From ad2642a8aac9facb217a8471df1d3e00f1214e92 Mon Sep 17 00:00:00 2001 From: Lauris BH Date: Tue, 11 Feb 2020 11:34:17 +0200 Subject: [PATCH] Language statistics bar for repositories (#8037) * Implementation for calculating language statistics Impement saving code language statistics to database Implement rendering langauge stats Add primary laguage to show in repository list Implement repository stats indexer queue Add indexer test Refactor to use queue module * Do not timeout for queues --- go.mod | 1 + go.sum | 9 + integrations/testlogger.go | 3 +- models/migrations/migrations.go | 2 + models/migrations/v127.go | 45 + models/models.go | 1 + models/repo.go | 9 +- models/repo_indexer.go | 81 +- models/repo_language_stats.go | 137 + models/repo_list.go | 31 +- modules/git/repo_language_stats.go | 116 + modules/indexer/code/bleve.go | 2 +- modules/indexer/code/git.go | 7 +- modules/indexer/code/queue.go | 2 +- modules/indexer/stats/db.go | 54 + modules/indexer/stats/indexer.go | 85 + modules/indexer/stats/indexer_test.go | 42 + modules/indexer/stats/queue.go | 43 + modules/notification/indexer/indexer.go | 7 + options/locale/locale_en-US.ini | 1 + routers/init.go | 4 + routers/org/home.go | 1 - routers/repo/view.go | 15 + routers/user/profile.go | 1 - templates/explore/repo_list.tmpl | 3 + templates/repo/sub_menu.tmpl | 51 +- vendor/github.com/src-d/enry/v2/.gitignore | 11 + vendor/github.com/src-d/enry/v2/.travis.yml | 132 + .../github.com/src-d/enry/v2/CONTRIBUTING.md | 61 + vendor/github.com/src-d/enry/v2/DCO | 25 + vendor/github.com/src-d/enry/v2/LICENSE | 201 + vendor/github.com/src-d/enry/v2/MAINTAINERS | 1 + vendor/github.com/src-d/enry/v2/Makefile | 82 + vendor/github.com/src-d/enry/v2/README.md | 328 + vendor/github.com/src-d/enry/v2/classifier.go | 107 + vendor/github.com/src-d/enry/v2/common.go | 472 + vendor/github.com/src-d/enry/v2/data/alias.go | 783 + .../github.com/src-d/enry/v2/data/colors.go | 254 + .../github.com/src-d/enry/v2/data/commit.go | 7 + .../github.com/src-d/enry/v2/data/content.go | 1358 + vendor/github.com/src-d/enry/v2/data/doc.go | 3 + .../src-d/enry/v2/data/documentation.go | 26 + .../src-d/enry/v2/data/extension.go | 1629 + .../github.com/src-d/enry/v2/data/filename.go | 241 + .../src-d/enry/v2/data/frequencies.go | 170527 +++++++++++++++ .../src-d/enry/v2/data/heuristics.go | 35 + .../src-d/enry/v2/data/interpreter.go | 124 + .../github.com/src-d/enry/v2/data/mimeType.go | 226 + .../src-d/enry/v2/data/rule/rule.go | 109 + vendor/github.com/src-d/enry/v2/data/type.go | 526 + .../github.com/src-d/enry/v2/data/vendor.go | 166 + vendor/github.com/src-d/enry/v2/enry.go | 16 + vendor/github.com/src-d/enry/v2/go.mod | 11 + vendor/github.com/src-d/enry/v2/go.sum | 17 + .../enry/v2/internal/tokenizer/common.go | 7 + .../internal/tokenizer/flex/lex.linguist_yy.c | 2226 + .../internal/tokenizer/flex/lex.linguist_yy.h | 336 + .../v2/internal/tokenizer/flex/linguist.h | 15 + .../v2/internal/tokenizer/flex/tokenize_c.go | 71 + .../enry/v2/internal/tokenizer/tokenize.go | 214 + .../enry/v2/internal/tokenizer/tokenize_c.go | 15 + .../src-d/enry/v2/regex/oniguruma.go | 17 + .../src-d/enry/v2/regex/standard.go | 17 + vendor/github.com/src-d/enry/v2/utils.go | 84 + .../github.com/src-d/go-oniguruma/.travis.yml | 20 + vendor/github.com/src-d/go-oniguruma/LICENSE | 19 + .../github.com/src-d/go-oniguruma/README.md | 20 + .../github.com/src-d/go-oniguruma/chelper.c | 184 + .../github.com/src-d/go-oniguruma/chelper.h | 14 + .../src-d/go-oniguruma/constants.go | 27 + vendor/github.com/src-d/go-oniguruma/go.mod | 1 + .../src-d/go-oniguruma/quotemeta.go | 36 + vendor/github.com/src-d/go-oniguruma/regex.go | 668 + vendor/github.com/toqueteos/trie/LICENSE.txt | 22 + vendor/github.com/toqueteos/trie/README.md | 7 + vendor/github.com/toqueteos/trie/go.mod | 1 + vendor/github.com/toqueteos/trie/trie.go | 102 + .../toqueteos/substring.v1/.gitignore | 24 + .../toqueteos/substring.v1/.travis.yml | 11 + .../gopkg.in/toqueteos/substring.v1/LICENSE | 22 + .../gopkg.in/toqueteos/substring.v1/README.md | 80 + .../gopkg.in/toqueteos/substring.v1/bytes.go | 229 + vendor/gopkg.in/toqueteos/substring.v1/lib.go | 10 + .../gopkg.in/toqueteos/substring.v1/string.go | 216 + vendor/modules.txt | 13 + web_src/js/index.js | 8 + web_src/less/_base.less | 10 + web_src/less/_repository.less | 20 + web_src/less/themes/theme-arc-green.less | 6 +- 89 files changed, 182948 insertions(+), 55 deletions(-) create mode 100644 models/migrations/v127.go create mode 100644 models/repo_language_stats.go create mode 100644 modules/git/repo_language_stats.go create mode 100644 modules/indexer/stats/db.go create mode 100644 modules/indexer/stats/indexer.go create mode 100644 modules/indexer/stats/indexer_test.go create mode 100644 modules/indexer/stats/queue.go create mode 100644 vendor/github.com/src-d/enry/v2/.gitignore create mode 100644 vendor/github.com/src-d/enry/v2/.travis.yml create mode 100644 vendor/github.com/src-d/enry/v2/CONTRIBUTING.md create mode 100644 vendor/github.com/src-d/enry/v2/DCO create mode 100644 vendor/github.com/src-d/enry/v2/LICENSE create mode 100644 vendor/github.com/src-d/enry/v2/MAINTAINERS create mode 100644 vendor/github.com/src-d/enry/v2/Makefile create mode 100644 vendor/github.com/src-d/enry/v2/README.md create mode 100644 vendor/github.com/src-d/enry/v2/classifier.go create mode 100644 vendor/github.com/src-d/enry/v2/common.go create mode 100644 vendor/github.com/src-d/enry/v2/data/alias.go create mode 100644 vendor/github.com/src-d/enry/v2/data/colors.go create mode 100644 vendor/github.com/src-d/enry/v2/data/commit.go create mode 100644 vendor/github.com/src-d/enry/v2/data/content.go create mode 100644 vendor/github.com/src-d/enry/v2/data/doc.go create mode 100644 vendor/github.com/src-d/enry/v2/data/documentation.go create mode 100644 vendor/github.com/src-d/enry/v2/data/extension.go create mode 100644 vendor/github.com/src-d/enry/v2/data/filename.go create mode 100644 vendor/github.com/src-d/enry/v2/data/frequencies.go create mode 100644 vendor/github.com/src-d/enry/v2/data/heuristics.go create mode 100644 vendor/github.com/src-d/enry/v2/data/interpreter.go create mode 100644 vendor/github.com/src-d/enry/v2/data/mimeType.go create mode 100644 vendor/github.com/src-d/enry/v2/data/rule/rule.go create mode 100644 vendor/github.com/src-d/enry/v2/data/type.go create mode 100644 vendor/github.com/src-d/enry/v2/data/vendor.go create mode 100644 vendor/github.com/src-d/enry/v2/enry.go create mode 100644 vendor/github.com/src-d/enry/v2/go.mod create mode 100644 vendor/github.com/src-d/enry/v2/go.sum create mode 100644 vendor/github.com/src-d/enry/v2/internal/tokenizer/common.go create mode 100644 vendor/github.com/src-d/enry/v2/internal/tokenizer/flex/lex.linguist_yy.c create mode 100644 vendor/github.com/src-d/enry/v2/internal/tokenizer/flex/lex.linguist_yy.h create mode 100644 vendor/github.com/src-d/enry/v2/internal/tokenizer/flex/linguist.h create mode 100644 vendor/github.com/src-d/enry/v2/internal/tokenizer/flex/tokenize_c.go create mode 100644 vendor/github.com/src-d/enry/v2/internal/tokenizer/tokenize.go create mode 100644 vendor/github.com/src-d/enry/v2/internal/tokenizer/tokenize_c.go create mode 100644 vendor/github.com/src-d/enry/v2/regex/oniguruma.go create mode 100644 vendor/github.com/src-d/enry/v2/regex/standard.go create mode 100644 vendor/github.com/src-d/enry/v2/utils.go create mode 100644 vendor/github.com/src-d/go-oniguruma/.travis.yml create mode 100644 vendor/github.com/src-d/go-oniguruma/LICENSE create mode 100644 vendor/github.com/src-d/go-oniguruma/README.md create mode 100644 vendor/github.com/src-d/go-oniguruma/chelper.c create mode 100644 vendor/github.com/src-d/go-oniguruma/chelper.h create mode 100644 vendor/github.com/src-d/go-oniguruma/constants.go create mode 100644 vendor/github.com/src-d/go-oniguruma/go.mod create mode 100644 vendor/github.com/src-d/go-oniguruma/quotemeta.go create mode 100644 vendor/github.com/src-d/go-oniguruma/regex.go create mode 100644 vendor/github.com/toqueteos/trie/LICENSE.txt create mode 100644 vendor/github.com/toqueteos/trie/README.md create mode 100644 vendor/github.com/toqueteos/trie/go.mod create mode 100644 vendor/github.com/toqueteos/trie/trie.go create mode 100644 vendor/gopkg.in/toqueteos/substring.v1/.gitignore create mode 100644 vendor/gopkg.in/toqueteos/substring.v1/.travis.yml create mode 100644 vendor/gopkg.in/toqueteos/substring.v1/LICENSE create mode 100644 vendor/gopkg.in/toqueteos/substring.v1/README.md create mode 100644 vendor/gopkg.in/toqueteos/substring.v1/bytes.go create mode 100644 vendor/gopkg.in/toqueteos/substring.v1/lib.go create mode 100644 vendor/gopkg.in/toqueteos/substring.v1/string.go diff --git a/go.mod b/go.mod index de67f582d..f28b199f0 100644 --- a/go.mod +++ b/go.mod @@ -85,6 +85,7 @@ require ( github.com/sergi/go-diff v1.0.0 github.com/shurcooL/httpfs v0.0.0-20190527155220-6a4d4a70508b // indirect github.com/shurcooL/vfsgen v0.0.0-20181202132449-6a9ea43bcacd + github.com/src-d/enry/v2 v2.1.0 github.com/steveyen/gtreap v0.0.0-20150807155958-0abe01ef9be2 // indirect github.com/stretchr/testify v1.4.0 github.com/tecbot/gorocksdb v0.0.0-20181010114359-8752a9433481 // indirect diff --git a/go.sum b/go.sum index ff5c52bc2..30109a24e 100644 --- a/go.sum +++ b/go.sum @@ -508,8 +508,13 @@ github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnIn github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/spf13/viper v1.4.0 h1:yXHLWeravcrgGyFSyCgdYpXQ9dR9c/WED3pg1RhxqEU= github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= +github.com/src-d/enry v1.7.3 h1:jG2fmEaQaURh0qqU/sn82BRzVa6d4EVHJIw6gc98bak= +github.com/src-d/enry/v2 v2.1.0 h1:z1L8t+B8bh3mmjPkJrgOTnVRpFGmTPJsplHX9wAn6BI= +github.com/src-d/enry/v2 v2.1.0/go.mod h1:qQeCMRwzMF3ckeGr+h0tJLdxXnq+NVZsIDMELj0t028= github.com/src-d/gcfg v1.4.0 h1:xXbNR5AlLSA315x2UO+fTSSAXCDf+Ar38/6oyGbDKQ4= github.com/src-d/gcfg v1.4.0/go.mod h1:p/UMsR43ujA89BJY9duynAwIpvqEujIH/jFlfL7jWoI= +github.com/src-d/go-oniguruma v1.1.0 h1:EG+Nm5n2JqWUaCjtM0NtutPxU7ZN5Tp50GWrrV8bTww= +github.com/src-d/go-oniguruma v1.1.0/go.mod h1:chVbff8kcVtmrhxtZ3yBVLLquXbzCS6DrxQaAK/CeqM= github.com/steveyen/gtreap v0.0.0-20150807155958-0abe01ef9be2 h1:JNEGSiWg6D3lcBCMCBqN3ELniXujt+0QNHLhNnO0w3s= github.com/steveyen/gtreap v0.0.0-20150807155958-0abe01ef9be2/go.mod h1:mjqs7N0Q6m5HpR7QfXVBZXZWSqTjQLeTujjA/xUp2uw= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -530,6 +535,8 @@ github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhV github.com/tinylib/msgp v1.1.0 h1:9fQd+ICuRIu/ue4vxJZu6/LzxN0HwMds2nq/0cFvxHU= github.com/tinylib/msgp v1.1.0/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/toqueteos/trie v1.0.0 h1:8i6pXxNUXNRAqP246iibb7w/pSFquNTQ+uNfriG7vlk= +github.com/toqueteos/trie v1.0.0/go.mod h1:Ywk48QhEqhU1+DwhMkJ2x7eeGxDHiGkAdc9+0DYcbsM= github.com/toqueteos/webbrowser v1.2.0 h1:tVP/gpK69Fx+qMJKsLE7TD8LuGWPnEV71wBN9rrstGQ= github.com/toqueteos/webbrowser v1.2.0/go.mod h1:XWoZq4cyp9WeUeak7w7LXRUQf1F1ATJMir8RTqb4ayM= github.com/tstranex/u2f v1.0.0 h1:HhJkSzDDlVSVIVt7pDJwCHQj67k7A5EeBgPmeD+pVsQ= @@ -747,6 +754,8 @@ gopkg.in/testfixtures.v2 v2.5.0 h1:N08B7l2GzFQenyYbzqthDnKAA+cmb17iAZhhFxr7JHw= gopkg.in/testfixtures.v2 v2.5.0/go.mod h1:vyAq+MYCgNpR29qitQdLZhdbLFf4mR/2MFJRFoQZZ2M= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/toqueteos/substring.v1 v1.0.2 h1:urLqCeMm6x/eTuQa1oZerNw8N1KNOIp5hD5kGL7lFsE= +gopkg.in/toqueteos/substring.v1 v1.0.2/go.mod h1:Eb2Z1UYehlVK8LYW2WBVR2rwbujsz3aX8XDrM1vbNew= gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= diff --git a/integrations/testlogger.go b/integrations/testlogger.go index b2ad257a9..eed0bf788 100644 --- a/integrations/testlogger.go +++ b/integrations/testlogger.go @@ -13,7 +13,6 @@ import ( "strings" "sync" "testing" - "time" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/queue" @@ -101,7 +100,7 @@ func PrintCurrentTest(t testing.TB, skip ...int) func() { } writerCloser.setT(&t) return func() { - if err := queue.GetManager().FlushAll(context.Background(), 20*time.Second); err != nil { + if err := queue.GetManager().FlushAll(context.Background(), -1); err != nil { t.Errorf("Flushing queues failed with error %v", err) } _ = writerCloser.Close() diff --git a/models/migrations/migrations.go b/models/migrations/migrations.go index 2de2a4556..ce3f77ba4 100644 --- a/models/migrations/migrations.go +++ b/models/migrations/migrations.go @@ -186,6 +186,8 @@ var migrations = []Migration{ NewMigration("Add some columns on review for migration", addReviewMigrateInfo), // v126 -> v127 NewMigration("Fix topic repository count", fixTopicRepositoryCount), + // v127 -> v128 + NewMigration("add repository code language statistics", addLanguageStats), } // Migrate database to current version diff --git a/models/migrations/v127.go b/models/migrations/v127.go new file mode 100644 index 000000000..d8f0de4a6 --- /dev/null +++ b/models/migrations/v127.go @@ -0,0 +1,45 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package migrations + +import ( + "fmt" + + "code.gitea.io/gitea/modules/timeutil" + + "xorm.io/xorm" +) + +func addLanguageStats(x *xorm.Engine) error { + // LanguageStat see models/repo_language_stats.go + type LanguageStat struct { + ID int64 `xorm:"pk autoincr"` + RepoID int64 `xorm:"UNIQUE(s) INDEX NOT NULL"` + CommitID string + IsPrimary bool + Language string `xorm:"VARCHAR(30) UNIQUE(s) INDEX NOT NULL"` + Percentage float32 `xorm:"NUMERIC(5,2) NOT NULL DEFAULT 0"` + Color string `xorm:"-"` + CreatedUnix timeutil.TimeStamp `xorm:"INDEX CREATED"` + } + + type RepoIndexerType int + + // RepoIndexerStatus see models/repo_stats_indexer.go + type RepoIndexerStatus struct { + ID int64 `xorm:"pk autoincr"` + RepoID int64 `xorm:"INDEX(s)"` + CommitSha string `xorm:"VARCHAR(40)"` + IndexerType RepoIndexerType `xorm:"INDEX(s) NOT NULL DEFAULT 0"` + } + + if err := x.Sync2(new(LanguageStat)); err != nil { + return fmt.Errorf("Sync2: %v", err) + } + if err := x.Sync2(new(RepoIndexerStatus)); err != nil { + return fmt.Errorf("Sync2: %v", err) + } + return nil +} diff --git a/models/models.go b/models/models.go index 239a9cf28..b84a179e3 100644 --- a/models/models.go +++ b/models/models.go @@ -116,6 +116,7 @@ func init() { new(OAuth2AuthorizationCode), new(OAuth2Grant), new(Task), + new(LanguageStat), ) gonicNames := []string{"SSL", "UID"} diff --git a/models/repo.go b/models/repo.go index 2d0f6fccb..4bb5e3eb0 100644 --- a/models/repo.go +++ b/models/repo.go @@ -175,8 +175,9 @@ type Repository struct { *Mirror `xorm:"-"` Status RepositoryStatus `xorm:"NOT NULL DEFAULT 0"` - RenderingMetas map[string]string `xorm:"-"` - Units []*RepoUnit `xorm:"-"` + RenderingMetas map[string]string `xorm:"-"` + Units []*RepoUnit `xorm:"-"` + PrimaryLanguage *LanguageStat `xorm:"-"` IsFork bool `xorm:"INDEX NOT NULL DEFAULT false"` ForkID int64 `xorm:"INDEX"` @@ -185,7 +186,8 @@ type Repository struct { TemplateID int64 `xorm:"INDEX"` TemplateRepo *Repository `xorm:"-"` Size int64 `xorm:"NOT NULL DEFAULT 0"` - IndexerStatus *RepoIndexerStatus `xorm:"-"` + CodeIndexerStatus *RepoIndexerStatus `xorm:"-"` + StatsIndexerStatus *RepoIndexerStatus `xorm:"-"` IsFsckEnabled bool `xorm:"NOT NULL DEFAULT true"` CloseIssuesViaCommitInAnyBranch bool `xorm:"NOT NULL DEFAULT false"` Topics []string `xorm:"TEXT JSON"` @@ -1504,6 +1506,7 @@ func DeleteRepository(doer *User, uid, repoID int64) error { &Notification{RepoID: repoID}, &CommitStatus{RepoID: repoID}, &RepoIndexerStatus{RepoID: repoID}, + &LanguageStat{RepoID: repoID}, &Comment{RefRepoID: repoID}, &Task{RepoID: repoID}, ); err != nil { diff --git a/models/repo_indexer.go b/models/repo_indexer.go index a9a516175..1f5ab928a 100644 --- a/models/repo_indexer.go +++ b/models/repo_indexer.go @@ -10,21 +10,32 @@ import ( "xorm.io/builder" ) +// RepoIndexerType specifies the repository indexer type +type RepoIndexerType int + +const ( + // RepoIndexerTypeCode code indexer + RepoIndexerTypeCode RepoIndexerType = iota // 0 + // RepoIndexerTypeStats repository stats indexer + RepoIndexerTypeStats // 1 +) + // RepoIndexerStatus status of a repo's entry in the repo indexer // For now, implicitly refers to default branch type RepoIndexerStatus struct { - ID int64 `xorm:"pk autoincr"` - RepoID int64 `xorm:"INDEX"` - CommitSha string `xorm:"VARCHAR(40)"` + ID int64 `xorm:"pk autoincr"` + RepoID int64 `xorm:"INDEX(s)"` + CommitSha string `xorm:"VARCHAR(40)"` + IndexerType RepoIndexerType `xorm:"INDEX(s) NOT NULL DEFAULT 0"` } // GetUnindexedRepos returns repos which do not have an indexer status -func GetUnindexedRepos(maxRepoID int64, page, pageSize int) ([]int64, error) { +func GetUnindexedRepos(indexerType RepoIndexerType, maxRepoID int64, page, pageSize int) ([]int64, error) { ids := make([]int64, 0, 50) cond := builder.Cond(builder.IsNull{ "repo_indexer_status.id", }) - sess := x.Table("repository").Join("LEFT OUTER", "repo_indexer_status", "repository.id = repo_indexer_status.repo_id") + sess := x.Table("repository").Join("LEFT OUTER", "repo_indexer_status", "repository.id = repo_indexer_status.repo_id AND repo_indexer_status.indexer_type = ?", indexerType) if maxRepoID > 0 { cond = builder.And(cond, builder.Lte{ "repository.id": maxRepoID, @@ -43,40 +54,64 @@ func GetUnindexedRepos(maxRepoID int64, page, pageSize int) ([]int64, error) { return ids, err } -// GetIndexerStatus loads repo codes indxer status -func (repo *Repository) GetIndexerStatus() error { - if repo.IndexerStatus != nil { - return nil +// getIndexerStatus loads repo codes indxer status +func (repo *Repository) getIndexerStatus(e Engine, indexerType RepoIndexerType) (*RepoIndexerStatus, error) { + switch indexerType { + case RepoIndexerTypeCode: + if repo.CodeIndexerStatus != nil { + return repo.CodeIndexerStatus, nil + } + case RepoIndexerTypeStats: + if repo.StatsIndexerStatus != nil { + return repo.StatsIndexerStatus, nil + } } - status := &RepoIndexerStatus{RepoID: repo.ID} - has, err := x.Get(status) + status := &RepoIndexerStatus{RepoID: repo.ID, IndexerType: indexerType} + has, err := e.Get(status) if err != nil { - return err + return nil, err } else if !has { status.CommitSha = "" } - repo.IndexerStatus = status - return nil + switch indexerType { + case RepoIndexerTypeCode: + repo.CodeIndexerStatus = status + case RepoIndexerTypeStats: + repo.StatsIndexerStatus = status + } + return status, nil } -// UpdateIndexerStatus updates indexer status -func (repo *Repository) UpdateIndexerStatus(sha string) error { - if err := repo.GetIndexerStatus(); err != nil { +// GetIndexerStatus loads repo codes indxer status +func (repo *Repository) GetIndexerStatus(indexerType RepoIndexerType) (*RepoIndexerStatus, error) { + return repo.getIndexerStatus(x, indexerType) +} + +// updateIndexerStatus updates indexer status +func (repo *Repository) updateIndexerStatus(e Engine, indexerType RepoIndexerType, sha string) error { + status, err := repo.getIndexerStatus(e, indexerType) + if err != nil { return fmt.Errorf("UpdateIndexerStatus: Unable to getIndexerStatus for repo: %s Error: %v", repo.FullName(), err) } - if len(repo.IndexerStatus.CommitSha) == 0 { - repo.IndexerStatus.CommitSha = sha - _, err := x.Insert(repo.IndexerStatus) + + if len(status.CommitSha) == 0 { + status.CommitSha = sha + _, err := e.Insert(status) if err != nil { return fmt.Errorf("UpdateIndexerStatus: Unable to insert repoIndexerStatus for repo: %s Sha: %s Error: %v", repo.FullName(), sha, err) } return nil } - repo.IndexerStatus.CommitSha = sha - _, err := x.ID(repo.IndexerStatus.ID).Cols("commit_sha"). - Update(repo.IndexerStatus) + status.CommitSha = sha + _, err = e.ID(status.ID).Cols("commit_sha"). + Update(status) if err != nil { return fmt.Errorf("UpdateIndexerStatus: Unable to update repoIndexerStatus for repo: %s Sha: %s Error: %v", repo.FullName(), sha, err) } return nil } + +// UpdateIndexerStatus updates indexer status +func (repo *Repository) UpdateIndexerStatus(indexerType RepoIndexerType, sha string) error { + return repo.updateIndexerStatus(x, indexerType, sha) +} diff --git a/models/repo_language_stats.go b/models/repo_language_stats.go new file mode 100644 index 000000000..4c3171e29 --- /dev/null +++ b/models/repo_language_stats.go @@ -0,0 +1,137 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package models + +import ( + "math" + "strings" + + "code.gitea.io/gitea/modules/timeutil" + + "github.com/src-d/enry/v2" +) + +// LanguageStat describes language statistics of a repository +type LanguageStat struct { + ID int64 `xorm:"pk autoincr"` + RepoID int64 `xorm:"UNIQUE(s) INDEX NOT NULL"` + CommitID string + IsPrimary bool + Language string `xorm:"VARCHAR(30) UNIQUE(s) INDEX NOT NULL"` + Percentage float32 `xorm:"NUMERIC(5,2) NOT NULL DEFAULT 0"` + Color string `xorm:"-"` + CreatedUnix timeutil.TimeStamp `xorm:"INDEX CREATED"` +} + +// LanguageStatList defines a list of language statistics +type LanguageStatList []*LanguageStat + +func (stats LanguageStatList) loadAttributes() { + for i := range stats { + stats[i].Color = enry.GetColor(stats[i].Language) + } +} + +func (repo *Repository) getLanguageStats(e Engine) (LanguageStatList, error) { + stats := make(LanguageStatList, 0, 6) + if err := e.Where("`repo_id` = ?", repo.ID).Desc("`percentage`").Find(&stats); err != nil { + return nil, err + } + stats.loadAttributes() + return stats, nil +} + +// GetLanguageStats returns the language statistics for a repository +func (repo *Repository) GetLanguageStats() (LanguageStatList, error) { + return repo.getLanguageStats(x) +} + +// GetTopLanguageStats returns the top language statistics for a repository +func (repo *Repository) GetTopLanguageStats(limit int) (LanguageStatList, error) { + stats, err := repo.getLanguageStats(x) + if err != nil { + return nil, err + } + topstats := make(LanguageStatList, 0, limit) + var other float32 + for i := range stats { + if stats[i].Language == "other" || len(topstats) >= limit { + other += stats[i].Percentage + continue + } + topstats = append(topstats, stats[i]) + } + if other > 0 { + topstats = append(topstats, &LanguageStat{ + RepoID: repo.ID, + Language: "other", + Color: "#cccccc", + Percentage: float32(math.Round(float64(other)*10) / 10), + }) + } + return topstats, nil +} + +// UpdateLanguageStats updates the language statistics for repository +func (repo *Repository) UpdateLanguageStats(commitID string, stats map[string]float32) error { + sess := x.NewSession() + if err := sess.Begin(); err != nil { + return err + } + defer sess.Close() + + oldstats, err := repo.getLanguageStats(sess) + if err != nil { + return err + } + var topLang string + var p float32 + for lang, perc := range stats { + if perc > p { + p = perc + topLang = strings.ToLower(lang) + } + } + + for lang, perc := range stats { + upd := false + llang := strings.ToLower(lang) + for _, s := range oldstats { + // Update already existing language + if strings.ToLower(s.Language) == llang { + s.CommitID = commitID + s.IsPrimary = llang == topLang + s.Percentage = perc + if _, err := sess.ID(s.ID).Cols("`commit_id`", "`percentage`", "`is_primary`").Update(s); err != nil { + return err + } + upd = true + break + } + } + // Insert new language + if !upd { + if _, err := sess.Insert(&LanguageStat{ + RepoID: repo.ID, + CommitID: commitID, + IsPrimary: llang == topLang, + Language: lang, + Percentage: perc, + }); err != nil { + return err + } + } + } + // Delete old languages + if _, err := sess.Where("`id` IN (SELECT `id` FROM `language_stat` WHERE `repo_id` = ? AND `commit_id` != ?)", repo.ID, commitID).Delete(&LanguageStat{}); err != nil { + return err + } + + if err = repo.updateIndexerStatus(sess, RepoIndexerTypeStats, commitID); err != nil { + return err + } + + return sess.Commit() +} diff --git a/models/repo_list.go b/models/repo_list.go index d3a113d26..6385de4b3 100644 --- a/models/repo_list.go +++ b/models/repo_list.go @@ -46,11 +46,14 @@ func (repos RepositoryList) loadAttributes(e Engine) error { return nil } - // Load owners. set := make(map[int64]struct{}) + repoIDs := make([]int64, len(repos)) for i := range repos { set[repos[i].OwnerID] = struct{}{} + repoIDs[i] = repos[i].ID } + + // Load owners. users := make(map[int64]*User, len(set)) if err := e. Where("id > 0"). @@ -61,6 +64,25 @@ func (repos RepositoryList) loadAttributes(e Engine) error { for i := range repos { repos[i].Owner = users[repos[i].OwnerID] } + + // Load primary language. + stats := make(LanguageStatList, 0, len(repos)) + if err := e. + Where("`is_primary` = ? AND `language` != ?", true, "other"). + In("`repo_id`", repoIDs). + Find(&stats); err != nil { + return fmt.Errorf("find primary languages: %v", err) + } + stats.loadAttributes() + for i := range repos { + for _, st := range stats { + if st.RepoID == repos[i].ID { + repos[i].PrimaryLanguage = st + break + } + } + } + return nil } @@ -119,7 +141,6 @@ type SearchRepoOptions struct { OrderBy SearchOrderBy Private bool // Include private repositories in results StarredByID int64 - IsProfile bool AllPublic bool // Include also all public repositories of users and public organisations AllLimited bool // Include also all public repositories of limited organisations // None -> include collaborative AND non-collaborative @@ -306,10 +327,8 @@ func SearchRepository(opts *SearchRepoOptions) (RepositoryList, int64, error) { return nil, 0, fmt.Errorf("Repo: %v", err) } - if !opts.IsProfile { - if err = repos.loadAttributes(sess); err != nil { - return nil, 0, fmt.Errorf("LoadAttributes: %v", err) - } + if err = repos.loadAttributes(sess); err != nil { + return nil, 0, fmt.Errorf("LoadAttributes: %v", err) } return repos, count, nil diff --git a/modules/git/repo_language_stats.go b/modules/git/repo_language_stats.go new file mode 100644 index 000000000..ffe6dd084 --- /dev/null +++ b/modules/git/repo_language_stats.go @@ -0,0 +1,116 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package git + +import ( + "bytes" + "io" + "io/ioutil" + "math" + "path/filepath" + + "github.com/src-d/enry/v2" + "gopkg.in/src-d/go-git.v4" + "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/object" +) + +const fileSizeLimit int64 = 16 * 1024 * 1024 + +// GetLanguageStats calculates language stats for git repository at specified commit +func (repo *Repository) GetLanguageStats(commitID string) (map[string]float32, error) { + r, err := git.PlainOpen(repo.Path) + if err != nil { + return nil, err + } + + rev, err := r.ResolveRevision(plumbing.Revision(commitID)) + if err != nil { + return nil, err + } + + commit, err := r.CommitObject(*rev) + if err != nil { + return nil, err + } + + tree, err := commit.Tree() + if err != nil { + return nil, err + } + + sizes := make(map[string]int64) + var total int64 + err = tree.Files().ForEach(func(f *object.File) error { + if enry.IsVendor(f.Name) || enry.IsDotFile(f.Name) || + enry.IsDocumentation(f.Name) || enry.IsConfiguration(f.Name) { + return nil + } + + // TODO: Use .gitattributes file for linguist overrides + + language, ok := enry.GetLanguageByExtension(f.Name) + if !ok { + if language, ok = enry.GetLanguageByFilename(f.Name); !ok { + content, err := readFile(f, fileSizeLimit) + if err != nil { + return nil + } + + language = enry.GetLanguage(filepath.Base(f.Name), content) + if language == enry.OtherLanguage { + return nil + } + } + } + + if language != "" { + sizes[language] += f.Size + total += f.Size + } + + return nil + }) + if err != nil { + return nil, err + } + + stats := make(map[string]float32) + var otherPerc float32 = 100 + for language, size := range sizes { + perc := float32(math.Round(float64(size)/float64(total)*1000) / 10) + if perc <= 0.1 { + continue + } + otherPerc -= perc + stats[language] = perc + } + otherPerc = float32(math.Round(float64(otherPerc)*10) / 10) + if otherPerc > 0 { + stats["other"] = otherPerc + } + return stats, nil +} + +func readFile(f *object.File, limit int64) ([]byte, error) { + r, err := f.Reader() + if err != nil { + return nil, err + } + defer r.Close() + + if limit <= 0 { + return ioutil.ReadAll(r) + } + + size := f.Size + if limit > 0 && size > limit { + size = limit + } + buf := bytes.NewBuffer(nil) + buf.Grow(int(size)) + _, err = io.Copy(buf, io.LimitReader(r, limit)) + return buf.Bytes(), err +} diff --git a/modules/indexer/code/bleve.go b/modules/indexer/code/bleve.go index 339dca74a..6052304f8 100644 --- a/modules/indexer/code/bleve.go +++ b/modules/indexer/code/bleve.go @@ -267,7 +267,7 @@ func (b *BleveIndexer) Index(repoID int64) error { if err = batch.Flush(); err != nil { return err } - return repo.UpdateIndexerStatus(sha) + return repo.UpdateIndexerStatus(models.RepoIndexerTypeCode, sha) } // Delete deletes indexes by ids diff --git a/modules/indexer/code/git.go b/modules/indexer/code/git.go index 114d5a9e6..37ab5ac3d 100644 --- a/modules/indexer/code/git.go +++ b/modules/indexer/code/git.go @@ -35,11 +35,12 @@ func getDefaultBranchSha(repo *models.Repository) (string, error) { // getRepoChanges returns changes to repo since last indexer update func getRepoChanges(repo *models.Repository, revision string) (*repoChanges, error) { - if err := repo.GetIndexerStatus(); err != nil { + status, err := repo.GetIndexerStatus(models.RepoIndexerTypeCode) + if err != nil { return nil, err } - if len(repo.IndexerStatus.CommitSha) == 0 { + if len(status.CommitSha) == 0 { return genesisChanges(repo, revision) } return nonGenesisChanges(repo, revision) @@ -98,7 +99,7 @@ func genesisChanges(repo *models.Repository, revision string) (*repoChanges, err // nonGenesisChanges get changes since the previous indexer update func nonGenesisChanges(repo *models.Repository, revision string) (*repoChanges, error) { diffCmd := git.NewCommand("diff", "--name-status", - repo.IndexerStatus.CommitSha, revision) + repo.CodeIndexerStatus.CommitSha, revision) stdout, err := diffCmd.RunInDir(repo.RepoPath()) if err != nil { // previous commit sha may have been removed by a force push, so diff --git a/modules/indexer/code/queue.go b/modules/indexer/code/queue.go index 4eeb6ac7d..94675559e 100644 --- a/modules/indexer/code/queue.go +++ b/modules/indexer/code/queue.go @@ -109,7 +109,7 @@ func populateRepoIndexer() { return default: } - ids, err := models.GetUnindexedRepos(maxRepoID, 0, 50) + ids, err := models.GetUnindexedRepos(models.RepoIndexerTypeCode, maxRepoID, 0, 50) if err != nil { log.Error("populateRepoIndexer: %v", err) return diff --git a/modules/indexer/stats/db.go b/modules/indexer/stats/db.go new file mode 100644 index 000000000..fe219b443 --- /dev/null +++ b/modules/indexer/stats/db.go @@ -0,0 +1,54 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package stats + +import ( + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/git" +) + +// DBIndexer implements Indexer interface to use database's like search +type DBIndexer struct { +} + +// Index repository status function +func (db *DBIndexer) Index(id int64) error { + repo, err := models.GetRepositoryByID(id) + if err != nil { + return err + } + status, err := repo.GetIndexerStatus(models.RepoIndexerTypeStats) + if err != nil { + return err + } + + gitRepo, err := git.OpenRepository(repo.RepoPath()) + if err != nil { + return err + } + defer gitRepo.Close() + + // Get latest commit for default branch + commitID, err := gitRepo.GetBranchCommitID(repo.DefaultBranch) + if err != nil { + return err + } + + // Do not recalculate stats if already calculated for this commit + if status.CommitSha == commitID { + return nil + } + + // Calculate and save language statistics to database + stats, err := gitRepo.GetLanguageStats(commitID) + if err != nil { + return err + } + return repo.UpdateLanguageStats(commitID, stats) +} + +// Close dummy function +func (db *DBIndexer) Close() { +} diff --git a/modules/indexer/stats/indexer.go b/modules/indexer/stats/indexer.go new file mode 100644 index 000000000..4d8a174ff --- /dev/null +++ b/modules/indexer/stats/indexer.go @@ -0,0 +1,85 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package stats + +import ( + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/graceful" + "code.gitea.io/gitea/modules/log" +) + +// Indexer defines an interface to index repository stats +type Indexer interface { + Index(id int64) error + Close() +} + +// indexer represents a indexer instance +var indexer Indexer + +// Init initialize the repo indexer +func Init() error { + indexer = &DBIndexer{} + + if err := initStatsQueue(); err != nil { + return err + } + + go populateRepoIndexer() + + return nil +} + +// populateRepoIndexer populate the repo indexer with pre-existing data. This +// should only be run when the indexer is created for the first time. +func populateRepoIndexer() { + log.Info("Populating the repo stats indexer with existing repositories") + + isShutdown := graceful.GetManager().IsShutdown() + + exist, err := models.IsTableNotEmpty("repository") + if err != nil { + log.Fatal("System error: %v", err) + } else if !exist { + return + } + + var maxRepoID int64 + if maxRepoID, err = models.GetMaxID("repository"); err != nil { + log.Fatal("System error: %v", err) + } + + // start with the maximum existing repo ID and work backwards, so that we + // don't include repos that are created after gitea starts; such repos will + // already be added to the indexer, and we don't need to add them again. + for maxRepoID > 0 { + select { + case <-isShutdown: + log.Info("Repository Stats Indexer population shutdown before completion") + return + default: + } + ids, err := models.GetUnindexedRepos(models.RepoIndexerTypeStats, maxRepoID, 0, 50) + if err != nil { + log.Error("populateRepoIndexer: %v", err) + return + } else if len(ids) == 0 { + break + } + for _, id := range ids { + select { + case <-isShutdown: + log.Info("Repository Stats Indexer population shutdown before completion") + return + default: + } + if err := statsQueue.Push(id); err != nil { + log.Error("statsQueue.Push: %v", err) + } + maxRepoID = id - 1 + } + } + log.Info("Done (re)populating the repo stats indexer with existing repositories") +} diff --git a/modules/indexer/stats/indexer_test.go b/modules/indexer/stats/indexer_test.go new file mode 100644 index 000000000..29d0f6dbe --- /dev/null +++ b/modules/indexer/stats/indexer_test.go @@ -0,0 +1,42 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package stats + +import ( + "path/filepath" + "testing" + "time" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/setting" + + "gopkg.in/ini.v1" + + "github.com/stretchr/testify/assert" +) + +func TestMain(m *testing.M) { + models.MainTest(m, filepath.Join("..", "..", "..")) +} + +func TestRepoStatsIndex(t *testing.T) { + assert.NoError(t, models.PrepareTestDatabase()) + setting.Cfg = ini.Empty() + + setting.NewQueueService() + + err := Init() + assert.NoError(t, err) + + time.Sleep(5 * time.Second) + + repo, err := models.GetRepositoryByID(1) + assert.NoError(t, err) + langs, err := repo.GetTopLanguageStats(5) + assert.NoError(t, err) + assert.Len(t, langs, 1) + assert.Equal(t, "other", langs[0].Language) + assert.Equal(t, float32(100), langs[0].Percentage) +} diff --git a/modules/indexer/stats/queue.go b/modules/indexer/stats/queue.go new file mode 100644 index 000000000..43a4de5ac --- /dev/null +++ b/modules/indexer/stats/queue.go @@ -0,0 +1,43 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package stats + +import ( + "fmt" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/graceful" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/queue" +) + +// statsQueue represents a queue to handle repository stats updates +var statsQueue queue.Queue + +// handle passed PR IDs and test the PRs +func handle(data ...queue.Data) { + for _, datum := range data { + opts := datum.(int64) + if err := indexer.Index(opts); err != nil { + log.Error("stats queue idexer.Index(%d) failed: %v", opts, err) + } + } +} + +func initStatsQueue() error { + statsQueue = queue.CreateQueue("repo_stats_update", handle, int64(0)).(queue.Queue) + if statsQueue == nil { + return fmt.Errorf("Unable to create repo_stats_update Queue") + } + + go graceful.GetManager().RunWithShutdownFns(statsQueue.Run) + + return nil +} + +// UpdateRepoIndexer update a repository's entries in the indexer +func UpdateRepoIndexer(repo *models.Repository) error { + return statsQueue.Push(repo.ID) +} diff --git a/modules/notification/indexer/indexer.go b/modules/notification/indexer/indexer.go index 4bce99073..6caae6fa6 100644 --- a/modules/notification/indexer/indexer.go +++ b/modules/notification/indexer/indexer.go @@ -9,6 +9,7 @@ import ( "code.gitea.io/gitea/modules/git" code_indexer "code.gitea.io/gitea/modules/indexer/code" issue_indexer "code.gitea.io/gitea/modules/indexer/issues" + stats_indexer "code.gitea.io/gitea/modules/indexer/stats" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/notification/base" "code.gitea.io/gitea/modules/repository" @@ -117,12 +118,18 @@ func (r *indexerNotifier) NotifyMigrateRepository(doer *models.User, u *models.U if setting.Indexer.RepoIndexerEnabled && !repo.IsEmpty { code_indexer.UpdateRepoIndexer(repo) } + if err := stats_indexer.UpdateRepoIndexer(repo); err != nil { + log.Error("stats_indexer.UpdateRepoIndexer(%d) failed: %v", repo.ID, err) + } } func (r *indexerNotifier) NotifyPushCommits(pusher *models.User, repo *models.Repository, refName, oldCommitID, newCommitID string, commits *repository.PushCommits) { if setting.Indexer.RepoIndexerEnabled && refName == git.BranchPrefix+repo.DefaultBranch { code_indexer.UpdateRepoIndexer(repo) } + if err := stats_indexer.UpdateRepoIndexer(repo); err != nil { + log.Error("stats_indexer.UpdateRepoIndexer(%d) failed: %v", repo.ID, err) + } } func (r *indexerNotifier) NotifyIssueChangeContent(doer *models.User, issue *models.Issue, oldContent string) { diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 8cd76ad2b..5b4d491bd 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -641,6 +641,7 @@ forks = Forks pick_reaction = Pick your reaction reactions_more = and %d more unit_disabled = The site administrator has disabled this repository section. +language_other = Other template.items = Template Items template.git_content = Git Content (Default Branch) diff --git a/routers/init.go b/routers/init.go index f86a7ad4b..724bf84c1 100644 --- a/routers/init.go +++ b/routers/init.go @@ -19,6 +19,7 @@ import ( "code.gitea.io/gitea/modules/highlight" code_indexer "code.gitea.io/gitea/modules/indexer/code" issue_indexer "code.gitea.io/gitea/modules/indexer/issues" + stats_indexer "code.gitea.io/gitea/modules/indexer/stats" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/markup" "code.gitea.io/gitea/modules/markup/external" @@ -111,6 +112,9 @@ func GlobalInit(ctx context.Context) { cron.NewContext() issue_indexer.InitIssueIndexer(false) code_indexer.Init() + if err := stats_indexer.Init(); err != nil { + log.Fatal("Failed to initialize repository stats indexer queue: %v", err) + } mirror_service.InitSyncMirrors() webhook.InitDeliverHooks() if err := pull_service.Init(); err != nil { diff --git a/routers/org/home.go b/routers/org/home.go index e1bea5b7a..fa61218d3 100644 --- a/routers/org/home.go +++ b/routers/org/home.go @@ -85,7 +85,6 @@ func Home(ctx *context.Context) { OrderBy: orderBy, Private: ctx.IsSigned, Actor: ctx.User, - IsProfile: true, IncludeDescription: setting.UI.SearchRepoDescription, }) if err != nil { diff --git a/routers/repo/view.go b/routers/repo/view.go index f56c52435..9183aea03 100644 --- a/routers/repo/view.go +++ b/routers/repo/view.go @@ -457,6 +457,16 @@ func Home(ctx *context.Context) { ctx.NotFound("Home", fmt.Errorf(ctx.Tr("units.error.no_unit_allowed_repo"))) } +func renderLanguageStats(ctx *context.Context) { + langs, err := ctx.Repo.Repository.GetTopLanguageStats(5) + if err != nil { + ctx.ServerError("Repo.GetTopLanguageStats", err) + return + } + + ctx.Data["LanguageStats"] = langs +} + func renderCode(ctx *context.Context) { ctx.Data["PageIsViewCode"] = true @@ -497,6 +507,11 @@ func renderCode(ctx *context.Context) { return } + renderLanguageStats(ctx) + if ctx.Written() { + return + } + if entry.IsDir() { renderDirectory(ctx, treeLink) } else { diff --git a/routers/user/profile.go b/routers/user/profile.go index a151884d7..215dff008 100644 --- a/routers/user/profile.go +++ b/routers/user/profile.go @@ -220,7 +220,6 @@ func Profile(ctx *context.Context) { OwnerID: ctxUser.ID, OrderBy: orderBy, Private: ctx.IsSigned, - IsProfile: true, Collaborate: util.OptionalBoolFalse, TopicOnly: topicOnly, IncludeDescription: setting.UI.SearchRepoDescription, diff --git a/templates/explore/repo_list.tmpl b/templates/explore/repo_list.tmpl index 8c7ba51a5..fec304cc9 100644 --- a/templates/explore/repo_list.tmpl +++ b/templates/explore/repo_list.tmpl @@ -21,6 +21,9 @@ {{end}} {{end}}
+ {{if .PrimaryLanguage }} + {{ .PrimaryLanguage.Language }} + {{end}} {{.NumStars}} {{.NumForks}}
diff --git a/templates/repo/sub_menu.tmpl b/templates/repo/sub_menu.tmpl index b97fe902e..96128fb2a 100644 --- a/templates/repo/sub_menu.tmpl +++ b/templates/repo/sub_menu.tmpl @@ -1,17 +1,42 @@ -
-
- {{if and (.Permission.CanRead $.UnitTypeCode) (not .IsEmptyRepo)}} -
- {{.CommitsCount}} {{.i18n.Tr (TrN .i18n.Lang .CommitsCount "repo.commit" "repo.commits") }} -
- {{end}} - {{if and (.Permission.CanRead $.UnitTypeCode) (not .IsEmptyRepo) }} -
- {{.BranchesCount}} {{.i18n.Tr (TrN .i18n.Lang .BranchesCount "repo.branch" "repo.branches") }} -
+
+
+
+ {{if and (.Permission.CanRead $.UnitTypeCode) (not .IsEmptyRepo)}} +
+ {{.CommitsCount}} {{.i18n.Tr (TrN .i18n.Lang .CommitsCount "repo.commit" "repo.commits") }} +
+ {{end}} + {{if and (.Permission.CanRead $.UnitTypeCode) (not .IsEmptyRepo) }} +
+ {{.BranchesCount}} {{.i18n.Tr (TrN .i18n.Lang .BranchesCount "repo.branch" "repo.branches") }} +
+
+ {{SizeFmt .Repository.Size}} +
+ {{end}} +
+
+ {{if and (.Permission.CanRead $.UnitTypeCode) (not .IsEmptyRepo) .LanguageStats }} + + + {{range .LanguageStats}} +
 
+ {{end}} + + {{end}}
diff --git a/vendor/github.com/src-d/enry/v2/.gitignore b/vendor/github.com/src-d/enry/v2/.gitignore new file mode 100644 index 000000000..78b3fb161 --- /dev/null +++ b/vendor/github.com/src-d/enry/v2/.gitignore @@ -0,0 +1,11 @@ +.linguist +benchmarks/output +.ci +Makefile.main +.shared +.idea +.docsrv-resources +build/ +vendor/ +java/lib/ +.vscode/ diff --git a/vendor/github.com/src-d/enry/v2/.travis.yml b/vendor/github.com/src-d/enry/v2/.travis.yml new file mode 100644 index 000000000..004bc52f9 --- /dev/null +++ b/vendor/github.com/src-d/enry/v2/.travis.yml @@ -0,0 +1,132 @@ +dist: trusty +language: go +go: + - '1.12.x' + - '1.11.x' +env: + global: + - GO_VERSION_FOR_JVM='1.11.x' + - CGO_ENABLED=0 + - GO111MODULE=on + - ONIGURUMA_VERSION='6.9.1' + matrix: + - ONIGURUMA=0 + - ONIGURUMA=1 +matrix: + fast_finish: true + +stages: + - name: test + - name: release + if: tag IS present + - name: publish + if: tag IS present + +stage: test +install: + - > + if [[ "${ONIGURUMA}" -gt 0 ]]; then + export CGO_ENABLED=1 + export GO_TAGS='oniguruma' + # install oniguruma manually as trusty has only ancient 5.x + sudo apt-get install -y dpkg # dpkg >= 1.17.5ubuntu5.8 fixes https://bugs.launchpad.net/ubuntu/+source/dpkg/+bug/1730627 + wget "http://archive.ubuntu.com/ubuntu/pool/universe/libo/libonig/libonig5_${ONIGURUMA_VERSION}-1_amd64.deb" + sudo dpkg -i "libonig5_${ONIGURUMA_VERSION}-1_amd64.deb" + wget "http://archive.ubuntu.com/ubuntu/pool/universe/libo/libonig/libonig-dev_${ONIGURUMA_VERSION}-1_amd64.deb" + sudo dpkg -i "libonig-dev_${ONIGURUMA_VERSION}-1_amd64.deb" + fi; +script: + - make test-coverage +after_success: + - bash <(curl -s https://codecov.io/bash) + +jobs: + include: + - name: 'java unit-tests' + stage: test + language: scala + jdk: oraclejdk8 + install: + - export CGO_ENABLED=1 + - eval "$(curl -sL https://raw.githubusercontent.com/travis-ci/gimme/master/gimme | GIMME_GO_VERSION=$GO_VERSION_FOR_JVM bash)" + - go version + before_script: + - cd java + - make + script: + - make test + + - name: 'linux packages' + stage: release + install: + - go version + script: make packages + deploy: + provider: releases + api_key: + secure: $GITHUB_TOKEN + file_glob: true + file: build/*.tar.gz + skip_cleanup: true + on: + tags: true + + - name: 'linux shared lib' + stage: release + install: + - go version + script: make linux-shared + deploy: + provider: releases + api_key: + secure: $GITHUB_TOKEN + file: + - ./.shared/linux-x86-64/libenry.so + skip_cleanup: true + on: + tags: true + + - name: 'macOS shared lib' + stage: release + env: + - OSXCROSS_PACKAGE="osxcross_3034f7149716d815bc473d0a7b35d17e4cf175aa.tar.gz" + - OSXCROSS_URL="https://github.com/bblfsh/client-scala/releases/download/v1.5.2/${OSXCROSS_PACKAGE}" + - PATH="/$HOME/osxcross/bin:$PATH" + install: + - go version + - sudo apt-get update + - sudo apt-get install -y --no-install-recommends clang g++ gcc gcc-multilib libc6-dev libc6-dev-i386 mingw-w64 patch xz-utils + - cd ${HOME} + - curl -sfSL ${OSXCROSS_URL} | tar -C ${HOME} -xzf - + - cd $GOPATH/src/github.com/src-d/enry + script: make darwin-shared + deploy: + provider: releases + api_key: + secure: $GITHUB_TOKEN + file: ./.shared/darwin/libenry.dylib + skip_cleanup: true + on: + tags: true + + - name: 'java: publish to maven' + stage: publish + language: scala + jdk: oraclejdk8 + install: + - export CGO_ENABLED=1 + - eval "$(curl -sL https://raw.githubusercontent.com/travis-ci/gimme/master/gimme | GIMME_GO_VERSION=$GO_VERSION_FOR_JVM bash)" + - go version + before_script: + - cd java + - make + - curl -o ./shared/linux-x86-64/libenry.so -sfL "https://github.com/$TRAVIS_REPO_SLUG/releases/download/$TRAVIS_TAG/libenry.so" || travis_terminate 1 + - mkdir -p ./shared/darwin + - curl -o ./shared/darwin/libenry.dylib -sfL "https://github.com/$TRAVIS_REPO_SLUG/releases/download/$TRAVIS_TAG/libenry.dylib" || travis_terminate 1 + - openssl aes-256-cbc -K $encrypted_a0e1c69dbbc7_key -iv $encrypted_a0e1c69dbbc7_iv -in key.asc.enc -out key.asc -d + - gpg --no-default-keyring --primary-keyring ./project/.gnupg/pubring.gpg --secret-keyring ./project/.gnupg/secring.gpg --keyring ./project/.gnupg/pubring.gpg --fingerprint --import key.asc + script: + - make test # ensure the shared objects are functional + - ./sbt publishLocal + - ./sbt publishSigned + - ./sbt sonatypeRelease diff --git a/vendor/github.com/src-d/enry/v2/CONTRIBUTING.md b/vendor/github.com/src-d/enry/v2/CONTRIBUTING.md new file mode 100644 index 000000000..6d2bcf9cc --- /dev/null +++ b/vendor/github.com/src-d/enry/v2/CONTRIBUTING.md @@ -0,0 +1,61 @@ +# source{d} Contributing Guidelines + +source{d} projects accept contributions via GitHub pull requests. +This document outlines some of the +conventions on development workflow, commit message formatting, contact points, +and other resources to make it easier to get your contribution accepted. + +## Certificate of Origin + +By contributing to this project, you agree to the [Developer Certificate of +Origin (DCO)](DCO). This document was created by the Linux Kernel community and is a +simple statement that you, as a contributor, have the legal right to make the +contribution. + +In order to show your agreement with the DCO you should include at the end of the commit message, +the following line: `Signed-off-by: John Doe `, using your real name. + +This can be done easily using the [`-s`](https://github.com/git/git/blob/b2c150d3aa82f6583b9aadfecc5f8fa1c74aca09/Documentation/git-commit.txt#L154-L161) flag on the `git commit`. + +If you find yourself pushed a few commits without `Signed-off-by`, you can still add it afterwards. We wrote a manual which can help: [fix-DCO.md](https://github.com/src-d/guide/blob/master/developer-community/fix-DCO.md). + +## Support Channels + +The official support channels, for both users and contributors, are: + +- GitHub issues: each repository has its own list of issues. +- Slack: join the [source{d} Slack](https://join.slack.com/t/sourced-community/shared_invite/enQtMjc4Njk5MzEyNzM2LTFjNzY4NjEwZGEwMzRiNTM4MzRlMzQ4MmIzZjkwZmZlM2NjODUxZmJjNDI1OTcxNDAyMmZlNmFjODZlNTg0YWM) community. + +*Before opening a new issue or submitting a new pull request, it's helpful to +search the project - it's likely that another user has already reported the +issue you're facing, or it's a known issue that we're already aware of. + + +## How to Contribute + +Pull Requests (PRs) are the main and exclusive way to contribute code to source{d} projects. +In order for a PR to be accepted it needs to pass this list of requirements: + +- The contribution must be correctly explained with natural language and providing a minimum working example that reproduces it. +- All PRs must be written idiomaticly: + - for Go: formatted according to [gofmt](https://golang.org/cmd/gofmt/), and without any warnings from [go lint](https://github.com/golang/lint) nor [go vet](https://golang.org/cmd/vet/) + - for other languages, similar constraints apply. +- They should in general include tests, and those shall pass. + - If the PR is a bug fix, it has to include a new unit test that fails before the patch is merged. + - If the PR is a new feature, it has to come with a suite of unit tests, that tests the new functionality. + - In any case, all the PRs have to pass the personal evaluation of at least one of the [maintainers](MAINTAINERS) of the project. + + +### Format of the commit message + +Every commit message should describe what was changed, under which context and, if applicable, the GitHub issue it relates to: + +``` +plumbing: packp, Skip argument validations for unknown capabilities. Fixes #623 +``` + +The format can be described more formally as follows: + +``` +: , . [Fixes #] +``` diff --git a/vendor/github.com/src-d/enry/v2/DCO b/vendor/github.com/src-d/enry/v2/DCO new file mode 100644 index 000000000..29c1b9208 --- /dev/null +++ b/vendor/github.com/src-d/enry/v2/DCO @@ -0,0 +1,25 @@ + Developer's Certificate of Origin 1.1 + + By making a contribution to this project, I certify that: + + (a) The contribution was created in whole or in part by me and I + have the right to submit it under the open source license + indicated in the file; or + + (b) The contribution is based upon previous work that, to the best + of my knowledge, is covered under an appropriate open source + license and I have the right under that license to submit that + work with modifications, whether created in whole or in part + by me, under the same open source license (unless I am + permitted to submit under a different license), as indicated + in the file; or + + (c) The contribution was provided directly to me by some other + person who certified (a), (b) or (c) and I have not modified + it. + + (d) I understand and agree that this project and the contribution + are public and that a record of the contribution (including all + personal information I submit with it, including my sign-off) is + maintained indefinitely and may be redistributed consistent with + this project or the open source license(s) involved. diff --git a/vendor/github.com/src-d/enry/v2/LICENSE b/vendor/github.com/src-d/enry/v2/LICENSE new file mode 100644 index 000000000..7f5eae069 --- /dev/null +++ b/vendor/github.com/src-d/enry/v2/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2017 Sourced Technologies, S.L. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/vendor/github.com/src-d/enry/v2/MAINTAINERS b/vendor/github.com/src-d/enry/v2/MAINTAINERS new file mode 100644 index 000000000..26868902d --- /dev/null +++ b/vendor/github.com/src-d/enry/v2/MAINTAINERS @@ -0,0 +1 @@ +Alexander Bezzubov (@bzz) diff --git a/vendor/github.com/src-d/enry/v2/Makefile b/vendor/github.com/src-d/enry/v2/Makefile new file mode 100644 index 000000000..fb978eb84 --- /dev/null +++ b/vendor/github.com/src-d/enry/v2/Makefile @@ -0,0 +1,82 @@ +# Package configuration +PROJECT = enry +COMMANDS = cmd/enry + +# Including ci Makefile +CI_REPOSITORY ?= https://github.com/src-d/ci.git +CI_BRANCH ?= v1 +CI_PATH ?= .ci +MAKEFILE := $(CI_PATH)/Makefile.main +$(MAKEFILE): + git clone --quiet --depth 1 -b $(CI_BRANCH) $(CI_REPOSITORY) $(CI_PATH); +-include $(MAKEFILE) + +# Docsrv: configure the languages whose api-doc can be auto generated +LANGUAGES = go +# Docs: do not edit this +DOCS_REPOSITORY := https://github.com/src-d/docs +SHARED_PATH ?= $(shell pwd)/.docsrv-resources +DOCS_PATH ?= $(SHARED_PATH)/.docs +$(DOCS_PATH)/Makefile.inc: + git clone --quiet --depth 1 $(DOCS_REPOSITORY) $(DOCS_PATH); +-include $(DOCS_PATH)/Makefile.inc + +LINGUIST_PATH = .linguist + +# shared objects +RESOURCES_DIR=./.shared +LINUX_DIR=$(RESOURCES_DIR)/linux-x86-64 +LINUX_SHARED_LIB=$(LINUX_DIR)/libenry.so +DARWIN_DIR=$(RESOURCES_DIR)/darwin +DARWIN_SHARED_LIB=$(DARWIN_DIR)/libenry.dylib +HEADER_FILE=libenry.h +NATIVE_LIB=./shared/enry.go + +$(LINGUIST_PATH): + git clone https://github.com/github/linguist.git $@ + +clean-linguist: + rm -rf $(LINGUIST_PATH) + +clean-shared: + rm -rf $(RESOURCES_DIR) + +clean: clean-linguist clean-shared + +code-generate: $(LINGUIST_PATH) + mkdir -p data && \ + go run internal/code-generator/main.go + ENRY_TEST_REPO="$${PWD}/.linguist" go test -v \ + -run Test_GeneratorTestSuite \ + ./internal/code-generator/generator \ + -testify.m TestUpdateGeneratorTestSuiteGold \ + -update_gold + +benchmarks: $(LINGUIST_PATH) + go test -run=NONE -bench=. && \ + benchmarks/linguist-total.rb + +benchmarks-samples: $(LINGUIST_PATH) + go test -run=NONE -bench=. -benchtime=5us && \ + benchmarks/linguist-samples.rb + +benchmarks-slow: $(LINGUIST_PATH) + mkdir -p benchmarks/output && \ + go test -run=NONE -bench=. -slow -benchtime=100ms -timeout=100h > benchmarks/output/enry_samples.bench && \ + benchmarks/linguist-samples.rb 5 > benchmarks/output/linguist_samples.bench + +linux-shared: $(LINUX_SHARED_LIB) + +darwin-shared: $(DARWIN_SHARED_LIB) + +$(DARWIN_SHARED_LIB): + mkdir -p $(DARWIN_DIR) && \ + CC="o64-clang" CXX="o64-clang++" CGO_ENABLED=1 GOOS=darwin go build -buildmode=c-shared -o $(DARWIN_SHARED_LIB) $(NATIVE_LIB) && \ + mv $(DARWIN_DIR)/$(HEADER_FILE) $(RESOURCES_DIR)/$(HEADER_FILE) + +$(LINUX_SHARED_LIB): + mkdir -p $(LINUX_DIR) && \ + CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go build -buildmode=c-shared -o $(LINUX_SHARED_LIB) $(NATIVE_LIB) && \ + mv $(LINUX_DIR)/$(HEADER_FILE) $(RESOURCES_DIR)/$(HEADER_FILE) + +.PHONY: benchmarks benchmarks-samples benchmarks-slow diff --git a/vendor/github.com/src-d/enry/v2/README.md b/vendor/github.com/src-d/enry/v2/README.md new file mode 100644 index 000000000..ed0d63e96 --- /dev/null +++ b/vendor/github.com/src-d/enry/v2/README.md @@ -0,0 +1,328 @@ +# enry [![GoDoc](https://godoc.org/github.com/src-d/enry?status.svg)](https://godoc.org/github.com/src-d/enry) [![Build Status](https://travis-ci.com/src-d/enry.svg?branch=master)](https://travis-ci.com/src-d/enry) [![codecov](https://codecov.io/gh/src-d/enry/branch/master/graph/badge.svg)](https://codecov.io/gh/src-d/enry) + +File programming language detector and toolbox to ignore binary or vendored files. *enry*, started as a port to _Go_ of the original [linguist](https://github.com/github/linguist) _Ruby_ library, that has an improved *2x performance*. + +* [Installation](#installation) +* [Examples](#examples) +* [CLI](#cli) +* [Java bindings](#java-bindings) +* [Python bindings](#python-bindings) +* [Divergences from linguist](#divergences-from-linguist) +* [Benchmarks](#benchmarks) +* [Why Enry?](#why-enry) +* [Development](#development) + * [Sync with github/linguist upstream](#sync-with-githublinguist-upstream) +* [Misc](#misc) + * [Benchmark](#benchmark) + * [Faster regexp engine (optional)](#faster-regexp-engine-optional) +* [License](#license) + +Installation +------------ + +The recommended way to install enry is to either [download a release](https://github.com/src-d/enry/releases) or + +``` +go get github.com/src-d/enry/cmd/enry +``` + +This project is now part of [source{d} Engine](https://sourced.tech/engine), +which provides the simplest way to get started with a single command. +Visit [sourced.tech/engine](https://sourced.tech/engine) for more information. + + +Examples +------------ + +```go +lang, safe := enry.GetLanguageByExtension("foo.go") +fmt.Println(lang, safe) +// result: Go true + +lang, safe := enry.GetLanguageByContent("foo.m", []byte("")) +fmt.Println(lang, safe) +// result: Matlab true + +lang, safe := enry.GetLanguageByContent("bar.m", []byte("")) +fmt.Println(lang, safe) +// result: Objective-C true + +// all strategies together +lang := enry.GetLanguage("foo.cpp", []byte("")) +// result: C++ true +``` + +Note that the returned boolean value `safe` is set either to `true`, if there is only one possible language detected, or to `false` otherwise. + +To get a list of possible languages for a given file, you can use the plural version of the detecting functions. + +```go +langs := enry.GetLanguages("foo.h", []byte("")) +// result: []string{"C", "C++", "Objective-C} + +langs := enry.GetLanguagesByExtension("foo.asc", []byte(""), nil) +// result: []string{"AGS Script", "AsciiDoc", "Public Key"} + +langs := enry.GetLanguagesByFilename("Gemfile", []byte(""), []string{}) +// result: []string{"Ruby"} +``` + + +CLI +------------ + +You can use enry as a command, + +```bash +$ enry --help +enry v2.0.0 build: 05-08-2019_20_40_35 commit: 6ccf0b6, based on linguist commit: e456098 +enry, A simple (and faster) implementation of github/linguist +usage: enry [-mode=(file|line|byte)] [-prog] + enry [-mode=(file|line|byte)] [-prog] [-json] [-breakdown] + enry [-mode=(file|line|byte)] [-prog] [-json] [-breakdown] + enry [-version] +``` + +and on repository root, it'll return an output similar to *linguist*'s output, + +```bash +$ enry +97.71% Go +1.60% C +0.31% Shell +0.22% Java +0.07% Ruby +0.05% Makefile +0.04% Scala +0.01% Gnuplot +``` + +but not only the output; its flags are also the same as *linguist*'s ones, + +```bash +$ enry --breakdown +97.71% Go +1.60% C +0.31% Shell +0.22% Java +0.07% Ruby +0.05% Makefile +0.04% Scala +0.01% Gnuplot + +Scala +java/build.sbt +java/project/plugins.sbt + +Java +java/src/main/java/tech/sourced/enry/Enry.java +java/src/main/java/tech/sourced/enry/GoUtils.java +java/src/main/java/tech/sourced/enry/Guess.java +java/src/test/java/tech/sourced/enry/EnryTest.java + +Makefile +Makefile +java/Makefile + +Go +benchmark_test.go +``` + +even the JSON flag, + +```bash +$ enry --json | jq . +{ + "C": [ + "internal/tokenizer/flex/lex.linguist_yy.c", + "internal/tokenizer/flex/lex.linguist_yy.h", + "internal/tokenizer/flex/linguist.h", + "python/_c_enry.c", + "python/enry.c" + ], + "Gnuplot": [ + "benchmarks/plot-histogram.gp" + ], + "Go": [ + "benchmark_test.go", +``` + +Note that enry's CLI **_doesn't need a git repository to work_**, which is intentionally different from the linguist. + +## Java bindings + + +Generated Java bindings using a C shared library and JNI are available under [`java`](https://github.com/src-d/enry/blob/master/java) and published on Maven at [tech.sourced:enry-java](https://mvnrepository.com/artifact/tech.sourced/enry-java) for macOS and linux. + + +## Python bindings +Generated Python bindings using a C shared library and cffi are not available yet and are WIP under [src-d/enry#154](https://github.com/src-d/enry/issues/154). + +Divergences from linguist +------------ + +The `enry` library is based on the data from `github/linguist` version **v7.5.1**. + +As opposed to linguist, `enry` [CLI tool](#cli) does *not* require a full Git repository in the filesystem in order to report languages. + +Parsing [linguist/samples](https://github.com/github/linguist/tree/master/samples) the following `enry` results are different from linguist: + +* [Heuristics for ".es" extension](https://github.com/github/linguist/blob/e761f9b013e5b61161481fcb898b59721ee40e3d/lib/linguist/heuristics.yml#L103) in JavaScript could not be parsed, due to unsupported backreference in RE2 regexp engine. + +* [Heuristics for ".rno" extension](https://github.com/github/linguist/blob/3a1bd3c3d3e741a8aaec4704f782e06f5cd2a00d/lib/linguist/heuristics.yml#L365) in RUNOFF could not be parsed, due to unsupported lookahead in RE2 regexp engine. + +* As of [Linguist v5.3.2](https://github.com/github/linguist/releases/tag/v5.3.2) it is using [flex-based scanner in C for tokenization](https://github.com/github/linguist/pull/3846). Enry still uses [extract_token](https://github.com/github/linguist/pull/3846/files#diff-d5179df0b71620e3fac4535cd1368d15L60) regex-based algorithm. See [#193](https://github.com/src-d/enry/issues/193). + +* Bayesian classifier can't distinguish "SQL" from "PLpgSQL. See [#194](https://github.com/src-d/enry/issues/194). + +* Detection of [generated files](https://github.com/github/linguist/blob/bf95666fc15e49d556f2def4d0a85338423c25f3/lib/linguist/generated.rb#L53) is not supported yet. + (Thus they are not excluded from CLI output). See [#213](https://github.com/src-d/enry/issues/213). + +* XML detection strategy is not implemented. See [#192](https://github.com/src-d/enry/issues/192). + +* Overriding languages and types though `.gitattributes` is not yet supported. See [#18](https://github.com/src-d/enry/issues/18). + +* `enry` CLI output does NOT exclude `.gitignore`ed files and git submodules, as linguist does + +In all the cases above that have an issue number - we plan to update enry to match Linguist behavior. + + +Benchmarks +------------ + +Enry's language detection has been compared with Linguist's one. In order to do that, Linguist's project directory [*linguist/samples*](https://github.com/github/linguist/tree/master/samples) was used as a set of files to run benchmarks against. + +We got these results: + +![histogram](benchmarks/histogram/distribution.png) + +The histogram shows the number of files detected (y-axis) per time interval bucket (x-axis). As one can see, most of the files were detected faster by enry. + +We found few cases where enry turns slower than linguist due to +Go regexp engine being slower than Ruby's, based on [oniguruma](https://github.com/kkos/oniguruma) library, written in C. + +See [instructions](#misc) for running enry with oniguruma. + + +Why Enry? +------------ + +In the movie [My Fair Lady](https://en.wikipedia.org/wiki/My_Fair_Lady), [Professor Henry Higgins](http://www.imdb.com/character/ch0011719/?ref_=tt_cl_t2) is one of the main characters. Henry is a linguist and at the very beginning of the movie enjoys guessing the origin of people based on their accent. + +"Enry Iggins" is how [Eliza Doolittle](http://www.imdb.com/character/ch0011720/?ref_=tt_cl_t1), [pronounces](https://www.youtube.com/watch?v=pwNKyTktDIE) the name of the Professor during the first half of the movie. + +## Development + +To build enry's CLI run: + + make build + +this will generate a binary in the project's root directory called `enry`. + +To run the tests: + + make test + + +### Sync with github/linguist upstream + +*enry* re-uses parts of the original [github/linguist](https://github.com/github/linguist) to generate internal data structures. +In order to update to the latest release of linguist do: + +```bash +$ git clone https://github.com/github/linguist.git .linguist +$ cd .linguist; git checkout ; cd .. + +# put the new release's commit sha in the generator_test.go (to re-generate .gold test fixtures) +# https://github.com/src-d/enry/blob/13d3d66d37a87f23a013246a1b0678c9ee3d524b/internal/code-generator/generator/generator_test.go#L18 + +$ make code-generate +``` + +To stay in sync, enry needs to be updated when a new release of the linguist includes changes to any of the following files: + +* [languages.yml](https://github.com/github/linguist/blob/master/lib/linguist/languages.yml) +* [heuristics.yml](https://github.com/github/linguist/blob/master/lib/linguist/heuristics.yml) +* [vendor.yml](https://github.com/github/linguist/blob/master/lib/linguist/vendor.yml) +* [documentation.yml](https://github.com/github/linguist/blob/master/lib/linguist/documentation.yml) + +There is no automation for detecting the changes in the linguist project, so this process above has to be done manually from time to time. + +When submitting a pull request syncing up to a new release, please make sure it only contains the changes in +the generated files (in [data](https://github.com/src-d/enry/blob/master/data) subdirectory). + +Separating all the necessary "manual" code changes to a different PR that includes some background description and an update to the documentation on ["divergences from linguist"](##divergences-from-linguist) is very much appreciated as it simplifies the maintenance (review/release notes/etc). + + + +## Misc + +
+ +### Benchmark + +All benchmark scripts are in [*benchmarks*](https://github.com/src-d/enry/blob/master/benchmarks) directory. + + +#### Dependencies +As benchmarks depend on Ruby and Github-Linguist gem make sure you have: + - Ruby (e.g using [`rbenv`](https://github.com/rbenv/rbenv)), [`bundler`](https://bundler.io/) installed + - Docker + - [native dependencies](https://github.com/github/linguist/#dependencies) installed + - Build the gem `cd .linguist && bundle install && rake build_gem && cd -` + - Install it `gem install --no-rdoc --no-ri --local .linguist/github-linguist-*.gem` + + +#### Quick benchmark +To run quicker benchmarks you can either: + + make benchmarks + +to get average times for the main detection function and strategies for the whole samples set or: + + make benchmarks-samples + +if you want to see measures per sample file. + + +#### Full benchmark +If you want to reproduce the same benchmarks as reported above: + - Make sure all [dependencies](#benchmark-dependencies) are installed + - Install [gnuplot](http://gnuplot.info) (in order to plot the histogram) + - Run `ENRY_TEST_REPO="$PWD/.linguist" benchmarks/run.sh` (takes ~15h) + +It will run the benchmarks for enry and linguist, parse the output, create csv files and plot the histogram. + +### Faster regexp engine (optional) + +[Oniguruma](https://github.com/kkos/oniguruma) is CRuby's regular expression engine. +It is very fast and performs better than the one built into Go runtime. *enry* supports swapping +between those two engines thanks to [rubex](https://github.com/moovweb/rubex) project. +The typical overall speedup from using Oniguruma is 1.5-2x. However, it requires CGo and the external shared library. +On macOS with [Homebrew](https://brew.sh/), it is: + +``` +brew install oniguruma +``` + +On Ubuntu, it is + +``` +sudo apt install libonig-dev +``` + +To build enry with Oniguruma regexps use the `oniguruma` build tag + +``` +go get -v -t --tags oniguruma ./... +``` + +and then rebuild the project. + +
+ + +License +------------ + +Apache License, Version 2.0. See [LICENSE](LICENSE) diff --git a/vendor/github.com/src-d/enry/v2/classifier.go b/vendor/github.com/src-d/enry/v2/classifier.go new file mode 100644 index 000000000..e70efc38f --- /dev/null +++ b/vendor/github.com/src-d/enry/v2/classifier.go @@ -0,0 +1,107 @@ +package enry + +import ( + "math" + "sort" + + "github.com/src-d/enry/v2/internal/tokenizer" +) + +// Classifier is the interface in charge to detect the possible languages of the given content based on a set of +// candidates. Candidates is a map which can be used to assign weights to languages dynamically. +type Classifier interface { + Classify(content []byte, candidates map[string]float64) (languages []string) +} + +type classifier struct { + languagesLogProbabilities map[string]float64 + tokensLogProbabilities map[string]map[string]float64 + tokensTotal float64 +} + +type scoredLanguage struct { + language string + score float64 +} + +// Classify returns a sorted slice of possible languages sorted by decreasing language's probability +func (c *classifier) Classify(content []byte, candidates map[string]float64) []string { + + var languages map[string]float64 + if len(candidates) == 0 { + languages = c.knownLangs() + } else { + languages = make(map[string]float64, len(candidates)) + for candidate, weight := range candidates { + if lang, ok := GetLanguageByAlias(candidate); ok { + candidate = lang + } + + languages[candidate] = weight + } + } + + empty := len(content) == 0 + scoredLangs := make([]*scoredLanguage, 0, len(languages)) + + var tokens []string + if !empty { + tokens = tokenizer.Tokenize(content) + } + + for language := range languages { + score := c.languagesLogProbabilities[language] + if !empty { + score += c.tokensLogProbability(tokens, language) + } + scoredLangs = append(scoredLangs, &scoredLanguage{ + language: language, + score: score, + }) + } + + return sortLanguagesByScore(scoredLangs) +} + +func sortLanguagesByScore(scoredLangs []*scoredLanguage) []string { + sort.Stable(byScore(scoredLangs)) + sortedLanguages := make([]string, 0, len(scoredLangs)) + for _, scoredLang := range scoredLangs { + sortedLanguages = append(sortedLanguages, scoredLang.language) + } + + return sortedLanguages +} + +func (c *classifier) knownLangs() map[string]float64 { + langs := make(map[string]float64, len(c.languagesLogProbabilities)) + for lang := range c.languagesLogProbabilities { + langs[lang]++ + } + + return langs +} + +func (c *classifier) tokensLogProbability(tokens []string, language string) float64 { + var sum float64 + for _, token := range tokens { + sum += c.tokenProbability(token, language) + } + + return sum +} + +func (c *classifier) tokenProbability(token, language string) float64 { + tokenProb, ok := c.tokensLogProbabilities[language][token] + if !ok { + tokenProb = math.Log(1.000000 / c.tokensTotal) + } + + return tokenProb +} + +type byScore []*scoredLanguage + +func (b byScore) Len() int { return len(b) } +func (b byScore) Swap(i, j int) { b[i], b[j] = b[j], b[i] } +func (b byScore) Less(i, j int) bool { return b[j].score < b[i].score } diff --git a/vendor/github.com/src-d/enry/v2/common.go b/vendor/github.com/src-d/enry/v2/common.go new file mode 100644 index 000000000..058d88486 --- /dev/null +++ b/vendor/github.com/src-d/enry/v2/common.go @@ -0,0 +1,472 @@ +package enry + +import ( + "bufio" + "bytes" + "path/filepath" + "strings" + + "github.com/src-d/enry/v2/data" + "github.com/src-d/enry/v2/regex" +) + +// OtherLanguage is used as a zero value when a function can not return a specific language. +const OtherLanguage = "" + +// Strategy type fix the signature for the functions that can be used as a strategy. +type Strategy func(filename string, content []byte, candidates []string) (languages []string) + +// DefaultStrategies is a sequence of strategies used by GetLanguage to detect languages. +var DefaultStrategies = []Strategy{ + GetLanguagesByModeline, + GetLanguagesByFilename, + GetLanguagesByShebang, + GetLanguagesByExtension, + GetLanguagesByContent, + GetLanguagesByClassifier, +} + +// DefaultClassifier is a Naive Bayes classifier trained on Linguist samples. +var DefaultClassifier Classifier = &classifier{ + languagesLogProbabilities: data.LanguagesLogProbabilities, + tokensLogProbabilities: data.TokensLogProbabilities, + tokensTotal: data.TokensTotal, +} + +// GetLanguage applies a sequence of strategies based on the given filename and content +// to find out the most probably language to return. +func GetLanguage(filename string, content []byte) (language string) { + languages := GetLanguages(filename, content) + return firstLanguage(languages) +} + +func firstLanguage(languages []string) string { + for _, l := range languages { + if l != "" { + return l + } + } + return OtherLanguage +} + +// GetLanguageByModeline returns detected language. If there are more than one possibles languages +// it returns the first language by alphabetically order and safe to false. +func GetLanguageByModeline(content []byte) (language string, safe bool) { + return getLanguageByStrategy(GetLanguagesByModeline, "", content, nil) +} + +// GetLanguageByEmacsModeline returns detected language. If there are more than one possibles languages +// it returns the first language by alphabetically order and safe to false. +func GetLanguageByEmacsModeline(content []byte) (language string, safe bool) { + return getLanguageByStrategy(GetLanguagesByEmacsModeline, "", content, nil) +} + +// GetLanguageByVimModeline returns detected language. If there are more than one possibles languages +// it returns the first language by alphabetically order and safe to false. +func GetLanguageByVimModeline(content []byte) (language string, safe bool) { + return getLanguageByStrategy(GetLanguagesByVimModeline, "", content, nil) +} + +// GetLanguageByFilename returns detected language. If there are more than one possibles languages +// it returns the first language by alphabetically order and safe to false. +func GetLanguageByFilename(filename string) (language string, safe bool) { + return getLanguageByStrategy(GetLanguagesByFilename, filename, nil, nil) +} + +// GetLanguageByShebang returns detected language. If there are more than one possibles languages +// it returns the first language by alphabetically order and safe to false. +func GetLanguageByShebang(content []byte) (language string, safe bool) { + return getLanguageByStrategy(GetLanguagesByShebang, "", content, nil) +} + +// GetLanguageByExtension returns detected language. If there are more than one possibles languages +// it returns the first language by alphabetically order and safe to false. +func GetLanguageByExtension(filename string) (language string, safe bool) { + return getLanguageByStrategy(GetLanguagesByExtension, filename, nil, nil) +} + +// GetLanguageByContent returns detected language. If there are more than one possibles languages +// it returns the first language by alphabetically order and safe to false. +func GetLanguageByContent(filename string, content []byte) (language string, safe bool) { + return getLanguageByStrategy(GetLanguagesByContent, filename, content, nil) +} + +// GetLanguageByClassifier returns the most probably language detected for the given content. It uses +// DefaultClassifier, if no candidates are provided it returns OtherLanguage. +func GetLanguageByClassifier(content []byte, candidates []string) (language string, safe bool) { + return getLanguageByStrategy(GetLanguagesByClassifier, "", content, candidates) +} + +func getLanguageByStrategy(strategy Strategy, filename string, content []byte, candidates []string) (string, bool) { + languages := strategy(filename, content, candidates) + return getFirstLanguageAndSafe(languages) +} + +func getFirstLanguageAndSafe(languages []string) (language string, safe bool) { + language = firstLanguage(languages) + safe = len(languages) == 1 + return +} + +// GetLanguageBySpecificClassifier returns the most probably language for the given content using +// classifier to detect language. +func GetLanguageBySpecificClassifier(content []byte, candidates []string, classifier Classifier) (language string, safe bool) { + languages := GetLanguagesBySpecificClassifier(content, candidates, classifier) + return getFirstLanguageAndSafe(languages) +} + +// GetLanguages applies a sequence of strategies based on the given filename and content +// to find out the most probably languages to return. +// At least one of arguments should be set. If content is missing, language detection will be based on the filename. +// The function won't read the file, given an empty content. +func GetLanguages(filename string, content []byte) []string { + if IsBinary(content) { + return nil + } + + var languages []string + candidates := []string{} + for _, strategy := range DefaultStrategies { + languages = strategy(filename, content, candidates) + if len(languages) == 1 { + return languages + } + + if len(languages) > 0 { + candidates = append(candidates, languages...) + } + } + + return languages +} + +// GetLanguagesByModeline returns a slice of possible languages for the given content. +// It complies with the signature to be a Strategy type. +func GetLanguagesByModeline(_ string, content []byte, candidates []string) []string { + headFoot := getHeaderAndFooter(content) + var languages []string + for _, getLang := range modelinesFunc { + languages = getLang("", headFoot, candidates) + if len(languages) > 0 { + break + } + } + + return languages +} + +var modelinesFunc = []Strategy{ + GetLanguagesByEmacsModeline, + GetLanguagesByVimModeline, +} + +func getHeaderAndFooter(content []byte) []byte { + const searchScope = 5 + + if len(content) == 0 { + return content + } + + if bytes.Count(content, []byte("\n")) < 2*searchScope { + return content + } + + header := headScope(content, searchScope) + footer := footScope(content, searchScope) + headerAndFooter := make([]byte, 0, len(content[:header])+len(content[footer:])) + headerAndFooter = append(headerAndFooter, content[:header]...) + headerAndFooter = append(headerAndFooter, content[footer:]...) + return headerAndFooter +} + +func headScope(content []byte, scope int) (index int) { + for i := 0; i < scope; i++ { + eol := bytes.IndexAny(content, "\n") + content = content[eol+1:] + index += eol + } + + return index + scope - 1 +} + +func footScope(content []byte, scope int) (index int) { + for i := 0; i < scope; i++ { + index = bytes.LastIndexAny(content, "\n") + content = content[:index] + } + + return index + 1 +} + +var ( + reEmacsModeline = regex.MustCompile(`.*-\*-\s*(.+?)\s*-\*-.*(?m:$)`) + reEmacsLang = regex.MustCompile(`.*(?i:mode)\s*:\s*([^\s;]+)\s*;*.*`) + reVimModeline = regex.MustCompile(`(?:(?m:\s|^)vi(?:m[<=>]?\d+|m)?|[\t\x20]*ex)\s*[:]\s*(.*)(?m:$)`) + reVimLang = regex.MustCompile(`(?i:filetype|ft|syntax)\s*=(\w+)(?:\s|:|$)`) +) + +// GetLanguagesByEmacsModeline returns a slice of possible languages for the given content. +// It complies with the signature to be a Strategy type. +func GetLanguagesByEmacsModeline(_ string, content []byte, _ []string) []string { + matched := reEmacsModeline.FindAllSubmatch(content, -1) + if matched == nil { + return nil + } + + // only take the last matched line, discard previous lines + lastLineMatched := matched[len(matched)-1][1] + matchedAlias := reEmacsLang.FindSubmatch(lastLineMatched) + var alias string + if matchedAlias != nil { + alias = string(matchedAlias[1]) + } else { + alias = string(lastLineMatched) + } + + language, ok := GetLanguageByAlias(alias) + if !ok { + return nil + } + + return []string{language} +} + +// GetLanguagesByVimModeline returns a slice of possible languages for the given content. +// It complies with the signature to be a Strategy type. +func GetLanguagesByVimModeline(_ string, content []byte, _ []string) []string { + matched := reVimModeline.FindAllSubmatch(content, -1) + if matched == nil { + return nil + } + + // only take the last matched line, discard previous lines + lastLineMatched := matched[len(matched)-1][1] + matchedAlias := reVimLang.FindAllSubmatch(lastLineMatched, -1) + if matchedAlias == nil { + return nil + } + + alias := string(matchedAlias[0][1]) + if len(matchedAlias) > 1 { + // cases: + // matchedAlias = [["syntax=ruby " "ruby"] ["ft=python " "python"] ["filetype=perl " "perl"]] returns OtherLanguage; + // matchedAlias = [["syntax=python " "python"] ["ft=python " "python"] ["filetype=python " "python"]] returns "Python"; + for _, match := range matchedAlias { + otherAlias := string(match[1]) + if otherAlias != alias { + return nil + } + } + } + + language, ok := GetLanguageByAlias(alias) + if !ok { + return nil + } + + return []string{language} +} + +// GetLanguagesByFilename returns a slice of possible languages for the given filename. +// It complies with the signature to be a Strategy type. +func GetLanguagesByFilename(filename string, _ []byte, _ []string) []string { + if filename == "" { + return nil + } + + return data.LanguagesByFilename[filepath.Base(filename)] +} + +// GetLanguagesByShebang returns a slice of possible languages for the given content. +// It complies with the signature to be a Strategy type. +func GetLanguagesByShebang(_ string, content []byte, _ []string) (languages []string) { + interpreter := getInterpreter(content) + return data.LanguagesByInterpreter[interpreter] +} + +var ( + shebangExecHack = regex.MustCompile(`exec (\w+).+\$0.+\$@`) + pythonVersion = regex.MustCompile(`python\d\.\d+`) +) + +func getInterpreter(data []byte) (interpreter string) { + line := getFirstLine(data) + if !hasShebang(line) { + return "" + } + + // skip shebang + line = bytes.TrimSpace(line[2:]) + splitted := bytes.Fields(line) + if len(splitted) == 0 { + return "" + } + + if bytes.Contains(splitted[0], []byte("env")) { + if len(splitted) > 1 { + interpreter = string(splitted[1]) + } + } else { + splittedPath := bytes.Split(splitted[0], []byte{'/'}) + interpreter = string(splittedPath[len(splittedPath)-1]) + } + + if interpreter == "sh" { + interpreter = lookForMultilineExec(data) + } + + if pythonVersion.MatchString(interpreter) { + interpreter = interpreter[:strings.Index(interpreter, `.`)] + } + + // If osascript is called with argument -l it could be different language so do not relay on it + // To match linguist behaviour, see ref https://github.com/github/linguist/blob/d95bae794576ab0ef2fcb41a39eb61ea5302c5b5/lib/linguist/shebang.rb#L63 + if interpreter == "osascript" && bytes.Contains(line, []byte("-l")) { + interpreter = "" + } + + return +} + +func getFirstLine(data []byte) []byte { + buf := bufio.NewScanner(bytes.NewReader(data)) + buf.Scan() + line := buf.Bytes() + if err := buf.Err(); err != nil { + return nil + } + + return line +} + +func hasShebang(line []byte) bool { + const shebang = `#!` + prefix := []byte(shebang) + return bytes.HasPrefix(line, prefix) +} + +func lookForMultilineExec(data []byte) string { + const magicNumOfLines = 5 + interpreter := "sh" + + buf := bufio.NewScanner(bytes.NewReader(data)) + for i := 0; i < magicNumOfLines && buf.Scan(); i++ { + line := buf.Bytes() + if shebangExecHack.Match(line) { + interpreter = shebangExecHack.FindStringSubmatch(string(line))[1] + break + } + } + + if err := buf.Err(); err != nil { + return interpreter + } + + return interpreter +} + +// GetLanguagesByExtension returns a slice of possible languages for the given filename. +// It complies with the signature to be a Strategy type. +func GetLanguagesByExtension(filename string, _ []byte, _ []string) []string { + if !strings.Contains(filename, ".") { + return nil + } + + filename = strings.ToLower(filename) + dots := getDotIndexes(filename) + for _, dot := range dots { + ext := filename[dot:] + languages, ok := data.LanguagesByExtension[ext] + if ok { + return languages + } + } + + return nil +} + +func getDotIndexes(filename string) []int { + dots := make([]int, 0, 2) + for i, letter := range filename { + if letter == rune('.') { + dots = append(dots, i) + } + } + + return dots +} + +// GetLanguagesByContent returns a slice of languages for the given content. +// It is a Strategy that uses content-based regexp heuristics and a filename extension. +func GetLanguagesByContent(filename string, content []byte, _ []string) []string { + if filename == "" { + return nil + } + + ext := strings.ToLower(filepath.Ext(filename)) + + heuristic, ok := data.ContentHeuristics[ext] + if !ok { + return nil + } + + return heuristic.Match(content) +} + +// GetLanguagesByClassifier uses DefaultClassifier as a Classifier and returns a sorted slice of possible languages ordered by +// decreasing language's probability. If there are not candidates it returns nil. It complies with the signature to be a Strategy type. +func GetLanguagesByClassifier(filename string, content []byte, candidates []string) (languages []string) { + if len(candidates) == 0 { + return nil + } + + return GetLanguagesBySpecificClassifier(content, candidates, DefaultClassifier) +} + +// GetLanguagesBySpecificClassifier returns a slice of possible languages. It takes in a Classifier to be used. +func GetLanguagesBySpecificClassifier(content []byte, candidates []string, classifier Classifier) (languages []string) { + mapCandidates := make(map[string]float64) + for _, candidate := range candidates { + mapCandidates[candidate]++ + } + + return classifier.Classify(content, mapCandidates) +} + +// GetLanguageExtensions returns the different extensions being used by the language. +func GetLanguageExtensions(language string) []string { + return data.ExtensionsByLanguage[language] +} + +// Type represent language's type. Either data, programming, markup, prose, or unknown. +type Type int + +// Type's values. +const ( + Unknown Type = iota + Data + Programming + Markup + Prose +) + +// GetLanguageType returns the type of the given language. +func GetLanguageType(language string) (langType Type) { + intType, ok := data.LanguagesType[language] + langType = Type(intType) + if !ok { + langType = Unknown + } + return langType +} + +// GetLanguageByAlias returns either the language related to the given alias and ok set to true +// or Otherlanguage and ok set to false if the alias is not recognized. +func GetLanguageByAlias(alias string) (lang string, ok bool) { + lang, ok = data.LanguageByAlias(alias) + if !ok { + lang = OtherLanguage + } + + return +} diff --git a/vendor/github.com/src-d/enry/v2/data/alias.go b/vendor/github.com/src-d/enry/v2/data/alias.go new file mode 100644 index 000000000..638be239e --- /dev/null +++ b/vendor/github.com/src-d/enry/v2/data/alias.go @@ -0,0 +1,783 @@ +// Code generated by github.com/src-d/enry/v2/internal/code-generator DO NOT EDIT. +// Extracted from github/linguist commit: 3a1bd3c3d3e741a8aaec4704f782e06f5cd2a00d + +package data + +import "strings" + +// LanguageByAliasMap keeps alias for different languages and use the name of the languages as an alias too. +// All the keys (alias or not) are written in lower case and the whitespaces has been replaced by underscores. +var LanguageByAliasMap = map[string]string{ + "1c_enterprise": "1C Enterprise", + "abap": "ABAP", + "abl": "OpenEdge ABL", + "abnf": "ABNF", + "abuild": "Alpine Abuild", + "acfm": "Adobe Font Metrics", + "aconf": "ApacheConf", + "actionscript": "ActionScript", + "actionscript3": "ActionScript", + "actionscript_3": "ActionScript", + "ada": "Ada", + "ada2005": "Ada", + "ada95": "Ada", + "adobe_composite_font_metrics": "Adobe Font Metrics", + "adobe_font_metrics": "Adobe Font Metrics", + "adobe_multiple_font_metrics": "Adobe Font Metrics", + "advpl": "xBase", + "afdko": "OpenType Feature File", + "agda": "Agda", + "ags": "AGS Script", + "ags_script": "AGS Script", + "ahk": "AutoHotkey", + "alloy": "Alloy", + "alpine_abuild": "Alpine Abuild", + "altium": "Altium Designer", + "altium_designer": "Altium Designer", + "amfm": "Adobe Font Metrics", + "ampl": "AMPL", + "angelscript": "AngelScript", + "ant_build_system": "Ant Build System", + "antlr": "ANTLR", + "apache": "ApacheConf", + "apacheconf": "ApacheConf", + "apex": "Apex", + "api_blueprint": "API Blueprint", + "apkbuild": "Alpine Abuild", + "apl": "APL", + "apollo_guidance_computer": "Apollo Guidance Computer", + "applescript": "AppleScript", + "arc": "Arc", + "arexx": "REXX", + "as3": "ActionScript", + "asciidoc": "AsciiDoc", + "asm": "Assembly", + "asn.1": "ASN.1", + "asp": "ASP", + "aspectj": "AspectJ", + "aspx": "ASP", + "aspx-vb": "ASP", + "assembly": "Assembly", + "asymptote": "Asymptote", + "ats": "ATS", + "ats2": "ATS", + "au3": "AutoIt", + "augeas": "Augeas", + "autoconf": "M4Sugar", + "autohotkey": "AutoHotkey", + "autoit": "AutoIt", + "autoit3": "AutoIt", + "autoitscript": "AutoIt", + "awk": "Awk", + "b3d": "BlitzBasic", + "ballerina": "Ballerina", + "bash": "Shell", + "bash_session": "ShellSession", + "bat": "Batchfile", + "batch": "Batchfile", + "batchfile": "Batchfile", + "befunge": "Befunge", + "bison": "Bison", + "bitbake": "BitBake", + "blade": "Blade", + "blitz3d": "BlitzBasic", + "blitzbasic": "BlitzBasic", + "blitzmax": "BlitzMax", + "blitzplus": "BlitzBasic", + "bluespec": "Bluespec", + "bmax": "BlitzMax", + "boo": "Boo", + "bplus": "BlitzBasic", + "brainfuck": "Brainfuck", + "brightscript": "Brightscript", + "bro": "Zeek", + "bsdmake": "Makefile", + "byond": "DM", + "c": "C", + "c#": "C#", + "c++": "C++", + "c++-objdump": "Cpp-ObjDump", + "c-objdump": "C-ObjDump", + "c2hs": "C2hs Haskell", + "c2hs_haskell": "C2hs Haskell", + "cabal": "Cabal Config", + "cabal_config": "Cabal Config", + "cap'n_proto": "Cap'n Proto", + "carto": "CartoCSS", + "cartocss": "CartoCSS", + "ceylon": "Ceylon", + "cfc": "ColdFusion CFC", + "cfm": "ColdFusion", + "cfml": "ColdFusion", + "chapel": "Chapel", + "charity": "Charity", + "chpl": "Chapel", + "chuck": "ChucK", + "cirru": "Cirru", + "clarion": "Clarion", + "clean": "Clean", + "click": "Click", + "clipper": "xBase", + "clips": "CLIPS", + "clojure": "Clojure", + "closure_templates": "Closure Templates", + "cloud_firestore_security_rules": "Cloud Firestore Security Rules", + "cmake": "CMake", + "cobol": "COBOL", + "coffee": "CoffeeScript", + "coffee-script": "CoffeeScript", + "coffeescript": "CoffeeScript", + "coldfusion": "ColdFusion", + "coldfusion_cfc": "ColdFusion CFC", + "coldfusion_html": "ColdFusion", + "collada": "COLLADA", + "common_lisp": "Common Lisp", + "common_workflow_language": "Common Workflow Language", + "component_pascal": "Component Pascal", + "conll": "CoNLL-U", + "conll-u": "CoNLL-U", + "conll-x": "CoNLL-U", + "console": "ShellSession", + "cool": "Cool", + "coq": "Coq", + "cperl": "Perl", + "cpp": "C++", + "cpp-objdump": "Cpp-ObjDump", + "creole": "Creole", + "crystal": "Crystal", + "csharp": "C#", + "cson": "CSON", + "csound": "Csound", + "csound-csd": "Csound Document", + "csound-orc": "Csound", + "csound-sco": "Csound Score", + "csound_document": "Csound Document", + "csound_score": "Csound Score", + "css": "CSS", + "csv": "CSV", + "cucumber": "Gherkin", + "cuda": "Cuda", + "cweb": "CWeb", + "cwl": "Common Workflow Language", + "cycript": "Cycript", + "cython": "Cython", + "d": "D", + "d-objdump": "D-ObjDump", + "darcs_patch": "Darcs Patch", + "dart": "Dart", + "dataweave": "DataWeave", + "dcl": "DIGITAL Command Language", + "delphi": "Component Pascal", + "desktop": "desktop", + "dhall": "Dhall", + "diff": "Diff", + "digital_command_language": "DIGITAL Command Language", + "django": "HTML+Django", + "dm": "DM", + "dns_zone": "DNS Zone", + "dockerfile": "Dockerfile", + "dogescript": "Dogescript", + "dosbatch": "Batchfile", + "dosini": "INI", + "dpatch": "Darcs Patch", + "dtrace": "DTrace", + "dtrace-script": "DTrace", + "dylan": "Dylan", + "e": "E", + "eagle": "Eagle", + "easybuild": "Easybuild", + "ebnf": "EBNF", + "ec": "eC", + "ecere_projects": "Ecere Projects", + "ecl": "ECL", + "eclipse": "ECLiPSe", + "ecr": "HTML+ECR", + "editor-config": "EditorConfig", + "editorconfig": "EditorConfig", + "edje_data_collection": "Edje Data Collection", + "edn": "edn", + "eeschema_schematic": "KiCad Schematic", + "eex": "HTML+EEX", + "eiffel": "Eiffel", + "ejs": "EJS", + "elisp": "Emacs Lisp", + "elixir": "Elixir", + "elm": "Elm", + "emacs": "Emacs Lisp", + "emacs_lisp": "Emacs Lisp", + "emberscript": "EmberScript", + "eml": "EML", + "eq": "EQ", + "erb": "HTML+ERB", + "erlang": "Erlang", + "f#": "F#", + "f*": "F*", + "factor": "Factor", + "fancy": "Fancy", + "fantom": "Fantom", + "figfont": "FIGlet Font", + "figlet_font": "FIGlet Font", + "filebench_wml": "Filebench WML", + "filterscript": "Filterscript", + "fish": "fish", + "flex": "Lex", + "flux": "FLUX", + "formatted": "Formatted", + "forth": "Forth", + "fortran": "Fortran", + "foxpro": "xBase", + "freemarker": "FreeMarker", + "frege": "Frege", + "fsharp": "F#", + "fstar": "F*", + "ftl": "FreeMarker", + "fundamental": "Text", + "g-code": "G-code", + "game_maker_language": "Game Maker Language", + "gams": "GAMS", + "gap": "GAP", + "gcc_machine_description": "GCC Machine Description", + "gdb": "GDB", + "gdscript": "GDScript", + "genie": "Genie", + "genshi": "Genshi", + "gentoo_ebuild": "Gentoo Ebuild", + "gentoo_eclass": "Gentoo Eclass", + "gerber_image": "Gerber Image", + "gettext_catalog": "Gettext Catalog", + "gf": "Grammatical Framework", + "gherkin": "Gherkin", + "git-ignore": "Ignore List", + "git_attributes": "Git Attributes", + "git_config": "Git Config", + "gitattributes": "Git Attributes", + "gitconfig": "Git Config", + "gitignore": "Ignore List", + "gitmodules": "Git Config", + "glsl": "GLSL", + "glyph": "Glyph", + "glyph_bitmap_distribution_format": "Glyph Bitmap Distribution Format", + "gn": "GN", + "gnuplot": "Gnuplot", + "go": "Go", + "golang": "Go", + "golo": "Golo", + "gosu": "Gosu", + "grace": "Grace", + "gradle": "Gradle", + "grammatical_framework": "Grammatical Framework", + "graph_modeling_language": "Graph Modeling Language", + "graphql": "GraphQL", + "graphviz_(dot)": "Graphviz (DOT)", + "groff": "Roff", + "groovy": "Groovy", + "groovy_server_pages": "Groovy Server Pages", + "gsp": "Groovy Server Pages", + "hack": "Hack", + "haml": "Haml", + "handlebars": "Handlebars", + "haproxy": "HAProxy", + "harbour": "Harbour", + "haskell": "Haskell", + "haxe": "Haxe", + "hbs": "Handlebars", + "hcl": "HCL", + "hiveql": "HiveQL", + "hlsl": "HLSL", + "holyc": "HolyC", + "html": "HTML", + "html+django": "HTML+Django", + "html+django/jinja": "HTML+Django", + "html+ecr": "HTML+ECR", + "html+eex": "HTML+EEX", + "html+erb": "HTML+ERB", + "html+jinja": "HTML+Django", + "html+php": "HTML+PHP", + "html+razor": "HTML+Razor", + "html+ruby": "RHTML", + "htmlbars": "Handlebars", + "htmldjango": "HTML+Django", + "http": "HTTP", + "hxml": "HXML", + "hy": "Hy", + "hylang": "Hy", + "hyphy": "HyPhy", + "i7": "Inform 7", + "idl": "IDL", + "idris": "Idris", + "ignore": "Ignore List", + "ignore_list": "Ignore List", + "igor": "IGOR Pro", + "igor_pro": "IGOR Pro", + "igorpro": "IGOR Pro", + "inc": "PHP", + "inform7": "Inform 7", + "inform_7": "Inform 7", + "ini": "INI", + "inno_setup": "Inno Setup", + "io": "Io", + "ioke": "Ioke", + "ipython_notebook": "Jupyter Notebook", + "irc": "IRC log", + "irc_log": "IRC log", + "irc_logs": "IRC log", + "isabelle": "Isabelle", + "isabelle_root": "Isabelle ROOT", + "j": "J", + "jasmin": "Jasmin", + "java": "Java", + "java_properties": "Java Properties", + "java_server_page": "Groovy Server Pages", + "java_server_pages": "Java Server Pages", + "javascript": "JavaScript", + "javascript+erb": "JavaScript+ERB", + "jflex": "JFlex", + "jison": "Jison", + "jison_lex": "Jison Lex", + "jolie": "Jolie", + "jruby": "Ruby", + "js": "JavaScript", + "json": "JSON", + "json5": "JSON5", + "json_with_comments": "JSON with Comments", + "jsonc": "JSON with Comments", + "jsoniq": "JSONiq", + "jsonld": "JSONLD", + "jsonnet": "Jsonnet", + "jsp": "Java Server Pages", + "jsx": "JSX", + "julia": "Julia", + "jupyter_notebook": "Jupyter Notebook", + "kicad_layout": "KiCad Layout", + "kicad_legacy_layout": "KiCad Legacy Layout", + "kicad_schematic": "KiCad Schematic", + "kit": "Kit", + "kotlin": "Kotlin", + "krl": "KRL", + "labview": "LabVIEW", + "lasso": "Lasso", + "lassoscript": "Lasso", + "latex": "TeX", + "latte": "Latte", + "lean": "Lean", + "less": "Less", + "lex": "Lex", + "lfe": "LFE", + "lhaskell": "Literate Haskell", + "lhs": "Literate Haskell", + "lilypond": "LilyPond", + "limbo": "Limbo", + "linker_script": "Linker Script", + "linux_kernel_module": "Linux Kernel Module", + "liquid": "Liquid", + "lisp": "Common Lisp", + "litcoffee": "Literate CoffeeScript", + "literate_agda": "Literate Agda", + "literate_coffeescript": "Literate CoffeeScript", + "literate_haskell": "Literate Haskell", + "live-script": "LiveScript", + "livescript": "LiveScript", + "llvm": "LLVM", + "logos": "Logos", + "logtalk": "Logtalk", + "lolcode": "LOLCODE", + "lookml": "LookML", + "loomscript": "LoomScript", + "ls": "LiveScript", + "lsl": "LSL", + "ltspice_symbol": "LTspice Symbol", + "lua": "Lua", + "m": "M", + "m4": "M4", + "m4sugar": "M4Sugar", + "macruby": "Ruby", + "make": "Makefile", + "makefile": "Makefile", + "mako": "Mako", + "man": "Roff", + "man-page": "Roff", + "man_page": "Roff", + "manpage": "Roff", + "markdown": "Markdown", + "marko": "Marko", + "markojs": "Marko", + "mask": "Mask", + "mathematica": "Mathematica", + "matlab": "MATLAB", + "maven_pom": "Maven POM", + "max": "Max", + "max/msp": "Max", + "maxmsp": "Max", + "maxscript": "MAXScript", + "mcfunction": "mcfunction", + "mdoc": "Roff", + "mediawiki": "MediaWiki", + "mercury": "Mercury", + "meson": "Meson", + "metal": "Metal", + "mf": "Makefile", + "minid": "MiniD", + "mirah": "Mirah", + "mma": "Mathematica", + "modelica": "Modelica", + "modula-2": "Modula-2", + "modula-3": "Modula-3", + "module_management_system": "Module Management System", + "monkey": "Monkey", + "moocode": "Moocode", + "moonscript": "MoonScript", + "motorola_68k_assembly": "Motorola 68K Assembly", + "mql4": "MQL4", + "mql5": "MQL5", + "mtml": "MTML", + "muf": "MUF", + "mumps": "M", + "mupad": "mupad", + "myghty": "Myghty", + "nanorc": "nanorc", + "nasm": "Assembly", + "ncl": "NCL", + "nearley": "Nearley", + "nemerle": "Nemerle", + "nesc": "nesC", + "netlinx": "NetLinx", + "netlinx+erb": "NetLinx+ERB", + "netlogo": "NetLogo", + "newlisp": "NewLisp", + "nextflow": "Nextflow", + "nginx": "Nginx", + "nginx_configuration_file": "Nginx", + "nim": "Nim", + "ninja": "Ninja", + "nit": "Nit", + "nix": "Nix", + "nixos": "Nix", + "njk": "HTML+Django", + "nl": "NL", + "node": "JavaScript", + "nroff": "Roff", + "nsis": "NSIS", + "nu": "Nu", + "numpy": "NumPy", + "nunjucks": "HTML+Django", + "nush": "Nu", + "nvim": "Vim script", + "obj-c": "Objective-C", + "obj-c++": "Objective-C++", + "obj-j": "Objective-J", + "objc": "Objective-C", + "objc++": "Objective-C++", + "objdump": "ObjDump", + "objective-c": "Objective-C", + "objective-c++": "Objective-C++", + "objective-j": "Objective-J", + "objectivec": "Objective-C", + "objectivec++": "Objective-C++", + "objectivej": "Objective-J", + "objectpascal": "Component Pascal", + "objectscript": "ObjectScript", + "objj": "Objective-J", + "ocaml": "OCaml", + "octave": "MATLAB", + "omgrofl": "Omgrofl", + "oncrpc": "RPC", + "ooc": "ooc", + "opa": "Opa", + "opal": "Opal", + "opencl": "OpenCL", + "openedge": "OpenEdge ABL", + "openedge_abl": "OpenEdge ABL", + "openrc": "OpenRC runscript", + "openrc_runscript": "OpenRC runscript", + "openscad": "OpenSCAD", + "opentype_feature_file": "OpenType Feature File", + "org": "Org", + "osascript": "AppleScript", + "ox": "Ox", + "oxygene": "Oxygene", + "oz": "Oz", + "p4": "P4", + "pan": "Pan", + "pandoc": "Markdown", + "papyrus": "Papyrus", + "parrot": "Parrot", + "parrot_assembly": "Parrot Assembly", + "parrot_internal_representation": "Parrot Internal Representation", + "pascal": "Pascal", + "pasm": "Parrot Assembly", + "pawn": "Pawn", + "pcbnew": "KiCad Layout", + "pep8": "Pep8", + "perl": "Perl", + "perl6": "Perl 6", + "perl_6": "Perl 6", + "php": "PHP", + "pic": "Pic", + "pickle": "Pickle", + "picolisp": "PicoLisp", + "piglatin": "PigLatin", + "pike": "Pike", + "pir": "Parrot Internal Representation", + "plpgsql": "PLpgSQL", + "plsql": "PLSQL", + "pod": "Pod", + "pod_6": "Pod 6", + "pogoscript": "PogoScript", + "pony": "Pony", + "posh": "PowerShell", + "postcss": "PostCSS", + "postscr": "PostScript", + "postscript": "PostScript", + "pot": "Gettext Catalog", + "pov-ray": "POV-Ray SDL", + "pov-ray_sdl": "POV-Ray SDL", + "povray": "POV-Ray SDL", + "powerbuilder": "PowerBuilder", + "powershell": "PowerShell", + "processing": "Processing", + "progress": "OpenEdge ABL", + "prolog": "Prolog", + "propeller_spin": "Propeller Spin", + "protobuf": "Protocol Buffer", + "protocol_buffer": "Protocol Buffer", + "protocol_buffers": "Protocol Buffer", + "public_key": "Public Key", + "pug": "Pug", + "puppet": "Puppet", + "pure_data": "Pure Data", + "purebasic": "PureBasic", + "purescript": "PureScript", + "pwsh": "PowerShell", + "pycon": "Python console", + "pyrex": "Cython", + "python": "Python", + "python3": "Python", + "python_console": "Python console", + "python_traceback": "Python traceback", + "q": "q", + "qmake": "QMake", + "qml": "QML", + "quake": "Quake", + "r": "R", + "racket": "Racket", + "ragel": "Ragel", + "ragel-rb": "Ragel", + "ragel-ruby": "Ragel", + "rake": "Ruby", + "raml": "RAML", + "rascal": "Rascal", + "raw": "Raw token data", + "raw_token_data": "Raw token data", + "razor": "HTML+Razor", + "rb": "Ruby", + "rbx": "Ruby", + "rdoc": "RDoc", + "realbasic": "REALbasic", + "reason": "Reason", + "rebol": "Rebol", + "red": "Red", + "red/system": "Red", + "redcode": "Redcode", + "regex": "Regular Expression", + "regexp": "Regular Expression", + "regular_expression": "Regular Expression", + "ren'py": "Ren'Py", + "renderscript": "RenderScript", + "renpy": "Ren'Py", + "restructuredtext": "reStructuredText", + "rexx": "REXX", + "rhtml": "RHTML", + "rich_text_format": "Rich Text Format", + "ring": "Ring", + "rmarkdown": "RMarkdown", + "robotframework": "RobotFramework", + "roff": "Roff", + "roff_manpage": "Roff Manpage", + "rouge": "Rouge", + "rpc": "RPC", + "rpcgen": "RPC", + "rpm_spec": "RPM Spec", + "rs-274x": "Gerber Image", + "rscript": "R", + "rss": "XML", + "rst": "reStructuredText", + "ruby": "Ruby", + "runoff": "RUNOFF", + "rust": "Rust", + "rusthon": "Python", + "sage": "Sage", + "salt": "SaltStack", + "saltstack": "SaltStack", + "saltstate": "SaltStack", + "sas": "SAS", + "sass": "Sass", + "scala": "Scala", + "scaml": "Scaml", + "scheme": "Scheme", + "scilab": "Scilab", + "scss": "SCSS", + "sed": "sed", + "self": "Self", + "sh": "Shell", + "shaderlab": "ShaderLab", + "shell": "Shell", + "shell-script": "Shell", + "shellsession": "ShellSession", + "shen": "Shen", + "slash": "Slash", + "slice": "Slice", + "slim": "Slim", + "smali": "Smali", + "smalltalk": "Smalltalk", + "smarty": "Smarty", + "sml": "Standard ML", + "smt": "SMT", + "snippet": "YASnippet", + "solidity": "Solidity", + "sourcemod": "SourcePawn", + "sourcepawn": "SourcePawn", + "soy": "Closure Templates", + "sparql": "SPARQL", + "specfile": "RPM Spec", + "spline_font_database": "Spline Font Database", + "splus": "R", + "sqf": "SQF", + "sql": "SQL", + "sqlpl": "SQLPL", + "squeak": "Smalltalk", + "squirrel": "Squirrel", + "srecode_template": "SRecode Template", + "ssh_config": "SSH Config", + "stan": "Stan", + "standard_ml": "Standard ML", + "stata": "Stata", + "ston": "STON", + "stylus": "Stylus", + "subrip_text": "SubRip Text", + "sugarss": "SugarSS", + "supercollider": "SuperCollider", + "svelte": "Svelte", + "svg": "SVG", + "swift": "Swift", + "systemverilog": "SystemVerilog", + "tcl": "Tcl", + "tcsh": "Tcsh", + "tea": "Tea", + "terra": "Terra", + "terraform": "HCL", + "tex": "TeX", + "text": "Text", + "textile": "Textile", + "thrift": "Thrift", + "ti_program": "TI Program", + "tl": "Type Language", + "tla": "TLA", + "toml": "TOML", + "troff": "Roff", + "ts": "TypeScript", + "tsql": "TSQL", + "tsx": "TSX", + "turing": "Turing", + "turtle": "Turtle", + "twig": "Twig", + "txl": "TXL", + "type_language": "Type Language", + "typescript": "TypeScript", + "udiff": "Diff", + "unified_parallel_c": "Unified Parallel C", + "unity3d_asset": "Unity3D Asset", + "unix_assembly": "Unix Assembly", + "uno": "Uno", + "unrealscript": "UnrealScript", + "ur": "UrWeb", + "ur/web": "UrWeb", + "urweb": "UrWeb", + "vala": "Vala", + "vb.net": "Visual Basic", + "vbnet": "Visual Basic", + "vcl": "VCL", + "verilog": "Verilog", + "vhdl": "VHDL", + "vim": "Vim script", + "vim_script": "Vim script", + "viml": "Vim script", + "visual_basic": "Visual Basic", + "volt": "Volt", + "vue": "Vue", + "wasm": "WebAssembly", + "wast": "WebAssembly", + "wavefront_material": "Wavefront Material", + "wavefront_object": "Wavefront Object", + "wdl": "wdl", + "web_ontology_language": "Web Ontology Language", + "webassembly": "WebAssembly", + "webidl": "WebIDL", + "webvtt": "WebVTT", + "winbatch": "Batchfile", + "windows_registry_entries": "Windows Registry Entries", + "wisp": "wisp", + "wollok": "Wollok", + "world_of_warcraft_addon_data": "World of Warcraft Addon Data", + "wsdl": "XML", + "x10": "X10", + "x_bitmap": "X BitMap", + "x_font_directory_index": "X Font Directory Index", + "x_pixmap": "X PixMap", + "xbase": "xBase", + "xbm": "X BitMap", + "xc": "XC", + "xcompose": "XCompose", + "xdr": "RPC", + "xhtml": "HTML", + "xml": "XML", + "xml+genshi": "Genshi", + "xml+kid": "Genshi", + "xojo": "Xojo", + "xpages": "XPages", + "xpm": "X PixMap", + "xproc": "XProc", + "xquery": "XQuery", + "xs": "XS", + "xsd": "XML", + "xsl": "XSLT", + "xslt": "XSLT", + "xten": "X10", + "xtend": "Xtend", + "yacc": "Yacc", + "yaml": "YAML", + "yang": "YANG", + "yara": "YARA", + "yas": "YASnippet", + "yasnippet": "YASnippet", + "yml": "YAML", + "zap": "ZAP", + "zeek": "Zeek", + "zenscript": "ZenScript", + "zephir": "Zephir", + "zig": "Zig", + "zil": "ZIL", + "zimpl": "Zimpl", + "zsh": "Shell", +} + +// LanguageByAlias looks up the language name by it's alias or name. +// It mirrors the logic of github linguist and is needed e.g for heuristcs.yml +// that mixes names and aliases in a language field (see XPM example). +func LanguageByAlias(langOrAlias string) (lang string, ok bool) { + k := convertToAliasKey(langOrAlias) + lang, ok = LanguageByAliasMap[k] + return +} + +// convertToAliasKey converts language name to a key in LanguageByAliasMap. +// Following +// - internal.code-generator.generator.convertToAliasKey() +// - GetLanguageByAlias() +// conventions. +// It is here to avoid dependency on "generate" and "enry" packages. +func convertToAliasKey(langName string) string { + ak := strings.SplitN(langName, `,`, 2)[0] + ak = strings.Replace(ak, ` `, `_`, -1) + ak = strings.ToLower(ak) + return ak +} diff --git a/vendor/github.com/src-d/enry/v2/data/colors.go b/vendor/github.com/src-d/enry/v2/data/colors.go new file mode 100644 index 000000000..39d2acc2d --- /dev/null +++ b/vendor/github.com/src-d/enry/v2/data/colors.go @@ -0,0 +1,254 @@ +// Code generated by github.com/src-d/enry/v2/internal/code-generator DO NOT EDIT. +// Extracted from github/linguist commit: 3a1bd3c3d3e741a8aaec4704f782e06f5cd2a00d + +package data + +var LanguagesColor = map[string]string{ + "1C Enterprise": "#814CCC", + "ABAP": "#E8274B", + "AGS Script": "#B9D9FF", + "AMPL": "#E6EFBB", + "ANTLR": "#9DC3FF", + "API Blueprint": "#2ACCA8", + "APL": "#5A8164", + "ASP": "#6a40fd", + "ATS": "#1ac620", + "ActionScript": "#882B0F", + "Ada": "#02f88c", + "Agda": "#315665", + "Alloy": "#64C800", + "AngelScript": "#C7D7DC", + "AppleScript": "#101F1F", + "Arc": "#aa2afe", + "AspectJ": "#a957b0", + "Assembly": "#6E4C13", + "Asymptote": "#4a0c0c", + "AutoHotkey": "#6594b9", + "AutoIt": "#1C3552", + "Ballerina": "#FF5000", + "Batchfile": "#C1F12E", + "BlitzMax": "#cd6400", + "Boo": "#d4bec1", + "Brainfuck": "#2F2530", + "C": "#555555", + "C#": "#178600", + "C++": "#f34b7d", + "CSS": "#563d7c", + "Ceylon": "#dfa535", + "Chapel": "#8dc63f", + "Cirru": "#ccccff", + "Clarion": "#db901e", + "Clean": "#3F85AF", + "Click": "#E4E6F3", + "Clojure": "#db5855", + "CoffeeScript": "#244776", + "ColdFusion": "#ed2cd6", + "Common Lisp": "#3fb68b", + "Common Workflow Language": "#B5314C", + "Component Pascal": "#B0CE4E", + "Crystal": "#000100", + "Cuda": "#3A4E3A", + "D": "#ba595e", + "DM": "#447265", + "Dart": "#00B4AB", + "DataWeave": "#003a52", + "Dhall": "#dfafff", + "Dockerfile": "#384d54", + "Dogescript": "#cca760", + "Dylan": "#6c616e", + "E": "#ccce35", + "ECL": "#8a1267", + "EQ": "#a78649", + "Eiffel": "#946d57", + "Elixir": "#6e4a7e", + "Elm": "#60B5CC", + "Emacs Lisp": "#c065db", + "EmberScript": "#FFF4F3", + "Erlang": "#B83998", + "F#": "#b845fc", + "F*": "#572e30", + "FLUX": "#88ccff", + "Factor": "#636746", + "Fancy": "#7b9db4", + "Fantom": "#14253c", + "Forth": "#341708", + "Fortran": "#4d41b1", + "FreeMarker": "#0050b2", + "Frege": "#00cafe", + "G-code": "#D08CF2", + "GDScript": "#355570", + "Game Maker Language": "#71b417", + "Genie": "#fb855d", + "Gherkin": "#5B2063", + "Glyph": "#c1ac7f", + "Gnuplot": "#f0a9f0", + "Go": "#00ADD8", + "Golo": "#88562A", + "Gosu": "#82937f", + "Grammatical Framework": "#79aa7a", + "Groovy": "#e69f56", + "HTML": "#e34c26", + "Hack": "#878787", + "Harbour": "#0e60e3", + "Haskell": "#5e5086", + "Haxe": "#df7900", + "HiveQL": "#dce200", + "HolyC": "#ffefaf", + "Hy": "#7790B2", + "IDL": "#a3522f", + "Idris": "#b30000", + "Io": "#a9188d", + "Ioke": "#078193", + "Isabelle": "#FEFE00", + "J": "#9EEDFF", + "JSONiq": "#40d47e", + "Java": "#b07219", + "JavaScript": "#f1e05a", + "Jolie": "#843179", + "Jsonnet": "#0064bd", + "Julia": "#a270ba", + "Jupyter Notebook": "#DA5B0B", + "KRL": "#28430A", + "Kotlin": "#F18E33", + "LFE": "#4C3023", + "LLVM": "#185619", + "LOLCODE": "#cc9900", + "LSL": "#3d9970", + "Lasso": "#999999", + "Lex": "#DBCA00", + "LiveScript": "#499886", + "LookML": "#652B81", + "Lua": "#000080", + "MATLAB": "#e16737", + "MAXScript": "#00a6a6", + "MQL4": "#62A8D6", + "MQL5": "#4A76B8", + "MTML": "#b7e1f4", + "Makefile": "#427819", + "Mask": "#f97732", + "Max": "#c4a79c", + "Mercury": "#ff2b2b", + "Meson": "#007800", + "Metal": "#8f14e9", + "Mirah": "#c7a938", + "Modula-3": "#223388", + "NCL": "#28431f", + "Nearley": "#990000", + "Nemerle": "#3d3c6e", + "NetLinx": "#0aa0ff", + "NetLinx+ERB": "#747faa", + "NetLogo": "#ff6375", + "NewLisp": "#87AED7", + "Nextflow": "#3ac486", + "Nim": "#37775b", + "Nit": "#009917", + "Nix": "#7e7eff", + "Nu": "#c9df40", + "OCaml": "#3be133", + "ObjectScript": "#424893", + "Objective-C": "#438eff", + "Objective-C++": "#6866fb", + "Objective-J": "#ff0c5a", + "Omgrofl": "#cabbff", + "Opal": "#f7ede0", + "Oxygene": "#cdd0e3", + "Oz": "#fab738", + "P4": "#7055b5", + "PHP": "#4F5D95", + "PLSQL": "#dad8d8", + "Pan": "#cc0000", + "Papyrus": "#6600cc", + "Parrot": "#f3ca0a", + "Pascal": "#E3F171", + "Pawn": "#dbb284", + "Pep8": "#C76F5B", + "Perl": "#0298c3", + "Perl 6": "#0000fb", + "PigLatin": "#fcd7de", + "Pike": "#005390", + "PogoScript": "#d80074", + "PostScript": "#da291c", + "PowerBuilder": "#8f0f8d", + "PowerShell": "#012456", + "Processing": "#0096D8", + "Prolog": "#74283c", + "Propeller Spin": "#7fa2a7", + "Puppet": "#302B6D", + "PureBasic": "#5a6986", + "PureScript": "#1D222D", + "Python": "#3572A5", + "QML": "#44a51c", + "Quake": "#882233", + "R": "#198CE7", + "RAML": "#77d9fb", + "RUNOFF": "#665a4e", + "Racket": "#3c5caa", + "Ragel": "#9d5200", + "Rascal": "#fffaa0", + "Rebol": "#358a5b", + "Red": "#f50000", + "Ren'Py": "#ff7f7f", + "Ring": "#2D54CB", + "Roff": "#ecdebe", + "Rouge": "#cc0088", + "Ruby": "#701516", + "Rust": "#dea584", + "SAS": "#B34936", + "SQF": "#3F3F3F", + "SRecode Template": "#348a34", + "SaltStack": "#646464", + "Scala": "#c22d40", + "Scheme": "#1e4aec", + "Self": "#0579aa", + "Shell": "#89e051", + "Shen": "#120F14", + "Slash": "#007eff", + "Slice": "#003fa2", + "Smalltalk": "#596706", + "Solidity": "#AA6746", + "SourcePawn": "#5c7611", + "Squirrel": "#800000", + "Stan": "#b2011d", + "Standard ML": "#dc566d", + "SuperCollider": "#46390b", + "Swift": "#ffac45", + "SystemVerilog": "#DAE1C2", + "TI Program": "#A0AA87", + "Tcl": "#e4cc98", + "TeX": "#3D6117", + "Terra": "#00004c", + "Turing": "#cf142b", + "TypeScript": "#2b7489", + "UnrealScript": "#a54c4d", + "VCL": "#148AA8", + "VHDL": "#adb2cb", + "Vala": "#fbe5cd", + "Verilog": "#b2b7f8", + "Vim script": "#199f4b", + "Visual Basic": "#945db7", + "Volt": "#1F1F1F", + "Vue": "#2c3e50", + "WebAssembly": "#04133b", + "Wollok": "#a23738", + "X10": "#4B6BEF", + "XC": "#99DA07", + "XQuery": "#5232e7", + "XSLT": "#EB8CEB", + "YARA": "#220000", + "YASnippet": "#32AB90", + "Yacc": "#4B6C4B", + "ZAP": "#0d665e", + "ZIL": "#dc75e5", + "ZenScript": "#00BCD1", + "Zephir": "#118f9e", + "Zig": "#ec915c", + "eC": "#913960", + "mcfunction": "#E22837", + "nesC": "#94B0C7", + "ooc": "#b0b77e", + "q": "#0040cd", + "sed": "#64b970", + "wdl": "#42f1f4", + "wisp": "#7582D1", + "xBase": "#403a40", +} diff --git a/vendor/github.com/src-d/enry/v2/data/commit.go b/vendor/github.com/src-d/enry/v2/data/commit.go new file mode 100644 index 000000000..e7cfc7bbe --- /dev/null +++ b/vendor/github.com/src-d/enry/v2/data/commit.go @@ -0,0 +1,7 @@ +// Code generated by github.com/src-d/enry/v2/internal/code-generator DO NOT EDIT. +// Extracted from github/linguist commit: 3a1bd3c3d3e741a8aaec4704f782e06f5cd2a00d + +package data + +// linguist's commit from which files were generated. +var LinguistCommit = "3a1bd3c3d3e741a8aaec4704f782e06f5cd2a00d" diff --git a/vendor/github.com/src-d/enry/v2/data/content.go b/vendor/github.com/src-d/enry/v2/data/content.go new file mode 100644 index 000000000..dab33dd86 --- /dev/null +++ b/vendor/github.com/src-d/enry/v2/data/content.go @@ -0,0 +1,1358 @@ +// Code generated by github.com/src-d/enry/v2/internal/code-generator DO NOT EDIT. +// Extracted from github/linguist commit: 3a1bd3c3d3e741a8aaec4704f782e06f5cd2a00d + +package data + +import ( + "regexp" + + "github.com/src-d/enry/v2/data/rule" +) + +var ContentHeuristics = map[string]*Heuristics{ + ".1": &Heuristics{ + rule.And( + rule.MatchingLanguages("Roff Manpage"), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[.'][ \t]*Dd +(?:[^"\s]+|"[^"]+")`), + ), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[.'][ \t]*Dt +(?:[^"\s]+|"[^"]+") +"?(?:[1-9]|@[^\s@]+@)`), + ), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[.'][ \t]*Sh +(?:[^"\s]|"[^"]+")`), + ), + ), + rule.And( + rule.MatchingLanguages("Roff Manpage"), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[.'][ \t]*TH +(?:[^"\s]+|"[^"]+") +"?(?:[1-9]|@[^\s@]+@)`), + ), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[.'][ \t]*SH +(?:[^"\s]+|"[^"\s]+)`), + ), + ), + rule.Always( + rule.MatchingLanguages("Roff"), + ), + }, + ".1in": &Heuristics{ + rule.And( + rule.MatchingLanguages("Roff Manpage"), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[.'][ \t]*Dd +(?:[^"\s]+|"[^"]+")`), + ), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[.'][ \t]*Dt +(?:[^"\s]+|"[^"]+") +"?(?:[1-9]|@[^\s@]+@)`), + ), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[.'][ \t]*Sh +(?:[^"\s]|"[^"]+")`), + ), + ), + rule.And( + rule.MatchingLanguages("Roff Manpage"), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[.'][ \t]*TH +(?:[^"\s]+|"[^"]+") +"?(?:[1-9]|@[^\s@]+@)`), + ), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[.'][ \t]*SH +(?:[^"\s]+|"[^"\s]+)`), + ), + ), + rule.Always( + rule.MatchingLanguages("Roff"), + ), + }, + ".1m": &Heuristics{ + rule.And( + rule.MatchingLanguages("Roff Manpage"), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[.'][ \t]*Dd +(?:[^"\s]+|"[^"]+")`), + ), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[.'][ \t]*Dt +(?:[^"\s]+|"[^"]+") +"?(?:[1-9]|@[^\s@]+@)`), + ), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[.'][ \t]*Sh +(?:[^"\s]|"[^"]+")`), + ), + ), + rule.And( + rule.MatchingLanguages("Roff Manpage"), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[.'][ \t]*TH +(?:[^"\s]+|"[^"]+") +"?(?:[1-9]|@[^\s@]+@)`), + ), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[.'][ \t]*SH +(?:[^"\s]+|"[^"\s]+)`), + ), + ), + rule.Always( + rule.MatchingLanguages("Roff"), + ), + }, + ".1x": &Heuristics{ + rule.And( + rule.MatchingLanguages("Roff Manpage"), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[.'][ \t]*Dd +(?:[^"\s]+|"[^"]+")`), + ), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[.'][ \t]*Dt +(?:[^"\s]+|"[^"]+") +"?(?:[1-9]|@[^\s@]+@)`), + ), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[.'][ \t]*Sh +(?:[^"\s]|"[^"]+")`), + ), + ), + rule.And( + rule.MatchingLanguages("Roff Manpage"), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[.'][ \t]*TH +(?:[^"\s]+|"[^"]+") +"?(?:[1-9]|@[^\s@]+@)`), + ), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[.'][ \t]*SH +(?:[^"\s]+|"[^"\s]+)`), + ), + ), + rule.Always( + rule.MatchingLanguages("Roff"), + ), + }, + ".2": &Heuristics{ + rule.And( + rule.MatchingLanguages("Roff Manpage"), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[.'][ \t]*Dd +(?:[^"\s]+|"[^"]+")`), + ), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[.'][ \t]*Dt +(?:[^"\s]+|"[^"]+") +"?(?:[1-9]|@[^\s@]+@)`), + ), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[.'][ \t]*Sh +(?:[^"\s]|"[^"]+")`), + ), + ), + rule.And( + rule.MatchingLanguages("Roff Manpage"), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[.'][ \t]*TH +(?:[^"\s]+|"[^"]+") +"?(?:[1-9]|@[^\s@]+@)`), + ), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[.'][ \t]*SH +(?:[^"\s]+|"[^"\s]+)`), + ), + ), + rule.Always( + rule.MatchingLanguages("Roff"), + ), + }, + ".3": &Heuristics{ + rule.And( + rule.MatchingLanguages("Roff Manpage"), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[.'][ \t]*Dd +(?:[^"\s]+|"[^"]+")`), + ), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[.'][ \t]*Dt +(?:[^"\s]+|"[^"]+") +"?(?:[1-9]|@[^\s@]+@)`), + ), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[.'][ \t]*Sh +(?:[^"\s]|"[^"]+")`), + ), + ), + rule.And( + rule.MatchingLanguages("Roff Manpage"), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[.'][ \t]*TH +(?:[^"\s]+|"[^"]+") +"?(?:[1-9]|@[^\s@]+@)`), + ), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[.'][ \t]*SH +(?:[^"\s]+|"[^"\s]+)`), + ), + ), + rule.Always( + rule.MatchingLanguages("Roff"), + ), + }, + ".3in": &Heuristics{ + rule.And( + rule.MatchingLanguages("Roff Manpage"), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[.'][ \t]*Dd +(?:[^"\s]+|"[^"]+")`), + ), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[.'][ \t]*Dt +(?:[^"\s]+|"[^"]+") +"?(?:[1-9]|@[^\s@]+@)`), + ), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[.'][ \t]*Sh +(?:[^"\s]|"[^"]+")`), + ), + ), + rule.And( + rule.MatchingLanguages("Roff Manpage"), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[.'][ \t]*TH +(?:[^"\s]+|"[^"]+") +"?(?:[1-9]|@[^\s@]+@)`), + ), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[.'][ \t]*SH +(?:[^"\s]+|"[^"\s]+)`), + ), + ), + rule.Always( + rule.MatchingLanguages("Roff"), + ), + }, + ".3m": &Heuristics{ + rule.And( + rule.MatchingLanguages("Roff Manpage"), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[.'][ \t]*Dd +(?:[^"\s]+|"[^"]+")`), + ), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[.'][ \t]*Dt +(?:[^"\s]+|"[^"]+") +"?(?:[1-9]|@[^\s@]+@)`), + ), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[.'][ \t]*Sh +(?:[^"\s]|"[^"]+")`), + ), + ), + rule.And( + rule.MatchingLanguages("Roff Manpage"), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[.'][ \t]*TH +(?:[^"\s]+|"[^"]+") +"?(?:[1-9]|@[^\s@]+@)`), + ), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[.'][ \t]*SH +(?:[^"\s]+|"[^"\s]+)`), + ), + ), + rule.Always( + rule.MatchingLanguages("Roff"), + ), + }, + ".3p": &Heuristics{ + rule.And( + rule.MatchingLanguages("Roff Manpage"), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[.'][ \t]*Dd +(?:[^"\s]+|"[^"]+")`), + ), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[.'][ \t]*Dt +(?:[^"\s]+|"[^"]+") +"?(?:[1-9]|@[^\s@]+@)`), + ), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[.'][ \t]*Sh +(?:[^"\s]|"[^"]+")`), + ), + ), + rule.And( + rule.MatchingLanguages("Roff Manpage"), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[.'][ \t]*TH +(?:[^"\s]+|"[^"]+") +"?(?:[1-9]|@[^\s@]+@)`), + ), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[.'][ \t]*SH +(?:[^"\s]+|"[^"\s]+)`), + ), + ), + rule.Always( + rule.MatchingLanguages("Roff"), + ), + }, + ".3pm": &Heuristics{ + rule.And( + rule.MatchingLanguages("Roff Manpage"), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[.'][ \t]*Dd +(?:[^"\s]+|"[^"]+")`), + ), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[.'][ \t]*Dt +(?:[^"\s]+|"[^"]+") +"?(?:[1-9]|@[^\s@]+@)`), + ), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[.'][ \t]*Sh +(?:[^"\s]|"[^"]+")`), + ), + ), + rule.And( + rule.MatchingLanguages("Roff Manpage"), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[.'][ \t]*TH +(?:[^"\s]+|"[^"]+") +"?(?:[1-9]|@[^\s@]+@)`), + ), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[.'][ \t]*SH +(?:[^"\s]+|"[^"\s]+)`), + ), + ), + rule.Always( + rule.MatchingLanguages("Roff"), + ), + }, + ".3qt": &Heuristics{ + rule.And( + rule.MatchingLanguages("Roff Manpage"), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[.'][ \t]*Dd +(?:[^"\s]+|"[^"]+")`), + ), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[.'][ \t]*Dt +(?:[^"\s]+|"[^"]+") +"?(?:[1-9]|@[^\s@]+@)`), + ), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[.'][ \t]*Sh +(?:[^"\s]|"[^"]+")`), + ), + ), + rule.And( + rule.MatchingLanguages("Roff Manpage"), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[.'][ \t]*TH +(?:[^"\s]+|"[^"]+") +"?(?:[1-9]|@[^\s@]+@)`), + ), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[.'][ \t]*SH +(?:[^"\s]+|"[^"\s]+)`), + ), + ), + rule.Always( + rule.MatchingLanguages("Roff"), + ), + }, + ".3x": &Heuristics{ + rule.And( + rule.MatchingLanguages("Roff Manpage"), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[.'][ \t]*Dd +(?:[^"\s]+|"[^"]+")`), + ), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[.'][ \t]*Dt +(?:[^"\s]+|"[^"]+") +"?(?:[1-9]|@[^\s@]+@)`), + ), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[.'][ \t]*Sh +(?:[^"\s]|"[^"]+")`), + ), + ), + rule.And( + rule.MatchingLanguages("Roff Manpage"), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[.'][ \t]*TH +(?:[^"\s]+|"[^"]+") +"?(?:[1-9]|@[^\s@]+@)`), + ), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[.'][ \t]*SH +(?:[^"\s]+|"[^"\s]+)`), + ), + ), + rule.Always( + rule.MatchingLanguages("Roff"), + ), + }, + ".4": &Heuristics{ + rule.And( + rule.MatchingLanguages("Roff Manpage"), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[.'][ \t]*Dd +(?:[^"\s]+|"[^"]+")`), + ), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[.'][ \t]*Dt +(?:[^"\s]+|"[^"]+") +"?(?:[1-9]|@[^\s@]+@)`), + ), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[.'][ \t]*Sh +(?:[^"\s]|"[^"]+")`), + ), + ), + rule.And( + rule.MatchingLanguages("Roff Manpage"), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[.'][ \t]*TH +(?:[^"\s]+|"[^"]+") +"?(?:[1-9]|@[^\s@]+@)`), + ), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[.'][ \t]*SH +(?:[^"\s]+|"[^"\s]+)`), + ), + ), + rule.Always( + rule.MatchingLanguages("Roff"), + ), + }, + ".5": &Heuristics{ + rule.And( + rule.MatchingLanguages("Roff Manpage"), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[.'][ \t]*Dd +(?:[^"\s]+|"[^"]+")`), + ), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[.'][ \t]*Dt +(?:[^"\s]+|"[^"]+") +"?(?:[1-9]|@[^\s@]+@)`), + ), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[.'][ \t]*Sh +(?:[^"\s]|"[^"]+")`), + ), + ), + rule.And( + rule.MatchingLanguages("Roff Manpage"), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[.'][ \t]*TH +(?:[^"\s]+|"[^"]+") +"?(?:[1-9]|@[^\s@]+@)`), + ), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[.'][ \t]*SH +(?:[^"\s]+|"[^"\s]+)`), + ), + ), + rule.Always( + rule.MatchingLanguages("Roff"), + ), + }, + ".6": &Heuristics{ + rule.And( + rule.MatchingLanguages("Roff Manpage"), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[.'][ \t]*Dd +(?:[^"\s]+|"[^"]+")`), + ), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[.'][ \t]*Dt +(?:[^"\s]+|"[^"]+") +"?(?:[1-9]|@[^\s@]+@)`), + ), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[.'][ \t]*Sh +(?:[^"\s]|"[^"]+")`), + ), + ), + rule.And( + rule.MatchingLanguages("Roff Manpage"), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[.'][ \t]*TH +(?:[^"\s]+|"[^"]+") +"?(?:[1-9]|@[^\s@]+@)`), + ), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[.'][ \t]*SH +(?:[^"\s]+|"[^"\s]+)`), + ), + ), + rule.Always( + rule.MatchingLanguages("Roff"), + ), + }, + ".7": &Heuristics{ + rule.And( + rule.MatchingLanguages("Roff Manpage"), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[.'][ \t]*Dd +(?:[^"\s]+|"[^"]+")`), + ), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[.'][ \t]*Dt +(?:[^"\s]+|"[^"]+") +"?(?:[1-9]|@[^\s@]+@)`), + ), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[.'][ \t]*Sh +(?:[^"\s]|"[^"]+")`), + ), + ), + rule.And( + rule.MatchingLanguages("Roff Manpage"), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[.'][ \t]*TH +(?:[^"\s]+|"[^"]+") +"?(?:[1-9]|@[^\s@]+@)`), + ), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[.'][ \t]*SH +(?:[^"\s]+|"[^"\s]+)`), + ), + ), + rule.Always( + rule.MatchingLanguages("Roff"), + ), + }, + ".8": &Heuristics{ + rule.And( + rule.MatchingLanguages("Roff Manpage"), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[.'][ \t]*Dd +(?:[^"\s]+|"[^"]+")`), + ), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[.'][ \t]*Dt +(?:[^"\s]+|"[^"]+") +"?(?:[1-9]|@[^\s@]+@)`), + ), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[.'][ \t]*Sh +(?:[^"\s]|"[^"]+")`), + ), + ), + rule.And( + rule.MatchingLanguages("Roff Manpage"), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[.'][ \t]*TH +(?:[^"\s]+|"[^"]+") +"?(?:[1-9]|@[^\s@]+@)`), + ), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[.'][ \t]*SH +(?:[^"\s]+|"[^"\s]+)`), + ), + ), + rule.Always( + rule.MatchingLanguages("Roff"), + ), + }, + ".9": &Heuristics{ + rule.And( + rule.MatchingLanguages("Roff Manpage"), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[.'][ \t]*Dd +(?:[^"\s]+|"[^"]+")`), + ), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[.'][ \t]*Dt +(?:[^"\s]+|"[^"]+") +"?(?:[1-9]|@[^\s@]+@)`), + ), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[.'][ \t]*Sh +(?:[^"\s]|"[^"]+")`), + ), + ), + rule.And( + rule.MatchingLanguages("Roff Manpage"), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[.'][ \t]*TH +(?:[^"\s]+|"[^"]+") +"?(?:[1-9]|@[^\s@]+@)`), + ), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[.'][ \t]*SH +(?:[^"\s]+|"[^"\s]+)`), + ), + ), + rule.Always( + rule.MatchingLanguages("Roff"), + ), + }, + ".as": &Heuristics{ + rule.Or( + rule.MatchingLanguages("ActionScript"), + regexp.MustCompile(`(?m)^\s*(package\s+[a-z0-9_\.]+|import\s+[a-zA-Z0-9_\.]+;|class\s+[A-Za-z0-9_]+\s+extends\s+[A-Za-z0-9_]+)`), + ), + rule.Always( + rule.MatchingLanguages("AngelScript"), + ), + }, + ".asc": &Heuristics{ + rule.Or( + rule.MatchingLanguages("Public Key"), + regexp.MustCompile(`(?m)^(----[- ]BEGIN|ssh-(rsa|dss)) `), + ), + rule.Or( + rule.MatchingLanguages("AsciiDoc"), + regexp.MustCompile(`(?m)^[=-]+(\s|\n)|{{[A-Za-z]`), + ), + rule.Or( + rule.MatchingLanguages("AGS Script"), + regexp.MustCompile(`(?m)^(\/\/.+|((import|export)\s+)?(function|int|float|char)\s+((room|repeatedly|on|game)_)?([A-Za-z]+[A-Za-z_0-9]+)\s*[;\(])`), + ), + }, + ".asy": &Heuristics{ + rule.Or( + rule.MatchingLanguages("LTspice Symbol"), + regexp.MustCompile(`(?m)^SymbolType[ \t]`), + ), + rule.Always( + rule.MatchingLanguages("Asymptote"), + ), + }, + ".bb": &Heuristics{ + rule.Or( + rule.MatchingLanguages("BlitzBasic"), + regexp.MustCompile(`(?m)(<^\s*; |End Function)`), + ), + rule.Or( + rule.MatchingLanguages("BitBake"), + regexp.MustCompile(`(?m)^\s*(# |include|require)\b`), + ), + }, + ".builds": &Heuristics{ + rule.Or( + rule.MatchingLanguages("XML"), + regexp.MustCompile(`(?m)^(\s*)(?i:)`), + ), + }, + ".f": &Heuristics{ + rule.Or( + rule.MatchingLanguages("Forth"), + regexp.MustCompile(`(?m)^: `), + ), + rule.Or( + rule.MatchingLanguages("Filebench WML"), + regexp.MustCompile(`(?m)flowop`), + ), + rule.Or( + rule.MatchingLanguages("Fortran"), + regexp.MustCompile(`(?m)^(?i:[c*][^abd-z]| (subroutine|program|end|data)\s|\s*!)`), + ), + }, + ".for": &Heuristics{ + rule.Or( + rule.MatchingLanguages("Forth"), + regexp.MustCompile(`(?m)^: `), + ), + rule.Or( + rule.MatchingLanguages("Fortran"), + regexp.MustCompile(`(?m)^(?i:[c*][^abd-z]| (subroutine|program|end|data)\s|\s*!)`), + ), + }, + ".fr": &Heuristics{ + rule.Or( + rule.MatchingLanguages("Forth"), + regexp.MustCompile(`(?m)^(: |also |new-device|previous )`), + ), + rule.Or( + rule.MatchingLanguages("Frege"), + regexp.MustCompile(`(?m)^\s*(import|module|package|data|type) `), + ), + rule.Always( + rule.MatchingLanguages("Text"), + ), + }, + ".fs": &Heuristics{ + rule.Or( + rule.MatchingLanguages("Forth"), + regexp.MustCompile(`(?m)^(: |new-device)`), + ), + rule.Or( + rule.MatchingLanguages("F#"), + regexp.MustCompile(`(?m)^\s*(#light|import|let|module|namespace|open|type)`), + ), + rule.Or( + rule.MatchingLanguages("GLSL"), + regexp.MustCompile(`(?m)^\s*(#version|precision|uniform|varying|vec[234])`), + ), + rule.Or( + rule.MatchingLanguages("Filterscript"), + regexp.MustCompile(`(?m)#include|#pragma\s+(rs|version)|__attribute__`), + ), + }, + ".gd": &Heuristics{ + rule.Or( + rule.MatchingLanguages("GAP"), + regexp.MustCompile(`(?m)\s*(Declare|BindGlobal|KeyDependentOperation)`), + ), + rule.Or( + rule.MatchingLanguages("GDScript"), + regexp.MustCompile(`(?m)\s*(extends|var|const|enum|func|class|signal|tool|yield|assert|onready)`), + ), + }, + ".gml": &Heuristics{ + rule.Or( + rule.MatchingLanguages("XML"), + regexp.MustCompile(`(?m)(?i:^\s*(\<\?xml|xmlns))`), + ), + rule.Or( + rule.MatchingLanguages("Graph Modeling Language"), + regexp.MustCompile(`(?m)(?i:^\s*(graph|node)\s+\[$)`), + ), + rule.Or( + rule.MatchingLanguages("Gerber Image"), + regexp.MustCompile(`(?m)\*\%$`), + ), + rule.Always( + rule.MatchingLanguages("Game Maker Language"), + ), + }, + ".gs": &Heuristics{ + rule.Or( + rule.MatchingLanguages("Gosu"), + regexp.MustCompile(`(?m)^uses java\.`), + ), + }, + ".h": &Heuristics{ + rule.Or( + rule.MatchingLanguages("Objective-C"), + regexp.MustCompile(`(?m)^\s*(@(interface|class|protocol|property|end|synchronised|selector|implementation)\b|#import\s+.+\.h[">])`), + ), + rule.Or( + rule.MatchingLanguages("C++"), + regexp.MustCompile(`(?m)^\s*#\s*include <(cstdint|string|vector|map|list|array|bitset|queue|stack|forward_list|unordered_map|unordered_set|(i|o|io)stream)>|^\s*template\s*<|^[ \t]*(try|constexpr)|^[ \t]*catch\s*\(|^[ \t]*(class|(using[ \t]+)?namespace)\s+\w+|^[ \t]*(private|public|protected):$|std::\w+`), + ), + }, + ".hh": &Heuristics{ + rule.Or( + rule.MatchingLanguages("Hack"), + regexp.MustCompile(`(?m)<\?hh`), + ), + }, + ".ice": &Heuristics{ + rule.Or( + rule.MatchingLanguages("JSON"), + regexp.MustCompile(`(?m)\A\s*[{\[]`), + ), + rule.Always( + rule.MatchingLanguages("Slice"), + ), + }, + ".inc": &Heuristics{ + rule.Or( + rule.MatchingLanguages("PHP"), + regexp.MustCompile(`(?m)^<\?(?:php)?`), + ), + rule.Or( + rule.MatchingLanguages("SourcePawn"), + regexp.MustCompile(`(?m)^public\s+(?:SharedPlugin(?:\s+|:)__pl_\w+\s*=(?:\s*{)?|(?:void\s+)?__pl_\w+_SetNTVOptional\(\)(?:\s*{)?)`), + ), + rule.Or( + rule.MatchingLanguages("POV-Ray SDL"), + regexp.MustCompile(`(?m)^\s*#(declare|local|macro|while)\s`), + ), + }, + ".l": &Heuristics{ + rule.Or( + rule.MatchingLanguages("Common Lisp"), + regexp.MustCompile(`(?m)\(def(un|macro)\s`), + ), + rule.Or( + rule.MatchingLanguages("Lex"), + regexp.MustCompile(`(?m)^(%[%{}]xs|<.*>)`), + ), + rule.Or( + rule.MatchingLanguages("Roff"), + regexp.MustCompile(`(?m)^\.[A-Za-z]{2}(\s|$)`), + ), + rule.Or( + rule.MatchingLanguages("PicoLisp"), + regexp.MustCompile(`(?m)^\((de|class|rel|code|data|must)\s`), + ), + }, + ".lisp": &Heuristics{ + rule.Or( + rule.MatchingLanguages("Common Lisp"), + regexp.MustCompile(`(?m)^\s*\((?i:defun|in-package|defpackage) `), + ), + rule.Or( + rule.MatchingLanguages("NewLisp"), + regexp.MustCompile(`(?m)^\s*\(define `), + ), + }, + ".ls": &Heuristics{ + rule.Or( + rule.MatchingLanguages("LoomScript"), + regexp.MustCompile(`(?m)^\s*package\s*[\w\.\/\*\s]*\s*{`), + ), + rule.Always( + rule.MatchingLanguages("LiveScript"), + ), + }, + ".lsp": &Heuristics{ + rule.Or( + rule.MatchingLanguages("Common Lisp"), + regexp.MustCompile(`(?m)^\s*\((?i:defun|in-package|defpackage) `), + ), + rule.Or( + rule.MatchingLanguages("NewLisp"), + regexp.MustCompile(`(?m)^\s*\(define `), + ), + }, + ".m": &Heuristics{ + rule.Or( + rule.MatchingLanguages("Objective-C"), + regexp.MustCompile(`(?m)^\s*(@(interface|class|protocol|property|end|synchronised|selector|implementation)\b|#import\s+.+\.h[">])`), + ), + rule.Or( + rule.MatchingLanguages("Mercury"), + regexp.MustCompile(`(?m):- module`), + ), + rule.Or( + rule.MatchingLanguages("MUF"), + regexp.MustCompile(`(?m)^: `), + ), + rule.Or( + rule.MatchingLanguages("M"), + regexp.MustCompile(`(?m)^\s*;`), + ), + rule.And( + rule.MatchingLanguages("Mathematica"), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)\(\*`), + ), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)\*\)$`), + ), + ), + rule.Or( + rule.MatchingLanguages("MATLAB"), + regexp.MustCompile(`(?m)^\s*%`), + ), + rule.Or( + rule.MatchingLanguages("Limbo"), + regexp.MustCompile(`(?m)^\w+\s*:\s*module\s*{`), + ), + }, + ".man": &Heuristics{ + rule.And( + rule.MatchingLanguages("Roff Manpage"), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[.'][ \t]*Dd +(?:[^"\s]+|"[^"]+")`), + ), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[.'][ \t]*Dt +(?:[^"\s]+|"[^"]+") +"?(?:[1-9]|@[^\s@]+@)`), + ), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[.'][ \t]*Sh +(?:[^"\s]|"[^"]+")`), + ), + ), + rule.And( + rule.MatchingLanguages("Roff Manpage"), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[.'][ \t]*TH +(?:[^"\s]+|"[^"]+") +"?(?:[1-9]|@[^\s@]+@)`), + ), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[.'][ \t]*SH +(?:[^"\s]+|"[^"\s]+)`), + ), + ), + rule.Always( + rule.MatchingLanguages("Roff"), + ), + }, + ".md": &Heuristics{ + rule.Or( + rule.MatchingLanguages("Markdown"), + regexp.MustCompile(`(?m)(^[-A-Za-z0-9=#!\*\[|>])|<\/|\A\z`), + ), + rule.Or( + rule.MatchingLanguages("GCC Machine Description"), + regexp.MustCompile(`(?m)^(;;|\(define_)`), + ), + rule.Always( + rule.MatchingLanguages("Markdown"), + ), + }, + ".mdoc": &Heuristics{ + rule.And( + rule.MatchingLanguages("Roff Manpage"), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[.'][ \t]*Dd +(?:[^"\s]+|"[^"]+")`), + ), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[.'][ \t]*Dt +(?:[^"\s]+|"[^"]+") +"?(?:[1-9]|@[^\s@]+@)`), + ), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[.'][ \t]*Sh +(?:[^"\s]|"[^"]+")`), + ), + ), + rule.And( + rule.MatchingLanguages("Roff Manpage"), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[.'][ \t]*TH +(?:[^"\s]+|"[^"]+") +"?(?:[1-9]|@[^\s@]+@)`), + ), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[.'][ \t]*SH +(?:[^"\s]+|"[^"\s]+)`), + ), + ), + rule.Always( + rule.MatchingLanguages("Roff"), + ), + }, + ".ml": &Heuristics{ + rule.Or( + rule.MatchingLanguages("OCaml"), + regexp.MustCompile(`(?m)(^\s*module)|let rec |match\s+(\S+\s)+with`), + ), + rule.Or( + rule.MatchingLanguages("Standard ML"), + regexp.MustCompile(`(?m)=> |case\s+(\S+\s)+of`), + ), + }, + ".mod": &Heuristics{ + rule.Or( + rule.MatchingLanguages("XML"), + regexp.MustCompile(`(?m)\s`), + ), + }, + ".pro": &Heuristics{ + rule.Or( + rule.MatchingLanguages("Prolog"), + regexp.MustCompile(`(?m)^[^\[#]+:-`), + ), + rule.Or( + rule.MatchingLanguages("INI"), + regexp.MustCompile(`(?m)last_client=`), + ), + rule.And( + rule.MatchingLanguages("QMake"), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)HEADERS`), + ), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)SOURCES`), + ), + ), + rule.Or( + rule.MatchingLanguages("IDL"), + regexp.MustCompile(`(?m)^\s*function[ \w,]+$`), + ), + }, + ".properties": &Heuristics{ + rule.And( + rule.MatchingLanguages("INI"), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[^#!;][^=]*=`), + ), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[;\[]`), + ), + ), + rule.And( + rule.MatchingLanguages("Java Properties"), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[^#!;][^=]*=`), + ), + rule.Or( + rule.MatchingLanguages(""), + regexp.MustCompile(`(?m)^[#!]`), + ), + ), + rule.Or( + rule.MatchingLanguages("INI"), + regexp.MustCompile(`(?m)^[^#!;][^=]*=`), + ), + rule.Or( + rule.MatchingLanguages("Java properties"), + regexp.MustCompile(`(?m)^[^#!][^:]*:`), + ), + }, + ".props": &Heuristics{ + rule.Or( + rule.MatchingLanguages("XML"), + regexp.MustCompile(`(?m)^(\s*)(?i:)\s*(\d{2}:\d{2}:\d{2},\d{3})$`), + ), + }, + ".t": &Heuristics{ + rule.Or( + rule.MatchingLanguages("Perl"), + regexp.MustCompile(`(?m)\buse\s+(?:strict\b|v?5\.)`), + ), + rule.Or( + rule.MatchingLanguages("Perl 6"), + regexp.MustCompile(`(?m)^\s*(?:use\s+v6\b|\bmodule\b|\b(?:my\s+)?class\b)`), + ), + rule.Or( + rule.MatchingLanguages("Turing"), + regexp.MustCompile(`(?m)^\s*%[ \t]+|^\s*var\s+\w+(\s*:\s*\w+)?\s*:=\s*\w+`), + ), + }, + ".toc": &Heuristics{ + rule.Or( + rule.MatchingLanguages("World of Warcraft Addon Data"), + regexp.MustCompile(`(?m)^## |@no-lib-strip@`), + ), + rule.Or( + rule.MatchingLanguages("TeX"), + regexp.MustCompile(`(?m)^\\(contentsline|defcounter|beamer|boolfalse)`), + ), + }, + ".ts": &Heuristics{ + rule.Or( + rule.MatchingLanguages("XML"), + regexp.MustCompile(`(?m) `), + ), + rule.Always( + rule.MatchingLanguages("Scilab"), + ), + }, + ".tsx": &Heuristics{ + rule.Or( + rule.MatchingLanguages("TSX"), + regexp.MustCompile(`(?m)^\s*(import.+(from\s+|require\()['"]react|\/\/\/\s*": -6.727180, + "?": -8.978471, + "DOM": -8.978471, + "DOM.": -9.671619, + "HTML": -8.978471, + "HTML.": -8.978471, + "Null": -9.671619, + "XML": -7.273723, + "XML.": -7.725708, + "_": -7.474394, + "А": -8.285324, + "Б": -8.062181, + "В": -4.701805, + "Г": -8.285324, + "Д": -4.722859, + "Е": -5.107270, + "Ж": -9.671619, + "З": -4.584022, + "И": -5.289592, + "К": -4.310326, + "Л": -6.237631, + "М": -6.270421, + "Н": -4.590214, + "О": -4.641181, + "П": -4.001738, + "Р": -5.842977, + "С": -4.435177, + "Т": -4.403760, + "У": -6.060701, + "Ф": -5.982739, + "Х": -8.978471, + "Ц": -5.611176, + "Ч": -6.580576, + "Ш": -6.627096, + "Э": -5.821471, + "а": -2.506125, + "б": -4.291721, + "в": -3.860478, + "г": -4.804084, + "д": -4.036829, + "е": -2.541520, + "ж": -5.139019, + "з": -4.827431, + "и": -2.859273, + "й": -4.688012, + "к": -3.479256, + "л": -3.214849, + "м": -4.073197, + "н": -2.957448, + "о": -2.573243, + "п": -4.273456, + "р": -3.199272, + "с": -3.569060, + "т": -2.863684, + "у": -4.029711, + "ф": -5.821471, + "х": -6.237631, + "ц": -4.319760, + "ч": -4.654339, + "ш": -7.369033, + "щ": -5.528484, + "ъ": -5.982739, + "ы": -3.805150, + "ь": -4.029711, + "ю": -6.116270, + "я": -4.565673, + "\ufeff": -8.978471, + }, + "ABAP": map[string]float64{ + "!": -4.637712, + "\"": -7.202661, + "(": -5.816367, + ")": -5.816367, + "*": -3.619142, + "+": -5.005437, + "-": -0.322277, + ".": -5.005437, + "<-()]>": -6.509514, + "": -6.509514, + "": -5.816367, + "": -6.104049, + "=": -4.900076, + ">": -5.123220, + "ABAP_BOOL": -7.202661, + "C": -7.202661, + "CLASS": -6.509514, + "CL_CSV_PARSER": -5.410902, + "CONSTRUCTOR": -7.202661, + "CSV": -7.202661, + "CSVSTRING": -7.202661, + "CX": -7.202661, + "CX_CSV_PARSE_ERROR": -7.202661, + "DEFINITION": -6.509514, + "DELEGATE": -7.202661, + "Get": -7.202661, + "IF_CSV_PARSER_DELEGATE": -7.202661, + "IMPLEMENTATION": -6.509514, + "Instance": -6.509514, + "Method": -6.509514, + "Parse": -7.202661, + "Private": -7.202661, + "Public": -7.202661, + "REF": -7.202661, + "RETURNING": -7.202661, + "SEPARATOR": -7.202661, + "SIGNATURE": -6.509514, + "SKIP_FIRST_LINE": -7.202661, + "STRING": -7.202661, + "STRINGTAB": -6.104049, + "Space": -6.509514, + "TO": -7.202661, + "TYPE": -5.256751, + "This": -7.202661, + "[": -5.593223, + "]": -5.816367, + "_LINES": -7.202661, + "_csvstring": -6.509514, + "_delegate": -7.202661, + "_lines": -7.202661, + "_lines.": -7.202661, + "_parse_line": -6.509514, + "_separator": -7.202661, + "_skip_first_line": -7.202661, + "_textindicator": -7.202661, + "`": -5.816367, + "abap": -7.202661, + "abap_bool": -6.509514, + "abap_true.": -6.509514, + "an": -7.202661, + "append": -6.509514, + "assigning": -7.202661, + "at": -6.509514, + "c": -6.509514, + "char": -6.509514, + "cl_abap_char_utilities": -7.202661, + "cl_csv_parser": -6.509514, + "cl_object": -7.202661, + "class": -6.509514, + "clear": -7.202661, + "concatenate": -5.816367, + "constants": -7.202661, + "constructor": -6.509514, + "constructor.": -7.202661, + "cr_lf": -7.202661, + "create": -7.202661, + "csv": -7.202661, + "csvstring": -7.202661, + "csvstring.": -7.202661, + "csvvalue": -5.410902, + "csvvalue.": -5.593223, + "csvvalues.": -6.509514, + "cx_csv_parse_error": -6.509514, + "data": -6.104049, + "definition": -7.202661, + "delegate": -7.202661, + "delegate.": -7.202661, + "do": -6.104049, + "e": -7.202661, + "else.": -5.816367, + "endclass.": -7.202661, + "endif.": -5.410902, + "endmethod.": -6.509514, + "endwhile.": -6.509514, + "error": -7.202661, + "exception": -7.202661, + "exporting": -7.202661, + "field": -7.202661, + "files": -6.104049, + "final": -7.202661, + "formatting": -7.202661, + "from": -7.202661, + "here": -6.104049, + "if_csv_parser_delegate": -7.202661, + "implementation.": -7.202661, + "importing": -7.202661, + "in": -7.202661, + "include": -6.104049, + "indicates": -7.202661, + "inheriting": -7.202661, + "into": -5.410902, + "is_first_line": -7.202661, + "line": -7.202661, + "lines": -5.816367, + "loop": -7.202661, + "message": -6.509514, + "method": -6.509514, + "methods": -6.509514, + "msg.": -6.509514, + "not": -6.104049, + "of": -6.509514, + "other": -6.104049, + "parse": -6.509514, + "pools": -7.202661, + "pos": -6.509514, + "private": -7.202661, + "protected": -7.202661, + "public": -6.104049, + "raise": -7.202661, + "raising": -7.202661, + "ref": -7.202661, + "returning.": -7.202661, + "section.": -6.104049, + "separator": -7.202661, + "separator.": -7.202661, + "skip_first_line": -7.202661, + "skip_first_line.": -7.202661, + "source": -6.104049, + "split": -7.202661, + "standard": -6.509514, + "string": -7.202661, + "string.": -6.104049, + "super": -7.202661, + "symbols": -7.202661, + "table": -6.104049, + "text_ended": -7.202661, + "the": -6.104049, + "to": -6.104049, + "type": -4.804766, + "value": -6.509514, + "values": -6.509514, + "|": -5.256751, + }, + "ABNF": map[string]float64{ + "%": -3.171745, + "'": -7.142037, + "(": -4.251665, + ")": -4.197598, + "*": -4.944812, + "*DIGIT": -7.142037, + "*basic": -6.448889, + "*literal": -7.142037, + "*newline": -7.142037, + "*non": -7.142037, + "+": -4.744141, + ",": -5.062595, + "-": -1.785450, + ".": -6.043424, + "/": -2.983153, + "//github.com/toml": -7.142037, + "//www.ietf.org/rfc/rfc": -7.142037, + ":": -5.755742, + ";": -2.588160, + "=": -2.747587, + "A": -5.196126, + "ABNF": -6.448889, + "ALPHA": -6.448889, + "Apostrophe": -7.142037, + "Array": -6.448889, + "B": -5.196126, + "B.": -7.142037, + "Basic": -6.043424, + "Boolean": -7.142037, + "Built": -7.142037, + "C": -5.350277, + "C.": -7.142037, + "CRLF": -7.142037, + "Comma": -6.448889, + "Comment": -7.142037, + "D": -4.839451, + "D.": -6.448889, + "DIGIT": -4.433986, + "Datetime": -7.142037, + "Double": -6.448889, + "E": -5.755742, + "Float": -7.142037, + "HEXDIG": -6.043424, + "Horizontal": -7.142037, + "Inline": -7.142037, + "Integer": -7.142037, + "Key": -7.142037, + "LF": -7.142037, + "Left": -7.142037, + "License": -7.142037, + "Literal": -6.448889, + "MIT": -7.142037, + "Multiline": -6.448889, + "Newline": -7.142037, + "Period": -7.142037, + "RFC": -6.448889, + "Right": -7.142037, + "See": -7.142037, + "Source": -7.142037, + "Space": -7.142037, + "Standard": -7.142037, + "String": -5.532599, + "Strings": -7.142037, + "TOML": -6.448889, + "Table": -5.755742, + "This": -7.142037, + "U": -4.839451, + "UXXXXXXXX": -7.142037, + "Value": -7.142037, + "Whitespace": -7.142037, + "XXXX": -7.142037, + "XXXXXXXX": -7.142037, + "Z": -6.448889, + "[": -4.577087, + "\\": -6.448889, + "]": -4.577087, + "_": -6.448889, + "a": -6.448889, + "according": -7.142037, + "an": -7.142037, + "apostraphe": -5.350277, + "array": -4.251665, + "as": -7.142037, + "attempt": -7.142037, + "b": -7.142037, + "backspace": -7.142037, + "based": -6.448889, + "basic": -4.502979, + "body": -5.755742, + "boolean": -6.448889, + "bracket": -5.755742, + "carriage": -7.142037, + "char": -5.062595, + "clarity": -7.142037, + "close": -5.062595, + "comment": -5.062595, + "date": -4.839451, + "decimal": -6.448889, + "define": -7.142037, + "defined": -6.448889, + "delim": -5.350277, + "digit": -6.448889, + "e": -6.043424, + "empty": -6.043424, + "eol": -7.142037, + "escape": -6.448889, + "escaped": -7.142037, + "exp": -6.043424, + "expression": -6.043424, + "f": -7.142037, + "false": -6.043424, + "feed": -6.448889, + "float": -6.448889, + "for": -7.142037, + "form": -7.142037, + "frac": -6.043424, + "full": -5.755742, + "fullyear": -6.448889, + "grammar": -7.142037, + "here": -7.142037, + "hour": -6.043424, + "http": -7.142037, + "https": -7.142037, + "in": -5.755742, + "inline": -4.577087, + "int": -5.755742, + "integer": -5.755742, + "is": -7.142037, + "key": -4.433986, + "keyval": -5.350277, + "keyvals": -5.532599, + "lang/toml": -7.142037, + "leap": -7.142037, + "left": -7.142037, + "line": -7.142037, + "literal": -4.657130, + "mark": -4.944812, + "mday": -6.448889, + "minus": -6.448889, + "minute": -6.043424, + "ml": -4.146304, + "month": -6.448889, + "month/year": -7.142037, + "n": -7.142037, + "newline": -5.532599, + "newlines": -5.532599, + "non": -6.043424, + "numoffset": -6.448889, + "offset": -6.448889, + "on": -6.448889, + "open": -5.062595, + "pairs": -7.142037, + "partial": -6.448889, + "plus": -6.448889, + "point": -6.448889, + "prefixable": -6.448889, + "quotation": -4.944812, + "quoted": -6.448889, + "r": -7.142037, + "reproduced": -7.142037, + "return": -7.142037, + "reverse": -7.142037, + "right": -7.142037, + "rules": -7.142037, + "secfrac": -6.448889, + "second": -6.043424, + "sep": -4.657130, + "solidus": -6.448889, + "square": -5.755742, + "start": -6.448889, + "std": -5.350277, + "string": -4.369448, + "symbol": -6.448889, + "t": -7.142037, + "tab": -6.448889, + "table": -3.740839, + "terms": -7.142037, + "the": -7.142037, + "time": -4.146304, + "to": -6.448889, + "toml": -7.142037, + "true": -6.043424, + "txt": -7.142037, + "uXXXX": -7.142037, + "underscore": -6.043424, + "unescaped": -6.043424, + "unquoted": -6.448889, + "val": -5.350277, + "values": -6.043424, + "ws": -4.006542, + "x": -3.171745, + "z": -6.448889, + "zero": -6.448889, + "{": -7.142037, + "}": -7.142037, + }, + "AGS Script": map[string]float64{ + "!": -5.850405, + "#define": -7.236699, + "&&": -6.138087, + "(": -2.606836, + ")": -2.606836, + "*control": -4.710971, + "*theGui": -7.929846, + "+": -5.983936, + ",": -3.978603, + "-": -2.601970, + "//": -5.531951, + "//****************************************************************************************************": -5.850405, + "//But": -7.929846, + "//Ctrl": -7.929846, + "//START": -7.929846, + "//Use": -7.929846, + "//if": -7.929846, + ":": -7.236699, + ";": -2.776555, + "<": -7.929846, + "=": -1.114206, + ">": -7.236699, + "CONTROL": -7.929846, + "DISTANCE": -4.710971, + "Debug": -6.543552, + "DeleteSaveSlot": -7.929846, + "Display": -6.543552, + "ESC": -7.929846, + "EventType": -7.929846, + "FUNCTIONS": -7.929846, + "GUI": -7.929846, + "GUIControl": -4.710971, + "IsGamePaused": -6.831234, + "IsInterfaceEnabled": -7.236699, + "IsKeyPressed": -5.096633, + "IsSpeechVoxAvailable": -7.929846, + "KeyboardMovement": -7.236699, + "KeyboardMovement_CurrentDirection": -5.983936, + "KeyboardMovement_Directions": -6.543552, + "KeyboardMovement_KeyDown": -6.320409, + "KeyboardMovement_KeyDownLeft": -6.831234, + "KeyboardMovement_KeyDownRight": -6.831234, + "KeyboardMovement_KeyLeft": -6.320409, + "KeyboardMovement_KeyRight": -6.320409, + "KeyboardMovement_KeyStop": -6.831234, + "KeyboardMovement_KeyUp": -6.320409, + "KeyboardMovement_KeyUpLeft": -6.831234, + "KeyboardMovement_KeyUpRight": -6.831234, + "KeyboardMovement_Mode": -6.543552, + "KeyboardMovement_Modes": -6.543552, + "KeyboardMovement_VERSION": -7.929846, + "MouseButton": -4.838804, + "OF": -7.929846, + "PANEL": -7.929846, + "ProcessClick": -7.929846, + "QuitGame": -7.236699, + "RestoreGameSlot": -7.929846, + "SaveGameSlot": -7.929846, + "SetMode": -7.236699, + "SetVoiceMode": -6.543552, + "System.SupportsGammaControl": -7.929846, + "System.Volume": -7.929846, + "W": -7.929846, + "Wait": -6.831234, + "[": -6.138087, + "]": -6.138087, + "appreciate.": -7.929846, + "area": -7.929846, + "as": -7.929846, + "btnAbout_OnClick": -7.929846, + "btnCancelRestore_OnClick": -7.929846, + "btnCancelSave_OnClick": -7.929846, + "btnDeleteSave_OnClick": -7.929846, + "btnIconAbout_Click": -7.929846, + "btnIconCurInv_Click": -7.929846, + "btnIconExit_Click": -7.929846, + "btnIconInv_Click": -7.929846, + "btnIconLoad": -7.929846, + "btnIconLoad_Click": -7.236699, + "btnIconSave": -7.929846, + "btnIconSave_Click": -7.236699, + "btnInvDown_Click": -7.929846, + "btnInvOK_Click": -7.929846, + "btnInvSelect_Click": -7.929846, + "btnInvUp_Click": -7.929846, + "btnLoad_OnClick": -7.929846, + "btnQuit_OnClick": -7.929846, + "btnRestoreGame_OnClick": -7.929846, + "btnResume_OnClick": -7.929846, + "btnSaveGame_OnClick": -7.236699, + "btnSave_OnClick": -7.929846, + "btnVoice.Text": -5.983936, + "btnVoice.Visible": -7.929846, + "button": -4.751793, + "cEgo_Interact": -7.929846, + "cEgo_Look": -7.929846, + "cEgo_Talk": -7.929846, + "cancel": -7.929846, + "close_restore_game_dialog": -6.831234, + "close_save_game_dialog": -6.831234, + "control": -7.929846, + "data": -7.929846, + "dx": -4.934114, + "dy": -4.934114, + "eEventLeaveRoom": -7.929846, + "eKeyCtrlA": -7.929846, + "eKeyCtrlS": -7.929846, + "eKeyCtrlV": -7.929846, + "eKeyCtrlW": -7.929846, + "eKeyCtrlX": -7.929846, + "eKeyEscape": -7.929846, + "eKeyboardMovement_Down": -6.320409, + "eKeyboardMovement_DownLeft": -6.320409, + "eKeyboardMovement_DownRight": -6.320409, + "eKeyboardMovement_Left": -6.320409, + "eKeyboardMovement_None": -7.236699, + "eKeyboardMovement_Pressing": -7.236699, + "eKeyboardMovement_Right": -6.320409, + "eKeyboardMovement_Stop": -5.732622, + "eKeyboardMovement_Tapping": -7.236699, + "eKeyboardMovement_Up": -6.320409, + "eKeyboardMovement_UpLeft": -6.320409, + "eKeyboardMovement_UpRight": -6.320409, + "eModeInteract": -7.929846, + "eModePickup": -7.929846, + "eModePointer": -6.320409, + "eModeTalkto": -7.929846, + "eModeUseinv": -7.236699, + "eModeWalkto": -7.929846, + "eMouseLeft": -6.831234, + "eMouseWheelNorth": -7.929846, + "eNoBlock": -7.236699, + "eSpeechTextOnly": -7.929846, + "eSpeechVoiceAndText": -7.236699, + "eSpeechVoiceOnly": -7.929846, + "else": -4.292260, + "enum": -7.236699, + "event": -7.236699, + "false": -4.985407, + "function": -4.240967, + "gControl_OnClick": -7.929846, + "gIconbar.Visible": -5.627261, + "gInventory.Visible": -7.929846, + "gPanel.Centre": -7.929846, + "gPanel.Visible": -5.983936, + "gRestartYN.Centre": -7.929846, + "gRestartYN.Visible": -7.236699, + "gRestoreGame.Visible": -7.236699, + "gSaveGame.Visible": -7.929846, + "game.debug_mode": -7.929846, + "gameSlotToSaveInto": -6.831234, + "give": -7.929846, + "i": -6.320409, + "if": -3.835502, + "import": -7.929846, + "initialize_control_panel": -7.929846, + "int": -5.096633, + "interface": -7.929846, + "interface_click": -7.929846, + "invCustomInv.ScrollDown": -7.929846, + "invCustomInv.ScrollUp": -7.929846, + "is": -7.929846, + "keyboard": -7.929846, + "keycode": -4.985407, + "lblVoice.Visible": -7.929846, + "lstRestoreGamesList.FillSaveGameList": -7.929846, + "lstRestoreGamesList.SaveGameSlots": -7.929846, + "lstRestoreGamesList.SelectedIndex": -7.236699, + "lstSaveGamesList.FillSaveGameList": -7.929846, + "lstSaveGamesList.ItemCount": -7.236699, + "lstSaveGamesList.Items": -6.831234, + "lstSaveGamesList.SaveGameSlots": -7.236699, + "lstSaveGamesList.SelectedIndex": -6.831234, + "lstSaveGamesList_OnSelectionCh": -7.929846, + "mode": -6.831234, + "mouse.Mode": -6.320409, + "mouse.UseDefaultGraphic": -5.983936, + "mouse.UseModeGraphic": -6.320409, + "mouse.x": -7.929846, + "mouse.y": -7.929846, + "move": -7.929846, + "much": -7.929846, + "newdirection": -4.168646, + "null": -7.929846, + "on_event": -7.929846, + "on_key_press": -7.929846, + "on_mouse_click": -7.929846, + "player.ActiveInventory": -7.929846, + "player.PlaceOnWalkableArea": -7.929846, + "player.StopMoving": -6.831234, + "player.WalkStraight": -7.236699, + "player.on": -7.236699, + "player.x": -7.236699, + "player.y": -7.236699, + "players": -7.929846, + "repeatedly_execute": -7.929846, + "restart.": -7.929846, + "return": -7.236699, + "shortcuts": -7.929846, + "show_inventory_window": -7.929846, + "show_restore_game_dialog": -7.236699, + "show_save_game_dialog": -7.929846, + "sldAudio.Value": -7.929846, + "sldAudio_OnChange": -7.929846, + "sldGamma_OnChange": -7.929846, + "sldVoice.Visible": -7.929846, + "sldVoice_OnChange": -7.929846, + "some": -7.929846, + "standard": -7.929846, + "static": -7.236699, + "struct": -7.929846, + "this": -7.929846, + "to": -7.236699, + "true": -5.531951, + "txtNewSaveName.Text": -6.320409, + "txtNewSaveName_OnActivate": -7.929846, + "very": -7.929846, + "walkable": -7.929846, + "while": -7.929846, + "will": -7.236699, + "you": -7.929846, + "your": -7.929846, + "{": -3.586041, + "||": -5.531951, + "}": -3.560399, + }, + "AMPL": map[string]float64{ + "(": -3.988984, + ")": -3.988984, + "*": -4.276666, + "+": -6.068426, + ",": -2.667228, + "-": -6.068426, + "..": -4.969813, + ":": -3.123987, + ";": -2.602690, + "<": -6.068426, + "=": -3.072693, + ">": -3.765840, + "BEAMS": -4.122515, + "COLUMNS": -4.276666, + "I": -4.122515, + "KnapsackBound": -4.969813, + "ROWS": -4.276666, + "S": -4.969813, + "T": -4.969813, + "Take": -4.969813, + "TotalValue": -6.068426, + "Value": -4.969813, + "Weight": -4.969813, + "WeightLimit": -6.068426, + "X": -4.458988, + "[": -3.178054, + "]": -3.178054, + "beam_values": -4.458988, + "binary": -6.068426, + "critical_area": -4.458988, + "critical_limit": -6.068426, + "critical_max": -5.375278, + "critical_values": -5.375278, + "data": -6.068426, + "h": -3.072693, + "i": -3.123987, + "in": -3.123987, + "integer": -5.375278, + "k": -3.072693, + "maximize": -5.375278, + "minimize": -4.969813, + "num_beams": -5.375278, + "num_cols": -5.375278, + "num_rows": -5.375278, + "param": -3.503476, + "s.t.": -6.068426, + "set": -4.276666, + "subject": -5.375278, + "sum": -3.765840, + "to": -5.375278, + "total_critical_dosage": -6.068426, + "total_critical_slack": -6.068426, + "total_tumor_dosage": -6.068426, + "total_tumor_slack": -6.068426, + "tumor_area": -4.458988, + "tumor_limit": -6.068426, + "tumor_min": -5.375278, + "tumor_values": -5.375278, + "var": -4.682131, + "{": -2.932931, + "}": -2.932931, + }, + "API Blueprint": map[string]float64{ + "!": -5.216746, + "#": -4.369448, + "##": -6.315358, + "%": -3.319626, + "'": -6.315358, + "(": -3.057261, + ")": -3.057261, + "**": -6.315358, + "**API": -6.315358, + "**Note": -6.315358, + "**parse**": -6.315358, + "**real": -6.315358, + "+": -3.096482, + ",": -3.542769, + "-": -3.917463, + ".": -5.622211, + "//github.com/apiaryio/drafter": -6.315358, + "//github.com/apiaryio/drafter#bindings": -6.315358, + "//raw.github.com/apiaryio/api": -4.929064, + ":": -2.760010, + "A": -4.523599, + "API": -4.118133, + "API.md": -5.622211, + "APIs": -6.315358, + "Action.md": -6.315358, + "Actions": -5.622211, + "Actions.md": -5.622211, + "Advanced": -4.705920, + "Also": -6.315358, + "As": -6.315358, + "Attributes": -5.216746, + "Attributes.md": -5.216746, + "Blueprint": -4.705920, + "Blueprint**": -6.315358, + "Blueprint**.": -6.315358, + "Body": -6.315358, + "Date": -6.315358, + "FORMAT": -5.216746, + "Github": -6.315358, + "Hello": -6.315358, + "ID.": -6.315358, + "In": -6.315358, + "JSON": -6.315358, + "Markdown": -6.315358, + "Model": -6.315358, + "Model.md": -6.315358, + "Next": -5.622211, + "One": -6.315358, + "Parameters": -4.929064, + "Parameters.md": -6.315358, + "Previous": -5.622211, + "Raw": -4.929064, + "Resource": -4.523599, + "Response": -4.705920, + "Retrieves": -6.315358, + "Simplest": -5.622211, + "The": -6.315358, + "This": -4.369448, + "Time": -6.315358, + "We": -6.315358, + "World": -6.315358, + "[": -3.750409, + "]": -3.750409, + "a": -4.369448, + "about": -6.315358, + "action": -5.622211, + "after": -6.315358, + "also": -6.315358, + "amount": -6.315358, + "an": -6.315358, + "and": -4.118133, + "another": -5.622211, + "application/json": -5.216746, + "apply": -6.315358, + "apply.": -6.315358, + "as": -5.622211, + "attributes": -5.622211, + "avoid": -6.315358, + "be": -6.315358, + "between": -6.315358, + "bindings": -6.315358, + "blueprint/master/examples/": -4.929064, + "body": -5.622211, + "can": -5.622211, + "case": -6.315358, + "code": -6.315358, + "combined": -6.315358, + "complementary": -6.315358, + "contains": -6.315358, + "coupon": -4.929064, + "course": -6.315358, + "created": -6.315358, + "customer.": -6.315358, + "demonstrate": -6.315358, + "demonstrates": -5.622211, + "describe": -6.315358, + "description": -6.315358, + "descriptions.": -6.315358, + "discount": -5.622211, + "do": -6.315358, + "duplicate": -6.315358, + "duplicates": -6.315358, + "every": -6.315358, + "example": -4.705920, + "examples": -6.315358, + "explain": -6.315358, + "fact": -6.315358, + "false": -5.622211, + "forget": -6.315358, + "given": -6.315358, + "going": -5.622211, + "how": -5.216746, + "https": -4.523599, + "id": -5.216746, + "in": -4.369448, + "information": -6.315358, + "installment": -6.315358, + "integer": -6.315358, + "is": -4.369448, + "it": -5.622211, + "its": -6.315358, + "just": -6.315358, + "keep": -6.315358, + "longer": -6.315358, + "message.": -6.315358, + "method": -6.315358, + "might": -6.315358, + "mind": -6.315358, + "next": -6.315358, + "no": -6.315358, + "not": -6.315358, + "null": -6.315358, + "number": -4.929064, + "object": -6.315358, + "of": -4.929064, + "off": -5.622211, + "on": -5.622211, + "one": -5.622211, + "opposed": -6.315358, + "or": -5.216746, + "output": -6.315358, + "parser": -6.315358, + "parser.": -6.315358, + "percent": -6.315358, + "percent_off": -6.315358, + "plain": -6.315358, + "please": -6.315358, + "positive": -6.315358, + "priority": -6.315358, + "progress": -6.315358, + "provided": -6.315358, + "really": -6.315358, + "redeem_by": -6.315358, + "redeemed": -6.315358, + "represents": -6.315358, + "request": -6.315358, + "resource": -5.216746, + "resource.": -6.315358, + "response": -6.315358, + "reuse": -6.315358, + "s": -6.315358, + "section.": -6.315358, + "see": -6.315358, + "seeing": -6.315358, + "simplest": -6.315358, + "single": -6.315358, + "stamp": -6.315358, + "state": -5.216746, + "status": -6.315358, + "string": -4.929064, + "such": -6.315358, + "text/plain": -6.315358, + "that": -5.216746, + "the": -3.482145, + "this": -5.622211, + "through": -6.315358, + "to": -3.917463, + "transition": -5.622211, + "transition.": -6.315358, + "true": -6.315358, + "view": -6.315358, + "want": -6.315358, + "we": -6.315358, + "what": -5.622211, + "which": -6.315358, + "will": -5.216746, + "with": -5.216746, + "written": -6.315358, + "you": -5.622211, + "{": -4.929064, + "}": -4.929064, + "–": -5.622211, + }, + "APL": map[string]float64{ + "!": -7.546446, + "#": -7.546446, + "#.": -6.853299, + "#.DISPLAY": -7.546446, + "#.Files.Dir": -7.546446, + "#.UT.appdir": -6.447834, + "#.UT.expect": -7.546446, + "#.UT.run": -7.546446, + "(": -3.521095, + ")": -3.503395, + "+": -5.061540, + ",": -3.080538, + "-": -5.349222, + "/": -4.907389, + "//tryapl.org/": -7.546446, + "/C": -7.546446, + "/Functions": -7.546446, + "/Page": -7.546446, + "/Z": -7.546446, + "/config": -7.546446, + "/input": -7.546446, + "/lines": -7.546446, + "/usr/local/bin/apl": -7.546446, + ":": -3.341754, + ";": -3.341754, + "": -7.546446, + "": -7.546446, + "": -7.546446, + "": -7.546446, + "": -6.853299, + "": -7.546446, + "": -7.546446, + "": -7.546446, + "
":                              -7.546446,
+		"":                       -5.793014,
+		"":                       -5.793014,
+		"":                         -5.793014,
+		"":                         -5.793014,
+		"":                         -5.793014,
+		"":         -5.793014,
+		"":           -4.694401,
+		"":             -8.567126,
+		"":             -6.082219,
+		"":             -8.567126,
+		"":                -5.234921,
+		"":              -6.775366,
+		"":             -7.180831,
+		"":                -5.928068,
+		"":                -6.487684,
+		"":                  -8.567126,
+		"
":                  -8.567126,
+		"":                  -4.318630,
+		"":               -6.775366,
+		"
":                -7.468513,
+		"
":                 -6.775366,
+		"