Use Go1.11 module (#5743)
* Migrate to go modules * make vendor * Update mvdan.cc/xurls * make vendor * Update code.gitea.io/git * make fmt-check * Update github.com/go-sql-driver/mysql * make vendortokarchuk/v1.17
Mura Li
6 years ago
committed by
Lunny Xiao
parent
d578b71d61
commit
d77176912b
File diff suppressed because it is too large
Load Diff
@ -1,119 +0,0 @@ |
|||||||
|
|
||||||
ignored = ["google.golang.org/appengine*"] |
|
||||||
|
|
||||||
[prune] |
|
||||||
go-tests = true |
|
||||||
unused-packages = true |
|
||||||
non-go = true |
|
||||||
|
|
||||||
[[constraint]] |
|
||||||
branch = "master" |
|
||||||
name = "code.gitea.io/git" |
|
||||||
|
|
||||||
[[constraint]] |
|
||||||
branch = "master" |
|
||||||
name = "code.gitea.io/sdk" |
|
||||||
|
|
||||||
[[constraint]] |
|
||||||
revision = "05d86ea8f6e30456949f612cf68cf4a27ce8c9c5" |
|
||||||
name = "github.com/blevesearch/bleve" |
|
||||||
|
|
||||||
[[constraint]] |
|
||||||
revision = "12dd70caea0268ac0d6c2707d0611ef601e7c64e" |
|
||||||
name = "golang.org/x/crypto" |
|
||||||
|
|
||||||
[[constraint]] |
|
||||||
branch = "master" |
|
||||||
name = "golang.org/x/sys" |
|
||||||
|
|
||||||
[[constraint]] |
|
||||||
revision = "2bf8f2a19ec09c670e931282edfe6567f6be21c9" |
|
||||||
name = "golang.org/x/text" |
|
||||||
|
|
||||||
[[constraint]] |
|
||||||
branch = "master" |
|
||||||
name = "golang.org/x/net" |
|
||||||
|
|
||||||
[[override]] |
|
||||||
name = "github.com/go-xorm/xorm" |
|
||||||
revision = "a6300f2a45e05a8f75f00a1d6188049fe7851915" |
|
||||||
|
|
||||||
[[override]] |
|
||||||
name = "github.com/go-xorm/builder" |
|
||||||
version = "0.3.3" |
|
||||||
|
|
||||||
[[override]] |
|
||||||
name = "github.com/go-sql-driver/mysql" |
|
||||||
revision = "c45f530f8e7fe40f4687eaa50d0c8c5f1b66f9e0" |
|
||||||
|
|
||||||
[[override]] |
|
||||||
name = "github.com/mattn/go-sqlite3" |
|
||||||
revision = "c7c4067b79cc51e6dfdcef5c702e74b1e0fa7c75" |
|
||||||
|
|
||||||
[[override]] |
|
||||||
name = "github.com/gorilla/mux" |
|
||||||
revision = "757bef944d0f21880861c2dd9c871ca543023cba" |
|
||||||
|
|
||||||
[[constraint]] |
|
||||||
name = "github.com/gorilla/context" |
|
||||||
version = "1.1.1" |
|
||||||
|
|
||||||
[[constraint]] |
|
||||||
name = "github.com/lafriks/xormstore" |
|
||||||
version = "1.0.0" |
|
||||||
|
|
||||||
[[constraint]] |
|
||||||
branch = "master" |
|
||||||
name = "github.com/lunny/dingtalk_webhook" |
|
||||||
|
|
||||||
[[constraint]] |
|
||||||
name = "github.com/markbates/goth" |
|
||||||
version = "1.47.2" |
|
||||||
|
|
||||||
[[constraint]] |
|
||||||
branch = "master" |
|
||||||
name = "github.com/mcuadros/go-version" |
|
||||||
|
|
||||||
[[constraint]] |
|
||||||
branch = "master" |
|
||||||
name = "github.com/russross/blackfriday" |
|
||||||
|
|
||||||
[[constraint]] |
|
||||||
branch = "master" |
|
||||||
name = "github.com/tstranex/u2f" |
|
||||||
|
|
||||||
[[constraint]] |
|
||||||
name = "gopkg.in/editorconfig/editorconfig-core-go.v1" |
|
||||||
version = "1.2.0" |
|
||||||
|
|
||||||
[[constraint]] |
|
||||||
branch = "v2" |
|
||||||
name = "gopkg.in/gomail.v2" |
|
||||||
|
|
||||||
[[constraint]] |
|
||||||
name = "gopkg.in/ini.v1" |
|
||||||
version = "1.31.1" |
|
||||||
|
|
||||||
[[constraint]] |
|
||||||
name = "gopkg.in/ldap.v3" |
|
||||||
version = "3.0.1" |
|
||||||
|
|
||||||
[[constraint]] |
|
||||||
name = "gopkg.in/macaron.v1" |
|
||||||
version = "1.2.4" |
|
||||||
|
|
||||||
[[constraint]] |
|
||||||
name = "gopkg.in/testfixtures.v2" |
|
||||||
version = "2.0.0" |
|
||||||
|
|
||||||
[[override]] |
|
||||||
branch = "master" |
|
||||||
name = "golang.org/x/oauth2" |
|
||||||
|
|
||||||
[[constraint]] |
|
||||||
name = "github.com/prometheus/client_golang" |
|
||||||
version = "0.9.0" |
|
||||||
|
|
||||||
[[constraint]] |
|
||||||
name = "github.com/mvdan/xurls" |
|
||||||
version = "2.0.0" |
|
||||||
@ -0,0 +1,139 @@ |
|||||||
|
module code.gitea.io/gitea |
||||||
|
|
||||||
|
go 1.12 |
||||||
|
|
||||||
|
require ( |
||||||
|
code.gitea.io/sdk v0.0.0-20190321154058-a669487e86e0 |
||||||
|
github.com/BurntSushi/toml v0.3.1 // indirect |
||||||
|
github.com/PuerkitoBio/goquery v0.0.0-20170324135448-ed7d758e9a34 |
||||||
|
github.com/RoaringBitmap/roaring v0.4.7 // indirect |
||||||
|
github.com/Unknwon/cae v0.0.0-20160715032808-c6aac99ea2ca |
||||||
|
github.com/Unknwon/com v0.0.0-20170819223952-7677a1d7c113 |
||||||
|
github.com/Unknwon/i18n v0.0.0-20171114194641-b64d33658966 |
||||||
|
github.com/Unknwon/paginater v0.0.0-20151104151617-7748a72e0141 |
||||||
|
github.com/andybalholm/cascadia v0.0.0-20161224141413-349dd0209470 // indirect |
||||||
|
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 // indirect |
||||||
|
github.com/bgentry/speakeasy v0.1.0 // indirect |
||||||
|
github.com/blevesearch/bleve v0.0.0-20190214220507-05d86ea8f6e3 |
||||||
|
github.com/blevesearch/blevex v0.0.0-20180227211930-4b158bb555a3 // indirect |
||||||
|
github.com/blevesearch/go-porterstemmer v0.0.0-20141230013033-23a2c8e5cf1f // indirect |
||||||
|
github.com/blevesearch/segment v0.0.0-20160105220820-db70c57796cc // indirect |
||||||
|
github.com/boombuler/barcode v0.0.0-20161226211916-fe0f26ff6d26 // indirect |
||||||
|
github.com/bradfitz/gomemcache v0.0.0-20160117192205-fb1f79c6b65a // indirect |
||||||
|
github.com/chaseadamsio/goorgeous v0.0.0-20170901132237-098da33fde5f |
||||||
|
github.com/couchbase/gomemcached v0.0.0-20181122193126-5125a94a666c // indirect |
||||||
|
github.com/couchbase/goutils v0.0.0-20180530154633-e865a1461c8a // indirect |
||||||
|
github.com/couchbase/vellum v0.0.0-20190111184608-e91b68ff3efe // indirect |
||||||
|
github.com/couchbaselabs/go-couchbase v0.0.0-20190117181324-d904413d884d // indirect |
||||||
|
github.com/cznic/b v0.0.0-20181122101859-a26611c4d92d // indirect |
||||||
|
github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 // indirect |
||||||
|
github.com/cznic/strutil v0.0.0-20181122101858-275e90344537 // indirect |
||||||
|
github.com/denisenkom/go-mssqldb v0.0.0-20181014144952-4e0d7dc8888f |
||||||
|
github.com/dgrijalva/jwt-go v0.0.0-20161101193935-9ed569b5d1ac |
||||||
|
github.com/edsrzf/mmap-go v0.0.0-20170320065105-0bce6a688712 // indirect |
||||||
|
github.com/elazarl/go-bindata-assetfs v0.0.0-20151224045452-57eb5e1fc594 // indirect |
||||||
|
github.com/emirpasic/gods v1.12.0 // indirect |
||||||
|
github.com/etcd-io/bbolt v1.3.2 // indirect |
||||||
|
github.com/ethantkoenig/rupture v0.0.0-20180203182544-0a76f03a811a |
||||||
|
github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a // indirect |
||||||
|
github.com/facebookgo/ensure v0.0.0-20160127193407-b4ab57deab51 // indirect |
||||||
|
github.com/facebookgo/freeport v0.0.0-20150612182905-d4adf43b75b9 // indirect |
||||||
|
github.com/facebookgo/grace v0.0.0-20160926231715-5729e484473f |
||||||
|
github.com/facebookgo/httpdown v0.0.0-20160323221027-a3b1354551a2 // indirect |
||||||
|
github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect |
||||||
|
github.com/facebookgo/stats v0.0.0-20151006221625-1b76add642e4 // indirect |
||||||
|
github.com/facebookgo/subset v0.0.0-20150612182917-8dac2c3c4870 // indirect |
||||||
|
github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd // indirect |
||||||
|
github.com/glycerine/goconvey v0.0.0-20190315024820-982ee783a72e // indirect |
||||||
|
github.com/go-macaron/bindata v0.0.0-20161222093048-85786f57eee3 |
||||||
|
github.com/go-macaron/binding v0.0.0-20160711225916-9440f336b443 |
||||||
|
github.com/go-macaron/cache v0.0.0-20151013081102-561735312776 |
||||||
|
github.com/go-macaron/captcha v0.0.0-20151123225153-8aa5919789ab |
||||||
|
github.com/go-macaron/csrf v0.0.0-20180426211211-503617c6b372 |
||||||
|
github.com/go-macaron/i18n v0.0.0-20160612092837-ef57533c3b0f |
||||||
|
github.com/go-macaron/inject v0.0.0-20160627170012-d8a0b8677191 |
||||||
|
github.com/go-macaron/session v0.0.0-20190131233854-0a0a789bf193 |
||||||
|
github.com/go-macaron/toolbox v0.0.0-20180818072302-a77f45a7ce90 |
||||||
|
github.com/go-sql-driver/mysql v1.4.0 |
||||||
|
github.com/go-xorm/builder v0.3.3 |
||||||
|
github.com/go-xorm/core v0.6.0 |
||||||
|
github.com/go-xorm/xorm v0.0.0-20190116032649-a6300f2a45e0 |
||||||
|
github.com/gogits/chardet v0.0.0-20150115103509-2404f7772561 |
||||||
|
github.com/gogits/cron v0.0.0-20160810035002-7f3990acf183 |
||||||
|
github.com/gogo/protobuf v1.2.1 // indirect |
||||||
|
github.com/gorilla/context v1.1.1 |
||||||
|
github.com/issue9/assert v1.3.2 // indirect |
||||||
|
github.com/issue9/identicon v0.0.0-20160320065130-d36b54562f4c |
||||||
|
github.com/jaytaylor/html2text v0.0.0-20160923191438-8fb95d837f7d |
||||||
|
github.com/jmhodges/levigo v1.0.0 // indirect |
||||||
|
github.com/joho/godotenv v1.3.0 // indirect |
||||||
|
github.com/kballard/go-shellquote v0.0.0-20170619183022-cd60e84ee657 |
||||||
|
github.com/keybase/go-crypto v0.0.0-20170605145657-00ac4db533f6 |
||||||
|
github.com/klauspost/compress v0.0.0-20161025140425-8df558b6cb6f |
||||||
|
github.com/klauspost/cpuid v0.0.0-20160302075316-09cded8978dc // indirect |
||||||
|
github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6 // indirect |
||||||
|
github.com/lafriks/xormstore v1.0.0 |
||||||
|
github.com/lib/pq v1.0.0 |
||||||
|
github.com/lunny/dingtalk_webhook v0.0.0-20171025031554-e3534c89ef96 |
||||||
|
github.com/lunny/levelqueue v0.0.0-20190217115915-02b525a4418e |
||||||
|
github.com/lunny/log v0.0.0-20160921050905-7887c61bf0de // indirect |
||||||
|
github.com/lunny/nodb v0.0.0-20160621015157-fc1ef06ad4af // indirect |
||||||
|
github.com/markbates/goth v1.49.0 |
||||||
|
github.com/mattn/go-oci8 v0.0.0-20190320171441-14ba190cf52d // indirect |
||||||
|
github.com/mattn/go-sqlite3 v1.10.0 |
||||||
|
github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect |
||||||
|
github.com/mcuadros/go-version v0.0.0-20171003094716-88e56e02bea1 |
||||||
|
github.com/microcosm-cc/bluemonday v0.0.0-20161012083705-f77f16ffc87a |
||||||
|
github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae // indirect |
||||||
|
github.com/msteinert/pam v0.0.0-20151204160544-02ccfbfaf0cc |
||||||
|
github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5 |
||||||
|
github.com/philhofer/fwd v1.0.0 // indirect |
||||||
|
github.com/pkg/errors v0.8.1 // indirect |
||||||
|
github.com/pquerna/otp v0.0.0-20160912161815-54653902c20e |
||||||
|
github.com/prometheus/client_golang v0.9.0 |
||||||
|
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 // indirect |
||||||
|
github.com/prometheus/common v0.0.0-20181020173914-7e9e6cabbd39 // indirect |
||||||
|
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d // indirect |
||||||
|
github.com/remyoudompheng/bigfft v0.0.0-20190321074620-2f0d2b0e0001 // indirect |
||||||
|
github.com/russross/blackfriday v0.0.0-20180428102519-11635eb403ff |
||||||
|
github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca // indirect |
||||||
|
github.com/satori/go.uuid v1.2.0 |
||||||
|
github.com/sergi/go-diff v1.0.0 |
||||||
|
github.com/shurcooL/sanitized_anchor_name v0.0.0-20160918041101-1dba4b3954bc // indirect |
||||||
|
github.com/siddontang/go-snappy v0.0.0-20140704025258-d8f7bb82a96d // indirect |
||||||
|
github.com/smartystreets/goconvey v0.0.0-20190306220146-200a235640ff // indirect |
||||||
|
github.com/steveyen/gtreap v0.0.0-20150807155958-0abe01ef9be2 // indirect |
||||||
|
github.com/stretchr/testify v1.2.2 |
||||||
|
github.com/tecbot/gorocksdb v0.0.0-20181010114359-8752a9433481 // indirect |
||||||
|
github.com/tinylib/msgp v0.0.0-20180516164116-c8cf64dff200 // indirect |
||||||
|
github.com/tstranex/u2f v1.0.0 |
||||||
|
github.com/urfave/cli v0.0.0-20161102131801-d86a009f5e13 |
||||||
|
github.com/willf/bitset v0.0.0-20180426185212-8ce1146b8621 // indirect |
||||||
|
github.com/yohcop/openid-go v0.0.0-20160914080427-2c050d2dae53 |
||||||
|
go.etcd.io/bbolt v1.3.2 // indirect |
||||||
|
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793 |
||||||
|
golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519 |
||||||
|
golang.org/x/oauth2 v0.0.0-20181101160152-c453e0c75759 // indirect |
||||||
|
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f |
||||||
|
golang.org/x/sys v0.0.0-20181026144532-2772b66316d2 |
||||||
|
golang.org/x/text v0.3.0 |
||||||
|
gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect |
||||||
|
gopkg.in/asn1-ber.v1 v1.0.0-20150924051756-4e86f4367175 // indirect |
||||||
|
gopkg.in/bufio.v1 v1.0.0-20140618132640-567b2bfa514e // indirect |
||||||
|
gopkg.in/editorconfig/editorconfig-core-go.v1 v1.2.0 |
||||||
|
gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df |
||||||
|
gopkg.in/ini.v1 v1.31.1 |
||||||
|
gopkg.in/ldap.v3 v3.0.2 |
||||||
|
gopkg.in/macaron.v1 v1.3.2 |
||||||
|
gopkg.in/redis.v2 v2.3.2 // indirect |
||||||
|
gopkg.in/src-d/go-billy.v4 v4.3.0 // indirect |
||||||
|
gopkg.in/src-d/go-git.v4 v4.8.0 |
||||||
|
gopkg.in/testfixtures.v2 v2.5.0 |
||||||
|
mvdan.cc/xurls/v2 v2.0.0 |
||||||
|
strk.kbt.io/projects/go/libravatar v0.0.0-20160628055650-5eed7bff870a |
||||||
|
) |
||||||
|
|
||||||
|
replace ( |
||||||
|
github.com/denisenkom/go-mssqldb v0.0.0-20181014144952-4e0d7dc8888f => github.com/denisenkom/go-mssqldb v0.0.0-20161128230840-e32ca5036449 |
||||||
|
github.com/go-sql-driver/mysql v1.4.0 => github.com/go-sql-driver/mysql v0.0.0-20181218123637-c45f530f8e7f |
||||||
|
) |
||||||
@ -0,0 +1,364 @@ |
|||||||
|
cloud.google.com/go v0.30.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= |
||||||
|
code.gitea.io/sdk v0.0.0-20190321154058-a669487e86e0 h1:pIKKrTUox+0pGcZwFl19Glw9gKfoeNkA02EqeiSmLjs= |
||||||
|
code.gitea.io/sdk v0.0.0-20190321154058-a669487e86e0/go.mod h1:5bZt0dRznpn2JysytQnV0yCru3FwDv9O5G91jo+lDAk= |
||||||
|
github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= |
||||||
|
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= |
||||||
|
github.com/PuerkitoBio/goquery v0.0.0-20170324135448-ed7d758e9a34 h1:UsHpWO0Elp6NaWVARdZHjiYwkhrspHVEGsyIKPb9OI8= |
||||||
|
github.com/PuerkitoBio/goquery v0.0.0-20170324135448-ed7d758e9a34/go.mod h1:T9ezsOHcCrDCgA8aF1Cqr3sSYbO/xgdy8/R/XiIMAhA= |
||||||
|
github.com/RoaringBitmap/roaring v0.4.7 h1:eGUudvFzvF7Kxh7JjYvXfI1f7l22/2duFby7r5+d4oc= |
||||||
|
github.com/RoaringBitmap/roaring v0.4.7/go.mod h1:8khRDP4HmeXns4xIj9oGrKSz7XTQiJx2zgh7AcNke4w= |
||||||
|
github.com/Unknwon/cae v0.0.0-20160715032808-c6aac99ea2ca h1:xU8R31tsvj6TesCBog973+UgI3TXjh/LqN5clki6hcc= |
||||||
|
github.com/Unknwon/cae v0.0.0-20160715032808-c6aac99ea2ca/go.mod h1:IRSre9/SEhVuy972TVuJLyaPTS73+8Owhe0Y0l9NXHc= |
||||||
|
github.com/Unknwon/com v0.0.0-20170819223952-7677a1d7c113 h1:YwXm6KwmrA5R5yJRhcnpqRUHmBXSKciHuWtK9zP5qKQ= |
||||||
|
github.com/Unknwon/com v0.0.0-20170819223952-7677a1d7c113/go.mod h1:KYCjqMOeHpNuTOiFQU6WEcTG7poCJrUs0YgyHNtn1no= |
||||||
|
github.com/Unknwon/i18n v0.0.0-20171114194641-b64d33658966 h1:Mp8GNJ/tdTZIEdLdZfykEJaL3mTyEYrSzYNcdoQKpJk= |
||||||
|
github.com/Unknwon/i18n v0.0.0-20171114194641-b64d33658966/go.mod h1:SFtfq0zFPsENI7DpE87QM2hcYu5QQ0fRdCgP+P1Hrqo= |
||||||
|
github.com/Unknwon/paginater v0.0.0-20151104151617-7748a72e0141 h1:SSvHGK7iMpeypcHjI8UzNMz7zW/K8/dcgqk/82lCYP0= |
||||||
|
github.com/Unknwon/paginater v0.0.0-20151104151617-7748a72e0141/go.mod h1:fw0McLecf/G5NFwddCRmDckU6yovtk1YsgWIoepMbYo= |
||||||
|
github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7 h1:uSoVVbwJiQipAclBbw+8quDsfcvFjOpI5iCf4p/cqCs= |
||||||
|
github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/ghQa61ZWa/C2Aw3RkjiTBOix7dkqa1VLIs= |
||||||
|
github.com/andybalholm/cascadia v0.0.0-20161224141413-349dd0209470 h1:4jHLmof+Hba81591gfH5xYA8QXzuvgksxwPNrmjR2BA= |
||||||
|
github.com/andybalholm/cascadia v0.0.0-20161224141413-349dd0209470/go.mod h1:3I+3V7B6gTBYfdpYgIG2ymALS9H+5VDKUl3lHH7ToM4= |
||||||
|
github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo4JG6LR5AXSUEsOjtdm0kw0FtQtMJA= |
||||||
|
github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= |
||||||
|
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0= |
||||||
|
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= |
||||||
|
github.com/bgentry/speakeasy v0.1.0 h1:ByYyxL9InA1OWqxJqqp2A5pYHUrCiAL6K3J+LKSsQkY= |
||||||
|
github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= |
||||||
|
github.com/blevesearch/bleve v0.0.0-20190214220507-05d86ea8f6e3 h1:vinCy/rcjbtxWnMiw11CbMKcuyNi+y4L4MbZUpk7m4M= |
||||||
|
github.com/blevesearch/bleve v0.0.0-20190214220507-05d86ea8f6e3/go.mod h1:Y2lmIkzV6mcNfAnAdOd+ZxHkHchhBfU/xroGIp61wfw= |
||||||
|
github.com/blevesearch/blevex v0.0.0-20180227211930-4b158bb555a3 h1:U6vnxZrTfItfiUiYx0lf/LgHjRSfaKK5QHSom3lEbnA= |
||||||
|
github.com/blevesearch/blevex v0.0.0-20180227211930-4b158bb555a3/go.mod h1:WH+MU2F4T0VmSdaPX+Wu5GYoZBrYWdOZWSjzvYcDmqQ= |
||||||
|
github.com/blevesearch/go-porterstemmer v0.0.0-20141230013033-23a2c8e5cf1f h1:J9ZVHbB2X6JNxbKw/f3Y4E9Xq+Ro+zPiivzgmi3RTvg= |
||||||
|
github.com/blevesearch/go-porterstemmer v0.0.0-20141230013033-23a2c8e5cf1f/go.mod h1:haWQqFT3RdOGz7PJuM3or/pWNJS1pKkoZJWCkWu0DVA= |
||||||
|
github.com/blevesearch/segment v0.0.0-20160105220820-db70c57796cc h1:7OfDAkuAGx71ruzOIFqCkHqGIsVZU0C7PMw5u1bIrwU= |
||||||
|
github.com/blevesearch/segment v0.0.0-20160105220820-db70c57796cc/go.mod h1:IInt5XRvpiGE09KOk9mmCMLjHhydIhNPKPPFLFBB7L8= |
||||||
|
github.com/boombuler/barcode v0.0.0-20161226211916-fe0f26ff6d26 h1:NGpwhs9FOwddM6TptNrq2ycby4s24TcppSe5uG4DA/Q= |
||||||
|
github.com/boombuler/barcode v0.0.0-20161226211916-fe0f26ff6d26/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= |
||||||
|
github.com/bradfitz/gomemcache v0.0.0-20160117192205-fb1f79c6b65a h1:k5TuEkqEYCRs8+66WdOkswWOj+L/YbP5ruainvn94wg= |
||||||
|
github.com/bradfitz/gomemcache v0.0.0-20160117192205-fb1f79c6b65a/go.mod h1:PmM6Mmwb0LSuEubjR8N7PtNe1KxZLtOUHtbeikc5h60= |
||||||
|
github.com/chaseadamsio/goorgeous v0.0.0-20170901132237-098da33fde5f h1:REH9VH5ubNR0skLaOxK7TRJeRbE2dDfvaouQo8FsRcA= |
||||||
|
github.com/chaseadamsio/goorgeous v0.0.0-20170901132237-098da33fde5f/go.mod h1:6QaC0vFoKWYDth94dHFNgRT2YkT5FHdQp/Yx15aAAi0= |
||||||
|
github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I= |
||||||
|
github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= |
||||||
|
github.com/couchbase/gomemcached v0.0.0-20181122193126-5125a94a666c h1:K4FIibkr4//ziZKOKmt4RL0YImuTjLLBtwElf+F2lSQ= |
||||||
|
github.com/couchbase/gomemcached v0.0.0-20181122193126-5125a94a666c/go.mod h1:srVSlQLB8iXBVXHgnqemxUXqN6FCvClgCMPCsjBDR7c= |
||||||
|
github.com/couchbase/goutils v0.0.0-20180530154633-e865a1461c8a h1:Y5XsLCEhtEI8qbD9RP3Qlv5FXdTDHxZM9UPUnMRgBp8= |
||||||
|
github.com/couchbase/goutils v0.0.0-20180530154633-e865a1461c8a/go.mod h1:BQwMFlJzDjFDG3DJUdU0KORxn88UlsOULuxLExMh3Hs= |
||||||
|
github.com/couchbase/vellum v0.0.0-20190111184608-e91b68ff3efe h1:2o6Y7KMjJNsuMTF8f2H2eTKRhqH7+bQbjr+D+LnhE5M= |
||||||
|
github.com/couchbase/vellum v0.0.0-20190111184608-e91b68ff3efe/go.mod h1:prYTC8EgTu3gwbqJihkud9zRXISvyulAplQ6exdCo1g= |
||||||
|
github.com/couchbaselabs/go-couchbase v0.0.0-20190117181324-d904413d884d h1:lsBRLJe/ET6DjCaRblGwls80dOcOzhFVNJrO6uaMrMQ= |
||||||
|
github.com/couchbaselabs/go-couchbase v0.0.0-20190117181324-d904413d884d/go.mod h1:mby/05p8HE5yHEAKiIH/555NoblMs7PtW6NrYshDruc= |
||||||
|
github.com/cznic/b v0.0.0-20181122101859-a26611c4d92d h1:SwD98825d6bdB+pEuTxWOXiSjBrHdOl/UVp75eI7JT8= |
||||||
|
github.com/cznic/b v0.0.0-20181122101859-a26611c4d92d/go.mod h1:URriBxXwVq5ijiJ12C7iIZqlA69nTlI+LgI6/pwftG8= |
||||||
|
github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 h1:iwZdTE0PVqJCos1vaoKsclOGD3ADKpshg3SRtYBbwso= |
||||||
|
github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548/go.mod h1:e6NPNENfs9mPDVNRekM7lKScauxd5kXTr1Mfyig6TDM= |
||||||
|
github.com/cznic/strutil v0.0.0-20181122101858-275e90344537 h1:MZRmHqDBd0vxNwenEbKSQqRVT24d3C05ft8kduSwlqM= |
||||||
|
github.com/cznic/strutil v0.0.0-20181122101858-275e90344537/go.mod h1:AHHPPPXTw0h6pVabbcbyGRK1DckRn7r/STdZEeIDzZc= |
||||||
|
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= |
||||||
|
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= |
||||||
|
github.com/denisenkom/go-mssqldb v0.0.0-20161128230840-e32ca5036449 h1:JpA+YMG4JLW8nzLmU05mTiuB0O17xHGxpWolEZ0zDuA= |
||||||
|
github.com/denisenkom/go-mssqldb v0.0.0-20161128230840-e32ca5036449/go.mod h1:xN/JuLBIz4bjkxNmByTiV1IbhfnYb6oo99phBn4Eqhc= |
||||||
|
github.com/dgrijalva/jwt-go v0.0.0-20161101193935-9ed569b5d1ac h1:xrQJVwQCGqDvOO7/0+RyIq5J2M3Q4ZF7Ug/BMQtML1E= |
||||||
|
github.com/dgrijalva/jwt-go v0.0.0-20161101193935-9ed569b5d1ac/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= |
||||||
|
github.com/edsrzf/mmap-go v0.0.0-20170320065105-0bce6a688712 h1:aaQcKT9WumO6JEJcRyTqFVq4XUZiUcKR2/GI31TOcz8= |
||||||
|
github.com/edsrzf/mmap-go v0.0.0-20170320065105-0bce6a688712/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= |
||||||
|
github.com/elazarl/go-bindata-assetfs v0.0.0-20151224045452-57eb5e1fc594 h1:McZ/pt/pP/XAbLMDQGzm/iQUwW6OXmKVbFtmH9klWmc= |
||||||
|
github.com/elazarl/go-bindata-assetfs v0.0.0-20151224045452-57eb5e1fc594/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4= |
||||||
|
github.com/emirpasic/gods v1.9.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= |
||||||
|
github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg= |
||||||
|
github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= |
||||||
|
github.com/etcd-io/bbolt v1.3.2 h1:RLRQ0TKLX7DlBRXAJHvbmXL17Q3KNnTBtZ9B6Qo+/Y0= |
||||||
|
github.com/etcd-io/bbolt v1.3.2/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw= |
||||||
|
github.com/ethantkoenig/rupture v0.0.0-20180203182544-0a76f03a811a h1:M1bRpaZAn4GSsqu3hdK2R8H0AH9O6vqCTCbm2oAFGfE= |
||||||
|
github.com/ethantkoenig/rupture v0.0.0-20180203182544-0a76f03a811a/go.mod h1:MkKY/CB98aVE4VxO63X5vTQKUgcn+3XP15LMASe3lYs= |
||||||
|
github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a h1:yDWHCSQ40h88yih2JAcL6Ls/kVkSE8GFACTGVnMPruw= |
||||||
|
github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a/go.mod h1:7Ga40egUymuWXxAe151lTNnCv97MddSOVsjpPPkityA= |
||||||
|
github.com/facebookgo/ensure v0.0.0-20160127193407-b4ab57deab51 h1:0JZ+dUmQeA8IIVUMzysrX4/AKuQwWhV2dYQuPZdvdSQ= |
||||||
|
github.com/facebookgo/ensure v0.0.0-20160127193407-b4ab57deab51/go.mod h1:Yg+htXGokKKdzcwhuNDwVvN+uBxDGXJ7G/VN1d8fa64= |
||||||
|
github.com/facebookgo/freeport v0.0.0-20150612182905-d4adf43b75b9 h1:wWke/RUCl7VRjQhwPlR/v0glZXNYzBHdNUzf/Am2Nmg= |
||||||
|
github.com/facebookgo/freeport v0.0.0-20150612182905-d4adf43b75b9/go.mod h1:uPmAp6Sws4L7+Q/OokbWDAK1ibXYhB3PXFP1kol5hPg= |
||||||
|
github.com/facebookgo/grace v0.0.0-20160926231715-5729e484473f h1:0mlfEUWnUDVZnqWEVHGerL5bKYDKMEmT/Qk/W/3nGuo= |
||||||
|
github.com/facebookgo/grace v0.0.0-20160926231715-5729e484473f/go.mod h1:KigFdumBXUPSwzLDbeuzyt0elrL7+CP7TKuhrhT4bcU= |
||||||
|
github.com/facebookgo/httpdown v0.0.0-20160323221027-a3b1354551a2 h1:3Zvf9wRhl1cOhckN1oRGWPOkIhOketmEcrQ4TeFAoR4= |
||||||
|
github.com/facebookgo/httpdown v0.0.0-20160323221027-a3b1354551a2/go.mod h1:TUV/fX3XrTtBQb5+ttSUJzcFgLNpILONFTKmBuk5RSw= |
||||||
|
github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 h1:JWuenKqqX8nojtoVVWjGfOF9635RETekkoH6Cc9SX0A= |
||||||
|
github.com/facebookgo/stack v0.0.0-20160209184415-751773369052/go.mod h1:UbMTZqLaRiH3MsBH8va0n7s1pQYcu3uTb8G4tygF4Zg= |
||||||
|
github.com/facebookgo/stats v0.0.0-20151006221625-1b76add642e4 h1:0YtRCqIZs2+Tz49QuH6cJVw/IFqzo39gEqZ0iYLxD2M= |
||||||
|
github.com/facebookgo/stats v0.0.0-20151006221625-1b76add642e4/go.mod h1:vsJz7uE339KUCpBXx3JAJzSRH7Uk4iGGyJzR529qDIA= |
||||||
|
github.com/facebookgo/subset v0.0.0-20150612182917-8dac2c3c4870 h1:E2s37DuLxFhQDg5gKsWoLBOB0n+ZW8s599zru8FJ2/Y= |
||||||
|
github.com/facebookgo/subset v0.0.0-20150612182917-8dac2c3c4870/go.mod h1:5tD+neXqOorC30/tWg0LCSkrqj/AR6gu8yY8/fpw1q0= |
||||||
|
github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BHsljHzVlRcyQhjrss6TZTdY2VfCqZPbv5k3iBFa2ZQ= |
||||||
|
github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= |
||||||
|
github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= |
||||||
|
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= |
||||||
|
github.com/gliderlabs/ssh v0.1.1 h1:j3L6gSLQalDETeEg/Jg0mGY0/y/N6zI2xX1978P0Uqw= |
||||||
|
github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= |
||||||
|
github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd h1:r04MMPyLHj/QwZuMJ5+7tJcBr1AQjpiAK/rZWRrQT7o= |
||||||
|
github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= |
||||||
|
github.com/glycerine/goconvey v0.0.0-20190315024820-982ee783a72e h1:SiEs4J3BKVIeaWrH3tKaz3QLZhJ68iJ/A4xrzIoE5+Y= |
||||||
|
github.com/glycerine/goconvey v0.0.0-20190315024820-982ee783a72e/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24= |
||||||
|
github.com/go-macaron/bindata v0.0.0-20161222093048-85786f57eee3 h1:n0H90987ZwasTc9r/RnuaU1G5ePfzLG6Bkc1cY3KqnY= |
||||||
|
github.com/go-macaron/bindata v0.0.0-20161222093048-85786f57eee3/go.mod h1:NkmXhFuAlCOqgV5EWhU1DKLrgztCEIVXGmr2DZv/+sk= |
||||||
|
github.com/go-macaron/binding v0.0.0-20160711225916-9440f336b443 h1:i801KPR7j76uRMLLlGVyb0hiYbgX1FM5+ur81TJWzIw= |
||||||
|
github.com/go-macaron/binding v0.0.0-20160711225916-9440f336b443/go.mod h1:u+H6rwW+HQwUL+w5uaEJSpIlVZDye1o9MB4Su0JfRfM= |
||||||
|
github.com/go-macaron/cache v0.0.0-20151013081102-561735312776 h1:UYIHS1r0WotqB5cIa0PAiV0m6GzD9rDBcn4alp5JgCw= |
||||||
|
github.com/go-macaron/cache v0.0.0-20151013081102-561735312776/go.mod h1:hHAsZm/oBZVcY+S7qdQL6Vbg5VrXF6RuKGuqsszt3Ok= |
||||||
|
github.com/go-macaron/captcha v0.0.0-20151123225153-8aa5919789ab h1:4VFhsA3GE5Wwq1Ymr8KWCmrOWi1wRLEgdj48LPfQjxI= |
||||||
|
github.com/go-macaron/captcha v0.0.0-20151123225153-8aa5919789ab/go.mod h1:j9TJ+0nwUOWBvNnm0bheHIPFf3cC62EQo7n7O6PbjZA= |
||||||
|
github.com/go-macaron/csrf v0.0.0-20180426211211-503617c6b372 h1:acrx8CnDmlKl+BPoOOLEK9Ko+SrWFB5pxRuGkKj4iqo= |
||||||
|
github.com/go-macaron/csrf v0.0.0-20180426211211-503617c6b372/go.mod h1:oZGMxI7MBnicI0jJqJvH4qQzyrWKhtiKxLSJKHC+ydc= |
||||||
|
github.com/go-macaron/i18n v0.0.0-20160612092837-ef57533c3b0f h1:wDKrZFc9pYJlqFOf7EzGbFMrSFFtyHt3plr2uTdo8Rg= |
||||||
|
github.com/go-macaron/i18n v0.0.0-20160612092837-ef57533c3b0f/go.mod h1:MePM/dStkAh+PNzAdNSNl4SGDM2EZvZGken+KpJhM7s= |
||||||
|
github.com/go-macaron/inject v0.0.0-20160627170012-d8a0b8677191 h1:NjHlg70DuOkcAMqgt0+XA+NHwtu66MkTVVgR4fFWbcI= |
||||||
|
github.com/go-macaron/inject v0.0.0-20160627170012-d8a0b8677191/go.mod h1:VFI2o2q9kYsC4o7VP1HrEVosiZZTd+MVT3YZx4gqvJw= |
||||||
|
github.com/go-macaron/session v0.0.0-20190131233854-0a0a789bf193 h1:z/nqwd+ql/r6Q3QGnwNd6B89UjPytM0be5pDQV9TuWw= |
||||||
|
github.com/go-macaron/session v0.0.0-20190131233854-0a0a789bf193/go.mod h1:ScEJm9Gk+ez5JJTml5WlBIqavAfuE5nF8e4Gvyz/X+A= |
||||||
|
github.com/go-macaron/toolbox v0.0.0-20180818072302-a77f45a7ce90 h1:3wYKrRg9IjUMfaf3H0Hh7M5Li9ge79Y7aw2yujHa2jQ= |
||||||
|
github.com/go-macaron/toolbox v0.0.0-20180818072302-a77f45a7ce90/go.mod h1:Ut/NmkIMGVYlEdJBzEZgWVWG5ZpYS9BLmUgXfAgi+qM= |
||||||
|
github.com/go-sql-driver/mysql v0.0.0-20181218123637-c45f530f8e7f h1:fbIzwEaXt5b2bl9mm+PIufKTSGKk6ZuwSSTQ7iZj7Lo= |
||||||
|
github.com/go-sql-driver/mysql v0.0.0-20181218123637-c45f530f8e7f/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= |
||||||
|
github.com/go-xorm/builder v0.3.2/go.mod h1:v8mE3MFBgtL+RGFNfUnAMUqqfk/Y4W5KuwCFQIEpQLk= |
||||||
|
github.com/go-xorm/builder v0.3.3 h1:v8grgrwOGv/iHXIEhIvOwHZIPLrpxRKSX8yWSMLFn/4= |
||||||
|
github.com/go-xorm/builder v0.3.3/go.mod h1:v8mE3MFBgtL+RGFNfUnAMUqqfk/Y4W5KuwCFQIEpQLk= |
||||||
|
github.com/go-xorm/core v0.6.0 h1:tp6hX+ku4OD9khFZS8VGBDRY3kfVCtelPfmkgCyHxL0= |
||||||
|
github.com/go-xorm/core v0.6.0/go.mod h1:d8FJ9Br8OGyQl12MCclmYBuBqqxsyeedpXciV5Myih8= |
||||||
|
github.com/go-xorm/sqlfiddle v0.0.0-20180821085327-62ce714f951a h1:9wScpmSP5A3Bk8V3XHWUcJmYTh+ZnlHVyc+A4oZYS3Y= |
||||||
|
github.com/go-xorm/sqlfiddle v0.0.0-20180821085327-62ce714f951a/go.mod h1:56xuuqnHyryaerycW3BfssRdxQstACi0Epw/yC5E2xM= |
||||||
|
github.com/go-xorm/xorm v0.0.0-20190116032649-a6300f2a45e0 h1:GBnJjWjp2SGXBZsyZfYksyp7QocvQwf9vZQ0NRN2FXM= |
||||||
|
github.com/go-xorm/xorm v0.0.0-20190116032649-a6300f2a45e0/go.mod h1:EHS1htMQFptzMaIHKyzqpHGw6C9Rtug75nsq6DA9unI= |
||||||
|
github.com/gogits/chardet v0.0.0-20150115103509-2404f7772561 h1:deE7ritpK04PgtpyVOS2TYcQEld9qLCD5b5EbVNOuLA= |
||||||
|
github.com/gogits/chardet v0.0.0-20150115103509-2404f7772561/go.mod h1:YgYOrVn3Nj9Tq0EvjmFbphRytDj7JNRoWSStJZWDJTQ= |
||||||
|
github.com/gogits/cron v0.0.0-20160810035002-7f3990acf183 h1:EBTlva3AOSb80G3JSwY6ZMdILEZJ1JKuewrbqrNjWuE= |
||||||
|
github.com/gogits/cron v0.0.0-20160810035002-7f3990acf183/go.mod h1:pX+V62FFmklia2fhP3P4YSY6iJdPO5jIDKFQ5fEd5QE= |
||||||
|
github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE= |
||||||
|
github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= |
||||||
|
github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= |
||||||
|
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= |
||||||
|
github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db h1:woRePGFeVFfLKN/pOkfl+p/TAqKOfFu+7KPlMVpok/w= |
||||||
|
github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= |
||||||
|
github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ= |
||||||
|
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= |
||||||
|
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= |
||||||
|
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= |
||||||
|
github.com/gorilla/context v1.1.1 h1:AWwleXJkX/nhcU9bZSnZoi3h/qGYqQAGhq6zZe/aQW8= |
||||||
|
github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= |
||||||
|
github.com/gorilla/mux v1.6.2 h1:Pgr17XVTNXAk3q/r4CpKzC5xBM/qW1uVLV+IhRZpIIk= |
||||||
|
github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= |
||||||
|
github.com/gorilla/pat v0.0.0-20180118222023-199c85a7f6d1 h1:LqbZZ9sNMWVjeXS4NN5oVvhMjDyLhmA1LG86oSo+IqY= |
||||||
|
github.com/gorilla/pat v0.0.0-20180118222023-199c85a7f6d1/go.mod h1:YeAe0gNeiNT5hoiZRI4yiOky6jVdNvfO2N6Kav/HmxY= |
||||||
|
github.com/gorilla/securecookie v1.1.1 h1:miw7JPhV+b/lAHSXz4qd/nN9jRiAFV5FwjeKyCS8BvQ= |
||||||
|
github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= |
||||||
|
github.com/gorilla/sessions v1.1.1 h1:YMDmfaK68mUixINzY/XjscuJ47uXFWSSHzFbBQM0PrE= |
||||||
|
github.com/gorilla/sessions v1.1.1/go.mod h1:8KCfur6+4Mqcc6S0FEfKuN15Vl5MgXW92AE8ovaJD0w= |
||||||
|
github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= |
||||||
|
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= |
||||||
|
github.com/issue9/assert v1.3.2 h1:IaTa37u4m1fUuTH9K9ldO5IONKVDXjLiUO1T9vj0OF0= |
||||||
|
github.com/issue9/assert v1.3.2/go.mod h1:9Ger+iz8X7r1zMYYwEhh++2wMGWcNN2oVI+zIQXxcio= |
||||||
|
github.com/issue9/identicon v0.0.0-20160320065130-d36b54562f4c h1:A/PDn117UYld5mlxe58EpMguqpkeTMw5/FCo0ZPS/Ko= |
||||||
|
github.com/issue9/identicon v0.0.0-20160320065130-d36b54562f4c/go.mod h1:5mTb/PQNkqmq2x3IxlQZE0aSnTksJg7fg/oWmJ5SKXQ= |
||||||
|
github.com/jackc/fake v0.0.0-20150926172116-812a484cc733 h1:vr3AYkKovP8uR8AvSGGUK1IDqRa5lAAvEkZG1LKaCRc= |
||||||
|
github.com/jackc/fake v0.0.0-20150926172116-812a484cc733/go.mod h1:WrMFNQdiFJ80sQsxDoMokWK1W5TQtxBFNpzWTD84ibQ= |
||||||
|
github.com/jackc/pgx v3.2.0+incompatible h1:0Vihzu20St42/UDsvZGdNE6jak7oi/UOeMzwMPHkgFY= |
||||||
|
github.com/jackc/pgx v3.2.0+incompatible/go.mod h1:0ZGrqGqkRlliWnWB4zKnWtjbSWbGkVEFm4TeybAXq+I= |
||||||
|
github.com/jarcoal/httpmock v0.0.0-20180424175123-9c70cfe4a1da/go.mod h1:ks+b9deReOc7jgqp+e7LuFiCBH6Rm5hL32cLcEAArb4= |
||||||
|
github.com/jaytaylor/html2text v0.0.0-20160923191438-8fb95d837f7d h1:ig/iUfDDg06RVW8OMby+GrmW6K2nPO3AFHlEIdvJSd4= |
||||||
|
github.com/jaytaylor/html2text v0.0.0-20160923191438-8fb95d837f7d/go.mod h1:CVKlgaMiht+LXvHG173ujK6JUhZXKb2u/BQtjPDIvyk= |
||||||
|
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= |
||||||
|
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= |
||||||
|
github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= |
||||||
|
github.com/jmhodges/levigo v1.0.0 h1:q5EC36kV79HWeTBWsod3mG11EgStG3qArTKcvlksN1U= |
||||||
|
github.com/jmhodges/levigo v1.0.0/go.mod h1:Q6Qx+uH3RAqyK4rFQroq9RL7mdkABMcfhEI+nNuzMJQ= |
||||||
|
github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc= |
||||||
|
github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= |
||||||
|
github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= |
||||||
|
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= |
||||||
|
github.com/kballard/go-shellquote v0.0.0-20170619183022-cd60e84ee657 h1:vE7J1m7cCpiRVEIr1B5ccDxRpbPsWT5JU3if2Di5nE4= |
||||||
|
github.com/kballard/go-shellquote v0.0.0-20170619183022-cd60e84ee657/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= |
||||||
|
github.com/kevinburke/ssh_config v0.0.0-20180830205328-81db2a75821e h1:RgQk53JHp/Cjunrr1WlsXSZpqXn+uREuHvUVcK82CV8= |
||||||
|
github.com/kevinburke/ssh_config v0.0.0-20180830205328-81db2a75821e/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= |
||||||
|
github.com/keybase/go-crypto v0.0.0-20170605145657-00ac4db533f6 h1:9mszGwKDxHEY2cy+9XxCQKWIfkGPSAEFrcN8ghzyAKg= |
||||||
|
github.com/keybase/go-crypto v0.0.0-20170605145657-00ac4db533f6/go.mod h1:ghbZscTyKdM07+Fw3KSi0hcJm+AlEUWj8QLlPtijN/M= |
||||||
|
github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= |
||||||
|
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= |
||||||
|
github.com/klauspost/compress v0.0.0-20161025140425-8df558b6cb6f h1:tCnZKEmDovgV4jmsclh6CuKk9AMzTzyVWfejgkgccVg= |
||||||
|
github.com/klauspost/compress v0.0.0-20161025140425-8df558b6cb6f/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= |
||||||
|
github.com/klauspost/cpuid v0.0.0-20160302075316-09cded8978dc h1:WW8B7p7QBnFlqRVv/k6ro/S8Z7tCnYjJHcQNScx9YVs= |
||||||
|
github.com/klauspost/cpuid v0.0.0-20160302075316-09cded8978dc/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= |
||||||
|
github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6 h1:KAZ1BW2TCmT6PRihDPpocIy1QTtsAsrx6TneU/4+CMg= |
||||||
|
github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6/go.mod h1:+ZoRqAPRLkC4NPOvfYeR5KNOrY6TD+/sAC3HXPZgDYg= |
||||||
|
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= |
||||||
|
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= |
||||||
|
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= |
||||||
|
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= |
||||||
|
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= |
||||||
|
github.com/lafriks/xormstore v1.0.0 h1:P/IJzNSIpjXl/Up3o2Td5ZU/x4v6DEKLMaPQJGtmJCk= |
||||||
|
github.com/lafriks/xormstore v1.0.0/go.mod h1:dD8vHNRfEp3Uy+JvX9cMi2SXcRKJ0x4pYKsZuy843Ic= |
||||||
|
github.com/lib/pq v1.0.0 h1:X5PMW56eZitiTeO7tKzZxFCSpbFZJtkMMooicw2us9A= |
||||||
|
github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= |
||||||
|
github.com/lunny/dingtalk_webhook v0.0.0-20171025031554-e3534c89ef96 h1:uNwtsDp7ci48vBTTxDuwcoTXz4lwtDTe7TjCQ0noaWY= |
||||||
|
github.com/lunny/dingtalk_webhook v0.0.0-20171025031554-e3534c89ef96/go.mod h1:mmIfjCSQlGYXmJ95jFN84AkQFnVABtKuJL8IrzwvUKQ= |
||||||
|
github.com/lunny/levelqueue v0.0.0-20190217115915-02b525a4418e h1:GSprKUrG9wNgwQgROvjPGXmcZrg4OLslOuZGB0uJjx8= |
||||||
|
github.com/lunny/levelqueue v0.0.0-20190217115915-02b525a4418e/go.mod h1:rQZVENnBOiVakCs97XvclbwJRTAv77CRFWcYVNDkVf8= |
||||||
|
github.com/lunny/log v0.0.0-20160921050905-7887c61bf0de h1:nyxwRdWHAVxpFcDThedEgQ07DbcRc5xgNObtbTp76fk= |
||||||
|
github.com/lunny/log v0.0.0-20160921050905-7887c61bf0de/go.mod h1:3q8WtuPQsoRbatJuy3nvq/hRSvuBJrHHr+ybPPiNvHQ= |
||||||
|
github.com/lunny/nodb v0.0.0-20160621015157-fc1ef06ad4af h1:UaWHNBdukWrSG3DRvHFR/hyfg681fceqQDYVTBncKfQ= |
||||||
|
github.com/lunny/nodb v0.0.0-20160621015157-fc1ef06ad4af/go.mod h1:Cqz6pqow14VObJ7peltM+2n3PWOz7yTrfUuGbVFkzN0= |
||||||
|
github.com/markbates/going v1.0.0/go.mod h1:I6mnB4BPnEeqo85ynXIx1ZFLLbtiLHNXVgWeFO9OGOA= |
||||||
|
github.com/markbates/goth v1.49.0 h1:qQ4Ti4WaqAxNAggOC+4s5M85sMVfMJwQn/Xkp73wfgI= |
||||||
|
github.com/markbates/goth v1.49.0/go.mod h1:zZmAw0Es0Dpm7TT/4AdN14QrkiWLMrrU9Xei1o+/mdA= |
||||||
|
github.com/mattn/go-oci8 v0.0.0-20190320171441-14ba190cf52d h1:m+dSK37rFf2fqppZhg15yI2IwC9BtucBiRwSDm9VL8g= |
||||||
|
github.com/mattn/go-oci8 v0.0.0-20190320171441-14ba190cf52d/go.mod h1:/M9VLO+lUPmxvoOK2PfWRZ8mTtB4q1Hy9lEGijv9Nr8= |
||||||
|
github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= |
||||||
|
github.com/mattn/go-sqlite3 v1.10.0 h1:jbhqpg7tQe4SupckyijYiy0mJJ/pRyHvXf7JdWK860o= |
||||||
|
github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= |
||||||
|
github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= |
||||||
|
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= |
||||||
|
github.com/mcuadros/go-version v0.0.0-20171003094716-88e56e02bea1 h1:4yXas1DDHpauOfuyfmAMm+EB+SiqPKEoTdc88XEJHsc= |
||||||
|
github.com/mcuadros/go-version v0.0.0-20171003094716-88e56e02bea1/go.mod h1:76rfSfYPWj01Z85hUf/ituArm797mNKcvINh1OlsZKo= |
||||||
|
github.com/microcosm-cc/bluemonday v0.0.0-20161012083705-f77f16ffc87a h1:d18LCO3ctH2kugUqt0pEyKKP8L+IYrocaPqGFilhTKk= |
||||||
|
github.com/microcosm-cc/bluemonday v0.0.0-20161012083705-f77f16ffc87a/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4= |
||||||
|
github.com/mitchellh/go-homedir v1.0.0 h1:vKb8ShqSby24Yrqr/yDYkuFz8d0WUjys40rvnGC8aR0= |
||||||
|
github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= |
||||||
|
github.com/mrjones/oauth v0.0.0-20180629183705-f4e24b6d100c h1:3wkDRdxK92dF+c1ke2dtj7ZzemFWBHB9plnJOtlwdFA= |
||||||
|
github.com/mrjones/oauth v0.0.0-20180629183705-f4e24b6d100c/go.mod h1:skjdDftzkFALcuGzYSklqYd8gvat6F1gZJ4YPVbkZpM= |
||||||
|
github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae h1:VeRdUYdCw49yizlSbMEn2SZ+gT+3IUKx8BqxyQdz+BY= |
||||||
|
github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg= |
||||||
|
github.com/msteinert/pam v0.0.0-20151204160544-02ccfbfaf0cc h1:z1PgdCCmYYVL0BoJTUgmAq1p7ca8fzYIPsNyfsN3xAU= |
||||||
|
github.com/msteinert/pam v0.0.0-20151204160544-02ccfbfaf0cc/go.mod h1:np1wUFZ6tyoke22qDJZY40URn9Ae51gX7ljIWXN5TJs= |
||||||
|
github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5 h1:BvoENQQU+fZ9uukda/RzCAL/191HHwJA5b13R6diVlY= |
||||||
|
github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8= |
||||||
|
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= |
||||||
|
github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs= |
||||||
|
github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= |
||||||
|
github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU= |
||||||
|
github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= |
||||||
|
github.com/pelletier/go-buffruneio v0.2.0 h1:U4t4R6YkofJ5xHm3dJzuRpPZ0mr5MMCoAWooScCR7aA= |
||||||
|
github.com/pelletier/go-buffruneio v0.2.0/go.mod h1:JkE26KsDizTr40EUHkXVtNPvgGtbSNq5BcowyYOWdKo= |
||||||
|
github.com/philhofer/fwd v1.0.0 h1:UbZqGr5Y38ApvM/V/jEljVxwocdweyH+vmYvRPBnbqQ= |
||||||
|
github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= |
||||||
|
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= |
||||||
|
github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= |
||||||
|
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= |
||||||
|
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= |
||||||
|
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= |
||||||
|
github.com/pquerna/otp v0.0.0-20160912161815-54653902c20e h1:ApqncJ84HYN8x8x5WV1T1YWDuPRF/0aXZhr91LnRMCQ= |
||||||
|
github.com/pquerna/otp v0.0.0-20160912161815-54653902c20e/go.mod h1:Zad1CMQfSQZI5KLpahDiSUX4tMMREnXw98IvL1nhgMk= |
||||||
|
github.com/prometheus/client_golang v0.9.0 h1:tXuTFVHC03mW0D+Ua1Q2d1EAVqLTuggX50V0VLICCzY= |
||||||
|
github.com/prometheus/client_golang v0.9.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= |
||||||
|
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 h1:idejC8f05m9MGOsuEi1ATq9shN03HrxNkD/luQvxCv8= |
||||||
|
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= |
||||||
|
github.com/prometheus/common v0.0.0-20181020173914-7e9e6cabbd39 h1:Cto4X6SVMWRPBkJ/3YHn1iDGDGc/Z+sW+AEMKHMVvN4= |
||||||
|
github.com/prometheus/common v0.0.0-20181020173914-7e9e6cabbd39/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= |
||||||
|
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d h1:GoAlyOgbOEIFdaDqxJVlbOQ1DtGmZWs/Qau0hIlk+WQ= |
||||||
|
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= |
||||||
|
github.com/remyoudompheng/bigfft v0.0.0-20190321074620-2f0d2b0e0001 h1:YDeskXpkNDhPdWN3REluVa46HQOVuVkjkd2sWnrABNQ= |
||||||
|
github.com/remyoudompheng/bigfft v0.0.0-20190321074620-2f0d2b0e0001/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= |
||||||
|
github.com/russross/blackfriday v0.0.0-20180428102519-11635eb403ff h1:g9ZlAHmkc/h5So+OjNCkZWh+FjuKEOOOoyRkqlGA8+c= |
||||||
|
github.com/russross/blackfriday v0.0.0-20180428102519-11635eb403ff/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= |
||||||
|
github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca h1:NugYot0LIVPxTvN8n+Kvkn6TrbMyxQiuvKdEwFdR9vI= |
||||||
|
github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU= |
||||||
|
github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= |
||||||
|
github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= |
||||||
|
github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ= |
||||||
|
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= |
||||||
|
github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24 h1:pntxY8Ary0t43dCZ5dqY4YTJCObLY1kIXl0uzMv+7DE= |
||||||
|
github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= |
||||||
|
github.com/shurcooL/sanitized_anchor_name v0.0.0-20160918041101-1dba4b3954bc h1:3wIrJvFb3Pf6B/2mDBnN1G5IfUVev4X5apadQlWOczE= |
||||||
|
github.com/shurcooL/sanitized_anchor_name v0.0.0-20160918041101-1dba4b3954bc/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= |
||||||
|
github.com/siddontang/go-snappy v0.0.0-20140704025258-d8f7bb82a96d h1:qQWKKOvHN7Q9c6GdmUteCef2F9ubxMpxY1IKwpIKz68= |
||||||
|
github.com/siddontang/go-snappy v0.0.0-20140704025258-d8f7bb82a96d/go.mod h1:vq0tzqLRu6TS7Id0wMo2N5QzJoKedVeovOpHjnykSzY= |
||||||
|
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= |
||||||
|
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= |
||||||
|
github.com/smartystreets/goconvey v0.0.0-20190306220146-200a235640ff h1:86HlEv0yBCry9syNuylzqznKXDK11p6D0DT596yNMys= |
||||||
|
github.com/smartystreets/goconvey v0.0.0-20190306220146-200a235640ff/go.mod h1:KSQcGKpxUMHk3nbYzs/tIBAM2iDooCn0BmttHOJEbLs= |
||||||
|
github.com/src-d/gcfg v1.4.0 h1:xXbNR5AlLSA315x2UO+fTSSAXCDf+Ar38/6oyGbDKQ4= |
||||||
|
github.com/src-d/gcfg v1.4.0/go.mod h1:p/UMsR43ujA89BJY9duynAwIpvqEujIH/jFlfL7jWoI= |
||||||
|
github.com/steveyen/gtreap v0.0.0-20150807155958-0abe01ef9be2 h1:JNEGSiWg6D3lcBCMCBqN3ELniXujt+0QNHLhNnO0w3s= |
||||||
|
github.com/steveyen/gtreap v0.0.0-20150807155958-0abe01ef9be2/go.mod h1:mjqs7N0Q6m5HpR7QfXVBZXZWSqTjQLeTujjA/xUp2uw= |
||||||
|
github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= |
||||||
|
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= |
||||||
|
github.com/syndtr/goleveldb v0.0.0-20190203031304-2f17a3356c66 h1:AwmkkZT+TucFotNCL+aNJ/0KCMsRtlXN9fs8uoOMSRk= |
||||||
|
github.com/syndtr/goleveldb v0.0.0-20190203031304-2f17a3356c66/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= |
||||||
|
github.com/tecbot/gorocksdb v0.0.0-20181010114359-8752a9433481 h1:HOxvxvnntLiPn123Fk+twfUhCQdMDaqmb0cclArW0T0= |
||||||
|
github.com/tecbot/gorocksdb v0.0.0-20181010114359-8752a9433481/go.mod h1:ahpPrc7HpcfEWDQRZEmnXMzHY03mLDYMCxeDzy46i+8= |
||||||
|
github.com/tinylib/msgp v0.0.0-20180516164116-c8cf64dff200 h1:ZVvr38DYEyOPyelySqvF0I9I++85NnUMsWkroBDS4fs= |
||||||
|
github.com/tinylib/msgp v0.0.0-20180516164116-c8cf64dff200/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= |
||||||
|
github.com/tstranex/u2f v1.0.0 h1:HhJkSzDDlVSVIVt7pDJwCHQj67k7A5EeBgPmeD+pVsQ= |
||||||
|
github.com/tstranex/u2f v1.0.0/go.mod h1:eahSLaqAS0zsIEv80+vXT7WanXs7MQQDg3j3wGBSayo= |
||||||
|
github.com/urfave/cli v0.0.0-20161102131801-d86a009f5e13 h1:niRuEF0NOlFnqraxzjuvvOdCM6gxmHiaBABjvg3/kDo= |
||||||
|
github.com/urfave/cli v0.0.0-20161102131801-d86a009f5e13/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= |
||||||
|
github.com/willf/bitset v0.0.0-20180426185212-8ce1146b8621 h1:E8u341JM/N8LCnPXBV6ZFD1RKo/j+qHl1XOqSV+GstA= |
||||||
|
github.com/willf/bitset v0.0.0-20180426185212-8ce1146b8621/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= |
||||||
|
github.com/xanzy/ssh-agent v0.2.0 h1:Adglfbi5p9Z0BmK2oKU9nTG+zKfniSfnaMYB+ULd+Ro= |
||||||
|
github.com/xanzy/ssh-agent v0.2.0/go.mod h1:0NyE30eGUDliuLEHJgYte/zncp2zdTStcOnWhgSqHD8= |
||||||
|
github.com/yohcop/openid-go v0.0.0-20160914080427-2c050d2dae53 h1:HsIQ6yAjfjQ3IxPGrTusxp6Qxn92gNVq2x5CbvQvx3w= |
||||||
|
github.com/yohcop/openid-go v0.0.0-20160914080427-2c050d2dae53/go.mod h1:f6elajwZV+xceiaqgRL090YzLEDGSbqr3poGL3ZgXYo= |
||||||
|
github.com/ziutek/mymysql v1.5.4 h1:GB0qdRGsTwQSBVYuVShFBKaXSnSnYYC2d9knnE1LHFs= |
||||||
|
github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0= |
||||||
|
go.etcd.io/bbolt v1.3.2 h1:Z/90sZLPOeCy2PwprqkFa25PdkusRzaj9P8zm/KNyvk= |
||||||
|
go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= |
||||||
|
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793 h1:u+LnwYTOOW7Ukr/fppxEb1Nwz0AtPflrblfvUudpo+I= |
||||||
|
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= |
||||||
|
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= |
||||||
|
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= |
||||||
|
golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519 h1:x6rhz8Y9CjbgQkccRGmELH6K+LJj7tOoh3XWeC1yaQM= |
||||||
|
golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= |
||||||
|
golang.org/x/oauth2 v0.0.0-20180620175406-ef147856a6dd/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= |
||||||
|
golang.org/x/oauth2 v0.0.0-20181101160152-c453e0c75759 h1:TMrx+Qdx7uJAeUbv15N72h5Hmyb5+VDjEiMufAEAM04= |
||||||
|
golang.org/x/oauth2 v0.0.0-20181101160152-c453e0c75759/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= |
||||||
|
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f h1:wMNYb4v58l5UBM7MYRLPG6ZhfOqbKu7X5eyFl8ZhKvA= |
||||||
|
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= |
||||||
|
golang.org/x/sys v0.0.0-20180903190138-2b024373dcd9/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= |
||||||
|
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= |
||||||
|
golang.org/x/sys v0.0.0-20181026144532-2772b66316d2 h1:W7CqTdBJ1CmxLKe7LptKDnBYV6PHrVLiGnoyBjaG/JQ= |
||||||
|
golang.org/x/sys v0.0.0-20181026144532-2772b66316d2/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= |
||||||
|
golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= |
||||||
|
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= |
||||||
|
golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= |
||||||
|
google.golang.org/appengine v1.2.0 h1:S0iUepdCWODXRvtE+gcRDd15L+k+k1AiHlMiMjefH24= |
||||||
|
google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= |
||||||
|
gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc h1:2gGKlE2+asNV9m7xrywl36YYNnBG5ZQ0r/BOOxqPpmk= |
||||||
|
gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc/go.mod h1:m7x9LTH6d71AHyAX77c9yqWCCa3UKHcVEj9y7hAtKDk= |
||||||
|
gopkg.in/asn1-ber.v1 v1.0.0-20150924051756-4e86f4367175 h1:nn6Zav2sOQHCFJHEspya8KqxhFwKci30UxHy3HXPTyQ= |
||||||
|
gopkg.in/asn1-ber.v1 v1.0.0-20150924051756-4e86f4367175/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= |
||||||
|
gopkg.in/bufio.v1 v1.0.0-20140618132640-567b2bfa514e h1:wGA78yza6bu/mWcc4QfBuIEHEtc06xdiU0X8sY36yUU= |
||||||
|
gopkg.in/bufio.v1 v1.0.0-20140618132640-567b2bfa514e/go.mod h1:xsQCaysVCudhrYTfzYWe577fCe7Ceci+6qjO2Rdc0Z4= |
||||||
|
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= |
||||||
|
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= |
||||||
|
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= |
||||||
|
gopkg.in/editorconfig/editorconfig-core-go.v1 v1.2.0 h1:CO465/foR4+bY1xNYjZEl6l8By1g/iMsImoruxfEt84= |
||||||
|
gopkg.in/editorconfig/editorconfig-core-go.v1 v1.2.0/go.mod h1:s2mQFI9McjArkyCwyEwU//+luQENTnD/Lfb/7Sj3/kQ= |
||||||
|
gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= |
||||||
|
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= |
||||||
|
gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df h1:n7WqCuqOuCbNr617RXOY0AWRXxgwEyPp2z+p0+hgMuE= |
||||||
|
gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df/go.mod h1:LRQQ+SO6ZHR7tOkpBDuZnXENFzX8qRjMDMyPD6BRkCw= |
||||||
|
gopkg.in/ini.v1 v1.31.1 h1:8EY/6KDwKM9Qg4vu1+01ZpsxClC/XV71R+nZ/TL7D4M= |
||||||
|
gopkg.in/ini.v1 v1.31.1/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= |
||||||
|
gopkg.in/ldap.v3 v3.0.2 h1:R6RBtabK6e1GO0eQKtkyOFbAHO73QesLzI2w2DZ6b9w= |
||||||
|
gopkg.in/ldap.v3 v3.0.2/go.mod h1:oxD7NyBuxchC+SgJDE1Q5Od05eGt29SDQVBmV+HYbzw= |
||||||
|
gopkg.in/macaron.v1 v1.3.2 h1:AvWIaPmwBUA87/OWzePkoxeaw6YJWDfBt1pDFPBnLf8= |
||||||
|
gopkg.in/macaron.v1 v1.3.2/go.mod h1:PrsiawTWAGZs6wFbT5hlr7SQ2Ns9h7cUVtcUu4lQOVo= |
||||||
|
gopkg.in/redis.v2 v2.3.2 h1:GPVIIB/JnL1wvfULefy3qXmPu1nfNu2d0yA09FHgwfs= |
||||||
|
gopkg.in/redis.v2 v2.3.2/go.mod h1:4wl9PJ/CqzeHk3LVq1hNLHH8krm3+AXEgut4jVc++LU= |
||||||
|
gopkg.in/src-d/go-billy.v4 v4.2.1/go.mod h1:tm33zBoOwxjYHZIE+OV8bxTWFMJLrconzFMd38aARFk= |
||||||
|
gopkg.in/src-d/go-billy.v4 v4.3.0 h1:KtlZ4c1OWbIs4jCv5ZXrTqG8EQocr0g/d4DjNg70aek= |
||||||
|
gopkg.in/src-d/go-billy.v4 v4.3.0/go.mod h1:tm33zBoOwxjYHZIE+OV8bxTWFMJLrconzFMd38aARFk= |
||||||
|
gopkg.in/src-d/go-git-fixtures.v3 v3.1.1 h1:XWW/s5W18RaJpmo1l0IYGqXKuJITWRFuA45iOf1dKJs= |
||||||
|
gopkg.in/src-d/go-git-fixtures.v3 v3.1.1/go.mod h1:dLBcvytrw/TYZsNTWCnkNF2DSIlzWYqTe3rJR56Ac7g= |
||||||
|
gopkg.in/src-d/go-git.v4 v4.8.0 h1:dDEbgvfNG9vUDM54uhCYPExiGa8uYgXpQ/MR8YvxcAM= |
||||||
|
gopkg.in/src-d/go-git.v4 v4.8.0/go.mod h1:Vtut8izDyrM8BUVQnzJ+YvmNcem2J89EmfZYCkLokZk= |
||||||
|
gopkg.in/stretchr/testify.v1 v1.2.2 h1:yhQC6Uy5CqibAIlk1wlusa/MJ3iAN49/BsR/dCCKz3M= |
||||||
|
gopkg.in/stretchr/testify.v1 v1.2.2/go.mod h1:QI5V/q6UbPmuhtm10CaFZxED9NreB8PnFYN9JcR6TxU= |
||||||
|
gopkg.in/testfixtures.v2 v2.5.0 h1:N08B7l2GzFQenyYbzqthDnKAA+cmb17iAZhhFxr7JHw= |
||||||
|
gopkg.in/testfixtures.v2 v2.5.0/go.mod h1:vyAq+MYCgNpR29qitQdLZhdbLFf4mR/2MFJRFoQZZ2M= |
||||||
|
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= |
||||||
|
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= |
||||||
|
gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= |
||||||
|
gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= |
||||||
|
gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE= |
||||||
|
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= |
||||||
|
mvdan.cc/xurls/v2 v2.0.0 h1:r1zSOSNS/kqtpmATyMMMvaZ4/djsesbYz5kr0+qMRWc= |
||||||
|
mvdan.cc/xurls/v2 v2.0.0/go.mod h1:2/webFPYOXN9jp/lzuj0zuAVlF+9g4KPFJANH1oJhRU= |
||||||
|
strk.kbt.io/projects/go/libravatar v0.0.0-20160628055650-5eed7bff870a h1:8q33ShxKXRwQ7JVd1ZnhIU3hZhwwn0Le+4fTeAackuM= |
||||||
|
strk.kbt.io/projects/go/libravatar v0.0.0-20160628055650-5eed7bff870a/go.mod h1:FJGmPh3vz9jSos1L/F91iAgnC/aejc0wIIrF2ZwJxdY= |
||||||
@ -0,0 +1,5 @@ |
|||||||
|
TAGS |
||||||
|
tags |
||||||
|
.*.swp |
||||||
|
tomlcheck/tomlcheck |
||||||
|
toml.test |
||||||
@ -0,0 +1,15 @@ |
|||||||
|
language: go |
||||||
|
go: |
||||||
|
- 1.1 |
||||||
|
- 1.2 |
||||||
|
- 1.3 |
||||||
|
- 1.4 |
||||||
|
- 1.5 |
||||||
|
- 1.6 |
||||||
|
- tip |
||||||
|
install: |
||||||
|
- go install ./... |
||||||
|
- go get github.com/BurntSushi/toml-test |
||||||
|
script: |
||||||
|
- export PATH="$PATH:$HOME/gopath/bin" |
||||||
|
- make test |
||||||
@ -0,0 +1,3 @@ |
|||||||
|
Compatible with TOML version |
||||||
|
[v0.4.0](https://github.com/toml-lang/toml/blob/v0.4.0/versions/en/toml-v0.4.0.md) |
||||||
|
|
||||||
@ -0,0 +1,19 @@ |
|||||||
|
install: |
||||||
|
go install ./...
|
||||||
|
|
||||||
|
test: install |
||||||
|
go test -v
|
||||||
|
toml-test toml-test-decoder
|
||||||
|
toml-test -encoder toml-test-encoder
|
||||||
|
|
||||||
|
fmt: |
||||||
|
gofmt -w *.go */*.go
|
||||||
|
colcheck *.go */*.go
|
||||||
|
|
||||||
|
tags: |
||||||
|
find ./ -name '*.go' -print0 | xargs -0 gotags > TAGS
|
||||||
|
|
||||||
|
push: |
||||||
|
git push origin master
|
||||||
|
git push github master
|
||||||
|
|
||||||
@ -0,0 +1,218 @@ |
|||||||
|
## TOML parser and encoder for Go with reflection |
||||||
|
|
||||||
|
TOML stands for Tom's Obvious, Minimal Language. This Go package provides a |
||||||
|
reflection interface similar to Go's standard library `json` and `xml` |
||||||
|
packages. This package also supports the `encoding.TextUnmarshaler` and |
||||||
|
`encoding.TextMarshaler` interfaces so that you can define custom data |
||||||
|
representations. (There is an example of this below.) |
||||||
|
|
||||||
|
Spec: https://github.com/toml-lang/toml |
||||||
|
|
||||||
|
Compatible with TOML version |
||||||
|
[v0.4.0](https://github.com/toml-lang/toml/blob/master/versions/en/toml-v0.4.0.md) |
||||||
|
|
||||||
|
Documentation: https://godoc.org/github.com/BurntSushi/toml |
||||||
|
|
||||||
|
Installation: |
||||||
|
|
||||||
|
```bash |
||||||
|
go get github.com/BurntSushi/toml |
||||||
|
``` |
||||||
|
|
||||||
|
Try the toml validator: |
||||||
|
|
||||||
|
```bash |
||||||
|
go get github.com/BurntSushi/toml/cmd/tomlv |
||||||
|
tomlv some-toml-file.toml |
||||||
|
``` |
||||||
|
|
||||||
|
[](https://travis-ci.org/BurntSushi/toml) [](https://godoc.org/github.com/BurntSushi/toml) |
||||||
|
|
||||||
|
### Testing |
||||||
|
|
||||||
|
This package passes all tests in |
||||||
|
[toml-test](https://github.com/BurntSushi/toml-test) for both the decoder |
||||||
|
and the encoder. |
||||||
|
|
||||||
|
### Examples |
||||||
|
|
||||||
|
This package works similarly to how the Go standard library handles `XML` |
||||||
|
and `JSON`. Namely, data is loaded into Go values via reflection. |
||||||
|
|
||||||
|
For the simplest example, consider some TOML file as just a list of keys |
||||||
|
and values: |
||||||
|
|
||||||
|
```toml |
||||||
|
Age = 25 |
||||||
|
Cats = [ "Cauchy", "Plato" ] |
||||||
|
Pi = 3.14 |
||||||
|
Perfection = [ 6, 28, 496, 8128 ] |
||||||
|
DOB = 1987-07-05T05:45:00Z |
||||||
|
``` |
||||||
|
|
||||||
|
Which could be defined in Go as: |
||||||
|
|
||||||
|
```go |
||||||
|
type Config struct { |
||||||
|
Age int |
||||||
|
Cats []string |
||||||
|
Pi float64 |
||||||
|
Perfection []int |
||||||
|
DOB time.Time // requires `import time` |
||||||
|
} |
||||||
|
``` |
||||||
|
|
||||||
|
And then decoded with: |
||||||
|
|
||||||
|
```go |
||||||
|
var conf Config |
||||||
|
if _, err := toml.Decode(tomlData, &conf); err != nil { |
||||||
|
// handle error |
||||||
|
} |
||||||
|
``` |
||||||
|
|
||||||
|
You can also use struct tags if your struct field name doesn't map to a TOML |
||||||
|
key value directly: |
||||||
|
|
||||||
|
```toml |
||||||
|
some_key_NAME = "wat" |
||||||
|
``` |
||||||
|
|
||||||
|
```go |
||||||
|
type TOML struct { |
||||||
|
ObscureKey string `toml:"some_key_NAME"` |
||||||
|
} |
||||||
|
``` |
||||||
|
|
||||||
|
### Using the `encoding.TextUnmarshaler` interface |
||||||
|
|
||||||
|
Here's an example that automatically parses duration strings into |
||||||
|
`time.Duration` values: |
||||||
|
|
||||||
|
```toml |
||||||
|
[[song]] |
||||||
|
name = "Thunder Road" |
||||||
|
duration = "4m49s" |
||||||
|
|
||||||
|
[[song]] |
||||||
|
name = "Stairway to Heaven" |
||||||
|
duration = "8m03s" |
||||||
|
``` |
||||||
|
|
||||||
|
Which can be decoded with: |
||||||
|
|
||||||
|
```go |
||||||
|
type song struct { |
||||||
|
Name string |
||||||
|
Duration duration |
||||||
|
} |
||||||
|
type songs struct { |
||||||
|
Song []song |
||||||
|
} |
||||||
|
var favorites songs |
||||||
|
if _, err := toml.Decode(blob, &favorites); err != nil { |
||||||
|
log.Fatal(err) |
||||||
|
} |
||||||
|
|
||||||
|
for _, s := range favorites.Song { |
||||||
|
fmt.Printf("%s (%s)\n", s.Name, s.Duration) |
||||||
|
} |
||||||
|
``` |
||||||
|
|
||||||
|
And you'll also need a `duration` type that satisfies the |
||||||
|
`encoding.TextUnmarshaler` interface: |
||||||
|
|
||||||
|
```go |
||||||
|
type duration struct { |
||||||
|
time.Duration |
||||||
|
} |
||||||
|
|
||||||
|
func (d *duration) UnmarshalText(text []byte) error { |
||||||
|
var err error |
||||||
|
d.Duration, err = time.ParseDuration(string(text)) |
||||||
|
return err |
||||||
|
} |
||||||
|
``` |
||||||
|
|
||||||
|
### More complex usage |
||||||
|
|
||||||
|
Here's an example of how to load the example from the official spec page: |
||||||
|
|
||||||
|
```toml |
||||||
|
# This is a TOML document. Boom. |
||||||
|
|
||||||
|
title = "TOML Example" |
||||||
|
|
||||||
|
[owner] |
||||||
|
name = "Tom Preston-Werner" |
||||||
|
organization = "GitHub" |
||||||
|
bio = "GitHub Cofounder & CEO\nLikes tater tots and beer." |
||||||
|
dob = 1979-05-27T07:32:00Z # First class dates? Why not? |
||||||
|
|
||||||
|
[database] |
||||||
|
server = "192.168.1.1" |
||||||
|
ports = [ 8001, 8001, 8002 ] |
||||||
|
connection_max = 5000 |
||||||
|
enabled = true |
||||||
|
|
||||||
|
[servers] |
||||||
|
|
||||||
|
# You can indent as you please. Tabs or spaces. TOML don't care. |
||||||
|
[servers.alpha] |
||||||
|
ip = "10.0.0.1" |
||||||
|
dc = "eqdc10" |
||||||
|
|
||||||
|
[servers.beta] |
||||||
|
ip = "10.0.0.2" |
||||||
|
dc = "eqdc10" |
||||||
|
|
||||||
|
[clients] |
||||||
|
data = [ ["gamma", "delta"], [1, 2] ] # just an update to make sure parsers support it |
||||||
|
|
||||||
|
# Line breaks are OK when inside arrays |
||||||
|
hosts = [ |
||||||
|
"alpha", |
||||||
|
"omega" |
||||||
|
] |
||||||
|
``` |
||||||
|
|
||||||
|
And the corresponding Go types are: |
||||||
|
|
||||||
|
```go |
||||||
|
type tomlConfig struct { |
||||||
|
Title string |
||||||
|
Owner ownerInfo |
||||||
|
DB database `toml:"database"` |
||||||
|
Servers map[string]server |
||||||
|
Clients clients |
||||||
|
} |
||||||
|
|
||||||
|
type ownerInfo struct { |
||||||
|
Name string |
||||||
|
Org string `toml:"organization"` |
||||||
|
Bio string |
||||||
|
DOB time.Time |
||||||
|
} |
||||||
|
|
||||||
|
type database struct { |
||||||
|
Server string |
||||||
|
Ports []int |
||||||
|
ConnMax int `toml:"connection_max"` |
||||||
|
Enabled bool |
||||||
|
} |
||||||
|
|
||||||
|
type server struct { |
||||||
|
IP string |
||||||
|
DC string |
||||||
|
} |
||||||
|
|
||||||
|
type clients struct { |
||||||
|
Data [][]interface{} |
||||||
|
Hosts []string |
||||||
|
} |
||||||
|
``` |
||||||
|
|
||||||
|
Note that a case insensitive match will be tried if an exact match can't be |
||||||
|
found. |
||||||
|
|
||||||
|
A working example of the above can be found in `_examples/example.{go,toml}`. |
||||||
@ -1,21 +0,0 @@ |
|||||||
The MIT License (MIT) |
|
||||||
|
|
||||||
Copyright (c) 2013 TOML authors |
|
||||||
|
|
||||||
Permission is hereby granted, free of charge, to any person obtaining a copy |
|
||||||
of this software and associated documentation files (the "Software"), to deal |
|
||||||
in the Software without restriction, including without limitation the rights |
|
||||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell |
|
||||||
copies of the Software, and to permit persons to whom the Software is |
|
||||||
furnished to do so, subject to the following conditions: |
|
||||||
|
|
||||||
The above copyright notice and this permission notice shall be included in |
|
||||||
all copies or substantial portions of the Software. |
|
||||||
|
|
||||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
|
||||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
|
||||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE |
|
||||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
|
||||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, |
|
||||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN |
|
||||||
THE SOFTWARE. |
|
||||||
@ -1,21 +0,0 @@ |
|||||||
The MIT License (MIT) |
|
||||||
|
|
||||||
Copyright (c) 2013 TOML authors |
|
||||||
|
|
||||||
Permission is hereby granted, free of charge, to any person obtaining a copy |
|
||||||
of this software and associated documentation files (the "Software"), to deal |
|
||||||
in the Software without restriction, including without limitation the rights |
|
||||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell |
|
||||||
copies of the Software, and to permit persons to whom the Software is |
|
||||||
furnished to do so, subject to the following conditions: |
|
||||||
|
|
||||||
The above copyright notice and this permission notice shall be included in |
|
||||||
all copies or substantial portions of the Software. |
|
||||||
|
|
||||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
|
||||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
|
||||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE |
|
||||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
|
||||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, |
|
||||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN |
|
||||||
THE SOFTWARE. |
|
||||||
@ -1,21 +0,0 @@ |
|||||||
The MIT License (MIT) |
|
||||||
|
|
||||||
Copyright (c) 2013 TOML authors |
|
||||||
|
|
||||||
Permission is hereby granted, free of charge, to any person obtaining a copy |
|
||||||
of this software and associated documentation files (the "Software"), to deal |
|
||||||
in the Software without restriction, including without limitation the rights |
|
||||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell |
|
||||||
copies of the Software, and to permit persons to whom the Software is |
|
||||||
furnished to do so, subject to the following conditions: |
|
||||||
|
|
||||||
The above copyright notice and this permission notice shall be included in |
|
||||||
all copies or substantial portions of the Software. |
|
||||||
|
|
||||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
|
||||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
|
||||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE |
|
||||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
|
||||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, |
|
||||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN |
|
||||||
THE SOFTWARE. |
|
||||||
@ -0,0 +1 @@ |
|||||||
|
au BufWritePost *.go silent!make tags > /dev/null 2>&1 |
||||||
@ -0,0 +1 @@ |
|||||||
|
testdata/* linguist-vendored |
||||||
@ -0,0 +1,16 @@ |
|||||||
|
# editor temporary files |
||||||
|
*.sublime-* |
||||||
|
.DS_Store |
||||||
|
*.swp |
||||||
|
#*.*# |
||||||
|
tags |
||||||
|
|
||||||
|
# direnv config |
||||||
|
.env* |
||||||
|
|
||||||
|
# test binaries |
||||||
|
*.test |
||||||
|
|
||||||
|
# coverage and profilte outputs |
||||||
|
*.out |
||||||
|
|
||||||
@ -0,0 +1,12 @@ |
|||||||
|
language: go |
||||||
|
|
||||||
|
go: |
||||||
|
- 1.1.x |
||||||
|
- 1.2.x |
||||||
|
- 1.3.x |
||||||
|
- 1.4.x |
||||||
|
- 1.5.x |
||||||
|
- 1.6.x |
||||||
|
- 1.7.x |
||||||
|
- 1.8.x |
||||||
|
- tip |
||||||
@ -0,0 +1,126 @@ |
|||||||
|
# goquery - a little like that j-thing, only in Go |
||||||
|
[](http://travis-ci.org/PuerkitoBio/goquery) [](http://godoc.org/github.com/PuerkitoBio/goquery) [](https://sourcegraph.com/github.com/PuerkitoBio/goquery?badge) |
||||||
|
|
||||||
|
|
||||||
|
goquery brings a syntax and a set of features similar to [jQuery][] to the [Go language][go]. It is based on Go's [net/html package][html] and the CSS Selector library [cascadia][]. Since the net/html parser returns nodes, and not a full-featured DOM tree, jQuery's stateful manipulation functions (like height(), css(), detach()) have been left off. |
||||||
|
|
||||||
|
Also, because the net/html parser requires UTF-8 encoding, so does goquery: it is the caller's responsibility to ensure that the source document provides UTF-8 encoded HTML. See the [wiki][] for various options to do this. |
||||||
|
|
||||||
|
Syntax-wise, it is as close as possible to jQuery, with the same function names when possible, and that warm and fuzzy chainable interface. jQuery being the ultra-popular library that it is, I felt that writing a similar HTML-manipulating library was better to follow its API than to start anew (in the same spirit as Go's `fmt` package), even though some of its methods are less than intuitive (looking at you, [index()][index]...). |
||||||
|
|
||||||
|
## Installation |
||||||
|
|
||||||
|
Please note that because of the net/html dependency, goquery requires Go1.1+. |
||||||
|
|
||||||
|
$ go get github.com/PuerkitoBio/goquery |
||||||
|
|
||||||
|
(optional) To run unit tests: |
||||||
|
|
||||||
|
$ cd $GOPATH/src/github.com/PuerkitoBio/goquery |
||||||
|
$ go test |
||||||
|
|
||||||
|
(optional) To run benchmarks (warning: it runs for a few minutes): |
||||||
|
|
||||||
|
$ cd $GOPATH/src/github.com/PuerkitoBio/goquery |
||||||
|
$ go test -bench=".*" |
||||||
|
|
||||||
|
## Changelog |
||||||
|
|
||||||
|
**Note that goquery's API is now stable, and will not break.** |
||||||
|
|
||||||
|
* **2017-02-12 (v1.1.0)** : Add `SetHtml` and `SetText` (thanks to @glebtv). |
||||||
|
* **2016-12-29 (v1.0.2)** : Optimize allocations for `Selection.Text` (thanks to @radovskyb). |
||||||
|
* **2016-08-28 (v1.0.1)** : Optimize performance for large documents. |
||||||
|
* **2016-07-27 (v1.0.0)** : Tag version 1.0.0. |
||||||
|
* **2016-06-15** : Invalid selector strings internally compile to a `Matcher` implementation that never matches any node (instead of a panic). So for example, `doc.Find("~")` returns an empty `*Selection` object. |
||||||
|
* **2016-02-02** : Add `NodeName` utility function similar to the DOM's `nodeName` property. It returns the tag name of the first element in a selection, and other relevant values of non-element nodes (see godoc for details). Add `OuterHtml` utility function similar to the DOM's `outerHTML` property (named `OuterHtml` in small caps for consistency with the existing `Html` method on the `Selection`). |
||||||
|
* **2015-04-20** : Add `AttrOr` helper method to return the attribute's value or a default value if absent. Thanks to [piotrkowalczuk][piotr]. |
||||||
|
* **2015-02-04** : Add more manipulation functions - Prepend* - thanks again to [Andrew Stone][thatguystone]. |
||||||
|
* **2014-11-28** : Add more manipulation functions - ReplaceWith*, Wrap* and Unwrap - thanks again to [Andrew Stone][thatguystone]. |
||||||
|
* **2014-11-07** : Add manipulation functions (thanks to [Andrew Stone][thatguystone]) and `*Matcher` functions, that receive compiled cascadia selectors instead of selector strings, thus avoiding potential panics thrown by goquery via `cascadia.MustCompile` calls. This results in better performance (selectors can be compiled once and reused) and more idiomatic error handling (you can handle cascadia's compilation errors, instead of recovering from panics, which had been bugging me for a long time). Note that the actual type expected is a `Matcher` interface, that `cascadia.Selector` implements. Other matcher implementations could be used. |
||||||
|
* **2014-11-06** : Change import paths of net/html to golang.org/x/net/html (see https://groups.google.com/forum/#!topic/golang-nuts/eD8dh3T9yyA). Make sure to update your code to use the new import path too when you call goquery with `html.Node`s. |
||||||
|
* **v0.3.2** : Add `NewDocumentFromReader()` (thanks jweir) which allows creating a goquery document from an io.Reader. |
||||||
|
* **v0.3.1** : Add `NewDocumentFromResponse()` (thanks assassingj) which allows creating a goquery document from an http response. |
||||||
|
* **v0.3.0** : Add `EachWithBreak()` which allows to break out of an `Each()` loop by returning false. This function was added instead of changing the existing `Each()` to avoid breaking compatibility. |
||||||
|
* **v0.2.1** : Make go-getable, now that [go.net/html is Go1.0-compatible][gonet] (thanks to @matrixik for pointing this out). |
||||||
|
* **v0.2.0** : Add support for negative indices in Slice(). **BREAKING CHANGE** `Document.Root` is removed, `Document` is now a `Selection` itself (a selection of one, the root element, just like `Document.Root` was before). Add jQuery's Closest() method. |
||||||
|
* **v0.1.1** : Add benchmarks to use as baseline for refactorings, refactor Next...() and Prev...() methods to use the new html package's linked list features (Next/PrevSibling, FirstChild). Good performance boost (40+% in some cases). |
||||||
|
* **v0.1.0** : Initial release. |
||||||
|
|
||||||
|
## API |
||||||
|
|
||||||
|
goquery exposes two structs, `Document` and `Selection`, and the `Matcher` interface. Unlike jQuery, which is loaded as part of a DOM document, and thus acts on its containing document, goquery doesn't know which HTML document to act upon. So it needs to be told, and that's what the `Document` type is for. It holds the root document node as the initial Selection value to manipulate. |
||||||
|
|
||||||
|
jQuery often has many variants for the same function (no argument, a selector string argument, a jQuery object argument, a DOM element argument, ...). Instead of exposing the same features in goquery as a single method with variadic empty interface arguments, statically-typed signatures are used following this naming convention: |
||||||
|
|
||||||
|
* When the jQuery equivalent can be called with no argument, it has the same name as jQuery for the no argument signature (e.g.: `Prev()`), and the version with a selector string argument is called `XxxFiltered()` (e.g.: `PrevFiltered()`) |
||||||
|
* When the jQuery equivalent **requires** one argument, the same name as jQuery is used for the selector string version (e.g.: `Is()`) |
||||||
|
* The signatures accepting a jQuery object as argument are defined in goquery as `XxxSelection()` and take a `*Selection` object as argument (e.g.: `FilterSelection()`) |
||||||
|
* The signatures accepting a DOM element as argument in jQuery are defined in goquery as `XxxNodes()` and take a variadic argument of type `*html.Node` (e.g.: `FilterNodes()`) |
||||||
|
* The signatures accepting a function as argument in jQuery are defined in goquery as `XxxFunction()` and take a function as argument (e.g.: `FilterFunction()`) |
||||||
|
* The goquery methods that can be called with a selector string have a corresponding version that take a `Matcher` interface and are defined as `XxxMatcher()` (e.g.: `IsMatcher()`) |
||||||
|
|
||||||
|
Utility functions that are not in jQuery but are useful in Go are implemented as functions (that take a `*Selection` as parameter), to avoid a potential naming clash on the `*Selection`'s methods (reserved for jQuery-equivalent behaviour). |
||||||
|
|
||||||
|
The complete [godoc reference documentation can be found here][doc]. |
||||||
|
|
||||||
|
Please note that Cascadia's selectors do not necessarily match all supported selectors of jQuery (Sizzle). See the [cascadia project][cascadia] for details. Invalid selector strings compile to a `Matcher` that fails to match any node. Behaviour of the various functions that take a selector string as argument follows from that fact, e.g. (where `~` is an invalid selector string): |
||||||
|
|
||||||
|
* `Find("~")` returns an empty selection because the selector string doesn't match anything. |
||||||
|
* `Add("~")` returns a new selection that holds the same nodes as the original selection, because it didn't add any node (selector string didn't match anything). |
||||||
|
* `ParentsFiltered("~")` returns an empty selection because the selector string doesn't match anything. |
||||||
|
* `ParentsUntil("~")` returns all parents of the selection because the selector string didn't match any element to stop before the top element. |
||||||
|
|
||||||
|
## Examples |
||||||
|
|
||||||
|
See some tips and tricks in the [wiki][]. |
||||||
|
|
||||||
|
Adapted from example_test.go: |
||||||
|
|
||||||
|
```Go |
||||||
|
package main |
||||||
|
|
||||||
|
import ( |
||||||
|
"fmt" |
||||||
|
"log" |
||||||
|
|
||||||
|
"github.com/PuerkitoBio/goquery" |
||||||
|
) |
||||||
|
|
||||||
|
func ExampleScrape() { |
||||||
|
doc, err := goquery.NewDocument("http://metalsucks.net") |
||||||
|
if err != nil { |
||||||
|
log.Fatal(err) |
||||||
|
} |
||||||
|
|
||||||
|
// Find the review items |
||||||
|
doc.Find(".sidebar-reviews article .content-block").Each(func(i int, s *goquery.Selection) { |
||||||
|
// For each item found, get the band and title |
||||||
|
band := s.Find("a").Text() |
||||||
|
title := s.Find("i").Text() |
||||||
|
fmt.Printf("Review %d: %s - %s\n", i, band, title) |
||||||
|
}) |
||||||
|
} |
||||||
|
|
||||||
|
func main() { |
||||||
|
ExampleScrape() |
||||||
|
} |
||||||
|
``` |
||||||
|
|
||||||
|
## License |
||||||
|
|
||||||
|
The [BSD 3-Clause license][bsd], the same as the [Go language][golic]. Cascadia's license is [here][caslic]. |
||||||
|
|
||||||
|
[jquery]: http://jquery.com/ |
||||||
|
[go]: http://golang.org/ |
||||||
|
[cascadia]: https://github.com/andybalholm/cascadia |
||||||
|
[bsd]: http://opensource.org/licenses/BSD-3-Clause |
||||||
|
[golic]: http://golang.org/LICENSE |
||||||
|
[caslic]: https://github.com/andybalholm/cascadia/blob/master/LICENSE |
||||||
|
[doc]: http://godoc.org/github.com/PuerkitoBio/goquery |
||||||
|
[index]: http://api.jquery.com/index/ |
||||||
|
[gonet]: https://github.com/golang/net/ |
||||||
|
[html]: http://godoc.org/golang.org/x/net/html |
||||||
|
[wiki]: https://github.com/PuerkitoBio/goquery/wiki/Tips-and-tricks |
||||||
|
[thatguystone]: https://github.com/thatguystone |
||||||
|
[piotr]: https://github.com/piotrkowalczuk |
||||||
@ -0,0 +1,6 @@ |
|||||||
|
*~ |
||||||
|
roaring-fuzz.zip |
||||||
|
workdir |
||||||
|
coverage.out |
||||||
|
testdata/all3.classic |
||||||
|
testdata/all3.msgp.snappy |
||||||
@ -0,0 +1,30 @@ |
|||||||
|
language: go |
||||||
|
sudo: false |
||||||
|
install: |
||||||
|
- go get -t github.com/RoaringBitmap/roaring |
||||||
|
- go get -t golang.org/x/tools/cmd/cover |
||||||
|
- go get -t github.com/mattn/goveralls |
||||||
|
- go get -t github.com/mschoch/smat |
||||||
|
notifications: |
||||||
|
email: false |
||||||
|
go: |
||||||
|
- 1.7.x |
||||||
|
- 1.8.x |
||||||
|
- 1.9.x |
||||||
|
- 1.10.x |
||||||
|
- tip |
||||||
|
|
||||||
|
# whitelist |
||||||
|
branches: |
||||||
|
only: |
||||||
|
- master |
||||||
|
script: |
||||||
|
- goveralls -v -service travis-ci -ignore arraycontainer_gen.go,bitmapcontainer_gen.go,rle16_gen.go,rle_gen.go,roaringarray_gen.go,rle.go || go test |
||||||
|
- go test -race -run TestConcurrent* |
||||||
|
- GOARCH=arm64 go build |
||||||
|
- GOARCH=386 go build |
||||||
|
- GOARCH=386 go test |
||||||
|
- GOARCH=arm go build |
||||||
|
matrix: |
||||||
|
allow_failures: |
||||||
|
- go: tip |
||||||
@ -0,0 +1,121 @@ |
|||||||
|
.PHONY: help all test format fmtcheck vet lint qa deps clean nuke rle backrle ser fetch-real-roaring-datasets |
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
# Display general help about this command
|
||||||
|
help: |
||||||
|
@echo ""
|
||||||
|
@echo "The following commands are available:"
|
||||||
|
@echo ""
|
||||||
|
@echo " make qa : Run all the tests"
|
||||||
|
@echo " make test : Run the unit tests"
|
||||||
|
@echo ""
|
||||||
|
@echo " make format : Format the source code"
|
||||||
|
@echo " make fmtcheck : Check if the source code has been formatted"
|
||||||
|
@echo " make vet : Check for suspicious constructs"
|
||||||
|
@echo " make lint : Check for style errors"
|
||||||
|
@echo ""
|
||||||
|
@echo " make deps : Get the dependencies"
|
||||||
|
@echo " make clean : Remove any build artifact"
|
||||||
|
@echo " make nuke : Deletes any intermediate file"
|
||||||
|
@echo ""
|
||||||
|
@echo " make fuzz-smat : Fuzzy testing with smat"
|
||||||
|
@echo " make fuzz-stream : Fuzzy testing with stream deserialization"
|
||||||
|
@echo " make fuzz-buffer : Fuzzy testing with buffer deserialization"
|
||||||
|
@echo ""
|
||||||
|
|
||||||
|
# Alias for help target
|
||||||
|
all: help |
||||||
|
test: |
||||||
|
go test
|
||||||
|
go test -race -run TestConcurrent*
|
||||||
|
# Format the source code
|
||||||
|
format: |
||||||
|
@find ./ -type f -name "*.go" -exec gofmt -w {} \;
|
||||||
|
|
||||||
|
# Check if the source code has been formatted
|
||||||
|
fmtcheck: |
||||||
|
@mkdir -p target
|
||||||
|
@find ./ -type f -name "*.go" -exec gofmt -d {} \; | tee target/format.diff
|
||||||
|
@test ! -s target/format.diff || { echo "ERROR: the source code has not been formatted - please use 'make format' or 'gofmt'"; exit 1; }
|
||||||
|
|
||||||
|
# Check for syntax errors
|
||||||
|
vet: |
||||||
|
GOPATH=$(GOPATH) go vet ./...
|
||||||
|
|
||||||
|
# Check for style errors
|
||||||
|
lint: |
||||||
|
GOPATH=$(GOPATH) PATH=$(GOPATH)/bin:$(PATH) golint ./...
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
# Alias to run all quality-assurance checks
|
||||||
|
qa: fmtcheck test vet lint |
||||||
|
|
||||||
|
# --- INSTALL ---
|
||||||
|
|
||||||
|
# Get the dependencies
|
||||||
|
deps: |
||||||
|
GOPATH=$(GOPATH) go get github.com/smartystreets/goconvey/convey
|
||||||
|
GOPATH=$(GOPATH) go get github.com/willf/bitset
|
||||||
|
GOPATH=$(GOPATH) go get github.com/golang/lint/golint
|
||||||
|
GOPATH=$(GOPATH) go get github.com/mschoch/smat
|
||||||
|
GOPATH=$(GOPATH) go get github.com/dvyukov/go-fuzz/go-fuzz
|
||||||
|
GOPATH=$(GOPATH) go get github.com/dvyukov/go-fuzz/go-fuzz-build
|
||||||
|
GOPATH=$(GOPATH) go get github.com/glycerine/go-unsnap-stream
|
||||||
|
GOPATH=$(GOPATH) go get github.com/philhofer/fwd
|
||||||
|
GOPATH=$(GOPATH) go get github.com/jtolds/gls
|
||||||
|
|
||||||
|
fuzz-smat: |
||||||
|
go test -tags=gofuzz -run=TestGenerateSmatCorpus
|
||||||
|
go-fuzz-build -func FuzzSmat github.com/RoaringBitmap/roaring
|
||||||
|
go-fuzz -bin=./roaring-fuzz.zip -workdir=workdir/ -timeout=200
|
||||||
|
|
||||||
|
|
||||||
|
fuzz-stream: |
||||||
|
go-fuzz-build -func FuzzSerializationStream github.com/RoaringBitmap/roaring
|
||||||
|
go-fuzz -bin=./roaring-fuzz.zip -workdir=workdir/ -timeout=200
|
||||||
|
|
||||||
|
|
||||||
|
fuzz-buffer: |
||||||
|
go-fuzz-build -func FuzzSerializationBuffer github.com/RoaringBitmap/roaring
|
||||||
|
go-fuzz -bin=./roaring-fuzz.zip -workdir=workdir/ -timeout=200
|
||||||
|
|
||||||
|
# Remove any build artifact
|
||||||
|
clean: |
||||||
|
GOPATH=$(GOPATH) go clean ./...
|
||||||
|
|
||||||
|
# Deletes any intermediate file
|
||||||
|
nuke: |
||||||
|
rm -rf ./target
|
||||||
|
GOPATH=$(GOPATH) go clean -i ./...
|
||||||
|
|
||||||
|
rle: |
||||||
|
cp rle.go rle16.go
|
||||||
|
perl -pi -e 's/32/16/g' rle16.go
|
||||||
|
cp rle_test.go rle16_test.go
|
||||||
|
perl -pi -e 's/32/16/g' rle16_test.go
|
||||||
|
|
||||||
|
backrle: |
||||||
|
cp rle16.go rle.go
|
||||||
|
perl -pi -e 's/16/32/g' rle.go
|
||||||
|
perl -pi -e 's/2032/2016/g' rle.go
|
||||||
|
|
||||||
|
ser: rle |
||||||
|
go generate
|
||||||
|
|
||||||
|
cover: |
||||||
|
go test -coverprofile=coverage.out
|
||||||
|
go tool cover -html=coverage.out
|
||||||
|
|
||||||
|
fetch-real-roaring-datasets: |
||||||
|
# pull github.com/RoaringBitmap/real-roaring-datasets -> testdata/real-roaring-datasets
|
||||||
|
git submodule init
|
||||||
|
git submodule update
|
||||||
@ -0,0 +1,246 @@ |
|||||||
|
roaring [](https://travis-ci.org/RoaringBitmap/roaring) [](https://coveralls.io/github/RoaringBitmap/roaring?branch=master) [](https://godoc.org/github.com/RoaringBitmap/roaring) [](https://goreportcard.com/report/github.com/RoaringBitmap/roaring) |
||||||
|
============= |
||||||
|
|
||||||
|
This is a go version of the Roaring bitmap data structure. |
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
Roaring bitmaps are used by several major systems such as [Apache Lucene][lucene] and derivative systems such as [Solr][solr] and |
||||||
|
[Elasticsearch][elasticsearch], [Metamarkets' Druid][druid], [LinkedIn Pinot][pinot], [Netflix Atlas][atlas], [Apache Spark][spark], [OpenSearchServer][opensearchserver], [Cloud Torrent][cloudtorrent], [Whoosh][whoosh], [Pilosa][pilosa], [Microsoft Visual Studio Team Services (VSTS)][vsts], and eBay's [Apache Kylin][kylin]. |
||||||
|
|
||||||
|
[lucene]: https://lucene.apache.org/ |
||||||
|
[solr]: https://lucene.apache.org/solr/ |
||||||
|
[elasticsearch]: https://www.elastic.co/products/elasticsearch |
||||||
|
[druid]: http://druid.io/ |
||||||
|
[spark]: https://spark.apache.org/ |
||||||
|
[opensearchserver]: http://www.opensearchserver.com |
||||||
|
[cloudtorrent]: https://github.com/jpillora/cloud-torrent |
||||||
|
[whoosh]: https://bitbucket.org/mchaput/whoosh/wiki/Home |
||||||
|
[pilosa]: https://www.pilosa.com/ |
||||||
|
[kylin]: http://kylin.apache.org/ |
||||||
|
[pinot]: http://github.com/linkedin/pinot/wiki |
||||||
|
[vsts]: https://www.visualstudio.com/team-services/ |
||||||
|
[atlas]: https://github.com/Netflix/atlas |
||||||
|
|
||||||
|
Roaring bitmaps are found to work well in many important applications: |
||||||
|
|
||||||
|
> Use Roaring for bitmap compression whenever possible. Do not use other bitmap compression methods ([Wang et al., SIGMOD 2017](http://db.ucsd.edu/wp-content/uploads/2017/03/sidm338-wangA.pdf)) |
||||||
|
|
||||||
|
|
||||||
|
The ``roaring`` Go library is used by |
||||||
|
* [Cloud Torrent](https://github.com/jpillora/cloud-torrent): a self-hosted remote torrent client |
||||||
|
* [runv](https://github.com/hyperhq/runv): an Hypervisor-based runtime for the Open Containers Initiative |
||||||
|
* [InfluxDB](https://www.influxdata.com) |
||||||
|
* [Pilosa](https://www.pilosa.com/) |
||||||
|
* [Bleve](http://www.blevesearch.com) |
||||||
|
|
||||||
|
This library is used in production in several systems, it is part of the [Awesome Go collection](https://awesome-go.com). |
||||||
|
|
||||||
|
|
||||||
|
There are also [Java](https://github.com/RoaringBitmap/RoaringBitmap) and [C/C++](https://github.com/RoaringBitmap/CRoaring) versions. The Java, C, C++ and Go version are binary compatible: e.g, you can save bitmaps |
||||||
|
from a Java program and load them back in Go, and vice versa. We have a [format specification](https://github.com/RoaringBitmap/RoaringFormatSpec). |
||||||
|
|
||||||
|
|
||||||
|
This code is licensed under Apache License, Version 2.0 (ASL2.0). |
||||||
|
|
||||||
|
Copyright 2016-... by the authors. |
||||||
|
|
||||||
|
|
||||||
|
### References |
||||||
|
|
||||||
|
- Daniel Lemire, Owen Kaser, Nathan Kurz, Luca Deri, Chris O'Hara, François Saint-Jacques, Gregory Ssi-Yan-Kai, Roaring Bitmaps: Implementation of an Optimized Software Library, Software: Practice and Experience 48 (4), 2018 [arXiv:1709.07821](https://arxiv.org/abs/1709.07821) |
||||||
|
- Samy Chambi, Daniel Lemire, Owen Kaser, Robert Godin, |
||||||
|
Better bitmap performance with Roaring bitmaps, |
||||||
|
Software: Practice and Experience 46 (5), 2016. |
||||||
|
http://arxiv.org/abs/1402.6407 This paper used data from http://lemire.me/data/realroaring2014.html |
||||||
|
- Daniel Lemire, Gregory Ssi-Yan-Kai, Owen Kaser, Consistently faster and smaller compressed bitmaps with Roaring, Software: Practice and Experience 46 (11), 2016. http://arxiv.org/abs/1603.06549 |
||||||
|
|
||||||
|
|
||||||
|
### Dependencies |
||||||
|
|
||||||
|
Dependencies are fetched automatically by giving the `-t` flag to `go get`. |
||||||
|
|
||||||
|
they include |
||||||
|
- github.com/smartystreets/goconvey/convey |
||||||
|
- github.com/willf/bitset |
||||||
|
- github.com/mschoch/smat |
||||||
|
- github.com/glycerine/go-unsnap-stream |
||||||
|
- github.com/philhofer/fwd |
||||||
|
- github.com/jtolds/gls |
||||||
|
|
||||||
|
Note that the smat library requires Go 1.6 or better. |
||||||
|
|
||||||
|
#### Installation |
||||||
|
|
||||||
|
- go get -t github.com/RoaringBitmap/roaring |
||||||
|
|
||||||
|
|
||||||
|
### Example |
||||||
|
|
||||||
|
Here is a simplified but complete example: |
||||||
|
|
||||||
|
```go |
||||||
|
package main |
||||||
|
|
||||||
|
import ( |
||||||
|
"fmt" |
||||||
|
"github.com/RoaringBitmap/roaring" |
||||||
|
"bytes" |
||||||
|
) |
||||||
|
|
||||||
|
|
||||||
|
func main() { |
||||||
|
// example inspired by https://github.com/fzandona/goroar |
||||||
|
fmt.Println("==roaring==") |
||||||
|
rb1 := roaring.BitmapOf(1, 2, 3, 4, 5, 100, 1000) |
||||||
|
fmt.Println(rb1.String()) |
||||||
|
|
||||||
|
rb2 := roaring.BitmapOf(3, 4, 1000) |
||||||
|
fmt.Println(rb2.String()) |
||||||
|
|
||||||
|
rb3 := roaring.New() |
||||||
|
fmt.Println(rb3.String()) |
||||||
|
|
||||||
|
fmt.Println("Cardinality: ", rb1.GetCardinality()) |
||||||
|
|
||||||
|
fmt.Println("Contains 3? ", rb1.Contains(3)) |
||||||
|
|
||||||
|
rb1.And(rb2) |
||||||
|
|
||||||
|
rb3.Add(1) |
||||||
|
rb3.Add(5) |
||||||
|
|
||||||
|
rb3.Or(rb1) |
||||||
|
|
||||||
|
// computes union of the three bitmaps in parallel using 4 workers |
||||||
|
roaring.ParOr(4, rb1, rb2, rb3) |
||||||
|
// computes intersection of the three bitmaps in parallel using 4 workers |
||||||
|
roaring.ParAnd(4, rb1, rb2, rb3) |
||||||
|
|
||||||
|
|
||||||
|
// prints 1, 3, 4, 5, 1000 |
||||||
|
i := rb3.Iterator() |
||||||
|
for i.HasNext() { |
||||||
|
fmt.Println(i.Next()) |
||||||
|
} |
||||||
|
fmt.Println() |
||||||
|
|
||||||
|
// next we include an example of serialization |
||||||
|
buf := new(bytes.Buffer) |
||||||
|
rb1.WriteTo(buf) // we omit error handling |
||||||
|
newrb:= roaring.New() |
||||||
|
newrb.ReadFrom(buf) |
||||||
|
if rb1.Equals(newrb) { |
||||||
|
fmt.Println("I wrote the content to a byte stream and read it back.") |
||||||
|
} |
||||||
|
} |
||||||
|
``` |
||||||
|
|
||||||
|
If you wish to use serialization and handle errors, you might want to |
||||||
|
consider the following sample of code: |
||||||
|
|
||||||
|
```go |
||||||
|
rb := BitmapOf(1, 2, 3, 4, 5, 100, 1000) |
||||||
|
buf := new(bytes.Buffer) |
||||||
|
size,err:=rb.WriteTo(buf) |
||||||
|
if err != nil { |
||||||
|
t.Errorf("Failed writing") |
||||||
|
} |
||||||
|
newrb:= New() |
||||||
|
size,err=newrb.ReadFrom(buf) |
||||||
|
if err != nil { |
||||||
|
t.Errorf("Failed reading") |
||||||
|
} |
||||||
|
if ! rb.Equals(newrb) { |
||||||
|
t.Errorf("Cannot retrieve serialized version") |
||||||
|
} |
||||||
|
``` |
||||||
|
|
||||||
|
Given N integers in [0,x), then the serialized size in bytes of |
||||||
|
a Roaring bitmap should never exceed this bound: |
||||||
|
|
||||||
|
`` 8 + 9 * ((long)x+65535)/65536 + 2 * N `` |
||||||
|
|
||||||
|
That is, given a fixed overhead for the universe size (x), Roaring |
||||||
|
bitmaps never use more than 2 bytes per integer. You can call |
||||||
|
``BoundSerializedSizeInBytes`` for a more precise estimate. |
||||||
|
|
||||||
|
|
||||||
|
### Documentation |
||||||
|
|
||||||
|
Current documentation is available at http://godoc.org/github.com/RoaringBitmap/roaring |
||||||
|
|
||||||
|
### Goroutine safety |
||||||
|
|
||||||
|
In general, it should not generally be considered safe to access |
||||||
|
the same bitmaps using different goroutines--they are left |
||||||
|
unsynchronized for performance. Should you want to access |
||||||
|
a Bitmap from more than one goroutine, you should |
||||||
|
provide synchronization. Typically this is done by using channels to pass |
||||||
|
the *Bitmap around (in Go style; so there is only ever one owner), |
||||||
|
or by using `sync.Mutex` to serialize operations on Bitmaps. |
||||||
|
|
||||||
|
### Coverage |
||||||
|
|
||||||
|
We test our software. For a report on our test coverage, see |
||||||
|
|
||||||
|
https://coveralls.io/github/RoaringBitmap/roaring?branch=master |
||||||
|
|
||||||
|
### Benchmark |
||||||
|
|
||||||
|
Type |
||||||
|
|
||||||
|
go test -bench Benchmark -run - |
||||||
|
|
||||||
|
To run benchmarks on [Real Roaring Datasets](https://github.com/RoaringBitmap/real-roaring-datasets) |
||||||
|
run the following: |
||||||
|
|
||||||
|
```sh |
||||||
|
go get github.com/RoaringBitmap/real-roaring-datasets |
||||||
|
BENCH_REAL_DATA=1 go test -bench BenchmarkRealData -run - |
||||||
|
``` |
||||||
|
|
||||||
|
### Iterative use |
||||||
|
|
||||||
|
You can use roaring with gore: |
||||||
|
|
||||||
|
- go get -u github.com/motemen/gore |
||||||
|
- Make sure that ``$GOPATH/bin`` is in your ``$PATH``. |
||||||
|
- go get github/RoaringBitmap/roaring |
||||||
|
|
||||||
|
```go |
||||||
|
$ gore |
||||||
|
gore version 0.2.6 :help for help |
||||||
|
gore> :import github.com/RoaringBitmap/roaring |
||||||
|
gore> x:=roaring.New() |
||||||
|
gore> x.Add(1) |
||||||
|
gore> x.String() |
||||||
|
"{1}" |
||||||
|
``` |
||||||
|
|
||||||
|
|
||||||
|
### Fuzzy testing |
||||||
|
|
||||||
|
You can help us test further the library with fuzzy testing: |
||||||
|
|
||||||
|
go get github.com/dvyukov/go-fuzz/go-fuzz |
||||||
|
go get github.com/dvyukov/go-fuzz/go-fuzz-build |
||||||
|
go test -tags=gofuzz -run=TestGenerateSmatCorpus |
||||||
|
go-fuzz-build github.com/RoaringBitmap/roaring |
||||||
|
go-fuzz -bin=./roaring-fuzz.zip -workdir=workdir/ -timeout=200 |
||||||
|
|
||||||
|
Let it run, and if the # of crashers is > 0, check out the reports in |
||||||
|
the workdir where you should be able to find the panic goroutine stack |
||||||
|
traces. |
||||||
|
|
||||||
|
### Alternative in Go |
||||||
|
|
||||||
|
There is a Go version wrapping the C/C++ implementation https://github.com/RoaringBitmap/gocroaring |
||||||
|
|
||||||
|
For an alternative implementation in Go, see https://github.com/fzandona/goroar |
||||||
|
The two versions were written independently. |
||||||
|
|
||||||
|
|
||||||
|
### Mailing list/discussion group |
||||||
|
|
||||||
|
https://groups.google.com/forum/#!forum/roaring-bitmaps |
||||||
@ -0,0 +1,25 @@ |
|||||||
|
# Compiled Object files, Static and Dynamic libs (Shared Objects) |
||||||
|
*.o |
||||||
|
*.a |
||||||
|
*.so |
||||||
|
|
||||||
|
# Folders |
||||||
|
_obj |
||||||
|
_test |
||||||
|
.idea |
||||||
|
|
||||||
|
# Architecture specific extensions/prefixes |
||||||
|
*.[568vq] |
||||||
|
[568vq].out |
||||||
|
|
||||||
|
*.cgo1.go |
||||||
|
*.cgo2.c |
||||||
|
_cgo_defun.c |
||||||
|
_cgo_gotypes.go |
||||||
|
_cgo_export.* |
||||||
|
|
||||||
|
_testmain.go |
||||||
|
|
||||||
|
cae.iml |
||||||
|
*.exe |
||||||
|
.DS_Store |
||||||
@ -0,0 +1,37 @@ |
|||||||
|
Compression and Archive Extensions |
||||||
|
================================== |
||||||
|
|
||||||
|
[](http://gowalker.org/github.com/Unknwon/cae) |
||||||
|
|
||||||
|
[中文文档](README_ZH.md) |
||||||
|
|
||||||
|
Package cae implements PHP-like Compression and Archive Extensions. |
||||||
|
|
||||||
|
But this package has some modifications depends on Go-style. |
||||||
|
|
||||||
|
Reference: [PHP:Compression and Archive Extensions](http://www.php.net/manual/en/refs.compression.php). |
||||||
|
|
||||||
|
Code Convention: based on [Go Code Convention](https://github.com/Unknwon/go-code-convention). |
||||||
|
|
||||||
|
### Implementations |
||||||
|
|
||||||
|
Package `zip`([Go Walker](http://gowalker.org/github.com/Unknwon/cae/zip)) and `tz`([Go Walker](http://gowalker.org/github.com/Unknwon/cae/tz)) both enable you to transparently read or write ZIP/TAR.GZ compressed archives and the files inside them. |
||||||
|
|
||||||
|
- Features: |
||||||
|
- Add file or directory from everywhere to archive, no one-to-one limitation. |
||||||
|
- Extract part of entries, not all at once. |
||||||
|
- Stream data directly into `io.Writer` without any file system storage. |
||||||
|
|
||||||
|
### Test cases and Coverage |
||||||
|
|
||||||
|
All subpackages use [GoConvey](http://goconvey.co/) to write test cases, and coverage is more than 80 percent. |
||||||
|
|
||||||
|
### Use cases |
||||||
|
|
||||||
|
- [Gogs](https://github.com/gogits/gogs): self hosted Git service in the Go Programming Language. |
||||||
|
- [GoBlog](https://github.com/fuxiaohei/GoBlog): personal blogging application. |
||||||
|
- [GoBuild](https://github.com/shxsun/gobuild/): online Go cross-platform compilation and download service. |
||||||
|
|
||||||
|
## License |
||||||
|
|
||||||
|
This project is under Apache v2 License. See the [LICENSE](LICENSE) file for the full license text. |
||||||
@ -0,0 +1,29 @@ |
|||||||
|
压缩与打包扩展 |
||||||
|
============= |
||||||
|
|
||||||
|
[](http://gowalker.org/github.com/Unknwon/cae) |
||||||
|
|
||||||
|
包 cae 实现了 PHP 风格的压缩与打包扩展。 |
||||||
|
|
||||||
|
但本包依据 Go 语言的风格进行了一些修改。 |
||||||
|
|
||||||
|
引用：[PHP:Compression and Archive Extensions](http://www.php.net/manual/en/refs.compression.php) |
||||||
|
|
||||||
|
编码规范：基于 [Go 编码规范](https://github.com/Unknwon/go-code-convention) |
||||||
|
|
||||||
|
### 实现 |
||||||
|
|
||||||
|
包 `zip`([Go Walker](http://gowalker.org/github.com/Unknwon/cae/zip)) 和 `tz`([Go Walker](http://gowalker.org/github.com/Unknwon/cae/tz)) 都允许你轻易的读取或写入 ZIP/TAR.GZ 压缩档案和其内部文件。 |
||||||
|
|
||||||
|
- 特性： |
||||||
|
- 将任意位置的文件或目录加入档案，没有一对一的操作限制。 |
||||||
|
- 只解压部分文件，而非一次性解压全部。 |
||||||
|
- 将数据以流的形式直接写入 `io.Writer` 而不需经过文件系统的存储。 |
||||||
|
|
||||||
|
### 测试用例与覆盖率 |
||||||
|
|
||||||
|
所有子包均采用 [GoConvey](http://goconvey.co/) 来书写测试用例，覆盖率均超过 80%。 |
||||||
|
|
||||||
|
## 授权许可 |
||||||
|
|
||||||
|
本项目采用 Apache v2 开源授权许可证，完整的授权说明已放置在 [LICENSE](LICENSE) 文件中。 |
||||||
@ -1 +0,0 @@ |
|||||||
test.zip |
|
||||||
@ -1 +0,0 @@ |
|||||||
testdir |
|
||||||
@ -1 +0,0 @@ |
|||||||
test.zip |
|
||||||
@ -1 +0,0 @@ |
|||||||
testdir |
|
||||||
@ -0,0 +1,24 @@ |
|||||||
|
# Compiled Object files, Static and Dynamic libs (Shared Objects) |
||||||
|
*.o |
||||||
|
*.a |
||||||
|
*.so |
||||||
|
|
||||||
|
# Folders |
||||||
|
_obj |
||||||
|
_test |
||||||
|
.idea |
||||||
|
|
||||||
|
# Architecture specific extensions/prefixes |
||||||
|
*.[568vq] |
||||||
|
[568vq].out |
||||||
|
|
||||||
|
*.cgo1.go |
||||||
|
*.cgo2.c |
||||||
|
_cgo_defun.c |
||||||
|
_cgo_gotypes.go |
||||||
|
_cgo_export.* |
||||||
|
|
||||||
|
_testmain.go |
||||||
|
|
||||||
|
*.exe |
||||||
|
*.iml |
||||||
@ -0,0 +1,13 @@ |
|||||||
|
language: go |
||||||
|
|
||||||
|
go: |
||||||
|
- 1.2 |
||||||
|
- 1.3 |
||||||
|
- 1.4 |
||||||
|
- tip |
||||||
|
|
||||||
|
install: go get -v -t |
||||||
|
|
||||||
|
notifications: |
||||||
|
email: |
||||||
|
- u@gogs.io |
||||||
@ -0,0 +1,20 @@ |
|||||||
|
Common Functions |
||||||
|
================ |
||||||
|
|
||||||
|
[](https://travis-ci.org/Unknwon/com) [](http://gowalker.org/github.com/Unknwon/com) |
||||||
|
|
||||||
|
This is an open source project for commonly used functions for the Go programming language. |
||||||
|
|
||||||
|
This package need >= **go 1.2** |
||||||
|
|
||||||
|
Code Convention: based on [Go Code Convention](https://github.com/Unknwon/go-code-convention). |
||||||
|
|
||||||
|
## Contribute |
||||||
|
|
||||||
|
Your contribute is welcome, but you have to check following steps after you added some functions and commit them: |
||||||
|
|
||||||
|
1. Make sure you wrote user-friendly comments for **all functions** . |
||||||
|
2. Make sure you wrote test cases with any possible condition for **all functions** in file `*_test.go`. |
||||||
|
3. Make sure you wrote benchmarks for **all functions** in file `*_test.go`. |
||||||
|
4. Make sure you wrote useful examples for **all functions** in file `example_test.go`. |
||||||
|
5. Make sure you ran `go test` and got **PASS** . |
||||||
@ -0,0 +1,22 @@ |
|||||||
|
# Compiled Object files, Static and Dynamic libs (Shared Objects) |
||||||
|
*.o |
||||||
|
*.a |
||||||
|
*.so |
||||||
|
|
||||||
|
# Folders |
||||||
|
_obj |
||||||
|
_test |
||||||
|
|
||||||
|
# Architecture specific extensions/prefixes |
||||||
|
*.[568vq] |
||||||
|
[568vq].out |
||||||
|
|
||||||
|
*.cgo1.go |
||||||
|
*.cgo2.c |
||||||
|
_cgo_defun.c |
||||||
|
_cgo_gotypes.go |
||||||
|
_cgo_export.* |
||||||
|
|
||||||
|
_testmain.go |
||||||
|
|
||||||
|
*.exe |
||||||
@ -0,0 +1,12 @@ |
|||||||
|
.PHONY: build test bench vet |
||||||
|
|
||||||
|
build: vet bench |
||||||
|
|
||||||
|
test: |
||||||
|
go test -v -cover
|
||||||
|
|
||||||
|
bench: |
||||||
|
go test -v -cover -test.bench=. -test.benchmem
|
||||||
|
|
||||||
|
vet: |
||||||
|
go vet
|
||||||
@ -0,0 +1,136 @@ |
|||||||
|
i18n [](https://godoc.org/github.com/Unknwon/i18n) [](https://sourcegraph.com/github.com/Unknwon/i18n?badge) |
||||||
|
==== |
||||||
|
|
||||||
|
Package i18n is for app Internationalization and Localization. |
||||||
|
|
||||||
|
## Introduction |
||||||
|
|
||||||
|
This package provides multiple-language options to improve user experience. Sites like [Go Walker](http://gowalker.org) and [gogs.io](http://gogs.io) are using this module to implement Chinese and English user interfaces. |
||||||
|
|
||||||
|
You can use following command to install this module: |
||||||
|
|
||||||
|
go get github.com/Unknwon/i18n |
||||||
|
|
||||||
|
## Usage |
||||||
|
|
||||||
|
First of all, you have to import this package: |
||||||
|
|
||||||
|
```go |
||||||
|
import "github.com/Unknwon/i18n" |
||||||
|
``` |
||||||
|
|
||||||
|
The format of locale files is very like INI format configuration file, which is basically key-value pairs. But this module has some improvements. Every language corresponding to a locale file, for example, under `conf/locale` folder of [gogsweb](https://github.com/gogits/gogsweb/tree/master/conf/locale), there are two files called `locale_en-US.ini` and `locale_zh-CN.ini`. |
||||||
|
|
||||||
|
The name and extensions of locale files can be anything, but we strongly recommend you to follow the style of gogsweb. |
||||||
|
|
||||||
|
## Minimal example |
||||||
|
|
||||||
|
Here are two simplest locale file examples: |
||||||
|
|
||||||
|
File `locale_en-US.ini`: |
||||||
|
|
||||||
|
```ini |
||||||
|
hi = hello, %s |
||||||
|
bye = goodbye |
||||||
|
``` |
||||||
|
|
||||||
|
File `locale_zh-CN.ini`: |
||||||
|
|
||||||
|
```ini |
||||||
|
hi = 您好，%s |
||||||
|
bye = 再见 |
||||||
|
``` |
||||||
|
|
||||||
|
### Do Translation |
||||||
|
|
||||||
|
There are two ways to do translation depends on which way is the best fit for your application or framework. |
||||||
|
|
||||||
|
Directly use package function to translate: |
||||||
|
|
||||||
|
```go |
||||||
|
i18n.Tr("en-US", "hi", "Unknwon") |
||||||
|
i18n.Tr("en-US", "bye") |
||||||
|
``` |
||||||
|
|
||||||
|
Or create a struct and embed it: |
||||||
|
|
||||||
|
```go |
||||||
|
type MyController struct{ |
||||||
|
// ...other fields |
||||||
|
i18n.Locale |
||||||
|
} |
||||||
|
|
||||||
|
//... |
||||||
|
|
||||||
|
func ... { |
||||||
|
c := &MyController{ |
||||||
|
Locale: i18n.Locale{"en-US"}, |
||||||
|
} |
||||||
|
_ = c.Tr("hi", "Unknwon") |
||||||
|
_ = c.Tr("bye") |
||||||
|
} |
||||||
|
``` |
||||||
|
|
||||||
|
Code above will produce correspondingly: |
||||||
|
|
||||||
|
- English `en-US`：`hello, Unknwon`, `goodbye` |
||||||
|
- Chinese `zh-CN`：`您好，Unknwon`, `再见` |
||||||
|
|
||||||
|
## Section |
||||||
|
|
||||||
|
For different pages, one key may map to different values. Therefore, i18n module also uses the section feature of INI format configuration to achieve section. |
||||||
|
|
||||||
|
For example, the key name is `about`, and we want to show `About` in the home page and `About Us` in about page. Then you can do following: |
||||||
|
|
||||||
|
Content in locale file: |
||||||
|
|
||||||
|
```ini |
||||||
|
about = About |
||||||
|
|
||||||
|
[about] |
||||||
|
about = About Us |
||||||
|
``` |
||||||
|
|
||||||
|
Get `about` in home page: |
||||||
|
|
||||||
|
```go |
||||||
|
i18n.Tr("en-US", "about") |
||||||
|
``` |
||||||
|
|
||||||
|
Get `about` in about page: |
||||||
|
|
||||||
|
```go |
||||||
|
i18n.Tr("en-US", "about.about") |
||||||
|
``` |
||||||
|
|
||||||
|
### Ambiguity |
||||||
|
|
||||||
|
Because dot `.` is sign of section in both [INI parser](https://github.com/go-ini/ini) and locale files, so when your key name contains `.` will cause ambiguity. At this point, you just need to add one more `.` in front of the key. |
||||||
|
|
||||||
|
For example, the key name is `about.`, then we can use: |
||||||
|
|
||||||
|
```go |
||||||
|
i18n.Tr("en-US", ".about.") |
||||||
|
``` |
||||||
|
|
||||||
|
to get expect result. |
||||||
|
|
||||||
|
## Helper tool |
||||||
|
|
||||||
|
Module i18n provides a command line helper tool beei18n for simplify steps of your development. You can install it as follows: |
||||||
|
|
||||||
|
go get github.com/Unknwon/i18n/ui18n |
||||||
|
|
||||||
|
### Sync locale files |
||||||
|
|
||||||
|
Command `sync` allows you use a exist local file as the template to create or sync other locale files: |
||||||
|
|
||||||
|
ui18n sync srouce_file.ini other1.ini other2.ini |
||||||
|
|
||||||
|
This command can operate 1 or more files in one command. |
||||||
|
|
||||||
|
## More information |
||||||
|
|
||||||
|
- The first locale you load to the module is considered as **default locale**. |
||||||
|
- When matching non-default locale and didn't find the string, i18n will have a second try on default locale. |
||||||
|
- If i18n still cannot find string in the default locale, raw string will be returned. For instance, when the string is `hi` and it does not exist in locale file, simply return `hi` as output. |
||||||
@ -0,0 +1,26 @@ |
|||||||
|
# Compiled Object files, Static and Dynamic libs (Shared Objects) |
||||||
|
*.o |
||||||
|
*.a |
||||||
|
*.so |
||||||
|
|
||||||
|
# Folders |
||||||
|
_obj |
||||||
|
_test |
||||||
|
|
||||||
|
# Architecture specific extensions/prefixes |
||||||
|
*.[568vq] |
||||||
|
[568vq].out |
||||||
|
|
||||||
|
*.cgo1.go |
||||||
|
*.cgo2.c |
||||||
|
_cgo_defun.c |
||||||
|
_cgo_gotypes.go |
||||||
|
_cgo_export.* |
||||||
|
|
||||||
|
_testmain.go |
||||||
|
|
||||||
|
*.exe |
||||||
|
*.test |
||||||
|
*.prof |
||||||
|
paginater.sublime-project |
||||||
|
paginater.sublime-workspace |
||||||
@ -0,0 +1,65 @@ |
|||||||
|
Paginater [](https://drone.io/github.com/Unknwon/paginater/latest) [](http://gocover.io/github.com/Unknwon/paginater) |
||||||
|
========= |
||||||
|
|
||||||
|
Package paginater is a helper module for custom pagination calculation. |
||||||
|
|
||||||
|
## Installation |
||||||
|
|
||||||
|
go get github.com/Unknwon/paginater |
||||||
|
|
||||||
|
## Getting Started |
||||||
|
|
||||||
|
The following code shows an example of how to use paginater: |
||||||
|
|
||||||
|
```go |
||||||
|
package main |
||||||
|
|
||||||
|
import "github.com/Unknwon/paginater" |
||||||
|
|
||||||
|
func main() { |
||||||
|
// Arguments: |
||||||
|
// - Total number of rows |
||||||
|
// - Number of rows in one page |
||||||
|
// - Current page number |
||||||
|
// - Number of page links |
||||||
|
p := paginater.New(45, 10, 3, 3) |
||||||
|
|
||||||
|
// Then use p as a template object named "Page" in "demo.html" |
||||||
|
// ... |
||||||
|
} |
||||||
|
``` |
||||||
|
|
||||||
|
`demo.html` |
||||||
|
|
||||||
|
```html |
||||||
|
{{if not .Page.IsFirst}}[First](1){{end}} |
||||||
|
{{if .Page.HasPrevious}}[Previous]({{.Page.Previous}}){{end}} |
||||||
|
|
||||||
|
{{range .Page.Pages}} |
||||||
|
{{if eq .Num -1}} |
||||||
|
... |
||||||
|
{{else}} |
||||||
|
{{.Num}}{{if .IsCurrent}}(current){{end}} |
||||||
|
{{end}} |
||||||
|
{{end}} |
||||||
|
|
||||||
|
{{if .Page.HasNext}}[Next]({{.Page.Next}}){{end}} |
||||||
|
{{if not .Page.IsLast}}[Last]({{.Page.TotalPages}}){{end}} |
||||||
|
``` |
||||||
|
|
||||||
|
Possible output: |
||||||
|
|
||||||
|
``` |
||||||
|
[First](1) [Previous](2) ... 2 3(current) 4 ... [Next](4) [Last](5) |
||||||
|
``` |
||||||
|
|
||||||
|
As you may guess, if the `Page` value is `-1`, you should print `...` in the HTML as common practice. |
||||||
|
|
||||||
|
## Getting Help |
||||||
|
|
||||||
|
- [API Documentation](https://gowalker.org/github.com/Unknwon/paginater) |
||||||
|
- [File An Issue](https://github.com/Unknwon/paginater/issues/new) |
||||||
|
|
||||||
|
## License |
||||||
|
|
||||||
|
This project is under Apache v2 License. See the [LICENSE](LICENSE) file for the full license text. |
||||||
@ -0,0 +1,14 @@ |
|||||||
|
language: go |
||||||
|
|
||||||
|
go: |
||||||
|
- 1.3 |
||||||
|
- 1.4 |
||||||
|
|
||||||
|
install: |
||||||
|
- go get github.com/andybalholm/cascadia |
||||||
|
|
||||||
|
script: |
||||||
|
- go test -v |
||||||
|
|
||||||
|
notifications: |
||||||
|
email: false |
||||||
@ -0,0 +1,7 @@ |
|||||||
|
# cascadia |
||||||
|
|
||||||
|
[](https://travis-ci.org/andybalholm/cascadia) |
||||||
|
|
||||||
|
The Cascadia package implements CSS selectors for use with the parse trees produced by the html package. |
||||||
|
|
||||||
|
To test CSS selectors without writing Go code, check out [cascadia](https://github.com/suntong/cascadia) the command line tool, a thin wrapper around this package. |
||||||
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,19 @@ |
|||||||
|
#* |
||||||
|
*.sublime-* |
||||||
|
*~ |
||||||
|
.#* |
||||||
|
.project |
||||||
|
.settings |
||||||
|
**/.idea/ |
||||||
|
**/*.iml |
||||||
|
.DS_Store |
||||||
|
query_string.y.go.tmp |
||||||
|
/analysis/token_filters/cld2/cld2-read-only |
||||||
|
/analysis/token_filters/cld2/libcld2_full.a |
||||||
|
/cmd/bleve/bleve |
||||||
|
vendor/** |
||||||
|
!vendor/manifest |
||||||
|
/y.output |
||||||
|
/search/query/y.output |
||||||
|
*.test |
||||||
|
tags |
||||||
@ -0,0 +1,24 @@ |
|||||||
|
sudo: false |
||||||
|
|
||||||
|
language: go |
||||||
|
|
||||||
|
go: |
||||||
|
- "1.9.x" |
||||||
|
- "1.10.x" |
||||||
|
- "1.11.x" |
||||||
|
|
||||||
|
script: |
||||||
|
- go get golang.org/x/tools/cmd/cover |
||||||
|
- go get github.com/mattn/goveralls |
||||||
|
- go get github.com/kisielk/errcheck |
||||||
|
- go get -u github.com/FiloSottile/gvt |
||||||
|
- gvt restore |
||||||
|
- go test -race -v $(go list ./... | grep -v vendor/) |
||||||
|
- go vet $(go list ./... | grep -v vendor/) |
||||||
|
- errcheck -ignorepkg fmt $(go list ./... | grep -v vendor/) |
||||||
|
- docs/project-code-coverage.sh |
||||||
|
- docs/build_children.sh |
||||||
|
|
||||||
|
notifications: |
||||||
|
email: |
||||||
|
- marty.schoch@gmail.com |
||||||
@ -0,0 +1,16 @@ |
|||||||
|
# Contributing to Bleve |
||||||
|
|
||||||
|
We look forward to your contributions, but ask that you first review these guidelines. |
||||||
|
|
||||||
|
### Sign the CLA |
||||||
|
|
||||||
|
As Bleve is a Couchbase project we require contributors accept the [Couchbase Contributor License Agreement](http://review.couchbase.org/static/individual_agreement.html). To sign this agreement log into the Couchbase [code review tool](http://review.couchbase.org/). The Bleve project does not use this code review tool but it is still used to track acceptance of the contributor license agreements. |
||||||
|
|
||||||
|
### Submitting a Pull Request |
||||||
|
|
||||||
|
All types of contributions are welcome, but please keep the following in mind: |
||||||
|
|
||||||
|
- If you're planning a large change, you should really discuss it in a github issue or on the google group first. This helps avoid duplicate effort and spending time on something that may not be merged. |
||||||
|
- Existing tests should continue to pass, new tests for the contribution are nice to have. |
||||||
|
- All code should have gone through `go fmt` |
||||||
|
- All code should pass `go vet` |
||||||
@ -0,0 +1,67 @@ |
|||||||
|
#  bleve |
||||||
|
|
||||||
|
[](https://travis-ci.org/blevesearch/bleve) [](https://coveralls.io/github/blevesearch/bleve?branch=master) [](https://godoc.org/github.com/blevesearch/bleve) |
||||||
|
[](https://gitter.im/blevesearch/bleve?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) |
||||||
|
[](https://codebeat.co/projects/github-com-blevesearch-bleve) |
||||||
|
[](https://goreportcard.com/report/blevesearch/bleve) |
||||||
|
[](https://sourcegraph.com/github.com/blevesearch/bleve?badge) [](https://opensource.org/licenses/Apache-2.0) |
||||||
|
|
||||||
|
modern text indexing in go - [blevesearch.com](http://www.blevesearch.com/) |
||||||
|
|
||||||
|
Try out bleve live by [searching the bleve website](http://www.blevesearch.com/search/?q=bleve). |
||||||
|
|
||||||
|
## Features |
||||||
|
|
||||||
|
* Index any go data structure (including JSON) |
||||||
|
* Intelligent defaults backed up by powerful configuration |
||||||
|
* Supported field types: |
||||||
|
* Text, Numeric, Date |
||||||
|
* Supported query types: |
||||||
|
* Term, Phrase, Match, Match Phrase, Prefix |
||||||
|
* Conjunction, Disjunction, Boolean |
||||||
|
* Numeric Range, Date Range |
||||||
|
* Simple query [syntax](http://www.blevesearch.com/docs/Query-String-Query/) for human entry |
||||||
|
* tf-idf Scoring |
||||||
|
* Search result match highlighting |
||||||
|
* Supports Aggregating Facets: |
||||||
|
* Terms Facet |
||||||
|
* Numeric Range Facet |
||||||
|
* Date Range Facet |
||||||
|
|
||||||
|
## Discussion |
||||||
|
|
||||||
|
Discuss usage and development of bleve in the [google group](https://groups.google.com/forum/#!forum/bleve). |
||||||
|
|
||||||
|
## Indexing |
||||||
|
|
||||||
|
```go |
||||||
|
message := struct{ |
||||||
|
Id string |
||||||
|
From string |
||||||
|
Body string |
||||||
|
}{ |
||||||
|
Id: "example", |
||||||
|
From: "marty.schoch@gmail.com", |
||||||
|
Body: "bleve indexing is easy", |
||||||
|
} |
||||||
|
|
||||||
|
mapping := bleve.NewIndexMapping() |
||||||
|
index, err := bleve.New("example.bleve", mapping) |
||||||
|
if err != nil { |
||||||
|
panic(err) |
||||||
|
} |
||||||
|
index.Index(message.Id, message) |
||||||
|
``` |
||||||
|
|
||||||
|
## Querying |
||||||
|
|
||||||
|
```go |
||||||
|
index, _ := bleve.Open("example.bleve") |
||||||
|
query := bleve.NewQueryStringQuery("bleve") |
||||||
|
searchRequest := bleve.NewSearchRequest(query) |
||||||
|
searchResult, _ := index.Search(searchRequest) |
||||||
|
``` |
||||||
|
|
||||||
|
## License |
||||||
|
|
||||||
|
Apache License Version 2.0 |
||||||
@ -0,0 +1,7 @@ |
|||||||
|
# full line comment |
||||||
|
marty |
||||||
|
steve # trailing comment |
||||||
|
| different format of comment |
||||||
|
dustin |
||||||
|
siri | different style trailing comment |
||||||
|
multiple words with different whitespace |
||||||
@ -0,0 +1,9 @@ |
|||||||
|
# geo support in bleve |
||||||
|
|
||||||
|
First, all of this geo code is a Go adaptation of the [Lucene 5.3.2 sandbox geo support](https://lucene.apache.org/core/5_3_2/sandbox/org/apache/lucene/util/package-summary.html). |
||||||
|
|
||||||
|
## Notes |
||||||
|
|
||||||
|
- All of the APIs will use float64 for lon/lat values. |
||||||
|
- When describing a point in function arguments or return values, we always use the order lon, lat. |
||||||
|
- High level APIs will use TopLeft and BottomRight to describe bounding boxes. This may not map cleanly to min/max lon/lat when crossing the dateline. The lower level APIs will use min/max lon/lat and require the higher-level code to split boxes accordingly. |
||||||
@ -0,0 +1,367 @@ |
|||||||
|
# scorch |
||||||
|
|
||||||
|
## Definitions |
||||||
|
|
||||||
|
Batch |
||||||
|
- A collection of Documents to mutate in the index. |
||||||
|
|
||||||
|
Document |
||||||
|
- Has a unique identifier (arbitrary bytes). |
||||||
|
- Is comprised of a list of fields. |
||||||
|
|
||||||
|
Field |
||||||
|
- Has a name (string). |
||||||
|
- Has a type (text, number, date, geopoint). |
||||||
|
- Has a value (depending on type). |
||||||
|
- Can be indexed, stored, or both. |
||||||
|
- If indexed, can be analyzed. |
||||||
|
-m If indexed, can optionally store term vectors. |
||||||
|
|
||||||
|
## Scope |
||||||
|
|
||||||
|
Scorch *MUST* implement the bleve.index API without requiring any changes to this API. |
||||||
|
|
||||||
|
Scorch *MAY* introduce new interfaces, which can be discovered to allow use of new capabilities not in the current API. |
||||||
|
|
||||||
|
## Implementation |
||||||
|
|
||||||
|
The scorch implementation starts with the concept of a segmented index. |
||||||
|
|
||||||
|
A segment is simply a slice, subset, or portion of the entire index. A segmented index is one which is composed of one or more segments. Although segments are created in a particular order, knowing this ordering is not required to achieve correct semantics when querying. Because there is no ordering, this means that when searching an index, you can (and should) search all the segments concurrently. |
||||||
|
|
||||||
|
### Internal Wrapper |
||||||
|
|
||||||
|
In order to accommodate the existing APIs while also improving the implementation, the scorch implementation includes some wrapper functionality that must be described. |
||||||
|
|
||||||
|
#### \_id field |
||||||
|
|
||||||
|
In scorch, field 0 is prearranged to be named \_id. All documents have a value for this field, which is the documents external identifier. In this version the field *MUST* be both indexed AND stored. The scorch wrapper adds this field, as it will not be present in the Document from the calling bleve code. |
||||||
|
|
||||||
|
NOTE: If a document already contains a field \_id, it will be replaced. If this is problematic, the caller must ensure such a scenario does not happen. |
||||||
|
|
||||||
|
### Proposed Structures |
||||||
|
|
||||||
|
``` |
||||||
|
type Segment interface { |
||||||
|
|
||||||
|
Dictionary(field string) TermDictionary |
||||||
|
|
||||||
|
} |
||||||
|
|
||||||
|
type TermDictionary interface { |
||||||
|
|
||||||
|
PostingsList(term string, excluding PostingsList) PostingsList |
||||||
|
|
||||||
|
} |
||||||
|
|
||||||
|
type PostingsList interface { |
||||||
|
|
||||||
|
Next() Posting |
||||||
|
|
||||||
|
And(other PostingsList) PostingsList |
||||||
|
Or(other PostingsList) PostingsList |
||||||
|
|
||||||
|
} |
||||||
|
|
||||||
|
type Posting interface { |
||||||
|
Number() uint64 |
||||||
|
|
||||||
|
Frequency() uint64 |
||||||
|
Norm() float64 |
||||||
|
|
||||||
|
Locations() Locations |
||||||
|
} |
||||||
|
|
||||||
|
type Locations interface { |
||||||
|
Start() uint64 |
||||||
|
End() uint64 |
||||||
|
Pos() uint64 |
||||||
|
ArrayPositions() ... |
||||||
|
} |
||||||
|
|
||||||
|
type DeletedDocs { |
||||||
|
|
||||||
|
} |
||||||
|
|
||||||
|
type SegmentSnapshot struct { |
||||||
|
segment Segment |
||||||
|
deleted PostingsList |
||||||
|
} |
||||||
|
|
||||||
|
type IndexSnapshot struct { |
||||||
|
segment []SegmentSnapshot |
||||||
|
} |
||||||
|
``` |
||||||
|
**What about errors?** |
||||||
|
**What about memory mgmnt or context?** |
||||||
|
**Postings List separate iterator to separate stateful from stateless** |
||||||
|
### Mutating the Index |
||||||
|
|
||||||
|
The bleve.index API has methods for directly making individual mutations (Update/Delete/SetInternal/DeleteInternal), however for this first implementation, we assume that all of these calls can simply be turned into a Batch of size 1. This may be highly inefficient, but it will be correct. This decision is made based on the fact that Couchbase FTS always uses Batches. |
||||||
|
|
||||||
|
NOTE: As a side-effect of this decision, it should be clear that performance tuning may depend on the batch size, which may in-turn require changes in FTS. |
||||||
|
|
||||||
|
From this point forward, only Batch mutations will be discussed. |
||||||
|
|
||||||
|
Sequence of Operations: |
||||||
|
|
||||||
|
1. For each document in the batch, search through all existing segments. The goal is to build up a per-segment bitset which tells us which documents in that segment are obsoleted by the addition of the new segment we're currently building. NOTE: we're not ready for this change to take effect yet, so rather than this operation mutating anything, they simply return bitsets, which we can apply later. Logically, this is something like: |
||||||
|
|
||||||
|
``` |
||||||
|
foreach segment { |
||||||
|
dict := segment.Dictionary("\_id") |
||||||
|
postings := empty postings list |
||||||
|
foreach docID { |
||||||
|
postings = postings.Or(dict.PostingsList(docID, nil)) |
||||||
|
} |
||||||
|
} |
||||||
|
``` |
||||||
|
|
||||||
|
NOTE: it is illustrated above as nested for loops, but some or all of these could be concurrently. The end result is that for each segment, we have (possibly empty) bitset. |
||||||
|
|
||||||
|
2. Also concurrent with 1, the documents in the batch are analyzed. This analysis proceeds using the existing analyzer pool. |
||||||
|
|
||||||
|
3. (after 2 completes) Analyzed documents are fed into a function which builds a new Segment representing this information. |
||||||
|
|
||||||
|
4. We now have everything we need to update the state of the system to include this new snapshot. |
||||||
|
|
||||||
|
- Acquire a lock |
||||||
|
- Create a new IndexSnapshot |
||||||
|
- For each SegmentSnapshot in the IndexSnapshot, take the deleted PostingsList and OR it with the new postings list for this Segment. Construct a new SegmentSnapshot for the segment using this new deleted PostingsList. Append this SegmentSnapshot to the IndexSnapshot. |
||||||
|
- Create a new SegmentSnapshot wrapping our new segment with nil deleted docs. |
||||||
|
- Append the new SegmentSnapshot to the IndexSnapshot |
||||||
|
- Release the lock |
||||||
|
|
||||||
|
An ASCII art example: |
||||||
|
``` |
||||||
|
0 - Empty Index |
||||||
|
|
||||||
|
No segments |
||||||
|
|
||||||
|
IndexSnapshot |
||||||
|
segments [] |
||||||
|
deleted [] |
||||||
|
|
||||||
|
|
||||||
|
1 - Index Batch [ A B C ] |
||||||
|
|
||||||
|
segment 0 |
||||||
|
numbers [ 1 2 3 ] |
||||||
|
\_id [ A B C ] |
||||||
|
|
||||||
|
IndexSnapshot |
||||||
|
segments [ 0 ] |
||||||
|
deleted [ nil ] |
||||||
|
|
||||||
|
|
||||||
|
2 - Index Batch [ B' ] |
||||||
|
|
||||||
|
segment 0 1 |
||||||
|
numbers [ 1 2 3 ] [ 1 ] |
||||||
|
\_id [ A B C ] [ B ] |
||||||
|
|
||||||
|
Compute bitset segment-0-deleted-by-1: |
||||||
|
[ 0 1 0 ] |
||||||
|
|
||||||
|
OR it with previous (nil) (call it 0-1) |
||||||
|
[ 0 1 0 ] |
||||||
|
|
||||||
|
IndexSnapshot |
||||||
|
segments [ 0 1 ] |
||||||
|
deleted [ 0-1 nil ] |
||||||
|
|
||||||
|
3 - Index Batch [ C' ] |
||||||
|
|
||||||
|
segment 0 1 2 |
||||||
|
numbers [ 1 2 3 ] [ 1 ] [ 1 ] |
||||||
|
\_id [ A B C ] [ B ] [ C ] |
||||||
|
|
||||||
|
Compute bitset segment-0-deleted-by-2: |
||||||
|
[ 0 0 1 ] |
||||||
|
|
||||||
|
OR it with previous ([ 0 1 0 ]) (call it 0-12) |
||||||
|
[ 0 1 1 ] |
||||||
|
|
||||||
|
Compute bitset segment-1-deleted-by-2: |
||||||
|
[ 0 ] |
||||||
|
|
||||||
|
OR it with previous (nil) |
||||||
|
still just nil |
||||||
|
|
||||||
|
|
||||||
|
IndexSnapshot |
||||||
|
segments [ 0 1 2 ] |
||||||
|
deleted [ 0-12 nil nil ] |
||||||
|
``` |
||||||
|
|
||||||
|
**is there opportunity to stop early when doc is found in one segment** |
||||||
|
**also, more efficient way to find bits for long lists of ids?** |
||||||
|
|
||||||
|
### Searching |
||||||
|
|
||||||
|
In the bleve.index API all searching starts by getting an IndexReader, which represents a snapshot of the index at a point in time. |
||||||
|
|
||||||
|
As described in the section above, our index implementation maintains a pointer to the current IndexSnapshot. When a caller gets an IndexReader, they get a copy of this pointer, and can use it as long as they like. The IndexSnapshot contains SegmentSnapshots, which only contain pointers to immutable segments. The deleted posting lists associated with a segment change over time, but the particular deleted posting list in YOUR snapshot is immutable. This gives a stable view of the data. |
||||||
|
|
||||||
|
#### Term Search |
||||||
|
|
||||||
|
Term search is the only searching primitive exposed in today's bleve.index API. This ultimately could limit our ability to take advantage of the indexing improvements, but it also means it will be easier to get a first version of this working. |
||||||
|
|
||||||
|
A term search for term T in field F will look something like this: |
||||||
|
|
||||||
|
``` |
||||||
|
searchResultPostings = empty |
||||||
|
foreach segment { |
||||||
|
dict := segment.Dictionary(F) |
||||||
|
segmentResultPostings = dict.PostingsList(T, segmentSnapshotDeleted) |
||||||
|
// make segmentLocal numbers into global numbers, and flip bits in searchResultPostings |
||||||
|
} |
||||||
|
``` |
||||||
|
|
||||||
|
The searchResultPostings will be a new implementation of the TermFieldReader inteface. |
||||||
|
|
||||||
|
As a reminder this interface is: |
||||||
|
|
||||||
|
``` |
||||||
|
// TermFieldReader is the interface exposing the enumeration of documents |
||||||
|
// containing a given term in a given field. Documents are returned in byte |
||||||
|
// lexicographic order over their identifiers. |
||||||
|
type TermFieldReader interface { |
||||||
|
// Next returns the next document containing the term in this field, or nil |
||||||
|
// when it reaches the end of the enumeration. The preAlloced TermFieldDoc |
||||||
|
// is optional, and when non-nil, will be used instead of allocating memory. |
||||||
|
Next(preAlloced *TermFieldDoc) (*TermFieldDoc, error) |
||||||
|
|
||||||
|
// Advance resets the enumeration at specified document or its immediate |
||||||
|
// follower. |
||||||
|
Advance(ID IndexInternalID, preAlloced *TermFieldDoc) (*TermFieldDoc, error) |
||||||
|
|
||||||
|
// Count returns the number of documents contains the term in this field. |
||||||
|
Count() uint64 |
||||||
|
Close() error |
||||||
|
} |
||||||
|
``` |
||||||
|
|
||||||
|
At first glance this appears problematic, we have no way to return documents in order of their identifiers. But it turns out the wording of this perhaps too strong, or a bit ambiguous. Originally, this referred to the external identifiers, but with the introduction of a distinction between internal/external identifiers, returning them in order of their internal identifiers is also acceptable. **ASIDE**: the reason for this is that most callers just use Next() and literally don't care what the order is, they could be in any order and it would be fine. There is only one search that cares and that is the ConjunctionSearcher, which relies on Next/Advance having very specific semantics. Later in this document we will have a proposal to split into multiple interfaces: |
||||||
|
|
||||||
|
- The weakest interface, only supports Next() no ordering at all. |
||||||
|
- Ordered, supporting Advance() |
||||||
|
- And/Or'able capable of internally efficiently doing these ops with like interfaces (if not capable then can always fall back to external walking) |
||||||
|
|
||||||
|
But, the good news is that we don't even have to do that for our first implementation. As long as the global numbers we use for internal identifiers are consistent within this IndexSnapshot, then Next() will be ordered by ascending document number, and Advance() will still work correctly. |
||||||
|
|
||||||
|
NOTE: there is another place where we rely on the ordering of these hits, and that is in the "\_id" sort order. Previously this was the natural order, and a NOOP for the collector, now it must be implemented by actually sorting on the "\_id" field. We probably should introduce at least a marker interface to detect this. |
||||||
|
|
||||||
|
An ASCII art example: |
||||||
|
|
||||||
|
``` |
||||||
|
Let's start with the IndexSnapshot we ended with earlier: |
||||||
|
|
||||||
|
3 - Index Batch [ C' ] |
||||||
|
|
||||||
|
segment 0 1 2 |
||||||
|
numbers [ 1 2 3 ] [ 1 ] [ 1 ] |
||||||
|
\_id [ A B C ] [ B ] [ C ] |
||||||
|
|
||||||
|
Compute bitset segment-0-deleted-by-2: |
||||||
|
[ 0 0 1 ] |
||||||
|
|
||||||
|
OR it with previous ([ 0 1 0 ]) (call it 0-12) |
||||||
|
[ 0 1 1 ] |
||||||
|
|
||||||
|
Compute bitset segment-1-deleted-by-2: |
||||||
|
[ 0 0 0 ] |
||||||
|
|
||||||
|
OR it with previous (nil) |
||||||
|
still just nil |
||||||
|
|
||||||
|
|
||||||
|
IndexSnapshot |
||||||
|
segments [ 0 1 2 ] |
||||||
|
deleted [ 0-12 nil nil ] |
||||||
|
|
||||||
|
Now let's search for the term 'cat' in the field 'desc' and let's assume that Document C (both versions) would match it. |
||||||
|
|
||||||
|
Concurrently: |
||||||
|
|
||||||
|
- Segment 0 |
||||||
|
- Get Term Dictionary For Field 'desc' |
||||||
|
- From it get Postings List for term 'cat' EXCLUDING 0-12 |
||||||
|
- raw segment matches [ 0 0 1 ] but excluding [ 0 1 1 ] gives [ 0 0 0 ] |
||||||
|
- Segment 1 |
||||||
|
- Get Term Dictionary For Field 'desc' |
||||||
|
- From it get Postings List for term 'cat' excluding nil |
||||||
|
- [ 0 ] |
||||||
|
- Segment 2 |
||||||
|
- Get Term Dictionary For Field 'desc' |
||||||
|
- From it get Postings List for term 'cat' excluding nil |
||||||
|
- [ 1 ] |
||||||
|
|
||||||
|
Map local bitsets into global number space (global meaning cross-segment but still unique to this snapshot) |
||||||
|
|
||||||
|
IndexSnapshot already should have mapping something like: |
||||||
|
0 - Offset 0 |
||||||
|
1 - Offset 3 (because segment 0 had 3 docs) |
||||||
|
2 - Offset 4 (because segment 1 had 1 doc) |
||||||
|
|
||||||
|
This maps to search result bitset: |
||||||
|
|
||||||
|
[ 0 0 0 0 1] |
||||||
|
|
||||||
|
Caller would call Next() and get doc number 5 (assuming 1 based indexing for now) |
||||||
|
|
||||||
|
Caller could then ask to get term locations, stored fields, external doc ID for document number 5. Internally in the IndexSnapshot, we can now convert that back, and realize doc number 5 comes from segment 2, 5-4=1 so we're looking for doc number 1 in segment 2. That happens to be C... |
||||||
|
|
||||||
|
``` |
||||||
|
|
||||||
|
#### Future improvements |
||||||
|
|
||||||
|
In the future, interfaces to detect these non-serially operating TermFieldReaders could expose their own And() and Or() up to the higher level Conjunction/Disjunction searchers. Doing this alone offers some win, but also means there would be greater burden on the Searcher code rewriting logical expressions for maximum performance. |
||||||
|
|
||||||
|
Another related topic is that of peak memory usage. With serially operating TermFieldReaders it was necessary to start them all at the same time and operate in unison. However, with these non-serially operating TermFieldReaders we have the option of doing a few at a time, consolidating them, dispoting the intermediaries, and then doing a few more. For very complex queries with many clauses this could reduce peak memory usage. |
||||||
|
|
||||||
|
|
||||||
|
### Memory Tracking |
||||||
|
|
||||||
|
All segments must be able to produce two statistics, an estimate of their explicit memory usage, and their actual size on disk (if any). For in-memory segments, disk usage could be zero, and the memory usage represents the entire information content. For mmap-based disk segments, the memory could be as low as the size of tracking structure itself (say just a few pointers). |
||||||
|
|
||||||
|
This would allow the implementation to throttle or block incoming mutations when a threshold memory usage has (or would be) exceeded. |
||||||
|
|
||||||
|
### Persistence |
||||||
|
|
||||||
|
Obviously, we want to support (but maybe not require) asynchronous persistence of segments. My expectation is that segments are initially built in memory. At some point they are persisted to disk. This poses some interesting challenges. |
||||||
|
|
||||||
|
At runtime, the state of an index (it's IndexSnapshot) is not only the contents of the segments, but also the bitmasks of deleted documents. These bitmasks indirectly encode an ordering in which the segments were added. The reason is that the bitmasks encode which items have been obsoleted by other (subsequent or more future) segments. In the runtime implementation we compute bitmask deltas and then merge them at the same time we bring the new segment in. One idea is that we could take a similar approach on disk. When we persist a segment, we persist the bitmask deltas of segments known to exist at that time, and eventually these can get merged up into a base segment deleted bitmask. |
||||||
|
|
||||||
|
This also relates to the topic rollback, addressed next... |
||||||
|
|
||||||
|
|
||||||
|
### Rollback |
||||||
|
|
||||||
|
One desirable property in the Couchbase ecosystem is the ability to rollback to some previous (though typically not long ago) state. One idea for keeping this property in this design is to protect some of the most recent segments from merging. Then, if necessary, they could be "undone" to reveal previous states of the system. In these scenarios "undone" has to properly undo the deleted bitmasks on the other segments. Again, the current thinking is that rather than "undo" anything, it could be work that was deferred in the first place, thus making it easier to logically undo. |
||||||
|
|
||||||
|
Another possibly related approach would be to tie this into our existing snapshot mechanism. Perhaps simulating a slow reader (holding onto index snapshots) for some period of time, can be the mechanism to achieve the desired end goal. |
||||||
|
|
||||||
|
|
||||||
|
### Internal Storage |
||||||
|
|
||||||
|
The bleve.index API has support for "internal storage". The ability to store information under a separate name space. |
||||||
|
|
||||||
|
This is not used for high volume storage, so it is tempting to think we could just put a small k/v store alongside the rest of the index. But, the reality is that this storage is used to maintain key information related to the rollback scenario. Because of this, its crucial that ordering and overwriting of key/value pairs correspond with actual segment persistence in the index. Based on this, I believe its important to put the internal key/value pairs inside the segments themselves. But, this also means that they must follow a similar "deleted" bitmask approach to obsolete values in older segments. But, this also seems to substantially increase the complexity of the solution because of the separate name space, it would appear to require its own bitmask. Further keys aren't numeric, which then implies yet another mapping from internal key to number, etc. |
||||||
|
|
||||||
|
More thought is required here. |
||||||
|
|
||||||
|
### Merging |
||||||
|
|
||||||
|
The segmented index approach requires merging to prevent the number of segments from growing too large. |
||||||
|
|
||||||
|
Recent experience with LSMs has taught us that having the correct merge strategy can make a huge difference in the overall performance of the system. In particular, a simple merge strategy which merges segments too aggressively can lead to high write amplification and unnecessarily rendering cached data useless. |
||||||
|
|
||||||
|
A few simple principles have been identified. |
||||||
|
|
||||||
|
- Roughly we merge multiple smaller segments into a single larger one. |
||||||
|
- The larger a segment gets the less likely we should be to ever merge it. |
||||||
|
- Segments with large numbers of deleted/obsoleted items are good candidates as the merge will result in a space savings. |
||||||
|
- Segments with all items deleted/obsoleted can be dropped. |
||||||
|
|
||||||
|
Merging of a segment should be able to proceed even if that segment is held by an ongoing snapshot, it should only delay the removal of it. |
||||||
@ -0,0 +1,158 @@ |
|||||||
|
# zap file format |
||||||
|
|
||||||
|
Advanced ZAP File Format Documentation is [here](zap.md). |
||||||
|
|
||||||
|
The file is written in the reverse order that we typically access data. This helps us write in one pass since later sections of the file require file offsets of things we've already written. |
||||||
|
|
||||||
|
Current usage: |
||||||
|
|
||||||
|
- mmap the entire file |
||||||
|
- crc-32 bytes and version are in fixed position at end of the file |
||||||
|
- reading remainder of footer could be version specific |
||||||
|
- remainder of footer gives us: |
||||||
|
- 3 important offsets (docValue , fields index and stored data index) |
||||||
|
- 2 important values (number of docs and chunk factor) |
||||||
|
- field data is processed once and memoized onto the heap so that we never have to go back to disk for it |
||||||
|
- access to stored data by doc number means first navigating to the stored data index, then accessing a fixed position offset into that slice, which gives us the actual address of the data. the first bytes of that section tell us the size of data so that we know where it ends. |
||||||
|
- access to all other indexed data follows the following pattern: |
||||||
|
- first know the field name -> convert to id |
||||||
|
- next navigate to term dictionary for that field |
||||||
|
- some operations stop here and do dictionary ops |
||||||
|
- next use dictionary to navigate to posting list for a specific term |
||||||
|
- walk posting list |
||||||
|
- if necessary, walk posting details as we go |
||||||
|
- if location info is desired, consult location bitmap to see if it is there |
||||||
|
|
||||||
|
## stored fields section |
||||||
|
|
||||||
|
- for each document |
||||||
|
- preparation phase: |
||||||
|
- produce a slice of metadata bytes and data bytes |
||||||
|
- produce these slices in field id order |
||||||
|
- field value is appended to the data slice |
||||||
|
- metadata slice is varint encoded with the following values for each field value |
||||||
|
- field id (uint16) |
||||||
|
- field type (byte) |
||||||
|
- field value start offset in uncompressed data slice (uint64) |
||||||
|
- field value length (uint64) |
||||||
|
- field number of array positions (uint64) |
||||||
|
- one additional value for each array position (uint64) |
||||||
|
- compress the data slice using snappy |
||||||
|
- file writing phase: |
||||||
|
- remember the start offset for this document |
||||||
|
- write out meta data length (varint uint64) |
||||||
|
- write out compressed data length (varint uint64) |
||||||
|
- write out the metadata bytes |
||||||
|
- write out the compressed data bytes |
||||||
|
|
||||||
|
## stored fields idx |
||||||
|
|
||||||
|
- for each document |
||||||
|
- write start offset (remembered from previous section) of stored data (big endian uint64) |
||||||
|
|
||||||
|
With this index and a known document number, we have direct access to all the stored field data. |
||||||
|
|
||||||
|
## posting details (freq/norm) section |
||||||
|
|
||||||
|
- for each posting list |
||||||
|
- produce a slice containing multiple consecutive chunks (each chunk is varint stream) |
||||||
|
- produce a slice remembering offsets of where each chunk starts |
||||||
|
- preparation phase: |
||||||
|
- for each hit in the posting list |
||||||
|
- if this hit is in next chunk close out encoding of last chunk and record offset start of next |
||||||
|
- encode term frequency (uint64) |
||||||
|
- encode norm factor (float32) |
||||||
|
- file writing phase: |
||||||
|
- remember start position for this posting list details |
||||||
|
- write out number of chunks that follow (varint uint64) |
||||||
|
- write out length of each chunk (each a varint uint64) |
||||||
|
- write out the byte slice containing all the chunk data |
||||||
|
|
||||||
|
If you know the doc number you're interested in, this format lets you jump to the correct chunk (docNum/chunkFactor) directly and then seek within that chunk until you find it. |
||||||
|
|
||||||
|
## posting details (location) section |
||||||
|
|
||||||
|
- for each posting list |
||||||
|
- produce a slice containing multiple consecutive chunks (each chunk is varint stream) |
||||||
|
- produce a slice remembering offsets of where each chunk starts |
||||||
|
- preparation phase: |
||||||
|
- for each hit in the posting list |
||||||
|
- if this hit is in next chunk close out encoding of last chunk and record offset start of next |
||||||
|
- encode field (uint16) |
||||||
|
- encode field pos (uint64) |
||||||
|
- encode field start (uint64) |
||||||
|
- encode field end (uint64) |
||||||
|
- encode number of array positions to follow (uint64) |
||||||
|
- encode each array position (each uint64) |
||||||
|
- file writing phase: |
||||||
|
- remember start position for this posting list details |
||||||
|
- write out number of chunks that follow (varint uint64) |
||||||
|
- write out length of each chunk (each a varint uint64) |
||||||
|
- write out the byte slice containing all the chunk data |
||||||
|
|
||||||
|
If you know the doc number you're interested in, this format lets you jump to the correct chunk (docNum/chunkFactor) directly and then seek within that chunk until you find it. |
||||||
|
|
||||||
|
## postings list section |
||||||
|
|
||||||
|
- for each posting list |
||||||
|
- preparation phase: |
||||||
|
- encode roaring bitmap posting list to bytes (so we know the length) |
||||||
|
- file writing phase: |
||||||
|
- remember the start position for this posting list |
||||||
|
- write freq/norm details offset (remembered from previous, as varint uint64) |
||||||
|
- write location details offset (remembered from previous, as varint uint64) |
||||||
|
- write length of encoded roaring bitmap |
||||||
|
- write the serialized roaring bitmap data |
||||||
|
|
||||||
|
## dictionary |
||||||
|
|
||||||
|
- for each field |
||||||
|
- preparation phase: |
||||||
|
- encode vellum FST with dictionary data pointing to file offset of posting list (remembered from previous) |
||||||
|
- file writing phase: |
||||||
|
- remember the start position of this persistDictionary |
||||||
|
- write length of vellum data (varint uint64) |
||||||
|
- write out vellum data |
||||||
|
|
||||||
|
## fields section |
||||||
|
|
||||||
|
- for each field |
||||||
|
- file writing phase: |
||||||
|
- remember start offset for each field |
||||||
|
- write dictionary address (remembered from previous) (varint uint64) |
||||||
|
- write length of field name (varint uint64) |
||||||
|
- write field name bytes |
||||||
|
|
||||||
|
## fields idx |
||||||
|
|
||||||
|
- for each field |
||||||
|
- file writing phase: |
||||||
|
- write big endian uint64 of start offset for each field |
||||||
|
|
||||||
|
NOTE: currently we don't know or record the length of this fields index. Instead we rely on the fact that we know it immediately precedes a footer of known size. |
||||||
|
|
||||||
|
## fields DocValue |
||||||
|
|
||||||
|
- for each field |
||||||
|
- preparation phase: |
||||||
|
- produce a slice containing multiple consecutive chunks, where each chunk is composed of a meta section followed by compressed columnar field data |
||||||
|
- produce a slice remembering the length of each chunk |
||||||
|
- file writing phase: |
||||||
|
- remember the start position of this first field DocValue offset in the footer |
||||||
|
- write out number of chunks that follow (varint uint64) |
||||||
|
- write out length of each chunk (each a varint uint64) |
||||||
|
- write out the byte slice containing all the chunk data |
||||||
|
|
||||||
|
NOTE: currently the meta header inside each chunk gives clue to the location offsets and size of the data pertaining to a given docID and any |
||||||
|
read operation leverage that meta information to extract the document specific data from the file. |
||||||
|
|
||||||
|
## footer |
||||||
|
|
||||||
|
- file writing phase |
||||||
|
- write number of docs (big endian uint64) |
||||||
|
- write stored field index location (big endian uint64) |
||||||
|
- write field index location (big endian uint64) |
||||||
|
- write field docValue location (big endian uint64) |
||||||
|
- write out chunk factor (big endian uint32) |
||||||
|
- write out version (big endian uint32) |
||||||
|
- write out file CRC of everything preceding this (big endian uint32) |
||||||
@ -0,0 +1,177 @@ |
|||||||
|
# ZAP File Format |
||||||
|
|
||||||
|
## Legend |
||||||
|
|
||||||
|
### Sections |
||||||
|
|
||||||
|
|========| |
||||||
|
| | section |
||||||
|
|========| |
||||||
|
|
||||||
|
### Fixed-size fields |
||||||
|
|
||||||
|
|--------| |----| |--| |-| |
||||||
|
| | uint64 | | uint32 | | uint16 | | uint8 |
||||||
|
|--------| |----| |--| |-| |
||||||
|
|
||||||
|
### Varints |
||||||
|
|
||||||
|
|~~~~~~~~| |
||||||
|
| | varint(up to uint64) |
||||||
|
|~~~~~~~~| |
||||||
|
|
||||||
|
### Arbitrary-length fields |
||||||
|
|
||||||
|
|--------...---| |
||||||
|
| | arbitrary-length field (string, vellum, roaring bitmap) |
||||||
|
|--------...---| |
||||||
|
|
||||||
|
### Chunked data |
||||||
|
|
||||||
|
[--------] |
||||||
|
[ ] |
||||||
|
[--------] |
||||||
|
|
||||||
|
## Overview |
||||||
|
|
||||||
|
Footer section describes the configuration of particular ZAP file. The format of footer is version-dependent, so it is necessary to check `V` field before the parsing. |
||||||
|
|
||||||
|
|==================================================| |
||||||
|
| Stored Fields | |
||||||
|
|==================================================| |
||||||
|
|-----> | Stored Fields Index | |
||||||
|
| |==================================================| |
||||||
|
| | Dictionaries + Postings + DocValues | |
||||||
|
| |==================================================| |
||||||
|
| |---> | DocValues Index | |
||||||
|
| | |==================================================| |
||||||
|
| | | Fields | |
||||||
|
| | |==================================================| |
||||||
|
| | |-> | Fields Index | |
||||||
|
| | | |========|========|========|========|====|====|====| |
||||||
|
| | | | D# | SF | F | FDV | CF | V | CC | (Footer) |
||||||
|
| | | |========|====|===|====|===|====|===|====|====|====| |
||||||
|
| | | | | | |
||||||
|
|-+-+-----------------| | | |
||||||
|
| |--------------------------| | |
||||||
|
|-------------------------------------| |
||||||
|
|
||||||
|
D#. Number of Docs. |
||||||
|
SF. Stored Fields Index Offset. |
||||||
|
F. Field Index Offset. |
||||||
|
FDV. Field DocValue Offset. |
||||||
|
CF. Chunk Factor. |
||||||
|
V. Version. |
||||||
|
CC. CRC32. |
||||||
|
|
||||||
|
## Stored Fields |
||||||
|
|
||||||
|
Stored Fields Index is `D#` consecutive 64-bit unsigned integers - offsets, where relevant Stored Fields Data records are located. |
||||||
|
|
||||||
|
0 [SF] [SF + D# * 8] |
||||||
|
| Stored Fields | Stored Fields Index | |
||||||
|
|================================|==================================| |
||||||
|
| | | |
||||||
|
| |--------------------| ||--------|--------|. . .|--------|| |
||||||
|
| |-> | Stored Fields Data | || 0 | 1 | | D# - 1 || |
||||||
|
| | |--------------------| ||--------|----|---|. . .|--------|| |
||||||
|
| | | | | |
||||||
|
|===|============================|==============|===================| |
||||||
|
| | |
||||||
|
|-------------------------------------------| |
||||||
|
|
||||||
|
Stored Fields Data is an arbitrary size record, which consists of metadata and [Snappy](https://github.com/golang/snappy)-compressed data. |
||||||
|
|
||||||
|
Stored Fields Data |
||||||
|
|~~~~~~~~|~~~~~~~~|~~~~~~~~...~~~~~~~~|~~~~~~~~...~~~~~~~~| |
||||||
|
| MDS | CDS | MD | CD | |
||||||
|
|~~~~~~~~|~~~~~~~~|~~~~~~~~...~~~~~~~~|~~~~~~~~...~~~~~~~~| |
||||||
|
|
||||||
|
MDS. Metadata size. |
||||||
|
CDS. Compressed data size. |
||||||
|
MD. Metadata. |
||||||
|
CD. Snappy-compressed data. |
||||||
|
|
||||||
|
## Fields |
||||||
|
|
||||||
|
Fields Index section located between addresses `F` and `len(file) - len(footer)` and consist of `uint64` values (`F1`, `F2`, ...) which are offsets to records in Fields section. We have `F# = (len(file) - len(footer) - F) / sizeof(uint64)` fields. |
||||||
|
|
||||||
|
|
||||||
|
(...) [F] [F + F#] |
||||||
|
| Fields | Fields Index. | |
||||||
|
|================================|================================| |
||||||
|
| | | |
||||||
|
| |~~~~~~~~|~~~~~~~~|---...---|||--------|--------|...|--------|| |
||||||
|
||->| Dict | Length | Name ||| 0 | 1 | | F# - 1 || |
||||||
|
|| |~~~~~~~~|~~~~~~~~|---...---|||--------|----|---|...|--------|| |
||||||
|
|| | | | |
||||||
|
||===============================|==============|=================| |
||||||
|
| | |
||||||
|
|----------------------------------------------| |
||||||
|
|
||||||
|
|
||||||
|
## Dictionaries + Postings |
||||||
|
|
||||||
|
Each of fields has its own dictionary, encoded in [Vellum](https://github.com/couchbase/vellum) format. Dictionary consists of pairs `(term, offset)`, where `offset` indicates the position of postings (list of documents) for this particular term. |
||||||
|
|
||||||
|
|================================================================|- Dictionaries + |
||||||
|
| | Postings + |
||||||
|
| | DocValues |
||||||
|
| Freq/Norm (chunked) | |
||||||
|
| [~~~~~~|~~~~~~~~~~~~~~~~~~~~~~~~~~~~~] | |
||||||
|
| |->[ Freq | Norm (float32 under varint) ] | |
||||||
|
| | [~~~~~~|~~~~~~~~~~~~~~~~~~~~~~~~~~~~~] | |
||||||
|
| | | |
||||||
|
| |------------------------------------------------------------| | |
||||||
|
| Location Details (chunked) | | |
||||||
|
| [~~~~~~|~~~~~|~~~~~~~|~~~~~|~~~~~~|~~~~~~~~|~~~~~] | | |
||||||
|
| |->[ Size | Pos | Start | End | Arr# | ArrPos | ... ] | | |
||||||
|
| | [~~~~~~|~~~~~|~~~~~~~|~~~~~|~~~~~~|~~~~~~~~|~~~~~] | | |
||||||
|
| | | | |
||||||
|
| |----------------------| | | |
||||||
|
| Postings List | | | |
||||||
|
| |~~~~~~~~|~~~~~|~~|~~~~~~~~|-----------...--| | | |
||||||
|
| |->| F/N | LD | Length | ROARING BITMAP | | | |
||||||
|
| | |~~~~~|~~|~~~~~~~~|~~~~~~~~|-----------...--| | | |
||||||
|
| | |----------------------------------------------| | |
||||||
|
| |--------------------------------------| | |
||||||
|
| Dictionary | | |
||||||
|
| |~~~~~~~~|--------------------------|-...-| | |
||||||
|
| |->| Length | VELLUM DATA : (TERM -> OFFSET) | | |
||||||
|
| | |~~~~~~~~|----------------------------...-| | |
||||||
|
| | | |
||||||
|
|======|=========================================================|- DocValues Index |
||||||
|
| | | |
||||||
|
|======|=========================================================|- Fields |
||||||
|
| | | |
||||||
|
| |~~~~|~~~|~~~~~~~~|---...---| | |
||||||
|
| | Dict | Length | Name | | |
||||||
|
| |~~~~~~~~|~~~~~~~~|---...---| | |
||||||
|
| | |
||||||
|
|================================================================| |
||||||
|
|
||||||
|
## DocValues |
||||||
|
|
||||||
|
DocValues Index is `F#` pairs of varints, one pair per field. Each pair of varints indicates start and end point of DocValues slice. |
||||||
|
|
||||||
|
|================================================================| |
||||||
|
| |------...--| | |
||||||
|
| |->| DocValues |<-| | |
||||||
|
| | |------...--| | | |
||||||
|
|==|=================|===========================================|- DocValues Index |
||||||
|
||~|~~~~~~~~~|~~~~~~~|~~| |~~~~~~~~~~~~~~|~~~~~~~~~~~~|| |
||||||
|
|| DV1 START | DV1 STOP | . . . . . | DV(F#) START | DV(F#) END || |
||||||
|
||~~~~~~~~~~~|~~~~~~~~~~| |~~~~~~~~~~~~~~|~~~~~~~~~~~~|| |
||||||
|
|================================================================| |
||||||
|
|
||||||
|
DocValues is chunked Snappy-compressed values for each document and field. |
||||||
|
|
||||||
|
[~~~~~~~~~~~~~~~|~~~~~~|~~~~~~~~~|-...-|~~~~~~|~~~~~~~~~|--------------------...-] |
||||||
|
[ Doc# in Chunk | Doc1 | Offset1 | ... | DocN | OffsetN | SNAPPY COMPRESSED DATA ] |
||||||
|
[~~~~~~~~~~~~~~~|~~~~~~|~~~~~~~~~|-...-|~~~~~~|~~~~~~~~~|--------------------...-] |
||||||
|
|
||||||
|
Last 16 bytes are description of chunks. |
||||||
|
|
||||||
|
|~~~~~~~~~~~~...~|----------------|----------------| |
||||||
|
| Chunk Sizes | Chunk Size Arr | Chunk# | |
||||||
|
|~~~~~~~~~~~~...~|----------------|----------------| |
||||||
@ -0,0 +1,8 @@ |
|||||||
|
#!/bin/sh |
||||||
|
|
||||||
|
BENCHMARKS=`grep "func Benchmark" *_test.go | sed 's/.*func //' | sed s/\(.*{//` |
||||||
|
|
||||||
|
for BENCHMARK in $BENCHMARKS |
||||||
|
do |
||||||
|
go test -v -run=xxx -bench=^$BENCHMARK$ -benchtime=10s -tags 'forestdb leveldb' | grep -v ok | grep -v PASS |
||||||
|
done |
||||||
@ -0,0 +1,14 @@ |
|||||||
|
message BackIndexTermsEntry { |
||||||
|
required uint32 field = 1; |
||||||
|
repeated string terms = 2; |
||||||
|
} |
||||||
|
|
||||||
|
message BackIndexStoreEntry { |
||||||
|
required uint32 field = 1; |
||||||
|
repeated uint64 arrayPositions = 2; |
||||||
|
} |
||||||
|
|
||||||
|
message BackIndexRowValue { |
||||||
|
repeated BackIndexTermsEntry termsEntries = 1; |
||||||
|
repeated BackIndexStoreEntry storedEntries = 2; |
||||||
|
} |
||||||
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,328 @@ |
|||||||
|
%{ |
||||||
|
package query |
||||||
|
import ( |
||||||
|
"fmt" |
||||||
|
"strconv" |
||||||
|
"strings" |
||||||
|
"time" |
||||||
|
) |
||||||
|
|
||||||
|
func logDebugGrammar(format string, v ...interface{}) { |
||||||
|
if debugParser { |
||||||
|
logger.Printf(format, v...) |
||||||
|
} |
||||||
|
} |
||||||
|
%} |
||||||
|
|
||||||
|
%union { |
||||||
|
s string |
||||||
|
n int |
||||||
|
f float64 |
||||||
|
q Query |
||||||
|
pf *float64} |
||||||
|
|
||||||
|
%token tSTRING tPHRASE tPLUS tMINUS tCOLON tBOOST tNUMBER tSTRING tGREATER tLESS |
||||||
|
tEQUAL tTILDE |
||||||
|
|
||||||
|
%type <s> tSTRING |
||||||
|
%type <s> tPHRASE |
||||||
|
%type <s> tNUMBER |
||||||
|
%type <s> posOrNegNumber |
||||||
|
%type <s> tTILDE |
||||||
|
%type <s> tBOOST |
||||||
|
%type <q> searchBase |
||||||
|
%type <pf> searchSuffix |
||||||
|
%type <n> searchPrefix |
||||||
|
|
||||||
|
%% |
||||||
|
|
||||||
|
input: |
||||||
|
searchParts { |
||||||
|
logDebugGrammar("INPUT") |
||||||
|
}; |
||||||
|
|
||||||
|
searchParts: |
||||||
|
searchPart searchParts { |
||||||
|
logDebugGrammar("SEARCH PARTS") |
||||||
|
} |
||||||
|
| |
||||||
|
searchPart { |
||||||
|
logDebugGrammar("SEARCH PART") |
||||||
|
}; |
||||||
|
|
||||||
|
searchPart: |
||||||
|
searchPrefix searchBase searchSuffix { |
||||||
|
query := $2 |
||||||
|
if $3 != nil { |
||||||
|
if query, ok := query.(BoostableQuery); ok { |
||||||
|
query.SetBoost(*$3) |
||||||
|
} |
||||||
|
} |
||||||
|
switch($1) { |
||||||
|
case queryShould: |
||||||
|
yylex.(*lexerWrapper).query.AddShould(query) |
||||||
|
case queryMust: |
||||||
|
yylex.(*lexerWrapper).query.AddMust(query) |
||||||
|
case queryMustNot: |
||||||
|
yylex.(*lexerWrapper).query.AddMustNot(query) |
||||||
|
} |
||||||
|
}; |
||||||
|
|
||||||
|
|
||||||
|
searchPrefix: |
||||||
|
/* empty */ { |
||||||
|
$$ = queryShould |
||||||
|
} |
||||||
|
| |
||||||
|
tPLUS { |
||||||
|
logDebugGrammar("PLUS") |
||||||
|
$$ = queryMust |
||||||
|
} |
||||||
|
| |
||||||
|
tMINUS { |
||||||
|
logDebugGrammar("MINUS") |
||||||
|
$$ = queryMustNot |
||||||
|
}; |
||||||
|
|
||||||
|
searchBase: |
||||||
|
tSTRING { |
||||||
|
str := $1 |
||||||
|
logDebugGrammar("STRING - %s", str) |
||||||
|
var q FieldableQuery |
||||||
|
if strings.HasPrefix(str, "/") && strings.HasSuffix(str, "/") { |
||||||
|
q = NewRegexpQuery(str[1:len(str)-1]) |
||||||
|
} else if strings.ContainsAny(str, "*?"){ |
||||||
|
q = NewWildcardQuery(str) |
||||||
|
} else { |
||||||
|
q = NewMatchQuery(str) |
||||||
|
} |
||||||
|
$$ = q |
||||||
|
} |
||||||
|
| |
||||||
|
tSTRING tTILDE { |
||||||
|
str := $1 |
||||||
|
fuzziness, err := strconv.ParseFloat($2, 64) |
||||||
|
if err != nil { |
||||||
|
yylex.(*lexerWrapper).lex.Error(fmt.Sprintf("invalid fuzziness value: %v", err)) |
||||||
|
} |
||||||
|
logDebugGrammar("FUZZY STRING - %s %f", str, fuzziness) |
||||||
|
q := NewMatchQuery(str) |
||||||
|
q.SetFuzziness(int(fuzziness)) |
||||||
|
$$ = q |
||||||
|
} |
||||||
|
| |
||||||
|
tSTRING tCOLON tSTRING tTILDE { |
||||||
|
field := $1 |
||||||
|
str := $3 |
||||||
|
fuzziness, err := strconv.ParseFloat($4, 64) |
||||||
|
if err != nil { |
||||||
|
yylex.(*lexerWrapper).lex.Error(fmt.Sprintf("invalid fuzziness value: %v", err)) |
||||||
|
} |
||||||
|
logDebugGrammar("FIELD - %s FUZZY STRING - %s %f", field, str, fuzziness) |
||||||
|
q := NewMatchQuery(str) |
||||||
|
q.SetFuzziness(int(fuzziness)) |
||||||
|
q.SetField(field) |
||||||
|
$$ = q |
||||||
|
} |
||||||
|
| |
||||||
|
tNUMBER { |
||||||
|
str := $1 |
||||||
|
logDebugGrammar("STRING - %s", str) |
||||||
|
q1 := NewMatchQuery(str) |
||||||
|
val, err := strconv.ParseFloat($1, 64) |
||||||
|
if err != nil { |
||||||
|
yylex.(*lexerWrapper).lex.Error(fmt.Sprintf("error parsing number: %v", err)) |
||||||
|
} |
||||||
|
inclusive := true |
||||||
|
q2 := NewNumericRangeInclusiveQuery(&val, &val, &inclusive, &inclusive) |
||||||
|
q := NewDisjunctionQuery([]Query{q1,q2}) |
||||||
|
q.queryStringMode = true |
||||||
|
$$ = q |
||||||
|
} |
||||||
|
| |
||||||
|
tPHRASE { |
||||||
|
phrase := $1 |
||||||
|
logDebugGrammar("PHRASE - %s", phrase) |
||||||
|
q := NewMatchPhraseQuery(phrase) |
||||||
|
$$ = q |
||||||
|
} |
||||||
|
| |
||||||
|
tSTRING tCOLON tSTRING { |
||||||
|
field := $1 |
||||||
|
str := $3 |
||||||
|
logDebugGrammar("FIELD - %s STRING - %s", field, str) |
||||||
|
var q FieldableQuery |
||||||
|
if strings.HasPrefix(str, "/") && strings.HasSuffix(str, "/") { |
||||||
|
q = NewRegexpQuery(str[1:len(str)-1]) |
||||||
|
} else if strings.ContainsAny(str, "*?"){ |
||||||
|
q = NewWildcardQuery(str) |
||||||
|
} else { |
||||||
|
q = NewMatchQuery(str) |
||||||
|
} |
||||||
|
q.SetField(field) |
||||||
|
$$ = q |
||||||
|
} |
||||||
|
| |
||||||
|
tSTRING tCOLON posOrNegNumber { |
||||||
|
field := $1 |
||||||
|
str := $3 |
||||||
|
logDebugGrammar("FIELD - %s STRING - %s", field, str) |
||||||
|
q1 := NewMatchQuery(str) |
||||||
|
q1.SetField(field) |
||||||
|
val, err := strconv.ParseFloat($3, 64) |
||||||
|
if err != nil { |
||||||
|
yylex.(*lexerWrapper).lex.Error(fmt.Sprintf("error parsing number: %v", err)) |
||||||
|
} |
||||||
|
inclusive := true |
||||||
|
q2 := NewNumericRangeInclusiveQuery(&val, &val, &inclusive, &inclusive) |
||||||
|
q2.SetField(field) |
||||||
|
q := NewDisjunctionQuery([]Query{q1,q2}) |
||||||
|
q.queryStringMode = true |
||||||
|
$$ = q |
||||||
|
} |
||||||
|
| |
||||||
|
tSTRING tCOLON tPHRASE { |
||||||
|
field := $1 |
||||||
|
phrase := $3 |
||||||
|
logDebugGrammar("FIELD - %s PHRASE - %s", field, phrase) |
||||||
|
q := NewMatchPhraseQuery(phrase) |
||||||
|
q.SetField(field) |
||||||
|
$$ = q |
||||||
|
} |
||||||
|
| |
||||||
|
tSTRING tCOLON tGREATER posOrNegNumber { |
||||||
|
field := $1 |
||||||
|
min, err := strconv.ParseFloat($4, 64) |
||||||
|
if err != nil { |
||||||
|
yylex.(*lexerWrapper).lex.Error(fmt.Sprintf("error parsing number: %v", err)) |
||||||
|
} |
||||||
|
minInclusive := false |
||||||
|
logDebugGrammar("FIELD - GREATER THAN %f", min) |
||||||
|
q := NewNumericRangeInclusiveQuery(&min, nil, &minInclusive, nil) |
||||||
|
q.SetField(field) |
||||||
|
$$ = q |
||||||
|
} |
||||||
|
| |
||||||
|
tSTRING tCOLON tGREATER tEQUAL posOrNegNumber { |
||||||
|
field := $1 |
||||||
|
min, err := strconv.ParseFloat($5, 64) |
||||||
|
if err != nil { |
||||||
|
yylex.(*lexerWrapper).lex.Error(fmt.Sprintf("error parsing number: %v", err)) |
||||||
|
} |
||||||
|
minInclusive := true |
||||||
|
logDebugGrammar("FIELD - GREATER THAN OR EQUAL %f", min) |
||||||
|
q := NewNumericRangeInclusiveQuery(&min, nil, &minInclusive, nil) |
||||||
|
q.SetField(field) |
||||||
|
$$ = q |
||||||
|
} |
||||||
|
| |
||||||
|
tSTRING tCOLON tLESS posOrNegNumber { |
||||||
|
field := $1 |
||||||
|
max, err := strconv.ParseFloat($4, 64) |
||||||
|
if err != nil { |
||||||
|
yylex.(*lexerWrapper).lex.Error(fmt.Sprintf("error parsing number: %v", err)) |
||||||
|
} |
||||||
|
maxInclusive := false |
||||||
|
logDebugGrammar("FIELD - LESS THAN %f", max) |
||||||
|
q := NewNumericRangeInclusiveQuery(nil, &max, nil, &maxInclusive) |
||||||
|
q.SetField(field) |
||||||
|
$$ = q |
||||||
|
} |
||||||
|
| |
||||||
|
tSTRING tCOLON tLESS tEQUAL posOrNegNumber { |
||||||
|
field := $1 |
||||||
|
max, err := strconv.ParseFloat($5, 64) |
||||||
|
if err != nil { |
||||||
|
yylex.(*lexerWrapper).lex.Error(fmt.Sprintf("error parsing number: %v", err)) |
||||||
|
} |
||||||
|
maxInclusive := true |
||||||
|
logDebugGrammar("FIELD - LESS THAN OR EQUAL %f", max) |
||||||
|
q := NewNumericRangeInclusiveQuery(nil, &max, nil, &maxInclusive) |
||||||
|
q.SetField(field) |
||||||
|
$$ = q |
||||||
|
} |
||||||
|
| |
||||||
|
tSTRING tCOLON tGREATER tPHRASE { |
||||||
|
field := $1 |
||||||
|
minInclusive := false |
||||||
|
phrase := $4 |
||||||
|
|
||||||
|
logDebugGrammar("FIELD - GREATER THAN DATE %s", phrase) |
||||||
|
minTime, err := queryTimeFromString(phrase) |
||||||
|
if err != nil { |
||||||
|
yylex.(*lexerWrapper).lex.Error(fmt.Sprintf("invalid time: %v", err)) |
||||||
|
} |
||||||
|
q := NewDateRangeInclusiveQuery(minTime, time.Time{}, &minInclusive, nil) |
||||||
|
q.SetField(field) |
||||||
|
$$ = q |
||||||
|
} |
||||||
|
| |
||||||
|
tSTRING tCOLON tGREATER tEQUAL tPHRASE { |
||||||
|
field := $1 |
||||||
|
minInclusive := true |
||||||
|
phrase := $5 |
||||||
|
|
||||||
|
logDebugGrammar("FIELD - GREATER THAN OR EQUAL DATE %s", phrase) |
||||||
|
minTime, err := queryTimeFromString(phrase) |
||||||
|
if err != nil { |
||||||
|
yylex.(*lexerWrapper).lex.Error(fmt.Sprintf("invalid time: %v", err)) |
||||||
|
} |
||||||
|
q := NewDateRangeInclusiveQuery(minTime, time.Time{}, &minInclusive, nil) |
||||||
|
q.SetField(field) |
||||||
|
$$ = q |
||||||
|
} |
||||||
|
| |
||||||
|
tSTRING tCOLON tLESS tPHRASE { |
||||||
|
field := $1 |
||||||
|
maxInclusive := false |
||||||
|
phrase := $4 |
||||||
|
|
||||||
|
logDebugGrammar("FIELD - LESS THAN DATE %s", phrase) |
||||||
|
maxTime, err := queryTimeFromString(phrase) |
||||||
|
if err != nil { |
||||||
|
yylex.(*lexerWrapper).lex.Error(fmt.Sprintf("invalid time: %v", err)) |
||||||
|
} |
||||||
|
q := NewDateRangeInclusiveQuery(time.Time{}, maxTime, nil, &maxInclusive) |
||||||
|
q.SetField(field) |
||||||
|
$$ = q |
||||||
|
} |
||||||
|
| |
||||||
|
tSTRING tCOLON tLESS tEQUAL tPHRASE { |
||||||
|
field := $1 |
||||||
|
maxInclusive := true |
||||||
|
phrase := $5 |
||||||
|
|
||||||
|
logDebugGrammar("FIELD - LESS THAN OR EQUAL DATE %s", phrase) |
||||||
|
maxTime, err := queryTimeFromString(phrase) |
||||||
|
if err != nil { |
||||||
|
yylex.(*lexerWrapper).lex.Error(fmt.Sprintf("invalid time: %v", err)) |
||||||
|
} |
||||||
|
q := NewDateRangeInclusiveQuery(time.Time{}, maxTime, nil, &maxInclusive) |
||||||
|
q.SetField(field) |
||||||
|
$$ = q |
||||||
|
}; |
||||||
|
|
||||||
|
searchSuffix: |
||||||
|
/* empty */ { |
||||||
|
$$ = nil |
||||||
|
} |
||||||
|
| |
||||||
|
tBOOST { |
||||||
|
$$ = nil |
||||||
|
boost, err := strconv.ParseFloat($1, 64) |
||||||
|
if err != nil { |
||||||
|
yylex.(*lexerWrapper).lex.Error(fmt.Sprintf("invalid boost value: %v", err)) |
||||||
|
} else { |
||||||
|
$$ = &boost |
||||||
|
} |
||||||
|
logDebugGrammar("BOOST %f", boost) |
||||||
|
}; |
||||||
|
|
||||||
|
posOrNegNumber: |
||||||
|
tNUMBER { |
||||||
|
$$ = $1 |
||||||
|
} |
||||||
|
| |
||||||
|
tMINUS tNUMBER { |
||||||
|
$$ = "-" + $2 |
||||||
|
}; |
||||||
@ -0,0 +1,8 @@ |
|||||||
|
#* |
||||||
|
*.sublime-* |
||||||
|
*~ |
||||||
|
.#* |
||||||
|
.project |
||||||
|
.settings |
||||||
|
.DS_Store |
||||||
|
/testdata |
||||||
@ -0,0 +1,16 @@ |
|||||||
|
language: go |
||||||
|
|
||||||
|
go: |
||||||
|
- 1.4 |
||||||
|
|
||||||
|
script: |
||||||
|
- go get golang.org/x/tools/cmd/vet |
||||||
|
- go get golang.org/x/tools/cmd/cover |
||||||
|
- go get github.com/mattn/goveralls |
||||||
|
- go test -v -covermode=count -coverprofile=profile.out |
||||||
|
- go vet |
||||||
|
- goveralls -service drone.io -coverprofile=profile.out -repotoken $COVERALLS |
||||||
|
|
||||||
|
notifications: |
||||||
|
email: |
||||||
|
- marty.schoch@gmail.com |
||||||
@ -0,0 +1,118 @@ |
|||||||
|
# This fork... |
||||||
|
|
||||||
|
I'm maintaining this fork because the original author was not replying to issues or pull requests. For now I plan on maintaining this fork as necessary. |
||||||
|
|
||||||
|
## Status |
||||||
|
|
||||||
|
[](https://travis-ci.org/blevesearch/go-porterstemmer) |
||||||
|
|
||||||
|
[](https://coveralls.io/r/blevesearch/go-porterstemmer?branch=HEAD) |
||||||
|
|
||||||
|
# Go Porter Stemmer |
||||||
|
|
||||||
|
A native Go clean room implementation of the Porter Stemming Algorithm. |
||||||
|
|
||||||
|
This algorithm is of interest to people doing Machine Learning or |
||||||
|
Natural Language Processing (NLP). |
||||||
|
|
||||||
|
This is NOT a port. This is a native Go implementation from the human-readable |
||||||
|
description of the algorithm. |
||||||
|
|
||||||
|
I've tried to make it (more) efficient by NOT internally using string's, but |
||||||
|
instead internally using []rune's and using the same (array) buffer used by |
||||||
|
the []rune slice (and sub-slices) at all steps of the algorithm. |
||||||
|
|
||||||
|
For Porter Stemmer algorithm, see: |
||||||
|
|
||||||
|
http://tartarus.org/martin/PorterStemmer/def.txt (URL #1) |
||||||
|
|
||||||
|
http://tartarus.org/martin/PorterStemmer/ (URL #2) |
||||||
|
|
||||||
|
# Departures |
||||||
|
|
||||||
|
Also, since when I initially implemented it, it failed the tests at... |
||||||
|
|
||||||
|
http://tartarus.org/martin/PorterStemmer/voc.txt (URL #3) |
||||||
|
|
||||||
|
http://tartarus.org/martin/PorterStemmer/output.txt (URL #4) |
||||||
|
|
||||||
|
... after reading the human-readble text over and over again to try to figure out |
||||||
|
what the error I made was (and doing all sorts of things to debug it) I came to the |
||||||
|
conclusion that the some of these tests were wrong according to the human-readable |
||||||
|
description of the algorithm. |
||||||
|
|
||||||
|
This led me to wonder if maybe other people's code that was passing these tests had |
||||||
|
rules that were not in the human-readable description. Which led me to look at the source |
||||||
|
code here... |
||||||
|
|
||||||
|
http://tartarus.org/martin/PorterStemmer/c.txt (URL #5) |
||||||
|
|
||||||
|
... When I looked there I noticed that there are some items marked as a "DEPARTURE", |
||||||
|
which differ from the original algorithm. (There are 2 of these.) |
||||||
|
|
||||||
|
I implemented these departures, and the tests at URL #3 and URL #4 all passed. |
||||||
|
|
||||||
|
## Usage |
||||||
|
|
||||||
|
To use this Golang library, use with something like: |
||||||
|
|
||||||
|
package main |
||||||
|
|
||||||
|
import ( |
||||||
|
"fmt" |
||||||
|
"github.com/reiver/go-porterstemmer" |
||||||
|
) |
||||||
|
|
||||||
|
func main() { |
||||||
|
|
||||||
|
word := "Waxes" |
||||||
|
|
||||||
|
stem := porterstemmer.StemString(word) |
||||||
|
|
||||||
|
fmt.Printf("The word [%s] has the stem [%s].\n", word, stem) |
||||||
|
} |
||||||
|
|
||||||
|
Alternatively, if you want to be a bit more efficient, use []rune slices instead, with code like: |
||||||
|
|
||||||
|
package main |
||||||
|
|
||||||
|
import ( |
||||||
|
"fmt" |
||||||
|
"github.com/reiver/go-porterstemmer" |
||||||
|
) |
||||||
|
|
||||||
|
func main() { |
||||||
|
|
||||||
|
word := []rune("Waxes") |
||||||
|
|
||||||
|
stem := porterstemmer.Stem(word) |
||||||
|
|
||||||
|
fmt.Printf("The word [%s] has the stem [%s].\n", string(word), string(stem)) |
||||||
|
} |
||||||
|
|
||||||
|
Although NOTE that the above code may modify original slice (named "word" in the example) as a side |
||||||
|
effect, for efficiency reasons. And that the slice named "stem" in the example above may be a |
||||||
|
sub-slice of the slice named "word". |
||||||
|
|
||||||
|
Also alternatively, if you already know that your word is already lowercase (and you don't need |
||||||
|
this library to lowercase your word for you) you can instead use code like: |
||||||
|
|
||||||
|
package main |
||||||
|
|
||||||
|
import ( |
||||||
|
"fmt" |
||||||
|
"github.com/reiver/go-porterstemmer" |
||||||
|
) |
||||||
|
|
||||||
|
func main() { |
||||||
|
|
||||||
|
word := []rune("waxes") |
||||||
|
|
||||||
|
stem := porterstemmer.StemWithoutLowerCasing(word) |
||||||
|
|
||||||
|
fmt.Printf("The word [%s] has the stem [%s].\n", string(word), string(stem)) |
||||||
|
} |
||||||
|
|
||||||
|
Again NOTE (like with the previous example) that the above code may modify original slice (named |
||||||
|
"word" in the example) as a side effect, for efficiency reasons. And that the slice named "stem" |
||||||
|
in the example above may be a sub-slice of the slice named "word". |
||||||
@ -0,0 +1,10 @@ |
|||||||
|
#* |
||||||
|
*.sublime-* |
||||||
|
*~ |
||||||
|
.#* |
||||||
|
.project |
||||||
|
.settings |
||||||
|
.DS_Store |
||||||
|
/maketesttables |
||||||
|
/workdir |
||||||
|
/segment-fuzz.zip |
||||||
@ -0,0 +1,16 @@ |
|||||||
|
language: go |
||||||
|
|
||||||
|
go: |
||||||
|
- 1.4 |
||||||
|
|
||||||
|
script: |
||||||
|
- go get golang.org/x/tools/cmd/vet |
||||||
|
- go get golang.org/x/tools/cmd/cover |
||||||
|
- go get github.com/mattn/goveralls |
||||||
|
- go test -v -covermode=count -coverprofile=profile.out |
||||||
|
- go vet |
||||||
|
- goveralls -service drone.io -coverprofile=profile.out -repotoken $COVERALLS |
||||||
|
|
||||||
|
notifications: |
||||||
|
email: |
||||||
|
- marty.schoch@gmail.com |
||||||
@ -0,0 +1,92 @@ |
|||||||
|
# segment |
||||||
|
|
||||||
|
A Go library for performing Unicode Text Segmentation |
||||||
|
as described in [Unicode Standard Annex #29](http://www.unicode.org/reports/tr29/) |
||||||
|
|
||||||
|
## Features |
||||||
|
|
||||||
|
* Currently only segmentation at Word Boundaries is supported. |
||||||
|
|
||||||
|
## License |
||||||
|
|
||||||
|
Apache License Version 2.0 |
||||||
|
|
||||||
|
## Usage |
||||||
|
|
||||||
|
The functionality is exposed in two ways: |
||||||
|
|
||||||
|
1. You can use a bufio.Scanner with the SplitWords implementation of SplitFunc. |
||||||
|
The SplitWords function will identify the appropriate word boundaries in the input |
||||||
|
text and the Scanner will return tokens at the appropriate place. |
||||||
|
|
||||||
|
scanner := bufio.NewScanner(...) |
||||||
|
scanner.Split(segment.SplitWords) |
||||||
|
for scanner.Scan() { |
||||||
|
tokenBytes := scanner.Bytes() |
||||||
|
} |
||||||
|
if err := scanner.Err(); err != nil { |
||||||
|
t.Fatal(err) |
||||||
|
} |
||||||
|
|
||||||
|
2. Sometimes you would also like information returned about the type of token. |
||||||
|
To do this we have introduce a new type named Segmenter. It works just like Scanner |
||||||
|
but additionally a token type is returned. |
||||||
|
|
||||||
|
segmenter := segment.NewWordSegmenter(...) |
||||||
|
for segmenter.Segment() { |
||||||
|
tokenBytes := segmenter.Bytes()) |
||||||
|
tokenType := segmenter.Type() |
||||||
|
} |
||||||
|
if err := segmenter.Err(); err != nil { |
||||||
|
t.Fatal(err) |
||||||
|
} |
||||||
|
|
||||||
|
## Choosing Implementation |
||||||
|
|
||||||
|
By default segment does NOT use the fastest runtime implementation. The reason is that it adds approximately 5s to compilation time and may require more than 1GB of ram on the machine performing compilation. |
||||||
|
|
||||||
|
However, you can choose to build with the fastest runtime implementation by passing the build tag as follows: |
||||||
|
|
||||||
|
-tags 'prod' |
||||||
|
|
||||||
|
## Generating Code |
||||||
|
|
||||||
|
Several components in this package are generated. |
||||||
|
|
||||||
|
1. Several Ragel rules files are generated from Unicode properties files. |
||||||
|
2. Ragel machine is generated from the Ragel rules. |
||||||
|
3. Test tables are generated from the Unicode test files. |
||||||
|
|
||||||
|
All of these can be generated by running: |
||||||
|
|
||||||
|
go generate |
||||||
|
|
||||||
|
## Fuzzing |
||||||
|
|
||||||
|
There is support for fuzzing the segment library with [go-fuzz](https://github.com/dvyukov/go-fuzz). |
||||||
|
|
||||||
|
1. Install go-fuzz if you haven't already: |
||||||
|
|
||||||
|
go get github.com/dvyukov/go-fuzz/go-fuzz |
||||||
|
go get github.com/dvyukov/go-fuzz/go-fuzz-build |
||||||
|
|
||||||
|
2. Build the package with go-fuzz: |
||||||
|
|
||||||
|
go-fuzz-build github.com/blevesearch/segment |
||||||
|
|
||||||
|
3. Convert the Unicode provided test cases into the initial corpus for go-fuzz: |
||||||
|
|
||||||
|
go test -v -run=TestGenerateWordSegmentFuzz -tags gofuzz_generate |
||||||
|
|
||||||
|
4. Run go-fuzz: |
||||||
|
|
||||||
|
go-fuzz -bin=segment-fuzz.zip -workdir=workdir |
||||||
|
|
||||||
|
## Status |
||||||
|
|
||||||
|
|
||||||
|
[](https://travis-ci.org/blevesearch/segment) |
||||||
|
|
||||||
|
[](https://coveralls.io/r/blevesearch/segment?branch=master) |
||||||
|
|
||||||
|
[](https://godoc.org/github.com/blevesearch/segment) |
||||||
@ -0,0 +1,285 @@ |
|||||||
|
// Copyright (c) 2015 Couchbase, Inc. |
||||||
|
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file |
||||||
|
// except in compliance with the License. You may obtain a copy of the License at |
||||||
|
// http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
// Unless required by applicable law or agreed to in writing, software distributed under the |
||||||
|
// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, |
||||||
|
// either express or implied. See the License for the specific language governing permissions |
||||||
|
// and limitations under the License. |
||||||
|
|
||||||
|
// +build BUILDTAGS |
||||||
|
|
||||||
|
package segment |
||||||
|
|
||||||
|
import ( |
||||||
|
"fmt" |
||||||
|
"unicode/utf8" |
||||||
|
) |
||||||
|
|
||||||
|
var RagelFlags = "RAGELFLAGS" |
||||||
|
|
||||||
|
var ParseError = fmt.Errorf("unicode word segmentation parse error") |
||||||
|
|
||||||
|
// Word Types |
||||||
|
const ( |
||||||
|
None = iota |
||||||
|
Number |
||||||
|
Letter |
||||||
|
Kana |
||||||
|
Ideo |
||||||
|
) |
||||||
|
|
||||||
|
%%{ |
||||||
|
machine s; |
||||||
|
write data; |
||||||
|
}%% |
||||||
|
|
||||||
|
func segmentWords(data []byte, maxTokens int, atEOF bool, val [][]byte, types []int) ([][]byte, []int, int, error) { |
||||||
|
cs, p, pe := 0, 0, len(data) |
||||||
|
cap := maxTokens |
||||||
|
if cap < 0 { |
||||||
|
cap = 1000 |
||||||
|
} |
||||||
|
if val == nil { |
||||||
|
val = make([][]byte, 0, cap) |
||||||
|
} |
||||||
|
if types == nil { |
||||||
|
types = make([]int, 0, cap) |
||||||
|
} |
||||||
|
|
||||||
|
// added for scanner |
||||||
|
ts := 0 |
||||||
|
te := 0 |
||||||
|
act := 0 |
||||||
|
eof := pe |
||||||
|
_ = ts // compiler not happy |
||||||
|
_ = te |
||||||
|
_ = act |
||||||
|
|
||||||
|
// our state |
||||||
|
startPos := 0 |
||||||
|
endPos := 0 |
||||||
|
totalConsumed := 0 |
||||||
|
%%{ |
||||||
|
|
||||||
|
include SCRIPTS "ragel/uscript.rl"; |
||||||
|
include WB "ragel/uwb.rl"; |
||||||
|
|
||||||
|
action startToken { |
||||||
|
startPos = p |
||||||
|
} |
||||||
|
|
||||||
|
action endToken { |
||||||
|
endPos = p |
||||||
|
} |
||||||
|
|
||||||
|
action finishNumericToken { |
||||||
|
if !atEOF { |
||||||
|
return val, types, totalConsumed, nil |
||||||
|
} |
||||||
|
|
||||||
|
val = append(val, data[startPos:endPos+1]) |
||||||
|
types = append(types, Number) |
||||||
|
totalConsumed = endPos+1 |
||||||
|
if maxTokens > 0 && len(val) >= maxTokens { |
||||||
|
return val, types, totalConsumed, nil |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
action finishHangulToken { |
||||||
|
if endPos+1 == pe && !atEOF { |
||||||
|
return val, types, totalConsumed, nil |
||||||
|
} else if dr, size := utf8.DecodeRune(data[endPos+1:]); dr == utf8.RuneError && size == 1 { |
||||||
|
return val, types, totalConsumed, nil |
||||||
|
} |
||||||
|
|
||||||
|
val = append(val, data[startPos:endPos+1]) |
||||||
|
types = append(types, Letter) |
||||||
|
totalConsumed = endPos+1 |
||||||
|
if maxTokens > 0 && len(val) >= maxTokens { |
||||||
|
return val, types, totalConsumed, nil |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
action finishKatakanaToken { |
||||||
|
if endPos+1 == pe && !atEOF { |
||||||
|
return val, types, totalConsumed, nil |
||||||
|
} else if dr, size := utf8.DecodeRune(data[endPos+1:]); dr == utf8.RuneError && size == 1 { |
||||||
|
return val, types, totalConsumed, nil |
||||||
|
} |
||||||
|
|
||||||
|
val = append(val, data[startPos:endPos+1]) |
||||||
|
types = append(types, Ideo) |
||||||
|
totalConsumed = endPos+1 |
||||||
|
if maxTokens > 0 && len(val) >= maxTokens { |
||||||
|
return val, types, totalConsumed, nil |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
action finishWordToken { |
||||||
|
if !atEOF { |
||||||
|
return val, types, totalConsumed, nil |
||||||
|
} |
||||||
|
val = append(val, data[startPos:endPos+1]) |
||||||
|
types = append(types, Letter) |
||||||
|
totalConsumed = endPos+1 |
||||||
|
if maxTokens > 0 && len(val) >= maxTokens { |
||||||
|
return val, types, totalConsumed, nil |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
action finishHanToken { |
||||||
|
if endPos+1 == pe && !atEOF { |
||||||
|
return val, types, totalConsumed, nil |
||||||
|
} else if dr, size := utf8.DecodeRune(data[endPos+1:]); dr == utf8.RuneError && size == 1 { |
||||||
|
return val, types, totalConsumed, nil |
||||||
|
} |
||||||
|
|
||||||
|
val = append(val, data[startPos:endPos+1]) |
||||||
|
types = append(types, Ideo) |
||||||
|
totalConsumed = endPos+1 |
||||||
|
if maxTokens > 0 && len(val) >= maxTokens { |
||||||
|
return val, types, totalConsumed, nil |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
action finishHiraganaToken { |
||||||
|
if endPos+1 == pe && !atEOF { |
||||||
|
return val, types, totalConsumed, nil |
||||||
|
} else if dr, size := utf8.DecodeRune(data[endPos+1:]); dr == utf8.RuneError && size == 1 { |
||||||
|
return val, types, totalConsumed, nil |
||||||
|
} |
||||||
|
|
||||||
|
val = append(val, data[startPos:endPos+1]) |
||||||
|
types = append(types, Ideo) |
||||||
|
totalConsumed = endPos+1 |
||||||
|
if maxTokens > 0 && len(val) >= maxTokens { |
||||||
|
return val, types, totalConsumed, nil |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
action finishNoneToken { |
||||||
|
lastPos := startPos |
||||||
|
for lastPos <= endPos { |
||||||
|
_, size := utf8.DecodeRune(data[lastPos:]) |
||||||
|
lastPos += size |
||||||
|
} |
||||||
|
endPos = lastPos -1 |
||||||
|
p = endPos |
||||||
|
|
||||||
|
if endPos+1 == pe && !atEOF { |
||||||
|
return val, types, totalConsumed, nil |
||||||
|
} else if dr, size := utf8.DecodeRune(data[endPos+1:]); dr == utf8.RuneError && size == 1 { |
||||||
|
return val, types, totalConsumed, nil |
||||||
|
} |
||||||
|
// otherwise, consume this as well |
||||||
|
val = append(val, data[startPos:endPos+1]) |
||||||
|
types = append(types, None) |
||||||
|
totalConsumed = endPos+1 |
||||||
|
if maxTokens > 0 && len(val) >= maxTokens { |
||||||
|
return val, types, totalConsumed, nil |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
HangulEx = Hangul ( Extend | Format )*; |
||||||
|
HebrewOrALetterEx = ( Hebrew_Letter | ALetter ) ( Extend | Format )*; |
||||||
|
NumericEx = Numeric ( Extend | Format )*; |
||||||
|
KatakanaEx = Katakana ( Extend | Format )*; |
||||||
|
MidLetterEx = ( MidLetter | MidNumLet | Single_Quote ) ( Extend | Format )*; |
||||||
|
MidNumericEx = ( MidNum | MidNumLet | Single_Quote ) ( Extend | Format )*; |
||||||
|
ExtendNumLetEx = ExtendNumLet ( Extend | Format )*; |
||||||
|
HanEx = Han ( Extend | Format )*; |
||||||
|
HiraganaEx = Hiragana ( Extend | Format )*; |
||||||
|
SingleQuoteEx = Single_Quote ( Extend | Format )*; |
||||||
|
DoubleQuoteEx = Double_Quote ( Extend | Format )*; |
||||||
|
HebrewLetterEx = Hebrew_Letter ( Extend | Format )*; |
||||||
|
RegionalIndicatorEx = Regional_Indicator ( Extend | Format )*; |
||||||
|
NLCRLF = Newline | CR | LF; |
||||||
|
OtherEx = ^(NLCRLF) ( Extend | Format )* ; |
||||||
|
|
||||||
|
# UAX#29 WB8. Numeric × Numeric |
||||||
|
# WB11. Numeric (MidNum | MidNumLet | Single_Quote) × Numeric |
||||||
|
# WB12. Numeric × (MidNum | MidNumLet | Single_Quote) Numeric |
||||||
|
# WB13a. (ALetter | Hebrew_Letter | Numeric | Katakana | ExtendNumLet) × ExtendNumLet |
||||||
|
# WB13b. ExtendNumLet × (ALetter | Hebrew_Letter | Numeric | Katakana) |
||||||
|
# |
||||||
|
WordNumeric = ( ( ExtendNumLetEx )* NumericEx ( ( ( ExtendNumLetEx )* | MidNumericEx ) NumericEx )* ( ExtendNumLetEx )* ) >startToken @endToken; |
||||||
|
|
||||||
|
# subset of the below for typing purposes only! |
||||||
|
WordHangul = ( HangulEx )+ >startToken @endToken; |
||||||
|
WordKatakana = ( KatakanaEx )+ >startToken @endToken; |
||||||
|
|
||||||
|
# UAX#29 WB5. (ALetter | Hebrew_Letter) × (ALetter | Hebrew_Letter) |
||||||
|
# WB6. (ALetter | Hebrew_Letter) × (MidLetter | MidNumLet | Single_Quote) (ALetter | Hebrew_Letter) |
||||||
|
# WB7. (ALetter | Hebrew_Letter) (MidLetter | MidNumLet | Single_Quote) × (ALetter | Hebrew_Letter) |
||||||
|
# WB7a. Hebrew_Letter × Single_Quote |
||||||
|
# WB7b. Hebrew_Letter × Double_Quote Hebrew_Letter |
||||||
|
# WB7c. Hebrew_Letter Double_Quote × Hebrew_Letter |
||||||
|
# WB9. (ALetter | Hebrew_Letter) × Numeric |
||||||
|
# WB10. Numeric × (ALetter | Hebrew_Letter) |
||||||
|
# WB13. Katakana × Katakana |
||||||
|
# WB13a. (ALetter | Hebrew_Letter | Numeric | Katakana | ExtendNumLet) × ExtendNumLet |
||||||
|
# WB13b. ExtendNumLet × (ALetter | Hebrew_Letter | Numeric | Katakana) |
||||||
|
# |
||||||
|
# Marty -deviated here to allow for (ExtendNumLetEx x ExtendNumLetEx) part of 13a |
||||||
|
# |
||||||
|
Word = ( ( ExtendNumLetEx )* ( KatakanaEx ( ( ExtendNumLetEx )* KatakanaEx )* |
||||||
|
| ( HebrewLetterEx ( SingleQuoteEx | DoubleQuoteEx HebrewLetterEx ) |
||||||
|
| NumericEx ( ( ( ExtendNumLetEx )* | MidNumericEx ) NumericEx )* |
||||||
|
| HebrewOrALetterEx ( ( ( ExtendNumLetEx )* | MidLetterEx ) HebrewOrALetterEx )* |
||||||
|
|ExtendNumLetEx |
||||||
|
)+ |
||||||
|
) |
||||||
|
( |
||||||
|
( ExtendNumLetEx )+ ( KatakanaEx ( ( ExtendNumLetEx )* KatakanaEx )* |
||||||
|
| ( HebrewLetterEx ( SingleQuoteEx | DoubleQuoteEx HebrewLetterEx ) |
||||||
|
| NumericEx ( ( ( ExtendNumLetEx )* | MidNumericEx ) NumericEx )* |
||||||
|
| HebrewOrALetterEx ( ( ( ExtendNumLetEx )* | MidLetterEx ) HebrewOrALetterEx )* |
||||||
|
)+ |
||||||
|
) |
||||||
|
)* ExtendNumLetEx*) >startToken @endToken; |
||||||
|
|
||||||
|
# UAX#29 WB14. Any ÷ Any |
||||||
|
WordHan = HanEx >startToken @endToken; |
||||||
|
WordHiragana = HiraganaEx >startToken @endToken; |
||||||
|
|
||||||
|
WordExt = ( ( Extend | Format )* ) >startToken @endToken; # maybe plus not star |
||||||
|
|
||||||
|
WordCRLF = (CR LF) >startToken @endToken; |
||||||
|
|
||||||
|
WordCR = CR >startToken @endToken; |
||||||
|
|
||||||
|
WordLF = LF >startToken @endToken; |
||||||
|
|
||||||
|
WordNL = Newline >startToken @endToken; |
||||||
|
|
||||||
|
WordRegional = (RegionalIndicatorEx+) >startToken @endToken; |
||||||
|
|
||||||
|
Other = OtherEx >startToken @endToken; |
||||||
|
|
||||||
|
main := |* |
||||||
|
WordNumeric => finishNumericToken; |
||||||
|
WordHangul => finishHangulToken; |
||||||
|
WordKatakana => finishKatakanaToken; |
||||||
|
Word => finishWordToken; |
||||||
|
WordHan => finishHanToken; |
||||||
|
WordHiragana => finishHiraganaToken; |
||||||
|
WordRegional =>finishNoneToken; |
||||||
|
WordCRLF => finishNoneToken; |
||||||
|
WordCR => finishNoneToken; |
||||||
|
WordLF => finishNoneToken; |
||||||
|
WordNL => finishNoneToken; |
||||||
|
WordExt => finishNoneToken; |
||||||
|
Other => finishNoneToken; |
||||||
|
*|; |
||||||
|
|
||||||
|
write init; |
||||||
|
write exec; |
||||||
|
}%% |
||||||
|
|
||||||
|
if cs < s_first_final { |
||||||
|
return val, types, totalConsumed, ParseError |
||||||
|
} |
||||||
|
|
||||||
|
return val, types, totalConsumed, nil |
||||||
|
} |
||||||
@ -0,0 +1,18 @@ |
|||||||
|
##Introduction## |
||||||
|
This is a package for GO which can be used to create different types of barcodes. |
||||||
|
|
||||||
|
##Supported Barcode Types## |
||||||
|
* Aztec Code |
||||||
|
* Codabar |
||||||
|
* Code 128 |
||||||
|
* Code 39 |
||||||
|
* EAN 8 |
||||||
|
* EAN 13 |
||||||
|
* Datamatrix |
||||||
|
* QR Codes |
||||||
|
* 2 of 5 |
||||||
|
|
||||||
|
##Documentation## |
||||||
|
See [GoDoc](https://godoc.org/github.com/boombuler/barcode) |
||||||
|
|
||||||
|
To create a barcode use the Encode function from one of the subpackages. |
||||||
@ -0,0 +1 @@ |
|||||||
|
.DS_Store |
||||||
@ -0,0 +1,12 @@ |
|||||||
|
language: go |
||||||
|
|
||||||
|
go: |
||||||
|
- 1.7 |
||||||
|
|
||||||
|
before_install: |
||||||
|
- go get golang.org/x/tools/cmd/cover |
||||||
|
- go get github.com/mattn/goveralls |
||||||
|
|
||||||
|
script: |
||||||
|
- go test -v -covermode=count -coverprofile=coverage.out |
||||||
|
- $HOME/gopath/bin/goveralls -coverprofile=coverage.out -service=travis-ci |
||||||
@ -0,0 +1,66 @@ |
|||||||
|
#+TITLE: chaseadamsio/goorgeous |
||||||
|
|
||||||
|
[[https://travis-ci.org/chaseadamsio/goorgeous.svg?branch=master]] |
||||||
|
[[https://coveralls.io/repos/github/chaseadamsio/goorgeous/badge.svg?branch=master]] |
||||||
|
|
||||||
|
/goorgeous is a Go Org to HTML Parser./ |
||||||
|
|
||||||
|
[[file:gopher_small.gif]] |
||||||
|
|
||||||
|
*Pronounced: Go? Org? Yes!* |
||||||
|
|
||||||
|
#+BEGIN_QUOTE |
||||||
|
"Org mode is for keeping notes, maintaining TODO lists, planning projects, and authoring documents with a fast and effective plain-text system." |
||||||
|
|
||||||
|
- [[orgmode.org]] |
||||||
|
#+END_QUOTE |
||||||
|
|
||||||
|
The purpose of this package is to come as close as possible as parsing an =*.org= document into HTML, the same way one might publish [[http://orgmode.org/worg/org-tutorials/org-publish-html-tutorial.html][with org-publish-html from Emacs]]. |
||||||
|
|
||||||
|
* Installation |
||||||
|
|
||||||
|
#+BEGIN_SRC sh |
||||||
|
go get -u github.com/chaseadamsio/goorgeous |
||||||
|
#+END_SRC |
||||||
|
|
||||||
|
* Usage |
||||||
|
|
||||||
|
** Org Headers |
||||||
|
|
||||||
|
To retrieve the headers from a =[]byte=, call =OrgHeaders= and it will return a =map[string]interface{}=: |
||||||
|
|
||||||
|
#+BEGIN_SRC go |
||||||
|
input := "#+title: goorgeous\n* Some Headline\n" |
||||||
|
out := goorgeous.OrgHeaders(input) |
||||||
|
#+END_SRC |
||||||
|
|
||||||
|
#+BEGIN_SRC go |
||||||
|
map[string]interface{}{ |
||||||
|
"title": "goorgeous" |
||||||
|
} |
||||||
|
#+END_SRC |
||||||
|
|
||||||
|
** Org Content |
||||||
|
|
||||||
|
After importing =github.com/chaseadamsio/goorgeous=, you can call =Org= with a =[]byte= and it will return an =html= version of the content as a =[]byte= |
||||||
|
|
||||||
|
#+BEGIN_SRC go |
||||||
|
input := "#+TITLE: goorgeous\n* Some Headline\n" |
||||||
|
out := goorgeous.Org(input) |
||||||
|
#+END_SRC |
||||||
|
|
||||||
|
=out= will be: |
||||||
|
|
||||||
|
#+BEGIN_SRC html |
||||||
|
<h1>Some Headline</h1>/n |
||||||
|
#+END_SRC |
||||||
|
|
||||||
|
* Why? |
||||||
|
|
||||||
|
First off, I've become an unapologetic user of Emacs & ever since finding =org-mode= I use it for anything having to do with writing content, organizing my life and keeping documentation of my days/weeks/months. |
||||||
|
|
||||||
|
Although I like Emacs & =emacs-lisp=, I publish all of my html sites with [[https://gohugo.io][Hugo Static Site Generator]] and wanted to be able to write my content in =org-mode= in Emacs rather than markdown. |
||||||
|
|
||||||
|
Hugo's implementation of templating and speed are unmatched, so the only way I knew for sure I could continue to use Hugo and write in =org-mode= seamlessly was to write a golang parser for org content and submit a PR for Hugo to use it. |
||||||
|
* Acknowledgements |
||||||
|
I leaned heavily on russross' [[https://github.com/russross/blackfriday][blackfriday markdown renderer]] as both an example of how to write a parser (with some updates to leverage the go we know today) and reusing the blackfriday HTML Renderer so I didn't have to write my own! |
||||||
|
After Width: | Height: | Size: 15 KiB |
|
After Width: | Height: | Size: 3.2 KiB |
@ -0,0 +1,6 @@ |
|||||||
|
#* |
||||||
|
*.[68] |
||||||
|
*~ |
||||||
|
*.swp |
||||||
|
/gocache/gocache |
||||||
|
c.out |
||||||
@ -0,0 +1,32 @@ |
|||||||
|
# gomemcached |
||||||
|
|
||||||
|
This is a memcached binary protocol toolkit in [go][go]. |
||||||
|
|
||||||
|
It provides client and server functionality as well as a little sample |
||||||
|
server showing how I might make a server if I valued purity over |
||||||
|
performance. |
||||||
|
|
||||||
|
## Server Design |
||||||
|
|
||||||
|
<div> |
||||||
|
<img src="http://dustin.github.com/images/gomemcached.png" |
||||||
|
alt="overview" style="float: right"/> |
||||||
|
</div> |
||||||
|
|
||||||
|
The basic design can be seen in [gocache]. A [storage |
||||||
|
server][storage] is run as a goroutine that receives a `MCRequest` on |
||||||
|
a channel, and then issues an `MCResponse` to a channel contained |
||||||
|
within the request. |
||||||
|
|
||||||
|
Each connection is a separate goroutine, of course, and is responsible |
||||||
|
for all IO for that connection until the connection drops or the |
||||||
|
`dataServer` decides it's stupid and sends a fatal response back over |
||||||
|
the channel. |
||||||
|
|
||||||
|
There is currently no work at all in making the thing perform (there |
||||||
|
are specific areas I know need work). This is just my attempt to |
||||||
|
learn the language somewhat. |
||||||
|
|
||||||
|
[go]: http://golang.org/ |
||||||
|
[gocache]: gomemcached/blob/master/gocache/gocache.go |
||||||
|
[storage]: gomemcached/blob/master/gocache/mc_storage.go |
||||||
@ -0,0 +1,22 @@ |
|||||||
|
sudo: false |
||||||
|
|
||||||
|
language: go |
||||||
|
|
||||||
|
go: |
||||||
|
- "1.9.x" |
||||||
|
- "1.10.x" |
||||||
|
- "1.11.x" |
||||||
|
|
||||||
|
script: |
||||||
|
- go get github.com/mattn/goveralls |
||||||
|
- go get -u github.com/kisielk/errcheck |
||||||
|
- go test -v $(go list ./... | grep -v vendor/) |
||||||
|
- go test -race |
||||||
|
- go vet |
||||||
|
- errcheck |
||||||
|
- go test -coverprofile=profile.out -covermode=count |
||||||
|
- 'if [ "$TRAVIS_PULL_REQUEST" = "false" ]; then goveralls -service=travis-ci -coverprofile=profile.out -repotoken $COVERALLS; fi' |
||||||
|
|
||||||
|
notifications: |
||||||
|
email: |
||||||
|
- marty.schoch@gmail.com |
||||||
@ -0,0 +1,16 @@ |
|||||||
|
# Contributing to Vellum |
||||||
|
|
||||||
|
We look forward to your contributions, but ask that you first review these guidelines. |
||||||
|
|
||||||
|
### Sign the CLA |
||||||
|
|
||||||
|
As Vellum is a Couchbase project we require contributors accept the [Couchbase Contributor License Agreement](http://review.couchbase.org/static/individual_agreement.html). To sign this agreement log into the Couchbase [code review tool](http://review.couchbase.org/). The Vellum project does not use this code review tool but it is still used to track acceptance of the contributor license agreements. |
||||||
|
|
||||||
|
### Submitting a Pull Request |
||||||
|
|
||||||
|
All types of contributions are welcome, but please keep the following in mind: |
||||||
|
|
||||||
|
- If you're planning a large change, you should really discuss it in a github issue first. This helps avoid duplicate effort and spending time on something that may not be merged. |
||||||
|
- Existing tests should continue to pass, new tests for the contribution are nice to have. |
||||||
|
- All code should have gone through `go fmt` |
||||||
|
- All code should pass `go vet` |
||||||
@ -0,0 +1,183 @@ |
|||||||
|
#  vellum |
||||||
|
|
||||||
|
[](https://travis-ci.org/couchbase/vellum) |
||||||
|
[](https://coveralls.io/github/couchbase/vellum?branch=master) |
||||||
|
[](https://godoc.org/github.com/couchbase/vellum) |
||||||
|
[](https://goreportcard.com/report/github.com/couchbase/vellum) |
||||||
|
[](https://opensource.org/licenses/Apache-2.0) |
||||||
|
|
||||||
|
A Go library implementing an FST (finite state transducer) capable of: |
||||||
|
- mapping between keys ([]byte) and a value (uint64) |
||||||
|
- enumerating keys in lexicographic order |
||||||
|
|
||||||
|
Some additional goals of this implementation: |
||||||
|
- bounded memory use while building the FST |
||||||
|
- streaming out FST data while building |
||||||
|
- mmap FST runtime to support very large FTSs (optional) |
||||||
|
|
||||||
|
## Usage |
||||||
|
|
||||||
|
### Building an FST |
||||||
|
|
||||||
|
To build an FST, create a new builder using the `New()` method. This method takes an `io.Writer` as an argument. As the FST is being built, data will be streamed to the writer as soon as possible. With this builder you **MUST** insert keys in lexicographic order. Inserting keys out of order will result in an error. After inserting the last key into the builder, you **MUST** call `Close()` on the builder. This will flush all remaining data to the underlying writer. |
||||||
|
|
||||||
|
In memory: |
||||||
|
```go |
||||||
|
var buf bytes.Buffer |
||||||
|
builder, err := vellum.New(&buf, nil) |
||||||
|
if err != nil { |
||||||
|
log.Fatal(err) |
||||||
|
} |
||||||
|
``` |
||||||
|
|
||||||
|
To disk: |
||||||
|
```go |
||||||
|
f, err := os.Create("/tmp/vellum.fst") |
||||||
|
if err != nil { |
||||||
|
log.Fatal(err) |
||||||
|
} |
||||||
|
builder, err := vellum.New(f, nil) |
||||||
|
if err != nil { |
||||||
|
log.Fatal(err) |
||||||
|
} |
||||||
|
``` |
||||||
|
|
||||||
|
**MUST** insert keys in lexicographic order: |
||||||
|
```go |
||||||
|
err = builder.Insert([]byte("cat"), 1) |
||||||
|
if err != nil { |
||||||
|
log.Fatal(err) |
||||||
|
} |
||||||
|
|
||||||
|
err = builder.Insert([]byte("dog"), 2) |
||||||
|
if err != nil { |
||||||
|
log.Fatal(err) |
||||||
|
} |
||||||
|
|
||||||
|
err = builder.Insert([]byte("fish"), 3) |
||||||
|
if err != nil { |
||||||
|
log.Fatal(err) |
||||||
|
} |
||||||
|
|
||||||
|
err = builder.Close() |
||||||
|
if err != nil { |
||||||
|
log.Fatal(err) |
||||||
|
} |
||||||
|
``` |
||||||
|
|
||||||
|
### Using an FST |
||||||
|
|
||||||
|
After closing the builder, the data can be used to instantiate an FST. If the data was written to disk, you can use the `Open()` method to mmap the file. If the data is already in memory, or you wish to load/mmap the data yourself, you can instantiate the FST with the `Load()` method. |
||||||
|
|
||||||
|
Load in memory: |
||||||
|
```go |
||||||
|
fst, err := vellum.Load(buf.Bytes()) |
||||||
|
if err != nil { |
||||||
|
log.Fatal(err) |
||||||
|
} |
||||||
|
``` |
||||||
|
|
||||||
|
Open from disk: |
||||||
|
```go |
||||||
|
fst, err := vellum.Open("/tmp/vellum.fst") |
||||||
|
if err != nil { |
||||||
|
log.Fatal(err) |
||||||
|
} |
||||||
|
``` |
||||||
|
|
||||||
|
Get key/value: |
||||||
|
```go |
||||||
|
val, exists, err = fst.Get([]byte("dog")) |
||||||
|
if err != nil { |
||||||
|
log.Fatal(err) |
||||||
|
} |
||||||
|
if exists { |
||||||
|
fmt.Printf("contains dog with val: %d\n", val) |
||||||
|
} else { |
||||||
|
fmt.Printf("does not contain dog") |
||||||
|
} |
||||||
|
``` |
||||||
|
|
||||||
|
Iterate key/values: |
||||||
|
```go |
||||||
|
itr, err := fst.Iterator(startKeyInclusive, endKeyExclusive) |
||||||
|
for err == nil { |
||||||
|
key, val := itr.Current() |
||||||
|
fmt.Printf("contains key: %s val: %d", key, val) |
||||||
|
err = itr.Next() |
||||||
|
} |
||||||
|
if err != nil { |
||||||
|
log.Fatal(err) |
||||||
|
} |
||||||
|
``` |
||||||
|
|
||||||
|
### How does the FST get built? |
||||||
|
|
||||||
|
A full example of the implementation is beyond the scope of this README, but let's consider a small example where we want to insert 3 key/value pairs. |
||||||
|
|
||||||
|
First we insert "are" with the value 4. |
||||||
|
|
||||||
|
 |
||||||
|
|
||||||
|
Next, we insert "ate" with the value 2. |
||||||
|
|
||||||
|
 |
||||||
|
|
||||||
|
Notice how the values associated with the transitions were adjusted so that by summing them while traversing we still get the expected value. |
||||||
|
|
||||||
|
At this point, we see that state 5 looks like state 3, and state 4 looks like state 2. But, we cannot yet combine them because future inserts could change this. |
||||||
|
|
||||||
|
Now, we insert "see" with value 3. Once it has been added, we now know that states 5 and 4 can longer change. Since they are identical to 3 and 2, we replace them. |
||||||
|
|
||||||
|
 |
||||||
|
|
||||||
|
Again, we see that states 7 and 8 appear to be identical to 2 and 3. |
||||||
|
|
||||||
|
Having inserted our last key, we call `Close()` on the builder. |
||||||
|
|
||||||
|
 |
||||||
|
|
||||||
|
Now, states 7 and 8 can safely be replaced with 2 and 3. |
||||||
|
|
||||||
|
For additional information, see the references at the bottom of this document. |
||||||
|
|
||||||
|
### What does the serialized format look like? |
||||||
|
|
||||||
|
We've broken out a separate document on the [vellum disk format v1](docs/format.md). |
||||||
|
|
||||||
|
### What if I want to use this on a system that doesn't have mmap? |
||||||
|
|
||||||
|
The mmap library itself is guarded with system/architecture build tags, but we've also added an additional build tag in vellum. If you'd like to Open() a file based representation of an FST, but not use mmap, you can build the library with the `nommap` build tag. NOTE: if you do this, the entire FST will be read into memory. |
||||||
|
|
||||||
|
### Can I use this with Unicode strings? |
||||||
|
|
||||||
|
Yes, however this implementation is only aware of the byte representation you choose. In order to find matches, you must work with some canonical byte representation of the string. In the future, some encoding-aware traversals may be possible on top of the lower-level byte transitions. |
||||||
|
|
||||||
|
### How did this library come to be? |
||||||
|
|
||||||
|
In my work on the [Bleve](https://github.com/blevesearch/bleve) project I became aware of the power of the FST for many search-related tasks. The obvious starting point for such a thing in Go was the [mafsa](https://github.com/smartystreets/mafsa) project. While working with mafsa I encountered some issues. First, it did not stream data to disk while building. Second, it chose to use a rune as the fundamental unit of transition in the FST, but I felt using a byte would be more powerful in the end. My hope is that higher-level encoding-aware traversals will be possible when necessary. Finally, as I reported bugs and submitted PRs I learned that the mafsa project was mainly a research project and no longer being maintained. I wanted to build something that could be used in production. As the project advanced more and more techniques from the [BurntSushi/fst](https://github.com/BurntSushi/fst) were adapted to our implementation. |
||||||
|
|
||||||
|
### Are there tools to work with vellum files? |
||||||
|
|
||||||
|
Under the cmd/vellum subdirectory, there's a command-line tool which |
||||||
|
features subcommands that can allow you to create, inspect and query |
||||||
|
vellum files. |
||||||
|
|
||||||
|
### How can I generate a state transition diagram from a vellum file? |
||||||
|
|
||||||
|
The vellum command-line tool has a "dot" subcommand that can emit |
||||||
|
graphviz dot output data from an input vellum file. The dot file can |
||||||
|
in turn be converted into an image using graphviz tools. Example... |
||||||
|
|
||||||
|
$ vellum dot myFile.vellum > output.dot |
||||||
|
$ dot -Tpng output.dot -o output.png |
||||||
|
|
||||||
|
## Related Work |
||||||
|
|
||||||
|
Much credit goes to two existing projects: |
||||||
|
- [mafsa](https://github.com/smartystreets/mafsa) |
||||||
|
- [BurntSushi/fst](https://github.com/BurntSushi/fst) |
||||||
|
|
||||||
|
Most of the original implementation here started with my digging into the internals of mafsa. As the implementation progressed, I continued to borrow ideas/approaches from the BurntSushi/fst library as well. |
||||||
|
|
||||||
|
For a great introduction to this topic, please read the blog post [Index 1,600,000,000 Keys with Automata and Rust](http://blog.burntsushi.net/transducers/) |
||||||
@ -0,0 +1,33 @@ |
|||||||
|
# levenshtein |
||||||
|
levenshtein automaton |
||||||
|
|
||||||
|
This package makes it fast and simple to build a finite determinic automaton that computes the levenshtein distance from a given string. |
||||||
|
|
||||||
|
# Sample usage: |
||||||
|
|
||||||
|
``` |
||||||
|
// build a re-usable builder |
||||||
|
lb := NewLevenshteinAutomatonBuilder(2, false) |
||||||
|
|
||||||
|
origTerm := "couchbasefts" |
||||||
|
dfa := lb.BuildDfa("couchbases", 2) |
||||||
|
ed := dfa.eval([]byte(origTerm)) |
||||||
|
if ed.distance() != 2 { |
||||||
|
log.Errorf("expected distance 2, actual: %d", ed.distance()) |
||||||
|
} |
||||||
|
|
||||||
|
``` |
||||||
|
|
||||||
|
This implementation is inspired by [blog post](https://fulmicoton.com/posts/levenshtein/) and is intended to be |
||||||
|
a port of original rust implementation: https://github.com/tantivy-search/levenshtein-automata |
||||||
|
|
||||||
|
|
||||||
|
Micro Benchmark Results against the current vellum/levenshtein is as below. |
||||||
|
|
||||||
|
``` |
||||||
|
BenchmarkNewEditDistance1-8 30000 52684 ns/op 89985 B/op 295 allocs/op |
||||||
|
BenchmarkOlderEditDistance1-8 10000 132931 ns/op 588892 B/op 363 allocs/op |
||||||
|
|
||||||
|
BenchmarkNewEditDistance2-8 10000 199127 ns/op 377532 B/op 1019 allocs/op |
||||||
|
BenchmarkOlderEditDistance2-8 2000 988109 ns/op 4236609 B/op 1898 allocs/op |
||||||
|
``` |
||||||
@ -0,0 +1,14 @@ |
|||||||
|
#* |
||||||
|
*.6 |
||||||
|
*.a |
||||||
|
*~ |
||||||
|
*.swp |
||||||
|
/examples/basic/basic |
||||||
|
/hello/hello |
||||||
|
/populate/populate |
||||||
|
/tools/view2go/view2go |
||||||
|
/tools/loadfile/loadfile |
||||||
|
gotags.files |
||||||
|
TAGS |
||||||
|
6.out |
||||||
|
_* |
||||||
@ -0,0 +1,5 @@ |
|||||||
|
language: go |
||||||
|
install: go get -v -d ./... && go build -v ./... |
||||||
|
script: go test -v ./... |
||||||
|
|
||||||
|
go: 1.1.1 |
||||||
@ -0,0 +1,37 @@ |
|||||||
|
# A smart client for couchbase in go |
||||||
|
|
||||||
|
This is a *unoffical* version of a Couchbase Golang client. If you are |
||||||
|
looking for the *Offical* Couchbase Golang client please see |
||||||
|
[CB-go])[https://github.com/couchbaselabs/gocb]. |
||||||
|
|
||||||
|
This is an evolving package, but does provide a useful interface to a |
||||||
|
[couchbase](http://www.couchbase.com/) server including all of the |
||||||
|
pool/bucket discovery features, compatible key distribution with other |
||||||
|
clients, and vbucket motion awareness so application can continue to |
||||||
|
operate during rebalances. |
||||||
|
|
||||||
|
It also supports view querying with source node randomization so you |
||||||
|
don't bang on all one node to do all the work. |
||||||
|
|
||||||
|
## Install |
||||||
|
|
||||||
|
go get github.com/couchbase/go-couchbase |
||||||
|
|
||||||
|
## Example |
||||||
|
|
||||||
|
c, err := couchbase.Connect("http://dev-couchbase.example.com:8091/") |
||||||
|
if err != nil { |
||||||
|
log.Fatalf("Error connecting: %v", err) |
||||||
|
} |
||||||
|
|
||||||
|
pool, err := c.GetPool("default") |
||||||
|
if err != nil { |
||||||
|
log.Fatalf("Error getting pool: %v", err) |
||||||
|
} |
||||||
|
|
||||||
|
bucket, err := pool.GetBucket("default") |
||||||
|
if err != nil { |
||||||
|
log.Fatalf("Error getting bucket: %v", err) |
||||||
|
} |
||||||
|
|
||||||
|
bucket.Set("someKey", 0, []string{"an", "example", "list"}) |
||||||
@ -0,0 +1,95 @@ |
|||||||
|
# A pure Go MSSQL driver for Go's database/sql package |
||||||
|
|
||||||
|
## Install |
||||||
|
|
||||||
|
go get github.com/denisenkom/go-mssqldb |
||||||
|
|
||||||
|
## Tests |
||||||
|
|
||||||
|
`go test` is used for testing. A running instance of MSSQL server is required. |
||||||
|
Environment variables are used to pass login information. |
||||||
|
|
||||||
|
Example: |
||||||
|
|
||||||
|
env HOST=localhost SQLUSER=sa SQLPASSWORD=sa DATABASE=test go test |
||||||
|
|
||||||
|
## Connection Parameters |
||||||
|
|
||||||
|
* "server" - host or host\instance (default localhost) |
||||||
|
* "port" - used only when there is no instance in server (default 1433) |
||||||
|
* "failoverpartner" - host or host\instance (default is no partner). |
||||||
|
* "failoverport" - used only when there is no instance in failoverpartner (default 1433) |
||||||
|
* "user id" - enter the SQL Server Authentication user id or the Windows Authentication user id in the DOMAIN\User format. On Windows, if user id is empty or missing Single-Sign-On is used. |
||||||
|
* "password" |
||||||
|
* "database" |
||||||
|
* "connection timeout" - in seconds (default is 30) |
||||||
|
* "dial timeout" - in seconds (default is 5) |
||||||
|
* "keepAlive" - in seconds; 0 to disable (default is 0) |
||||||
|
* "log" - logging flags (default 0/no logging, 63 for full logging) |
||||||
|
* 1 log errors |
||||||
|
* 2 log messages |
||||||
|
* 4 log rows affected |
||||||
|
* 8 trace sql statements |
||||||
|
* 16 log statement parameters |
||||||
|
* 32 log transaction begin/end |
||||||
|
* "encrypt" |
||||||
|
* disable - Data send between client and server is not encrypted. |
||||||
|
* false - Data sent between client and server is not encrypted beyond the login packet. (Default) |
||||||
|
* true - Data sent between client and server is encrypted. |
||||||
|
* "TrustServerCertificate" |
||||||
|
* false - Server certificate is checked. Default is false if encypt is specified. |
||||||
|
* true - Server certificate is not checked. Default is true if encrypt is not specified. If trust server certificate is true, driver accepts any certificate presented by the server and any host name in that certificate. In this mode, TLS is susceptible to man-in-the-middle attacks. This should be used only for testing. |
||||||
|
* "certificate" - The file that contains the public key certificate of the CA that signed the SQL Server certificate. The specified certificate overrides the go platform specific CA certificates. |
||||||
|
* "hostNameInCertificate" - Specifies the Common Name (CN) in the server certificate. Default value is the server host. |
||||||
|
* "ServerSPN" - The kerberos SPN (Service Principal Name) for the server. Default is MSSQLSvc/host:port. |
||||||
|
* "Workstation ID" - The workstation name (default is the host name) |
||||||
|
* "app name" - The application name (default is go-mssqldb) |
||||||
|
* "ApplicationIntent" - Can be given the value "ReadOnly" to initiate a read-only connection to an Availability Group listener. |
||||||
|
|
||||||
|
Example: |
||||||
|
|
||||||
|
```go |
||||||
|
db, err := sql.Open("mssql", "server=localhost;user id=sa") |
||||||
|
``` |
||||||
|
|
||||||
|
## Statement Parameters |
||||||
|
|
||||||
|
In the SQL statement text, literals may be replaced by a parameter that matches one of the following: |
||||||
|
|
||||||
|
* ? |
||||||
|
* ?nnn |
||||||
|
* :nnn |
||||||
|
* $nnn |
||||||
|
|
||||||
|
where nnn represents an integer that specifies a 1-indexed positional parameter. Ex: |
||||||
|
|
||||||
|
```go |
||||||
|
db.Query("SELECT * FROM t WHERE a = ?3, b = ?2, c = ?1", "x", "y", "z") |
||||||
|
``` |
||||||
|
|
||||||
|
will expand to roughly |
||||||
|
|
||||||
|
```sql |
||||||
|
SELECT * FROM t WHERE a = 'z', b = 'y', c = 'x' |
||||||
|
``` |
||||||
|
|
||||||
|
|
||||||
|
## Features |
||||||
|
|
||||||
|
* Can be used with SQL Server 2005 or newer |
||||||
|
* Can be used with Microsoft Azure SQL Database |
||||||
|
* Can be used on all go supported platforms (e.g. Linux, Mac OS X and Windows) |
||||||
|
* Supports new date/time types: date, time, datetime2, datetimeoffset |
||||||
|
* Supports string parameters longer than 8000 characters |
||||||
|
* Supports encryption using SSL/TLS |
||||||
|
* Supports SQL Server and Windows Authentication |
||||||
|
* Supports Single-Sign-On on Windows |
||||||
|
* Supports connections to AlwaysOn Availability Group listeners, including re-direction to read-only replicas. |
||||||
|
* Supports query notifications |
||||||
|
|
||||||
|
## Known Issues |
||||||
|
|
||||||
|
* SQL Server 2008 and 2008 R2 engine cannot handle login records when SSL encryption is not disabled. |
||||||
|
To fix SQL Server 2008 R2 issue, install SQL Server 2008 R2 Service Pack 2. |
||||||
|
To fix SQL Server 2008 issue, install Microsoft SQL Server 2008 Service Pack 3 and Cumulative update package 3 for SQL Server 2008 SP3. |
||||||
|
More information: http://support.microsoft.com/kb/2653857 |
||||||
@ -0,0 +1,4 @@ |
|||||||
|
.DS_Store |
||||||
|
bin |
||||||
|
|
||||||
|
|
||||||
@ -0,0 +1,13 @@ |
|||||||
|
language: go |
||||||
|
|
||||||
|
script: |
||||||
|
- go vet ./... |
||||||
|
- go test -v ./... |
||||||
|
|
||||||
|
go: |
||||||
|
- 1.3 |
||||||
|
- 1.4 |
||||||
|
- 1.5 |
||||||
|
- 1.6 |
||||||
|
- 1.7 |
||||||
|
- tip |
||||||
@ -0,0 +1,97 @@ |
|||||||
|
## Migration Guide from v2 -> v3 |
||||||
|
|
||||||
|
Version 3 adds several new, frequently requested features. To do so, it introduces a few breaking changes. We've worked to keep these as minimal as possible. This guide explains the breaking changes and how you can quickly update your code. |
||||||
|
|
||||||
|
### `Token.Claims` is now an interface type |
||||||
|
|
||||||
|
The most requested feature from the 2.0 verison of this library was the ability to provide a custom type to the JSON parser for claims. This was implemented by introducing a new interface, `Claims`, to replace `map[string]interface{}`. We also included two concrete implementations of `Claims`: `MapClaims` and `StandardClaims`. |
||||||
|
|
||||||
|
`MapClaims` is an alias for `map[string]interface{}` with built in validation behavior. It is the default claims type when using `Parse`. The usage is unchanged except you must type cast the claims property. |
||||||
|
|
||||||
|
The old example for parsing a token looked like this.. |
||||||
|
|
||||||
|
```go |
||||||
|
if token, err := jwt.Parse(tokenString, keyLookupFunc); err == nil { |
||||||
|
fmt.Printf("Token for user %v expires %v", token.Claims["user"], token.Claims["exp"]) |
||||||
|
} |
||||||
|
``` |
||||||
|
|
||||||
|
is now directly mapped to... |
||||||
|
|
||||||
|
```go |
||||||
|
if token, err := jwt.Parse(tokenString, keyLookupFunc); err == nil { |
||||||
|
claims := token.Claims.(jwt.MapClaims) |
||||||
|
fmt.Printf("Token for user %v expires %v", claims["user"], claims["exp"]) |
||||||
|
} |
||||||
|
``` |
||||||
|
|
||||||
|
`StandardClaims` is designed to be embedded in your custom type. You can supply a custom claims type with the new `ParseWithClaims` function. Here's an example of using a custom claims type. |
||||||
|
|
||||||
|
```go |
||||||
|
type MyCustomClaims struct { |
||||||
|
User string |
||||||
|
*StandardClaims |
||||||
|
} |
||||||
|
|
||||||
|
if token, err := jwt.ParseWithClaims(tokenString, &MyCustomClaims{}, keyLookupFunc); err == nil { |
||||||
|
claims := token.Claims.(*MyCustomClaims) |
||||||
|
fmt.Printf("Token for user %v expires %v", claims.User, claims.StandardClaims.ExpiresAt) |
||||||
|
} |
||||||
|
``` |
||||||
|
|
||||||
|
### `ParseFromRequest` has been moved |
||||||
|
|
||||||
|
To keep this library focused on the tokens without becoming overburdened with complex request processing logic, `ParseFromRequest` and its new companion `ParseFromRequestWithClaims` have been moved to a subpackage, `request`. The method signatues have also been augmented to receive a new argument: `Extractor`. |
||||||
|
|
||||||
|
`Extractors` do the work of picking the token string out of a request. The interface is simple and composable. |
||||||
|
|
||||||
|
This simple parsing example: |
||||||
|
|
||||||
|
```go |
||||||
|
if token, err := jwt.ParseFromRequest(tokenString, req, keyLookupFunc); err == nil { |
||||||
|
fmt.Printf("Token for user %v expires %v", token.Claims["user"], token.Claims["exp"]) |
||||||
|
} |
||||||
|
``` |
||||||
|
|
||||||
|
is directly mapped to: |
||||||
|
|
||||||
|
```go |
||||||
|
if token, err := request.ParseFromRequest(req, request.OAuth2Extractor, keyLookupFunc); err == nil { |
||||||
|
claims := token.Claims.(jwt.MapClaims) |
||||||
|
fmt.Printf("Token for user %v expires %v", claims["user"], claims["exp"]) |
||||||
|
} |
||||||
|
``` |
||||||
|
|
||||||
|
There are several concrete `Extractor` types provided for your convenience: |
||||||
|
|
||||||
|
* `HeaderExtractor` will search a list of headers until one contains content. |
||||||
|
* `ArgumentExtractor` will search a list of keys in request query and form arguments until one contains content. |
||||||
|
* `MultiExtractor` will try a list of `Extractors` in order until one returns content. |
||||||
|
* `AuthorizationHeaderExtractor` will look in the `Authorization` header for a `Bearer` token. |
||||||
|
* `OAuth2Extractor` searches the places an OAuth2 token would be specified (per the spec): `Authorization` header and `access_token` argument |
||||||
|
* `PostExtractionFilter` wraps an `Extractor`, allowing you to process the content before it's parsed. A simple example is stripping the `Bearer ` text from a header |
||||||
|
|
||||||
|
|
||||||
|
### RSA signing methods no longer accept `[]byte` keys |
||||||
|
|
||||||
|
Due to a [critical vulnerability](https://auth0.com/blog/2015/03/31/critical-vulnerabilities-in-json-web-token-libraries/), we've decided the convenience of accepting `[]byte` instead of `rsa.PublicKey` or `rsa.PrivateKey` isn't worth the risk of misuse. |
||||||
|
|
||||||
|
To replace this behavior, we've added two helper methods: `ParseRSAPrivateKeyFromPEM(key []byte) (*rsa.PrivateKey, error)` and `ParseRSAPublicKeyFromPEM(key []byte) (*rsa.PublicKey, error)`. These are just simple helpers for unpacking PEM encoded PKCS1 and PKCS8 keys. If your keys are encoded any other way, all you need to do is convert them to the `crypto/rsa` package's types. |
||||||
|
|
||||||
|
```go |
||||||
|
func keyLookupFunc(*Token) (interface{}, error) { |
||||||
|
// Don't forget to validate the alg is what you expect: |
||||||
|
if _, ok := token.Method.(*jwt.SigningMethodRSA); !ok { |
||||||
|
return nil, fmt.Errorf("Unexpected signing method: %v", token.Header["alg"]) |
||||||
|
} |
||||||
|
|
||||||
|
// Look up key |
||||||
|
key, err := lookupPublicKey(token.Header["kid"]) |
||||||
|
if err != nil { |
||||||
|
return nil, err |
||||||
|
} |
||||||
|
|
||||||
|
// Unpack key from PEM encoded PKCS8 |
||||||
|
return jwt.ParseRSAPublicKeyFromPEM(key) |
||||||
|
} |
||||||
|
``` |
||||||
@ -0,0 +1,85 @@ |
|||||||
|
A [go](http://www.golang.org) (or 'golang' for search engine friendliness) implementation of [JSON Web Tokens](http://self-issued.info/docs/draft-ietf-oauth-json-web-token.html) |
||||||
|
|
||||||
|
[](https://travis-ci.org/dgrijalva/jwt-go) |
||||||
|
|
||||||
|
**BREAKING CHANGES:*** Version 3.0.0 is here. It includes _a lot_ of changes including a few that break the API. We've tried to break as few things as possible, so there should just be a few type signature changes. A full list of breaking changes is available in `VERSION_HISTORY.md`. See `MIGRATION_GUIDE.md` for more information on updating your code. |
||||||
|
|
||||||
|
**NOTICE:** A vulnerability in JWT was [recently published](https://auth0.com/blog/2015/03/31/critical-vulnerabilities-in-json-web-token-libraries/). As this library doesn't force users to validate the `alg` is what they expected, it's possible your usage is effected. There will be an update soon to remedy this, and it will likey require backwards-incompatible changes to the API. In the short term, please make sure your implementation verifies the `alg` is what you expect. |
||||||
|
|
||||||
|
|
||||||
|
## What the heck is a JWT? |
||||||
|
|
||||||
|
JWT.io has [a great introduction](https://jwt.io/introduction) to JSON Web Tokens. |
||||||
|
|
||||||
|
In short, it's a signed JSON object that does something useful (for example, authentication). It's commonly used for `Bearer` tokens in Oauth 2. A token is made of three parts, separated by `.`'s. The first two parts are JSON objects, that have been [base64url](http://tools.ietf.org/html/rfc4648) encoded. The last part is the signature, encoded the same way. |
||||||
|
|
||||||
|
The first part is called the header. It contains the necessary information for verifying the last part, the signature. For example, which encryption method was used for signing and what key was used. |
||||||
|
|
||||||
|
The part in the middle is the interesting bit. It's called the Claims and contains the actual stuff you care about. Refer to [the RFC](http://self-issued.info/docs/draft-jones-json-web-token.html) for information about reserved keys and the proper way to add your own. |
||||||
|
|
||||||
|
## What's in the box? |
||||||
|
|
||||||
|
This library supports the parsing and verification as well as the generation and signing of JWTs. Current supported signing algorithms are HMAC SHA, RSA, RSA-PSS, and ECDSA, though hooks are present for adding your own. |
||||||
|
|
||||||
|
## Examples |
||||||
|
|
||||||
|
See [the project documentation](https://godoc.org/github.com/dgrijalva/jwt-go) for examples of usage: |
||||||
|
|
||||||
|
* [Simple example of parsing and validating a token](https://godoc.org/github.com/dgrijalva/jwt-go#example-Parse--Hmac) |
||||||
|
* [Simple example of building and signing a token](https://godoc.org/github.com/dgrijalva/jwt-go#example-New--Hmac) |
||||||
|
* [Directory of Examples](https://godoc.org/github.com/dgrijalva/jwt-go#pkg-examples) |
||||||
|
|
||||||
|
## Extensions |
||||||
|
|
||||||
|
This library publishes all the necessary components for adding your own signing methods. Simply implement the `SigningMethod` interface and register a factory method using `RegisterSigningMethod`. |
||||||
|
|
||||||
|
Here's an example of an extension that integrates with the Google App Engine signing tools: https://github.com/someone1/gcp-jwt-go |
||||||
|
|
||||||
|
## Compliance |
||||||
|
|
||||||
|
This library was last reviewed to comply with [RTF 7519](http://www.rfc-editor.org/info/rfc7519) dated May 2015 with a few notable differences: |
||||||
|
|
||||||
|
* In order to protect against accidental use of [Unsecured JWTs](http://self-issued.info/docs/draft-ietf-oauth-json-web-token.html#UnsecuredJWT), tokens using `alg=none` will only be accepted if the constant `jwt.UnsafeAllowNoneSignatureType` is provided as the key. |
||||||
|
|
||||||
|
## Project Status & Versioning |
||||||
|
|
||||||
|
This library is considered production ready. Feedback and feature requests are appreciated. The API should be considered stable. There should be very few backwards-incompatible changes outside of major version updates (and only with good reason). |
||||||
|
|
||||||
|
This project uses [Semantic Versioning 2.0.0](http://semver.org). Accepted pull requests will land on `master`. Periodically, versions will be tagged from `master`. You can find all the releases on [the project releases page](https://github.com/dgrijalva/jwt-go/releases). |
||||||
|
|
||||||
|
While we try to make it obvious when we make breaking changes, there isn't a great mechanism for pushing announcements out to users. You may want to use this alternative package include: `gopkg.in/dgrijalva/jwt-go.v2`. It will do the right thing WRT semantic versioning. |
||||||
|
|
||||||
|
## Usage Tips |
||||||
|
|
||||||
|
### Signing vs Encryption |
||||||
|
|
||||||
|
A token is simply a JSON object that is signed by its author. this tells you exactly two things about the data: |
||||||
|
|
||||||
|
* The author of the token was in the possession of the signing secret |
||||||
|
* The data has not been modified since it was signed |
||||||
|
|
||||||
|
It's important to know that JWT does not provide encryption, which means anyone who has access to the token can read its contents. If you need to protect (encrypt) the data, there is a companion spec, `JWE`, that provides this functionality. JWE is currently outside the scope of this library. |
||||||
|
|
||||||
|
### Choosing a Signing Method |
||||||
|
|
||||||
|
There are several signing methods available, and you should probably take the time to learn about the various options before choosing one. The principal design decision is most likely going to be symmetric vs asymmetric. |
||||||
|
|
||||||
|
Symmetric signing methods, such as HSA, use only a single secret. This is probably the simplest signing method to use since any `[]byte` can be used as a valid secret. They are also slightly computationally faster to use, though this rarely is enough to matter. Symmetric signing methods work the best when both producers and consumers of tokens are trusted, or even the same system. Since the same secret is used to both sign and validate tokens, you can't easily distribute the key for validation. |
||||||
|
|
||||||
|
Asymmetric signing methods, such as RSA, use different keys for signing and verifying tokens. This makes it possible to produce tokens with a private key, and allow any consumer to access the public key for verification. |
||||||
|
|
||||||
|
### JWT and OAuth |
||||||
|
|
||||||
|
It's worth mentioning that OAuth and JWT are not the same thing. A JWT token is simply a signed JSON object. It can be used anywhere such a thing is useful. There is some confusion, though, as JWT is the most common type of bearer token used in OAuth2 authentication. |
||||||
|
|
||||||
|
Without going too far down the rabbit hole, here's a description of the interaction of these technologies: |
||||||
|
|
||||||
|
* OAuth is a protocol for allowing an identity provider to be separate from the service a user is logging in to. For example, whenever you use Facebook to log into a different service (Yelp, Spotify, etc), you are using OAuth. |
||||||
|
* OAuth defines several options for passing around authentication data. One popular method is called a "bearer token". A bearer token is simply a string that _should_ only be held by an authenticated user. Thus, simply presenting this token proves your identity. You can probably derive from here why a JWT might make a good bearer token. |
||||||
|
* Because bearer tokens are used for authentication, it's important they're kept secret. This is why transactions that use bearer tokens typically happen over SSL. |
||||||
|
|
||||||
|
## More |
||||||
|
|
||||||
|
Documentation can be found [on godoc.org](http://godoc.org/github.com/dgrijalva/jwt-go). |
||||||
|
|
||||||
|
The command line utility included in this project (cmd/jwt) provides a straightforward example of token creation and parsing as well as a useful tool for debugging your own integration. You'll also find several implementation examples in to documentation. |
||||||
@ -0,0 +1,105 @@ |
|||||||
|
## `jwt-go` Version History |
||||||
|
|
||||||
|
#### 3.0.0 |
||||||
|
|
||||||
|
* **Compatibility Breaking Changes**: See MIGRATION_GUIDE.md for tips on updating your code |
||||||
|
* Dropped support for `[]byte` keys when using RSA signing methods. This convenience feature could contribute to security vulnerabilities involving mismatched key types with signing methods. |
||||||
|
* `ParseFromRequest` has been moved to `request` subpackage and usage has changed |
||||||
|
* The `Claims` property on `Token` is now type `Claims` instead of `map[string]interface{}`. The default value is type `MapClaims`, which is an alias to `map[string]interface{}`. This makes it possible to use a custom type when decoding claims. |
||||||
|
* Other Additions and Changes |
||||||
|
* Added `Claims` interface type to allow users to decode the claims into a custom type |
||||||
|
* Added `ParseWithClaims`, which takes a third argument of type `Claims`. Use this function instead of `Parse` if you have a custom type you'd like to decode into. |
||||||
|
* Dramatically improved the functionality and flexibility of `ParseFromRequest`, which is now in the `request` subpackage |
||||||
|
* Added `ParseFromRequestWithClaims` which is the `FromRequest` equivalent of `ParseWithClaims` |
||||||
|
* Added new interface type `Extractor`, which is used for extracting JWT strings from http requests. Used with `ParseFromRequest` and `ParseFromRequestWithClaims`. |
||||||
|
* Added several new, more specific, validation errors to error type bitmask |
||||||
|
* Moved examples from README to executable example files |
||||||
|
* Signing method registry is now thread safe |
||||||
|
* Added new property to `ValidationError`, which contains the raw error returned by calls made by parse/verify (such as those returned by keyfunc or json parser) |
||||||
|
|
||||||
|
#### 2.7.0 |
||||||
|
|
||||||
|
This will likely be the last backwards compatible release before 3.0.0, excluding essential bug fixes. |
||||||
|
|
||||||
|
* Added new option `-show` to the `jwt` command that will just output the decoded token without verifying |
||||||
|
* Error text for expired tokens includes how long it's been expired |
||||||
|
* Fixed incorrect error returned from `ParseRSAPublicKeyFromPEM` |
||||||
|
* Documentation updates |
||||||
|
|
||||||
|
#### 2.6.0 |
||||||
|
|
||||||
|
* Exposed inner error within ValidationError |
||||||
|
* Fixed validation errors when using UseJSONNumber flag |
||||||
|
* Added several unit tests |
||||||
|
|
||||||
|
#### 2.5.0 |
||||||
|
|
||||||
|
* Added support for signing method none. You shouldn't use this. The API tries to make this clear. |
||||||
|
* Updated/fixed some documentation |
||||||
|
* Added more helpful error message when trying to parse tokens that begin with `BEARER ` |
||||||
|
|
||||||
|
#### 2.4.0 |
||||||
|
|
||||||
|
* Added new type, Parser, to allow for configuration of various parsing parameters |
||||||
|
* You can now specify a list of valid signing methods. Anything outside this set will be rejected. |
||||||
|
* You can now opt to use the `json.Number` type instead of `float64` when parsing token JSON |
||||||
|
* Added support for [Travis CI](https://travis-ci.org/dgrijalva/jwt-go) |
||||||
|
* Fixed some bugs with ECDSA parsing |
||||||
|
|
||||||
|
#### 2.3.0 |
||||||
|
|
||||||
|
* Added support for ECDSA signing methods |
||||||
|
* Added support for RSA PSS signing methods (requires go v1.4) |
||||||
|
|
||||||
|
#### 2.2.0 |
||||||
|
|
||||||
|
* Gracefully handle a `nil` `Keyfunc` being passed to `Parse`. Result will now be the parsed token and an error, instead of a panic. |
||||||
|
|
||||||
|
#### 2.1.0 |
||||||
|
|
||||||
|
Backwards compatible API change that was missed in 2.0.0. |
||||||
|
|
||||||
|
* The `SignedString` method on `Token` now takes `interface{}` instead of `[]byte` |
||||||
|
|
||||||
|
#### 2.0.0 |
||||||
|
|
||||||
|
There were two major reasons for breaking backwards compatibility with this update. The first was a refactor required to expand the width of the RSA and HMAC-SHA signing implementations. There will likely be no required code changes to support this change. |
||||||
|
|
||||||
|
The second update, while unfortunately requiring a small change in integration, is required to open up this library to other signing methods. Not all keys used for all signing methods have a single standard on-disk representation. Requiring `[]byte` as the type for all keys proved too limiting. Additionally, this implementation allows for pre-parsed tokens to be reused, which might matter in an application that parses a high volume of tokens with a small set of keys. Backwards compatibilty has been maintained for passing `[]byte` to the RSA signing methods, but they will also accept `*rsa.PublicKey` and `*rsa.PrivateKey`. |
||||||
|
|
||||||
|
It is likely the only integration change required here will be to change `func(t *jwt.Token) ([]byte, error)` to `func(t *jwt.Token) (interface{}, error)` when calling `Parse`. |
||||||
|
|
||||||
|
* **Compatibility Breaking Changes** |
||||||
|
* `SigningMethodHS256` is now `*SigningMethodHMAC` instead of `type struct` |
||||||
|
* `SigningMethodRS256` is now `*SigningMethodRSA` instead of `type struct` |
||||||
|
* `KeyFunc` now returns `interface{}` instead of `[]byte` |
||||||
|
* `SigningMethod.Sign` now takes `interface{}` instead of `[]byte` for the key |
||||||
|
* `SigningMethod.Verify` now takes `interface{}` instead of `[]byte` for the key |
||||||
|
* Renamed type `SigningMethodHS256` to `SigningMethodHMAC`. Specific sizes are now just instances of this type. |
||||||
|
* Added public package global `SigningMethodHS256` |
||||||
|
* Added public package global `SigningMethodHS384` |
||||||
|
* Added public package global `SigningMethodHS512` |
||||||
|
* Renamed type `SigningMethodRS256` to `SigningMethodRSA`. Specific sizes are now just instances of this type. |
||||||
|
* Added public package global `SigningMethodRS256` |
||||||
|
* Added public package global `SigningMethodRS384` |
||||||
|
* Added public package global `SigningMethodRS512` |
||||||
|
* Moved sample private key for HMAC tests from an inline value to a file on disk. Value is unchanged. |
||||||
|
* Refactored the RSA implementation to be easier to read |
||||||
|
* Exposed helper methods `ParseRSAPrivateKeyFromPEM` and `ParseRSAPublicKeyFromPEM` |
||||||
|
|
||||||
|
#### 1.0.2 |
||||||
|
|
||||||
|
* Fixed bug in parsing public keys from certificates |
||||||
|
* Added more tests around the parsing of keys for RS256 |
||||||
|
* Code refactoring in RS256 implementation. No functional changes |
||||||
|
|
||||||
|
#### 1.0.1 |
||||||
|
|
||||||
|
* Fixed panic if RS256 signing method was passed an invalid key |
||||||
|
|
||||||
|
#### 1.0.0 |
||||||
|
|
||||||
|
* First versioned release |
||||||
|
* API stabilized |
||||||
|
* Supports creating, signing, parsing, and validating JWT tokens |
||||||
|
* Supports RS256 and HS256 signing methods |
||||||
@ -0,0 +1,8 @@ |
|||||||
|
*.out |
||||||
|
*.5 |
||||||
|
*.6 |
||||||
|
*.8 |
||||||
|
*.swp |
||||||
|
_obj |
||||||
|
_test |
||||||
|
testdata |
||||||
@ -0,0 +1,12 @@ |
|||||||
|
mmap-go |
||||||
|
======= |
||||||
|
|
||||||
|
mmap-go is a portable mmap package for the [Go programming language](http://golang.org). |
||||||
|
It has been tested on Linux (386, amd64), OS X, and Windows (386). It should also |
||||||
|
work on other Unix-like platforms, but hasn't been tested with them. I'm interested |
||||||
|
to hear about the results. |
||||||
|
|
||||||
|
I haven't been able to add more features without adding significant complexity, |
||||||
|
so mmap-go doesn't support mprotect, mincore, and maybe a few other things. |
||||||
|
If you're running on a Unix-like platform and need some of these features, |
||||||
|
I suggest Gustavo Niemeyer's [gommap](http://labix.org/gommap). |
||||||
@ -0,0 +1,46 @@ |
|||||||
|
# go-bindata-assetfs |
||||||
|
|
||||||
|
Serve embedded files from [jteeuwen/go-bindata](https://github.com/jteeuwen/go-bindata) with `net/http`. |
||||||
|
|
||||||
|
[GoDoc](http://godoc.org/github.com/elazarl/go-bindata-assetfs) |
||||||
|
|
||||||
|
### Installation |
||||||
|
|
||||||
|
Install with |
||||||
|
|
||||||
|
$ go get github.com/jteeuwen/go-bindata/... |
||||||
|
$ go get github.com/elazarl/go-bindata-assetfs/... |
||||||
|
|
||||||
|
### Creating embedded data |
||||||
|
|
||||||
|
Usage is identical to [jteeuwen/go-bindata](https://github.com/jteeuwen/go-bindata) usage, |
||||||
|
instead of running `go-bindata` run `go-bindata-assetfs`. |
||||||
|
|
||||||
|
The tool will create a `bindata_assetfs.go` file, which contains the embedded data. |
||||||
|
|
||||||
|
A typical use case is |
||||||
|
|
||||||
|
$ go-bindata-assetfs data/... |
||||||
|
|
||||||
|
### Using assetFS in your code |
||||||
|
|
||||||
|
The generated file provides an `assetFS()` function that returns a `http.Filesystem` |
||||||
|
wrapping the embedded files. What you usually want to do is: |
||||||
|
|
||||||
|
http.Handle("/", http.FileServer(assetFS())) |
||||||
|
|
||||||
|
This would run an HTTP server serving the embedded files. |
||||||
|
|
||||||
|
## Without running binary tool |
||||||
|
|
||||||
|
You can always just run the `go-bindata` tool, and then |
||||||
|
|
||||||
|
use |
||||||
|
|
||||||
|
import "github.com/elazarl/go-bindata-assetfs" |
||||||
|
... |
||||||
|
http.Handle("/", |
||||||
|
http.FileServer( |
||||||
|
&assetfs.AssetFS{Asset: Asset, AssetDir: AssetDir, AssetInfo: AssetInfo, Prefix: "data"})) |
||||||
|
|
||||||
|
to serve files embedded from the `data` directory. |
||||||
@ -0,0 +1,5 @@ |
|||||||
|
*.prof |
||||||
|
*.test |
||||||
|
*.swp |
||||||
|
/bin/ |
||||||
|
cover.out |
||||||
@ -0,0 +1,17 @@ |
|||||||
|
language: go |
||||||
|
go_import_path: go.etcd.io/bbolt |
||||||
|
|
||||||
|
sudo: false |
||||||
|
|
||||||
|
go: |
||||||
|
- 1.11 |
||||||
|
|
||||||
|
before_install: |
||||||
|
- go get -v honnef.co/go/tools/... |
||||||
|
- go get -v github.com/kisielk/errcheck |
||||||
|
|
||||||
|
script: |
||||||
|
- make fmt |
||||||
|
- make test |
||||||
|
- make race |
||||||
|
# - make errcheck |
||||||
@ -0,0 +1,38 @@ |
|||||||
|
BRANCH=`git rev-parse --abbrev-ref HEAD`
|
||||||
|
COMMIT=`git rev-parse --short HEAD`
|
||||||
|
GOLDFLAGS="-X main.branch $(BRANCH) -X main.commit $(COMMIT)"
|
||||||
|
|
||||||
|
default: build |
||||||
|
|
||||||
|
race: |
||||||
|
@TEST_FREELIST_TYPE=hashmap go test -v -race -test.run="TestSimulate_(100op|1000op)"
|
||||||
|
@echo "array freelist test"
|
||||||
|
@TEST_FREELIST_TYPE=array go test -v -race -test.run="TestSimulate_(100op|1000op)"
|
||||||
|
|
||||||
|
fmt: |
||||||
|
!(gofmt -l -s -d $(shell find . -name \*.go) | grep '[a-z]')
|
||||||
|
|
||||||
|
# go get honnef.co/go/tools/simple
|
||||||
|
gosimple: |
||||||
|
gosimple ./...
|
||||||
|
|
||||||
|
# go get honnef.co/go/tools/unused
|
||||||
|
unused: |
||||||
|
unused ./...
|
||||||
|
|
||||||
|
# go get github.com/kisielk/errcheck
|
||||||
|
errcheck: |
||||||
|
@errcheck -ignorepkg=bytes -ignore=os:Remove go.etcd.io/bbolt
|
||||||
|
|
||||||
|
test: |
||||||
|
TEST_FREELIST_TYPE=hashmap go test -timeout 20m -v -coverprofile cover.out -covermode atomic
|
||||||
|
# Note: gets "program not an importable package" in out of path builds
|
||||||
|
TEST_FREELIST_TYPE=hashmap go test -v ./cmd/bbolt
|
||||||
|
|
||||||
|
@echo "array freelist test"
|
||||||
|
|
||||||
|
@TEST_FREELIST_TYPE=array go test -timeout 20m -v -coverprofile cover.out -covermode atomic
|
||||||
|
# Note: gets "program not an importable package" in out of path builds
|
||||||
|
@TEST_FREELIST_TYPE=array go test -v ./cmd/bbolt
|
||||||
|
|
||||||
|
.PHONY: race fmt errcheck test gosimple unused |
||||||
@ -0,0 +1,954 @@ |
|||||||
|
bbolt |
||||||
|
===== |
||||||
|
|
||||||
|
[](https://goreportcard.com/report/github.com/etcd-io/bbolt) |
||||||
|
[](https://codecov.io/gh/etcd-io/bbolt) |
||||||
|
[](https://travis-ci.com/etcd-io/bbolt) |
||||||
|
[](https://godoc.org/github.com/etcd-io/bbolt) |
||||||
|
[](https://github.com/etcd-io/bbolt/releases) |
||||||
|
[](https://github.com/etcd-io/bbolt/blob/master/LICENSE) |
||||||
|
|
||||||
|
bbolt is a fork of [Ben Johnson's][gh_ben] [Bolt][bolt] key/value |
||||||
|
store. The purpose of this fork is to provide the Go community with an active |
||||||
|
maintenance and development target for Bolt; the goal is improved reliability |
||||||
|
and stability. bbolt includes bug fixes, performance enhancements, and features |
||||||
|
not found in Bolt while preserving backwards compatibility with the Bolt API. |
||||||
|
|
||||||
|
Bolt is a pure Go key/value store inspired by [Howard Chu's][hyc_symas] |
||||||
|
[LMDB project][lmdb]. The goal of the project is to provide a simple, |
||||||
|
fast, and reliable database for projects that don't require a full database |
||||||
|
server such as Postgres or MySQL. |
||||||
|
|
||||||
|
Since Bolt is meant to be used as such a low-level piece of functionality, |
||||||
|
simplicity is key. The API will be small and only focus on getting values |
||||||
|
and setting values. That's it. |
||||||
|
|
||||||
|
[gh_ben]: https://github.com/benbjohnson |
||||||
|
[bolt]: https://github.com/boltdb/bolt |
||||||
|
[hyc_symas]: https://twitter.com/hyc_symas |
||||||
|
[lmdb]: http://symas.com/mdb/ |
||||||
|
|
||||||
|
## Project Status |
||||||
|
|
||||||
|
Bolt is stable, the API is fixed, and the file format is fixed. Full unit |
||||||
|
test coverage and randomized black box testing are used to ensure database |
||||||
|
consistency and thread safety. Bolt is currently used in high-load production |
||||||
|
environments serving databases as large as 1TB. Many companies such as |
||||||
|
Shopify and Heroku use Bolt-backed services every day. |
||||||
|
|
||||||
|
## Project versioning |
||||||
|
|
||||||
|
bbolt uses [semantic versioning](http://semver.org). |
||||||
|
API should not change between patch and minor releases. |
||||||
|
New minor versions may add additional features to the API. |
||||||
|
|
||||||
|
## Table of Contents |
||||||
|
|
||||||
|
- [Getting Started](#getting-started) |
||||||
|
- [Installing](#installing) |
||||||
|
- [Opening a database](#opening-a-database) |
||||||
|
- [Transactions](#transactions) |
||||||
|
- [Read-write transactions](#read-write-transactions) |
||||||
|
- [Read-only transactions](#read-only-transactions) |
||||||
|
- [Batch read-write transactions](#batch-read-write-transactions) |
||||||
|
- [Managing transactions manually](#managing-transactions-manually) |
||||||
|
- [Using buckets](#using-buckets) |
||||||
|
- [Using key/value pairs](#using-keyvalue-pairs) |
||||||
|
- [Autoincrementing integer for the bucket](#autoincrementing-integer-for-the-bucket) |
||||||
|
- [Iterating over keys](#iterating-over-keys) |
||||||
|
- [Prefix scans](#prefix-scans) |
||||||
|
- [Range scans](#range-scans) |
||||||
|
- [ForEach()](#foreach) |
||||||
|
- [Nested buckets](#nested-buckets) |
||||||
|
- [Database backups](#database-backups) |
||||||
|
- [Statistics](#statistics) |
||||||
|
- [Read-Only Mode](#read-only-mode) |
||||||
|
- [Mobile Use (iOS/Android)](#mobile-use-iosandroid) |
||||||
|
- [Resources](#resources) |
||||||
|
- [Comparison with other databases](#comparison-with-other-databases) |
||||||
|
- [Postgres, MySQL, & other relational databases](#postgres-mysql--other-relational-databases) |
||||||
|
- [LevelDB, RocksDB](#leveldb-rocksdb) |
||||||
|
- [LMDB](#lmdb) |
||||||
|
- [Caveats & Limitations](#caveats--limitations) |
||||||
|
- [Reading the Source](#reading-the-source) |
||||||
|
- [Other Projects Using Bolt](#other-projects-using-bolt) |
||||||
|
|
||||||
|
## Getting Started |
||||||
|
|
||||||
|
### Installing |
||||||
|
|
||||||
|
To start using Bolt, install Go and run `go get`: |
||||||
|
|
||||||
|
```sh |
||||||
|
$ go get go.etcd.io/bbolt/... |
||||||
|
``` |
||||||
|
|
||||||
|
This will retrieve the library and install the `bolt` command line utility into |
||||||
|
your `$GOBIN` path. |
||||||
|
|
||||||
|
|
||||||
|
### Importing bbolt |
||||||
|
|
||||||
|
To use bbolt as an embedded key-value store, import as: |
||||||
|
|
||||||
|
```go |
||||||
|
import bolt "go.etcd.io/bbolt" |
||||||
|
|
||||||
|
db, err := bolt.Open(path, 0666, nil) |
||||||
|
if err != nil { |
||||||
|
return err |
||||||
|
} |
||||||
|
defer db.Close() |
||||||
|
``` |
||||||
|
|
||||||
|
|
||||||
|
### Opening a database |
||||||
|
|
||||||
|
The top-level object in Bolt is a `DB`. It is represented as a single file on |
||||||
|
your disk and represents a consistent snapshot of your data. |
||||||
|
|
||||||
|
To open your database, simply use the `bolt.Open()` function: |
||||||
|
|
||||||
|
```go |
||||||
|
package main |
||||||
|
|
||||||
|
import ( |
||||||
|
"log" |
||||||
|
|
||||||
|
bolt "go.etcd.io/bbolt" |
||||||
|
) |
||||||
|
|
||||||
|
func main() { |
||||||
|
// Open the my.db data file in your current directory. |
||||||
|
// It will be created if it doesn't exist. |
||||||
|
db, err := bolt.Open("my.db", 0600, nil) |
||||||
|
if err != nil { |
||||||
|
log.Fatal(err) |
||||||
|
} |
||||||
|
defer db.Close() |
||||||
|
|
||||||
|
... |
||||||
|
} |
||||||
|
``` |
||||||
|
|
||||||
|
Please note that Bolt obtains a file lock on the data file so multiple processes |
||||||
|
cannot open the same database at the same time. Opening an already open Bolt |
||||||
|
database will cause it to hang until the other process closes it. To prevent |
||||||
|
an indefinite wait you can pass a timeout option to the `Open()` function: |
||||||
|
|
||||||
|
```go |
||||||
|
db, err := bolt.Open("my.db", 0600, &bolt.Options{Timeout: 1 * time.Second}) |
||||||
|
``` |
||||||
|
|
||||||
|
|
||||||
|
### Transactions |
||||||
|
|
||||||
|
Bolt allows only one read-write transaction at a time but allows as many |
||||||
|
read-only transactions as you want at a time. Each transaction has a consistent |
||||||
|
view of the data as it existed when the transaction started. |
||||||
|
|
||||||
|
Individual transactions and all objects created from them (e.g. buckets, keys) |
||||||
|
are not thread safe. To work with data in multiple goroutines you must start |
||||||
|
a transaction for each one or use locking to ensure only one goroutine accesses |
||||||
|
a transaction at a time. Creating transaction from the `DB` is thread safe. |
||||||
|
|
||||||
|
Read-only transactions and read-write transactions should not depend on one |
||||||
|
another and generally shouldn't be opened simultaneously in the same goroutine. |
||||||
|
This can cause a deadlock as the read-write transaction needs to periodically |
||||||
|
re-map the data file but it cannot do so while a read-only transaction is open. |
||||||
|
|
||||||
|
|
||||||
|
#### Read-write transactions |
||||||
|
|
||||||
|
To start a read-write transaction, you can use the `DB.Update()` function: |
||||||
|
|
||||||
|
```go |
||||||
|
err := db.Update(func(tx *bolt.Tx) error { |
||||||
|
... |
||||||
|
return nil |
||||||
|
}) |
||||||
|
``` |
||||||
|
|
||||||
|
Inside the closure, you have a consistent view of the database. You commit the |
||||||
|
transaction by returning `nil` at the end. You can also rollback the transaction |
||||||
|
at any point by returning an error. All database operations are allowed inside |
||||||
|
a read-write transaction. |
||||||
|
|
||||||
|
Always check the return error as it will report any disk failures that can cause |
||||||
|
your transaction to not complete. If you return an error within your closure |
||||||
|
it will be passed through. |
||||||
|
|
||||||
|
|
||||||
|
#### Read-only transactions |
||||||
|
|
||||||
|
To start a read-only transaction, you can use the `DB.View()` function: |
||||||
|
|
||||||
|
```go |
||||||
|
err := db.View(func(tx *bolt.Tx) error { |
||||||
|
... |
||||||
|
return nil |
||||||
|
}) |
||||||
|
``` |
||||||
|
|
||||||
|
You also get a consistent view of the database within this closure, however, |
||||||
|
no mutating operations are allowed within a read-only transaction. You can only |
||||||
|
retrieve buckets, retrieve values, and copy the database within a read-only |
||||||
|
transaction. |
||||||
|
|
||||||
|
|
||||||
|
#### Batch read-write transactions |
||||||
|
|
||||||
|
Each `DB.Update()` waits for disk to commit the writes. This overhead |
||||||
|
can be minimized by combining multiple updates with the `DB.Batch()` |
||||||
|
function: |
||||||
|
|
||||||
|
```go |
||||||
|
err := db.Batch(func(tx *bolt.Tx) error { |
||||||
|
... |
||||||
|
return nil |
||||||
|
}) |
||||||
|
``` |
||||||
|
|
||||||
|
Concurrent Batch calls are opportunistically combined into larger |
||||||
|
transactions. Batch is only useful when there are multiple goroutines |
||||||
|
calling it. |
||||||
|
|
||||||
|
The trade-off is that `Batch` can call the given |
||||||
|
function multiple times, if parts of the transaction fail. The |
||||||
|
function must be idempotent and side effects must take effect only |
||||||
|
after a successful return from `DB.Batch()`. |
||||||
|
|
||||||
|
For example: don't display messages from inside the function, instead |
||||||
|
set variables in the enclosing scope: |
||||||
|
|
||||||
|
```go |
||||||
|
var id uint64 |
||||||
|
err := db.Batch(func(tx *bolt.Tx) error { |
||||||
|
// Find last key in bucket, decode as bigendian uint64, increment |
||||||
|
// by one, encode back to []byte, and add new key. |
||||||
|
... |
||||||
|
id = newValue |
||||||
|
return nil |
||||||
|
}) |
||||||
|
if err != nil { |
||||||
|
return ... |
||||||
|
} |
||||||
|
fmt.Println("Allocated ID %d", id) |
||||||
|
``` |
||||||
|
|
||||||
|
|
||||||
|
#### Managing transactions manually |
||||||
|
|
||||||
|
The `DB.View()` and `DB.Update()` functions are wrappers around the `DB.Begin()` |
||||||
|
function. These helper functions will start the transaction, execute a function, |
||||||
|
and then safely close your transaction if an error is returned. This is the |
||||||
|
recommended way to use Bolt transactions. |
||||||
|
|
||||||
|
However, sometimes you may want to manually start and end your transactions. |
||||||
|
You can use the `DB.Begin()` function directly but **please** be sure to close |
||||||
|
the transaction. |
||||||
|
|
||||||
|
```go |
||||||
|
// Start a writable transaction. |
||||||
|
tx, err := db.Begin(true) |
||||||
|
if err != nil { |
||||||
|
return err |
||||||
|
} |
||||||
|
defer tx.Rollback() |
||||||
|
|
||||||
|
// Use the transaction... |
||||||
|
_, err := tx.CreateBucket([]byte("MyBucket")) |
||||||
|
if err != nil { |
||||||
|
return err |
||||||
|
} |
||||||
|
|
||||||
|
// Commit the transaction and check for error. |
||||||
|
if err := tx.Commit(); err != nil { |
||||||
|
return err |
||||||
|
} |
||||||
|
``` |
||||||
|
|
||||||
|
The first argument to `DB.Begin()` is a boolean stating if the transaction |
||||||
|
should be writable. |
||||||
|
|
||||||
|
|
||||||
|
### Using buckets |
||||||
|
|
||||||
|
Buckets are collections of key/value pairs within the database. All keys in a |
||||||
|
bucket must be unique. You can create a bucket using the `DB.CreateBucket()` |
||||||
|
function: |
||||||
|
|
||||||
|
```go |
||||||
|
db.Update(func(tx *bolt.Tx) error { |
||||||
|
b, err := tx.CreateBucket([]byte("MyBucket")) |
||||||
|
if err != nil { |
||||||
|
return fmt.Errorf("create bucket: %s", err) |
||||||
|
} |
||||||
|
return nil |
||||||
|
}) |
||||||
|
``` |
||||||
|
|
||||||
|
You can also create a bucket only if it doesn't exist by using the |
||||||
|
`Tx.CreateBucketIfNotExists()` function. It's a common pattern to call this |
||||||
|
function for all your top-level buckets after you open your database so you can |
||||||
|
guarantee that they exist for future transactions. |
||||||
|
|
||||||
|
To delete a bucket, simply call the `Tx.DeleteBucket()` function. |
||||||
|
|
||||||
|
|
||||||
|
### Using key/value pairs |
||||||
|
|
||||||
|
To save a key/value pair to a bucket, use the `Bucket.Put()` function: |
||||||
|
|
||||||
|
```go |
||||||
|
db.Update(func(tx *bolt.Tx) error { |
||||||
|
b := tx.Bucket([]byte("MyBucket")) |
||||||
|
err := b.Put([]byte("answer"), []byte("42")) |
||||||
|
return err |
||||||
|
}) |
||||||
|
``` |
||||||
|
|
||||||
|
This will set the value of the `"answer"` key to `"42"` in the `MyBucket` |
||||||
|
bucket. To retrieve this value, we can use the `Bucket.Get()` function: |
||||||
|
|
||||||
|
```go |
||||||
|
db.View(func(tx *bolt.Tx) error { |
||||||
|
b := tx.Bucket([]byte("MyBucket")) |
||||||
|
v := b.Get([]byte("answer")) |
||||||
|
fmt.Printf("The answer is: %s\n", v) |
||||||
|
return nil |
||||||
|
}) |
||||||
|
``` |
||||||
|
|
||||||
|
The `Get()` function does not return an error because its operation is |
||||||
|
guaranteed to work (unless there is some kind of system failure). If the key |
||||||
|
exists then it will return its byte slice value. If it doesn't exist then it |
||||||
|
will return `nil`. It's important to note that you can have a zero-length value |
||||||
|
set to a key which is different than the key not existing. |
||||||
|
|
||||||
|
Use the `Bucket.Delete()` function to delete a key from the bucket. |
||||||
|
|
||||||
|
Please note that values returned from `Get()` are only valid while the |
||||||
|
transaction is open. If you need to use a value outside of the transaction |
||||||
|
then you must use `copy()` to copy it to another byte slice. |
||||||
|
|
||||||
|
|
||||||
|
### Autoincrementing integer for the bucket |
||||||
|
By using the `NextSequence()` function, you can let Bolt determine a sequence |
||||||
|
which can be used as the unique identifier for your key/value pairs. See the |
||||||
|
example below. |
||||||
|
|
||||||
|
```go |
||||||
|
// CreateUser saves u to the store. The new user ID is set on u once the data is persisted. |
||||||
|
func (s *Store) CreateUser(u *User) error { |
||||||
|
return s.db.Update(func(tx *bolt.Tx) error { |
||||||
|
// Retrieve the users bucket. |
||||||
|
// This should be created when the DB is first opened. |
||||||
|
b := tx.Bucket([]byte("users")) |
||||||
|
|
||||||
|
// Generate ID for the user. |
||||||
|
// This returns an error only if the Tx is closed or not writeable. |
||||||
|
// That can't happen in an Update() call so I ignore the error check. |
||||||
|
id, _ := b.NextSequence() |
||||||
|
u.ID = int(id) |
||||||
|
|
||||||
|
// Marshal user data into bytes. |
||||||
|
buf, err := json.Marshal(u) |
||||||
|
if err != nil { |
||||||
|
return err |
||||||
|
} |
||||||
|
|
||||||
|
// Persist bytes to users bucket. |
||||||
|
return b.Put(itob(u.ID), buf) |
||||||
|
}) |
||||||
|
} |
||||||
|
|
||||||
|
// itob returns an 8-byte big endian representation of v. |
||||||
|
func itob(v int) []byte { |
||||||
|
b := make([]byte, 8) |
||||||
|
binary.BigEndian.PutUint64(b, uint64(v)) |
||||||
|
return b |
||||||
|
} |
||||||
|
|
||||||
|
type User struct { |
||||||
|
ID int |
||||||
|
... |
||||||
|
} |
||||||
|
``` |
||||||
|
|
||||||
|
### Iterating over keys |
||||||
|
|
||||||
|
Bolt stores its keys in byte-sorted order within a bucket. This makes sequential |
||||||
|
iteration over these keys extremely fast. To iterate over keys we'll use a |
||||||
|
`Cursor`: |
||||||
|
|
||||||
|
```go |
||||||
|
db.View(func(tx *bolt.Tx) error { |
||||||
|
// Assume bucket exists and has keys |
||||||
|
b := tx.Bucket([]byte("MyBucket")) |
||||||
|
|
||||||
|
c := b.Cursor() |
||||||
|
|
||||||
|
for k, v := c.First(); k != nil; k, v = c.Next() { |
||||||
|
fmt.Printf("key=%s, value=%s\n", k, v) |
||||||
|
} |
||||||
|
|
||||||
|
return nil |
||||||
|
}) |
||||||
|
``` |
||||||
|
|
||||||
|
The cursor allows you to move to a specific point in the list of keys and move |
||||||
|
forward or backward through the keys one at a time. |
||||||
|
|
||||||
|
The following functions are available on the cursor: |
||||||
|
|
||||||
|
``` |
||||||
|
First() Move to the first key. |
||||||
|
Last() Move to the last key. |
||||||
|
Seek() Move to a specific key. |
||||||
|
Next() Move to the next key. |
||||||
|
Prev() Move to the previous key. |
||||||
|
``` |
||||||
|
|
||||||
|
Each of those functions has a return signature of `(key []byte, value []byte)`. |
||||||
|
When you have iterated to the end of the cursor then `Next()` will return a |
||||||
|
`nil` key. You must seek to a position using `First()`, `Last()`, or `Seek()` |
||||||
|
before calling `Next()` or `Prev()`. If you do not seek to a position then |
||||||
|
these functions will return a `nil` key. |
||||||
|
|
||||||
|
During iteration, if the key is non-`nil` but the value is `nil`, that means |
||||||
|
the key refers to a bucket rather than a value. Use `Bucket.Bucket()` to |
||||||
|
access the sub-bucket. |
||||||
|
|
||||||
|
|
||||||
|
#### Prefix scans |
||||||
|
|
||||||
|
To iterate over a key prefix, you can combine `Seek()` and `bytes.HasPrefix()`: |
||||||
|
|
||||||
|
```go |
||||||
|
db.View(func(tx *bolt.Tx) error { |
||||||
|
// Assume bucket exists and has keys |
||||||
|
c := tx.Bucket([]byte("MyBucket")).Cursor() |
||||||
|
|
||||||
|
prefix := []byte("1234") |
||||||
|
for k, v := c.Seek(prefix); k != nil && bytes.HasPrefix(k, prefix); k, v = c.Next() { |
||||||
|
fmt.Printf("key=%s, value=%s\n", k, v) |
||||||
|
} |
||||||
|
|
||||||
|
return nil |
||||||
|
}) |
||||||
|
``` |
||||||
|
|
||||||
|
#### Range scans |
||||||
|
|
||||||
|
Another common use case is scanning over a range such as a time range. If you |
||||||
|
use a sortable time encoding such as RFC3339 then you can query a specific |
||||||
|
date range like this: |
||||||
|
|
||||||
|
```go |
||||||
|
db.View(func(tx *bolt.Tx) error { |
||||||
|
// Assume our events bucket exists and has RFC3339 encoded time keys. |
||||||
|
c := tx.Bucket([]byte("Events")).Cursor() |
||||||
|
|
||||||
|
// Our time range spans the 90's decade. |
||||||
|
min := []byte("1990-01-01T00:00:00Z") |
||||||
|
max := []byte("2000-01-01T00:00:00Z") |
||||||
|
|
||||||
|
// Iterate over the 90's. |
||||||
|
for k, v := c.Seek(min); k != nil && bytes.Compare(k, max) <= 0; k, v = c.Next() { |
||||||
|
fmt.Printf("%s: %s\n", k, v) |
||||||
|
} |
||||||
|
|
||||||
|
return nil |
||||||
|
}) |
||||||
|
``` |
||||||
|
|
||||||
|
Note that, while RFC3339 is sortable, the Golang implementation of RFC3339Nano does not use a fixed number of digits after the decimal point and is therefore not sortable. |
||||||
|
|
||||||
|
|
||||||
|
#### ForEach() |
||||||
|
|
||||||
|
You can also use the function `ForEach()` if you know you'll be iterating over |
||||||
|
all the keys in a bucket: |
||||||
|
|
||||||
|
```go |
||||||
|
db.View(func(tx *bolt.Tx) error { |
||||||
|
// Assume bucket exists and has keys |
||||||
|
b := tx.Bucket([]byte("MyBucket")) |
||||||
|
|
||||||
|
b.ForEach(func(k, v []byte) error { |
||||||
|
fmt.Printf("key=%s, value=%s\n", k, v) |
||||||
|
return nil |
||||||
|
}) |
||||||
|
return nil |
||||||
|
}) |
||||||
|
``` |
||||||
|
|
||||||
|
Please note that keys and values in `ForEach()` are only valid while |
||||||
|
the transaction is open. If you need to use a key or value outside of |
||||||
|
the transaction, you must use `copy()` to copy it to another byte |
||||||
|
slice. |
||||||
|
|
||||||
|
### Nested buckets |
||||||
|
|
||||||
|
You can also store a bucket in a key to create nested buckets. The API is the |
||||||
|
same as the bucket management API on the `DB` object: |
||||||
|
|
||||||
|
```go |
||||||
|
func (*Bucket) CreateBucket(key []byte) (*Bucket, error) |
||||||
|
func (*Bucket) CreateBucketIfNotExists(key []byte) (*Bucket, error) |
||||||
|
func (*Bucket) DeleteBucket(key []byte) error |
||||||
|
``` |
||||||
|
|
||||||
|
Say you had a multi-tenant application where the root level bucket was the account bucket. Inside of this bucket was a sequence of accounts which themselves are buckets. And inside the sequence bucket you could have many buckets pertaining to the Account itself (Users, Notes, etc) isolating the information into logical groupings. |
||||||
|
|
||||||
|
```go |
||||||
|
|
||||||
|
// createUser creates a new user in the given account. |
||||||
|
func createUser(accountID int, u *User) error { |
||||||
|
// Start the transaction. |
||||||
|
tx, err := db.Begin(true) |
||||||
|
if err != nil { |
||||||
|
return err |
||||||
|
} |
||||||
|
defer tx.Rollback() |
||||||
|
|
||||||
|
// Retrieve the root bucket for the account. |
||||||
|
// Assume this has already been created when the account was set up. |
||||||
|
root := tx.Bucket([]byte(strconv.FormatUint(accountID, 10))) |
||||||
|
|
||||||
|
// Setup the users bucket. |
||||||
|
bkt, err := root.CreateBucketIfNotExists([]byte("USERS")) |
||||||
|
if err != nil { |
||||||
|
return err |
||||||
|
} |
||||||
|
|
||||||
|
// Generate an ID for the new user. |
||||||
|
userID, err := bkt.NextSequence() |
||||||
|
if err != nil { |
||||||
|
return err |
||||||
|
} |
||||||
|
u.ID = userID |
||||||
|
|
||||||
|
// Marshal and save the encoded user. |
||||||
|
if buf, err := json.Marshal(u); err != nil { |
||||||
|
return err |
||||||
|
} else if err := bkt.Put([]byte(strconv.FormatUint(u.ID, 10)), buf); err != nil { |
||||||
|
return err |
||||||
|
} |
||||||
|
|
||||||
|
// Commit the transaction. |
||||||
|
if err := tx.Commit(); err != nil { |
||||||
|
return err |
||||||
|
} |
||||||
|
|
||||||
|
return nil |
||||||
|
} |
||||||
|
|
||||||
|
``` |
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
### Database backups |
||||||
|
|
||||||
|
Bolt is a single file so it's easy to backup. You can use the `Tx.WriteTo()` |
||||||
|
function to write a consistent view of the database to a writer. If you call |
||||||
|
this from a read-only transaction, it will perform a hot backup and not block |
||||||
|
your other database reads and writes. |
||||||
|
|
||||||
|
By default, it will use a regular file handle which will utilize the operating |
||||||
|
system's page cache. See the [`Tx`](https://godoc.org/go.etcd.io/bbolt#Tx) |
||||||
|
documentation for information about optimizing for larger-than-RAM datasets. |
||||||
|
|
||||||
|
One common use case is to backup over HTTP so you can use tools like `cURL` to |
||||||
|
do database backups: |
||||||
|
|
||||||
|
```go |
||||||
|
func BackupHandleFunc(w http.ResponseWriter, req *http.Request) { |
||||||
|
err := db.View(func(tx *bolt.Tx) error { |
||||||
|
w.Header().Set("Content-Type", "application/octet-stream") |
||||||
|
w.Header().Set("Content-Disposition", `attachment; filename="my.db"`) |
||||||
|
w.Header().Set("Content-Length", strconv.Itoa(int(tx.Size()))) |
||||||
|
_, err := tx.WriteTo(w) |
||||||
|
return err |
||||||
|
}) |
||||||
|
if err != nil { |
||||||
|
http.Error(w, err.Error(), http.StatusInternalServerError) |
||||||
|
} |
||||||
|
} |
||||||
|
``` |
||||||
|
|
||||||
|
Then you can backup using this command: |
||||||
|
|
||||||
|
```sh |
||||||
|
$ curl http://localhost/backup > my.db |
||||||
|
``` |
||||||
|
|
||||||
|
Or you can open your browser to `http://localhost/backup` and it will download |
||||||
|
automatically. |
||||||
|
|
||||||
|
If you want to backup to another file you can use the `Tx.CopyFile()` helper |
||||||
|
function. |
||||||
|
|
||||||
|
|
||||||
|
### Statistics |
||||||
|
|
||||||
|
The database keeps a running count of many of the internal operations it |
||||||
|
performs so you can better understand what's going on. By grabbing a snapshot |
||||||
|
of these stats at two points in time we can see what operations were performed |
||||||
|
in that time range. |
||||||
|
|
||||||
|
For example, we could start a goroutine to log stats every 10 seconds: |
||||||
|
|
||||||
|
```go |
||||||
|
go func() { |
||||||
|
// Grab the initial stats. |
||||||
|
prev := db.Stats() |
||||||
|
|
||||||
|
for { |
||||||
|
// Wait for 10s. |
||||||
|
time.Sleep(10 * time.Second) |
||||||
|
|
||||||
|
// Grab the current stats and diff them. |
||||||
|
stats := db.Stats() |
||||||
|
diff := stats.Sub(&prev) |
||||||
|
|
||||||
|
// Encode stats to JSON and print to STDERR. |
||||||
|
json.NewEncoder(os.Stderr).Encode(diff) |
||||||
|
|
||||||
|
// Save stats for the next loop. |
||||||
|
prev = stats |
||||||
|
} |
||||||
|
}() |
||||||
|
``` |
||||||
|
|
||||||
|
It's also useful to pipe these stats to a service such as statsd for monitoring |
||||||
|
or to provide an HTTP endpoint that will perform a fixed-length sample. |
||||||
|
|
||||||
|
|
||||||
|
### Read-Only Mode |
||||||
|
|
||||||
|
Sometimes it is useful to create a shared, read-only Bolt database. To this, |
||||||
|
set the `Options.ReadOnly` flag when opening your database. Read-only mode |
||||||
|
uses a shared lock to allow multiple processes to read from the database but |
||||||
|
it will block any processes from opening the database in read-write mode. |
||||||
|
|
||||||
|
```go |
||||||
|
db, err := bolt.Open("my.db", 0666, &bolt.Options{ReadOnly: true}) |
||||||
|
if err != nil { |
||||||
|
log.Fatal(err) |
||||||
|
} |
||||||
|
``` |
||||||
|
|
||||||
|
### Mobile Use (iOS/Android) |
||||||
|
|
||||||
|
Bolt is able to run on mobile devices by leveraging the binding feature of the |
||||||
|
[gomobile](https://github.com/golang/mobile) tool. Create a struct that will |
||||||
|
contain your database logic and a reference to a `*bolt.DB` with a initializing |
||||||
|
constructor that takes in a filepath where the database file will be stored. |
||||||
|
Neither Android nor iOS require extra permissions or cleanup from using this method. |
||||||
|
|
||||||
|
```go |
||||||
|
func NewBoltDB(filepath string) *BoltDB { |
||||||
|
db, err := bolt.Open(filepath+"/demo.db", 0600, nil) |
||||||
|
if err != nil { |
||||||
|
log.Fatal(err) |
||||||
|
} |
||||||
|
|
||||||
|
return &BoltDB{db} |
||||||
|
} |
||||||
|
|
||||||
|
type BoltDB struct { |
||||||
|
db *bolt.DB |
||||||
|
... |
||||||
|
} |
||||||
|
|
||||||
|
func (b *BoltDB) Path() string { |
||||||
|
return b.db.Path() |
||||||
|
} |
||||||
|
|
||||||
|
func (b *BoltDB) Close() { |
||||||
|
b.db.Close() |
||||||
|
} |
||||||
|
``` |
||||||
|
|
||||||
|
Database logic should be defined as methods on this wrapper struct. |
||||||
|
|
||||||
|
To initialize this struct from the native language (both platforms now sync |
||||||
|
their local storage to the cloud. These snippets disable that functionality for the |
||||||
|
database file): |
||||||
|
|
||||||
|
#### Android |
||||||
|
|
||||||
|
```java |
||||||
|
String path; |
||||||
|
if (android.os.Build.VERSION.SDK_INT >=android.os.Build.VERSION_CODES.LOLLIPOP){ |
||||||
|
path = getNoBackupFilesDir().getAbsolutePath(); |
||||||
|
} else{ |
||||||
|
path = getFilesDir().getAbsolutePath(); |
||||||
|
} |
||||||
|
Boltmobiledemo.BoltDB boltDB = Boltmobiledemo.NewBoltDB(path) |
||||||
|
``` |
||||||
|
|
||||||
|
#### iOS |
||||||
|
|
||||||
|
```objc |
||||||
|
- (void)demo { |
||||||
|
NSString* path = [NSSearchPathForDirectoriesInDomains(NSLibraryDirectory, |
||||||
|
NSUserDomainMask, |
||||||
|
YES) objectAtIndex:0]; |
||||||
|
GoBoltmobiledemoBoltDB * demo = GoBoltmobiledemoNewBoltDB(path); |
||||||
|
[self addSkipBackupAttributeToItemAtPath:demo.path]; |
||||||
|
//Some DB Logic would go here |
||||||
|
[demo close]; |
||||||
|
} |
||||||
|
|
||||||
|
- (BOOL)addSkipBackupAttributeToItemAtPath:(NSString *) filePathString |
||||||
|
{ |
||||||
|
NSURL* URL= [NSURL fileURLWithPath: filePathString]; |
||||||
|
assert([[NSFileManager defaultManager] fileExistsAtPath: [URL path]]); |
||||||
|
|
||||||
|
NSError *error = nil; |
||||||
|
BOOL success = [URL setResourceValue: [NSNumber numberWithBool: YES] |
||||||
|
forKey: NSURLIsExcludedFromBackupKey error: &error]; |
||||||
|
if(!success){ |
||||||
|
NSLog(@"Error excluding %@ from backup %@", [URL lastPathComponent], error); |
||||||
|
} |
||||||
|
return success; |
||||||
|
} |
||||||
|
|
||||||
|
``` |
||||||
|
|
||||||
|
## Resources |
||||||
|
|
||||||
|
For more information on getting started with Bolt, check out the following articles: |
||||||
|
|
||||||
|
* [Intro to BoltDB: Painless Performant Persistence](http://npf.io/2014/07/intro-to-boltdb-painless-performant-persistence/) by [Nate Finch](https://github.com/natefinch). |
||||||
|
* [Bolt -- an embedded key/value database for Go](https://www.progville.com/go/bolt-embedded-db-golang/) by Progville |
||||||
|
|
||||||
|
|
||||||
|
## Comparison with other databases |
||||||
|
|
||||||
|
### Postgres, MySQL, & other relational databases |
||||||
|
|
||||||
|
Relational databases structure data into rows and are only accessible through |
||||||
|
the use of SQL. This approach provides flexibility in how you store and query |
||||||
|
your data but also incurs overhead in parsing and planning SQL statements. Bolt |
||||||
|
accesses all data by a byte slice key. This makes Bolt fast to read and write |
||||||
|
data by key but provides no built-in support for joining values together. |
||||||
|
|
||||||
|
Most relational databases (with the exception of SQLite) are standalone servers |
||||||
|
that run separately from your application. This gives your systems |
||||||
|
flexibility to connect multiple application servers to a single database |
||||||
|
server but also adds overhead in serializing and transporting data over the |
||||||
|
network. Bolt runs as a library included in your application so all data access |
||||||
|
has to go through your application's process. This brings data closer to your |
||||||
|
application but limits multi-process access to the data. |
||||||
|
|
||||||
|
|
||||||
|
### LevelDB, RocksDB |
||||||
|
|
||||||
|
LevelDB and its derivatives (RocksDB, HyperLevelDB) are similar to Bolt in that |
||||||
|
they are libraries bundled into the application, however, their underlying |
||||||
|
structure is a log-structured merge-tree (LSM tree). An LSM tree optimizes |
||||||
|
random writes by using a write ahead log and multi-tiered, sorted files called |
||||||
|
SSTables. Bolt uses a B+tree internally and only a single file. Both approaches |
||||||
|
have trade-offs. |
||||||
|
|
||||||
|
If you require a high random write throughput (>10,000 w/sec) or you need to use |
||||||
|
spinning disks then LevelDB could be a good choice. If your application is |
||||||
|
read-heavy or does a lot of range scans then Bolt could be a good choice. |
||||||
|
|
||||||
|
One other important consideration is that LevelDB does not have transactions. |
||||||
|
It supports batch writing of key/values pairs and it supports read snapshots |
||||||
|
but it will not give you the ability to do a compare-and-swap operation safely. |
||||||
|
Bolt supports fully serializable ACID transactions. |
||||||
|
|
||||||
|
|
||||||
|
### LMDB |
||||||
|
|
||||||
|
Bolt was originally a port of LMDB so it is architecturally similar. Both use |
||||||
|
a B+tree, have ACID semantics with fully serializable transactions, and support |
||||||
|
lock-free MVCC using a single writer and multiple readers. |
||||||
|
|
||||||
|
The two projects have somewhat diverged. LMDB heavily focuses on raw performance |
||||||
|
while Bolt has focused on simplicity and ease of use. For example, LMDB allows |
||||||
|
several unsafe actions such as direct writes for the sake of performance. Bolt |
||||||
|
opts to disallow actions which can leave the database in a corrupted state. The |
||||||
|
only exception to this in Bolt is `DB.NoSync`. |
||||||
|
|
||||||
|
There are also a few differences in API. LMDB requires a maximum mmap size when |
||||||
|
opening an `mdb_env` whereas Bolt will handle incremental mmap resizing |
||||||
|
automatically. LMDB overloads the getter and setter functions with multiple |
||||||
|
flags whereas Bolt splits these specialized cases into their own functions. |
||||||
|
|
||||||
|
|
||||||
|
## Caveats & Limitations |
||||||
|
|
||||||
|
It's important to pick the right tool for the job and Bolt is no exception. |
||||||
|
Here are a few things to note when evaluating and using Bolt: |
||||||
|
|
||||||
|
* Bolt is good for read intensive workloads. Sequential write performance is |
||||||
|
also fast but random writes can be slow. You can use `DB.Batch()` or add a |
||||||
|
write-ahead log to help mitigate this issue. |
||||||
|
|
||||||
|
* Bolt uses a B+tree internally so there can be a lot of random page access. |
||||||
|
SSDs provide a significant performance boost over spinning disks. |
||||||
|
|
||||||
|
* Try to avoid long running read transactions. Bolt uses copy-on-write so |
||||||
|
old pages cannot be reclaimed while an old transaction is using them. |
||||||
|
|
||||||
|
* Byte slices returned from Bolt are only valid during a transaction. Once the |
||||||
|
transaction has been committed or rolled back then the memory they point to |
||||||
|
can be reused by a new page or can be unmapped from virtual memory and you'll |
||||||
|
see an `unexpected fault address` panic when accessing it. |
||||||
|
|
||||||
|
* Bolt uses an exclusive write lock on the database file so it cannot be |
||||||
|
shared by multiple processes. |
||||||
|
|
||||||
|
* Be careful when using `Bucket.FillPercent`. Setting a high fill percent for |
||||||
|
buckets that have random inserts will cause your database to have very poor |
||||||
|
page utilization. |
||||||
|
|
||||||
|
* Use larger buckets in general. Smaller buckets causes poor page utilization |
||||||
|
once they become larger than the page size (typically 4KB). |
||||||
|
|
||||||
|
* Bulk loading a lot of random writes into a new bucket can be slow as the |
||||||
|
page will not split until the transaction is committed. Randomly inserting |
||||||
|
more than 100,000 key/value pairs into a single new bucket in a single |
||||||
|
transaction is not advised. |
||||||
|
|
||||||
|
* Bolt uses a memory-mapped file so the underlying operating system handles the |
||||||
|
caching of the data. Typically, the OS will cache as much of the file as it |
||||||
|
can in memory and will release memory as needed to other processes. This means |
||||||
|
that Bolt can show very high memory usage when working with large databases. |
||||||
|
However, this is expected and the OS will release memory as needed. Bolt can |
||||||
|
handle databases much larger than the available physical RAM, provided its |
||||||
|
memory-map fits in the process virtual address space. It may be problematic |
||||||
|
on 32-bits systems. |
||||||
|
|
||||||
|
* The data structures in the Bolt database are memory mapped so the data file |
||||||
|
will be endian specific. This means that you cannot copy a Bolt file from a |
||||||
|
little endian machine to a big endian machine and have it work. For most |
||||||
|
users this is not a concern since most modern CPUs are little endian. |
||||||
|
|
||||||
|
* Because of the way pages are laid out on disk, Bolt cannot truncate data files |
||||||
|
and return free pages back to the disk. Instead, Bolt maintains a free list |
||||||
|
of unused pages within its data file. These free pages can be reused by later |
||||||
|
transactions. This works well for many use cases as databases generally tend |
||||||
|
to grow. However, it's important to note that deleting large chunks of data |
||||||
|
will not allow you to reclaim that space on disk. |
||||||
|
|
||||||
|
For more information on page allocation, [see this comment][page-allocation]. |
||||||
|
|
||||||
|
[page-allocation]: https://github.com/boltdb/bolt/issues/308#issuecomment-74811638 |
||||||
|
|
||||||
|
|
||||||
|
## Reading the Source |
||||||
|
|
||||||
|
Bolt is a relatively small code base (<5KLOC) for an embedded, serializable, |
||||||
|
transactional key/value database so it can be a good starting point for people |
||||||
|
interested in how databases work. |
||||||
|
|
||||||
|
The best places to start are the main entry points into Bolt: |
||||||
|
|
||||||
|
- `Open()` - Initializes the reference to the database. It's responsible for |
||||||
|
creating the database if it doesn't exist, obtaining an exclusive lock on the |
||||||
|
file, reading the meta pages, & memory-mapping the file. |
||||||
|
|
||||||
|
- `DB.Begin()` - Starts a read-only or read-write transaction depending on the |
||||||
|
value of the `writable` argument. This requires briefly obtaining the "meta" |
||||||
|
lock to keep track of open transactions. Only one read-write transaction can |
||||||
|
exist at a time so the "rwlock" is acquired during the life of a read-write |
||||||
|
transaction. |
||||||
|
|
||||||
|
- `Bucket.Put()` - Writes a key/value pair into a bucket. After validating the |
||||||
|
arguments, a cursor is used to traverse the B+tree to the page and position |
||||||
|
where they key & value will be written. Once the position is found, the bucket |
||||||
|
materializes the underlying page and the page's parent pages into memory as |
||||||
|
"nodes". These nodes are where mutations occur during read-write transactions. |
||||||
|
These changes get flushed to disk during commit. |
||||||
|
|
||||||
|
- `Bucket.Get()` - Retrieves a key/value pair from a bucket. This uses a cursor |
||||||
|
to move to the page & position of a key/value pair. During a read-only |
||||||
|
transaction, the key and value data is returned as a direct reference to the |
||||||
|
underlying mmap file so there's no allocation overhead. For read-write |
||||||
|
transactions, this data may reference the mmap file or one of the in-memory |
||||||
|
node values. |
||||||
|
|
||||||
|
- `Cursor` - This object is simply for traversing the B+tree of on-disk pages |
||||||
|
or in-memory nodes. It can seek to a specific key, move to the first or last |
||||||
|
value, or it can move forward or backward. The cursor handles the movement up |
||||||
|
and down the B+tree transparently to the end user. |
||||||
|
|
||||||
|
- `Tx.Commit()` - Converts the in-memory dirty nodes and the list of free pages |
||||||
|
into pages to be written to disk. Writing to disk then occurs in two phases. |
||||||
|
First, the dirty pages are written to disk and an `fsync()` occurs. Second, a |
||||||
|
new meta page with an incremented transaction ID is written and another |
||||||
|
`fsync()` occurs. This two phase write ensures that partially written data |
||||||
|
pages are ignored in the event of a crash since the meta page pointing to them |
||||||
|
is never written. Partially written meta pages are invalidated because they |
||||||
|
are written with a checksum. |
||||||
|
|
||||||
|
If you have additional notes that could be helpful for others, please submit |
||||||
|
them via pull request. |
||||||
|
|
||||||
|
|
||||||
|
## Other Projects Using Bolt |
||||||
|
|
||||||
|
Below is a list of public, open source projects that use Bolt: |
||||||
|
|
||||||
|
* [Algernon](https://github.com/xyproto/algernon) - A HTTP/2 web server with built-in support for Lua. Uses BoltDB as the default database backend. |
||||||
|
* [Bazil](https://bazil.org/) - A file system that lets your data reside where it is most convenient for it to reside. |
||||||
|
* [bolter](https://github.com/hasit/bolter) - Command-line app for viewing BoltDB file in your terminal. |
||||||
|
* [boltcli](https://github.com/spacewander/boltcli) - the redis-cli for boltdb with Lua script support. |
||||||
|
* [BoltHold](https://github.com/timshannon/bolthold) - An embeddable NoSQL store for Go types built on BoltDB |
||||||
|
* [BoltStore](https://github.com/yosssi/boltstore) - Session store using Bolt. |
||||||
|
* [Boltdb Boilerplate](https://github.com/bobintornado/boltdb-boilerplate) - Boilerplate wrapper around bolt aiming to make simple calls one-liners. |
||||||
|
* [BoltDbWeb](https://github.com/evnix/boltdbweb) - A web based GUI for BoltDB files. |
||||||
|
* [bleve](http://www.blevesearch.com/) - A pure Go search engine similar to ElasticSearch that uses Bolt as the default storage backend. |
||||||
|
* [btcwallet](https://github.com/btcsuite/btcwallet) - A bitcoin wallet. |
||||||
|
* [buckets](https://github.com/joyrexus/buckets) - a bolt wrapper streamlining |
||||||
|
simple tx and key scans. |
||||||
|
* [cayley](https://github.com/google/cayley) - Cayley is an open-source graph database using Bolt as optional backend. |
||||||
|
* [ChainStore](https://github.com/pressly/chainstore) - Simple key-value interface to a variety of storage engines organized as a chain of operations. |
||||||
|
* [Consul](https://github.com/hashicorp/consul) - Consul is service discovery and configuration made easy. Distributed, highly available, and datacenter-aware. |
||||||
|
* [DVID](https://github.com/janelia-flyem/dvid) - Added Bolt as optional storage engine and testing it against Basho-tuned leveldb. |
||||||
|
* [dcrwallet](https://github.com/decred/dcrwallet) - A wallet for the Decred cryptocurrency. |
||||||
|
* [drive](https://github.com/odeke-em/drive) - drive is an unofficial Google Drive command line client for \*NIX operating systems. |
||||||
|
* [event-shuttle](https://github.com/sclasen/event-shuttle) - A Unix system service to collect and reliably deliver messages to Kafka. |
||||||
|
* [Freehold](http://tshannon.bitbucket.org/freehold/) - An open, secure, and lightweight platform for your files and data. |
||||||
|
* [Go Report Card](https://goreportcard.com/) - Go code quality report cards as a (free and open source) service. |
||||||
|
* [GoWebApp](https://github.com/josephspurrier/gowebapp) - A basic MVC web application in Go using BoltDB. |
||||||
|
* [GoShort](https://github.com/pankajkhairnar/goShort) - GoShort is a URL shortener written in Golang and BoltDB for persistent key/value storage and for routing it's using high performent HTTPRouter. |
||||||
|
* [gopherpit](https://github.com/gopherpit/gopherpit) - A web service to manage Go remote import paths with custom domains |
||||||
|
* [Gitchain](https://github.com/gitchain/gitchain) - Decentralized, peer-to-peer Git repositories aka "Git meets Bitcoin". |
||||||
|
* [InfluxDB](https://influxdata.com) - Scalable datastore for metrics, events, and real-time analytics. |
||||||
|
* [ipLocator](https://github.com/AndreasBriese/ipLocator) - A fast ip-geo-location-server using bolt with bloom filters. |
||||||
|
* [ipxed](https://github.com/kelseyhightower/ipxed) - Web interface and api for ipxed. |
||||||
|
* [Ironsmith](https://github.com/timshannon/ironsmith) - A simple, script-driven continuous integration (build - > test -> release) tool, with no external dependencies |
||||||
|
* [Kala](https://github.com/ajvb/kala) - Kala is a modern job scheduler optimized to run on a single node. It is persistent, JSON over HTTP API, ISO 8601 duration notation, and dependent jobs. |
||||||
|
* [Key Value Access Langusge (KVAL)](https://github.com/kval-access-language) - A proposed grammar for key-value datastores offering a bbolt binding. |
||||||
|
* [LedisDB](https://github.com/siddontang/ledisdb) - A high performance NoSQL, using Bolt as optional storage. |
||||||
|
* [lru](https://github.com/crowdriff/lru) - Easy to use Bolt-backed Least-Recently-Used (LRU) read-through cache with chainable remote stores. |
||||||
|
* [mbuckets](https://github.com/abhigupta912/mbuckets) - A Bolt wrapper that allows easy operations on multi level (nested) buckets. |
||||||
|
* [MetricBase](https://github.com/msiebuhr/MetricBase) - Single-binary version of Graphite. |
||||||
|
* [MuLiFS](https://github.com/dankomiocevic/mulifs) - Music Library Filesystem creates a filesystem to organise your music files. |
||||||
|
* [Operation Go: A Routine Mission](http://gocode.io) - An online programming game for Golang using Bolt for user accounts and a leaderboard. |
||||||
|
* [photosite/session](https://godoc.org/bitbucket.org/kardianos/photosite/session) - Sessions for a photo viewing site. |
||||||
|
* [Prometheus Annotation Server](https://github.com/oliver006/prom_annotation_server) - Annotation server for PromDash & Prometheus service monitoring system. |
||||||
|
* [reef-pi](https://github.com/reef-pi/reef-pi) - reef-pi is an award winning, modular, DIY reef tank controller using easy to learn electronics based on a Raspberry Pi. |
||||||
|
* [Request Baskets](https://github.com/darklynx/request-baskets) - A web service to collect arbitrary HTTP requests and inspect them via REST API or simple web UI, similar to [RequestBin](http://requestb.in/) service |
||||||
|
* [Seaweed File System](https://github.com/chrislusf/seaweedfs) - Highly scalable distributed key~file system with O(1) disk read. |
||||||
|
* [stow](https://github.com/djherbis/stow) - a persistence manager for objects |
||||||
|
backed by boltdb. |
||||||
|
* [Storm](https://github.com/asdine/storm) - Simple and powerful ORM for BoltDB. |
||||||
|
* [SimpleBolt](https://github.com/xyproto/simplebolt) - A simple way to use BoltDB. Deals mainly with strings. |
||||||
|
* [Skybox Analytics](https://github.com/skybox/skybox) - A standalone funnel analysis tool for web analytics. |
||||||
|
* [Scuttlebutt](https://github.com/benbjohnson/scuttlebutt) - Uses Bolt to store and process all Twitter mentions of GitHub projects. |
||||||
|
* [tentacool](https://github.com/optiflows/tentacool) - REST api server to manage system stuff (IP, DNS, Gateway...) on a linux server. |
||||||
|
* [torrent](https://github.com/anacrolix/torrent) - Full-featured BitTorrent client package and utilities in Go. BoltDB is a storage backend in development. |
||||||
|
* [Wiki](https://github.com/peterhellberg/wiki) - A tiny wiki using Goji, BoltDB and Blackfriday. |
||||||
|
|
||||||
|
If you are using Bolt in a project please send a pull request to add it to the list. |
||||||
@ -0,0 +1,17 @@ |
|||||||
|
# Binaries for programs and plugins |
||||||
|
*.exe |
||||||
|
*.dll |
||||||
|
*.so |
||||||
|
*.dylib |
||||||
|
|
||||||
|
# Test binary, build with `go test -c` |
||||||
|
*.test |
||||||
|
|
||||||
|
# Output of the go coverage tool, specifically when used with LiteIDE |
||||||
|
*.out |
||||||
|
|
||||||
|
# Project-local glide cache, RE: https://github.com/Masterminds/glide/issues/736 |
||||||
|
.glide/ |
||||||
|
|
||||||
|
# vendor dependencies |
||||||
|
vendor/ |
||||||
@ -0,0 +1,14 @@ |
|||||||
|
language: go |
||||||
|
|
||||||
|
go: |
||||||
|
- 1.8.x |
||||||
|
- 1.9.x |
||||||
|
|
||||||
|
before_script: |
||||||
|
- go get -u github.com/golang/lint/golint |
||||||
|
|
||||||
|
script: |
||||||
|
- go vet |
||||||
|
- golint -set_exit_status |
||||||
|
- go build |
||||||
|
- go test |
||||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in new issue