remove lightningpub
This commit is contained in:
parent
14a8af3f16
commit
1e06ae9014
34 changed files with 112 additions and 2211 deletions
72
alby/well-known.go
Normal file
72
alby/well-known.go
Normal file
|
|
@ -0,0 +1,72 @@
|
||||||
|
package alby
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"net"
|
||||||
|
"net/http"
|
||||||
|
|
||||||
|
"git.devvul.com/asara/gologger"
|
||||||
|
"git.devvul.com/asara/well-goknown/config"
|
||||||
|
"github.com/jmoiron/sqlx"
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
DB *sqlx.DB
|
||||||
|
)
|
||||||
|
|
||||||
|
type lnurlp struct {
|
||||||
|
Status string `json:"status"`
|
||||||
|
Tag string `json:"tag"`
|
||||||
|
CommentAllowed int32 `json:"commentAllowed"`
|
||||||
|
Callback string `json:"callback"`
|
||||||
|
MinSendable int64 `json:"minSendable"`
|
||||||
|
MaxSendable int64 `json:"maxSendable"`
|
||||||
|
Metadata string `json:"metadata"`
|
||||||
|
AllowsNostr bool `json:"allowsNostr"`
|
||||||
|
NostrPubkey string `json:"nostrPubkey"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func GetLnurlp(w http.ResponseWriter, r *http.Request) {
|
||||||
|
l := gologger.Get(config.GetConfig().LogLevel).With().Caller().Logger()
|
||||||
|
|
||||||
|
// normalize domain
|
||||||
|
domain, _, err := net.SplitHostPort(r.Host)
|
||||||
|
if err != nil {
|
||||||
|
domain = r.Host
|
||||||
|
}
|
||||||
|
|
||||||
|
name := r.PathValue("name")
|
||||||
|
var lnwallet string
|
||||||
|
err = DB.QueryRow("SELECT wallet FROM lnwallets WHERE name=$1 AND domain=$2", strings.Lower(name), domain).Scan(&lnwallet)
|
||||||
|
if err != nil {
|
||||||
|
l.Debug().Msgf("user (%s@%s) doesn't exist: %s", name, domain, err.Error())
|
||||||
|
http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
lnurlpReturn := &lnurlp{
|
||||||
|
Status: "OK",
|
||||||
|
Tag: "payRequest",
|
||||||
|
CommentAllowed: 255,
|
||||||
|
Callback: fmt.Sprintf("https://%s/.well-known/lnurlp/%s/callback", name),
|
||||||
|
MinSendable: 1000,
|
||||||
|
MaxSendable: 10000000,
|
||||||
|
Metadata: fmt.Sprintf("[[\"text/plain\", \"ln address payment to %s on the devvul server\"],[\"text/identifier\", \"%s@%s\"]]", name, name, domain),
|
||||||
|
AllowNostr: true,
|
||||||
|
NostrPubkey: lnwallet,
|
||||||
|
}
|
||||||
|
|
||||||
|
ret, err := json.Marshal(lnurlpReturn)
|
||||||
|
if err != nil {
|
||||||
|
l.Debug().Msgf("unable to marshal json for %s@%s: %s", name, domain, err.Error())
|
||||||
|
http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
l.Debug().Msgf("returning lnwallet for %s@%s", name, domain)
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
w.WriteHeader(http.StatusOK)
|
||||||
|
w.Write(ret)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
@ -17,6 +17,7 @@ type (
|
||||||
RelayDescription string
|
RelayDescription string
|
||||||
RelayIcon string
|
RelayIcon string
|
||||||
RelayContact string
|
RelayContact string
|
||||||
|
AlbyAdminAuth string
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
@ -33,6 +34,7 @@ func GetConfig() Config {
|
||||||
RelayDescription: getEnv("RELAY_DESCRIPTION", ""),
|
RelayDescription: getEnv("RELAY_DESCRIPTION", ""),
|
||||||
RelayIcon: getEnv("RELAY_ICON", ""),
|
RelayIcon: getEnv("RELAY_ICON", ""),
|
||||||
RelayContact: getEnv("RELAY_CONTACT", ""),
|
RelayContact: getEnv("RELAY_CONTACT", ""),
|
||||||
|
AlbyAdminAuth: getEnv("ALBY_ADMIN_AUTH", ""),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
4
go.mod
4
go.mod
|
|
@ -5,10 +5,10 @@ go 1.23.3
|
||||||
require (
|
require (
|
||||||
git.devvul.com/asara/gologger v0.9.0
|
git.devvul.com/asara/gologger v0.9.0
|
||||||
github.com/fiatjaf/eventstore v0.14.0
|
github.com/fiatjaf/eventstore v0.14.0
|
||||||
github.com/fiatjaf/khatru v0.12.0
|
github.com/fiatjaf/khatru v0.12.1
|
||||||
github.com/jmoiron/sqlx v1.4.0
|
github.com/jmoiron/sqlx v1.4.0
|
||||||
github.com/lib/pq v1.10.9
|
github.com/lib/pq v1.10.9
|
||||||
github.com/nbd-wtf/go-nostr v0.42.2
|
github.com/nbd-wtf/go-nostr v0.42.3
|
||||||
)
|
)
|
||||||
|
|
||||||
require (
|
require (
|
||||||
|
|
|
||||||
4
go.sum
4
go.sum
|
|
@ -28,6 +28,8 @@ github.com/fiatjaf/eventstore v0.14.0 h1:eAyugJGFRCrXYJLCc2nC/BIApmBbQN/Z4dxvNz1
|
||||||
github.com/fiatjaf/eventstore v0.14.0/go.mod h1:XOl5B6WGBX1a0ww6s3WT94QVOmye/6zDTtyWHVtHQ5U=
|
github.com/fiatjaf/eventstore v0.14.0/go.mod h1:XOl5B6WGBX1a0ww6s3WT94QVOmye/6zDTtyWHVtHQ5U=
|
||||||
github.com/fiatjaf/khatru v0.12.0 h1:pOWyahXl9UoyFTj/tX4Y3eM8nqGRHwMqM4F8ed7O3A0=
|
github.com/fiatjaf/khatru v0.12.0 h1:pOWyahXl9UoyFTj/tX4Y3eM8nqGRHwMqM4F8ed7O3A0=
|
||||||
github.com/fiatjaf/khatru v0.12.0/go.mod h1:GfKKAR27sMxBmepv709QnL7C9lEmlhaj41LFm/ueATc=
|
github.com/fiatjaf/khatru v0.12.0/go.mod h1:GfKKAR27sMxBmepv709QnL7C9lEmlhaj41LFm/ueATc=
|
||||||
|
github.com/fiatjaf/khatru v0.12.1 h1:J7GlQy/Be0nAXH9JdS9jVMv2JdwLQhSu7TK3ZbiFZh4=
|
||||||
|
github.com/fiatjaf/khatru v0.12.1/go.mod h1:GfKKAR27sMxBmepv709QnL7C9lEmlhaj41LFm/ueATc=
|
||||||
github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y=
|
github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y=
|
||||||
github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg=
|
github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg=
|
||||||
github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU=
|
github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU=
|
||||||
|
|
@ -59,6 +61,8 @@ github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o
|
||||||
github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
|
github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
|
||||||
github.com/nbd-wtf/go-nostr v0.42.2 h1:X8vpfLutvmyxqjsroKPHdIyPliNa6sYD8+CA0kDVySw=
|
github.com/nbd-wtf/go-nostr v0.42.2 h1:X8vpfLutvmyxqjsroKPHdIyPliNa6sYD8+CA0kDVySw=
|
||||||
github.com/nbd-wtf/go-nostr v0.42.2/go.mod h1:FBa4FBJO7NuANvkeKSlrf0BIyxGufmrUbuelr6Q4Ick=
|
github.com/nbd-wtf/go-nostr v0.42.2/go.mod h1:FBa4FBJO7NuANvkeKSlrf0BIyxGufmrUbuelr6Q4Ick=
|
||||||
|
github.com/nbd-wtf/go-nostr v0.42.3 h1:wimwmXLhF9ScrNTG4by3eSj2p7HUGkLUospX4bHjxQk=
|
||||||
|
github.com/nbd-wtf/go-nostr v0.42.3/go.mod h1:p29g9i1UiSBKdyXkNa6V8rFqE+wrIn4UY0Emabwdu6A=
|
||||||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||||
|
|
|
||||||
|
|
@ -1,87 +0,0 @@
|
||||||
package lightningpub
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/json"
|
|
||||||
"fmt"
|
|
||||||
"io/ioutil"
|
|
||||||
"net"
|
|
||||||
"net/http"
|
|
||||||
|
|
||||||
"git.devvul.com/asara/gologger"
|
|
||||||
"git.devvul.com/asara/well-goknown/config"
|
|
||||||
"github.com/jmoiron/sqlx"
|
|
||||||
)
|
|
||||||
|
|
||||||
var (
|
|
||||||
DB *sqlx.DB
|
|
||||||
)
|
|
||||||
|
|
||||||
type lnurlp struct {
|
|
||||||
Tag string `json:"tag"`
|
|
||||||
Callback string `json:"callback"`
|
|
||||||
MaxSendable int64 `json:"maxSendable"`
|
|
||||||
MinSendable int64 `json:"minSendable"`
|
|
||||||
Metadata string `json:"metadata"`
|
|
||||||
AllowsNostr bool `json:"allowsNostr"`
|
|
||||||
NostrPubkey string `json:"nostrPubkey"`
|
|
||||||
}
|
|
||||||
|
|
||||||
func GetLnurlp(w http.ResponseWriter, r *http.Request) {
|
|
||||||
l := gologger.Get(config.GetConfig().LogLevel).With().Caller().Logger()
|
|
||||||
|
|
||||||
// normalize domain
|
|
||||||
domain, _, err := net.SplitHostPort(r.Host)
|
|
||||||
if err != nil {
|
|
||||||
domain = r.Host
|
|
||||||
}
|
|
||||||
|
|
||||||
name := r.PathValue("name")
|
|
||||||
var lnwallet string
|
|
||||||
err = DB.QueryRow("SELECT wallet FROM lnwallets WHERE name=$1 AND domain=$2", name, domain).Scan(&lnwallet)
|
|
||||||
if err != nil {
|
|
||||||
l.Debug().Msgf("user (%s@%s) doesn't exist: %s", name, domain, err.Error())
|
|
||||||
http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
//upstreamUrl := fmt.Sprintf("https://%s/api/guest/lnurl_pay/info?k1=%s", domain, lnwallet)
|
|
||||||
upstreamUrl := fmt.Sprintf("https://%s/api/guest/lnurl_pay/info?k1=%s", domain, lnwallet)
|
|
||||||
upstreamPayload, err := http.Get(upstreamUrl)
|
|
||||||
if err != nil {
|
|
||||||
l.Debug().Msgf("user (%s@%s) doesn't exist: %s", name, domain, err.Error())
|
|
||||||
http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
defer upstreamPayload.Body.Close()
|
|
||||||
body, err := ioutil.ReadAll(upstreamPayload.Body)
|
|
||||||
if err != nil {
|
|
||||||
l.Debug().Msgf("user (%s@%s) doesn't exist: %s", name, domain, err.Error())
|
|
||||||
http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
lnurlpReturn := lnurlp{}
|
|
||||||
err = json.Unmarshal(body, &lnurlpReturn)
|
|
||||||
if err != nil {
|
|
||||||
l.Debug().Msgf("user (%s@%s) doesn't exist: %s", name, domain, err.Error())
|
|
||||||
http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
m := fmt.Sprintf("[[\"text/plain\", \"ln address payment to a user on the devvul server\"],[\"text/identifier\", \"%s@%s\"]]", name, domain)
|
|
||||||
lnurlpReturn.Metadata = m
|
|
||||||
|
|
||||||
ret, err := json.Marshal(lnurlpReturn)
|
|
||||||
if err != nil {
|
|
||||||
l.Debug().Msgf("user (%s@%s) doesn't exist: %s", name, domain, err.Error())
|
|
||||||
http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
l.Debug().Msgf("returning lnwallet for %s@%s", name, domain)
|
|
||||||
w.Header().Set("Content-Type", "application/json")
|
|
||||||
w.WriteHeader(http.StatusOK)
|
|
||||||
w.Write(ret)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
6
main.go
6
main.go
|
|
@ -6,7 +6,6 @@ import (
|
||||||
"git.devvul.com/asara/gologger"
|
"git.devvul.com/asara/gologger"
|
||||||
"git.devvul.com/asara/well-goknown/config"
|
"git.devvul.com/asara/well-goknown/config"
|
||||||
"git.devvul.com/asara/well-goknown/db"
|
"git.devvul.com/asara/well-goknown/db"
|
||||||
"git.devvul.com/asara/well-goknown/lightningpub"
|
|
||||||
"git.devvul.com/asara/well-goknown/matrix"
|
"git.devvul.com/asara/well-goknown/matrix"
|
||||||
"git.devvul.com/asara/well-goknown/nostr"
|
"git.devvul.com/asara/well-goknown/nostr"
|
||||||
"github.com/fiatjaf/eventstore/postgresql"
|
"github.com/fiatjaf/eventstore/postgresql"
|
||||||
|
|
@ -22,7 +21,6 @@ func main() {
|
||||||
db, _ := db.NewDB()
|
db, _ := db.NewDB()
|
||||||
defer db.Close()
|
defer db.Close()
|
||||||
|
|
||||||
lightningpub.DB = db
|
|
||||||
nostr.DB = db
|
nostr.DB = db
|
||||||
nostr.RelayDb = postgresql.PostgresBackend{DatabaseURL: config.GetConfig().DbUrl}
|
nostr.RelayDb = postgresql.PostgresBackend{DatabaseURL: config.GetConfig().DbUrl}
|
||||||
if err := nostr.RelayDb.Init(); err != nil {
|
if err := nostr.RelayDb.Init(); err != nil {
|
||||||
|
|
@ -41,8 +39,8 @@ func main() {
|
||||||
http.Handle("/relay", relay)
|
http.Handle("/relay", relay)
|
||||||
|
|
||||||
// lnurlp endpoint
|
// lnurlp endpoint
|
||||||
l.Debug().Msg("enabling lnurlp well-known endpoint")
|
//l.Debug().Msg("enabling lnurlp well-known endpoint")
|
||||||
http.HandleFunc("/.well-known/lnurlp/{name}", lightningpub.GetLnurlp)
|
//http.HandleFunc("/.well-known/lnurlp/{name}", alby.GetLnurlp)
|
||||||
|
|
||||||
// start server
|
// start server
|
||||||
port := config.GetConfig().ListenAddr
|
port := config.GetConfig().ListenAddr
|
||||||
|
|
|
||||||
|
|
@ -15,12 +15,16 @@ func RejectUnregisteredNpubs(ctx context.Context, event *nostr.Event) (reject bo
|
||||||
|
|
||||||
// always allow the following kinds
|
// always allow the following kinds
|
||||||
// 13: nip-59 seals
|
// 13: nip-59 seals
|
||||||
|
// 9734: nip-57 zap request
|
||||||
// 9735: nip-57 zap receipt
|
// 9735: nip-57 zap receipt
|
||||||
// 21000: lightning.pub rpc
|
// 13194: nip-47 info event
|
||||||
// 22242: nip-42 client auth
|
// 22242: nip-42 client auth
|
||||||
|
// 23194: nip-47 request
|
||||||
|
// 23195: nip-47 response
|
||||||
|
// 23196: nip-47 notification
|
||||||
// 30078: nip-78 addressable events
|
// 30078: nip-78 addressable events
|
||||||
switch event.Kind {
|
switch event.Kind {
|
||||||
case 13, 9735, 21000, 22242, 30078:
|
case 13, 9734, 9735, 13194, 22242, 23194, 23195, 23196, 30078:
|
||||||
return false, ""
|
return false, ""
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -8,3 +8,4 @@ export RELAY_DESCRIPTION="nostr relay running via git.devvul.com/asara/well-gokn
|
||||||
export RELAY_ICON=""
|
export RELAY_ICON=""
|
||||||
export RELAY_NAME="Nostr Relay"
|
export RELAY_NAME="Nostr Relay"
|
||||||
export RELAY_PUBKEY=""
|
export RELAY_PUBKEY=""
|
||||||
|
export ALBY_ADMIN_AUTH=""
|
||||||
|
|
|
||||||
22
vendor/github.com/cespare/xxhash/LICENSE.txt
generated
vendored
22
vendor/github.com/cespare/xxhash/LICENSE.txt
generated
vendored
|
|
@ -1,22 +0,0 @@
|
||||||
Copyright (c) 2016 Caleb Spare
|
|
||||||
|
|
||||||
MIT License
|
|
||||||
|
|
||||||
Permission is hereby granted, free of charge, to any person obtaining
|
|
||||||
a copy of this software and associated documentation files (the
|
|
||||||
"Software"), to deal in the Software without restriction, including
|
|
||||||
without limitation the rights to use, copy, modify, merge, publish,
|
|
||||||
distribute, sublicense, and/or sell copies of the Software, and to
|
|
||||||
permit persons to whom the Software is furnished to do so, subject to
|
|
||||||
the following conditions:
|
|
||||||
|
|
||||||
The above copyright notice and this permission notice shall be
|
|
||||||
included in all copies or substantial portions of the Software.
|
|
||||||
|
|
||||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
|
||||||
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
|
||||||
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
|
||||||
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
|
||||||
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
|
||||||
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
|
||||||
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
|
||||||
50
vendor/github.com/cespare/xxhash/README.md
generated
vendored
50
vendor/github.com/cespare/xxhash/README.md
generated
vendored
|
|
@ -1,50 +0,0 @@
|
||||||
# xxhash
|
|
||||||
|
|
||||||
[](https://godoc.org/github.com/cespare/xxhash)
|
|
||||||
|
|
||||||
xxhash is a Go implementation of the 64-bit
|
|
||||||
[xxHash](http://cyan4973.github.io/xxHash/) algorithm, XXH64. This is a
|
|
||||||
high-quality hashing algorithm that is much faster than anything in the Go
|
|
||||||
standard library.
|
|
||||||
|
|
||||||
The API is very small, taking its cue from the other hashing packages in the
|
|
||||||
standard library:
|
|
||||||
|
|
||||||
$ go doc github.com/cespare/xxhash !
|
|
||||||
package xxhash // import "github.com/cespare/xxhash"
|
|
||||||
|
|
||||||
Package xxhash implements the 64-bit variant of xxHash (XXH64) as described
|
|
||||||
at http://cyan4973.github.io/xxHash/.
|
|
||||||
|
|
||||||
func New() hash.Hash64
|
|
||||||
func Sum64(b []byte) uint64
|
|
||||||
func Sum64String(s string) uint64
|
|
||||||
|
|
||||||
This implementation provides a fast pure-Go implementation and an even faster
|
|
||||||
assembly implementation for amd64.
|
|
||||||
|
|
||||||
## Benchmarks
|
|
||||||
|
|
||||||
Here are some quick benchmarks comparing the pure-Go and assembly
|
|
||||||
implementations of Sum64 against another popular Go XXH64 implementation,
|
|
||||||
[github.com/OneOfOne/xxhash](https://github.com/OneOfOne/xxhash):
|
|
||||||
|
|
||||||
| input size | OneOfOne | cespare (purego) | cespare |
|
|
||||||
| --- | --- | --- | --- |
|
|
||||||
| 5 B | 416 MB/s | 720 MB/s | 872 MB/s |
|
|
||||||
| 100 B | 3980 MB/s | 5013 MB/s | 5252 MB/s |
|
|
||||||
| 4 KB | 12727 MB/s | 12999 MB/s | 13026 MB/s |
|
|
||||||
| 10 MB | 9879 MB/s | 10775 MB/s | 10913 MB/s |
|
|
||||||
|
|
||||||
These numbers were generated with:
|
|
||||||
|
|
||||||
```
|
|
||||||
$ go test -benchtime 10s -bench '/OneOfOne,'
|
|
||||||
$ go test -tags purego -benchtime 10s -bench '/xxhash,'
|
|
||||||
$ go test -benchtime 10s -bench '/xxhash,'
|
|
||||||
```
|
|
||||||
|
|
||||||
## Projects using this package
|
|
||||||
|
|
||||||
- [InfluxDB](https://github.com/influxdata/influxdb)
|
|
||||||
- [Prometheus](https://github.com/prometheus/prometheus)
|
|
||||||
14
vendor/github.com/cespare/xxhash/rotate.go
generated
vendored
14
vendor/github.com/cespare/xxhash/rotate.go
generated
vendored
|
|
@ -1,14 +0,0 @@
|
||||||
// +build !go1.9
|
|
||||||
|
|
||||||
package xxhash
|
|
||||||
|
|
||||||
// TODO(caleb): After Go 1.10 comes out, remove this fallback code.
|
|
||||||
|
|
||||||
func rol1(x uint64) uint64 { return (x << 1) | (x >> (64 - 1)) }
|
|
||||||
func rol7(x uint64) uint64 { return (x << 7) | (x >> (64 - 7)) }
|
|
||||||
func rol11(x uint64) uint64 { return (x << 11) | (x >> (64 - 11)) }
|
|
||||||
func rol12(x uint64) uint64 { return (x << 12) | (x >> (64 - 12)) }
|
|
||||||
func rol18(x uint64) uint64 { return (x << 18) | (x >> (64 - 18)) }
|
|
||||||
func rol23(x uint64) uint64 { return (x << 23) | (x >> (64 - 23)) }
|
|
||||||
func rol27(x uint64) uint64 { return (x << 27) | (x >> (64 - 27)) }
|
|
||||||
func rol31(x uint64) uint64 { return (x << 31) | (x >> (64 - 31)) }
|
|
||||||
14
vendor/github.com/cespare/xxhash/rotate19.go
generated
vendored
14
vendor/github.com/cespare/xxhash/rotate19.go
generated
vendored
|
|
@ -1,14 +0,0 @@
|
||||||
// +build go1.9
|
|
||||||
|
|
||||||
package xxhash
|
|
||||||
|
|
||||||
import "math/bits"
|
|
||||||
|
|
||||||
func rol1(x uint64) uint64 { return bits.RotateLeft64(x, 1) }
|
|
||||||
func rol7(x uint64) uint64 { return bits.RotateLeft64(x, 7) }
|
|
||||||
func rol11(x uint64) uint64 { return bits.RotateLeft64(x, 11) }
|
|
||||||
func rol12(x uint64) uint64 { return bits.RotateLeft64(x, 12) }
|
|
||||||
func rol18(x uint64) uint64 { return bits.RotateLeft64(x, 18) }
|
|
||||||
func rol23(x uint64) uint64 { return bits.RotateLeft64(x, 23) }
|
|
||||||
func rol27(x uint64) uint64 { return bits.RotateLeft64(x, 27) }
|
|
||||||
func rol31(x uint64) uint64 { return bits.RotateLeft64(x, 31) }
|
|
||||||
168
vendor/github.com/cespare/xxhash/xxhash.go
generated
vendored
168
vendor/github.com/cespare/xxhash/xxhash.go
generated
vendored
|
|
@ -1,168 +0,0 @@
|
||||||
// Package xxhash implements the 64-bit variant of xxHash (XXH64) as described
|
|
||||||
// at http://cyan4973.github.io/xxHash/.
|
|
||||||
package xxhash
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/binary"
|
|
||||||
"hash"
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
prime1 uint64 = 11400714785074694791
|
|
||||||
prime2 uint64 = 14029467366897019727
|
|
||||||
prime3 uint64 = 1609587929392839161
|
|
||||||
prime4 uint64 = 9650029242287828579
|
|
||||||
prime5 uint64 = 2870177450012600261
|
|
||||||
)
|
|
||||||
|
|
||||||
// NOTE(caleb): I'm using both consts and vars of the primes. Using consts where
|
|
||||||
// possible in the Go code is worth a small (but measurable) performance boost
|
|
||||||
// by avoiding some MOVQs. Vars are needed for the asm and also are useful for
|
|
||||||
// convenience in the Go code in a few places where we need to intentionally
|
|
||||||
// avoid constant arithmetic (e.g., v1 := prime1 + prime2 fails because the
|
|
||||||
// result overflows a uint64).
|
|
||||||
var (
|
|
||||||
prime1v = prime1
|
|
||||||
prime2v = prime2
|
|
||||||
prime3v = prime3
|
|
||||||
prime4v = prime4
|
|
||||||
prime5v = prime5
|
|
||||||
)
|
|
||||||
|
|
||||||
type xxh struct {
|
|
||||||
v1 uint64
|
|
||||||
v2 uint64
|
|
||||||
v3 uint64
|
|
||||||
v4 uint64
|
|
||||||
total int
|
|
||||||
mem [32]byte
|
|
||||||
n int // how much of mem is used
|
|
||||||
}
|
|
||||||
|
|
||||||
// New creates a new hash.Hash64 that implements the 64-bit xxHash algorithm.
|
|
||||||
func New() hash.Hash64 {
|
|
||||||
var x xxh
|
|
||||||
x.Reset()
|
|
||||||
return &x
|
|
||||||
}
|
|
||||||
|
|
||||||
func (x *xxh) Reset() {
|
|
||||||
x.n = 0
|
|
||||||
x.total = 0
|
|
||||||
x.v1 = prime1v + prime2
|
|
||||||
x.v2 = prime2
|
|
||||||
x.v3 = 0
|
|
||||||
x.v4 = -prime1v
|
|
||||||
}
|
|
||||||
|
|
||||||
func (x *xxh) Size() int { return 8 }
|
|
||||||
func (x *xxh) BlockSize() int { return 32 }
|
|
||||||
|
|
||||||
// Write adds more data to x. It always returns len(b), nil.
|
|
||||||
func (x *xxh) Write(b []byte) (n int, err error) {
|
|
||||||
n = len(b)
|
|
||||||
x.total += len(b)
|
|
||||||
|
|
||||||
if x.n+len(b) < 32 {
|
|
||||||
// This new data doesn't even fill the current block.
|
|
||||||
copy(x.mem[x.n:], b)
|
|
||||||
x.n += len(b)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if x.n > 0 {
|
|
||||||
// Finish off the partial block.
|
|
||||||
copy(x.mem[x.n:], b)
|
|
||||||
x.v1 = round(x.v1, u64(x.mem[0:8]))
|
|
||||||
x.v2 = round(x.v2, u64(x.mem[8:16]))
|
|
||||||
x.v3 = round(x.v3, u64(x.mem[16:24]))
|
|
||||||
x.v4 = round(x.v4, u64(x.mem[24:32]))
|
|
||||||
b = b[32-x.n:]
|
|
||||||
x.n = 0
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(b) >= 32 {
|
|
||||||
// One or more full blocks left.
|
|
||||||
b = writeBlocks(x, b)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Store any remaining partial block.
|
|
||||||
copy(x.mem[:], b)
|
|
||||||
x.n = len(b)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
func (x *xxh) Sum(b []byte) []byte {
|
|
||||||
s := x.Sum64()
|
|
||||||
return append(
|
|
||||||
b,
|
|
||||||
byte(s>>56),
|
|
||||||
byte(s>>48),
|
|
||||||
byte(s>>40),
|
|
||||||
byte(s>>32),
|
|
||||||
byte(s>>24),
|
|
||||||
byte(s>>16),
|
|
||||||
byte(s>>8),
|
|
||||||
byte(s),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (x *xxh) Sum64() uint64 {
|
|
||||||
var h uint64
|
|
||||||
|
|
||||||
if x.total >= 32 {
|
|
||||||
v1, v2, v3, v4 := x.v1, x.v2, x.v3, x.v4
|
|
||||||
h = rol1(v1) + rol7(v2) + rol12(v3) + rol18(v4)
|
|
||||||
h = mergeRound(h, v1)
|
|
||||||
h = mergeRound(h, v2)
|
|
||||||
h = mergeRound(h, v3)
|
|
||||||
h = mergeRound(h, v4)
|
|
||||||
} else {
|
|
||||||
h = x.v3 + prime5
|
|
||||||
}
|
|
||||||
|
|
||||||
h += uint64(x.total)
|
|
||||||
|
|
||||||
i, end := 0, x.n
|
|
||||||
for ; i+8 <= end; i += 8 {
|
|
||||||
k1 := round(0, u64(x.mem[i:i+8]))
|
|
||||||
h ^= k1
|
|
||||||
h = rol27(h)*prime1 + prime4
|
|
||||||
}
|
|
||||||
if i+4 <= end {
|
|
||||||
h ^= uint64(u32(x.mem[i:i+4])) * prime1
|
|
||||||
h = rol23(h)*prime2 + prime3
|
|
||||||
i += 4
|
|
||||||
}
|
|
||||||
for i < end {
|
|
||||||
h ^= uint64(x.mem[i]) * prime5
|
|
||||||
h = rol11(h) * prime1
|
|
||||||
i++
|
|
||||||
}
|
|
||||||
|
|
||||||
h ^= h >> 33
|
|
||||||
h *= prime2
|
|
||||||
h ^= h >> 29
|
|
||||||
h *= prime3
|
|
||||||
h ^= h >> 32
|
|
||||||
|
|
||||||
return h
|
|
||||||
}
|
|
||||||
|
|
||||||
func u64(b []byte) uint64 { return binary.LittleEndian.Uint64(b) }
|
|
||||||
func u32(b []byte) uint32 { return binary.LittleEndian.Uint32(b) }
|
|
||||||
|
|
||||||
func round(acc, input uint64) uint64 {
|
|
||||||
acc += input * prime2
|
|
||||||
acc = rol31(acc)
|
|
||||||
acc *= prime1
|
|
||||||
return acc
|
|
||||||
}
|
|
||||||
|
|
||||||
func mergeRound(acc, val uint64) uint64 {
|
|
||||||
val = round(0, val)
|
|
||||||
acc ^= val
|
|
||||||
acc = acc*prime1 + prime4
|
|
||||||
return acc
|
|
||||||
}
|
|
||||||
12
vendor/github.com/cespare/xxhash/xxhash_amd64.go
generated
vendored
12
vendor/github.com/cespare/xxhash/xxhash_amd64.go
generated
vendored
|
|
@ -1,12 +0,0 @@
|
||||||
// +build !appengine
|
|
||||||
// +build gc
|
|
||||||
// +build !purego
|
|
||||||
|
|
||||||
package xxhash
|
|
||||||
|
|
||||||
// Sum64 computes the 64-bit xxHash digest of b.
|
|
||||||
//
|
|
||||||
//go:noescape
|
|
||||||
func Sum64(b []byte) uint64
|
|
||||||
|
|
||||||
func writeBlocks(x *xxh, b []byte) []byte
|
|
||||||
233
vendor/github.com/cespare/xxhash/xxhash_amd64.s
generated
vendored
233
vendor/github.com/cespare/xxhash/xxhash_amd64.s
generated
vendored
|
|
@ -1,233 +0,0 @@
|
||||||
// +build !appengine
|
|
||||||
// +build gc
|
|
||||||
// +build !purego
|
|
||||||
|
|
||||||
#include "textflag.h"
|
|
||||||
|
|
||||||
// Register allocation:
|
|
||||||
// AX h
|
|
||||||
// CX pointer to advance through b
|
|
||||||
// DX n
|
|
||||||
// BX loop end
|
|
||||||
// R8 v1, k1
|
|
||||||
// R9 v2
|
|
||||||
// R10 v3
|
|
||||||
// R11 v4
|
|
||||||
// R12 tmp
|
|
||||||
// R13 prime1v
|
|
||||||
// R14 prime2v
|
|
||||||
// R15 prime4v
|
|
||||||
|
|
||||||
// round reads from and advances the buffer pointer in CX.
|
|
||||||
// It assumes that R13 has prime1v and R14 has prime2v.
|
|
||||||
#define round(r) \
|
|
||||||
MOVQ (CX), R12 \
|
|
||||||
ADDQ $8, CX \
|
|
||||||
IMULQ R14, R12 \
|
|
||||||
ADDQ R12, r \
|
|
||||||
ROLQ $31, r \
|
|
||||||
IMULQ R13, r
|
|
||||||
|
|
||||||
// mergeRound applies a merge round on the two registers acc and val.
|
|
||||||
// It assumes that R13 has prime1v, R14 has prime2v, and R15 has prime4v.
|
|
||||||
#define mergeRound(acc, val) \
|
|
||||||
IMULQ R14, val \
|
|
||||||
ROLQ $31, val \
|
|
||||||
IMULQ R13, val \
|
|
||||||
XORQ val, acc \
|
|
||||||
IMULQ R13, acc \
|
|
||||||
ADDQ R15, acc
|
|
||||||
|
|
||||||
// func Sum64(b []byte) uint64
|
|
||||||
TEXT ·Sum64(SB), NOSPLIT, $0-32
|
|
||||||
// Load fixed primes.
|
|
||||||
MOVQ ·prime1v(SB), R13
|
|
||||||
MOVQ ·prime2v(SB), R14
|
|
||||||
MOVQ ·prime4v(SB), R15
|
|
||||||
|
|
||||||
// Load slice.
|
|
||||||
MOVQ b_base+0(FP), CX
|
|
||||||
MOVQ b_len+8(FP), DX
|
|
||||||
LEAQ (CX)(DX*1), BX
|
|
||||||
|
|
||||||
// The first loop limit will be len(b)-32.
|
|
||||||
SUBQ $32, BX
|
|
||||||
|
|
||||||
// Check whether we have at least one block.
|
|
||||||
CMPQ DX, $32
|
|
||||||
JLT noBlocks
|
|
||||||
|
|
||||||
// Set up initial state (v1, v2, v3, v4).
|
|
||||||
MOVQ R13, R8
|
|
||||||
ADDQ R14, R8
|
|
||||||
MOVQ R14, R9
|
|
||||||
XORQ R10, R10
|
|
||||||
XORQ R11, R11
|
|
||||||
SUBQ R13, R11
|
|
||||||
|
|
||||||
// Loop until CX > BX.
|
|
||||||
blockLoop:
|
|
||||||
round(R8)
|
|
||||||
round(R9)
|
|
||||||
round(R10)
|
|
||||||
round(R11)
|
|
||||||
|
|
||||||
CMPQ CX, BX
|
|
||||||
JLE blockLoop
|
|
||||||
|
|
||||||
MOVQ R8, AX
|
|
||||||
ROLQ $1, AX
|
|
||||||
MOVQ R9, R12
|
|
||||||
ROLQ $7, R12
|
|
||||||
ADDQ R12, AX
|
|
||||||
MOVQ R10, R12
|
|
||||||
ROLQ $12, R12
|
|
||||||
ADDQ R12, AX
|
|
||||||
MOVQ R11, R12
|
|
||||||
ROLQ $18, R12
|
|
||||||
ADDQ R12, AX
|
|
||||||
|
|
||||||
mergeRound(AX, R8)
|
|
||||||
mergeRound(AX, R9)
|
|
||||||
mergeRound(AX, R10)
|
|
||||||
mergeRound(AX, R11)
|
|
||||||
|
|
||||||
JMP afterBlocks
|
|
||||||
|
|
||||||
noBlocks:
|
|
||||||
MOVQ ·prime5v(SB), AX
|
|
||||||
|
|
||||||
afterBlocks:
|
|
||||||
ADDQ DX, AX
|
|
||||||
|
|
||||||
// Right now BX has len(b)-32, and we want to loop until CX > len(b)-8.
|
|
||||||
ADDQ $24, BX
|
|
||||||
|
|
||||||
CMPQ CX, BX
|
|
||||||
JG fourByte
|
|
||||||
|
|
||||||
wordLoop:
|
|
||||||
// Calculate k1.
|
|
||||||
MOVQ (CX), R8
|
|
||||||
ADDQ $8, CX
|
|
||||||
IMULQ R14, R8
|
|
||||||
ROLQ $31, R8
|
|
||||||
IMULQ R13, R8
|
|
||||||
|
|
||||||
XORQ R8, AX
|
|
||||||
ROLQ $27, AX
|
|
||||||
IMULQ R13, AX
|
|
||||||
ADDQ R15, AX
|
|
||||||
|
|
||||||
CMPQ CX, BX
|
|
||||||
JLE wordLoop
|
|
||||||
|
|
||||||
fourByte:
|
|
||||||
ADDQ $4, BX
|
|
||||||
CMPQ CX, BX
|
|
||||||
JG singles
|
|
||||||
|
|
||||||
MOVL (CX), R8
|
|
||||||
ADDQ $4, CX
|
|
||||||
IMULQ R13, R8
|
|
||||||
XORQ R8, AX
|
|
||||||
|
|
||||||
ROLQ $23, AX
|
|
||||||
IMULQ R14, AX
|
|
||||||
ADDQ ·prime3v(SB), AX
|
|
||||||
|
|
||||||
singles:
|
|
||||||
ADDQ $4, BX
|
|
||||||
CMPQ CX, BX
|
|
||||||
JGE finalize
|
|
||||||
|
|
||||||
singlesLoop:
|
|
||||||
MOVBQZX (CX), R12
|
|
||||||
ADDQ $1, CX
|
|
||||||
IMULQ ·prime5v(SB), R12
|
|
||||||
XORQ R12, AX
|
|
||||||
|
|
||||||
ROLQ $11, AX
|
|
||||||
IMULQ R13, AX
|
|
||||||
|
|
||||||
CMPQ CX, BX
|
|
||||||
JL singlesLoop
|
|
||||||
|
|
||||||
finalize:
|
|
||||||
MOVQ AX, R12
|
|
||||||
SHRQ $33, R12
|
|
||||||
XORQ R12, AX
|
|
||||||
IMULQ R14, AX
|
|
||||||
MOVQ AX, R12
|
|
||||||
SHRQ $29, R12
|
|
||||||
XORQ R12, AX
|
|
||||||
IMULQ ·prime3v(SB), AX
|
|
||||||
MOVQ AX, R12
|
|
||||||
SHRQ $32, R12
|
|
||||||
XORQ R12, AX
|
|
||||||
|
|
||||||
MOVQ AX, ret+24(FP)
|
|
||||||
RET
|
|
||||||
|
|
||||||
// writeBlocks uses the same registers as above except that it uses AX to store
|
|
||||||
// the x pointer.
|
|
||||||
|
|
||||||
// func writeBlocks(x *xxh, b []byte) []byte
|
|
||||||
TEXT ·writeBlocks(SB), NOSPLIT, $0-56
|
|
||||||
// Load fixed primes needed for round.
|
|
||||||
MOVQ ·prime1v(SB), R13
|
|
||||||
MOVQ ·prime2v(SB), R14
|
|
||||||
|
|
||||||
// Load slice.
|
|
||||||
MOVQ b_base+8(FP), CX
|
|
||||||
MOVQ CX, ret_base+32(FP) // initialize return base pointer; see NOTE below
|
|
||||||
MOVQ b_len+16(FP), DX
|
|
||||||
LEAQ (CX)(DX*1), BX
|
|
||||||
SUBQ $32, BX
|
|
||||||
|
|
||||||
// Load vN from x.
|
|
||||||
MOVQ x+0(FP), AX
|
|
||||||
MOVQ 0(AX), R8 // v1
|
|
||||||
MOVQ 8(AX), R9 // v2
|
|
||||||
MOVQ 16(AX), R10 // v3
|
|
||||||
MOVQ 24(AX), R11 // v4
|
|
||||||
|
|
||||||
// We don't need to check the loop condition here; this function is
|
|
||||||
// always called with at least one block of data to process.
|
|
||||||
blockLoop:
|
|
||||||
round(R8)
|
|
||||||
round(R9)
|
|
||||||
round(R10)
|
|
||||||
round(R11)
|
|
||||||
|
|
||||||
CMPQ CX, BX
|
|
||||||
JLE blockLoop
|
|
||||||
|
|
||||||
// Copy vN back to x.
|
|
||||||
MOVQ R8, 0(AX)
|
|
||||||
MOVQ R9, 8(AX)
|
|
||||||
MOVQ R10, 16(AX)
|
|
||||||
MOVQ R11, 24(AX)
|
|
||||||
|
|
||||||
// Construct return slice.
|
|
||||||
// NOTE: It's important that we don't construct a slice that has a base
|
|
||||||
// pointer off the end of the original slice, as in Go 1.7+ this will
|
|
||||||
// cause runtime crashes. (See discussion in, for example,
|
|
||||||
// https://github.com/golang/go/issues/16772.)
|
|
||||||
// Therefore, we calculate the length/cap first, and if they're zero, we
|
|
||||||
// keep the old base. This is what the compiler does as well if you
|
|
||||||
// write code like
|
|
||||||
// b = b[len(b):]
|
|
||||||
|
|
||||||
// New length is 32 - (CX - BX) -> BX+32 - CX.
|
|
||||||
ADDQ $32, BX
|
|
||||||
SUBQ CX, BX
|
|
||||||
JZ afterSetBase
|
|
||||||
|
|
||||||
MOVQ CX, ret_base+32(FP)
|
|
||||||
|
|
||||||
afterSetBase:
|
|
||||||
MOVQ BX, ret_len+40(FP)
|
|
||||||
MOVQ BX, ret_cap+48(FP) // set cap == len
|
|
||||||
|
|
||||||
RET
|
|
||||||
75
vendor/github.com/cespare/xxhash/xxhash_other.go
generated
vendored
75
vendor/github.com/cespare/xxhash/xxhash_other.go
generated
vendored
|
|
@ -1,75 +0,0 @@
|
||||||
// +build !amd64 appengine !gc purego
|
|
||||||
|
|
||||||
package xxhash
|
|
||||||
|
|
||||||
// Sum64 computes the 64-bit xxHash digest of b.
|
|
||||||
func Sum64(b []byte) uint64 {
|
|
||||||
// A simpler version would be
|
|
||||||
// x := New()
|
|
||||||
// x.Write(b)
|
|
||||||
// return x.Sum64()
|
|
||||||
// but this is faster, particularly for small inputs.
|
|
||||||
|
|
||||||
n := len(b)
|
|
||||||
var h uint64
|
|
||||||
|
|
||||||
if n >= 32 {
|
|
||||||
v1 := prime1v + prime2
|
|
||||||
v2 := prime2
|
|
||||||
v3 := uint64(0)
|
|
||||||
v4 := -prime1v
|
|
||||||
for len(b) >= 32 {
|
|
||||||
v1 = round(v1, u64(b[0:8:len(b)]))
|
|
||||||
v2 = round(v2, u64(b[8:16:len(b)]))
|
|
||||||
v3 = round(v3, u64(b[16:24:len(b)]))
|
|
||||||
v4 = round(v4, u64(b[24:32:len(b)]))
|
|
||||||
b = b[32:len(b):len(b)]
|
|
||||||
}
|
|
||||||
h = rol1(v1) + rol7(v2) + rol12(v3) + rol18(v4)
|
|
||||||
h = mergeRound(h, v1)
|
|
||||||
h = mergeRound(h, v2)
|
|
||||||
h = mergeRound(h, v3)
|
|
||||||
h = mergeRound(h, v4)
|
|
||||||
} else {
|
|
||||||
h = prime5
|
|
||||||
}
|
|
||||||
|
|
||||||
h += uint64(n)
|
|
||||||
|
|
||||||
i, end := 0, len(b)
|
|
||||||
for ; i+8 <= end; i += 8 {
|
|
||||||
k1 := round(0, u64(b[i:i+8:len(b)]))
|
|
||||||
h ^= k1
|
|
||||||
h = rol27(h)*prime1 + prime4
|
|
||||||
}
|
|
||||||
if i+4 <= end {
|
|
||||||
h ^= uint64(u32(b[i:i+4:len(b)])) * prime1
|
|
||||||
h = rol23(h)*prime2 + prime3
|
|
||||||
i += 4
|
|
||||||
}
|
|
||||||
for ; i < end; i++ {
|
|
||||||
h ^= uint64(b[i]) * prime5
|
|
||||||
h = rol11(h) * prime1
|
|
||||||
}
|
|
||||||
|
|
||||||
h ^= h >> 33
|
|
||||||
h *= prime2
|
|
||||||
h ^= h >> 29
|
|
||||||
h *= prime3
|
|
||||||
h ^= h >> 32
|
|
||||||
|
|
||||||
return h
|
|
||||||
}
|
|
||||||
|
|
||||||
func writeBlocks(x *xxh, b []byte) []byte {
|
|
||||||
v1, v2, v3, v4 := x.v1, x.v2, x.v3, x.v4
|
|
||||||
for len(b) >= 32 {
|
|
||||||
v1 = round(v1, u64(b[0:8:len(b)]))
|
|
||||||
v2 = round(v2, u64(b[8:16:len(b)]))
|
|
||||||
v3 = round(v3, u64(b[16:24:len(b)]))
|
|
||||||
v4 = round(v4, u64(b[24:32:len(b)]))
|
|
||||||
b = b[32:len(b):len(b)]
|
|
||||||
}
|
|
||||||
x.v1, x.v2, x.v3, x.v4 = v1, v2, v3, v4
|
|
||||||
return b
|
|
||||||
}
|
|
||||||
10
vendor/github.com/cespare/xxhash/xxhash_safe.go
generated
vendored
10
vendor/github.com/cespare/xxhash/xxhash_safe.go
generated
vendored
|
|
@ -1,10 +0,0 @@
|
||||||
// +build appengine
|
|
||||||
|
|
||||||
// This file contains the safe implementations of otherwise unsafe-using code.
|
|
||||||
|
|
||||||
package xxhash
|
|
||||||
|
|
||||||
// Sum64String computes the 64-bit xxHash digest of s.
|
|
||||||
func Sum64String(s string) uint64 {
|
|
||||||
return Sum64([]byte(s))
|
|
||||||
}
|
|
||||||
30
vendor/github.com/cespare/xxhash/xxhash_unsafe.go
generated
vendored
30
vendor/github.com/cespare/xxhash/xxhash_unsafe.go
generated
vendored
|
|
@ -1,30 +0,0 @@
|
||||||
// +build !appengine
|
|
||||||
|
|
||||||
// This file encapsulates usage of unsafe.
|
|
||||||
// xxhash_safe.go contains the safe implementations.
|
|
||||||
|
|
||||||
package xxhash
|
|
||||||
|
|
||||||
import (
|
|
||||||
"reflect"
|
|
||||||
"unsafe"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Sum64String computes the 64-bit xxHash digest of s.
|
|
||||||
// It may be faster than Sum64([]byte(s)) by avoiding a copy.
|
|
||||||
//
|
|
||||||
// TODO(caleb): Consider removing this if an optimization is ever added to make
|
|
||||||
// it unnecessary: https://golang.org/issue/2205.
|
|
||||||
//
|
|
||||||
// TODO(caleb): We still have a function call; we could instead write Go/asm
|
|
||||||
// copies of Sum64 for strings to squeeze out a bit more speed.
|
|
||||||
func Sum64String(s string) uint64 {
|
|
||||||
// See https://groups.google.com/d/msg/golang-nuts/dcjzJy-bSpw/tcZYBzQqAQAJ
|
|
||||||
// for some discussion about this unsafe conversion.
|
|
||||||
var b []byte
|
|
||||||
bh := (*reflect.SliceHeader)(unsafe.Pointer(&b))
|
|
||||||
bh.Data = (*reflect.StringHeader)(unsafe.Pointer(&s)).Data
|
|
||||||
bh.Len = len(s)
|
|
||||||
bh.Cap = len(s)
|
|
||||||
return Sum64(b)
|
|
||||||
}
|
|
||||||
14
vendor/github.com/fiatjaf/khatru/nip86.go
generated
vendored
14
vendor/github.com/fiatjaf/khatru/nip86.go
generated
vendored
|
|
@ -65,16 +65,22 @@ func (rl *Relay) HandleNIP86(w http.ResponseWriter, r *http.Request) {
|
||||||
resp.Error = "missing auth"
|
resp.Error = "missing auth"
|
||||||
goto respond
|
goto respond
|
||||||
}
|
}
|
||||||
if evtj, err := base64.StdEncoding.DecodeString(spl[1]); err != nil {
|
|
||||||
|
evtj, err := base64.StdEncoding.DecodeString(spl[1])
|
||||||
|
if err != nil {
|
||||||
resp.Error = "invalid base64 auth"
|
resp.Error = "invalid base64 auth"
|
||||||
goto respond
|
goto respond
|
||||||
} else if err := json.Unmarshal(evtj, &evt); err != nil {
|
}
|
||||||
|
if err := json.Unmarshal(evtj, &evt); err != nil {
|
||||||
resp.Error = "invalid auth event json"
|
resp.Error = "invalid auth event json"
|
||||||
goto respond
|
goto respond
|
||||||
} else if ok, _ := evt.CheckSignature(); !ok {
|
}
|
||||||
|
if ok, _ := evt.CheckSignature(); !ok {
|
||||||
resp.Error = "invalid auth event"
|
resp.Error = "invalid auth event"
|
||||||
goto respond
|
goto respond
|
||||||
} else if uTag := evt.Tags.GetFirst([]string{"u", ""}); uTag == nil || getServiceBaseURL(r) != (*uTag)[1] {
|
}
|
||||||
|
|
||||||
|
if uTag := evt.Tags.GetFirst([]string{"u", ""}); uTag == nil || rl.ServiceURL != (*uTag)[1] {
|
||||||
resp.Error = "invalid 'u' tag"
|
resp.Error = "invalid 'u' tag"
|
||||||
goto respond
|
goto respond
|
||||||
} else if pht := evt.Tags.GetFirst([]string{"payload", hex.EncodeToString(payloadHash[:])}); pht == nil {
|
} else if pht := evt.Tags.GetFirst([]string{"payload", hex.EncodeToString(payloadHash[:])}); pht == nil {
|
||||||
|
|
|
||||||
2
vendor/github.com/greatroar/blobloom/.gitattributes
generated
vendored
2
vendor/github.com/greatroar/blobloom/.gitattributes
generated
vendored
|
|
@ -1,2 +0,0 @@
|
||||||
# Work around https://github.com/golang/go/issues/52268.
|
|
||||||
**/testdata/fuzz/*/* eol=lf
|
|
||||||
25
vendor/github.com/greatroar/blobloom/.golangci.yml
generated
vendored
25
vendor/github.com/greatroar/blobloom/.golangci.yml
generated
vendored
|
|
@ -1,25 +0,0 @@
|
||||||
# Configuration for golangci-lint.
|
|
||||||
|
|
||||||
linters:
|
|
||||||
disable:
|
|
||||||
- asciicheck
|
|
||||||
enable:
|
|
||||||
- gocognit
|
|
||||||
- gocyclo
|
|
||||||
- godot
|
|
||||||
- gofumpt
|
|
||||||
- lll
|
|
||||||
- misspell
|
|
||||||
- nakedret
|
|
||||||
- thelper
|
|
||||||
|
|
||||||
issues:
|
|
||||||
exclude-rules:
|
|
||||||
- path: _test\.go
|
|
||||||
linters:
|
|
||||||
errcheck
|
|
||||||
|
|
||||||
linters-settings:
|
|
||||||
govet:
|
|
||||||
enable:
|
|
||||||
- atomicalign
|
|
||||||
202
vendor/github.com/greatroar/blobloom/LICENSE
generated
vendored
202
vendor/github.com/greatroar/blobloom/LICENSE
generated
vendored
|
|
@ -1,202 +0,0 @@
|
||||||
|
|
||||||
Apache License
|
|
||||||
Version 2.0, January 2004
|
|
||||||
http://www.apache.org/licenses/
|
|
||||||
|
|
||||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
|
||||||
|
|
||||||
1. Definitions.
|
|
||||||
|
|
||||||
"License" shall mean the terms and conditions for use, reproduction,
|
|
||||||
and distribution as defined by Sections 1 through 9 of this document.
|
|
||||||
|
|
||||||
"Licensor" shall mean the copyright owner or entity authorized by
|
|
||||||
the copyright owner that is granting the License.
|
|
||||||
|
|
||||||
"Legal Entity" shall mean the union of the acting entity and all
|
|
||||||
other entities that control, are controlled by, or are under common
|
|
||||||
control with that entity. For the purposes of this definition,
|
|
||||||
"control" means (i) the power, direct or indirect, to cause the
|
|
||||||
direction or management of such entity, whether by contract or
|
|
||||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
|
||||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
|
||||||
|
|
||||||
"You" (or "Your") shall mean an individual or Legal Entity
|
|
||||||
exercising permissions granted by this License.
|
|
||||||
|
|
||||||
"Source" form shall mean the preferred form for making modifications,
|
|
||||||
including but not limited to software source code, documentation
|
|
||||||
source, and configuration files.
|
|
||||||
|
|
||||||
"Object" form shall mean any form resulting from mechanical
|
|
||||||
transformation or translation of a Source form, including but
|
|
||||||
not limited to compiled object code, generated documentation,
|
|
||||||
and conversions to other media types.
|
|
||||||
|
|
||||||
"Work" shall mean the work of authorship, whether in Source or
|
|
||||||
Object form, made available under the License, as indicated by a
|
|
||||||
copyright notice that is included in or attached to the work
|
|
||||||
(an example is provided in the Appendix below).
|
|
||||||
|
|
||||||
"Derivative Works" shall mean any work, whether in Source or Object
|
|
||||||
form, that is based on (or derived from) the Work and for which the
|
|
||||||
editorial revisions, annotations, elaborations, or other modifications
|
|
||||||
represent, as a whole, an original work of authorship. For the purposes
|
|
||||||
of this License, Derivative Works shall not include works that remain
|
|
||||||
separable from, or merely link (or bind by name) to the interfaces of,
|
|
||||||
the Work and Derivative Works thereof.
|
|
||||||
|
|
||||||
"Contribution" shall mean any work of authorship, including
|
|
||||||
the original version of the Work and any modifications or additions
|
|
||||||
to that Work or Derivative Works thereof, that is intentionally
|
|
||||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
|
||||||
or by an individual or Legal Entity authorized to submit on behalf of
|
|
||||||
the copyright owner. For the purposes of this definition, "submitted"
|
|
||||||
means any form of electronic, verbal, or written communication sent
|
|
||||||
to the Licensor or its representatives, including but not limited to
|
|
||||||
communication on electronic mailing lists, source code control systems,
|
|
||||||
and issue tracking systems that are managed by, or on behalf of, the
|
|
||||||
Licensor for the purpose of discussing and improving the Work, but
|
|
||||||
excluding communication that is conspicuously marked or otherwise
|
|
||||||
designated in writing by the copyright owner as "Not a Contribution."
|
|
||||||
|
|
||||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
|
||||||
on behalf of whom a Contribution has been received by Licensor and
|
|
||||||
subsequently incorporated within the Work.
|
|
||||||
|
|
||||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
|
||||||
this License, each Contributor hereby grants to You a perpetual,
|
|
||||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
||||||
copyright license to reproduce, prepare Derivative Works of,
|
|
||||||
publicly display, publicly perform, sublicense, and distribute the
|
|
||||||
Work and such Derivative Works in Source or Object form.
|
|
||||||
|
|
||||||
3. Grant of Patent License. Subject to the terms and conditions of
|
|
||||||
this License, each Contributor hereby grants to You a perpetual,
|
|
||||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
||||||
(except as stated in this section) patent license to make, have made,
|
|
||||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
|
||||||
where such license applies only to those patent claims licensable
|
|
||||||
by such Contributor that are necessarily infringed by their
|
|
||||||
Contribution(s) alone or by combination of their Contribution(s)
|
|
||||||
with the Work to which such Contribution(s) was submitted. If You
|
|
||||||
institute patent litigation against any entity (including a
|
|
||||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
|
||||||
or a Contribution incorporated within the Work constitutes direct
|
|
||||||
or contributory patent infringement, then any patent licenses
|
|
||||||
granted to You under this License for that Work shall terminate
|
|
||||||
as of the date such litigation is filed.
|
|
||||||
|
|
||||||
4. Redistribution. You may reproduce and distribute copies of the
|
|
||||||
Work or Derivative Works thereof in any medium, with or without
|
|
||||||
modifications, and in Source or Object form, provided that You
|
|
||||||
meet the following conditions:
|
|
||||||
|
|
||||||
(a) You must give any other recipients of the Work or
|
|
||||||
Derivative Works a copy of this License; and
|
|
||||||
|
|
||||||
(b) You must cause any modified files to carry prominent notices
|
|
||||||
stating that You changed the files; and
|
|
||||||
|
|
||||||
(c) You must retain, in the Source form of any Derivative Works
|
|
||||||
that You distribute, all copyright, patent, trademark, and
|
|
||||||
attribution notices from the Source form of the Work,
|
|
||||||
excluding those notices that do not pertain to any part of
|
|
||||||
the Derivative Works; and
|
|
||||||
|
|
||||||
(d) If the Work includes a "NOTICE" text file as part of its
|
|
||||||
distribution, then any Derivative Works that You distribute must
|
|
||||||
include a readable copy of the attribution notices contained
|
|
||||||
within such NOTICE file, excluding those notices that do not
|
|
||||||
pertain to any part of the Derivative Works, in at least one
|
|
||||||
of the following places: within a NOTICE text file distributed
|
|
||||||
as part of the Derivative Works; within the Source form or
|
|
||||||
documentation, if provided along with the Derivative Works; or,
|
|
||||||
within a display generated by the Derivative Works, if and
|
|
||||||
wherever such third-party notices normally appear. The contents
|
|
||||||
of the NOTICE file are for informational purposes only and
|
|
||||||
do not modify the License. You may add Your own attribution
|
|
||||||
notices within Derivative Works that You distribute, alongside
|
|
||||||
or as an addendum to the NOTICE text from the Work, provided
|
|
||||||
that such additional attribution notices cannot be construed
|
|
||||||
as modifying the License.
|
|
||||||
|
|
||||||
You may add Your own copyright statement to Your modifications and
|
|
||||||
may provide additional or different license terms and conditions
|
|
||||||
for use, reproduction, or distribution of Your modifications, or
|
|
||||||
for any such Derivative Works as a whole, provided Your use,
|
|
||||||
reproduction, and distribution of the Work otherwise complies with
|
|
||||||
the conditions stated in this License.
|
|
||||||
|
|
||||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
|
||||||
any Contribution intentionally submitted for inclusion in the Work
|
|
||||||
by You to the Licensor shall be under the terms and conditions of
|
|
||||||
this License, without any additional terms or conditions.
|
|
||||||
Notwithstanding the above, nothing herein shall supersede or modify
|
|
||||||
the terms of any separate license agreement you may have executed
|
|
||||||
with Licensor regarding such Contributions.
|
|
||||||
|
|
||||||
6. Trademarks. This License does not grant permission to use the trade
|
|
||||||
names, trademarks, service marks, or product names of the Licensor,
|
|
||||||
except as required for reasonable and customary use in describing the
|
|
||||||
origin of the Work and reproducing the content of the NOTICE file.
|
|
||||||
|
|
||||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
|
||||||
agreed to in writing, Licensor provides the Work (and each
|
|
||||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
|
||||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
||||||
implied, including, without limitation, any warranties or conditions
|
|
||||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
|
||||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
|
||||||
appropriateness of using or redistributing the Work and assume any
|
|
||||||
risks associated with Your exercise of permissions under this License.
|
|
||||||
|
|
||||||
8. Limitation of Liability. In no event and under no legal theory,
|
|
||||||
whether in tort (including negligence), contract, or otherwise,
|
|
||||||
unless required by applicable law (such as deliberate and grossly
|
|
||||||
negligent acts) or agreed to in writing, shall any Contributor be
|
|
||||||
liable to You for damages, including any direct, indirect, special,
|
|
||||||
incidental, or consequential damages of any character arising as a
|
|
||||||
result of this License or out of the use or inability to use the
|
|
||||||
Work (including but not limited to damages for loss of goodwill,
|
|
||||||
work stoppage, computer failure or malfunction, or any and all
|
|
||||||
other commercial damages or losses), even if such Contributor
|
|
||||||
has been advised of the possibility of such damages.
|
|
||||||
|
|
||||||
9. Accepting Warranty or Additional Liability. While redistributing
|
|
||||||
the Work or Derivative Works thereof, You may choose to offer,
|
|
||||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
|
||||||
or other liability obligations and/or rights consistent with this
|
|
||||||
License. However, in accepting such obligations, You may act only
|
|
||||||
on Your own behalf and on Your sole responsibility, not on behalf
|
|
||||||
of any other Contributor, and only if You agree to indemnify,
|
|
||||||
defend, and hold each Contributor harmless for any liability
|
|
||||||
incurred by, or claims asserted against, such Contributor by reason
|
|
||||||
of your accepting any such warranty or additional liability.
|
|
||||||
|
|
||||||
END OF TERMS AND CONDITIONS
|
|
||||||
|
|
||||||
APPENDIX: How to apply the Apache License to your work.
|
|
||||||
|
|
||||||
To apply the Apache License to your work, attach the following
|
|
||||||
boilerplate notice, with the fields enclosed by brackets "[]"
|
|
||||||
replaced with your own identifying information. (Don't include
|
|
||||||
the brackets!) The text should be enclosed in the appropriate
|
|
||||||
comment syntax for the file format. We also recommend that a
|
|
||||||
file or class name and description of purpose be included on the
|
|
||||||
same "printed page" as the copyright notice for easier
|
|
||||||
identification within third-party archives.
|
|
||||||
|
|
||||||
Copyright [yyyy] [name of copyright owner]
|
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
you may not use this file except in compliance with the License.
|
|
||||||
You may obtain a copy of the License at
|
|
||||||
|
|
||||||
http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
|
|
||||||
Unless required by applicable law or agreed to in writing, software
|
|
||||||
distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
See the License for the specific language governing permissions and
|
|
||||||
limitations under the License.
|
|
||||||
86
vendor/github.com/greatroar/blobloom/README.md
generated
vendored
86
vendor/github.com/greatroar/blobloom/README.md
generated
vendored
|
|
@ -1,86 +0,0 @@
|
||||||
Blobloom
|
|
||||||
========
|
|
||||||
|
|
||||||
A Bloom filter package for Go (golang) with no compile-time dependencies.
|
|
||||||
|
|
||||||
This package implements a version of Bloom filters called [blocked Bloom filters](
|
|
||||||
https://algo2.iti.kit.edu/documents/cacheefficientbloomfilters-jea.pdf),
|
|
||||||
which get a speed boost from using the CPU cache more efficiently
|
|
||||||
than regular Bloom filters.
|
|
||||||
|
|
||||||
Unlike most Bloom filter packages for Go,
|
|
||||||
this one doesn't run a hash function for you.
|
|
||||||
That's a benefit if you need a custom hash
|
|
||||||
or you want pick the fastest one for an application.
|
|
||||||
|
|
||||||
Usage
|
|
||||||
-----
|
|
||||||
|
|
||||||
To construct a Bloom filter, you need to know how many keys you want to store
|
|
||||||
and what rate of false positives you find acceptable.
|
|
||||||
|
|
||||||
f := blobloom.NewOptimized(blobloom.Config{
|
|
||||||
Capacity: nkeys, // Expected number of keys.
|
|
||||||
FPRate: 1e-4, // Accept one false positive per 10,000 lookups.
|
|
||||||
})
|
|
||||||
|
|
||||||
To add a key:
|
|
||||||
|
|
||||||
// import "github.com/cespare/xxhash/v2"
|
|
||||||
f.Add(xxhash.Sum64(key))
|
|
||||||
|
|
||||||
To test for the presence of a key in the filter:
|
|
||||||
|
|
||||||
if f.Has(xxhash.Sum64(key)) {
|
|
||||||
// Key is probably in f.
|
|
||||||
} else {
|
|
||||||
// Key is certainly not in f.
|
|
||||||
}
|
|
||||||
|
|
||||||
The false positive rate is defined as usual:
|
|
||||||
if you look up 10,000 random keys in a Bloom filter filled to capacity,
|
|
||||||
an expected one of those is a false positive for FPRate 1e-4.
|
|
||||||
|
|
||||||
See the examples/ directory and the
|
|
||||||
[package documentation](https://pkg.go.dev/github.com/greatroar/blobloom)
|
|
||||||
for further usage information and examples.
|
|
||||||
|
|
||||||
Hash functions
|
|
||||||
--------------
|
|
||||||
|
|
||||||
Blobloom does not provide hash functions. Instead, it requires client code to
|
|
||||||
represent each key as a single 64-bit hash value, leaving it to the user to
|
|
||||||
pick the right hash function for a particular problem. Here are some general
|
|
||||||
suggestions:
|
|
||||||
|
|
||||||
* If you use Bloom filters to speed up access to a key-value store, you might
|
|
||||||
want to look at [xxh3](https://github.com/zeebo/xxh3) or [xxhash](
|
|
||||||
https://github.com/cespare/xxhash).
|
|
||||||
* If your keys are cryptographic hashes, consider using the first 8 bytes of those hashes.
|
|
||||||
* If you use Bloom filters to make probabilistic decisions, a randomized hash
|
|
||||||
function such as [maphash](https://golang.org/pkg/hash/maphash) should prevent
|
|
||||||
the same false positives occurring every time.
|
|
||||||
|
|
||||||
When evaluating a hash function, or designing a custom one,
|
|
||||||
make sure it is a 64-bit hash that properly mixes its input bits.
|
|
||||||
Casting a 32-bit hash to uint64 gives suboptimal results.
|
|
||||||
So does passing integer keys in without running them through a mixing function.
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
License
|
|
||||||
-------
|
|
||||||
|
|
||||||
Copyright © 2020-2023 the Blobloom authors
|
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
you may not use this file except in compliance with the License.
|
|
||||||
You may obtain a copy of the License at
|
|
||||||
|
|
||||||
http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
|
|
||||||
Unless required by applicable law or agreed to in writing, software
|
|
||||||
distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
See the License for the specific language governing permissions and
|
|
||||||
limitations under the License.
|
|
||||||
279
vendor/github.com/greatroar/blobloom/bloomfilter.go
generated
vendored
279
vendor/github.com/greatroar/blobloom/bloomfilter.go
generated
vendored
|
|
@ -1,279 +0,0 @@
|
||||||
// Copyright 2020-2022 the Blobloom authors
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
// Package blobloom implements blocked Bloom filters.
|
|
||||||
//
|
|
||||||
// Blocked Bloom filters are an approximate set data structure: if a key has
|
|
||||||
// been added to a filter, a lookup of that key returns true, but if the key
|
|
||||||
// has not been added, there is a non-zero probability that the lookup still
|
|
||||||
// returns true (a false positive). False negatives are impossible: if the
|
|
||||||
// lookup for a key returns false, that key has not been added.
|
|
||||||
//
|
|
||||||
// In this package, keys are represented exclusively as hashes. Client code
|
|
||||||
// is responsible for supplying a 64-bit hash value.
|
|
||||||
//
|
|
||||||
// Compared to standard Bloom filters, blocked Bloom filters use the CPU
|
|
||||||
// cache more efficiently. A blocked Bloom filter is an array of ordinary
|
|
||||||
// Bloom filters of fixed size BlockBits (the blocks). The lower half of the
|
|
||||||
// hash selects the block to use.
|
|
||||||
//
|
|
||||||
// To achieve the same false positive rate (FPR) as a standard Bloom filter,
|
|
||||||
// a blocked Bloom filter requires more memory. For an FPR of at most 2e-6
|
|
||||||
// (two in a million), it uses ~20% more memory. At 1e-10, the space required
|
|
||||||
// is double that of standard Bloom filter.
|
|
||||||
//
|
|
||||||
// For more details, see the 2010 paper by Putze, Sanders and Singler,
|
|
||||||
// https://algo2.iti.kit.edu/documents/cacheefficientbloomfilters-jea.pdf.
|
|
||||||
package blobloom
|
|
||||||
|
|
||||||
import "math"
|
|
||||||
|
|
||||||
// BlockBits is the number of bits per block and the minimum number of bits
|
|
||||||
// in a Filter.
|
|
||||||
//
|
|
||||||
// The value of this constant is chosen to match the L1 cache line size
|
|
||||||
// of popular architectures (386, amd64, arm64).
|
|
||||||
const BlockBits = 512
|
|
||||||
|
|
||||||
// MaxBits is the maximum number of bits supported by a Filter.
|
|
||||||
const MaxBits = BlockBits << 32 // 256GiB.
|
|
||||||
|
|
||||||
// A Filter is a blocked Bloom filter.
|
|
||||||
type Filter struct {
|
|
||||||
b []block // Shards.
|
|
||||||
k int // Number of hash functions required.
|
|
||||||
}
|
|
||||||
|
|
||||||
// New constructs a Bloom filter with given numbers of bits and hash functions.
|
|
||||||
//
|
|
||||||
// The number of bits should be at least BlockBits; smaller values are silently
|
|
||||||
// increased.
|
|
||||||
//
|
|
||||||
// The number of hashes reflects the number of hashes synthesized from the
|
|
||||||
// single hash passed in by the client. It is silently increased to two if
|
|
||||||
// a lower value is given.
|
|
||||||
func New(nbits uint64, nhashes int) *Filter {
|
|
||||||
nbits, nhashes = fixBitsAndHashes(nbits, nhashes)
|
|
||||||
|
|
||||||
return &Filter{
|
|
||||||
b: make([]block, nbits/BlockBits),
|
|
||||||
k: nhashes,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func fixBitsAndHashes(nbits uint64, nhashes int) (uint64, int) {
|
|
||||||
if nbits < 1 {
|
|
||||||
nbits = BlockBits
|
|
||||||
}
|
|
||||||
if nhashes < 2 {
|
|
||||||
nhashes = 2
|
|
||||||
}
|
|
||||||
if nbits > MaxBits {
|
|
||||||
panic("nbits exceeds MaxBits")
|
|
||||||
}
|
|
||||||
|
|
||||||
// Round nbits up to a multiple of BlockBits.
|
|
||||||
if nbits%BlockBits != 0 {
|
|
||||||
nbits += BlockBits - nbits%BlockBits
|
|
||||||
}
|
|
||||||
|
|
||||||
return nbits, nhashes
|
|
||||||
}
|
|
||||||
|
|
||||||
// Add insert a key with hash value h into f.
|
|
||||||
func (f *Filter) Add(h uint64) {
|
|
||||||
h1, h2 := uint32(h>>32), uint32(h)
|
|
||||||
b := getblock(f.b, h2)
|
|
||||||
|
|
||||||
for i := 1; i < f.k; i++ {
|
|
||||||
h1, h2 = doublehash(h1, h2, i)
|
|
||||||
b.setbit(h1)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// log(1 - 1/BlockBits) computed with 128 bits precision.
|
|
||||||
// Note that this is extremely close to -1/BlockBits,
|
|
||||||
// which is what Wikipedia would have us use:
|
|
||||||
// https://en.wikipedia.org/wiki/Bloom_filter#Approximating_the_number_of_items_in_a_Bloom_filter.
|
|
||||||
const log1minus1divBlockbits = -0.0019550348358033505576274922418668121377
|
|
||||||
|
|
||||||
// Cardinality estimates the number of distinct keys added to f.
|
|
||||||
//
|
|
||||||
// The estimate is most reliable when f is filled to roughly its capacity.
|
|
||||||
// It gets worse as f gets more densely filled. When one of the blocks is
|
|
||||||
// entirely filled, the estimate becomes +Inf.
|
|
||||||
//
|
|
||||||
// The return value is the maximum likelihood estimate of Papapetrou, Siberski
|
|
||||||
// and Nejdl, summed over the blocks
|
|
||||||
// (https://www.win.tue.nl/~opapapetrou/papers/Bloomfilters-DAPD.pdf).
|
|
||||||
func (f *Filter) Cardinality() float64 {
|
|
||||||
return cardinality(f.k, f.b, onescount)
|
|
||||||
}
|
|
||||||
|
|
||||||
func cardinality(nhashes int, b []block, onescount func(*block) int) float64 {
|
|
||||||
k := float64(nhashes - 1)
|
|
||||||
|
|
||||||
// The probability of some bit not being set in a single insertion is
|
|
||||||
// p0 = (1-1/BlockBits)^k.
|
|
||||||
//
|
|
||||||
// logProb0Inv = 1 / log(p0) = 1 / (k*log(1-1/BlockBits)).
|
|
||||||
logProb0Inv := 1 / (k * log1minus1divBlockbits)
|
|
||||||
|
|
||||||
var n float64
|
|
||||||
for i := range b {
|
|
||||||
ones := onescount(&b[i])
|
|
||||||
if ones == 0 {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
n += math.Log1p(-float64(ones) / BlockBits)
|
|
||||||
}
|
|
||||||
return n * logProb0Inv
|
|
||||||
}
|
|
||||||
|
|
||||||
// Clear resets f to its empty state.
|
|
||||||
func (f *Filter) Clear() {
|
|
||||||
for i := 0; i < len(f.b); i++ {
|
|
||||||
f.b[i] = block{}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Empty reports whether f contains no keys.
|
|
||||||
func (f *Filter) Empty() bool {
|
|
||||||
for i := 0; i < len(f.b); i++ {
|
|
||||||
if f.b[i] != (block{}) {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
// Equals returns true if f and g contain the same keys (in terms of Has)
|
|
||||||
// when used with the same hash function.
|
|
||||||
func (f *Filter) Equals(g *Filter) bool {
|
|
||||||
if g.k != f.k || len(g.b) != len(f.b) {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
for i := range g.b {
|
|
||||||
if f.b[i] != g.b[i] {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
// Fill set f to a completely full filter.
|
|
||||||
// After Fill, Has returns true for any key.
|
|
||||||
func (f *Filter) Fill() {
|
|
||||||
for i := 0; i < len(f.b); i++ {
|
|
||||||
for j := 0; j < blockWords; j++ {
|
|
||||||
f.b[i][j] = ^uint32(0)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Has reports whether a key with hash value h has been added.
|
|
||||||
// It may return a false positive.
|
|
||||||
func (f *Filter) Has(h uint64) bool {
|
|
||||||
h1, h2 := uint32(h>>32), uint32(h)
|
|
||||||
b := getblock(f.b, h2)
|
|
||||||
|
|
||||||
for i := 1; i < f.k; i++ {
|
|
||||||
h1, h2 = doublehash(h1, h2, i)
|
|
||||||
if !b.getbit(h1) {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
// doublehash generates the hash values to use in iteration i of
|
|
||||||
// enhanced double hashing from the values h1, h2 of the previous iteration.
|
|
||||||
// See https://www.ccs.neu.edu/home/pete/pub/bloom-filters-verification.pdf.
|
|
||||||
func doublehash(h1, h2 uint32, i int) (uint32, uint32) {
|
|
||||||
h1 = h1 + h2
|
|
||||||
h2 = h2 + uint32(i)
|
|
||||||
return h1, h2
|
|
||||||
}
|
|
||||||
|
|
||||||
// NumBits returns the number of bits of f.
|
|
||||||
func (f *Filter) NumBits() uint64 {
|
|
||||||
return BlockBits * uint64(len(f.b))
|
|
||||||
}
|
|
||||||
|
|
||||||
func checkBinop(f, g *Filter) {
|
|
||||||
if len(f.b) != len(g.b) {
|
|
||||||
panic("Bloom filters do not have the same number of bits")
|
|
||||||
}
|
|
||||||
if f.k != g.k {
|
|
||||||
panic("Bloom filters do not have the same number of hash functions")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Intersect sets f to the intersection of f and g.
|
|
||||||
//
|
|
||||||
// Intersect panics when f and g do not have the same number of bits and
|
|
||||||
// hash functions. Both Filters must be using the same hash function(s),
|
|
||||||
// but Intersect cannot check this.
|
|
||||||
//
|
|
||||||
// Since Bloom filters may return false positives, Has may return true for
|
|
||||||
// a key that was not in both f and g.
|
|
||||||
//
|
|
||||||
// After Intersect, the estimates from f.Cardinality and f.FPRate should be
|
|
||||||
// considered unreliable.
|
|
||||||
func (f *Filter) Intersect(g *Filter) {
|
|
||||||
checkBinop(f, g)
|
|
||||||
f.intersect(g)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Union sets f to the union of f and g.
|
|
||||||
//
|
|
||||||
// Union panics when f and g do not have the same number of bits and
|
|
||||||
// hash functions. Both Filters must be using the same hash function(s),
|
|
||||||
// but Union cannot check this.
|
|
||||||
func (f *Filter) Union(g *Filter) {
|
|
||||||
checkBinop(f, g)
|
|
||||||
f.union(g)
|
|
||||||
}
|
|
||||||
|
|
||||||
const (
|
|
||||||
wordSize = 32
|
|
||||||
blockWords = BlockBits / wordSize
|
|
||||||
)
|
|
||||||
|
|
||||||
// A block is a fixed-size Bloom filter, used as a shard of a Filter.
|
|
||||||
type block [blockWords]uint32
|
|
||||||
|
|
||||||
func getblock(b []block, h2 uint32) *block {
|
|
||||||
i := reducerange(h2, uint32(len(b)))
|
|
||||||
return &b[i]
|
|
||||||
}
|
|
||||||
|
|
||||||
// reducerange maps i to an integer in the range [0,n).
|
|
||||||
// https://lemire.me/blog/2016/06/27/a-fast-alternative-to-the-modulo-reduction/
|
|
||||||
func reducerange(i, n uint32) uint32 {
|
|
||||||
return uint32((uint64(i) * uint64(n)) >> 32)
|
|
||||||
}
|
|
||||||
|
|
||||||
// getbit reports whether bit (i modulo BlockBits) is set.
|
|
||||||
func (b *block) getbit(i uint32) bool {
|
|
||||||
bit := uint32(1) << (i % wordSize)
|
|
||||||
x := (*b)[(i/wordSize)%blockWords] & bit
|
|
||||||
return x != 0
|
|
||||||
}
|
|
||||||
|
|
||||||
// setbit sets bit (i modulo BlockBits) of b.
|
|
||||||
func (b *block) setbit(i uint32) {
|
|
||||||
bit := uint32(1) << (i % wordSize)
|
|
||||||
(*b)[(i/wordSize)%blockWords] |= bit
|
|
||||||
}
|
|
||||||
246
vendor/github.com/greatroar/blobloom/io.go
generated
vendored
246
vendor/github.com/greatroar/blobloom/io.go
generated
vendored
|
|
@ -1,246 +0,0 @@
|
||||||
// Copyright 2023 the Blobloom authors
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
package blobloom
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"encoding/binary"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"strings"
|
|
||||||
"sync/atomic"
|
|
||||||
)
|
|
||||||
|
|
||||||
const maxCommentLen = 44
|
|
||||||
|
|
||||||
// Dump writes f to w, with an optional comment string, in the binary format
|
|
||||||
// that a Loader accepts. It returns the number of bytes written to w.
|
|
||||||
//
|
|
||||||
// The comment may contain arbitrary data, within the limits layed out by the
|
|
||||||
// format description. It can be used to record the hash function to be used
|
|
||||||
// with a Filter.
|
|
||||||
func Dump(w io.Writer, f *Filter, comment string) (int64, error) {
|
|
||||||
return dump(w, f.b, f.k, comment)
|
|
||||||
}
|
|
||||||
|
|
||||||
// DumpSync is like Dump, but for SyncFilters.
|
|
||||||
//
|
|
||||||
// If other goroutines are simultaneously modifying f,
|
|
||||||
// their modifications may not be reflected in the dump.
|
|
||||||
// Separate synchronization is required to prevent this.
|
|
||||||
//
|
|
||||||
// The format produced is the same as Dump's. The fact that
|
|
||||||
// the argument is a SyncFilter is not encoded in the dump.
|
|
||||||
func DumpSync(w io.Writer, f *SyncFilter, comment string) (n int64, err error) {
|
|
||||||
return dump(w, f.b, f.k, comment)
|
|
||||||
}
|
|
||||||
|
|
||||||
func dump(w io.Writer, b []block, nhashes int, comment string) (n int64, err error) {
|
|
||||||
switch {
|
|
||||||
case len(b) == 0 || nhashes == 0:
|
|
||||||
err = errors.New("blobloom: won't dump uninitialized Filter")
|
|
||||||
case len(comment) > maxCommentLen:
|
|
||||||
err = fmt.Errorf("blobloom: comment of length %d too long", len(comment))
|
|
||||||
case strings.IndexByte(comment, 0) != -1:
|
|
||||||
err = fmt.Errorf("blobloom: comment %q contains zero byte", len(comment))
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
return 0, err
|
|
||||||
}
|
|
||||||
|
|
||||||
var buf [64]byte
|
|
||||||
copy(buf[:8], "blobloom")
|
|
||||||
// As documented in the comment for Loader, we store one less than the
|
|
||||||
// number of blocks. This way, we can use the otherwise invalid value 0
|
|
||||||
// and store 2³² blocks instead of at most 2³²-1.
|
|
||||||
binary.LittleEndian.PutUint32(buf[12:], uint32(len(b)-1))
|
|
||||||
binary.LittleEndian.PutUint32(buf[16:], uint32(nhashes))
|
|
||||||
copy(buf[20:], comment)
|
|
||||||
|
|
||||||
k, err := w.Write(buf[:])
|
|
||||||
n = int64(k)
|
|
||||||
if err != nil {
|
|
||||||
return n, err
|
|
||||||
}
|
|
||||||
|
|
||||||
for i := range b {
|
|
||||||
for j := range b[i] {
|
|
||||||
x := atomic.LoadUint32(&b[i][j])
|
|
||||||
binary.LittleEndian.PutUint32(buf[4*j:], x)
|
|
||||||
}
|
|
||||||
k, err = w.Write(buf[:])
|
|
||||||
n += int64(k)
|
|
||||||
if err != nil {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return n, err
|
|
||||||
}
|
|
||||||
|
|
||||||
// A Loader reads a Filter or SyncFilter from an io.Reader.
|
|
||||||
//
|
|
||||||
// A Loader accepts the binary format produced by Dump. The format starts
|
|
||||||
// with a 64-byte header:
|
|
||||||
// - the string "blobloom", in ASCII;
|
|
||||||
// - a four-byte version number, which must be zero;
|
|
||||||
// - the number of Bloom filter blocks, minus one, as a 32-bit integer;
|
|
||||||
// - the number of hashes, as a 32-bit integer;
|
|
||||||
// - a comment of at most 44 non-zero bytes, padded to 44 bytes with zeros.
|
|
||||||
//
|
|
||||||
// After the header come the 512-bit blocks, divided into sixteen 32-bit limbs.
|
|
||||||
// All integers are little-endian.
|
|
||||||
type Loader struct {
|
|
||||||
buf [64]byte
|
|
||||||
r io.Reader
|
|
||||||
err error
|
|
||||||
|
|
||||||
Comment string // Comment field. Filled in by NewLoader.
|
|
||||||
nblocks uint64
|
|
||||||
nhashes int
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewLoader parses the format header from r and returns a Loader
|
|
||||||
// that can be used to load a Filter from it.
|
|
||||||
func NewLoader(r io.Reader) (*Loader, error) {
|
|
||||||
l := &Loader{r: r}
|
|
||||||
|
|
||||||
err := l.fillbuf()
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
version := binary.LittleEndian.Uint32(l.buf[8:])
|
|
||||||
// See comment in dump for the +1.
|
|
||||||
l.nblocks = 1 + uint64(binary.LittleEndian.Uint32(l.buf[12:]))
|
|
||||||
l.nhashes = int(binary.LittleEndian.Uint32(l.buf[16:]))
|
|
||||||
comment := l.buf[20:]
|
|
||||||
|
|
||||||
switch {
|
|
||||||
case string(l.buf[:8]) != "blobloom":
|
|
||||||
err = errors.New("blobloom: not a Bloom filter dump")
|
|
||||||
case version != 0:
|
|
||||||
err = errors.New("blobloom: unsupported dump version")
|
|
||||||
case l.nhashes == 0:
|
|
||||||
err = errors.New("blobloom: zero hashes in Bloom filter dump")
|
|
||||||
}
|
|
||||||
if err == nil {
|
|
||||||
comment, err = checkComment(comment)
|
|
||||||
l.Comment = string(comment)
|
|
||||||
}
|
|
||||||
|
|
||||||
if err != nil {
|
|
||||||
l = nil
|
|
||||||
}
|
|
||||||
return l, err
|
|
||||||
}
|
|
||||||
|
|
||||||
// Load sets f to the union of f and the Loader's filter, then returns f.
|
|
||||||
// If f is nil, a new Filter of the appropriate size is constructed.
|
|
||||||
//
|
|
||||||
// If f is not nil and an error occurs while reading from the Loader,
|
|
||||||
// f may end up in an inconsistent state.
|
|
||||||
func (l *Loader) Load(f *Filter) (*Filter, error) {
|
|
||||||
if f == nil {
|
|
||||||
nbits := BlockBits * l.nblocks
|
|
||||||
if nbits > MaxBits {
|
|
||||||
return nil, fmt.Errorf("blobloom: %d blocks is too large", l.nblocks)
|
|
||||||
}
|
|
||||||
f = New(nbits, int(l.nhashes))
|
|
||||||
} else if err := l.checkBitsAndHashes(len(f.b), f.k); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
for i := range f.b {
|
|
||||||
if err := l.fillbuf(); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
for j := range f.b[i] {
|
|
||||||
f.b[i][j] |= binary.LittleEndian.Uint32(l.buf[4*j:])
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return f, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Load sets f to the union of f and the Loader's filter, then returns f.
|
|
||||||
// If f is nil, a new SyncFilter of the appropriate size is constructed.
|
|
||||||
// Else, LoadSync may run concurrently with other modifications to f.
|
|
||||||
//
|
|
||||||
// If f is not nil and an error occurs while reading from the Loader,
|
|
||||||
// f may end up in an inconsistent state.
|
|
||||||
func (l *Loader) LoadSync(f *SyncFilter) (*SyncFilter, error) {
|
|
||||||
if f == nil {
|
|
||||||
nbits := BlockBits * l.nblocks
|
|
||||||
if nbits > MaxBits {
|
|
||||||
return nil, fmt.Errorf("blobloom: %d blocks is too large", l.nblocks)
|
|
||||||
}
|
|
||||||
f = NewSync(nbits, int(l.nhashes))
|
|
||||||
} else if err := l.checkBitsAndHashes(len(f.b), f.k); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
for i := range f.b {
|
|
||||||
if err := l.fillbuf(); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
for j := range f.b[i] {
|
|
||||||
p := &f.b[i][j]
|
|
||||||
x := binary.LittleEndian.Uint32(l.buf[4*j:])
|
|
||||||
|
|
||||||
for {
|
|
||||||
old := atomic.LoadUint32(p)
|
|
||||||
if atomic.CompareAndSwapUint32(p, old, old|x) {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return f, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (l *Loader) checkBitsAndHashes(nblocks, nhashes int) error {
|
|
||||||
switch {
|
|
||||||
case nblocks != int(l.nblocks):
|
|
||||||
return fmt.Errorf("blobloom: Filter has %d blocks, but dump has %d", nblocks, l.nblocks)
|
|
||||||
case nhashes != l.nhashes:
|
|
||||||
return fmt.Errorf("blobloom: Filter has %d hashes, but dump has %d", nhashes, l.nhashes)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (l *Loader) fillbuf() error {
|
|
||||||
_, err := io.ReadFull(l.r, l.buf[:])
|
|
||||||
if err == io.EOF {
|
|
||||||
err = io.ErrUnexpectedEOF
|
|
||||||
}
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
func checkComment(p []byte) ([]byte, error) {
|
|
||||||
eos := bytes.IndexByte(p, 0)
|
|
||||||
if eos != -1 {
|
|
||||||
tail := p[eos+1:]
|
|
||||||
if !bytes.Equal(tail, make([]byte, len(tail))) {
|
|
||||||
return nil, fmt.Errorf("blobloom: comment block %q contains zero byte", p)
|
|
||||||
}
|
|
||||||
p = p[:eos]
|
|
||||||
}
|
|
||||||
return p, nil
|
|
||||||
}
|
|
||||||
201
vendor/github.com/greatroar/blobloom/optimize.go
generated
vendored
201
vendor/github.com/greatroar/blobloom/optimize.go
generated
vendored
|
|
@ -1,201 +0,0 @@
|
||||||
// Copyright 2020 the Blobloom authors
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
package blobloom
|
|
||||||
|
|
||||||
import "math"
|
|
||||||
|
|
||||||
// A Config holds parameters for Optimize or NewOptimized.
|
|
||||||
type Config struct {
|
|
||||||
// Trigger the "contains filtered or unexported fields" message for
|
|
||||||
// forward compatibility and force the caller to use named fields.
|
|
||||||
_ struct{}
|
|
||||||
|
|
||||||
// Capacity is the expected number of distinct keys to be added.
|
|
||||||
// More keys can always be added, but the false positive rate can be
|
|
||||||
// expected to drop below FPRate if their number exceeds the Capacity.
|
|
||||||
Capacity uint64
|
|
||||||
|
|
||||||
// Desired lower bound on the false positive rate when the Bloom filter
|
|
||||||
// has been filled to its capacity. FPRate must be between zero
|
|
||||||
// (exclusive) and one (inclusive).
|
|
||||||
FPRate float64
|
|
||||||
|
|
||||||
// Maximum size of the Bloom filter in bits. Zero means the global
|
|
||||||
// MaxBits constant. A value less than BlockBits means BlockBits.
|
|
||||||
MaxBits uint64
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewOptimized is shorthand for New(Optimize(config)).
|
|
||||||
func NewOptimized(config Config) *Filter {
|
|
||||||
return New(Optimize(config))
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewSyncOptimized is shorthand for New(Optimize(config)).
|
|
||||||
func NewSyncOptimized(config Config) *SyncFilter {
|
|
||||||
return NewSync(Optimize(config))
|
|
||||||
}
|
|
||||||
|
|
||||||
// Optimize returns numbers of keys and hash functions that achieve the
|
|
||||||
// desired false positive described by config.
|
|
||||||
//
|
|
||||||
// Optimize panics when config.FPRate is invalid.
|
|
||||||
//
|
|
||||||
// The estimated number of bits is imprecise for false positives rates below
|
|
||||||
// ca. 1e-15.
|
|
||||||
func Optimize(config Config) (nbits uint64, nhashes int) {
|
|
||||||
n := float64(config.Capacity)
|
|
||||||
p := config.FPRate
|
|
||||||
|
|
||||||
if p <= 0 || p > 1 {
|
|
||||||
panic("false positive rate for a Bloom filter must be > 0, <= 1")
|
|
||||||
}
|
|
||||||
if n == 0 {
|
|
||||||
// Assume the client wants to add at least one key; log2(0) = -inf.
|
|
||||||
n = 1
|
|
||||||
}
|
|
||||||
|
|
||||||
// The optimal nbits/n is c = -log2(p) / ln(2) for a vanilla Bloom filter.
|
|
||||||
c := math.Ceil(-math.Log2(p) / math.Ln2)
|
|
||||||
if c < float64(len(correctC)) {
|
|
||||||
c = float64(correctC[int(c)])
|
|
||||||
} else {
|
|
||||||
// We can't achieve the desired FPR. Just triple the number of bits.
|
|
||||||
c *= 3
|
|
||||||
}
|
|
||||||
nbits = uint64(c * n)
|
|
||||||
|
|
||||||
// Round up to a multiple of BlockBits.
|
|
||||||
if nbits%BlockBits != 0 {
|
|
||||||
nbits += BlockBits - nbits%BlockBits
|
|
||||||
}
|
|
||||||
|
|
||||||
var maxbits uint64 = MaxBits
|
|
||||||
if config.MaxBits != 0 && config.MaxBits < maxbits {
|
|
||||||
maxbits = config.MaxBits
|
|
||||||
if maxbits < BlockBits {
|
|
||||||
maxbits = BlockBits
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if nbits > maxbits {
|
|
||||||
nbits = maxbits
|
|
||||||
// Round down to a multiple of BlockBits.
|
|
||||||
nbits -= nbits % BlockBits
|
|
||||||
}
|
|
||||||
|
|
||||||
// The corresponding optimal number of hash functions is k = c * log(2).
|
|
||||||
// Try rounding up and down to see which rounding is better.
|
|
||||||
c = float64(nbits) / n
|
|
||||||
k := c * math.Ln2
|
|
||||||
if k < 1 {
|
|
||||||
nhashes = 1
|
|
||||||
return nbits, nhashes
|
|
||||||
}
|
|
||||||
|
|
||||||
ceilK, floorK := math.Floor(k), math.Ceil(k)
|
|
||||||
if ceilK == floorK {
|
|
||||||
return nbits, int(ceilK)
|
|
||||||
}
|
|
||||||
|
|
||||||
fprCeil, _ := fpRate(c, math.Ceil(k))
|
|
||||||
fprFloor, _ := fpRate(c, math.Floor(k))
|
|
||||||
if fprFloor < fprCeil {
|
|
||||||
k = floorK
|
|
||||||
} else {
|
|
||||||
k = ceilK
|
|
||||||
}
|
|
||||||
|
|
||||||
return nbits, int(k)
|
|
||||||
}
|
|
||||||
|
|
||||||
// correctC maps c = m/n for a vanilla Bloom filter to the c' for a
|
|
||||||
// blocked Bloom filter.
|
|
||||||
//
|
|
||||||
// This is Putze et al.'s Table I, extended down to zero.
|
|
||||||
// For c > 34, the values become huge and are hard to compute.
|
|
||||||
var correctC = []byte{
|
|
||||||
1, 1, 2, 4, 5,
|
|
||||||
6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 17, 18, 20, 21, 23,
|
|
||||||
25, 26, 28, 30, 32, 35, 38, 40, 44, 48, 51, 58, 64, 74, 90,
|
|
||||||
}
|
|
||||||
|
|
||||||
// FPRate computes an estimate of the false positive rate of a Bloom filter
|
|
||||||
// after nkeys distinct keys have been added.
|
|
||||||
func FPRate(nkeys, nbits uint64, nhashes int) float64 {
|
|
||||||
if nkeys == 0 {
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
p, _ := fpRate(float64(nbits)/float64(nkeys), float64(nhashes))
|
|
||||||
return p
|
|
||||||
}
|
|
||||||
|
|
||||||
func fpRate(c, k float64) (p float64, iter int) {
|
|
||||||
switch {
|
|
||||||
case c == 0:
|
|
||||||
panic("0 bits per key is too few")
|
|
||||||
case k == 0:
|
|
||||||
panic("0 hashes is too few")
|
|
||||||
}
|
|
||||||
|
|
||||||
// Putze et al.'s Equation (3).
|
|
||||||
//
|
|
||||||
// The Poisson distribution has a single spike around its mean
|
|
||||||
// BlockBits/c that gets slimmer and further away from zero as c tends
|
|
||||||
// to zero (the Bloom filter gets more filled). We start at the mean,
|
|
||||||
// then add terms left and right of it until their relative contribution
|
|
||||||
// drops below ε.
|
|
||||||
const ε = 1e-9
|
|
||||||
mean := BlockBits / c
|
|
||||||
|
|
||||||
// Ceil to make sure we start at one, not zero.
|
|
||||||
i := math.Ceil(mean)
|
|
||||||
p = math.Exp(logPoisson(mean, i) + logFprBlock(BlockBits/i, k))
|
|
||||||
|
|
||||||
for j := i - 1; j > 0; j-- {
|
|
||||||
add := math.Exp(logPoisson(mean, j) + logFprBlock(BlockBits/j, k))
|
|
||||||
p += add
|
|
||||||
iter++
|
|
||||||
if add/p < ε {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
for j := i + 1; ; j++ {
|
|
||||||
add := math.Exp(logPoisson(mean, j) + logFprBlock(BlockBits/j, k))
|
|
||||||
p += add
|
|
||||||
iter++
|
|
||||||
if add/p < ε {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return p, iter
|
|
||||||
}
|
|
||||||
|
|
||||||
// FPRate computes an estimate of f's false positive rate after nkeys distinct
|
|
||||||
// keys have been added.
|
|
||||||
func (f *Filter) FPRate(nkeys uint64) float64 {
|
|
||||||
return FPRate(nkeys, f.NumBits(), f.k)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Log of the FPR of a single block, FPR = (1 - exp(-k/c))^k.
|
|
||||||
func logFprBlock(c, k float64) float64 {
|
|
||||||
return k * math.Log1p(-math.Exp(-k/c))
|
|
||||||
}
|
|
||||||
|
|
||||||
// Log of the Poisson distribution's pmf.
|
|
||||||
func logPoisson(λ, k float64) float64 {
|
|
||||||
lg, _ := math.Lgamma(k + 1)
|
|
||||||
return k*math.Log(λ) - λ - lg
|
|
||||||
}
|
|
||||||
148
vendor/github.com/greatroar/blobloom/setop_64bit.go
generated
vendored
148
vendor/github.com/greatroar/blobloom/setop_64bit.go
generated
vendored
|
|
@ -1,148 +0,0 @@
|
||||||
// Copyright 2020-2022 the Blobloom authors
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
//go:build (amd64 || arm64) && !nounsafe
|
|
||||||
// +build amd64 arm64
|
|
||||||
// +build !nounsafe
|
|
||||||
|
|
||||||
package blobloom
|
|
||||||
|
|
||||||
import (
|
|
||||||
"math/bits"
|
|
||||||
"sync/atomic"
|
|
||||||
"unsafe"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Block reinterpreted as array of uint64.
|
|
||||||
type block64 [BlockBits / 64]uint64
|
|
||||||
|
|
||||||
func (f *Filter) intersect(g *Filter) {
|
|
||||||
a, b := f.b, g.b
|
|
||||||
for len(a) >= 2 && len(b) >= 2 {
|
|
||||||
p := (*block64)(unsafe.Pointer(&a[0]))
|
|
||||||
q := (*block64)(unsafe.Pointer(&b[0]))
|
|
||||||
|
|
||||||
p[0] &= q[0]
|
|
||||||
p[1] &= q[1]
|
|
||||||
p[2] &= q[2]
|
|
||||||
p[3] &= q[3]
|
|
||||||
p[4] &= q[4]
|
|
||||||
p[5] &= q[5]
|
|
||||||
p[6] &= q[6]
|
|
||||||
p[7] &= q[7]
|
|
||||||
|
|
||||||
p = (*block64)(unsafe.Pointer(&a[1]))
|
|
||||||
q = (*block64)(unsafe.Pointer(&b[1]))
|
|
||||||
|
|
||||||
p[0] &= q[0]
|
|
||||||
p[1] &= q[1]
|
|
||||||
p[2] &= q[2]
|
|
||||||
p[3] &= q[3]
|
|
||||||
p[4] &= q[4]
|
|
||||||
p[5] &= q[5]
|
|
||||||
p[6] &= q[6]
|
|
||||||
p[7] &= q[7]
|
|
||||||
|
|
||||||
a, b = a[2:], b[2:]
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(a) > 0 && len(b) > 0 {
|
|
||||||
p := (*block64)(unsafe.Pointer(&a[0]))
|
|
||||||
q := (*block64)(unsafe.Pointer(&b[0]))
|
|
||||||
|
|
||||||
p[0] &= q[0]
|
|
||||||
p[1] &= q[1]
|
|
||||||
p[2] &= q[2]
|
|
||||||
p[3] &= q[3]
|
|
||||||
p[4] &= q[4]
|
|
||||||
p[5] &= q[5]
|
|
||||||
p[6] &= q[6]
|
|
||||||
p[7] &= q[7]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f *Filter) union(g *Filter) {
|
|
||||||
a, b := f.b, g.b
|
|
||||||
for len(a) >= 2 && len(b) >= 2 {
|
|
||||||
p := (*block64)(unsafe.Pointer(&a[0]))
|
|
||||||
q := (*block64)(unsafe.Pointer(&b[0]))
|
|
||||||
|
|
||||||
p[0] |= q[0]
|
|
||||||
p[1] |= q[1]
|
|
||||||
p[2] |= q[2]
|
|
||||||
p[3] |= q[3]
|
|
||||||
p[4] |= q[4]
|
|
||||||
p[5] |= q[5]
|
|
||||||
p[6] |= q[6]
|
|
||||||
p[7] |= q[7]
|
|
||||||
|
|
||||||
p = (*block64)(unsafe.Pointer(&a[1]))
|
|
||||||
q = (*block64)(unsafe.Pointer(&b[1]))
|
|
||||||
|
|
||||||
p[0] |= q[0]
|
|
||||||
p[1] |= q[1]
|
|
||||||
p[2] |= q[2]
|
|
||||||
p[3] |= q[3]
|
|
||||||
p[4] |= q[4]
|
|
||||||
p[5] |= q[5]
|
|
||||||
p[6] |= q[6]
|
|
||||||
p[7] |= q[7]
|
|
||||||
|
|
||||||
a, b = a[2:], b[2:]
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(a) > 0 && len(b) > 0 {
|
|
||||||
p := (*block64)(unsafe.Pointer(&a[0]))
|
|
||||||
q := (*block64)(unsafe.Pointer(&b[0]))
|
|
||||||
|
|
||||||
p[0] |= q[0]
|
|
||||||
p[1] |= q[1]
|
|
||||||
p[2] |= q[2]
|
|
||||||
p[3] |= q[3]
|
|
||||||
p[4] |= q[4]
|
|
||||||
p[5] |= q[5]
|
|
||||||
p[6] |= q[6]
|
|
||||||
p[7] |= q[7]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func onescount(b *block) (n int) {
|
|
||||||
p := (*block64)(unsafe.Pointer(&b[0]))
|
|
||||||
|
|
||||||
n += bits.OnesCount64(p[0])
|
|
||||||
n += bits.OnesCount64(p[1])
|
|
||||||
n += bits.OnesCount64(p[2])
|
|
||||||
n += bits.OnesCount64(p[3])
|
|
||||||
n += bits.OnesCount64(p[4])
|
|
||||||
n += bits.OnesCount64(p[5])
|
|
||||||
n += bits.OnesCount64(p[6])
|
|
||||||
n += bits.OnesCount64(p[7])
|
|
||||||
|
|
||||||
return n
|
|
||||||
}
|
|
||||||
|
|
||||||
func onescountAtomic(b *block) (n int) {
|
|
||||||
p := (*block64)(unsafe.Pointer(&b[0]))
|
|
||||||
|
|
||||||
n += bits.OnesCount64(atomic.LoadUint64(&p[0]))
|
|
||||||
n += bits.OnesCount64(atomic.LoadUint64(&p[1]))
|
|
||||||
n += bits.OnesCount64(atomic.LoadUint64(&p[2]))
|
|
||||||
n += bits.OnesCount64(atomic.LoadUint64(&p[3]))
|
|
||||||
n += bits.OnesCount64(atomic.LoadUint64(&p[4]))
|
|
||||||
n += bits.OnesCount64(atomic.LoadUint64(&p[5]))
|
|
||||||
n += bits.OnesCount64(atomic.LoadUint64(&p[6]))
|
|
||||||
n += bits.OnesCount64(atomic.LoadUint64(&p[7]))
|
|
||||||
|
|
||||||
return n
|
|
||||||
}
|
|
||||||
115
vendor/github.com/greatroar/blobloom/setop_other.go
generated
vendored
115
vendor/github.com/greatroar/blobloom/setop_other.go
generated
vendored
|
|
@ -1,115 +0,0 @@
|
||||||
// Copyright 2020-2022 the Blobloom authors
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
//go:build (!amd64 && !arm64) || nounsafe
|
|
||||||
// +build !amd64,!arm64 nounsafe
|
|
||||||
|
|
||||||
package blobloom
|
|
||||||
|
|
||||||
import (
|
|
||||||
"math/bits"
|
|
||||||
"sync/atomic"
|
|
||||||
)
|
|
||||||
|
|
||||||
func (f *Filter) intersect(g *Filter) {
|
|
||||||
for i := range f.b {
|
|
||||||
f.b[i].intersect(&g.b[i])
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f *Filter) union(g *Filter) {
|
|
||||||
for i := range f.b {
|
|
||||||
f.b[i].union(&g.b[i])
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (b *block) intersect(c *block) {
|
|
||||||
b[0] &= c[0]
|
|
||||||
b[1] &= c[1]
|
|
||||||
b[2] &= c[2]
|
|
||||||
b[3] &= c[3]
|
|
||||||
b[4] &= c[4]
|
|
||||||
b[5] &= c[5]
|
|
||||||
b[6] &= c[6]
|
|
||||||
b[7] &= c[7]
|
|
||||||
b[8] &= c[8]
|
|
||||||
b[9] &= c[9]
|
|
||||||
b[10] &= c[10]
|
|
||||||
b[11] &= c[11]
|
|
||||||
b[12] &= c[12]
|
|
||||||
b[13] &= c[13]
|
|
||||||
b[14] &= c[14]
|
|
||||||
b[15] &= c[15]
|
|
||||||
}
|
|
||||||
|
|
||||||
func (b *block) union(c *block) {
|
|
||||||
b[0] |= c[0]
|
|
||||||
b[1] |= c[1]
|
|
||||||
b[2] |= c[2]
|
|
||||||
b[3] |= c[3]
|
|
||||||
b[4] |= c[4]
|
|
||||||
b[5] |= c[5]
|
|
||||||
b[6] |= c[6]
|
|
||||||
b[7] |= c[7]
|
|
||||||
b[8] |= c[8]
|
|
||||||
b[9] |= c[9]
|
|
||||||
b[10] |= c[10]
|
|
||||||
b[11] |= c[11]
|
|
||||||
b[12] |= c[12]
|
|
||||||
b[13] |= c[13]
|
|
||||||
b[14] |= c[14]
|
|
||||||
b[15] |= c[15]
|
|
||||||
}
|
|
||||||
|
|
||||||
func onescount(b *block) (n int) {
|
|
||||||
n += bits.OnesCount32(b[0])
|
|
||||||
n += bits.OnesCount32(b[1])
|
|
||||||
n += bits.OnesCount32(b[2])
|
|
||||||
n += bits.OnesCount32(b[3])
|
|
||||||
n += bits.OnesCount32(b[4])
|
|
||||||
n += bits.OnesCount32(b[5])
|
|
||||||
n += bits.OnesCount32(b[6])
|
|
||||||
n += bits.OnesCount32(b[7])
|
|
||||||
n += bits.OnesCount32(b[8])
|
|
||||||
n += bits.OnesCount32(b[9])
|
|
||||||
n += bits.OnesCount32(b[10])
|
|
||||||
n += bits.OnesCount32(b[11])
|
|
||||||
n += bits.OnesCount32(b[12])
|
|
||||||
n += bits.OnesCount32(b[13])
|
|
||||||
n += bits.OnesCount32(b[14])
|
|
||||||
n += bits.OnesCount32(b[15])
|
|
||||||
|
|
||||||
return n
|
|
||||||
}
|
|
||||||
|
|
||||||
func onescountAtomic(b *block) (n int) {
|
|
||||||
n += bits.OnesCount32(atomic.LoadUint32(&b[0]))
|
|
||||||
n += bits.OnesCount32(atomic.LoadUint32(&b[1]))
|
|
||||||
n += bits.OnesCount32(atomic.LoadUint32(&b[2]))
|
|
||||||
n += bits.OnesCount32(atomic.LoadUint32(&b[3]))
|
|
||||||
n += bits.OnesCount32(atomic.LoadUint32(&b[4]))
|
|
||||||
n += bits.OnesCount32(atomic.LoadUint32(&b[5]))
|
|
||||||
n += bits.OnesCount32(atomic.LoadUint32(&b[6]))
|
|
||||||
n += bits.OnesCount32(atomic.LoadUint32(&b[7]))
|
|
||||||
n += bits.OnesCount32(atomic.LoadUint32(&b[8]))
|
|
||||||
n += bits.OnesCount32(atomic.LoadUint32(&b[9]))
|
|
||||||
n += bits.OnesCount32(atomic.LoadUint32(&b[10]))
|
|
||||||
n += bits.OnesCount32(atomic.LoadUint32(&b[11]))
|
|
||||||
n += bits.OnesCount32(atomic.LoadUint32(&b[12]))
|
|
||||||
n += bits.OnesCount32(atomic.LoadUint32(&b[13]))
|
|
||||||
n += bits.OnesCount32(atomic.LoadUint32(&b[14]))
|
|
||||||
n += bits.OnesCount32(atomic.LoadUint32(&b[15]))
|
|
||||||
|
|
||||||
return n
|
|
||||||
}
|
|
||||||
145
vendor/github.com/greatroar/blobloom/sync.go
generated
vendored
145
vendor/github.com/greatroar/blobloom/sync.go
generated
vendored
|
|
@ -1,145 +0,0 @@
|
||||||
// Copyright 2021-2022 the Blobloom authors
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
package blobloom
|
|
||||||
|
|
||||||
import "sync/atomic"
|
|
||||||
|
|
||||||
// A SyncFilter is a Bloom filter that can be accessed and updated
|
|
||||||
// by multiple goroutines concurrently.
|
|
||||||
//
|
|
||||||
// A SyncFilter mostly behaves as a regular filter protected by a lock,
|
|
||||||
//
|
|
||||||
// type SyncFilter struct {
|
|
||||||
// Filter
|
|
||||||
// lock sync.Mutex
|
|
||||||
// }
|
|
||||||
//
|
|
||||||
// with each method taking and releasing the lock,
|
|
||||||
// but is implemented much more efficiently.
|
|
||||||
// See the method descriptions for exceptions to the previous rule.
|
|
||||||
type SyncFilter struct {
|
|
||||||
b []block // Shards.
|
|
||||||
k int // Number of hash functions required.
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewSync constructs a Bloom filter with given numbers of bits and hash functions.
|
|
||||||
//
|
|
||||||
// The number of bits should be at least BlockBits; smaller values are silently
|
|
||||||
// increased.
|
|
||||||
//
|
|
||||||
// The number of hashes reflects the number of hashes synthesized from the
|
|
||||||
// single hash passed in by the client. It is silently increased to two if
|
|
||||||
// a lower value is given.
|
|
||||||
func NewSync(nbits uint64, nhashes int) *SyncFilter {
|
|
||||||
nbits, nhashes = fixBitsAndHashes(nbits, nhashes)
|
|
||||||
|
|
||||||
return &SyncFilter{
|
|
||||||
b: make([]block, nbits/BlockBits),
|
|
||||||
k: nhashes,
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
// Add insert a key with hash value h into f.
|
|
||||||
func (f *SyncFilter) Add(h uint64) {
|
|
||||||
h1, h2 := uint32(h>>32), uint32(h)
|
|
||||||
b := getblock(f.b, h2)
|
|
||||||
|
|
||||||
for i := 1; i < f.k; i++ {
|
|
||||||
h1, h2 = doublehash(h1, h2, i)
|
|
||||||
setbitAtomic(b, h1)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Cardinality estimates the number of distinct keys added to f.
|
|
||||||
//
|
|
||||||
// The estimate is most reliable when f is filled to roughly its capacity.
|
|
||||||
// It gets worse as f gets more densely filled. When one of the blocks is
|
|
||||||
// entirely filled, the estimate becomes +Inf.
|
|
||||||
//
|
|
||||||
// The return value is the maximum likelihood estimate of Papapetrou, Siberski
|
|
||||||
// and Nejdl, summed over the blocks
|
|
||||||
// (https://www.win.tue.nl/~opapapetrou/papers/Bloomfilters-DAPD.pdf).
|
|
||||||
//
|
|
||||||
// If other goroutines are concurrently adding keys,
|
|
||||||
// the estimate may lie in between what would have been returned
|
|
||||||
// before the concurrent updates started and what is returned
|
|
||||||
// after the updates complete.
|
|
||||||
func (f *SyncFilter) Cardinality() float64 {
|
|
||||||
return cardinality(f.k, f.b, onescountAtomic)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Empty reports whether f contains no keys.
|
|
||||||
//
|
|
||||||
// If other goroutines are concurrently adding keys,
|
|
||||||
// Empty may return a false positive.
|
|
||||||
func (f *SyncFilter) Empty() bool {
|
|
||||||
for i := 0; i < len(f.b); i++ {
|
|
||||||
for j := 0; j < blockWords; j++ {
|
|
||||||
if atomic.LoadUint32(&f.b[i][j]) != 0 {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
// Fill sets f to a completely full filter.
|
|
||||||
// After Fill, Has returns true for any key.
|
|
||||||
func (f *SyncFilter) Fill() {
|
|
||||||
for i := 0; i < len(f.b); i++ {
|
|
||||||
for j := 0; j < blockWords; j++ {
|
|
||||||
atomic.StoreUint32(&f.b[i][j], ^uint32(0))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Has reports whether a key with hash value h has been added.
|
|
||||||
// It may return a false positive.
|
|
||||||
func (f *SyncFilter) Has(h uint64) bool {
|
|
||||||
h1, h2 := uint32(h>>32), uint32(h)
|
|
||||||
b := getblock(f.b, h2)
|
|
||||||
|
|
||||||
for i := 1; i < f.k; i++ {
|
|
||||||
h1, h2 = doublehash(h1, h2, i)
|
|
||||||
if !getbitAtomic(b, h1) {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
// getbitAtomic reports whether bit (i modulo BlockBits) is set.
|
|
||||||
func getbitAtomic(b *block, i uint32) bool {
|
|
||||||
bit := uint32(1) << (i % wordSize)
|
|
||||||
x := atomic.LoadUint32(&(*b)[(i/wordSize)%blockWords])
|
|
||||||
return x&bit != 0
|
|
||||||
}
|
|
||||||
|
|
||||||
// setbit sets bit (i modulo BlockBits) of b, atomically.
|
|
||||||
func setbitAtomic(b *block, i uint32) {
|
|
||||||
bit := uint32(1) << (i % wordSize)
|
|
||||||
p := &(*b)[(i/wordSize)%blockWords]
|
|
||||||
|
|
||||||
for {
|
|
||||||
old := atomic.LoadUint32(p)
|
|
||||||
if old&bit != 0 {
|
|
||||||
// Checking here instead of checking the return value from
|
|
||||||
// the CAS is between 50% and 80% faster on the benchmark.
|
|
||||||
return
|
|
||||||
}
|
|
||||||
atomic.CompareAndSwapUint32(p, old, old|bit)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
16
vendor/github.com/greatroar/blobloom/test.sh
generated
vendored
16
vendor/github.com/greatroar/blobloom/test.sh
generated
vendored
|
|
@ -1,16 +0,0 @@
|
||||||
#!/bin/sh
|
|
||||||
|
|
||||||
set -e -x
|
|
||||||
|
|
||||||
golangci-lint run . examples/*
|
|
||||||
|
|
||||||
go test
|
|
||||||
|
|
||||||
if [ "$(go env GOARCH)" = amd64 ]; then
|
|
||||||
go test -tags nounsafe
|
|
||||||
GOARCH=386 go test
|
|
||||||
fi
|
|
||||||
|
|
||||||
for e in examples/*; do
|
|
||||||
(cd $e && go build && rm $(basename $e))
|
|
||||||
done
|
|
||||||
2
vendor/github.com/nbd-wtf/go-nostr/kinds.go
generated
vendored
2
vendor/github.com/nbd-wtf/go-nostr/kinds.go
generated
vendored
|
|
@ -26,7 +26,7 @@ const (
|
||||||
KindChess int = 64
|
KindChess int = 64
|
||||||
KindMergeRequests int = 818
|
KindMergeRequests int = 818
|
||||||
KindBid int = 1021
|
KindBid int = 1021
|
||||||
KIndBidConfirmation int = 1022
|
KindBidConfirmation int = 1022
|
||||||
KindOpenTimestamps int = 1040
|
KindOpenTimestamps int = 1040
|
||||||
KindGiftWrap int = 1059
|
KindGiftWrap int = 1059
|
||||||
KindFileMetadata int = 1063
|
KindFileMetadata int = 1063
|
||||||
|
|
|
||||||
12
vendor/github.com/nbd-wtf/go-nostr/nip77/nip77.go
generated
vendored
12
vendor/github.com/nbd-wtf/go-nostr/nip77/nip77.go
generated
vendored
|
|
@ -5,8 +5,6 @@ import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"sync"
|
"sync"
|
||||||
|
|
||||||
"github.com/cespare/xxhash"
|
|
||||||
"github.com/greatroar/blobloom"
|
|
||||||
"github.com/nbd-wtf/go-nostr"
|
"github.com/nbd-wtf/go-nostr"
|
||||||
"github.com/nbd-wtf/go-nostr/nip77/negentropy"
|
"github.com/nbd-wtf/go-nostr/nip77/negentropy"
|
||||||
"github.com/nbd-wtf/go-nostr/nip77/negentropy/storage/vector"
|
"github.com/nbd-wtf/go-nostr/nip77/negentropy/storage/vector"
|
||||||
|
|
@ -88,10 +86,7 @@ func NegentropySync(ctx context.Context, store nostr.RelayStore, url string, fil
|
||||||
go func(dir direction) {
|
go func(dir direction) {
|
||||||
defer wg.Done()
|
defer wg.Done()
|
||||||
|
|
||||||
seen := blobloom.NewOptimized(blobloom.Config{
|
seen := make(map[string]struct{})
|
||||||
Capacity: 10000,
|
|
||||||
FPRate: 0.01,
|
|
||||||
})
|
|
||||||
|
|
||||||
doSync := func(ids []string) {
|
doSync := func(ids []string) {
|
||||||
defer wg.Done()
|
defer wg.Done()
|
||||||
|
|
@ -112,12 +107,11 @@ func NegentropySync(ctx context.Context, store nostr.RelayStore, url string, fil
|
||||||
|
|
||||||
ids := pool.grab()
|
ids := pool.grab()
|
||||||
for item := range dir.items {
|
for item := range dir.items {
|
||||||
h := xxhash.Sum64([]byte(item))
|
if _, ok := seen[item]; ok {
|
||||||
if seen.Has(h) {
|
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
seen[item] = struct{}{}
|
||||||
|
|
||||||
seen.Add(h)
|
|
||||||
ids = append(ids, item)
|
ids = append(ids, item)
|
||||||
if len(ids) == 50 {
|
if len(ids) == 50 {
|
||||||
wg.Add(1)
|
wg.Add(1)
|
||||||
|
|
|
||||||
2
vendor/github.com/nbd-wtf/go-nostr/relay.go
generated
vendored
2
vendor/github.com/nbd-wtf/go-nostr/relay.go
generated
vendored
|
|
@ -182,12 +182,14 @@ func (r *Relay) ConnectWithTLS(ctx context.Context, tlsConfig *tls.Config) error
|
||||||
for {
|
for {
|
||||||
select {
|
select {
|
||||||
case <-ticker.C:
|
case <-ticker.C:
|
||||||
|
if r.Connection != nil {
|
||||||
err := wsutil.WriteClientMessage(r.Connection.conn, ws.OpPing, nil)
|
err := wsutil.WriteClientMessage(r.Connection.conn, ws.OpPing, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
InfoLogger.Printf("{%s} error writing ping: %v; closing websocket", r.URL, err)
|
InfoLogger.Printf("{%s} error writing ping: %v; closing websocket", r.URL, err)
|
||||||
r.Close() // this should trigger a context cancelation
|
r.Close() // this should trigger a context cancelation
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
}
|
||||||
case writeRequest := <-r.writeQueue:
|
case writeRequest := <-r.writeQueue:
|
||||||
// all write requests will go through this to prevent races
|
// all write requests will go through this to prevent races
|
||||||
debugLogf("{%s} sending %v\n", r.URL, string(writeRequest.msg))
|
debugLogf("{%s} sending %v\n", r.URL, string(writeRequest.msg))
|
||||||
|
|
|
||||||
6
vendor/modules.txt
vendored
6
vendor/modules.txt
vendored
|
|
@ -17,7 +17,6 @@ github.com/btcsuite/btcd/btcec/v2/schnorr
|
||||||
github.com/btcsuite/btcd/chaincfg/chainhash
|
github.com/btcsuite/btcd/chaincfg/chainhash
|
||||||
# github.com/cespare/xxhash v1.1.0
|
# github.com/cespare/xxhash v1.1.0
|
||||||
## explicit
|
## explicit
|
||||||
github.com/cespare/xxhash
|
|
||||||
# github.com/decred/dcrd/crypto/blake256 v1.1.0
|
# github.com/decred/dcrd/crypto/blake256 v1.1.0
|
||||||
## explicit; go 1.17
|
## explicit; go 1.17
|
||||||
github.com/decred/dcrd/crypto/blake256
|
github.com/decred/dcrd/crypto/blake256
|
||||||
|
|
@ -33,7 +32,7 @@ github.com/fasthttp/websocket
|
||||||
## explicit; go 1.23.1
|
## explicit; go 1.23.1
|
||||||
github.com/fiatjaf/eventstore
|
github.com/fiatjaf/eventstore
|
||||||
github.com/fiatjaf/eventstore/postgresql
|
github.com/fiatjaf/eventstore/postgresql
|
||||||
# github.com/fiatjaf/khatru v0.12.0
|
# github.com/fiatjaf/khatru v0.12.1
|
||||||
## explicit; go 1.23.1
|
## explicit; go 1.23.1
|
||||||
github.com/fiatjaf/khatru
|
github.com/fiatjaf/khatru
|
||||||
github.com/fiatjaf/khatru/policies
|
github.com/fiatjaf/khatru/policies
|
||||||
|
|
@ -53,7 +52,6 @@ github.com/gobwas/ws/wsflate
|
||||||
github.com/gobwas/ws/wsutil
|
github.com/gobwas/ws/wsutil
|
||||||
# github.com/greatroar/blobloom v0.8.0
|
# github.com/greatroar/blobloom v0.8.0
|
||||||
## explicit; go 1.14
|
## explicit; go 1.14
|
||||||
github.com/greatroar/blobloom
|
|
||||||
# github.com/jmoiron/sqlx v1.4.0
|
# github.com/jmoiron/sqlx v1.4.0
|
||||||
## explicit; go 1.10
|
## explicit; go 1.10
|
||||||
github.com/jmoiron/sqlx
|
github.com/jmoiron/sqlx
|
||||||
|
|
@ -90,7 +88,7 @@ github.com/mattn/go-colorable
|
||||||
# github.com/mattn/go-isatty v0.0.20
|
# github.com/mattn/go-isatty v0.0.20
|
||||||
## explicit; go 1.15
|
## explicit; go 1.15
|
||||||
github.com/mattn/go-isatty
|
github.com/mattn/go-isatty
|
||||||
# github.com/nbd-wtf/go-nostr v0.42.2
|
# github.com/nbd-wtf/go-nostr v0.42.3
|
||||||
## explicit; go 1.23.1
|
## explicit; go 1.23.1
|
||||||
github.com/nbd-wtf/go-nostr
|
github.com/nbd-wtf/go-nostr
|
||||||
github.com/nbd-wtf/go-nostr/nip11
|
github.com/nbd-wtf/go-nostr/nip11
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue