This commit is contained in:
Amarpreet Minhas 2024-10-12 16:50:18 -04:00
parent b8c301c92a
commit 4f6b5f519b
9 changed files with 157 additions and 39 deletions

View file

@ -21,6 +21,15 @@ func (e *Encoder) Encode(dst []byte, src []byte, matches []matchfinder.Match, la
e.wroteHeader = true
}
if len(src) == 0 {
if lastBlock {
e.bw.writeBits(2, 3) // islast + isempty
e.bw.jumpToByteBoundary()
return e.bw.dst
}
return dst
}
var literalHisto [256]uint32
var commandHisto [704]uint32
var distanceHisto [64]uint32

View file

@ -32,14 +32,3 @@ func (e *matchEmitter) emit(m absoluteMatch) {
})
e.NextEmit = m.End
}
// trim shortens m if it extends past maxEnd. Then if the length is at least
// minLength, the match is emitted.
func (e *matchEmitter) trim(m absoluteMatch, maxEnd int, minLength int) {
if m.End > maxEnd {
m.End = maxEnd
}
if m.End-m.Start >= minLength {
e.emit(m)
}
}

View file

@ -56,7 +56,7 @@ func (q *M4) Reset() {
}
func (q *M4) score(m absoluteMatch) int {
return (m.End-m.Start)*256 + bits.LeadingZeros32(uint32(m.Start-m.Match))*q.DistanceBitCost
return (m.End-m.Start)*256 + (bits.LeadingZeros32(uint32(m.Start-m.Match))-32)*q.DistanceBitCost
}
func (q *M4) FindMatches(dst []Match, src []byte) []Match {
@ -112,7 +112,12 @@ func (q *M4) FindMatches(dst []Match, src []byte) []Match {
// We have found some matches, and we're far enough along that we probably
// won't find overlapping matches, so we might as well emit them.
if matches[1] != (absoluteMatch{}) {
e.trim(matches[1], matches[0].Start, q.MinLength)
if matches[1].End > matches[0].Start {
matches[1].End = matches[0].Start
}
if matches[1].End-matches[1].Start >= q.MinLength && q.score(matches[1]) > 0 {
e.emit(matches[1])
}
}
e.emit(matches[0])
matches = [3]absoluteMatch{}
@ -139,12 +144,10 @@ func (q *M4) FindMatches(dst []Match, src []byte) []Match {
// Look for a match.
var currentMatch absoluteMatch
if i-candidate != matches[0].Start-matches[0].Match {
if binary.LittleEndian.Uint32(src[candidate:]) == binary.LittleEndian.Uint32(src[i:]) {
m := extendMatch2(src, i, candidate, e.NextEmit)
if m.End-m.Start > q.MinLength {
currentMatch = m
}
if binary.LittleEndian.Uint32(src[candidate:]) == binary.LittleEndian.Uint32(src[i:]) {
m := extendMatch2(src, i, candidate, e.NextEmit)
if m.End-m.Start > q.MinLength && q.score(m) > 0 {
currentMatch = m
}
}
@ -157,12 +160,10 @@ func (q *M4) FindMatches(dst []Match, src []byte) []Match {
if candidate <= 0 || i-candidate > q.MaxDistance {
break
}
if i-candidate != matches[0].Start-matches[0].Match {
if binary.LittleEndian.Uint32(src[candidate:]) == binary.LittleEndian.Uint32(src[i:]) {
m := extendMatch2(src, i, candidate, e.NextEmit)
if m.End-m.Start > q.MinLength && q.score(m) > q.score(currentMatch) {
currentMatch = m
}
if binary.LittleEndian.Uint32(src[candidate:]) == binary.LittleEndian.Uint32(src[i:]) {
m := extendMatch2(src, i, candidate, e.NextEmit)
if m.End-m.Start > q.MinLength && q.score(m) > q.score(currentMatch) {
currentMatch = m
}
}
}
@ -217,14 +218,24 @@ func (q *M4) FindMatches(dst []Match, src []byte) []Match {
default:
// Emit the first match, shortening it if necessary to avoid overlap with the second.
e.trim(matches[2], matches[1].Start, q.MinLength)
if matches[2].End > matches[1].Start {
matches[2].End = matches[1].Start
}
if matches[2].End-matches[2].Start >= q.MinLength && q.score(matches[2]) > 0 {
e.emit(matches[2])
}
matches[2] = absoluteMatch{}
}
}
// We've found all the matches now; emit the remaining ones.
if matches[1] != (absoluteMatch{}) {
e.trim(matches[1], matches[0].Start, q.MinLength)
if matches[1].End > matches[0].Start {
matches[1].End = matches[0].Start
}
if matches[1].End-matches[1].Start >= q.MinLength && q.score(matches[1]) > 0 {
e.emit(matches[1])
}
}
if matches[0] != (absoluteMatch{}) {
e.emit(matches[0])

View file

@ -1,5 +1,5 @@
# This is an example goreleaser.yaml file with some sane defaults.
# Make sure to check the documentation at http://goreleaser.com
version: 2
before:
hooks:
- ./gen.sh
@ -99,7 +99,7 @@ archives:
checksum:
name_template: 'checksums.txt'
snapshot:
name_template: "{{ .Tag }}-next"
version_template: "{{ .Tag }}-next"
changelog:
sort: asc
filters:

View file

@ -16,6 +16,13 @@ This package provides various compression algorithms.
# changelog
* Sep 23rd, 2024 - [1.17.10](https://github.com/klauspost/compress/releases/tag/v1.17.10)
* gzhttp: Add TransportAlwaysDecompress option. https://github.com/klauspost/compress/pull/978
* gzhttp: Add supported decompress request body by @mirecl in https://github.com/klauspost/compress/pull/1002
* s2: Add EncodeBuffer buffer recycling callback https://github.com/klauspost/compress/pull/982
* zstd: Improve memory usage on small streaming encodes https://github.com/klauspost/compress/pull/1007
* flate: read data written with partial flush by @vajexal in https://github.com/klauspost/compress/pull/996
* Jun 12th, 2024 - [1.17.9](https://github.com/klauspost/compress/releases/tag/v1.17.9)
* s2: Reduce ReadFrom temporary allocations https://github.com/klauspost/compress/pull/949
* flate, zstd: Shave some bytes off amd64 matchLen by @greatroar in https://github.com/klauspost/compress/pull/963

View file

@ -6,6 +6,7 @@ package zstd
import (
"crypto/rand"
"errors"
"fmt"
"io"
"math"
@ -149,6 +150,9 @@ func (e *Encoder) ResetContentSize(w io.Writer, size int64) {
// and write CRC if requested.
func (e *Encoder) Write(p []byte) (n int, err error) {
s := &e.state
if s.eofWritten {
return 0, ErrEncoderClosed
}
for len(p) > 0 {
if len(p)+len(s.filling) < e.o.blockSize {
if e.o.crc {
@ -288,6 +292,9 @@ func (e *Encoder) nextBlock(final bool) error {
s.filling, s.current, s.previous = s.previous[:0], s.filling, s.current
s.nInput += int64(len(s.current))
s.wg.Add(1)
if final {
s.eofWritten = true
}
go func(src []byte) {
if debugEncoder {
println("Adding block,", len(src), "bytes, final:", final)
@ -303,9 +310,6 @@ func (e *Encoder) nextBlock(final bool) error {
blk := enc.Block()
enc.Encode(blk, src)
blk.last = final
if final {
s.eofWritten = true
}
// Wait for pending writes.
s.wWg.Wait()
if s.writeErr != nil {
@ -401,12 +405,20 @@ func (e *Encoder) Flush() error {
if len(s.filling) > 0 {
err := e.nextBlock(false)
if err != nil {
// Ignore Flush after Close.
if errors.Is(s.err, ErrEncoderClosed) {
return nil
}
return err
}
}
s.wg.Wait()
s.wWg.Wait()
if s.err != nil {
// Ignore Flush after Close.
if errors.Is(s.err, ErrEncoderClosed) {
return nil
}
return s.err
}
return s.writeErr
@ -422,6 +434,9 @@ func (e *Encoder) Close() error {
}
err := e.nextBlock(true)
if err != nil {
if errors.Is(s.err, ErrEncoderClosed) {
return nil
}
return err
}
if s.frameContentSize > 0 {
@ -459,6 +474,11 @@ func (e *Encoder) Close() error {
}
_, s.err = s.w.Write(frame)
}
if s.err == nil {
s.err = ErrEncoderClosed
return nil
}
return s.err
}

View file

@ -88,6 +88,10 @@ var (
// Close has been called.
ErrDecoderClosed = errors.New("decoder used after Close")
// ErrEncoderClosed will be returned if the Encoder was used after
// Close has been called.
ErrEncoderClosed = errors.New("encoder used after Close")
// ErrDecoderNilInput is returned when a nil Reader was provided
// and an operation other than Reset/DecodeAll/Close was attempted.
ErrDecoderNilInput = errors.New("nil input provided as reader")

View file

@ -9,16 +9,45 @@ const (
KindDeletion int = 5
KindRepost int = 6
KindReaction int = 7
KindBadgeAward int = 8
KindSimpleGroupChatMessage int = 9
KindSimpleGroupThreadedReply int = 10
KindSimpleGroupThread int = 11
KindSimpleGroupReply int = 12
KindSeal int = 13
KindDirectMessage int = 14
KindGenericRepost int = 16
KindReactionToWebsite int = 17
KindChannelCreation int = 40
KindChannelMetadata int = 41
KindChannelMessage int = 42
KindChannelHideMessage int = 43
KindChannelMuteUser int = 44
KindPatch int = 1617
KindChess int = 64
KindMergeRequests int = 818
KindBid int = 1021
KIndBidConfirmation int = 1022
KindOpenTimestamps int = 1040
KindGiftWrap int = 1059
KindFileMetadata int = 1063
KindLiveChatMessage int = 1311
KindPatch int = 1617
KindIssue int = 1621
KindReply int = 1622
KindStatusOpen int = 1630
KindStatusApplied int = 1631
KindStatusClosed int = 1632
KindStatusDraft int = 1633
KindProblemTracker int = 1971
KindReporting int = 1984
KindLabel int = 1985
KindRelayReviews int = 1986
KindAIEmbeddings int = 1987
KindTorrent int = 2003
KindTorrentComment int = 2004
KindCoinjoinPool int = 2022
KindCommunityPostApproval int = 4550
KindJobFeedback int = 7000
KindSimpleGroupAddUser int = 9000
KindSimpleGroupRemoveUser int = 9001
KindSimpleGroupEditMetadata int = 9002
@ -30,27 +59,76 @@ const (
KindSimpleGroupDeleteGroup int = 9008
KindSimpleGroupJoinRequest int = 9021
KindSimpleGroupLeaveRequest int = 9022
KindZapGoal int = 9041
KindTidalLogin int = 9467
KindZapRequest int = 9734
KindZap int = 9735
KindHighlights int = 9802
KindMuteList int = 10000
KindPinList int = 10001
KindRelayListMetadata int = 10002
KindBookmarkList int = 10003
KindCommunityList int = 10004
KindPublicChatList int = 10005
KindBlockedRelayList int = 10006
KindSearchRelayList int = 10007
KindSimpleGroupList int = 10009
KindInterestList int = 10015
KindEmojiList int = 10030
KindDMRelayList int = 10050
KindUserServerList int = 10063
KindFileStorageServerList int = 10096
KindGoodWikiAuthorList int = 10101
KindGoodWikiRelayList int = 10102
KindNWCWalletInfo int = 13194
KindLightningPubRPC int = 21000
KindClientAuthentication int = 22242
KindNWCWalletRequest int = 23194
KindNWCWalletResponse int = 23195
KindNostrConnect int = 24133
KindBlobs int = 24242
KindHTTPAuth int = 27235
KindCategorizedPeopleList int = 30000
KindCategorizedBookmarksList int = 30001
KindRelaySets int = 30002
KindBookmarkSets int = 30003
KindCuratedSets int = 30004
KindCuratedVideoSets int = 30005
KindMuteSets int = 30007
KindProfileBadges int = 30008
KindBadgeDefinition int = 30009
KindInterestSets int = 30015
KindStallDefinition int = 30017
KindProductDefinition int = 30018
KindMarketplaceUI int = 30019
KindProductSoldAsAuction int = 30020
KindArticle int = 30023
KindDraftArticle int = 30024
KindEmojiSets int = 30030
KindModularArticleHeader int = 30040
KindModularArticleContent int = 30041
KindReleaseArtifactSets int = 30063
KindApplicationSpecificData int = 30078
KindLiveEvent int = 30311
KindUserStatuses int = 30315
KindClassifiedListing int = 30402
KindDraftClassifiedListing int = 30403
KindRepositoryAnnouncement int = 30617
KindRepositoryState int = 30618
KindSimpleGroupMetadata int = 39000
KindWikiArticle int = 30818
KindRedirects int = 30819
KindFeed int = 31890
KindDateCalendarEvent int = 31922
KindTimeCalendarEvent int = 31923
KindCalendar int = 31924
KindCalendarEventRSVP int = 31925
KindHandlerRecommendation int = 31989
KindHandlerInformation int = 31990
KindVideoEvent int = 34235
KindShortVideoEvent int = 34236
KindVideoViewEvent int = 34237
KindCommunityDefinition int = 34550
KindSimpleGroupAdmins int = 39001
KindSimpleGroupMembers int = 39002
)

10
vendor/modules.txt vendored
View file

@ -1,7 +1,7 @@
# git.devvul.com/asara/gologger v0.9.0
## explicit; go 1.22.5
git.devvul.com/asara/gologger
# github.com/andybalholm/brotli v1.1.0
# github.com/andybalholm/brotli v1.1.1
## explicit; go 1.13
github.com/andybalholm/brotli
github.com/andybalholm/brotli/matchfinder
@ -23,7 +23,7 @@ github.com/decred/dcrd/dcrec/secp256k1/v4/schnorr
# github.com/fasthttp/websocket v1.5.10
## explicit; go 1.20
github.com/fasthttp/websocket
# github.com/fiatjaf/eventstore v0.11.0
# github.com/fiatjaf/eventstore v0.11.3
## explicit; go 1.23.0
github.com/fiatjaf/eventstore
github.com/fiatjaf/eventstore/postgresql
@ -52,7 +52,7 @@ github.com/jmoiron/sqlx/reflectx
# github.com/josharian/intern v1.0.0
## explicit; go 1.5
github.com/josharian/intern
# github.com/klauspost/compress v1.17.10
# github.com/klauspost/compress v1.17.11
## explicit; go 1.21
github.com/klauspost/compress
github.com/klauspost/compress/flate
@ -81,7 +81,7 @@ github.com/mattn/go-colorable
# github.com/mattn/go-isatty v0.0.20
## explicit; go 1.15
github.com/mattn/go-isatty
# github.com/nbd-wtf/go-nostr v0.38.1
# github.com/nbd-wtf/go-nostr v0.38.2
## explicit; go 1.23.0
github.com/nbd-wtf/go-nostr
github.com/nbd-wtf/go-nostr/nip11
@ -119,7 +119,7 @@ github.com/valyala/bytebufferpool
github.com/valyala/fasthttp
github.com/valyala/fasthttp/fasthttputil
github.com/valyala/fasthttp/stackless
# golang.org/x/exp v0.0.0-20241004190924-225e2abe05e6
# golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c
## explicit; go 1.22.0
golang.org/x/exp/constraints
# golang.org/x/net v0.30.0