This commit is contained in:
Amarpreet Minhas 2024-10-12 16:50:18 -04:00
parent b8c301c92a
commit 4f6b5f519b
9 changed files with 157 additions and 39 deletions

View file

@ -21,6 +21,15 @@ func (e *Encoder) Encode(dst []byte, src []byte, matches []matchfinder.Match, la
e.wroteHeader = true e.wroteHeader = true
} }
if len(src) == 0 {
if lastBlock {
e.bw.writeBits(2, 3) // islast + isempty
e.bw.jumpToByteBoundary()
return e.bw.dst
}
return dst
}
var literalHisto [256]uint32 var literalHisto [256]uint32
var commandHisto [704]uint32 var commandHisto [704]uint32
var distanceHisto [64]uint32 var distanceHisto [64]uint32

View file

@ -32,14 +32,3 @@ func (e *matchEmitter) emit(m absoluteMatch) {
}) })
e.NextEmit = m.End e.NextEmit = m.End
} }
// trim shortens m if it extends past maxEnd. Then if the length is at least
// minLength, the match is emitted.
func (e *matchEmitter) trim(m absoluteMatch, maxEnd int, minLength int) {
if m.End > maxEnd {
m.End = maxEnd
}
if m.End-m.Start >= minLength {
e.emit(m)
}
}

View file

@ -56,7 +56,7 @@ func (q *M4) Reset() {
} }
func (q *M4) score(m absoluteMatch) int { func (q *M4) score(m absoluteMatch) int {
return (m.End-m.Start)*256 + bits.LeadingZeros32(uint32(m.Start-m.Match))*q.DistanceBitCost return (m.End-m.Start)*256 + (bits.LeadingZeros32(uint32(m.Start-m.Match))-32)*q.DistanceBitCost
} }
func (q *M4) FindMatches(dst []Match, src []byte) []Match { func (q *M4) FindMatches(dst []Match, src []byte) []Match {
@ -112,7 +112,12 @@ func (q *M4) FindMatches(dst []Match, src []byte) []Match {
// We have found some matches, and we're far enough along that we probably // We have found some matches, and we're far enough along that we probably
// won't find overlapping matches, so we might as well emit them. // won't find overlapping matches, so we might as well emit them.
if matches[1] != (absoluteMatch{}) { if matches[1] != (absoluteMatch{}) {
e.trim(matches[1], matches[0].Start, q.MinLength) if matches[1].End > matches[0].Start {
matches[1].End = matches[0].Start
}
if matches[1].End-matches[1].Start >= q.MinLength && q.score(matches[1]) > 0 {
e.emit(matches[1])
}
} }
e.emit(matches[0]) e.emit(matches[0])
matches = [3]absoluteMatch{} matches = [3]absoluteMatch{}
@ -139,14 +144,12 @@ func (q *M4) FindMatches(dst []Match, src []byte) []Match {
// Look for a match. // Look for a match.
var currentMatch absoluteMatch var currentMatch absoluteMatch
if i-candidate != matches[0].Start-matches[0].Match {
if binary.LittleEndian.Uint32(src[candidate:]) == binary.LittleEndian.Uint32(src[i:]) { if binary.LittleEndian.Uint32(src[candidate:]) == binary.LittleEndian.Uint32(src[i:]) {
m := extendMatch2(src, i, candidate, e.NextEmit) m := extendMatch2(src, i, candidate, e.NextEmit)
if m.End-m.Start > q.MinLength { if m.End-m.Start > q.MinLength && q.score(m) > 0 {
currentMatch = m currentMatch = m
} }
} }
}
for j := 0; j < q.ChainLength; j++ { for j := 0; j < q.ChainLength; j++ {
delta := q.chain[candidate] delta := q.chain[candidate]
@ -157,7 +160,6 @@ func (q *M4) FindMatches(dst []Match, src []byte) []Match {
if candidate <= 0 || i-candidate > q.MaxDistance { if candidate <= 0 || i-candidate > q.MaxDistance {
break break
} }
if i-candidate != matches[0].Start-matches[0].Match {
if binary.LittleEndian.Uint32(src[candidate:]) == binary.LittleEndian.Uint32(src[i:]) { if binary.LittleEndian.Uint32(src[candidate:]) == binary.LittleEndian.Uint32(src[i:]) {
m := extendMatch2(src, i, candidate, e.NextEmit) m := extendMatch2(src, i, candidate, e.NextEmit)
if m.End-m.Start > q.MinLength && q.score(m) > q.score(currentMatch) { if m.End-m.Start > q.MinLength && q.score(m) > q.score(currentMatch) {
@ -165,7 +167,6 @@ func (q *M4) FindMatches(dst []Match, src []byte) []Match {
} }
} }
} }
}
if currentMatch.End-currentMatch.Start < q.MinLength { if currentMatch.End-currentMatch.Start < q.MinLength {
continue continue
@ -217,14 +218,24 @@ func (q *M4) FindMatches(dst []Match, src []byte) []Match {
default: default:
// Emit the first match, shortening it if necessary to avoid overlap with the second. // Emit the first match, shortening it if necessary to avoid overlap with the second.
e.trim(matches[2], matches[1].Start, q.MinLength) if matches[2].End > matches[1].Start {
matches[2].End = matches[1].Start
}
if matches[2].End-matches[2].Start >= q.MinLength && q.score(matches[2]) > 0 {
e.emit(matches[2])
}
matches[2] = absoluteMatch{} matches[2] = absoluteMatch{}
} }
} }
// We've found all the matches now; emit the remaining ones. // We've found all the matches now; emit the remaining ones.
if matches[1] != (absoluteMatch{}) { if matches[1] != (absoluteMatch{}) {
e.trim(matches[1], matches[0].Start, q.MinLength) if matches[1].End > matches[0].Start {
matches[1].End = matches[0].Start
}
if matches[1].End-matches[1].Start >= q.MinLength && q.score(matches[1]) > 0 {
e.emit(matches[1])
}
} }
if matches[0] != (absoluteMatch{}) { if matches[0] != (absoluteMatch{}) {
e.emit(matches[0]) e.emit(matches[0])

View file

@ -1,5 +1,5 @@
# This is an example goreleaser.yaml file with some sane defaults. version: 2
# Make sure to check the documentation at http://goreleaser.com
before: before:
hooks: hooks:
- ./gen.sh - ./gen.sh
@ -99,7 +99,7 @@ archives:
checksum: checksum:
name_template: 'checksums.txt' name_template: 'checksums.txt'
snapshot: snapshot:
name_template: "{{ .Tag }}-next" version_template: "{{ .Tag }}-next"
changelog: changelog:
sort: asc sort: asc
filters: filters:

View file

@ -16,6 +16,13 @@ This package provides various compression algorithms.
# changelog # changelog
* Sep 23rd, 2024 - [1.17.10](https://github.com/klauspost/compress/releases/tag/v1.17.10)
* gzhttp: Add TransportAlwaysDecompress option. https://github.com/klauspost/compress/pull/978
* gzhttp: Add supported decompress request body by @mirecl in https://github.com/klauspost/compress/pull/1002
* s2: Add EncodeBuffer buffer recycling callback https://github.com/klauspost/compress/pull/982
* zstd: Improve memory usage on small streaming encodes https://github.com/klauspost/compress/pull/1007
* flate: read data written with partial flush by @vajexal in https://github.com/klauspost/compress/pull/996
* Jun 12th, 2024 - [1.17.9](https://github.com/klauspost/compress/releases/tag/v1.17.9) * Jun 12th, 2024 - [1.17.9](https://github.com/klauspost/compress/releases/tag/v1.17.9)
* s2: Reduce ReadFrom temporary allocations https://github.com/klauspost/compress/pull/949 * s2: Reduce ReadFrom temporary allocations https://github.com/klauspost/compress/pull/949
* flate, zstd: Shave some bytes off amd64 matchLen by @greatroar in https://github.com/klauspost/compress/pull/963 * flate, zstd: Shave some bytes off amd64 matchLen by @greatroar in https://github.com/klauspost/compress/pull/963

View file

@ -6,6 +6,7 @@ package zstd
import ( import (
"crypto/rand" "crypto/rand"
"errors"
"fmt" "fmt"
"io" "io"
"math" "math"
@ -149,6 +150,9 @@ func (e *Encoder) ResetContentSize(w io.Writer, size int64) {
// and write CRC if requested. // and write CRC if requested.
func (e *Encoder) Write(p []byte) (n int, err error) { func (e *Encoder) Write(p []byte) (n int, err error) {
s := &e.state s := &e.state
if s.eofWritten {
return 0, ErrEncoderClosed
}
for len(p) > 0 { for len(p) > 0 {
if len(p)+len(s.filling) < e.o.blockSize { if len(p)+len(s.filling) < e.o.blockSize {
if e.o.crc { if e.o.crc {
@ -288,6 +292,9 @@ func (e *Encoder) nextBlock(final bool) error {
s.filling, s.current, s.previous = s.previous[:0], s.filling, s.current s.filling, s.current, s.previous = s.previous[:0], s.filling, s.current
s.nInput += int64(len(s.current)) s.nInput += int64(len(s.current))
s.wg.Add(1) s.wg.Add(1)
if final {
s.eofWritten = true
}
go func(src []byte) { go func(src []byte) {
if debugEncoder { if debugEncoder {
println("Adding block,", len(src), "bytes, final:", final) println("Adding block,", len(src), "bytes, final:", final)
@ -303,9 +310,6 @@ func (e *Encoder) nextBlock(final bool) error {
blk := enc.Block() blk := enc.Block()
enc.Encode(blk, src) enc.Encode(blk, src)
blk.last = final blk.last = final
if final {
s.eofWritten = true
}
// Wait for pending writes. // Wait for pending writes.
s.wWg.Wait() s.wWg.Wait()
if s.writeErr != nil { if s.writeErr != nil {
@ -401,12 +405,20 @@ func (e *Encoder) Flush() error {
if len(s.filling) > 0 { if len(s.filling) > 0 {
err := e.nextBlock(false) err := e.nextBlock(false)
if err != nil { if err != nil {
// Ignore Flush after Close.
if errors.Is(s.err, ErrEncoderClosed) {
return nil
}
return err return err
} }
} }
s.wg.Wait() s.wg.Wait()
s.wWg.Wait() s.wWg.Wait()
if s.err != nil { if s.err != nil {
// Ignore Flush after Close.
if errors.Is(s.err, ErrEncoderClosed) {
return nil
}
return s.err return s.err
} }
return s.writeErr return s.writeErr
@ -422,6 +434,9 @@ func (e *Encoder) Close() error {
} }
err := e.nextBlock(true) err := e.nextBlock(true)
if err != nil { if err != nil {
if errors.Is(s.err, ErrEncoderClosed) {
return nil
}
return err return err
} }
if s.frameContentSize > 0 { if s.frameContentSize > 0 {
@ -459,6 +474,11 @@ func (e *Encoder) Close() error {
} }
_, s.err = s.w.Write(frame) _, s.err = s.w.Write(frame)
} }
if s.err == nil {
s.err = ErrEncoderClosed
return nil
}
return s.err return s.err
} }

View file

@ -88,6 +88,10 @@ var (
// Close has been called. // Close has been called.
ErrDecoderClosed = errors.New("decoder used after Close") ErrDecoderClosed = errors.New("decoder used after Close")
// ErrEncoderClosed will be returned if the Encoder was used after
// Close has been called.
ErrEncoderClosed = errors.New("encoder used after Close")
// ErrDecoderNilInput is returned when a nil Reader was provided // ErrDecoderNilInput is returned when a nil Reader was provided
// and an operation other than Reset/DecodeAll/Close was attempted. // and an operation other than Reset/DecodeAll/Close was attempted.
ErrDecoderNilInput = errors.New("nil input provided as reader") ErrDecoderNilInput = errors.New("nil input provided as reader")

View file

@ -9,16 +9,45 @@ const (
KindDeletion int = 5 KindDeletion int = 5
KindRepost int = 6 KindRepost int = 6
KindReaction int = 7 KindReaction int = 7
KindBadgeAward int = 8
KindSimpleGroupChatMessage int = 9 KindSimpleGroupChatMessage int = 9
KindSimpleGroupThreadedReply int = 10
KindSimpleGroupThread int = 11 KindSimpleGroupThread int = 11
KindSimpleGroupReply int = 12 KindSimpleGroupReply int = 12
KindSeal int = 13
KindDirectMessage int = 14
KindGenericRepost int = 16
KindReactionToWebsite int = 17
KindChannelCreation int = 40 KindChannelCreation int = 40
KindChannelMetadata int = 41 KindChannelMetadata int = 41
KindChannelMessage int = 42 KindChannelMessage int = 42
KindChannelHideMessage int = 43 KindChannelHideMessage int = 43
KindChannelMuteUser int = 44 KindChannelMuteUser int = 44
KindPatch int = 1617 KindChess int = 64
KindMergeRequests int = 818
KindBid int = 1021
KIndBidConfirmation int = 1022
KindOpenTimestamps int = 1040
KindGiftWrap int = 1059
KindFileMetadata int = 1063 KindFileMetadata int = 1063
KindLiveChatMessage int = 1311
KindPatch int = 1617
KindIssue int = 1621
KindReply int = 1622
KindStatusOpen int = 1630
KindStatusApplied int = 1631
KindStatusClosed int = 1632
KindStatusDraft int = 1633
KindProblemTracker int = 1971
KindReporting int = 1984
KindLabel int = 1985
KindRelayReviews int = 1986
KindAIEmbeddings int = 1987
KindTorrent int = 2003
KindTorrentComment int = 2004
KindCoinjoinPool int = 2022
KindCommunityPostApproval int = 4550
KindJobFeedback int = 7000
KindSimpleGroupAddUser int = 9000 KindSimpleGroupAddUser int = 9000
KindSimpleGroupRemoveUser int = 9001 KindSimpleGroupRemoveUser int = 9001
KindSimpleGroupEditMetadata int = 9002 KindSimpleGroupEditMetadata int = 9002
@ -30,27 +59,76 @@ const (
KindSimpleGroupDeleteGroup int = 9008 KindSimpleGroupDeleteGroup int = 9008
KindSimpleGroupJoinRequest int = 9021 KindSimpleGroupJoinRequest int = 9021
KindSimpleGroupLeaveRequest int = 9022 KindSimpleGroupLeaveRequest int = 9022
KindZapGoal int = 9041
KindTidalLogin int = 9467
KindZapRequest int = 9734 KindZapRequest int = 9734
KindZap int = 9735 KindZap int = 9735
KindHighlights int = 9802
KindMuteList int = 10000 KindMuteList int = 10000
KindPinList int = 10001 KindPinList int = 10001
KindRelayListMetadata int = 10002 KindRelayListMetadata int = 10002
KindBookmarkList int = 10003
KindCommunityList int = 10004
KindPublicChatList int = 10005
KindBlockedRelayList int = 10006
KindSearchRelayList int = 10007
KindSimpleGroupList int = 10009
KindInterestList int = 10015
KindEmojiList int = 10030
KindDMRelayList int = 10050
KindUserServerList int = 10063
KindFileStorageServerList int = 10096
KindGoodWikiAuthorList int = 10101
KindGoodWikiRelayList int = 10102
KindNWCWalletInfo int = 13194 KindNWCWalletInfo int = 13194
KindLightningPubRPC int = 21000
KindClientAuthentication int = 22242 KindClientAuthentication int = 22242
KindNWCWalletRequest int = 23194 KindNWCWalletRequest int = 23194
KindNWCWalletResponse int = 23195 KindNWCWalletResponse int = 23195
KindNostrConnect int = 24133 KindNostrConnect int = 24133
KindBlobs int = 24242
KindHTTPAuth int = 27235
KindCategorizedPeopleList int = 30000 KindCategorizedPeopleList int = 30000
KindCategorizedBookmarksList int = 30001 KindCategorizedBookmarksList int = 30001
KindRelaySets int = 30002
KindBookmarkSets int = 30003
KindCuratedSets int = 30004
KindCuratedVideoSets int = 30005
KindMuteSets int = 30007
KindProfileBadges int = 30008 KindProfileBadges int = 30008
KindBadgeDefinition int = 30009 KindBadgeDefinition int = 30009
KindInterestSets int = 30015
KindStallDefinition int = 30017 KindStallDefinition int = 30017
KindProductDefinition int = 30018 KindProductDefinition int = 30018
KindMarketplaceUI int = 30019
KindProductSoldAsAuction int = 30020
KindArticle int = 30023 KindArticle int = 30023
KindDraftArticle int = 30024
KindEmojiSets int = 30030
KindModularArticleHeader int = 30040
KindModularArticleContent int = 30041
KindReleaseArtifactSets int = 30063
KindApplicationSpecificData int = 30078 KindApplicationSpecificData int = 30078
KindLiveEvent int = 30311
KindUserStatuses int = 30315
KindClassifiedListing int = 30402
KindDraftClassifiedListing int = 30403
KindRepositoryAnnouncement int = 30617 KindRepositoryAnnouncement int = 30617
KindRepositoryState int = 30618 KindRepositoryState int = 30618
KindSimpleGroupMetadata int = 39000 KindSimpleGroupMetadata int = 39000
KindWikiArticle int = 30818
KindRedirects int = 30819
KindFeed int = 31890
KindDateCalendarEvent int = 31922
KindTimeCalendarEvent int = 31923
KindCalendar int = 31924
KindCalendarEventRSVP int = 31925
KindHandlerRecommendation int = 31989
KindHandlerInformation int = 31990
KindVideoEvent int = 34235
KindShortVideoEvent int = 34236
KindVideoViewEvent int = 34237
KindCommunityDefinition int = 34550
KindSimpleGroupAdmins int = 39001 KindSimpleGroupAdmins int = 39001
KindSimpleGroupMembers int = 39002 KindSimpleGroupMembers int = 39002
) )

10
vendor/modules.txt vendored
View file

@ -1,7 +1,7 @@
# git.devvul.com/asara/gologger v0.9.0 # git.devvul.com/asara/gologger v0.9.0
## explicit; go 1.22.5 ## explicit; go 1.22.5
git.devvul.com/asara/gologger git.devvul.com/asara/gologger
# github.com/andybalholm/brotli v1.1.0 # github.com/andybalholm/brotli v1.1.1
## explicit; go 1.13 ## explicit; go 1.13
github.com/andybalholm/brotli github.com/andybalholm/brotli
github.com/andybalholm/brotli/matchfinder github.com/andybalholm/brotli/matchfinder
@ -23,7 +23,7 @@ github.com/decred/dcrd/dcrec/secp256k1/v4/schnorr
# github.com/fasthttp/websocket v1.5.10 # github.com/fasthttp/websocket v1.5.10
## explicit; go 1.20 ## explicit; go 1.20
github.com/fasthttp/websocket github.com/fasthttp/websocket
# github.com/fiatjaf/eventstore v0.11.0 # github.com/fiatjaf/eventstore v0.11.3
## explicit; go 1.23.0 ## explicit; go 1.23.0
github.com/fiatjaf/eventstore github.com/fiatjaf/eventstore
github.com/fiatjaf/eventstore/postgresql github.com/fiatjaf/eventstore/postgresql
@ -52,7 +52,7 @@ github.com/jmoiron/sqlx/reflectx
# github.com/josharian/intern v1.0.0 # github.com/josharian/intern v1.0.0
## explicit; go 1.5 ## explicit; go 1.5
github.com/josharian/intern github.com/josharian/intern
# github.com/klauspost/compress v1.17.10 # github.com/klauspost/compress v1.17.11
## explicit; go 1.21 ## explicit; go 1.21
github.com/klauspost/compress github.com/klauspost/compress
github.com/klauspost/compress/flate github.com/klauspost/compress/flate
@ -81,7 +81,7 @@ github.com/mattn/go-colorable
# github.com/mattn/go-isatty v0.0.20 # github.com/mattn/go-isatty v0.0.20
## explicit; go 1.15 ## explicit; go 1.15
github.com/mattn/go-isatty github.com/mattn/go-isatty
# github.com/nbd-wtf/go-nostr v0.38.1 # github.com/nbd-wtf/go-nostr v0.38.2
## explicit; go 1.23.0 ## explicit; go 1.23.0
github.com/nbd-wtf/go-nostr github.com/nbd-wtf/go-nostr
github.com/nbd-wtf/go-nostr/nip11 github.com/nbd-wtf/go-nostr/nip11
@ -119,7 +119,7 @@ github.com/valyala/bytebufferpool
github.com/valyala/fasthttp github.com/valyala/fasthttp
github.com/valyala/fasthttp/fasthttputil github.com/valyala/fasthttp/fasthttputil
github.com/valyala/fasthttp/stackless github.com/valyala/fasthttp/stackless
# golang.org/x/exp v0.0.0-20241004190924-225e2abe05e6 # golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c
## explicit; go 1.22.0 ## explicit; go 1.22.0
golang.org/x/exp/constraints golang.org/x/exp/constraints
# golang.org/x/net v0.30.0 # golang.org/x/net v0.30.0