Skip to content

Commit c226f64

Browse files
committed
strconv: pre-allocate in appendQuotedWith
The byte-at-a-time allocation done quoting strings in appendQuotedWith grows the output incrementally, which is poor behavior for very large strings. An easy fix is to make sure the buffer has enough room at least for an unquoted string. Add a benchmark with a megabyte of non-ASCII data. Before: 39 allocations. After: 7 allocations. We could do better by doing a lot more work but this seems like a big result for little effort. Fixes golang#31472. Change-Id: I852139e0a2bd13722c4dd329ded8ae1759abad5b Reviewed-on: https://go-review.googlesource.com/c/go/+/172677 Reviewed-by: Ian Lance Taylor <[email protected]> Run-TryBot: Ian Lance Taylor <[email protected]> TryBot-Result: Gobot Gobot <[email protected]>
1 parent cbaa8e5 commit c226f64

2 files changed

Lines changed: 17 additions & 0 deletions

File tree

src/strconv/quote.go

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,13 @@ func quoteRuneWith(r rune, quote byte, ASCIIonly, graphicOnly bool) string {
2525
}
2626

2727
func appendQuotedWith(buf []byte, s string, quote byte, ASCIIonly, graphicOnly bool) []byte {
28+
// Often called with big strings, so preallocate. If there's quoting,
29+
// this is conservative but still helps a lot.
30+
if cap(buf)-len(buf) < len(s) {
31+
nBuf := make([]byte, len(buf), len(buf)+1+len(s)+1)
32+
copy(nBuf, buf)
33+
buf = nBuf
34+
}
2835
buf = append(buf, quote)
2936
for width := 0; len(s) > 0; s = s[width:] {
3037
r := rune(s[0])

src/strconv/strconv_test.go

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,9 @@ var (
3030
AppendFloat(localBuf[:0], 1.23, 'g', 5, 64)
3131
}},
3232
{0, `AppendFloat(globalBuf[:0], 1.23, 'g', 5, 64)`, func() { AppendFloat(globalBuf[:0], 1.23, 'g', 5, 64) }},
33+
// In practice we see 7 for the next one, but allow some slop.
34+
// Before pre-allocation in appendQuotedWith, we saw 39.
35+
{10, `AppendQuoteToASCII(nil, oneMB)`, func() { AppendQuoteToASCII(nil, string(oneMB)) }},
3336
{0, `ParseFloat("123.45", 64)`, func() { ParseFloat("123.45", 64) }},
3437
{0, `ParseFloat("123.456789123456789", 64)`, func() { ParseFloat("123.456789123456789", 64) }},
3538
{0, `ParseFloat("1.000000000000000111022302462515654042363166809082031251", 64)`, func() {
@@ -41,13 +44,20 @@ var (
4144
}
4245
)
4346

47+
var oneMB []byte // Will be allocated to 1MB of random data by TestCountMallocs.
48+
4449
func TestCountMallocs(t *testing.T) {
4550
if testing.Short() {
4651
t.Skip("skipping malloc count in short mode")
4752
}
4853
if runtime.GOMAXPROCS(0) > 1 {
4954
t.Skip("skipping; GOMAXPROCS>1")
5055
}
56+
// Allocate a big messy buffer for AppendQuoteToASCII's test.
57+
oneMB = make([]byte, 1e6)
58+
for i := range oneMB {
59+
oneMB[i] = byte(i)
60+
}
5161
for _, mt := range mallocTest {
5262
allocs := testing.AllocsPerRun(100, mt.fn)
5363
if max := float64(mt.count); allocs > max {

0 commit comments

Comments
 (0)