Skip to content

Commit dbe3228

Browse files
cherrymuicherrymui
committed
cmd/link: mmap object data
This resurrects CL 121198, except that this time we map read-only. In case that we need to apply relocations to the symbol's content that is backed by read-only memory, we do our own copy- on-write. This can happen if we failed to mmap the output file, or we build for Wasm. Memory profile for building k8s.io/kubernetes/cmd/kube-apiserver on Linux/AMD64: Old (before this sequence of CLs): inuse_space 1598.75MB total 669.87MB 41.90% 41.90% 669.87MB 41.90% cmd/link/internal/objfile.(*objReader).readSlices New: inuse_space 1280.45MB total 441.18MB 34.46% 34.46% 441.18MB 34.46% cmd/link/internal/objfile.(*objReader).readSlices Change-Id: I6b4d29d6eee9828089ea3120eb38c212db21330b Reviewed-on: https://go-review.googlesource.com/c/go/+/170741 Run-TryBot: Cherry Zhang <[email protected]> Reviewed-by: Austin Clements <[email protected]> TryBot-Result: Gobot Gobot <[email protected]>
1 parent f957a7e commit dbe3228

7 files changed

Lines changed: 144 additions & 21 deletions

File tree

‎src/cmd/internal/bio/buf.go‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ package bio
77

88
import (
99
"bufio"
10+
"io"
1011
"log"
1112
"os"
1213
)
@@ -105,3 +106,26 @@ func (r *Reader) File() *os.File {
105106
func (w *Writer) File() *os.File {
106107
return w.f
107108
}
109+
110+
// Slice reads the next length bytes of r into a slice.
111+
//
112+
// This slice may be backed by mmap'ed memory. Currently, this memory
113+
// will never be unmapped. The second result reports whether the
114+
// backing memory is read-only.
115+
func (r *Reader) Slice(length uint64) ([]byte, bool, error) {
116+
if length == 0 {
117+
return []byte{}, false, nil
118+
}
119+
120+
data, ok := r.sliceOS(length)
121+
if ok {
122+
return data, true, nil
123+
}
124+
125+
data = make([]byte, length)
126+
_, err := io.ReadFull(r, data)
127+
if err != nil {
128+
return nil, false, err
129+
}
130+
return data, false, nil
131+
}

‎src/cmd/internal/bio/buf_mmap.go‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
// Copyright 2019 The Go Authors. All rights reserved.
2+
// Use of this source code is governed by a BSD-style
3+
// license that can be found in the LICENSE file.
4+
5+
// +build darwin dragonfly freebsd linux netbsd openbsd
6+
7+
package bio
8+
9+
import (
10+
"runtime"
11+
"sync/atomic"
12+
"syscall"
13+
)
14+
15+
// mmapLimit is the maximum number of mmaped regions to create before
16+
// falling back to reading into a heap-allocated slice. This exists
17+
// because some operating systems place a limit on the number of
18+
// distinct mapped regions per process. As of this writing:
19+
//
20+
// Darwin unlimited
21+
// DragonFly 1000000 (vm.max_proc_mmap)
22+
// FreeBSD unlimited
23+
// Linux 65530 (vm.max_map_count) // TODO: query /proc/sys/vm/max_map_count?
24+
// NetBSD unlimited
25+
// OpenBSD unlimited
26+
var mmapLimit int32 = 1<<31 - 1
27+
28+
func init() {
29+
// Linux is the only practically concerning OS.
30+
if runtime.GOOS == "linux" {
31+
mmapLimit = 30000
32+
}
33+
}
34+
35+
func (r *Reader) sliceOS(length uint64) ([]byte, bool) {
36+
// For small slices, don't bother with the overhead of a
37+
// mapping, especially since we have no way to unmap it.
38+
const threshold = 16 << 10
39+
if length < threshold {
40+
return nil, false
41+
}
42+
43+
// Have we reached the mmap limit?
44+
if atomic.AddInt32(&mmapLimit, -1) < 0 {
45+
atomic.AddInt32(&mmapLimit, 1)
46+
return nil, false
47+
}
48+
49+
// Page-align the offset.
50+
off := r.Offset()
51+
align := syscall.Getpagesize()
52+
aoff := off &^ int64(align-1)
53+
54+
data, err := syscall.Mmap(int(r.f.Fd()), aoff, int(length+uint64(off-aoff)), syscall.PROT_READ, syscall.MAP_SHARED|syscall.MAP_FILE)
55+
if err != nil {
56+
return nil, false
57+
}
58+
59+
data = data[off-aoff:]
60+
r.Seek(int64(length), 1)
61+
return data, true
62+
}

‎src/cmd/internal/bio/buf_nommap.go‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
// Copyright 2019 The Go Authors. All rights reserved.
2+
// Use of this source code is governed by a BSD-style
3+
// license that can be found in the LICENSE file.
4+
5+
// +build !darwin,!dragonfly,!freebsd,!linux,!netbsd,!openbsd
6+
7+
package bio
8+
9+
func (r *Reader) sliceOS(length uint64) ([]byte, bool) {
10+
return nil, false
11+
}

‎src/cmd/link/internal/ld/data.go‎

Lines changed: 30 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -127,6 +127,15 @@ func trampoline(ctxt *Link, s *sym.Symbol) {
127127
// This is a performance-critical function for the linker; be careful
128128
// to avoid introducing unnecessary allocations in the main loop.
129129
func relocsym(ctxt *Link, s *sym.Symbol) {
130+
if len(s.R) == 0 {
131+
return
132+
}
133+
if s.Attr.ReadOnly() {
134+
// The symbol's content is backed by read-only memory.
135+
// Copy it to writable memory to apply relocations.
136+
s.P = append([]byte(nil), s.P...)
137+
s.Attr.Set(sym.AttrReadOnly, false)
138+
}
130139
for ri := int32(0); ri < int32(len(s.R)); ri++ {
131140
r := &s.R[ri]
132141
if r.Done {
@@ -2384,17 +2393,21 @@ func compressSyms(ctxt *Link, syms []*sym.Symbol) []byte {
23842393
if err != nil {
23852394
log.Fatalf("NewWriterLevel failed: %s", err)
23862395
}
2387-
for _, sym := range syms {
2388-
// sym.P may be read-only. Apply relocations in a
2396+
for _, s := range syms {
2397+
// s.P may be read-only. Apply relocations in a
23892398
// temporary buffer, and immediately write it out.
2390-
oldP := sym.P
2391-
ctxt.relocbuf = append(ctxt.relocbuf[:0], sym.P...)
2392-
sym.P = ctxt.relocbuf
2393-
relocsym(ctxt, sym)
2394-
if _, err := z.Write(sym.P); err != nil {
2399+
oldP := s.P
2400+
wasReadOnly := s.Attr.ReadOnly()
2401+
if len(s.R) != 0 && wasReadOnly {
2402+
ctxt.relocbuf = append(ctxt.relocbuf[:0], s.P...)
2403+
s.P = ctxt.relocbuf
2404+
s.Attr.Set(sym.AttrReadOnly, false)
2405+
}
2406+
relocsym(ctxt, s)
2407+
if _, err := z.Write(s.P); err != nil {
23952408
log.Fatalf("compression failed: %s", err)
23962409
}
2397-
for i := sym.Size - int64(len(sym.P)); i > 0; {
2410+
for i := s.Size - int64(len(s.P)); i > 0; {
23982411
b := zeros[:]
23992412
if i < int64(len(b)) {
24002413
b = b[:i]
@@ -2405,13 +2418,15 @@ func compressSyms(ctxt *Link, syms []*sym.Symbol) []byte {
24052418
}
24062419
i -= int64(n)
24072420
}
2408-
// Restore sym.P, for 1. not holding temp buffer live
2409-
// unnecessarily, 2. if compression is not beneficial,
2410-
// we'll go back to use the uncompressed contents, in
2411-
// which case we still need sym.P.
2412-
sym.P = oldP
2413-
for i := range sym.R {
2414-
sym.R[i].Done = false
2421+
// Restore s.P if a temporary buffer was used. If compression
2422+
// is not beneficial, we'll go back to use the uncompressed
2423+
// contents, in which case we still need s.P.
2424+
if len(s.R) != 0 && wasReadOnly {
2425+
s.P = oldP
2426+
s.Attr.Set(sym.AttrReadOnly, wasReadOnly)
2427+
for i := range s.R {
2428+
s.R[i].Done = false
2429+
}
24152430
}
24162431
}
24172432
if err := z.Close(); err != nil {

‎src/cmd/link/internal/ld/outbuf.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -158,6 +158,7 @@ func (out *OutBuf) WriteSym(s *sym.Symbol) {
158158
start := out.off
159159
out.Write(s.P)
160160
s.P = out.buf[start:out.off]
161+
s.Attr.Set(sym.AttrReadOnly, false)
161162
} else {
162163
out.Write(s.P)
163164
}

‎src/cmd/link/internal/objfile/objfile.go‎

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ var emptyPkg = []byte(`"".`)
3434

3535
// objReader reads Go object files.
3636
type objReader struct {
37-
rd *bufio.Reader
37+
rd *bio.Reader
3838
arch *sys.Arch
3939
syms *sym.Symbols
4040
lib *sym.Library
@@ -43,6 +43,7 @@ type objReader struct {
4343
localSymVersion int
4444
flags int
4545
strictDupMsgs int
46+
dataSize int
4647

4748
// rdBuf is used by readString and readSymName as scratch for reading strings.
4849
rdBuf []byte
@@ -56,6 +57,8 @@ type objReader struct {
5657
funcdata []*sym.Symbol
5758
funcdataoff []int64
5859
file []*sym.Symbol
60+
61+
dataReadOnly bool // whether data is backed by read-only memory
5962
}
6063

6164
// Flags to enable optional behavior during object loading/reading.
@@ -76,7 +79,7 @@ const (
7679
func Load(arch *sys.Arch, syms *sym.Symbols, f *bio.Reader, lib *sym.Library, length int64, pn string, flags int) int {
7780
start := f.Offset()
7881
r := &objReader{
79-
rd: f.Reader,
82+
rd: f,
8083
lib: lib,
8184
arch: arch,
8285
syms: syms,
@@ -133,7 +136,10 @@ func (r *objReader) loadObjFile() {
133136
r.readSlices()
134137

135138
// Data section
136-
r.readFull(r.data)
139+
r.data, r.dataReadOnly, err = r.rd.Slice(uint64(r.dataSize))
140+
if err != nil {
141+
log.Fatalf("%s: error reading %s", r.pn, err)
142+
}
137143

138144
// Defined symbols
139145
for {
@@ -156,9 +162,8 @@ func (r *objReader) loadObjFile() {
156162
}
157163

158164
func (r *objReader) readSlices() {
165+
r.dataSize = r.readInt()
159166
n := r.readInt()
160-
r.data = make([]byte, n)
161-
n = r.readInt()
162167
r.reloc = make([]sym.Reloc, n)
163168
n = r.readInt()
164169
r.pcdata = make([]sym.Pcdata, n)
@@ -249,6 +254,7 @@ overwrite:
249254
dup.Gotype = typ
250255
}
251256
s.P = data
257+
s.Attr.Set(sym.AttrReadOnly, r.dataReadOnly)
252258
if nreloc > 0 {
253259
s.R = r.reloc[:nreloc:nreloc]
254260
if !isdup {

‎src/cmd/link/internal/sym/attribute.go‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,10 @@ const (
7878
// AttrTopFrame means that the function is an entry point and unwinders
7979
// should stop when they hit this function.
8080
AttrTopFrame
81-
// 18 attributes defined so far.
81+
// AttrReadOnly indicates whether the symbol's content (Symbol.P) is backed by
82+
// read-only memory.
83+
AttrReadOnly
84+
// 19 attributes defined so far.
8285
)
8386

8487
func (a Attribute) DuplicateOK() bool { return a&AttrDuplicateOK != 0 }
@@ -99,6 +102,7 @@ func (a Attribute) VisibilityHidden() bool { return a&AttrVisibilityHidden != 0
99102
func (a Attribute) SubSymbol() bool { return a&AttrSubSymbol != 0 }
100103
func (a Attribute) Container() bool { return a&AttrContainer != 0 }
101104
func (a Attribute) TopFrame() bool { return a&AttrTopFrame != 0 }
105+
func (a Attribute) ReadOnly() bool { return a&AttrReadOnly != 0 }
102106

103107
func (a Attribute) CgoExport() bool {
104108
return a.CgoExportDynamic() || a.CgoExportStatic()

0 commit comments

Comments
 (0)