diff options
author | Filippo Valsorda <filippo@golang.org> | 2020-06-09 13:02:10 -0400 |
---|---|---|
committer | Filippo Valsorda <filippo@golang.org> | 2020-06-09 13:02:10 -0400 |
commit | fcdb6aa6ee5ddeacc7472e38c0065b63e9d954ce (patch) | |
tree | 6e259dbaec400d37530c0bec6ecdda5647364e43 | |
parent | 2b0d842f4b24b3be4bcf02c7a796e3a4d3d952f6 (diff) | |
parent | 83b181c68bf332ac7948f145f33d128377a09c42 (diff) | |
download | go-fcdb6aa6ee5ddeacc7472e38c0065b63e9d954ce.tar.gz go-fcdb6aa6ee5ddeacc7472e38c0065b63e9d954ce.zip |
[dev.boringcrypto.go1.14] all: merge go1.14.4 into dev.boringcrypto.go1.14
Change-Id: I71db92e18e105a2082a51aec0bdee8e5dd0727ed
36 files changed, 524 insertions, 102 deletions
diff --git a/misc/cgo/test/testx.go b/misc/cgo/test/testx.go index 42979b5f4d..eb9d7fa47d 100644 --- a/misc/cgo/test/testx.go +++ b/misc/cgo/test/testx.go @@ -124,6 +124,11 @@ typedef struct { } Issue31891B; void callIssue31891(void); + +typedef struct { + int i; +} Issue38408, *PIssue38408; + */ import "C" @@ -552,3 +557,8 @@ func useIssue31891B(c *C.Issue31891B) {} func test31891(t *testing.T) { C.callIssue31891() } + +// issue 38408 +// A typedef pointer can be used as the element type. +// No runtime test; just make sure it compiles. +var _ C.PIssue38408 = &C.Issue38408{i: 1} diff --git a/src/cmd/cgo/gcc.go b/src/cmd/cgo/gcc.go index c4128e9502..7f99057a49 100644 --- a/src/cmd/cgo/gcc.go +++ b/src/cmd/cgo/gcc.go @@ -2060,6 +2060,10 @@ var goIdent = make(map[string]*ast.Ident) // that may contain a pointer. This is used for cgo pointer checking. var unionWithPointer = make(map[ast.Expr]bool) +// anonymousStructTag provides a consistent tag for an anonymous struct. +// The same dwarf.StructType pointer will always get the same tag. +var anonymousStructTag = make(map[*dwarf.StructType]string) + func (c *typeConv) Init(ptrSize, intSize int64) { c.ptrSize = ptrSize c.intSize = intSize @@ -2408,8 +2412,12 @@ func (c *typeConv) loadType(dtype dwarf.Type, pos token.Pos, parent string) *Typ break } if tag == "" { - tag = "__" + strconv.Itoa(tagGen) - tagGen++ + tag = anonymousStructTag[dt] + if tag == "" { + tag = "__" + strconv.Itoa(tagGen) + tagGen++ + anonymousStructTag[dt] = tag + } } else if t.C.Empty() { t.C.Set(dt.Kind + " " + tag) } diff --git a/src/cmd/compile/internal/gc/subr.go b/src/cmd/compile/internal/gc/subr.go index 3b085a0d64..73ed34f86a 100644 --- a/src/cmd/compile/internal/gc/subr.go +++ b/src/cmd/compile/internal/gc/subr.go @@ -542,7 +542,7 @@ func methtype(t *types.Type) *types.Type { // Is type src assignment compatible to type dst? // If so, return op code to use in conversion. // If not, return 0. -func assignop(src *types.Type, dst *types.Type, why *string) Op { +func assignop(src, dst *types.Type, why *string) Op { if why != nil { *why = "" } @@ -665,7 +665,8 @@ func assignop(src *types.Type, dst *types.Type, why *string) Op { // Can we convert a value of type src to a value of type dst? // If so, return op code to use in conversion (maybe OCONVNOP). // If not, return 0. -func convertop(src *types.Type, dst *types.Type, why *string) Op { +// srcConstant indicates whether the value of type src is a constant. +func convertop(srcConstant bool, src, dst *types.Type, why *string) Op { if why != nil { *why = "" } @@ -741,6 +742,13 @@ func convertop(src *types.Type, dst *types.Type, why *string) Op { return OCONV } + // Special case for constant conversions: any numeric + // conversion is potentially okay. We'll validate further + // within evconst. See #38117. + if srcConstant && (src.IsInteger() || src.IsFloat() || src.IsComplex()) && (dst.IsInteger() || dst.IsFloat() || dst.IsComplex()) { + return OCONV + } + // 6. src is an integer or has type []byte or []rune // and dst is a string type. if src.IsInteger() && dst.IsString() { diff --git a/src/cmd/compile/internal/gc/typecheck.go b/src/cmd/compile/internal/gc/typecheck.go index ae2e16760d..2ed736367f 100644 --- a/src/cmd/compile/internal/gc/typecheck.go +++ b/src/cmd/compile/internal/gc/typecheck.go @@ -1634,7 +1634,7 @@ func typecheck1(n *Node, top int) (res *Node) { return n } var why string - n.Op = convertop(t, n.Type, &why) + n.Op = convertop(n.Left.Op == OLITERAL, t, n.Type, &why) if n.Op == 0 { if !n.Diag() && !n.Type.Broke() && !n.Left.Diag() { yyerror("cannot convert %L to type %v%s", n.Left, n.Type, why) diff --git a/src/cmd/compile/internal/ssa/regalloc.go b/src/cmd/compile/internal/ssa/regalloc.go index e125ae4239..95e732d744 100644 --- a/src/cmd/compile/internal/ssa/regalloc.go +++ b/src/cmd/compile/internal/ssa/regalloc.go @@ -977,25 +977,22 @@ func (s *regAllocState) regalloc(f *Func) { } } - // Second pass - deallocate any phi inputs which are now dead. + // Second pass - deallocate all in-register phi inputs. for i, v := range phis { if !s.values[v.ID].needReg { continue } a := v.Args[idx] - if !regValLiveSet.contains(a.ID) { - // Input is dead beyond the phi, deallocate - // anywhere else it might live. - s.freeRegs(s.values[a.ID].regs) - } else { - // Input is still live. + r := phiRegs[i] + if r == noRegister { + continue + } + if regValLiveSet.contains(a.ID) { + // Input value is still live (it is used by something other than Phi). // Try to move it around before kicking out, if there is a free register. // We generate a Copy in the predecessor block and record it. It will be - // deleted if never used. - r := phiRegs[i] - if r == noRegister { - continue - } + // deleted later if never used. + // // Pick a free register. At this point some registers used in the predecessor // block may have been deallocated. Those are the ones used for Phis. Exclude // them (and they are not going to be helpful anyway). @@ -1011,8 +1008,8 @@ func (s *regAllocState) regalloc(f *Func) { s.assignReg(r2, a, c) s.endRegs[p.ID] = append(s.endRegs[p.ID], endReg{r2, a, c}) } - s.freeReg(r) } + s.freeReg(r) } // Copy phi ops into new schedule. @@ -1852,6 +1849,11 @@ func (s *regAllocState) shuffle(stacklive [][]ID) { e.process() } } + + if s.f.pass.debug > regDebug { + fmt.Printf("post shuffle %s\n", s.f.Name) + fmt.Println(s.f.String()) + } } type edgeState struct { diff --git a/src/cmd/doc/doc_test.go b/src/cmd/doc/doc_test.go index c0959acca1..fd2ae30827 100644 --- a/src/cmd/doc/doc_test.go +++ b/src/cmd/doc/doc_test.go @@ -724,6 +724,40 @@ var tests = []test{ }, }, + // Merging comments with -src. + { + "merge comments with -src A", + []string{"-src", p + "/merge", `A`}, + []string{ + `A doc`, + `func A`, + `A comment`, + }, + []string{ + `Package A doc`, + `Package B doc`, + `B doc`, + `B comment`, + `B doc`, + }, + }, + { + "merge comments with -src B", + []string{"-src", p + "/merge", `B`}, + []string{ + `B doc`, + `func B`, + `B comment`, + }, + []string{ + `Package A doc`, + `Package B doc`, + `A doc`, + `A comment`, + `A doc`, + }, + }, + // No dups with -u. Issue 21797. { "case matching on, no dups", diff --git a/src/cmd/doc/testdata/merge/aa.go b/src/cmd/doc/testdata/merge/aa.go new file mode 100644 index 0000000000..f8ab92dfd0 --- /dev/null +++ b/src/cmd/doc/testdata/merge/aa.go @@ -0,0 +1,7 @@ +// Package comment A. +package merge + +// A doc. +func A() { + // A comment. +} diff --git a/src/cmd/doc/testdata/merge/bb.go b/src/cmd/doc/testdata/merge/bb.go new file mode 100644 index 0000000000..fd8cf3c446 --- /dev/null +++ b/src/cmd/doc/testdata/merge/bb.go @@ -0,0 +1,7 @@ +// Package comment B. +package merge + +// B doc. +func B() { + // B comment. +} diff --git a/src/encoding/json/decode.go b/src/encoding/json/decode.go index b43484692e..b60e2bb0b2 100644 --- a/src/encoding/json/decode.go +++ b/src/encoding/json/decode.go @@ -1217,6 +1217,11 @@ func (d *decodeState) unquoteBytes(s []byte) (t []byte, ok bool) { if r == -1 { return s, true } + // Only perform up to one safe unquote for each re-scanned string + // literal. In some edge cases, the decoder unquotes a literal a second + // time, even after another literal has been re-scanned. Thus, only the + // first unquote can safely use safeUnquote. + d.safeUnquote = 0 b := make([]byte, len(s)+2*utf8.UTFMax) w := copy(b, s[0:r]) diff --git a/src/encoding/json/decode_test.go b/src/encoding/json/decode_test.go index 498bd97b46..a49181e982 100644 --- a/src/encoding/json/decode_test.go +++ b/src/encoding/json/decode_test.go @@ -2419,7 +2419,7 @@ func (m *textUnmarshalerString) UnmarshalText(text []byte) error { return nil } -// Test unmarshal to a map, with map key is a user defined type. +// Test unmarshal to a map, where the map key is a user defined type. // See golang.org/issues/34437. func TestUnmarshalMapWithTextUnmarshalerStringKey(t *testing.T) { var p map[textUnmarshalerString]string @@ -2428,6 +2428,35 @@ func TestUnmarshalMapWithTextUnmarshalerStringKey(t *testing.T) { } if _, ok := p["foo"]; !ok { - t.Errorf(`Key "foo" is not existed in map: %v`, p) + t.Errorf(`Key "foo" does not exist in map: %v`, p) + } +} + +func TestUnmarshalRescanLiteralMangledUnquote(t *testing.T) { + // See golang.org/issues/38105. + var p map[textUnmarshalerString]string + if err := Unmarshal([]byte(`{"开源":"12345开源"}`), &p); err != nil { + t.Fatalf("Unmarshal unexpected error: %v", err) + } + if _, ok := p["开源"]; !ok { + t.Errorf(`Key "开源" does not exist in map: %v`, p) + } + + // See golang.org/issues/38126. + type T struct { + F1 string `json:"F1,string"` + } + t1 := T{"aaa\tbbb"} + + b, err := Marshal(t1) + if err != nil { + t.Fatalf("Marshal unexpected error: %v", err) + } + var t2 T + if err := Unmarshal(b, &t2); err != nil { + t.Fatalf("Unmarshal unexpected error: %v", err) + } + if t1 != t2 { + t.Errorf("Marshal and Unmarshal roundtrip mismatch: want %q got %q", t1, t2) } } diff --git a/src/encoding/json/encode.go b/src/encoding/json/encode.go index 39cdaebde7..b351cf3f44 100644 --- a/src/encoding/json/encode.go +++ b/src/encoding/json/encode.go @@ -635,11 +635,12 @@ func stringEncoder(e *encodeState, v reflect.Value, opts encOpts) { return } if opts.quoted { - b := make([]byte, 0, v.Len()+2) - b = append(b, '"') - b = append(b, []byte(v.String())...) - b = append(b, '"') - e.stringBytes(b, opts.escapeHTML) + e2 := newEncodeState() + // Since we encode the string twice, we only need to escape HTML + // the first time. + e2.string(v.String(), opts.escapeHTML) + e.stringBytes(e2.Bytes(), false) + encodeStatePool.Put(e2) } else { e.string(v.String(), opts.escapeHTML) } diff --git a/src/encoding/json/encode_test.go b/src/encoding/json/encode_test.go index 5110c7de9b..7290eca06f 100644 --- a/src/encoding/json/encode_test.go +++ b/src/encoding/json/encode_test.go @@ -79,37 +79,66 @@ type StringTag struct { NumberStr Number `json:",string"` } -var stringTagExpected = `{ - "BoolStr": "true", - "IntStr": "42", - "UintptrStr": "44", - "StrStr": "\"xzbit\"", - "NumberStr": "46" -}` - -func TestStringTag(t *testing.T) { - var s StringTag - s.BoolStr = true - s.IntStr = 42 - s.UintptrStr = 44 - s.StrStr = "xzbit" - s.NumberStr = "46" - got, err := MarshalIndent(&s, "", " ") - if err != nil { - t.Fatal(err) - } - if got := string(got); got != stringTagExpected { - t.Fatalf(" got: %s\nwant: %s\n", got, stringTagExpected) +func TestRoundtripStringTag(t *testing.T) { + tests := []struct { + name string + in StringTag + want string // empty to just test that we roundtrip + }{ + { + name: "AllTypes", + in: StringTag{ + BoolStr: true, + IntStr: 42, + UintptrStr: 44, + StrStr: "xzbit", + NumberStr: "46", + }, + want: `{ + "BoolStr": "true", + "IntStr": "42", + "UintptrStr": "44", + "StrStr": "\"xzbit\"", + "NumberStr": "46" + }`, + }, + { + // See golang.org/issues/38173. + name: "StringDoubleEscapes", + in: StringTag{ + StrStr: "\b\f\n\r\t\"\\", + NumberStr: "0", // just to satisfy the roundtrip + }, + want: `{ + "BoolStr": "false", + "IntStr": "0", + "UintptrStr": "0", + "StrStr": "\"\\u0008\\u000c\\n\\r\\t\\\"\\\\\"", + "NumberStr": "0" + }`, + }, } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + // Indent with a tab prefix to make the multi-line string + // literals in the table nicer to read. + got, err := MarshalIndent(&test.in, "\t\t\t", "\t") + if err != nil { + t.Fatal(err) + } + if got := string(got); got != test.want { + t.Fatalf(" got: %s\nwant: %s\n", got, test.want) + } - // Verify that it round-trips. - var s2 StringTag - err = NewDecoder(bytes.NewReader(got)).Decode(&s2) - if err != nil { - t.Fatalf("Decode: %v", err) - } - if !reflect.DeepEqual(s, s2) { - t.Fatalf("decode didn't match.\nsource: %#v\nEncoded as:\n%s\ndecode: %#v", s, string(got), s2) + // Verify that it round-trips. + var s2 StringTag + if err := Unmarshal(got, &s2); err != nil { + t.Fatalf("Decode: %v", err) + } + if !reflect.DeepEqual(test.in, s2) { + t.Fatalf("decode didn't match.\nsource: %#v\nEncoded as:\n%s\ndecode: %#v", test.in, string(got), s2) + } + }) } } diff --git a/src/encoding/json/stream_test.go b/src/encoding/json/stream_test.go index ebb4f231d1..c9e5334337 100644 --- a/src/encoding/json/stream_test.go +++ b/src/encoding/json/stream_test.go @@ -144,14 +144,15 @@ func TestEncoderSetEscapeHTML(t *testing.T) { }, { "stringOption", stringOption, - `{"bar":"\"\u003chtml\u003efoobar\u003c/html\u003e\""}`, + `{"bar":"\"\\u003chtml\\u003efoobar\\u003c/html\\u003e\""}`, `{"bar":"\"<html>foobar</html>\""}`, }, } { var buf bytes.Buffer enc := NewEncoder(&buf) if err := enc.Encode(tt.v); err != nil { - t.Fatalf("Encode(%s): %s", tt.name, err) + t.Errorf("Encode(%s): %s", tt.name, err) + continue } if got := strings.TrimSpace(buf.String()); got != tt.wantEscape { t.Errorf("Encode(%s) = %#q, want %#q", tt.name, got, tt.wantEscape) @@ -159,7 +160,8 @@ func TestEncoderSetEscapeHTML(t *testing.T) { buf.Reset() enc.SetEscapeHTML(false) if err := enc.Encode(tt.v); err != nil { - t.Fatalf("SetEscapeHTML(false) Encode(%s): %s", tt.name, err) + t.Errorf("SetEscapeHTML(false) Encode(%s): %s", tt.name, err) + continue } if got := strings.TrimSpace(buf.String()); got != tt.want { t.Errorf("SetEscapeHTML(false) Encode(%s) = %#q, want %#q", diff --git a/src/go/doc/example.go b/src/go/doc/example.go index a010d3a85a..ebf81189b5 100644 --- a/src/go/doc/example.go +++ b/src/go/doc/example.go @@ -62,9 +62,6 @@ func Examples(testFiles ...*ast.File) []*Example { if !ok || f.Recv != nil { continue } - if params := f.Type.Params; len(params.List) != 0 { - continue // function has params; not a valid example - } numDecl++ name := f.Name.Name if isTest(name, "Test") || isTest(name, "Benchmark") { @@ -74,6 +71,9 @@ func Examples(testFiles ...*ast.File) []*Example { if !isTest(name, "Example") { continue } + if params := f.Type.Params; len(params.List) != 0 { + continue // function has params; not a valid example + } if f.Body == nil { // ast.File.Body nil dereference (see issue 28044) continue } diff --git a/src/go/doc/example_test.go b/src/go/doc/example_test.go index cd2f469c2f..32db3cd7da 100644 --- a/src/go/doc/example_test.go +++ b/src/go/doc/example_test.go @@ -331,25 +331,65 @@ func main() { } ` +const exampleWholeFileFunction = `package foo_test + +func Foo(x int) { +} + +func Example() { + fmt.Println("Hello, world!") + // Output: Hello, world! +} +` + +const exampleWholeFileFunctionOutput = `package main + +func Foo(x int) { +} + +func main() { + fmt.Println("Hello, world!") +} +` + +var exampleWholeFileTestCases = []struct { + Title, Source, Play, Output string +}{ + { + "Methods", + exampleWholeFile, + exampleWholeFileOutput, + "Hello, world!\n", + }, + { + "Function", + exampleWholeFileFunction, + exampleWholeFileFunctionOutput, + "Hello, world!\n", + }, +} + func TestExamplesWholeFile(t *testing.T) { - fset := token.NewFileSet() - file, err := parser.ParseFile(fset, "test.go", strings.NewReader(exampleWholeFile), parser.ParseComments) - if err != nil { - t.Fatal(err) - } - es := doc.Examples(file) - if len(es) != 1 { - t.Fatalf("wrong number of examples; got %d want 1", len(es)) - } - e := es[0] - if e.Name != "" { - t.Errorf("got Name == %q, want %q", e.Name, "") - } - if g, w := formatFile(t, fset, e.Play), exampleWholeFileOutput; g != w { - t.Errorf("got Play == %q, want %q", g, w) - } - if g, w := e.Output, "Hello, world!\n"; g != w { - t.Errorf("got Output == %q, want %q", g, w) + for _, c := range exampleWholeFileTestCases { + fset := token.NewFileSet() + file, err := parser.ParseFile(fset, "test.go", strings.NewReader(c.Source), parser.ParseComments) + if err != nil { + t.Fatal(err) + } + es := doc.Examples(file) + if len(es) != 1 { + t.Fatalf("%s: wrong number of examples; got %d want 1", c.Title, len(es)) + } + e := es[0] + if e.Name != "" { + t.Errorf("%s: got Name == %q, want %q", c.Title, e.Name, "") + } + if g, w := formatFile(t, fset, e.Play), c.Play; g != w { + t.Errorf("%s: got Play == %q, want %q", c.Title, g, w) + } + if g, w := e.Output, c.Output; g != w { + t.Errorf("%s: got Output == %q, want %q", c.Title, g, w) + } } } diff --git a/src/go/parser/interface.go b/src/go/parser/interface.go index 500c98d496..54f9d7b80a 100644 --- a/src/go/parser/interface.go +++ b/src/go/parser/interface.go @@ -133,13 +133,7 @@ func ParseFile(fset *token.FileSet, filename string, src interface{}, mode Mode) // first error encountered are returned. // func ParseDir(fset *token.FileSet, path string, filter func(os.FileInfo) bool, mode Mode) (pkgs map[string]*ast.Package, first error) { - fd, err := os.Open(path) - if err != nil { - return nil, err - } - defer fd.Close() - - list, err := fd.Readdir(-1) + list, err := ioutil.ReadDir(path) if err != nil { return nil, err } diff --git a/src/math/big/nat.go b/src/math/big/nat.go index 1b771ca7c6..c31ec5156b 100644 --- a/src/math/big/nat.go +++ b/src/math/big/nat.go @@ -740,7 +740,8 @@ func (z nat) divLarge(u, uIn, vIn nat) (q, r nat) { // The remainder overwrites input u. // // Precondition: -// - len(q) >= len(u)-len(v) +// - q is large enough to hold the quotient u / v +// which has a maximum length of len(u)-len(v)+1. func (q nat) divBasic(u, v nat) { n := len(v) m := len(u) - n @@ -779,6 +780,8 @@ func (q nat) divBasic(u, v nat) { } // D4. + // Compute the remainder u - (q̂*v) << (_W*j). + // The subtraction may overflow if q̂ estimate was off by one. qhatv[n] = mulAddVWW(qhatv[0:n], v, qhat, 0) qhl := len(qhatv) if j+qhl > len(u) && qhatv[n] == 0 { @@ -787,7 +790,11 @@ func (q nat) divBasic(u, v nat) { c := subVV(u[j:j+qhl], u[j:], qhatv) if c != 0 { c := addVV(u[j:j+n], u[j:], v) - u[j+n] += c + // If n == qhl, the carry from subVV and the carry from addVV + // cancel out and don't affect u[j+n]. + if n < qhl { + u[j+n] += c + } qhat-- } @@ -827,6 +834,10 @@ func (z nat) divRecursive(u, v nat) { putNat(tmp) } +// divRecursiveStep computes the division of u by v. +// - z must be large enough to hold the quotient +// - the quotient will overwrite z +// - the remainder will overwrite u func (z nat) divRecursiveStep(u, v nat, depth int, tmp *nat, temps []*nat) { u = u.norm() v = v.norm() diff --git a/src/math/big/nat_test.go b/src/math/big/nat_test.go index 32f29e3876..89e913fc16 100644 --- a/src/math/big/nat_test.go +++ b/src/math/big/nat_test.go @@ -786,3 +786,21 @@ func TestNatDiv(t *testing.T) { } } } + +// TestIssue37499 triggers the edge case of divBasic where +// the inaccurate estimate of the first word's quotient +// happens at the very beginning of the loop. +func TestIssue37499(t *testing.T) { + // Choose u and v such that v is slightly larger than u >> N. + // This tricks divBasic into choosing 1 as the first word + // of the quotient. This works in both 32-bit and 64-bit settings. + u := natFromString("0x2b6c385a05be027f5c22005b63c42a1165b79ff510e1706b39f8489c1d28e57bb5ba4ef9fd9387a3e344402c0a453381") + v := natFromString("0x2b6c385a05be027f5c22005b63c42a1165b79ff510e1706c") + + q := nat(nil).make(8) + q.divBasic(u, v) + q = q.norm() + if s := string(q.utoa(16)); s != "fffffffffffffffffffffffffffffffffffffffffffffffb" { + t.Fatalf("incorrect quotient: %s", s) + } +} diff --git a/src/os/os_test.go b/src/os/os_test.go index 278c19e44b..31b3e67319 100644 --- a/src/os/os_test.go +++ b/src/os/os_test.go @@ -2448,3 +2448,38 @@ func TestDirSeek(t *testing.T) { } } } + +// Test that opening a file does not change its permissions. Issue 38225. +func TestOpenFileKeepsPermissions(t *testing.T) { + t.Parallel() + dir, err := ioutil.TempDir("", "TestOpenFileKeepsPermissions") + if err != nil { + t.Fatal(err) + } + defer RemoveAll(dir) + name := filepath.Join(dir, "x") + f, err := Create(name) + if err != nil { + t.Fatal(err) + } + if err := f.Close(); err != nil { + t.Error(err) + } + f, err = OpenFile(name, O_WRONLY|O_CREATE|O_TRUNC, 0) + if err != nil { + t.Fatal(err) + } + if fi, err := f.Stat(); err != nil { + t.Error(err) + } else if fi.Mode()&0222 == 0 { + t.Errorf("f.Stat.Mode after OpenFile is %v, should be writable", fi.Mode()) + } + if err := f.Close(); err != nil { + t.Error(err) + } + if fi, err := Stat(name); err != nil { + t.Error(err) + } else if fi.Mode()&0222 == 0 { + t.Errorf("Stat after OpenFile is %v, should be writable", fi.Mode()) + } +} diff --git a/src/runtime/crash_test.go b/src/runtime/crash_test.go index 5333b60646..34f30c9a37 100644 --- a/src/runtime/crash_test.go +++ b/src/runtime/crash_test.go @@ -55,6 +55,16 @@ func runTestProg(t *testing.T, binary, name string, env ...string) string { t.Fatal(err) } + return runBuiltTestProg(t, exe, name, env...) +} + +func runBuiltTestProg(t *testing.T, exe, name string, env ...string) string { + if *flagQuick { + t.Skip("-quick") + } + + testenv.MustHaveGoBuild(t) + cmd := testenv.CleanCmdEnv(exec.Command(exe, name)) cmd.Env = append(cmd.Env, env...) if testing.Short() { @@ -64,7 +74,7 @@ func runTestProg(t *testing.T, binary, name string, env ...string) string { cmd.Stdout = &b cmd.Stderr = &b if err := cmd.Start(); err != nil { - t.Fatalf("starting %s %s: %v", binary, name, err) + t.Fatalf("starting %s %s: %v", exe, name, err) } // If the process doesn't complete within 1 minute, @@ -92,7 +102,7 @@ func runTestProg(t *testing.T, binary, name string, env ...string) string { }() if err := cmd.Wait(); err != nil { - t.Logf("%s %s exit status: %v", binary, name, err) + t.Logf("%s %s exit status: %v", exe, name, err) } close(done) diff --git a/src/runtime/mgcscavenge.go b/src/runtime/mgcscavenge.go index c2625095f6..7a217ddcf3 100644 --- a/src/runtime/mgcscavenge.go +++ b/src/runtime/mgcscavenge.go @@ -286,6 +286,28 @@ func bgscavenge(c chan int) { continue } + if released < physPageSize { + // If this happens, it means that we may have attempted to release part + // of a physical page, but the likely effect of that is that it released + // the whole physical page, some of which may have still been in-use. + // This could lead to memory corruption. Throw. + throw("released less than one physical page of memory") + } + + // On some platforms we may see crit as zero if the time it takes to scavenge + // memory is less than the minimum granularity of its clock (e.g. Windows). + // In this case, just assume scavenging takes 10 µs per regular physical page + // (determined empirically), and conservatively ignore the impact of huge pages + // on timing. + // + // We shouldn't ever see a crit value less than zero unless there's a bug of + // some kind, either on our side or in the platform we're running on, but be + // defensive in that case as well. + const approxCritNSPerPhysicalPage = 10e3 + if crit <= 0 { + crit = approxCritNSPerPhysicalPage * float64(released/physPageSize) + } + // Multiply the critical time by 1 + the ratio of the costs of using // scavenged memory vs. scavenging memory. This forces us to pay down // the cost of reusing this memory eagerly by sleeping for a longer period diff --git a/src/runtime/mpagecache.go b/src/runtime/mpagecache.go index 9fc338bd8e..a074961840 100644 --- a/src/runtime/mpagecache.go +++ b/src/runtime/mpagecache.go @@ -148,9 +148,14 @@ func (s *pageAlloc) allocToCache() pageCache { // Update as an allocation, but note that it's not contiguous. s.update(c.base, pageCachePages, false, true) - // We're always searching for the first free page, and we always know the - // up to pageCache size bits will be allocated, so we can always move the - // searchAddr past the cache. - s.searchAddr = c.base + pageSize*pageCachePages + // Set the search address to the last page represented by the cache. + // Since all of the pages in this block are going to the cache, and we + // searched for the first free page, we can confidently start at the + // next page. + // + // However, s.searchAddr is not allowed to point into unmapped heap memory + // unless it is maxSearchAddr, so make it the last page as opposed to + // the page after. + s.searchAddr = c.base + pageSize*(pageCachePages-1) return c } diff --git a/src/runtime/mpagecache_test.go b/src/runtime/mpagecache_test.go index b8cc0bd965..2ed0c0aa6a 100644 --- a/src/runtime/mpagecache_test.go +++ b/src/runtime/mpagecache_test.go @@ -260,12 +260,13 @@ func TestPageAllocAllocToCache(t *testing.T) { if GOOS == "openbsd" && testing.Short() { t.Skip("skipping because virtual memory is limited; see #36210") } - tests := map[string]struct { + type test struct { before map[ChunkIdx][]BitRange scav map[ChunkIdx][]BitRange hits []PageCache // expected base addresses and patterns after map[ChunkIdx][]BitRange - }{ + } + tests := map[string]test{ "AllFree": { before: map[ChunkIdx][]BitRange{ BaseChunkIdx: {}, @@ -349,6 +350,34 @@ func TestPageAllocAllocToCache(t *testing.T) { }, }, } + if PageAlloc64Bit != 0 { + const chunkIdxBigJump = 0x100000 // chunk index offset which translates to O(TiB) + + // This test is similar to the one with the same name for + // pageAlloc.alloc and serves the same purpose. + // See mpagealloc_test.go for details. + sumsPerPhysPage := ChunkIdx(PhysPageSize / PallocSumBytes) + baseChunkIdx := BaseChunkIdx &^ (sumsPerPhysPage - 1) + tests["DiscontiguousMappedSumBoundary"] = test{ + before: map[ChunkIdx][]BitRange{ + baseChunkIdx + sumsPerPhysPage - 1: {{0, PallocChunkPages - 1}}, + baseChunkIdx + chunkIdxBigJump: {{1, PallocChunkPages - 1}}, + }, + scav: map[ChunkIdx][]BitRange{ + baseChunkIdx + sumsPerPhysPage - 1: {}, + baseChunkIdx + chunkIdxBigJump: {}, + }, + hits: []PageCache{ + NewPageCache(PageBase(baseChunkIdx+sumsPerPhysPage-1, PallocChunkPages-64), 1<<63, 0), + NewPageCache(PageBase(baseChunkIdx+chunkIdxBigJump, 0), 1, 0), + NewPageCache(0, 0, 0), + }, + after: map[ChunkIdx][]BitRange{ + baseChunkIdx + sumsPerPhysPage - 1: {{0, PallocChunkPages}}, + baseChunkIdx + chunkIdxBigJump: {{0, PallocChunkPages}}, + }, + } + } for name, v := range tests { v := v t.Run(name, func(t *testing.T) { diff --git a/src/runtime/proc.go b/src/runtime/proc.go index 2a91e82185..a66b4d0251 100644 --- a/src/runtime/proc.go +++ b/src/runtime/proc.go @@ -1762,10 +1762,16 @@ func startTemplateThread() { if GOARCH == "wasm" { // no threads on wasm yet return } + + // Disable preemption to guarantee that the template thread will be + // created before a park once haveTemplateThread is set. + mp := acquirem() if !atomic.Cas(&newmHandoff.haveTemplateThread, 0, 1) { + releasem(mp) return } newm(templateThread, nil) + releasem(mp) } // templateThread is a thread in a known-good state that exists solely diff --git a/src/runtime/proc_test.go b/src/runtime/proc_test.go index acee7a1819..8c70c19d92 100644 --- a/src/runtime/proc_test.go +++ b/src/runtime/proc_test.go @@ -6,6 +6,7 @@ package runtime_test import ( "fmt" + "internal/testenv" "math" "net" "runtime" @@ -923,6 +924,29 @@ func TestLockOSThreadAvoidsStatePropagation(t *testing.T) { } } +func TestLockOSThreadTemplateThreadRace(t *testing.T) { + testenv.MustHaveGoRun(t) + + exe, err := buildTestProg(t, "testprog") + if err != nil { + t.Fatal(err) + } + + iterations := 100 + if testing.Short() { + // Reduce run time to ~100ms, with much lower probability of + // catching issues. + iterations = 5 + } + for i := 0; i < iterations; i++ { + want := "OK\n" + output := runBuiltTestProg(t, exe, "LockOSThreadTemplateThreadRace") + if output != want { + t.Fatalf("run %d: want %q, got %q", i, want, output) + } + } +} + // fakeSyscall emulates a system call. //go:nosplit func fakeSyscall(duration time.Duration) { diff --git a/src/runtime/race/README b/src/runtime/race/README index 7e948cb681..357a5ed786 100644 --- a/src/runtime/race/README +++ b/src/runtime/race/README @@ -4,10 +4,10 @@ the LLVM project (https://github.com/llvm/llvm-project/tree/master/compiler-rt). To update the .syso files use golang.org/x/build/cmd/racebuild. -race_darwin_amd64.syso built with LLVM 810ae8ddac890a6613d814c0b5415c7fcb7f5cca and Go 8c6876e9a481a2ea48070d3285a07163f564877b. -race_freebsd_amd64.syso built with LLVM 810ae8ddac890a6613d814c0b5415c7fcb7f5cca and Go 8c6876e9a481a2ea48070d3285a07163f564877b. -race_linux_amd64.syso built with LLVM 810ae8ddac890a6613d814c0b5415c7fcb7f5cca and Go 8c6876e9a481a2ea48070d3285a07163f564877b. -race_linux_ppc64le.syso built with LLVM 810ae8ddac890a6613d814c0b5415c7fcb7f5cca and Go 8c6876e9a481a2ea48070d3285a07163f564877b. -race_netbsd_amd64.syso built with LLVM 810ae8ddac890a6613d814c0b5415c7fcb7f5cca and Go 8c6876e9a481a2ea48070d3285a07163f564877b. -race_windows_amd64.syso built with LLVM 810ae8ddac890a6613d814c0b5415c7fcb7f5cca and Go 8c6876e9a481a2ea48070d3285a07163f564877b. -race_linux_arm64.syso built with LLVM 810ae8ddac890a6613d814c0b5415c7fcb7f5cca and Go 8c6876e9a481a2ea48070d3285a07163f564877b. +race_darwin_amd64.syso built with LLVM 0fb8a5356214c47bbb832e89fbb3da1c755eeb73 and Go 95773ab9b053edc43ba07a182f3d5e0e29775a45. +race_freebsd_amd64.syso built with LLVM 0fb8a5356214c47bbb832e89fbb3da1c755eeb73 and Go 95773ab9b053edc43ba07a182f3d5e0e29775a45. +race_linux_amd64.syso built with LLVM 0fb8a5356214c47bbb832e89fbb3da1c755eeb73 and Go 95773ab9b053edc43ba07a182f3d5e0e29775a45. +race_linux_ppc64le.syso built with LLVM 2db63723a87527bac797996b8aa9d2f5a176b2f7 and Go e31d741801514f2a008625fd246644d2214f4516. +race_netbsd_amd64.syso built with LLVM efeb35e19563df911febe6a53151103c3b6011a5 and Go fd18f3ba5031079102ca4dc4cf425c2b496408ba. +race_windows_amd64.syso built with LLVM 0fb8a5356214c47bbb832e89fbb3da1c755eeb73 and Go 95773ab9b053edc43ba07a182f3d5e0e29775a45. +race_linux_arm64.syso built with LLVM 0fb8a5356214c47bbb832e89fbb3da1c755eeb73 and Go 95773ab9b053edc43ba07a182f3d5e0e29775a45. diff --git a/src/runtime/race/race_darwin_amd64.syso b/src/runtime/race/race_darwin_amd64.syso Binary files differindex bd904b98a7..4339f40a41 100644 --- a/src/runtime/race/race_darwin_amd64.syso +++ b/src/runtime/race/race_darwin_amd64.syso diff --git a/src/runtime/race/race_freebsd_amd64.syso b/src/runtime/race/race_freebsd_amd64.syso Binary files differindex 300684343b..01d34b993c 100644 --- a/src/runtime/race/race_freebsd_amd64.syso +++ b/src/runtime/race/race_freebsd_amd64.syso diff --git a/src/runtime/race/race_linux_amd64.syso b/src/runtime/race/race_linux_amd64.syso Binary files differindex 10d335c37f..a00b62839a 100644 --- a/src/runtime/race/race_linux_amd64.syso +++ b/src/runtime/race/race_linux_amd64.syso diff --git a/src/runtime/race/race_linux_arm64.syso b/src/runtime/race/race_linux_arm64.syso Binary files differindex 5b26e9f4ce..4da7102f03 100644 --- a/src/runtime/race/race_linux_arm64.syso +++ b/src/runtime/race/race_linux_arm64.syso diff --git a/src/runtime/race/race_linux_ppc64le.syso b/src/runtime/race/race_linux_ppc64le.syso Binary files differindex 244e38734e..2a5a0c2a2c 100644 --- a/src/runtime/race/race_linux_ppc64le.syso +++ b/src/runtime/race/race_linux_ppc64le.syso diff --git a/src/runtime/race/race_netbsd_amd64.syso b/src/runtime/race/race_netbsd_amd64.syso Binary files differindex 332b4ea55a..18e596fc80 100644 --- a/src/runtime/race/race_netbsd_amd64.syso +++ b/src/runtime/race/race_netbsd_amd64.syso diff --git a/src/runtime/race/race_windows_amd64.syso b/src/runtime/race/race_windows_amd64.syso Binary files differindex dc6a1931be..fc1c3a4d29 100644 --- a/src/runtime/race/race_windows_amd64.syso +++ b/src/runtime/race/race_windows_amd64.syso diff --git a/src/runtime/testdata/testprog/lockosthread.go b/src/runtime/testdata/testprog/lockosthread.go index fd3123e647..098cc4dd72 100644 --- a/src/runtime/testdata/testprog/lockosthread.go +++ b/src/runtime/testdata/testprog/lockosthread.go @@ -7,6 +7,7 @@ package main import ( "os" "runtime" + "sync" "time" ) @@ -30,6 +31,7 @@ func init() { runtime.LockOSThread() }) register("LockOSThreadAvoidsStatePropagation", LockOSThreadAvoidsStatePropagation) + register("LockOSThreadTemplateThreadRace", LockOSThreadTemplateThreadRace) } func LockOSThreadMain() { @@ -195,3 +197,50 @@ func LockOSThreadAvoidsStatePropagation() { runtime.UnlockOSThread() println("OK") } + +func LockOSThreadTemplateThreadRace() { + // This test attempts to reproduce the race described in + // golang.org/issue/38931. To do so, we must have a stop-the-world + // (achieved via ReadMemStats) racing with two LockOSThread calls. + // + // While this test attempts to line up the timing, it is only expected + // to fail (and thus hang) around 2% of the time if the race is + // present. + + // Ensure enough Ps to actually run everything in parallel. Though on + // <4 core machines, we are still at the whim of the kernel scheduler. + runtime.GOMAXPROCS(4) + + go func() { + // Stop the world; race with LockOSThread below. + var m runtime.MemStats + for { + runtime.ReadMemStats(&m) + } + }() + + // Try to synchronize both LockOSThreads. + start := time.Now().Add(10*time.Millisecond) + + var wg sync.WaitGroup + wg.Add(2) + + for i := 0; i < 2; i++ { + go func() { + for time.Now().Before(start) { + } + + // Add work to the local runq to trigger early startm + // in handoffp. + go func(){}() + + runtime.LockOSThread() + runtime.Gosched() // add a preemption point. + wg.Done() + }() + } + + wg.Wait() + // If both LockOSThreads completed then we did not hit the race. + println("OK") +} diff --git a/src/syscall/syscall_windows.go b/src/syscall/syscall_windows.go index 950c281e4d..41bc42a44e 100644 --- a/src/syscall/syscall_windows.go +++ b/src/syscall/syscall_windows.go @@ -334,6 +334,26 @@ func Open(path string, mode int, perm uint32) (fd Handle, err error) { var attrs uint32 = FILE_ATTRIBUTE_NORMAL if perm&S_IWRITE == 0 { attrs = FILE_ATTRIBUTE_READONLY + if createmode == CREATE_ALWAYS { + // We have been asked to create a read-only file. + // If the file already exists, the semantics of + // the Unix open system call is to preserve the + // existing permissions. If we pass CREATE_ALWAYS + // and FILE_ATTRIBUTE_READONLY to CreateFile, + // and the file already exists, CreateFile will + // change the file permissions. + // Avoid that to preserve the Unix semantics. + h, e := CreateFile(pathp, access, sharemode, sa, TRUNCATE_EXISTING, FILE_ATTRIBUTE_NORMAL, 0) + switch e { + case ERROR_FILE_NOT_FOUND, _ERROR_BAD_NETPATH, ERROR_PATH_NOT_FOUND: + // File does not exist. These are the same + // errors as Errno.Is checks for ErrNotExist. + // Carry on to create the file. + default: + // Success or some different error. + return h, e + } + } } h, e := CreateFile(pathp, access, sharemode, sa, createmode, attrs, 0) return h, e diff --git a/test/fixedbugs/issue38117.go b/test/fixedbugs/issue38117.go new file mode 100644 index 0000000000..11edef7f25 --- /dev/null +++ b/test/fixedbugs/issue38117.go @@ -0,0 +1,17 @@ +// errorcheck + +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// cmd/compile erroneously rejected conversions of constant values +// between int/float and complex types. + +package p + +const ( + _ = int(complex64(int(0))) + _ = float64(complex128(float64(0))) + + _ = int8(complex128(1000)) // ERROR "overflow" +) |