From 91de29ec8f069ceb6cd9e2007db6795ce1cc3506 Mon Sep 17 00:00:00 2001 From: Andrew Date: Wed, 12 Aug 2020 17:45:48 -0400 Subject: [release-branch.go1.15] doc/go1.15: include behavior updates to the context package Fixes #40737 Change-Id: I8e2c1e1653d427af1ded6d61df1aa450e3c4d35c Reviewed-on: https://go-review.googlesource.com/c/go/+/248329 Run-TryBot: Andrew Bonventre TryBot-Result: Gobot Gobot Reviewed-by: Ian Lance Taylor (cherry picked from commit b2353174db1349f15113416b4df2a833db6058a6) Reviewed-on: https://go-review.googlesource.com/c/go/+/248331 Reviewed-by: Andrew Gerrand --- doc/go1.15.html | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/doc/go1.15.html b/doc/go1.15.html index 8872d71138..e7b7456059 100644 --- a/doc/go1.15.html +++ b/doc/go1.15.html @@ -433,6 +433,19 @@ Do not send CLs removing the interior tags from such phrases. +
context
+
+

+ Creating a derived Context using a nil parent is now explicitly + disallowed. Any attempt to do so with the + WithValue, + WithDeadline, or + WithCancel functions + will cause a panic. +

+
+
+
crypto

-- cgit v1.2.3-54-g00ecf From 46a34f93a20bd7e249bebde5c937765dd779bf4f Mon Sep 17 00:00:00 2001 From: Cherry Zhang Date: Wed, 12 Aug 2020 13:30:43 -0400 Subject: [release-branch.go1.15] doc/go1.15: clarify external linking can still be used for building PIE In Go 1.15 we switched the default linking mode for PIE on Linux/AMD64 and Linux/ARM64 to internal linking. Clarify that the previous behavior (external linking) can still be used with a flag. Updates #40719. Change-Id: Ib7042622bc91e1b1aa31f520990d03b5eb6c56bb Reviewed-on: https://go-review.googlesource.com/c/go/+/248199 Reviewed-by: Ian Lance Taylor (cherry picked from commit 50f63a7ae4b7f951fa894b96633b1716adca55fa) Reviewed-on: https://go-review.googlesource.com/c/go/+/248330 --- doc/go1.15.html | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/doc/go1.15.html b/doc/go1.15.html index e7b7456059..c691bf3bd5 100644 --- a/doc/go1.15.html +++ b/doc/go1.15.html @@ -357,7 +357,10 @@ Do not send CLs removing the interior tags from such phrases. The linker now defaults to internal linking mode for -buildmode=pie on linux/amd64 and linux/arm64, so these - configurations no longer require a C linker. + configurations no longer require a C linker. External linking + mode (which was the default in Go 1.14 for + -buildmode=pie) can still be requested with + -ldflags=-linkmode=external flag.

Objdump

-- cgit v1.2.3-54-g00ecf From 9c17883f28ceb2aba94ae7f51269f1c982fd7f8b Mon Sep 17 00:00:00 2001 From: Josh Bleecher Snyder Date: Thu, 13 Aug 2020 12:39:04 -0700 Subject: cmd/compile: correct type of CvtBoolToUint8 values Fixes #40772 Change-Id: I539f07d1f958dacee87d846171a8889d03182d25 Reviewed-on: https://go-review.googlesource.com/c/go/+/248397 Run-TryBot: Josh Bleecher Snyder TryBot-Result: Gobot Gobot Reviewed-by: Keith Randall Reviewed-on: https://go-review.googlesource.com/c/go/+/248401 --- src/cmd/compile/internal/ssa/phiopt.go | 2 +- src/cmd/compile/internal/ssa/prove.go | 2 +- test/fixedbugs/issue40746.go | 19 +++++++++++++++++++ 3 files changed, 21 insertions(+), 2 deletions(-) create mode 100644 test/fixedbugs/issue40746.go diff --git a/src/cmd/compile/internal/ssa/phiopt.go b/src/cmd/compile/internal/ssa/phiopt.go index 8643fa584c..db7b02275c 100644 --- a/src/cmd/compile/internal/ssa/phiopt.go +++ b/src/cmd/compile/internal/ssa/phiopt.go @@ -154,7 +154,7 @@ func phioptint(v *Value, b0 *Block, reverse int) { } v.AddArg(a) - cvt := v.Block.NewValue1(v.Pos, OpCvtBoolToUint8, a.Type, a) + cvt := v.Block.NewValue1(v.Pos, OpCvtBoolToUint8, v.Block.Func.Config.Types.UInt8, a) switch v.Type.Size() { case 1: v.reset(OpCopy) diff --git a/src/cmd/compile/internal/ssa/prove.go b/src/cmd/compile/internal/ssa/prove.go index 6c6be39d34..ce7d689f93 100644 --- a/src/cmd/compile/internal/ssa/prove.go +++ b/src/cmd/compile/internal/ssa/prove.go @@ -1334,7 +1334,7 @@ func removeBranch(b *Block, branch branch) { // isNonNegative reports whether v is known to be greater or equal to zero. func isNonNegative(v *Value) bool { if !v.Type.IsInteger() { - panic("isNonNegative bad type") + v.Fatalf("isNonNegative bad type: %v", v.Type) } // TODO: return true if !v.Type.IsSigned() // SSA isn't type-safe enough to do that now (issue 37753). diff --git a/test/fixedbugs/issue40746.go b/test/fixedbugs/issue40746.go new file mode 100644 index 0000000000..235282fd90 --- /dev/null +++ b/test/fixedbugs/issue40746.go @@ -0,0 +1,19 @@ +// compile + +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package p + +func f(x byte, b bool) byte { + var c byte + if b { + c = 1 + } + + if int8(c) < 0 { + x++ + } + return x +} -- cgit v1.2.3-54-g00ecf From f2b710998902d6754d62810e1577c01313763342 Mon Sep 17 00:00:00 2001 From: Ian Lance Taylor Date: Wed, 12 Aug 2020 12:09:35 -0700 Subject: [release-branch.go1.15] internal/poll: treat copy_file_range EOPNOTSUPP as not-handled For #40731 Fixes #40739 Change-Id: I3e29878d597318acf5edcc38497aa2624f72be35 Reviewed-on: https://go-review.googlesource.com/c/go/+/248258 Run-TryBot: Ian Lance Taylor TryBot-Result: Gobot Gobot Reviewed-by: Emmanuel Odeke Reviewed-by: Tobias Klauser (cherry picked from commit d3a411b6debccb665da3497e7fa597c9a5ff16f1) Reviewed-on: https://go-review.googlesource.com/c/go/+/249197 Run-TryBot: Tobias Klauser --- src/internal/poll/copy_file_range_linux.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/internal/poll/copy_file_range_linux.go b/src/internal/poll/copy_file_range_linux.go index 604607f774..7e67125818 100644 --- a/src/internal/poll/copy_file_range_linux.go +++ b/src/internal/poll/copy_file_range_linux.go @@ -41,7 +41,7 @@ func CopyFileRange(dst, src *FD, remain int64) (written int64, handled bool, err // use copy_file_range(2) again. atomic.StoreInt32(©FileRangeSupported, 0) return 0, false, nil - case syscall.EXDEV, syscall.EINVAL: + case syscall.EXDEV, syscall.EINVAL, syscall.EOPNOTSUPP: // Prior to Linux 5.3, it was not possible to // copy_file_range across file systems. Similarly to // the ENOSYS case above, if we see EXDEV, we have @@ -52,6 +52,9 @@ func CopyFileRange(dst, src *FD, remain int64) (written int64, handled bool, err // dst or src refer to a pipe rather than a regular // file. This is another case where no data has been // transfered, so we consider it unhandled. + // + // If the file is on NFS, we can see EOPNOTSUPP. + // See issue #40731. return 0, false, nil case nil: if n == 0 { -- cgit v1.2.3-54-g00ecf From f454f70344703bd08e1f0a104fd2e6879e1d846c Mon Sep 17 00:00:00 2001 From: Matthew Dempsky Date: Wed, 19 Aug 2020 21:39:12 -0700 Subject: [release-branch.go1.15] cmd/compile: fix checkptr handling of &^ checkptr has code to recognize &^ expressions, but it didn't take into account that "p &^ x" gets rewritten to "p & ^x" during walk, which resulted in false positive diagnostics. This CL changes walkexpr to mark OANDNOT expressions with Implicit when they're rewritten to OAND, so that walkCheckPtrArithmetic can still recognize them later. It would be slightly more idiomatic to instead mark the OBITNOT expression as Implicit (as it's a compiler-generated Node), but the OBITNOT expression might get constant folded. It's not worth the extra complexity/subtlety of relying on n.Right.Orig, so we set Implicit on the OAND node instead. To atone for this transgression, I add documentation for nodeImplicit. Updates #40917. Fixes #40934. Change-Id: I386304171ad299c530e151e5924f179e9a5fd5b8 Reviewed-on: https://go-review.googlesource.com/c/go/+/249477 Run-TryBot: Matthew Dempsky TryBot-Result: Gobot Gobot Reviewed-by: Keith Randall Reviewed-by: Cuong Manh Le (cherry picked from commit e94544cf012535da6b3c9e735bc4026e2db1c99c) Reviewed-on: https://go-review.googlesource.com/c/go/+/249879 Run-TryBot: Dmitri Shuralyov Reviewed-by: Matthew Dempsky --- src/cmd/compile/internal/gc/syntax.go | 4 ++-- src/cmd/compile/internal/gc/walk.go | 7 ++++++- src/runtime/checkptr_test.go | 1 + src/runtime/testdata/testprog/checkptr.go | 8 ++++++++ test/fixedbugs/issue40917.go | 23 +++++++++++++++++++++++ 5 files changed, 40 insertions(+), 3 deletions(-) create mode 100644 test/fixedbugs/issue40917.go diff --git a/src/cmd/compile/internal/gc/syntax.go b/src/cmd/compile/internal/gc/syntax.go index b658410c53..47e5e59156 100644 --- a/src/cmd/compile/internal/gc/syntax.go +++ b/src/cmd/compile/internal/gc/syntax.go @@ -141,8 +141,8 @@ const ( nodeInitorder, _ // tracks state during init1; two bits _, _ // second nodeInitorder bit _, nodeHasBreak - _, nodeNoInline // used internally by inliner to indicate that a function call should not be inlined; set for OCALLFUNC and OCALLMETH only - _, nodeImplicit + _, nodeNoInline // used internally by inliner to indicate that a function call should not be inlined; set for OCALLFUNC and OCALLMETH only + _, nodeImplicit // implicit OADDR or ODEREF; ++/-- statement represented as OASOP; or ANDNOT lowered to OAND _, nodeIsDDD // is the argument variadic _, nodeDiag // already printed error about this _, nodeColas // OAS resulting from := diff --git a/src/cmd/compile/internal/gc/walk.go b/src/cmd/compile/internal/gc/walk.go index 1e6d913ae6..b7cf313938 100644 --- a/src/cmd/compile/internal/gc/walk.go +++ b/src/cmd/compile/internal/gc/walk.go @@ -968,6 +968,7 @@ opswitch: case OANDNOT: n.Left = walkexpr(n.Left, init) n.Op = OAND + n.SetImplicit(true) // for walkCheckPtrArithmetic n.Right = nod(OBITNOT, n.Right, nil) n.Right = typecheck(n.Right, ctxExpr) n.Right = walkexpr(n.Right, init) @@ -3993,8 +3994,12 @@ func walkCheckPtrArithmetic(n *Node, init *Nodes) *Node { case OADD: walk(n.Left) walk(n.Right) - case OSUB, OANDNOT: + case OSUB: walk(n.Left) + case OAND: + if n.Implicit() { // was OANDNOT + walk(n.Left) + } case OCONVNOP: if n.Left.Type.Etype == TUNSAFEPTR { n.Left = cheapexpr(n.Left, init) diff --git a/src/runtime/checkptr_test.go b/src/runtime/checkptr_test.go index 8ab8a4937c..194cc1243a 100644 --- a/src/runtime/checkptr_test.go +++ b/src/runtime/checkptr_test.go @@ -27,6 +27,7 @@ func TestCheckPtr(t *testing.T) { {"CheckPtrAlignmentPtr", "fatal error: checkptr: misaligned pointer conversion\n"}, {"CheckPtrAlignmentNoPtr", ""}, {"CheckPtrArithmetic", "fatal error: checkptr: pointer arithmetic result points to invalid allocation\n"}, + {"CheckPtrArithmetic2", "fatal error: checkptr: pointer arithmetic result points to invalid allocation\n"}, {"CheckPtrSize", "fatal error: checkptr: converted pointer straddles multiple allocations\n"}, {"CheckPtrSmall", "fatal error: checkptr: pointer arithmetic computed bad pointer value\n"}, } diff --git a/src/runtime/testdata/testprog/checkptr.go b/src/runtime/testdata/testprog/checkptr.go index 45e6fb1aa5..e0a2794f4c 100644 --- a/src/runtime/testdata/testprog/checkptr.go +++ b/src/runtime/testdata/testprog/checkptr.go @@ -10,6 +10,7 @@ func init() { register("CheckPtrAlignmentNoPtr", CheckPtrAlignmentNoPtr) register("CheckPtrAlignmentPtr", CheckPtrAlignmentPtr) register("CheckPtrArithmetic", CheckPtrArithmetic) + register("CheckPtrArithmetic2", CheckPtrArithmetic2) register("CheckPtrSize", CheckPtrSize) register("CheckPtrSmall", CheckPtrSmall) } @@ -32,6 +33,13 @@ func CheckPtrArithmetic() { sink2 = (*int)(unsafe.Pointer(i)) } +func CheckPtrArithmetic2() { + var x [2]int64 + p := unsafe.Pointer(&x[1]) + var one uintptr = 1 + sink2 = unsafe.Pointer(uintptr(p) & ^one) +} + func CheckPtrSize() { p := new(int64) sink2 = p diff --git a/test/fixedbugs/issue40917.go b/test/fixedbugs/issue40917.go new file mode 100644 index 0000000000..2128be5eca --- /dev/null +++ b/test/fixedbugs/issue40917.go @@ -0,0 +1,23 @@ +// run -gcflags=-d=checkptr + +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import "unsafe" + +func main() { + var x [2]uint64 + a := unsafe.Pointer(&x[1]) + + b := a + b = unsafe.Pointer(uintptr(b) + 2) + b = unsafe.Pointer(uintptr(b) - 1) + b = unsafe.Pointer(uintptr(b) &^ 1) + + if a != b { + panic("pointer arithmetic failed") + } +} -- cgit v1.2.3-54-g00ecf From 8bf83b323cb5542ac3c41394a4bb1d3ae115c640 Mon Sep 17 00:00:00 2001 From: Tobias Klauser Date: Wed, 19 Aug 2020 16:54:36 +0200 Subject: [release-branch.go1.15] internal/poll: treat copy_file_range EPERM as not-handled Updates #40893. Fixes #40900. Change-Id: I938ea4796c1e1d1e136117fe78b06ad6da8e40de Reviewed-on: https://go-review.googlesource.com/c/go/+/249257 Run-TryBot: Tobias Klauser TryBot-Result: Gobot Gobot Reviewed-by: Antonio Troina Reviewed-by: Ian Lance Taylor (cherry picked from commit b0cc02e8c2bdba5401838d9d70a859191af9bfa5) Reviewed-on: https://go-review.googlesource.com/c/go/+/249897 Run-TryBot: Dmitri Shuralyov --- src/internal/poll/copy_file_range_linux.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/internal/poll/copy_file_range_linux.go b/src/internal/poll/copy_file_range_linux.go index 7e67125818..09de299ff7 100644 --- a/src/internal/poll/copy_file_range_linux.go +++ b/src/internal/poll/copy_file_range_linux.go @@ -41,7 +41,7 @@ func CopyFileRange(dst, src *FD, remain int64) (written int64, handled bool, err // use copy_file_range(2) again. atomic.StoreInt32(©FileRangeSupported, 0) return 0, false, nil - case syscall.EXDEV, syscall.EINVAL, syscall.EOPNOTSUPP: + case syscall.EXDEV, syscall.EINVAL, syscall.EOPNOTSUPP, syscall.EPERM: // Prior to Linux 5.3, it was not possible to // copy_file_range across file systems. Similarly to // the ENOSYS case above, if we see EXDEV, we have @@ -55,6 +55,11 @@ func CopyFileRange(dst, src *FD, remain int64) (written int64, handled bool, err // // If the file is on NFS, we can see EOPNOTSUPP. // See issue #40731. + // + // If the process is running inside a Docker container, + // we might see EPERM instead of ENOSYS. See issue + // #40893. Since EPERM might also be a legitimate error, + // don't mark copy_file_range(2) as unsupported. return 0, false, nil case nil: if n == 0 { -- cgit v1.2.3-54-g00ecf From b1253d24e159129c778377c3a2a0bde15904a417 Mon Sep 17 00:00:00 2001 From: "Jason A. Donenfeld" Date: Tue, 14 Jul 2020 01:41:03 -0600 Subject: [release-branch.go1.15] runtime: detect services in signal handler The service handler needs to handle CTRL+C-like events -- including those sent by the service manager itself -- using the default Windows implementation if no signal handler from Go is already listening to those events. Ordinarily, the signal handler would call exit(2), but we actually need to allow this to be passed onward to the service handler. So, we detect if we're in a service and skip calling exit(2) in that case, just like we do for shared libraries. Updates #40167. Updates #40074. Fixes #40412. Change-Id: Ia77871737a80e1e94f85b02d26af1fd2f646af96 Reviewed-on: https://go-review.googlesource.com/c/go/+/244959 Run-TryBot: Jason A. Donenfeld TryBot-Result: Gobot Gobot Reviewed-by: Alex Brainman --- src/runtime/os_windows.go | 73 ++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 69 insertions(+), 4 deletions(-) diff --git a/src/runtime/os_windows.go b/src/runtime/os_windows.go index a584ada702..769197db46 100644 --- a/src/runtime/os_windows.go +++ b/src/runtime/os_windows.go @@ -36,7 +36,10 @@ const ( //go:cgo_import_dynamic runtime._SetThreadContext SetThreadContext%2 "kernel32.dll" //go:cgo_import_dynamic runtime._LoadLibraryW LoadLibraryW%1 "kernel32.dll" //go:cgo_import_dynamic runtime._LoadLibraryA LoadLibraryA%1 "kernel32.dll" +//go:cgo_import_dynamic runtime._OpenProcess OpenProcess%3 "kernel32.dll" //go:cgo_import_dynamic runtime._PostQueuedCompletionStatus PostQueuedCompletionStatus%4 "kernel32.dll" +//go:cgo_import_dynamic runtime._ProcessIdToSessionId ProcessIdToSessionId%2 "kernel32.dll" +//go:cgo_import_dynamic runtime._QueryFullProcessImageNameA QueryFullProcessImageNameA%4 "kernel32.dll" //go:cgo_import_dynamic runtime._ResumeThread ResumeThread%1 "kernel32.dll" //go:cgo_import_dynamic runtime._SetConsoleCtrlHandler SetConsoleCtrlHandler%2 "kernel32.dll" //go:cgo_import_dynamic runtime._SetErrorMode SetErrorMode%1 "kernel32.dll" @@ -84,7 +87,10 @@ var ( _SetThreadContext, _LoadLibraryW, _LoadLibraryA, + _OpenProcess, _PostQueuedCompletionStatus, + _ProcessIdToSessionId, + _QueryFullProcessImageNameA, _QueryPerformanceCounter, _QueryPerformanceFrequency, _ResumeThread, @@ -128,7 +134,8 @@ var ( // Load ntdll.dll manually during startup, otherwise Mingw // links wrong printf function to cgo executable (see issue // 12030 for details). - _NtWaitForSingleObject stdFunction + _NtWaitForSingleObject stdFunction + _NtQueryInformationProcess stdFunction // These are from non-kernel32.dll, so we prefer to LoadLibraryEx them. _timeBeginPeriod, @@ -255,6 +262,7 @@ func loadOptionalSyscalls() { throw("ntdll.dll not found") } _NtWaitForSingleObject = windowsFindfunc(n32, []byte("NtWaitForSingleObject\000")) + _NtQueryInformationProcess = windowsFindfunc(n32, []byte("NtQueryInformationProcess\000")) if GOARCH == "arm" { _QueryPerformanceCounter = windowsFindfunc(k32, []byte("QueryPerformanceCounter\000")) @@ -995,6 +1003,63 @@ func usleep(us uint32) { onosstack(usleep2Addr, 10*us) } +// isWindowsService returns whether the process is currently executing as a +// Windows service. The below technique looks a bit hairy, but it's actually +// exactly what the .NET framework does for the similarly named function: +// https://github.com/dotnet/extensions/blob/f4066026ca06984b07e90e61a6390ac38152ba93/src/Hosting/WindowsServices/src/WindowsServiceHelpers.cs#L26-L31 +// Specifically, it looks up whether the parent process has session ID zero +// and is called "services". +func isWindowsService() bool { + const ( + _CURRENT_PROCESS = ^uintptr(0) + _PROCESS_QUERY_LIMITED_INFORMATION = 0x1000 + ) + // pbi is a PROCESS_BASIC_INFORMATION struct, where we just care about + // the 6th pointer inside of it, which contains the pid of the process + // parent: + // https://github.com/wine-mirror/wine/blob/42cb7d2ad1caba08de235e6319b9967296b5d554/include/winternl.h#L1294 + var pbi [6]uintptr + var pbiLen uint32 + err := stdcall5(_NtQueryInformationProcess, _CURRENT_PROCESS, 0, uintptr(unsafe.Pointer(&pbi[0])), uintptr(unsafe.Sizeof(pbi)), uintptr(unsafe.Pointer(&pbiLen))) + if err != 0 { + return false + } + var psid uint32 + err = stdcall2(_ProcessIdToSessionId, pbi[5], uintptr(unsafe.Pointer(&psid))) + if err == 0 || psid != 0 { + return false + } + pproc := stdcall3(_OpenProcess, _PROCESS_QUERY_LIMITED_INFORMATION, 0, pbi[5]) + if pproc == 0 { + return false + } + defer stdcall1(_CloseHandle, pproc) + // exeName gets the path to the executable image of the parent process + var exeName [261]byte + exeNameLen := uint32(len(exeName) - 1) + err = stdcall4(_QueryFullProcessImageNameA, pproc, 0, uintptr(unsafe.Pointer(&exeName[0])), uintptr(unsafe.Pointer(&exeNameLen))) + if err == 0 || exeNameLen == 0 { + return false + } + servicesLower := "services.exe" + servicesUpper := "SERVICES.EXE" + i := int(exeNameLen) - 1 + j := len(servicesLower) - 1 + if i < j { + return false + } + for { + if j == -1 { + return i == -1 || exeName[i] == '\\' + } + if exeName[i] != servicesLower[j] && exeName[i] != servicesUpper[j] { + return false + } + i-- + j-- + } +} + func ctrlhandler1(_type uint32) uint32 { var s uint32 @@ -1010,9 +1075,9 @@ func ctrlhandler1(_type uint32) uint32 { if sigsend(s) { return 1 } - if !islibrary && !isarchive { - // Only exit the program if we don't have a DLL. - // See https://golang.org/issues/35965. + if !islibrary && !isarchive && !isWindowsService() { + // Only exit the program if we don't have a DLL or service. + // See https://golang.org/issues/35965 and https://golang.org/issues/40167 exit(2) // SIGINT, SIGTERM, etc } return 0 -- cgit v1.2.3-54-g00ecf From 45265c210c64b10ef86e120a3616602047036ca9 Mon Sep 17 00:00:00 2001 From: Michał Łowicki Date: Sun, 23 Aug 2020 23:53:04 +0100 Subject: [release-branch.go1.15] testing: fix Cleanup race with Logf and Errorf MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Updates #40908 Fixes #41034 Change-Id: I25561a3f18e730a50e6fbf85aa7bd85bf1b73b6e Reviewed-on: https://go-review.googlesource.com/c/go/+/250078 Reviewed-by: Tobias Klauser Run-TryBot: Bryan C. Mills TryBot-Result: Gobot Gobot (cherry picked from commit 00a053bd4b2c19b2d9680f78f4c8657fcc6f1c88) Reviewed-on: https://go-review.googlesource.com/c/go/+/250617 Reviewed-by: Emmanuel Odeke Reviewed-by: Michał Łowicki --- src/cmd/go/testdata/script/testing_issue40908.txt | 21 +++++++++++++++++++++ src/testing/testing.go | 4 ++++ 2 files changed, 25 insertions(+) create mode 100644 src/cmd/go/testdata/script/testing_issue40908.txt diff --git a/src/cmd/go/testdata/script/testing_issue40908.txt b/src/cmd/go/testdata/script/testing_issue40908.txt new file mode 100644 index 0000000000..4939de080c --- /dev/null +++ b/src/cmd/go/testdata/script/testing_issue40908.txt @@ -0,0 +1,21 @@ +[short] skip +[!race] skip + +go test -race testrace + +-- testrace/race_test.go -- +package testrace + +import "testing" + +func TestRace(t *testing.T) { + helperDone := make(chan struct{}) + go func() { + t.Logf("Something happened before cleanup.") + close(helperDone) + }() + + t.Cleanup(func() { + <-helperDone + }) +} diff --git a/src/testing/testing.go b/src/testing/testing.go index 061142b9ab..1bfcbb27e4 100644 --- a/src/testing/testing.go +++ b/src/testing/testing.go @@ -851,11 +851,15 @@ func (c *common) Cleanup(f func()) { c.cleanup = func() { if oldCleanup != nil { defer func() { + c.mu.Lock() c.cleanupPc = oldCleanupPc + c.mu.Unlock() oldCleanup() }() } + c.mu.Lock() c.cleanupName = callerName(0) + c.mu.Unlock() f() } var pc [maxStackLen]uintptr -- cgit v1.2.3-54-g00ecf From 19c8546cd9cd53da0f4604d20f47714554fed051 Mon Sep 17 00:00:00 2001 From: Changkun Ou Date: Mon, 24 Aug 2020 13:45:27 +0200 Subject: [release-branch.go1.15] sync: delete dirty keys inside Map.LoadAndDelete Updates #40999 Fixes #41011 Change-Id: Ie32427e5cb5ed512b976b554850f50be156ce9f2 Reviewed-on: https://go-review.googlesource.com/c/go/+/250197 Run-TryBot: Emmanuel Odeke TryBot-Result: Gobot Gobot Reviewed-by: Bryan C. Mills (cherry picked from commit 94953d3e5928c8a577bad7911aabbf627269ef77) Reviewed-on: https://go-review.googlesource.com/c/go/+/250297 Run-TryBot: Bryan C. Mills Reviewed-by: Emmanuel Odeke --- src/sync/map.go | 1 + src/sync/map_test.go | 24 ++++++++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/src/sync/map.go b/src/sync/map.go index a61e2ebdd6..9ad25353ff 100644 --- a/src/sync/map.go +++ b/src/sync/map.go @@ -274,6 +274,7 @@ func (m *Map) LoadAndDelete(key interface{}) (value interface{}, loaded bool) { e, ok = read.m[key] if !ok && read.amended { e, ok = m.dirty[key] + delete(m.dirty, key) // Regardless of whether the entry was present, record a miss: this key // will take the slow path until the dirty map is promoted to the read // map. diff --git a/src/sync/map_test.go b/src/sync/map_test.go index 4ae989a6d5..7f163caa5c 100644 --- a/src/sync/map_test.go +++ b/src/sync/map_test.go @@ -9,6 +9,7 @@ import ( "reflect" "runtime" "sync" + "sync/atomic" "testing" "testing/quick" ) @@ -171,3 +172,26 @@ func TestConcurrentRange(t *testing.T) { } } } + +func TestIssue40999(t *testing.T) { + var m sync.Map + + // Since the miss-counting in missLocked (via Delete) + // compares the miss count with len(m.dirty), + // add an initial entry to bias len(m.dirty) above the miss count. + m.Store(nil, struct{}{}) + + var finalized uint32 + + // Set finalizers that count for collected keys. A non-zero count + // indicates that keys have not been leaked. + for atomic.LoadUint32(&finalized) == 0 { + p := new(int) + runtime.SetFinalizer(p, func(*int) { + atomic.AddUint32(&finalized, 1) + }) + m.Store(p, struct{}{}) + m.Delete(p) + runtime.GC() + } +} -- cgit v1.2.3-54-g00ecf From 1fa328f8fbb6f4fae494b00c7c1a6fe1c288af79 Mon Sep 17 00:00:00 2001 From: Ian Lance Taylor Date: Fri, 14 Aug 2020 13:29:07 -0700 Subject: [release-branch.go1.15] net/mail: return error on empty address list MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This restores the handling accidentally changed in CL 217377. Fixes #40804 For #40803 For #36959 Change-Id: If77fbc0c2a1dde4799f760affdfb8dde9bcaf458 Reviewed-on: https://go-review.googlesource.com/c/go/+/248598 Run-TryBot: Ian Lance Taylor TryBot-Result: Gobot Gobot Reviewed-by: Daniel Martí Reviewed-by: Jeremy Fishman (cherry picked from commit 3e636ab9ad31040aff2d484237808907a776cec6) Reviewed-on: https://go-review.googlesource.com/c/go/+/251167 --- src/net/mail/message.go | 13 +++++++++---- src/net/mail/message_test.go | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 4 deletions(-) diff --git a/src/net/mail/message.go b/src/net/mail/message.go index 6833cfaec1..09fb794005 100644 --- a/src/net/mail/message.go +++ b/src/net/mail/message.go @@ -279,9 +279,6 @@ func (p *addrParser) parseAddressList() ([]*Address, error) { if p.consume(',') { continue } - if p.empty() { - break - } addrs, err := p.parseAddress(true) if err != nil { @@ -295,9 +292,17 @@ func (p *addrParser) parseAddressList() ([]*Address, error) { if p.empty() { break } - if !p.consume(',') { + if p.peek() != ',' { return nil, errors.New("mail: expected comma") } + + // Skip empty entries for obs-addr-list. + for p.consume(',') { + p.skipSpace() + } + if p.empty() { + break + } } return list, nil } diff --git a/src/net/mail/message_test.go b/src/net/mail/message_test.go index 75db767547..67e3643aeb 100644 --- a/src/net/mail/message_test.go +++ b/src/net/mail/message_test.go @@ -445,6 +445,19 @@ func TestAddressParsing(t *testing.T) { }, }, }, + { + ` , joe@where.test,,John ,,`, + []*Address{ + { + Name: "", + Address: "joe@where.test", + }, + { + Name: "John", + Address: "jdoe@one.test", + }, + }, + }, { `Group1: ;, Group 2: addr2@example.com;, John `, []*Address{ @@ -1067,3 +1080,22 @@ func TestAddressFormattingAndParsing(t *testing.T) { } } } + +func TestEmptyAddress(t *testing.T) { + parsed, err := ParseAddress("") + if parsed != nil || err == nil { + t.Errorf(`ParseAddress("") = %v, %v, want nil, error`, parsed, err) + } + list, err := ParseAddressList("") + if len(list) > 0 || err == nil { + t.Errorf(`ParseAddressList("") = %v, %v, want nil, error`, list, err) + } + list, err = ParseAddressList(",") + if len(list) > 0 || err == nil { + t.Errorf(`ParseAddressList("") = %v, %v, want nil, error`, list, err) + } + list, err = ParseAddressList("a@b c@d") + if len(list) > 0 || err == nil { + t.Errorf(`ParseAddressList("") = %v, %v, want nil, error`, list, err) + } +} -- cgit v1.2.3-54-g00ecf From eb07103a083237414145a45f029c873d57037e06 Mon Sep 17 00:00:00 2001 From: Roberto Clapis Date: Wed, 26 Aug 2020 08:53:03 +0200 Subject: [release-branch.go1.15-security] net/http/cgi,net/http/fcgi: add Content-Type detection This CL ensures that responses served via CGI and FastCGI have a Content-Type header based on the content of the response if not explicitly set by handlers. If the implementers of the handler did not explicitly specify a Content-Type both CGI implementations would default to "text/html", potentially causing cross-site scripting. Thanks to RedTeam Pentesting GmbH for reporting this. Fixes CVE-2020-24553 Change-Id: I82cfc396309b5ab2e8d6e9a87eda8ea7e3799473 Reviewed-on: https://team-review.git.corp.google.com/c/golang/go-private/+/823217 Reviewed-by: Russ Cox (cherry picked from commit 23d675d07fdc56aafd67c0a0b63d5b7e14708ff0) Reviewed-on: https://team-review.git.corp.google.com/c/golang/go-private/+/835311 Reviewed-by: Dmitri Shuralyov --- src/net/http/cgi/child.go | 36 +++++++++++++------ src/net/http/cgi/child_test.go | 69 ++++++++++++++++++++++++++++++++++++ src/net/http/cgi/integration_test.go | 53 ++++++++++++++++++++++++++- src/net/http/fcgi/child.go | 39 ++++++++++++++------ src/net/http/fcgi/fcgi_test.go | 52 +++++++++++++++++++++++++++ 5 files changed, 227 insertions(+), 22 deletions(-) diff --git a/src/net/http/cgi/child.go b/src/net/http/cgi/child.go index 9474175f17..61de6165f6 100644 --- a/src/net/http/cgi/child.go +++ b/src/net/http/cgi/child.go @@ -163,10 +163,12 @@ func Serve(handler http.Handler) error { } type response struct { - req *http.Request - header http.Header - bufw *bufio.Writer - headerSent bool + req *http.Request + header http.Header + code int + wroteHeader bool + wroteCGIHeader bool + bufw *bufio.Writer } func (r *response) Flush() { @@ -178,26 +180,38 @@ func (r *response) Header() http.Header { } func (r *response) Write(p []byte) (n int, err error) { - if !r.headerSent { + if !r.wroteHeader { r.WriteHeader(http.StatusOK) } + if !r.wroteCGIHeader { + r.writeCGIHeader(p) + } return r.bufw.Write(p) } func (r *response) WriteHeader(code int) { - if r.headerSent { + if r.wroteHeader { // Note: explicitly using Stderr, as Stdout is our HTTP output. fmt.Fprintf(os.Stderr, "CGI attempted to write header twice on request for %s", r.req.URL) return } - r.headerSent = true - fmt.Fprintf(r.bufw, "Status: %d %s\r\n", code, http.StatusText(code)) + r.wroteHeader = true + r.code = code +} - // Set a default Content-Type +// writeCGIHeader finalizes the header sent to the client and writes it to the output. +// p is not written by writeHeader, but is the first chunk of the body +// that will be written. It is sniffed for a Content-Type if none is +// set explicitly. +func (r *response) writeCGIHeader(p []byte) { + if r.wroteCGIHeader { + return + } + r.wroteCGIHeader = true + fmt.Fprintf(r.bufw, "Status: %d %s\r\n", r.code, http.StatusText(r.code)) if _, hasType := r.header["Content-Type"]; !hasType { - r.header.Add("Content-Type", "text/html; charset=utf-8") + r.header.Set("Content-Type", http.DetectContentType(p)) } - r.header.Write(r.bufw) r.bufw.WriteString("\r\n") r.bufw.Flush() diff --git a/src/net/http/cgi/child_test.go b/src/net/http/cgi/child_test.go index 14e0af475f..f6ecb6eb80 100644 --- a/src/net/http/cgi/child_test.go +++ b/src/net/http/cgi/child_test.go @@ -7,6 +7,11 @@ package cgi import ( + "bufio" + "bytes" + "net/http" + "net/http/httptest" + "strings" "testing" ) @@ -148,3 +153,67 @@ func TestRequestWithoutRemotePort(t *testing.T) { t.Errorf("RemoteAddr: got %q; want %q", g, e) } } + +type countingWriter int + +func (c *countingWriter) Write(p []byte) (int, error) { + *c += countingWriter(len(p)) + return len(p), nil +} +func (c *countingWriter) WriteString(p string) (int, error) { + *c += countingWriter(len(p)) + return len(p), nil +} + +func TestResponse(t *testing.T) { + var tests = []struct { + name string + body string + wantCT string + }{ + { + name: "no body", + wantCT: "text/plain; charset=utf-8", + }, + { + name: "html", + body: "test pageThis is a body", + wantCT: "text/html; charset=utf-8", + }, + { + name: "text", + body: strings.Repeat("gopher", 86), + wantCT: "text/plain; charset=utf-8", + }, + { + name: "jpg", + body: "\xFF\xD8\xFF" + strings.Repeat("B", 1024), + wantCT: "image/jpeg", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var buf bytes.Buffer + resp := response{ + req: httptest.NewRequest("GET", "/", nil), + header: http.Header{}, + bufw: bufio.NewWriter(&buf), + } + n, err := resp.Write([]byte(tt.body)) + if err != nil { + t.Errorf("Write: unexpected %v", err) + } + if want := len(tt.body); n != want { + t.Errorf("reported short Write: got %v want %v", n, want) + } + resp.writeCGIHeader(nil) + resp.Flush() + if got := resp.Header().Get("Content-Type"); got != tt.wantCT { + t.Errorf("wrong content-type: got %q, want %q", got, tt.wantCT) + } + if !bytes.HasSuffix(buf.Bytes(), []byte(tt.body)) { + t.Errorf("body was not correctly written") + } + }) + } +} diff --git a/src/net/http/cgi/integration_test.go b/src/net/http/cgi/integration_test.go index 32d59c09a3..295c3b82d4 100644 --- a/src/net/http/cgi/integration_test.go +++ b/src/net/http/cgi/integration_test.go @@ -16,7 +16,9 @@ import ( "io" "net/http" "net/http/httptest" + "net/url" "os" + "strings" "testing" "time" ) @@ -52,7 +54,7 @@ func TestHostingOurselves(t *testing.T) { } replay := runCgiTest(t, h, "GET /test.go?foo=bar&a=b HTTP/1.0\nHost: example.com\n\n", expectedMap) - if expected, got := "text/html; charset=utf-8", replay.Header().Get("Content-Type"); got != expected { + if expected, got := "text/plain; charset=utf-8", replay.Header().Get("Content-Type"); got != expected { t.Errorf("got a Content-Type of %q; expected %q", got, expected) } if expected, got := "X-Test-Value", replay.Header().Get("X-Test-Header"); got != expected { @@ -152,6 +154,51 @@ func TestChildOnlyHeaders(t *testing.T) { } } +func TestChildContentType(t *testing.T) { + testenv.MustHaveExec(t) + + h := &Handler{ + Path: os.Args[0], + Root: "/test.go", + Args: []string{"-test.run=TestBeChildCGIProcess"}, + } + var tests = []struct { + name string + body string + wantCT string + }{ + { + name: "no body", + wantCT: "text/plain; charset=utf-8", + }, + { + name: "html", + body: "test pageThis is a body", + wantCT: "text/html; charset=utf-8", + }, + { + name: "text", + body: strings.Repeat("gopher", 86), + wantCT: "text/plain; charset=utf-8", + }, + { + name: "jpg", + body: "\xFF\xD8\xFF" + strings.Repeat("B", 1024), + wantCT: "image/jpeg", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + expectedMap := map[string]string{"_body": tt.body} + req := fmt.Sprintf("GET /test.go?exact-body=%s HTTP/1.0\nHost: example.com\n\n", url.QueryEscape(tt.body)) + replay := runCgiTest(t, h, req, expectedMap) + if got := replay.Header().Get("Content-Type"); got != tt.wantCT { + t.Errorf("got a Content-Type of %q; expected it to start with %q", got, tt.wantCT) + } + }) + } +} + // golang.org/issue/7198 func Test500WithNoHeaders(t *testing.T) { want500Test(t, "/immediate-disconnect") } func Test500WithNoContentType(t *testing.T) { want500Test(t, "/no-content-type") } @@ -203,6 +250,10 @@ func TestBeChildCGIProcess(t *testing.T) { if req.FormValue("no-body") == "1" { return } + if eb, ok := req.Form["exact-body"]; ok { + io.WriteString(rw, eb[0]) + return + } if req.FormValue("write-forever") == "1" { io.Copy(rw, neverEnding('a')) for { diff --git a/src/net/http/fcgi/child.go b/src/net/http/fcgi/child.go index 30a6b2ce2d..a31273b3ec 100644 --- a/src/net/http/fcgi/child.go +++ b/src/net/http/fcgi/child.go @@ -74,10 +74,12 @@ func (r *request) parseParams() { // response implements http.ResponseWriter. type response struct { - req *request - header http.Header - w *bufWriter - wroteHeader bool + req *request + header http.Header + code int + wroteHeader bool + wroteCGIHeader bool + w *bufWriter } func newResponse(c *child, req *request) *response { @@ -92,11 +94,14 @@ func (r *response) Header() http.Header { return r.header } -func (r *response) Write(data []byte) (int, error) { +func (r *response) Write(p []byte) (n int, err error) { if !r.wroteHeader { r.WriteHeader(http.StatusOK) } - return r.w.Write(data) + if !r.wroteCGIHeader { + r.writeCGIHeader(p) + } + return r.w.Write(p) } func (r *response) WriteHeader(code int) { @@ -104,22 +109,34 @@ func (r *response) WriteHeader(code int) { return } r.wroteHeader = true + r.code = code if code == http.StatusNotModified { // Must not have body. r.header.Del("Content-Type") r.header.Del("Content-Length") r.header.Del("Transfer-Encoding") - } else if r.header.Get("Content-Type") == "" { - r.header.Set("Content-Type", "text/html; charset=utf-8") } - if r.header.Get("Date") == "" { r.header.Set("Date", time.Now().UTC().Format(http.TimeFormat)) } +} - fmt.Fprintf(r.w, "Status: %d %s\r\n", code, http.StatusText(code)) +// writeCGIHeader finalizes the header sent to the client and writes it to the output. +// p is not written by writeHeader, but is the first chunk of the body +// that will be written. It is sniffed for a Content-Type if none is +// set explicitly. +func (r *response) writeCGIHeader(p []byte) { + if r.wroteCGIHeader { + return + } + r.wroteCGIHeader = true + fmt.Fprintf(r.w, "Status: %d %s\r\n", r.code, http.StatusText(r.code)) + if _, hasType := r.header["Content-Type"]; r.code != http.StatusNotModified && !hasType { + r.header.Set("Content-Type", http.DetectContentType(p)) + } r.header.Write(r.w) r.w.WriteString("\r\n") + r.w.Flush() } func (r *response) Flush() { @@ -290,6 +307,8 @@ func (c *child) serveRequest(req *request, body io.ReadCloser) { httpReq = httpReq.WithContext(envVarCtx) c.handler.ServeHTTP(r, httpReq) } + // Make sure we serve something even if nothing was written to r + r.Write(nil) r.Close() c.mu.Lock() delete(c.requests, req.reqId) diff --git a/src/net/http/fcgi/fcgi_test.go b/src/net/http/fcgi/fcgi_test.go index e9d2b34023..4a27a12c35 100644 --- a/src/net/http/fcgi/fcgi_test.go +++ b/src/net/http/fcgi/fcgi_test.go @@ -10,6 +10,7 @@ import ( "io" "io/ioutil" "net/http" + "strings" "testing" ) @@ -344,3 +345,54 @@ func TestChildServeReadsEnvVars(t *testing.T) { <-done } } + +func TestResponseWriterSniffsContentType(t *testing.T) { + var tests = []struct { + name string + body string + wantCT string + }{ + { + name: "no body", + wantCT: "text/plain; charset=utf-8", + }, + { + name: "html", + body: "test pageThis is a body", + wantCT: "text/html; charset=utf-8", + }, + { + name: "text", + body: strings.Repeat("gopher", 86), + wantCT: "text/plain; charset=utf-8", + }, + { + name: "jpg", + body: "\xFF\xD8\xFF" + strings.Repeat("B", 1024), + wantCT: "image/jpeg", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + input := make([]byte, len(streamFullRequestStdin)) + copy(input, streamFullRequestStdin) + rc := nopWriteCloser{bytes.NewBuffer(input)} + done := make(chan bool) + var resp *response + c := newChild(rc, http.HandlerFunc(func( + w http.ResponseWriter, + r *http.Request, + ) { + io.WriteString(w, tt.body) + resp = w.(*response) + done <- true + })) + defer c.cleanUp() + go c.serve() + <-done + if got := resp.Header().Get("Content-Type"); got != tt.wantCT { + t.Errorf("got a Content-Type of %q; expected it to start with %q", got, tt.wantCT) + } + }) + } +} -- cgit v1.2.3-54-g00ecf From 01af46f7cc419da19f8a6a444da8f6022c016803 Mon Sep 17 00:00:00 2001 From: Dmitri Shuralyov Date: Tue, 1 Sep 2020 08:58:41 -0400 Subject: [release-branch.go1.15-security] go1.15.1 Change-Id: I4103c524ce46d50215af5097460e514609b513c6 Reviewed-on: https://team-review.git.corp.google.com/c/golang/go-private/+/835373 Reviewed-by: Filippo Valsorda --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 4b9a234fa9..29a7367f57 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -go1.15 \ No newline at end of file +go1.15.1 \ No newline at end of file -- cgit v1.2.3-54-g00ecf From 0ffc5672df75accd06fb7477c46d4a7d1740401d Mon Sep 17 00:00:00 2001 From: "Bryan C. Mills" Date: Mon, 17 Aug 2020 17:31:21 -0400 Subject: [release-branch.go1.15] testing: treat PAUSE lines as changing the active test name We could instead fix cmd/test2json to treat PAUSE lines as *not* changing the active test name, but that seems like it would be more confusing to humans, and also wouldn't fix tools that parse output using existing builds of cmd/test2json. Fixes #40849 Updates #40657 Change-Id: I937611778f5b1e7dd1d6e9f44424d7e725a589ed Reviewed-on: https://go-review.googlesource.com/c/go/+/248727 Run-TryBot: Bryan C. Mills Reviewed-by: Jean de Klerk (cherry picked from commit cdc77d34d7770ed02d84b9193380f9646017dce6) Reviewed-on: https://go-review.googlesource.com/c/go/+/249097 TryBot-Result: Gobot Gobot --- .../go/testdata/script/test_json_interleaved.txt | 27 ++++++++++++++++++++++ src/testing/testing.go | 25 ++++++++++++++++---- 2 files changed, 47 insertions(+), 5 deletions(-) create mode 100644 src/cmd/go/testdata/script/test_json_interleaved.txt diff --git a/src/cmd/go/testdata/script/test_json_interleaved.txt b/src/cmd/go/testdata/script/test_json_interleaved.txt new file mode 100644 index 0000000000..e2d349e3fb --- /dev/null +++ b/src/cmd/go/testdata/script/test_json_interleaved.txt @@ -0,0 +1,27 @@ +# Regression test for https://golang.org/issue/40657: output from the main test +# function should be attributed correctly even if interleaved with the PAUSE +# line for a new parallel subtest. + +[short] skip + +go test -json +stdout '"Test":"TestWeirdTiming","Output":"[^"]* logging to outer again\\n"' + +-- go.mod -- +module example.com +go 1.15 +-- main_test.go -- +package main + +import ( + "testing" +) + +func TestWeirdTiming(outer *testing.T) { + outer.Run("pauser", func(pauser *testing.T) { + outer.Logf("logging to outer") + pauser.Parallel() + }) + + outer.Logf("logging to outer again") +} diff --git a/src/testing/testing.go b/src/testing/testing.go index 1bfcbb27e4..bf83df8863 100644 --- a/src/testing/testing.go +++ b/src/testing/testing.go @@ -357,10 +357,19 @@ func (p *testPrinter) Fprint(w io.Writer, testName, out string) { defer p.lastNameMu.Unlock() if !p.chatty || - strings.HasPrefix(out, "--- PASS") || - strings.HasPrefix(out, "--- FAIL") || - strings.HasPrefix(out, "=== CONT") || - strings.HasPrefix(out, "=== RUN") { + strings.HasPrefix(out, "--- PASS: ") || + strings.HasPrefix(out, "--- FAIL: ") || + strings.HasPrefix(out, "--- SKIP: ") || + strings.HasPrefix(out, "=== RUN ") || + strings.HasPrefix(out, "=== CONT ") || + strings.HasPrefix(out, "=== PAUSE ") { + // If we're buffering test output (!p.chatty), we don't really care which + // test is emitting which line so long as they are serialized. + // + // If the message already implies an association with a specific new test, + // we don't need to check what the old test name was or log an extra CONT + // line for it. (We're updating it anyway, and the current message already + // includes the test name.) p.lastName = testName fmt.Fprint(w, out) return @@ -980,7 +989,13 @@ func (t *T) Parallel() { for ; root.parent != nil; root = root.parent { } root.mu.Lock() - fmt.Fprintf(root.w, "=== PAUSE %s\n", t.name) + // Unfortunately, even though PAUSE indicates that the named test is *no + // longer* running, cmd/test2json interprets it as changing the active test + // for the purpose of log parsing. We could fix cmd/test2json, but that + // won't fix existing deployments of third-party tools that already shell + // out to older builds of cmd/test2json — so merely fixing cmd/test2json + // isn't enough for now. + printer.Fprint(root.w, t.name, fmt.Sprintf("=== PAUSE %s\n", t.name)) root.mu.Unlock() } -- cgit v1.2.3-54-g00ecf From a269e5f9391ee462cd94ccfb59c8c4460d4a7eb7 Mon Sep 17 00:00:00 2001 From: chainhelen Date: Fri, 21 Aug 2020 16:44:52 +0000 Subject: [release-branch.go1.15] runtime: fix panic if newstack at runtime.acquireLockRank Process may crash becaues acquireLockRank and releaseLockRank may be called in nosplit context. With optimizations and inlining disabled, these functions won't get inlined or have their morestack calls eliminated. Nosplit is not strictly required for lockWithRank, unlockWithRank and lockWithRankMayAcquire, just keep consistency with lockrank_on.go here. Updates #40843. Fixes #40845. Change-Id: I5824119f98a1da66d767cdb9a60dffe768f13c81 GitHub-Last-Rev: 38fd3ccf6ea03b670c7561c060ccdbccc42fff40 GitHub-Pull-Request: golang/go#40844 Reviewed-on: https://go-review.googlesource.com/c/go/+/248878 Reviewed-by: Dan Scales Run-TryBot: Emmanuel Odeke TryBot-Result: Gobot Gobot (cherry picked from commit b246c0e12fd41caf45a0f81eaa4f8fe249fbbc01) Reviewed-on: https://go-review.googlesource.com/c/go/+/252339 Run-TryBot: Dmitri Shuralyov --- src/runtime/lockrank_off.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/runtime/lockrank_off.go b/src/runtime/lockrank_off.go index 425ca8dd93..32378a9627 100644 --- a/src/runtime/lockrank_off.go +++ b/src/runtime/lockrank_off.go @@ -18,19 +18,29 @@ func getLockRank(l *mutex) lockRank { return 0 } +// The following functions may be called in nosplit context. +// Nosplit is not strictly required for lockWithRank, unlockWithRank +// and lockWithRankMayAcquire, but these nosplit annotations must +// be kept consistent with the equivalent functions in lockrank_on.go. + +//go:nosplit func lockWithRank(l *mutex, rank lockRank) { lock2(l) } +//go:nosplit func acquireLockRank(rank lockRank) { } +//go:nosplit func unlockWithRank(l *mutex) { unlock2(l) } +//go:nosplit func releaseLockRank(rank lockRank) { } +//go:nosplit func lockWithRankMayAcquire(l *mutex, rank lockRank) { } -- cgit v1.2.3-54-g00ecf From 60fdbce6fd02e226454ae28c8644b3052a793c3e Mon Sep 17 00:00:00 2001 From: ShihCheng Tu Date: Mon, 15 Jun 2020 00:07:10 +0800 Subject: [release-branch.go1.15] doc/go1.14: document json.Umarshal map key support of TextUnmarshaler MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Document that json.Unmarshal supports map keys whose underlying types implement encoding.TextUnmarshaler. Updates #38801. Fixes #41178. Change-Id: Icb9414e9067517531ba0da910bd4a2bb3daace65 Reviewed-on: https://go-review.googlesource.com/c/go/+/237857 Reviewed-by: Daniel Martí Run-TryBot: Daniel Martí TryBot-Result: Gobot Gobot (cherry picked from commit 47b450997778163dfed6f58cae379d928fc37687) Reviewed-on: https://go-review.googlesource.com/c/go/+/252618 Run-TryBot: Dmitri Shuralyov Reviewed-by: Carlos Amedee --- doc/go1.14.html | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/doc/go1.14.html b/doc/go1.14.html index 35a9f3c2f3..410e0cbf7c 100644 --- a/doc/go1.14.html +++ b/doc/go1.14.html @@ -609,6 +609,12 @@ Do not send CLs removing the interior tags from such phrases. If a program needs to accept invalid numbers like the empty string, consider wrapping the type with Unmarshaler.

+ +

+ Unmarshal + can now support map keys with string underlying type which implement + encoding.TextUnmarshaler. +

-- cgit v1.2.3-54-g00ecf From 76a89d6ca0436f027c9c3df59a7a542a99ee790f Mon Sep 17 00:00:00 2001 From: "Bryan C. Mills" Date: Fri, 14 Aug 2020 15:47:49 -0400 Subject: [release-branch.go1.15] cmd/go/internal/test: keep looking for go command flags after ambiguous test flag Updates #40763 Fixes #40802 Change-Id: I275970d1f8561414571a5b93e368d68fa052c60f Reviewed-on: https://go-review.googlesource.com/c/go/+/248618 Run-TryBot: Bryan C. Mills TryBot-Result: Gobot Gobot Reviewed-by: Jay Conrod (cherry picked from commit 797124f5ff4bb80957007adbf3115287a4e90870) Reviewed-on: https://go-review.googlesource.com/c/go/+/248726 --- src/cmd/go/internal/test/testflag.go | 29 +++++++++++++++++++++---- src/cmd/go/testdata/script/test_flags.txt | 35 +++++++++++++++++++++++++------ 2 files changed, 54 insertions(+), 10 deletions(-) diff --git a/src/cmd/go/internal/test/testflag.go b/src/cmd/go/internal/test/testflag.go index 1ff34f7445..4f0a8924f1 100644 --- a/src/cmd/go/internal/test/testflag.go +++ b/src/cmd/go/internal/test/testflag.go @@ -214,9 +214,13 @@ func testFlags(args []string) (packageNames, passToTest []string) { explicitArgs := make([]string, 0, len(args)) inPkgList := false + afterFlagWithoutValue := false for len(args) > 0 { f, remainingArgs, err := cmdflag.ParseOne(&CmdTest.Flag, args) + wasAfterFlagWithoutValue := afterFlagWithoutValue + afterFlagWithoutValue = false // provisionally + if errors.Is(err, flag.ErrHelp) { exitWithUsage() } @@ -233,10 +237,24 @@ func testFlags(args []string) (packageNames, passToTest []string) { if nf := (cmdflag.NonFlagError{}); errors.As(err, &nf) { if !inPkgList && packageNames != nil { // We already saw the package list previously, and this argument is not - // a flag, so it — and everything after it — must be a literal argument - // to the test binary. - explicitArgs = append(explicitArgs, args...) - break + // a flag, so it — and everything after it — must be either a value for + // a preceding flag or a literal argument to the test binary. + if wasAfterFlagWithoutValue { + // This argument could syntactically be a flag value, so + // optimistically assume that it is and keep looking for go command + // flags after it. + // + // (If we're wrong, we'll at least be consistent with historical + // behavior; see https://golang.org/issue/40763.) + explicitArgs = append(explicitArgs, nf.RawArg) + args = remainingArgs + continue + } else { + // This argument syntactically cannot be a flag value, so it must be a + // positional argument, and so must everything after it. + explicitArgs = append(explicitArgs, args...) + break + } } inPkgList = true @@ -272,6 +290,9 @@ func testFlags(args []string) (packageNames, passToTest []string) { explicitArgs = append(explicitArgs, nd.RawArg) args = remainingArgs + if !nd.HasValue { + afterFlagWithoutValue = true + } continue } diff --git a/src/cmd/go/testdata/script/test_flags.txt b/src/cmd/go/testdata/script/test_flags.txt index d38e37f238..63385e6997 100644 --- a/src/cmd/go/testdata/script/test_flags.txt +++ b/src/cmd/go/testdata/script/test_flags.txt @@ -10,7 +10,7 @@ stdout '\Aok\s+example.com/x\s+[0-9.s]+\n\z' ! stderr . # For backward-compatibility with previous releases of the 'go' command, -# arguments that appear after unrecognized flags should not be treated +# arguments that appear after unrecognized flags should not be treated # as packages, even if they are unambiguously not arguments to flags. # Even though ./x looks like a package path, the real package should be # the implicit '.'. @@ -18,6 +18,22 @@ stdout '\Aok\s+example.com/x\s+[0-9.s]+\n\z' stderr '^no Go files in .+$' ! stderr '/x' +# However, *flags* that appear after unrecognized flags should still be +# interpreted as flags, under the (possibly-erroneous) assumption that +# unrecognized flags are non-boolean. + +go test -v -x ./x -timeout 24h -boolflag=true foo -timeout 25h +stdout 'args: foo -timeout 25h' +stdout 'timeout: 24h0m0s$' # -timeout is unambiguously not a flag, so the real flag wins. + +go test -v -x ./x -timeout 24h -boolflag foo -timeout 25h +stdout 'args: foo -test\.timeout=25h0m0s' # For legacy reasons, '-timeout ' is erroneously rewritten to -test.timeout; see https://golang.org/issue/40763. +stdout 'timeout: 24h0m0s$' # Actual flag wins. + +go test -v -x ./x -timeout 24h -stringflag foo -timeout 25h +stdout 'args: $' +stdout 'timeout: 25h0m0s$' # Later flag wins. + # An explicit '-outputdir=' argument should set test.outputdir # to the 'go' command's working directory, not zero it out # for the test binary. @@ -30,23 +46,23 @@ exists ./cover.out # with the 'test.' prefix in the GOFLAGS entry... env GOFLAGS='-test.timeout=24h0m0s -count=1' go test -v -x ./x -stdout '.*: 24h0m0s$' +stdout 'timeout: 24h0m0s$' stderr '-test.count=1' # ...or without. env GOFLAGS='-timeout=24h0m0s -count=1' go test -v -x ./x -stdout '.*: 24h0m0s$' +stdout 'timeout: 24h0m0s$' stderr '-test.count=1' # Arguments from the command line should override GOFLAGS... go test -v -x -timeout=25h0m0s ./x -stdout '.*: 25h0m0s$' +stdout 'timeout: 25h0m0s$' stderr '-test.count=1' # ...even if they use a different flag name. go test -v -x -test.timeout=26h0m0s ./x -stdout '.*: 26h0m0s$' +stdout 'timeout: 26h0m0s$' stderr '-test\.timeout=26h0m0s' ! stderr 'timeout=24h0m0s' stderr '-test.count=1' @@ -99,11 +115,18 @@ package x import ( "flag" + "strings" "testing" ) var _ = flag.String("usage_message", "", "dummy flag to check usage message") +var boolflag = flag.Bool("boolflag", false, "ignored boolean flag") +var stringflag = flag.String("stringflag", "", "ignored string flag") func TestLogTimeout(t *testing.T) { - t.Log(flag.Lookup("test.timeout").Value) + t.Logf("timeout: %v", flag.Lookup("test.timeout").Value) +} + +func TestLogArgs(t *testing.T) { + t.Logf("args: %s", strings.Join(flag.Args(), " ")) } -- cgit v1.2.3-54-g00ecf From 09b28977f599d846aca968102c8d6e8b0283a18b Mon Sep 17 00:00:00 2001 From: "Bryan C. Mills" Date: Fri, 14 Aug 2020 17:44:22 -0400 Subject: [release-branch.go1.15] cmd/test2json: do not emit a final Action if the result is not known MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If we are parsing a test output, and the test does not end in the usual PASS or FAIL line (say, because it panicked), then we need the exit status of the test binary in order to determine whether the test passed or failed. If we don't have that status available, we shouldn't guess arbitrarily — instead, we should omit the final "pass" or "fail" action entirely. (In practice, we nearly always DO have the final status, such as when running 'go test' or 'go tool test2json some.exe'.) Updates #40132 Fixes #40805 Change-Id: Iae482577361a6033395fe4a05d746b980e18c3de Reviewed-on: https://go-review.googlesource.com/c/go/+/248624 Run-TryBot: Bryan C. Mills TryBot-Result: Gobot Gobot Reviewed-by: Jay Conrod (cherry picked from commit 1b86bdbdc3991c13c6ed156100a5f4918fdd9c6b) Reviewed-on: https://go-review.googlesource.com/c/go/+/248725 --- src/cmd/go/internal/test/test.go | 8 +- src/cmd/go/testdata/script/test_json_exit.txt | 102 +++++++++++++++++++++ src/cmd/internal/test2json/test2json.go | 44 +++++---- .../internal/test2json/testdata/benchshort.json | 1 - src/cmd/internal/test2json/testdata/empty.json | 1 - src/cmd/test2json/main.go | 6 +- 6 files changed, 139 insertions(+), 23 deletions(-) create mode 100644 src/cmd/go/testdata/script/test_json_exit.txt diff --git a/src/cmd/go/internal/test/test.go b/src/cmd/go/internal/test/test.go index 873a76aa38..77bfc11fe9 100644 --- a/src/cmd/go/internal/test/test.go +++ b/src/cmd/go/internal/test/test.go @@ -1079,9 +1079,13 @@ func (c *runCache) builderRunTest(b *work.Builder, a *work.Action) error { } var stdout io.Writer = os.Stdout + var err error if testJSON { json := test2json.NewConverter(lockedStdout{}, a.Package.ImportPath, test2json.Timestamp) - defer json.Close() + defer func() { + json.Exited(err) + json.Close() + }() stdout = json } @@ -1185,7 +1189,7 @@ func (c *runCache) builderRunTest(b *work.Builder, a *work.Action) error { } t0 := time.Now() - err := cmd.Start() + err = cmd.Start() // This is a last-ditch deadline to detect and // stop wedged test binaries, to keep the builders diff --git a/src/cmd/go/testdata/script/test_json_exit.txt b/src/cmd/go/testdata/script/test_json_exit.txt new file mode 100644 index 0000000000..dc7ffb06cf --- /dev/null +++ b/src/cmd/go/testdata/script/test_json_exit.txt @@ -0,0 +1,102 @@ +[short] skip + +go test -c -o mainpanic.exe ./mainpanic & +go test -c -o mainexit0.exe ./mainexit0 & +go test -c -o testpanic.exe ./testpanic & +go test -c -o testbgpanic.exe ./testbgpanic & +wait + +# Test binaries that panic in TestMain should be marked as failing. + +! go test -json ./mainpanic +stdout '"Action":"fail"' +! stdout '"Action":"pass"' + +! go tool test2json ./mainpanic.exe +stdout '"Action":"fail"' +! stdout '"Action":"pass"' + +# Test binaries that exit with status 0 should be marked as passing. + +go test -json ./mainexit0 +stdout '"Action":"pass"' +! stdout '"Action":"fail"' + +go tool test2json ./mainexit0.exe +stdout '"Action":"pass"' +! stdout '"Action":"fail"' + +# Test functions that panic should never be marked as passing +# (https://golang.org/issue/40132). + +! go test -json ./testpanic +stdout '"Action":"fail"' +! stdout '"Action":"pass"' + +! go tool test2json ./testpanic.exe -test.v +stdout '"Action":"fail"' +! stdout '"Action":"pass"' + +! go tool test2json ./testpanic.exe +stdout '"Action":"fail"' +! stdout '"Action":"pass"' + +# Tests that panic in a background goroutine should be marked as failing. + +! go test -json ./testbgpanic +stdout '"Action":"fail"' +! stdout '"Action":"pass"' + +! go tool test2json ./testbgpanic.exe -test.v +stdout '"Action":"fail"' +! stdout '"Action":"pass"' + +! go tool test2json ./testbgpanic.exe +stdout '"Action":"fail"' +! stdout '"Action":"pass"' + +-- go.mod -- +module m +go 1.14 +-- mainpanic/mainpanic_test.go -- +package mainpanic_test + +import "testing" + +func TestMain(m *testing.M) { + panic("haha no") +} +-- mainexit0/mainexit0_test.go -- +package mainexit0_test + +import ( + "fmt" + "os" + "testing" +) + +func TestMain(m *testing.M) { + fmt.Println("nothing to do") + os.Exit(0) +} +-- testpanic/testpanic_test.go -- +package testpanic_test + +import "testing" + +func TestPanic(*testing.T) { + panic("haha no") +} +-- testbgpanic/testbgpanic_test.go -- +package testbgpanic_test + +import "testing" + +func TestPanicInBackground(*testing.T) { + c := make(chan struct{}) + go func() { + panic("haha no") + close(c) + }() + <-c +} diff --git a/src/cmd/internal/test2json/test2json.go b/src/cmd/internal/test2json/test2json.go index a01a8900e8..4eb6dd4838 100644 --- a/src/cmd/internal/test2json/test2json.go +++ b/src/cmd/internal/test2json/test2json.go @@ -45,10 +45,10 @@ type textBytes []byte func (b textBytes) MarshalText() ([]byte, error) { return b, nil } -// A converter holds the state of a test-to-JSON conversion. +// A Converter holds the state of a test-to-JSON conversion. // It implements io.WriteCloser; the caller writes test output in, // and the converter writes JSON output to w. -type converter struct { +type Converter struct { w io.Writer // JSON output stream pkg string // package to name in events mode Mode // mode bits @@ -100,9 +100,9 @@ var ( // // The pkg string, if present, specifies the import path to // report in the JSON stream. -func NewConverter(w io.Writer, pkg string, mode Mode) io.WriteCloser { - c := new(converter) - *c = converter{ +func NewConverter(w io.Writer, pkg string, mode Mode) *Converter { + c := new(Converter) + *c = Converter{ w: w, pkg: pkg, mode: mode, @@ -122,11 +122,20 @@ func NewConverter(w io.Writer, pkg string, mode Mode) io.WriteCloser { } // Write writes the test input to the converter. -func (c *converter) Write(b []byte) (int, error) { +func (c *Converter) Write(b []byte) (int, error) { c.input.write(b) return len(b), nil } +// Exited marks the test process as having exited with the given error. +func (c *Converter) Exited(err error) { + if err == nil { + c.result = "pass" + } else { + c.result = "fail" + } +} + var ( // printed by test on successful run. bigPass = []byte("PASS\n") @@ -160,7 +169,7 @@ var ( // handleInputLine handles a single whole test output line. // It must write the line to c.output but may choose to do so // before or after emitting other events. -func (c *converter) handleInputLine(line []byte) { +func (c *Converter) handleInputLine(line []byte) { // Final PASS or FAIL. if bytes.Equal(line, bigPass) || bytes.Equal(line, bigFail) || bytes.HasPrefix(line, bigFailErrorPrefix) { c.flushReport(0) @@ -286,7 +295,7 @@ func (c *converter) handleInputLine(line []byte) { } // flushReport flushes all pending PASS/FAIL reports at levels >= depth. -func (c *converter) flushReport(depth int) { +func (c *Converter) flushReport(depth int) { c.testName = "" for len(c.report) > depth { e := c.report[len(c.report)-1] @@ -298,23 +307,22 @@ func (c *converter) flushReport(depth int) { // Close marks the end of the go test output. // It flushes any pending input and then output (only partial lines at this point) // and then emits the final overall package-level pass/fail event. -func (c *converter) Close() error { +func (c *Converter) Close() error { c.input.flush() c.output.flush() - e := &event{Action: "pass"} if c.result != "" { - e.Action = c.result - } - if c.mode&Timestamp != 0 { - dt := time.Since(c.start).Round(1 * time.Millisecond).Seconds() - e.Elapsed = &dt + e := &event{Action: c.result} + if c.mode&Timestamp != 0 { + dt := time.Since(c.start).Round(1 * time.Millisecond).Seconds() + e.Elapsed = &dt + } + c.writeEvent(e) } - c.writeEvent(e) return nil } // writeOutputEvent writes a single output event with the given bytes. -func (c *converter) writeOutputEvent(out []byte) { +func (c *Converter) writeOutputEvent(out []byte) { c.writeEvent(&event{ Action: "output", Output: (*textBytes)(&out), @@ -323,7 +331,7 @@ func (c *converter) writeOutputEvent(out []byte) { // writeEvent writes a single event. // It adds the package, time (if requested), and test name (if needed). -func (c *converter) writeEvent(e *event) { +func (c *Converter) writeEvent(e *event) { e.Package = c.pkg if c.mode&Timestamp != 0 { t := time.Now() diff --git a/src/cmd/internal/test2json/testdata/benchshort.json b/src/cmd/internal/test2json/testdata/benchshort.json index 28e287c848..34b03b9362 100644 --- a/src/cmd/internal/test2json/testdata/benchshort.json +++ b/src/cmd/internal/test2json/testdata/benchshort.json @@ -4,4 +4,3 @@ {"Action":"output","Output":"# but to avoid questions of timing, we just use a file with no \\n at all.\n"} {"Action":"output","Output":"BenchmarkFoo \t"} {"Action":"output","Output":"10000 early EOF"} -{"Action":"pass"} diff --git a/src/cmd/internal/test2json/testdata/empty.json b/src/cmd/internal/test2json/testdata/empty.json index 80b5217501..e69de29bb2 100644 --- a/src/cmd/internal/test2json/testdata/empty.json +++ b/src/cmd/internal/test2json/testdata/empty.json @@ -1 +0,0 @@ -{"Action":"pass"} diff --git a/src/cmd/test2json/main.go b/src/cmd/test2json/main.go index 0385d8f246..57a874193e 100644 --- a/src/cmd/test2json/main.go +++ b/src/cmd/test2json/main.go @@ -118,12 +118,16 @@ func main() { w := &countWriter{0, c} cmd.Stdout = w cmd.Stderr = w - if err := cmd.Run(); err != nil { + err := cmd.Run() + if err != nil { if w.n > 0 { // Assume command printed why it failed. } else { fmt.Fprintf(c, "test2json: %v\n", err) } + } + c.Exited(err) + if err != nil { c.Close() os.Exit(1) } -- cgit v1.2.3-54-g00ecf From 33e86004f1c80fd5114696faec7c08e5c1cbb915 Mon Sep 17 00:00:00 2001 From: Filippo Valsorda Date: Wed, 2 Sep 2020 18:19:08 +0200 Subject: [release-branch.go1.15] net/http/fgci: skip flaky test A test introduced in the security release is flaky due to a pre-existing issue that does not qualify for backport itself. Updates #41167 Fixes #41193 Change-Id: Ie6014e0796c1baee7b077881b5a799f9947fc9c2 Reviewed-on: https://go-review.googlesource.com/c/go/+/252717 Run-TryBot: Dmitri Shuralyov Reviewed-by: Dmitri Shuralyov TryBot-Result: Gobot Gobot --- src/net/http/fcgi/fcgi_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/src/net/http/fcgi/fcgi_test.go b/src/net/http/fcgi/fcgi_test.go index 4a27a12c35..59246c26cc 100644 --- a/src/net/http/fcgi/fcgi_test.go +++ b/src/net/http/fcgi/fcgi_test.go @@ -347,6 +347,7 @@ func TestChildServeReadsEnvVars(t *testing.T) { } func TestResponseWriterSniffsContentType(t *testing.T) { + t.Skip("this test is flaky, see Issue 41167") var tests = []struct { name string body string -- cgit v1.2.3-54-g00ecf From a5fb3b1f3d0de39805462abce5a722c9c3ce8647 Mon Sep 17 00:00:00 2001 From: Lynn Boger Date: Tue, 11 Aug 2020 12:04:25 -0400 Subject: [release-branch.go1.15] cmd/internal/obj/ppc64: don't remove NOP in assembler Previously, the assembler removed NOPs from the Prog list in obj9.go. NOPs shouldn't be removed if they were added as an inline mark, as described in the issue below. Fixes #40767 Once the NOPs were left in the Prog list, some instructions were flagged as invalid because they had an operand which was not represented in optab. In order to preserve the previous assembler behavior, entries were added to optab for those operand cases. They were not flagged as errors before because the NOP instructions were removed before the code to check the valid opcode/operand combinations. Change-Id: Iae5145f94459027cf458e914d7c5d6089807ccf8 Reviewed-on: https://go-review.googlesource.com/c/go/+/247842 Run-TryBot: Lynn Boger TryBot-Result: Gobot Gobot Reviewed-by: Paul Murphy Reviewed-by: Michael Munday Reviewed-by: Keith Randall (cherry picked from commit 7d7bd5abc7f7ac901830b79496f63ce86895e262) Reviewed-on: https://go-review.googlesource.com/c/go/+/248381 Run-TryBot: Dmitri Shuralyov --- src/cmd/internal/obj/ppc64/asm9.go | 3 +++ src/cmd/internal/obj/ppc64/obj9.go | 11 +++-------- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/src/cmd/internal/obj/ppc64/asm9.go b/src/cmd/internal/obj/ppc64/asm9.go index 0fd0744a42..238ca8f0b7 100644 --- a/src/cmd/internal/obj/ppc64/asm9.go +++ b/src/cmd/internal/obj/ppc64/asm9.go @@ -613,6 +613,9 @@ var optab = []Optab{ {obj.APCDATA, C_LCON, C_NONE, C_NONE, C_LCON, 0, 0, 0}, {obj.AFUNCDATA, C_SCON, C_NONE, C_NONE, C_ADDR, 0, 0, 0}, {obj.ANOP, C_NONE, C_NONE, C_NONE, C_NONE, 0, 0, 0}, + {obj.ANOP, C_LCON, C_NONE, C_NONE, C_NONE, 0, 0, 0}, // NOP operand variations added for #40689 + {obj.ANOP, C_REG, C_NONE, C_NONE, C_NONE, 0, 0, 0}, // to preserve previous behavior + {obj.ANOP, C_FREG, C_NONE, C_NONE, C_NONE, 0, 0, 0}, {obj.ADUFFZERO, C_NONE, C_NONE, C_NONE, C_LBRA, 11, 4, 0}, // same as ABR/ABL {obj.ADUFFCOPY, C_NONE, C_NONE, C_NONE, C_LBRA, 11, 4, 0}, // same as ABR/ABL {obj.APCALIGN, C_LCON, C_NONE, C_NONE, C_NONE, 0, 0, 0}, // align code diff --git a/src/cmd/internal/obj/ppc64/obj9.go b/src/cmd/internal/obj/ppc64/obj9.go index 7135488f9d..a2f16f2e2a 100644 --- a/src/cmd/internal/obj/ppc64/obj9.go +++ b/src/cmd/internal/obj/ppc64/obj9.go @@ -427,7 +427,6 @@ func preprocess(ctxt *obj.Link, cursym *obj.LSym, newprog obj.ProgAlloc) { /* * find leaf subroutines - * strip NOPs * expand RET * expand BECOME pseudo */ @@ -557,10 +556,7 @@ func preprocess(ctxt *obj.Link, cursym *obj.LSym, newprog obj.ProgAlloc) { q = p q1 = p.Pcond if q1 != nil { - for q1.As == obj.ANOP { - q1 = q1.Link - p.Pcond = q1 - } + // NOPs are not removed due to #40689. if q1.Mark&LEAF == 0 { q1.Mark |= LABEL @@ -587,9 +583,8 @@ func preprocess(ctxt *obj.Link, cursym *obj.LSym, newprog obj.ProgAlloc) { continue case obj.ANOP: - q1 = p.Link - q.Link = q1 /* q is non-nop */ - q1.Mark |= p.Mark + // NOPs are not removed due to + // #40689 continue default: -- cgit v1.2.3-54-g00ecf From b616f745ef6eb14daeb1d7d9b41a2908835bc3fd Mon Sep 17 00:00:00 2001 From: Keith Randall Date: Sat, 15 Aug 2020 08:08:37 -0700 Subject: [release-branch.go1.15] cmd/internal/obj: stop removing NOPs from instruction stream This has already been done for s390x, ppc64. This CL is for all the other architectures. Fixes #40798 Change-Id: Idd1816e057df63022d47e99fa06617811d8c8489 Reviewed-on: https://go-review.googlesource.com/c/go/+/248684 Run-TryBot: Cherry Zhang TryBot-Result: Gobot Gobot Reviewed-by: Cherry Zhang (cherry picked from commit 46ca7b5ee2a8582736f1ddac27d8660e1104c345) Reviewed-on: https://go-review.googlesource.com/c/go/+/249444 Run-TryBot: Dmitri Shuralyov --- src/cmd/internal/obj/arm/asm5.go | 3 ++ src/cmd/internal/obj/arm/obj5.go | 46 ------------------------------- src/cmd/internal/obj/arm64/asm7.go | 3 ++ src/cmd/internal/obj/arm64/obj7.go | 56 ++------------------------------------ src/cmd/internal/obj/mips/asm0.go | 3 ++ src/cmd/internal/obj/mips/obj0.go | 26 ++---------------- 6 files changed, 14 insertions(+), 123 deletions(-) diff --git a/src/cmd/internal/obj/arm/asm5.go b/src/cmd/internal/obj/arm/asm5.go index f66f8aaf84..7b7e42ee2e 100644 --- a/src/cmd/internal/obj/arm/asm5.go +++ b/src/cmd/internal/obj/arm/asm5.go @@ -327,6 +327,9 @@ var optab = []Optab{ {obj.APCDATA, C_LCON, C_NONE, C_LCON, 0, 0, 0, 0, 0, 0}, {obj.AFUNCDATA, C_LCON, C_NONE, C_ADDR, 0, 0, 0, 0, 0, 0}, {obj.ANOP, C_NONE, C_NONE, C_NONE, 0, 0, 0, 0, 0, 0}, + {obj.ANOP, C_LCON, C_NONE, C_NONE, 0, 0, 0, 0, 0, 0}, // nop variants, see #40689 + {obj.ANOP, C_REG, C_NONE, C_NONE, 0, 0, 0, 0, 0, 0}, + {obj.ANOP, C_FREG, C_NONE, C_NONE, 0, 0, 0, 0, 0, 0}, {obj.ADUFFZERO, C_NONE, C_NONE, C_SBRA, 5, 4, 0, 0, 0, 0}, // same as ABL {obj.ADUFFCOPY, C_NONE, C_NONE, C_SBRA, 5, 4, 0, 0, 0, 0}, // same as ABL {obj.AXXX, C_NONE, C_NONE, C_NONE, 0, 4, 0, 0, 0, 0}, diff --git a/src/cmd/internal/obj/arm/obj5.go b/src/cmd/internal/obj/arm/obj5.go index 008118c47b..86831f2b44 100644 --- a/src/cmd/internal/obj/arm/obj5.go +++ b/src/cmd/internal/obj/arm/obj5.go @@ -276,67 +276,21 @@ func preprocess(ctxt *obj.Link, cursym *obj.LSym, newprog obj.ProgAlloc) { /* * find leaf subroutines - * strip NOPs - * expand RET - * expand BECOME pseudo */ - var q1 *obj.Prog - var q *obj.Prog for p := cursym.Func.Text; p != nil; p = p.Link { switch p.As { case obj.ATEXT: p.Mark |= LEAF - case obj.ARET: - break - case ADIV, ADIVU, AMOD, AMODU: - q = p cursym.Func.Text.Mark &^= LEAF - continue - - case obj.ANOP: - q1 = p.Link - q.Link = q1 /* q is non-nop */ - if q1 != nil { - q1.Mark |= p.Mark - } - continue case ABL, ABX, obj.ADUFFZERO, obj.ADUFFCOPY: cursym.Func.Text.Mark &^= LEAF - fallthrough - - case AB, - ABEQ, - ABNE, - ABCS, - ABHS, - ABCC, - ABLO, - ABMI, - ABPL, - ABVS, - ABVC, - ABHI, - ABLS, - ABGE, - ABLT, - ABGT, - ABLE: - q1 = p.Pcond - if q1 != nil { - for q1.As == obj.ANOP { - q1 = q1.Link - p.Pcond = q1 - } - } } - - q = p } var q2 *obj.Prog diff --git a/src/cmd/internal/obj/arm64/asm7.go b/src/cmd/internal/obj/arm64/asm7.go index df17729a76..f18799b60a 100644 --- a/src/cmd/internal/obj/arm64/asm7.go +++ b/src/cmd/internal/obj/arm64/asm7.go @@ -837,6 +837,9 @@ var optab = []Optab{ {obj.APCDATA, C_VCON, C_NONE, C_NONE, C_VCON, 0, 0, 0, 0, 0}, {obj.AFUNCDATA, C_VCON, C_NONE, C_NONE, C_ADDR, 0, 0, 0, 0, 0}, {obj.ANOP, C_NONE, C_NONE, C_NONE, C_NONE, 0, 0, 0, 0, 0}, + {obj.ANOP, C_LCON, C_NONE, C_NONE, C_NONE, 0, 0, 0, 0, 0}, // nop variants, see #40689 + {obj.ANOP, C_REG, C_NONE, C_NONE, C_NONE, 0, 0, 0, 0, 0}, + {obj.ANOP, C_VREG, C_NONE, C_NONE, C_NONE, 0, 0, 0, 0, 0}, {obj.ADUFFZERO, C_NONE, C_NONE, C_NONE, C_SBRA, 5, 4, 0, 0, 0}, // same as AB/ABL {obj.ADUFFCOPY, C_NONE, C_NONE, C_NONE, C_SBRA, 5, 4, 0, 0, 0}, // same as AB/ABL {obj.APCALIGN, C_LCON, C_NONE, C_NONE, C_NONE, 0, 0, 0, 0, 0}, // align code diff --git a/src/cmd/internal/obj/arm64/obj7.go b/src/cmd/internal/obj/arm64/obj7.go index b046685ada..0d74430053 100644 --- a/src/cmd/internal/obj/arm64/obj7.go +++ b/src/cmd/internal/obj/arm64/obj7.go @@ -468,73 +468,21 @@ func preprocess(ctxt *obj.Link, cursym *obj.LSym, newprog obj.ProgAlloc) { /* * find leaf subroutines - * strip NOPs - * expand RET */ - q := (*obj.Prog)(nil) - var q1 *obj.Prog for p := c.cursym.Func.Text; p != nil; p = p.Link { switch p.As { case obj.ATEXT: p.Mark |= LEAF - case obj.ARET: - break - - case obj.ANOP: - if p.Link != nil { - q1 = p.Link - q.Link = q1 /* q is non-nop */ - q1.Mark |= p.Mark - } - continue - case ABL, obj.ADUFFZERO, obj.ADUFFCOPY: c.cursym.Func.Text.Mark &^= LEAF - fallthrough - - case ACBNZ, - ACBZ, - ACBNZW, - ACBZW, - ATBZ, - ATBNZ, - AB, - ABEQ, - ABNE, - ABCS, - ABHS, - ABCC, - ABLO, - ABMI, - ABPL, - ABVS, - ABVC, - ABHI, - ABLS, - ABGE, - ABLT, - ABGT, - ABLE, - AADR, /* strange */ - AADRP: - q1 = p.Pcond - - if q1 != nil { - for q1.As == obj.ANOP { - q1 = q1.Link - p.Pcond = q1 - } - } - - break } - - q = p } + var q *obj.Prog + var q1 *obj.Prog var retjmp *obj.LSym for p := c.cursym.Func.Text; p != nil; p = p.Link { o := p.As diff --git a/src/cmd/internal/obj/mips/asm0.go b/src/cmd/internal/obj/mips/asm0.go index faa12bf133..faa827da9f 100644 --- a/src/cmd/internal/obj/mips/asm0.go +++ b/src/cmd/internal/obj/mips/asm0.go @@ -391,6 +391,9 @@ var optab = []Optab{ {obj.APCDATA, C_LCON, C_NONE, C_LCON, 0, 0, 0, 0, 0}, {obj.AFUNCDATA, C_SCON, C_NONE, C_ADDR, 0, 0, 0, 0, 0}, {obj.ANOP, C_NONE, C_NONE, C_NONE, 0, 0, 0, 0, 0}, + {obj.ANOP, C_LCON, C_NONE, C_NONE, 0, 0, 0, 0, 0}, // nop variants, see #40689 + {obj.ANOP, C_REG, C_NONE, C_NONE, 0, 0, 0, 0, 0}, + {obj.ANOP, C_FREG, C_NONE, C_NONE, 0, 0, 0, 0, 0}, {obj.ADUFFZERO, C_NONE, C_NONE, C_LBRA, 11, 4, 0, 0, 0}, // same as AJMP {obj.ADUFFCOPY, C_NONE, C_NONE, C_LBRA, 11, 4, 0, 0, 0}, // same as AJMP diff --git a/src/cmd/internal/obj/mips/obj0.go b/src/cmd/internal/obj/mips/obj0.go index 3106143844..77cad979a6 100644 --- a/src/cmd/internal/obj/mips/obj0.go +++ b/src/cmd/internal/obj/mips/obj0.go @@ -158,19 +158,14 @@ func preprocess(ctxt *obj.Link, cursym *obj.LSym, newprog obj.ProgAlloc) { /* * find leaf subroutines - * strip NOPs * expand RET * expand BECOME pseudo */ - var q *obj.Prog - var q1 *obj.Prog for p := c.cursym.Func.Text; p != nil; p = p.Link { switch p.As { /* too hard, just leave alone */ case obj.ATEXT: - q = p - p.Mark |= LABEL | LEAF | SYNC if p.Link != nil { p.Link.Mark |= LABEL @@ -179,7 +174,6 @@ func preprocess(ctxt *obj.Link, cursym *obj.LSym, newprog obj.ProgAlloc) { /* too hard, just leave alone */ case AMOVW, AMOVV: - q = p if p.To.Type == obj.TYPE_REG && p.To.Reg >= REG_SPECIAL { p.Mark |= LABEL | SYNC break @@ -195,11 +189,9 @@ func preprocess(ctxt *obj.Link, cursym *obj.LSym, newprog obj.ProgAlloc) { ATLBWI, ATLBP, ATLBR: - q = p p.Mark |= LABEL | SYNC case ANOR: - q = p if p.To.Type == obj.TYPE_REG { if p.To.Reg == REGZERO { p.Mark |= LABEL | SYNC @@ -235,8 +227,7 @@ func preprocess(ctxt *obj.Link, cursym *obj.LSym, newprog obj.ProgAlloc) { } else { p.Mark |= BRANCH } - q = p - q1 = p.Pcond + q1 := p.Pcond if q1 != nil { for q1.As == obj.ANOP { q1 = q1.Link @@ -254,24 +245,11 @@ func preprocess(ctxt *obj.Link, cursym *obj.LSym, newprog obj.ProgAlloc) { if q1 != nil { q1.Mark |= LABEL } - continue case ARET: - q = p if p.Link != nil { p.Link.Mark |= LABEL } - continue - - case obj.ANOP: - q1 = p.Link - q.Link = q1 /* q is non-nop */ - q1.Mark |= p.Mark - continue - - default: - q = p - continue } } @@ -284,6 +262,8 @@ func preprocess(ctxt *obj.Link, cursym *obj.LSym, newprog obj.ProgAlloc) { mov = AMOVW } + var q *obj.Prog + var q1 *obj.Prog autosize := int32(0) var p1 *obj.Prog var p2 *obj.Prog -- cgit v1.2.3-54-g00ecf From f50cde9a704542faf6c499e1a69c6bdd6e451f30 Mon Sep 17 00:00:00 2001 From: Cherry Zhang Date: Mon, 17 Aug 2020 19:06:19 -0400 Subject: [release-branch.go1.15] cmd/compile, runtime: mark R12 clobbered for write barrier call on PPC64 When external linking, for large binaries, the external linker may insert a trampoline for the write barrier call, which looks 0000000005a98cc8 <__long_branch_runtime.gcWriteBarrier>: 5a98cc8: 86 01 82 3d addis r12,r2,390 5a98ccc: d8 bd 8c e9 ld r12,-16936(r12) 5a98cd0: a6 03 89 7d mtctr r12 5a98cd4: 20 04 80 4e bctr It clobbers R12 (and CTR, which is never live across a call). As at compile time we don't know whether the binary is big and what link mode will be used, I think we need to mark R12 as clobbered for write barrier call. For extra safety (future-proof) we mark caller-saved register that cannot be used for function arguments, which includes R11, as potentially clobbered as well. Updates #40851. Fixes #40868. Change-Id: Iedd901c5072f1127cc59b0a48cfeb4aaec81b519 Reviewed-on: https://go-review.googlesource.com/c/go/+/248917 Run-TryBot: Cherry Zhang TryBot-Result: Gobot Gobot Reviewed-by: Austin Clements (cherry picked from commit b58d29741650c7bf10b17f455666e2727e1cdd2e) Reviewed-on: https://go-review.googlesource.com/c/go/+/249019 --- src/cmd/compile/internal/ssa/gen/PPC64Ops.go | 4 +-- src/cmd/compile/internal/ssa/opGen.go | 2 +- src/runtime/asm_ppc64x.s | 41 ++++++++++++++-------------- 3 files changed, 24 insertions(+), 23 deletions(-) diff --git a/src/cmd/compile/internal/ssa/gen/PPC64Ops.go b/src/cmd/compile/internal/ssa/gen/PPC64Ops.go index f8bc6cb20b..0261dc283b 100644 --- a/src/cmd/compile/internal/ssa/gen/PPC64Ops.go +++ b/src/cmd/compile/internal/ssa/gen/PPC64Ops.go @@ -645,9 +645,9 @@ func init() { {name: "LoweredAtomicOr8", argLength: 3, reg: gpstore, asm: "OR", faultOnNilArg0: true, hasSideEffects: true}, // LoweredWB invokes runtime.gcWriteBarrier. arg0=destptr, arg1=srcptr, arg2=mem, aux=runtime.gcWriteBarrier - // It preserves R0 through R15, g, and its arguments R20 and R21, + // It preserves R0 through R17 (except special registers R1, R2, R11, R12, R13), g, and its arguments R20 and R21, // but may clobber anything else, including R31 (REGTMP). - {name: "LoweredWB", argLength: 3, reg: regInfo{inputs: []regMask{buildReg("R20"), buildReg("R21")}, clobbers: (callerSave &^ buildReg("R0 R3 R4 R5 R6 R7 R8 R9 R10 R11 R12 R14 R15 R20 R21 g")) | buildReg("R31")}, clobberFlags: true, aux: "Sym", symEffect: "None"}, + {name: "LoweredWB", argLength: 3, reg: regInfo{inputs: []regMask{buildReg("R20"), buildReg("R21")}, clobbers: (callerSave &^ buildReg("R0 R3 R4 R5 R6 R7 R8 R9 R10 R14 R15 R16 R17 R20 R21 g")) | buildReg("R31")}, clobberFlags: true, aux: "Sym", symEffect: "None"}, // There are three of these functions so that they can have three different register inputs. // When we check 0 <= c <= cap (A), then 0 <= b <= c (B), then 0 <= a <= b (C), we want the diff --git a/src/cmd/compile/internal/ssa/opGen.go b/src/cmd/compile/internal/ssa/opGen.go index 9efa1bfcc4..8a27c4bb2e 100644 --- a/src/cmd/compile/internal/ssa/opGen.go +++ b/src/cmd/compile/internal/ssa/opGen.go @@ -26885,7 +26885,7 @@ var opcodeTable = [...]opInfo{ {0, 1048576}, // R20 {1, 2097152}, // R21 }, - clobbers: 576460746931503104, // R16 R17 R18 R19 R22 R23 R24 R25 R26 R27 R28 R29 R31 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 + clobbers: 576460746931312640, // R11 R12 R18 R19 R22 R23 R24 R25 R26 R27 R28 R29 R31 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 }, }, { diff --git a/src/runtime/asm_ppc64x.s b/src/runtime/asm_ppc64x.s index 11d2f2f51a..23387a2165 100644 --- a/src/runtime/asm_ppc64x.s +++ b/src/runtime/asm_ppc64x.s @@ -916,23 +916,23 @@ TEXT ·checkASM(SB),NOSPLIT,$0-1 // - R20 is the destination of the write // - R21 is the value being written at R20. // It clobbers condition codes. -// It does not clobber R0 through R15, +// It does not clobber R0 through R17 (except special registers), // but may clobber any other register, *including* R31. TEXT runtime·gcWriteBarrier(SB),NOSPLIT,$112 // The standard prologue clobbers R31. - // We use R16 and R17 as scratch registers. - MOVD g_m(g), R16 - MOVD m_p(R16), R16 - MOVD (p_wbBuf+wbBuf_next)(R16), R17 + // We use R18 and R19 as scratch registers. + MOVD g_m(g), R18 + MOVD m_p(R18), R18 + MOVD (p_wbBuf+wbBuf_next)(R18), R19 // Increment wbBuf.next position. - ADD $16, R17 - MOVD R17, (p_wbBuf+wbBuf_next)(R16) - MOVD (p_wbBuf+wbBuf_end)(R16), R16 - CMP R16, R17 + ADD $16, R19 + MOVD R19, (p_wbBuf+wbBuf_next)(R18) + MOVD (p_wbBuf+wbBuf_end)(R18), R18 + CMP R18, R19 // Record the write. - MOVD R21, -16(R17) // Record value - MOVD (R20), R16 // TODO: This turns bad writes into bad reads. - MOVD R16, -8(R17) // Record *slot + MOVD R21, -16(R19) // Record value + MOVD (R20), R18 // TODO: This turns bad writes into bad reads. + MOVD R18, -8(R19) // Record *slot // Is the buffer full? (flags set in CMP above) BEQ flush ret: @@ -956,11 +956,12 @@ flush: MOVD R8, (FIXED_FRAME+56)(R1) MOVD R9, (FIXED_FRAME+64)(R1) MOVD R10, (FIXED_FRAME+72)(R1) - MOVD R11, (FIXED_FRAME+80)(R1) - MOVD R12, (FIXED_FRAME+88)(R1) + // R11, R12 may be clobbered by external-linker-inserted trampoline // R13 is REGTLS - MOVD R14, (FIXED_FRAME+96)(R1) - MOVD R15, (FIXED_FRAME+104)(R1) + MOVD R14, (FIXED_FRAME+80)(R1) + MOVD R15, (FIXED_FRAME+88)(R1) + MOVD R16, (FIXED_FRAME+96)(R1) + MOVD R17, (FIXED_FRAME+104)(R1) // This takes arguments R20 and R21. CALL runtime·wbBufFlush(SB) @@ -975,10 +976,10 @@ flush: MOVD (FIXED_FRAME+56)(R1), R8 MOVD (FIXED_FRAME+64)(R1), R9 MOVD (FIXED_FRAME+72)(R1), R10 - MOVD (FIXED_FRAME+80)(R1), R11 - MOVD (FIXED_FRAME+88)(R1), R12 - MOVD (FIXED_FRAME+96)(R1), R14 - MOVD (FIXED_FRAME+104)(R1), R15 + MOVD (FIXED_FRAME+80)(R1), R14 + MOVD (FIXED_FRAME+88)(R1), R15 + MOVD (FIXED_FRAME+96)(R1), R16 + MOVD (FIXED_FRAME+104)(R1), R17 JMP ret // Note: these functions use a special calling convention to save generated code space. -- cgit v1.2.3-54-g00ecf From 9706f510a5e2754595d716bd64be8375997311fb Mon Sep 17 00:00:00 2001 From: Dmitri Shuralyov Date: Wed, 9 Sep 2020 10:01:49 -0400 Subject: [release-branch.go1.15] go1.15.2 Change-Id: I876e199ac276e42cd98bf36d22eaacf26506a975 Reviewed-on: https://go-review.googlesource.com/c/go/+/253717 Reviewed-by: Alexander Rakoczy Run-TryBot: Dmitri Shuralyov TryBot-Result: Gobot Gobot --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 29a7367f57..6792f7e519 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -go1.15.1 \ No newline at end of file +go1.15.2 \ No newline at end of file -- cgit v1.2.3-54-g00ecf