diff options
| author | Austin Clements <austin@google.com> | 2017-01-11 11:32:40 -0500 |
|---|---|---|
| committer | Austin Clements <austin@google.com> | 2017-01-11 11:34:07 -0500 |
| commit | 8b25a00e6d889c8a919922f747791478c8bdfe6f (patch) | |
| tree | 6faa7aece11fb62a1afd3eb0bd7350440fb13244 | |
| parent | 42afbd9e63ed870c67d7c46b4ec38d4bae0e3e63 (diff) | |
| parent | d9a05791566aa8d54113ac88ae77b982872f9be7 (diff) | |
| download | go-8b25a00e6d889c8a919922f747791478c8bdfe6f.tar.gz go-8b25a00e6d889c8a919922f747791478c8bdfe6f.zip | |
Merge branch 'master' into dev.garbage
Change-Id: I36274cf72b8e1908efc8e375cab7880d7b0b3f43
346 files changed, 6192 insertions, 1823 deletions
diff --git a/.gitignore b/.gitignore index 7173067a75..552cf187ae 100644 --- a/.gitignore +++ b/.gitignore @@ -18,28 +18,28 @@ _cgo_* _obj _test _testmain.go -build.out -test.out -doc/articles/wiki/*.bin -misc/cgo/life/run.out -misc/cgo/stdio/run.out -misc/cgo/testso/main -src/cmd/cgo/zdefaultcc.go -src/cmd/go/zdefaultcc.go -src/cmd/go/zosarch.go -src/cmd/internal/obj/zbootstrap.go -src/go/build/zcgo.go -src/go/doc/headscan -src/runtime/internal/sys/zversion.go -src/unicode/maketables -src/*.*/ -test/pass.out -test/run.out -test/times.out -test/garbage/*.out -goinstall.log -last-change -VERSION.cache -bin/ -pkg/ +/VERSION.cache +/bin/ +/build.out +/doc/articles/wiki/*.bin +/goinstall.log +/last-change +/misc/cgo/life/run.out +/misc/cgo/stdio/run.out +/misc/cgo/testso/main +/pkg/ +/src/*.*/ +/src/cmd/cgo/zdefaultcc.go +/src/cmd/go/zdefaultcc.go +/src/cmd/go/zosarch.go +/src/cmd/internal/obj/zbootstrap.go +/src/go/build/zcgo.go +/src/go/doc/headscan +/src/runtime/internal/sys/zversion.go +/src/unicode/maketables +/test.out +/test/garbage/*.out +/test/pass.out +/test/run.out +/test/times.out @@ -29,11 +29,13 @@ Akshat Kumar <seed@mail.nanosouffle.net> Alan Shreve <alan@inconshreveable.com> Albert Nigmatzianov <albertnigma@gmail.com> Albert Strasheim <fullung@gmail.com> +Albert Yu <yukinying@gmail.com> Alberto Bertogli <albertito@blitiri.com.ar> Alberto Donizetti <alb.donizetti@gmail.com> Alberto García Hierro <alberto@garciahierro.com> <alberto.garcia.hierro@gmail.com> Aleksandar Dezelin <dezelin@gmail.com> Alessandro Arzilli <alessandro.arzilli@gmail.com> +Alessandro Baffa <alessandro.baffa@gmail.com> Alex A Skinner <alex@lx.lc> Alex Brainman <alex.brainman@gmail.com> Alex Browne <stephenalexbrowne@gmail.com> @@ -45,6 +47,7 @@ Alex Sergeyev <abc@alexsergeyev.com> Alexander Demakin <alexander.demakin@gmail.com> Alexander Döring <email@alexd.ch> Alexander Larsson <alexander.larsson@gmail.com> +Alexander Menzhinsky <amenzhinsky@gmail.com> Alexander Morozov <lk4d4math@gmail.com> Alexander Neumann <alexander@bumpern.de> Alexander Orlov <alexander.orlov@loxal.net> @@ -53,6 +56,7 @@ Alexander Surma <surma@surmair.de> Alexander Zhavnerchik <alex.vizor@gmail.com> Alexander Zolotov <goldifit@gmail.com> Alexandre Cesaro <alexandre.cesaro@gmail.com> +Alexandre Fiori <fiorix@gmail.com> Alexandre Normand <alexandre.normand@gmail.com> Alexei Sholik <alcosholik@gmail.com> Alexey Borzenkov <snaury@gmail.com> @@ -69,6 +73,7 @@ Andreas Auernhammer <aead@mail.de> Andreas Litt <andreas.litt@gmail.com> Andrei Korzhevskii <a.korzhevskiy@gmail.com> Andrei Vieru <euvieru@gmail.com> +Andrew Austin <andrewaclt@gmail.com> Andrew Balholm <andybalholm@gmail.com> Andrew Bonventre <andybons@chromium.org> Andrew Bursavich <abursavich@gmail.com> @@ -88,6 +93,7 @@ Andrey Petrov <andrey.petrov@shazow.net> Andriy Lytvynov <lytvynov.a.v@gmail.com> Andy Balholm <andy@balholm.com> Andy Davis <andy@bigandian.com> +Andy Finkenstadt <afinkenstadt@zynga.com> Andy Maloney <asmaloney@gmail.com> Anfernee Yongkun Gui <anfernee.gui@gmail.com> Angelo Bulfone <mbulfone@gmail.com> @@ -98,6 +104,8 @@ Anthony Canino <anthony.canino1@gmail.com> Anthony Eufemio <anthony.eufemio@gmail.com> Anthony Martin <ality@pbrane.org> Anthony Starks <ajstarks@gmail.com> +Anthony Woods <awoods@raintank.io> +Antonio Bibiano <antbbn@gmail.com> Apisak Darakananda <pongad@gmail.com> Aram Hăvărneanu <aram@mgk.ro> Areski Belaid <areski@gmail.com> @@ -117,7 +125,9 @@ Aulus Egnatius Varialus <varialus@gmail.com> awaw fumin <awawfumin@gmail.com> Ayanamist Yang <ayanamist@gmail.com> Aymerick Jéhanne <aymerick@jehanne.org> +Baiju Muthukadan <baiju.m.mail@gmail.com> Ben Burkert <ben@benburkert.com> +Ben Lubar <ben.lubar@gmail.com> Ben Olive <sionide21@gmail.com> Benjamin Black <b@b3k.us> Benny Siegert <bsiegert@gmail.com> @@ -163,6 +173,7 @@ Chris Jones <chris@cjones.org> Chris Kastorff <encryptio@gmail.com> Chris Lennert <calennert@gmail.com> Chris McGee <sirnewton_01@yahoo.ca> <newton688@gmail.com> +Chris Stockton <chrisstocktonaz@gmail.com> Christian Couder <chriscool@tuxfamily.org> Christian Himpel <chressie@googlemail.com> Christine Hansmann <chhansmann@gmail.com> @@ -258,6 +269,7 @@ Egon Elbre <egonelbre@gmail.com> Ehren Kret <ehren.kret@gmail.com> Eivind Uggedal <eivind@uggedal.com> Elias Naur <elias.naur@gmail.com> +Elliot Morrison-Reed <elliotmr@gmail.com> Emil Hessman <c.emil.hessman@gmail.com> <emil@hessman.se> Emmanuel Odeke <emm.odeke@gmail.com> <odeke@ualberta.ca> Empirical Interfaces Inc. @@ -273,6 +285,7 @@ Erik St. Martin <alakriti@gmail.com> Erik Westrup <erik.westrup@gmail.com> Ernest Chiang <ernest_chiang@htc.com> Esko Luontola <esko.luontola@gmail.com> +Euan Kemp <euank@euank.com> Evan Phoenix <evan@phx.io> Evan Shaw <chickencha@gmail.com> Ewan Chou <coocood@gmail.com> @@ -328,6 +341,7 @@ Hajime Hoshi <hajimehoshi@gmail.com> Hari haran <hariharan.uno@gmail.com> Hariharan Srinath <srinathh@gmail.com> Harley Laue <losinggeneration@gmail.com> +Harry Moreno <morenoh149@gmail.com> Harshavardhana <hrshvardhana@gmail.com> Håvard Haugen <havard.haugen@gmail.com> Hector Chu <hectorchu@gmail.com> @@ -395,6 +409,7 @@ Jens Frederich <jfrederich@gmail.com> Jeremy Jackins <jeremyjackins@gmail.com> Jeroen Bobbeldijk <jerbob92@gmail.com> Jess Frazelle <me@jessfraz.com> +Jesse Szwedko <jesse.szwedko@gmail.com> Jihyun Yu <yjh0502@gmail.com> Jim McGrath <jimmc2@gmail.com> Jimmy Zelinskie <jimmyzelinskie@gmail.com> @@ -429,6 +444,8 @@ Jonathan Rudenberg <jonathan@titanous.com> Jonathan Wills <runningwild@gmail.com> Jongmin Kim <atomaths@gmail.com> Joonas Kuorilehto <joneskoo@derbian.fi> +Joop Kiefte <ikojba@gmail.com> <joop@kiefte.net> +Jordan Lewis <jordanthelewis@gmail.com> Jose Luis Vázquez González <josvazg@gmail.com> Joseph Holsten <joseph@josephholsten.com> Josh Bleecher Snyder <josharian@gmail.com> @@ -450,6 +467,8 @@ Kamil Kisiel <kamil@kamilkisiel.net> <kamil.kisiel@gmail.com> Kang Hu <hukangustc@gmail.com> Kato Kazuyoshi <kato.kazuyoshi@gmail.com> Katrina Owen <katrina.owen@gmail.com> +Kaviraj Kanagaraj <kavirajkanagaraj@gmail.com> +Keegan Carruthers-Smith <keegan.csmith@gmail.com> Kei Son <hey.calmdown@gmail.com> Keith Ball <inflatablewoman@gmail.com> Keith Rarick <kr@xph.us> @@ -492,12 +511,15 @@ Luigi Riefolo <luigi.riefolo@gmail.com> Luit van Drongelen <luitvd@gmail.com> Luka Zakrajšek <tr00.g33k@gmail.com> Luke Curley <qpingu@gmail.com> +Maksym Trykur <maksym.trykur@gmail.com> Mal Curtis <mal@mal.co.nz> Manfred Touron <m@42.am> Manu S Ajith <neo@codingarena.in> Manuel Mendez <mmendez534@gmail.com> Marc Weistroff <marc@weistroff.net> +Marcel Edmund Franke <marcel.edmund.franke@gmail.com> Marco Hennings <marco.hennings@freiheit.com> +Marin Bašić <marin.basic02@gmail.com> Mark Bucciarelli <mkbucc@gmail.com> Mark Severson <miquella@gmail.com> Mark Theunissen <mark.theunissen@gmail.com> @@ -535,6 +557,8 @@ Matthew Denton <mdenton@skyportsystems.com> Matthew Holt <Matthew.Holt+git@gmail.com> Matthew Horsnell <matthew.horsnell@gmail.com> Matthieu Hauglustaine <matt.hauglustaine@gmail.com> +Matthieu Olivier <olivier.matthieu@gmail.com> +Max Riveiro <kavu13@gmail.com> Maxim Khitrov <max@mxcrypt.com> Maxwell Krohn <themax@gmail.com> MediaMath, Inc @@ -599,6 +623,7 @@ Nicholas Presta <nick@nickpresta.ca> <nick1presta@gmail.com> Nicholas Sullivan <nicholas.sullivan@gmail.com> Nicholas Waples <nwaples@gmail.com> Nick Craig-Wood <nick@craig-wood.com> <nickcw@gmail.com> +Nick Leli <nicholasleli@gmail.com> Nick Patavalis <nick.patavalis@gmail.com> Nick Petroni <npetroni@cs.umd.edu> Nicolas Kaiser <nikai@nikai.net> @@ -606,10 +631,12 @@ Nicolas Owens <mischief@offblast.org> Nicolas S. Dade <nic.dade@gmail.com> Niels Widger <niels.widger@gmail.com> Nigel Kerr <nigel.kerr@gmail.com> +Nik Nyby <nnyby@columbia.edu> Niko Dziemba <niko@dziemba.com> Nikolay Turpitko <nikolay@turpitko.com> Noah Campbell <noahcampbell@gmail.com> Norberto Lopes <nlopes.ml@gmail.com> +Odin Ugedal <odin@ugedal.com> Oleg Vakheta <helginet@gmail.com> Oleku Konko <oleku.konko@gmail.com> Oling Cat <olingcat@gmail.com> @@ -630,6 +657,7 @@ Pascal S. de Kloe <pascal@quies.net> Patrick Crosby <patrick@stathat.com> Patrick Gavlin <pgavlin@gmail.com> Patrick Higgins <patrick.allen.higgins@gmail.com> +Patrick Lee <pattyshack101@gmail.com> Patrick Mézard <patrick@mezard.eu> Patrick Mylund Nielsen <patrick@patrickmn.com> Patrick Smith <pat42smith@gmail.com> @@ -673,9 +701,11 @@ Quentin Perez <qperez@ocs.online.net> Quoc-Viet Nguyen <afelion@gmail.com> RackTop Systems Inc. Radu Berinde <radu@cockroachlabs.com> +Rafal Jeczalik <rjeczalik@gmail.com> Raif S. Naffah <go@naffah-raif.name> Rajat Goel <rajat.goel2010@gmail.com> Ralph Corderoy <ralph@inputplus.co.uk> +Raphael Geronimi <raphael.geronimi@gmail.com> Red Hat, Inc. Reinaldo de Souza Jr <juniorz@gmail.com> Rémy Oudompheng <oudomphe@phare.normalesup.org> @@ -706,10 +736,12 @@ Ron Minnich <rminnich@gmail.com> Ross Light <rlight2@gmail.com> Rowan Worth <sqweek@gmail.com> Russell Haering <russellhaering@gmail.com> +Ryan Bagwell <ryanbagwell@outlook.com> Ryan Hitchman <hitchmanr@gmail.com> Ryan Lower <rpjlower@gmail.com> Ryan Seys <ryan@ryanseys.com> Ryan Slade <ryanslade@gmail.com> +Ryuzo Yamamoto <ryuzo.yamamoto@gmail.com> S.Çağlar Onur <caglar@10ur.org> Salmān Aljammāz <s@0x65.net> Sam Hug <samuel.b.hug@gmail.com> @@ -744,6 +776,7 @@ Simon Whitehead <chemnova@gmail.com> Sina Siadat <siadat@gmail.com> Sokolov Yura <funny.falcon@gmail.com> Song Gao <song@gao.io> +Sourcegraph Inc Spencer Nelson <s@spenczar.com> Spring Mc <heresy.mc@gmail.com> Square, Inc. @@ -767,6 +800,7 @@ Szabolcs Nagy <nsz@port70.net> Tad Glines <tad.glines@gmail.com> Taj Khattra <taj.khattra@gmail.com> Takeshi YAMANASHI <9.nashi@gmail.com> +Takuya Ueda <uedatakuya@gmail.com> Tal Shprecher <tshprecher@gmail.com> Tamir Duberstein <tamird@gmail.com> Tarmigan Casebolt <tarmigan@gmail.com> @@ -780,6 +814,7 @@ Thomas de Zeeuw <thomasdezeeuw@gmail.com> Thomas Desrosiers <thomasdesr@gmail.com> Thomas Kappler <tkappler@gmail.com> Thorben Krueger <thorben.krueger@gmail.com> +Thordur Bjornsson <thorduri@secnorth.net> Tilman Dilo <tilman.dilo@gmail.com> Tim Cooijmans <timcooijmans@gmail.com> Tim Ebringer <tim.ebringer@gmail.com> @@ -798,6 +833,7 @@ Totoro W <tw19881113@gmail.com> Travis Cline <travis.cline@gmail.com> Trey Lawrence <lawrence.trey@gmail.com> Trey Tacon <ttacon@gmail.com> +Tristan Colgate <tcolgate@gmail.com> Tristan Ooohry <ooohry@gmail.com> Tudor Golubenco <tudor.g@gmail.com> Tuo Shan <sturbo89@gmail.com> @@ -845,8 +881,10 @@ Yoshiyuki Kanno <nekotaroh@gmail.com> <yoshiyuki.kanno@stoic.co.jp> Yusuke Kagiwada <block.rxckin.beats@gmail.com> Yuusei Kuwana <kuwana@kumama.org> Yuval Pavel Zholkover <paulzhol@gmail.com> +Zac Bergquist <zbergquist99@gmail.com> Zemanta d.o.o. Zev Goldstein <zev.goldstein@gmail.com> Ziad Hatahet <hatahet@gmail.com> Zorion Arrizabalaga <zorionk@gmail.com> +Фахриддин Балтаев <faxriddinjon@gmail.com> 申习之 <bronze1man@gmail.com> diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4120daf281..9620d81a89 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -7,6 +7,11 @@ It is the work of hundreds of contributors. We appreciate your help! ## Filing issues +General questions should go to the +[golang-nuts mailing list](https://groups.google.com/group/golang-nuts) or +[other forum](https://golang.org/wiki/Questions) instead of the issue tracker. +The gophers there will answer or ask you to file an issue if you've tripped over a bug. + When filing an issue, make sure to answer these five questions: 1. What version of Go are you using (`go version`)? @@ -15,8 +20,7 @@ When filing an issue, make sure to answer these five questions: 4. What did you expect to see? 5. What did you see instead? -General questions should go to the [golang-nuts mailing list](https://groups.google.com/group/golang-nuts) instead of the issue tracker. -The gophers there will answer or ask you to file an issue if you've tripped over a bug. +For change proposals, see [Proposing Changes To Go](https://github.com/golang/proposal/). Sensitive security-related issues should be reported to [security@golang.org](mailto:security@golang.org). @@ -28,6 +32,7 @@ before sending patches. **We do not accept GitHub pull requests** (we use [an instance](https://go-review.googlesource.com/) of the [Gerrit](https://www.gerritcodereview.com/) code review system instead). +Also, please do not post patches on the issue tracker. Unless otherwise noted, the Go source files are distributed under the BSD-style license found in the LICENSE file. diff --git a/CONTRIBUTORS b/CONTRIBUTORS index 43d1d9a0d4..d410b36d6d 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -55,11 +55,13 @@ Alan Donovan <adonovan@google.com> Alan Shreve <alan@inconshreveable.com> Albert Nigmatzianov <albertnigma@gmail.com> Albert Strasheim <fullung@gmail.com> +Albert Yu <yukinying@gmail.com> Alberto Bertogli <albertito@blitiri.com.ar> Alberto Donizetti <alb.donizetti@gmail.com> Alberto García Hierro <alberto@garciahierro.com> <alberto.garcia.hierro@gmail.com> Aleksandar Dezelin <dezelin@gmail.com> Alessandro Arzilli <alessandro.arzilli@gmail.com> +Alessandro Baffa <alessandro.baffa@gmail.com> Alex A Skinner <alex@lx.lc> Alex Brainman <alex.brainman@gmail.com> Alex Bramley <abramley@google.com> @@ -73,6 +75,7 @@ Alex Vaghin <crhyme@google.com> Alexander Demakin <alexander.demakin@gmail.com> Alexander Döring <email@alexd.ch> Alexander Larsson <alexander.larsson@gmail.com> +Alexander Menzhinsky <amenzhinsky@gmail.com> Alexander Morozov <lk4d4math@gmail.com> Alexander Neumann <alexander@bumpern.de> Alexander Orlov <alexander.orlov@loxal.net> @@ -81,6 +84,7 @@ Alexander Surma <surma@surmair.de> Alexander Zhavnerchik <alex.vizor@gmail.com> Alexander Zolotov <goldifit@gmail.com> Alexandre Cesaro <alexandre.cesaro@gmail.com> +Alexandre Fiori <fiorix@gmail.com> Alexandre Normand <alexandre.normand@gmail.com> Alexandru Moșoi <brtzsnr@gmail.com> Alexei Sholik <alcosholik@gmail.com> @@ -101,6 +105,7 @@ Andreas Litt <andreas.litt@gmail.com> Andrei Korzhevskii <a.korzhevskiy@gmail.com> Andrei Vieru <euvieru@gmail.com> Andres Erbsen <andreser@google.com> +Andrew Austin <andrewaclt@gmail.com> Andrew Balholm <andybalholm@gmail.com> Andrew Bonventre <andybons@chromium.org> Andrew Bursavich <abursavich@gmail.com> @@ -123,6 +128,7 @@ Andrey Petrov <andrey.petrov@shazow.net> Andriy Lytvynov <lytvynov.a.v@gmail.com> Andy Balholm <andy@balholm.com> Andy Davis <andy@bigandian.com> +Andy Finkenstadt <afinkenstadt@zynga.com> Andy Maloney <asmaloney@gmail.com> Anfernee Yongkun Gui <anfernee.gui@gmail.com> Angelo Bulfone <mbulfone@gmail.com> @@ -133,6 +139,8 @@ Anthony Canino <anthony.canino1@gmail.com> Anthony Eufemio <anthony.eufemio@gmail.com> Anthony Martin <ality@pbrane.org> Anthony Starks <ajstarks@gmail.com> +Anthony Woods <awoods@raintank.io> +Antonio Bibiano <antbbn@gmail.com> Antonio Murdaca <runcom@redhat.com> Apisak Darakananda <pongad@gmail.com> Aram Hăvărneanu <aram@mgk.ro> @@ -155,10 +163,12 @@ Austin Clements <austin@google.com> <aclements@csail.mit.edu> awaw fumin <awawfumin@gmail.com> Ayanamist Yang <ayanamist@gmail.com> Aymerick Jéhanne <aymerick@jehanne.org> +Baiju Muthukadan <baiju.m.mail@gmail.com> Balazs Lecz <leczb@google.com> Ben Burkert <ben@benburkert.com> Ben Eitzen <eitzenb@golang.org> Ben Fried <ben.fried@gmail.com> +Ben Lubar <ben.lubar@gmail.com> Ben Lynn <benlynn@gmail.com> Ben Olive <sionide21@gmail.com> Benjamin Black <b@b3k.us> @@ -233,6 +243,7 @@ Chris Kastorff <encryptio@gmail.com> Chris Lennert <calennert@gmail.com> Chris Manghane <cmang@golang.org> Chris McGee <sirnewton_01@yahoo.ca> <newton688@gmail.com> +Chris Stockton <chrisstocktonaz@gmail.com> Chris Zou <chriszou@ca.ibm.com> Christian Couder <chriscool@tuxfamily.org> Christian Himpel <chressie@googlemail.com> <chressie@gmail.com> @@ -305,6 +316,7 @@ David Glasser <glasser@meteor.com> David Howden <dhowden@gmail.com> David Hubbard <dsp@google.com> David Jakob Fritz <david.jakob.fritz@gmail.com> +David Lazar <lazard@golang.org> David Leon Gil <coruus@gmail.com> David McLeish <davemc@google.com> David Presotto <presotto@gmail.com> @@ -360,6 +372,7 @@ Egon Elbre <egonelbre@gmail.com> Ehren Kret <ehren.kret@gmail.com> Eivind Uggedal <eivind@uggedal.com> Elias Naur <elias.naur@gmail.com> +Elliot Morrison-Reed <elliotmr@gmail.com> Emil Hessman <c.emil.hessman@gmail.com> <emil@hessman.se> Emmanuel Odeke <emm.odeke@gmail.com> <odeke@ualberta.ca> Eoghan Sherry <ejsherry@gmail.com> @@ -379,6 +392,7 @@ Ernest Chiang <ernest_chiang@htc.com> Esko Luontola <esko.luontola@gmail.com> Ethan Burns <eaburns@google.com> Ethan Miller <eamiller@us.ibm.com> +Euan Kemp <euank@euank.com> Evan Broder <evan@stripe.com> Evan Brown <evanbrown@google.com> Evan Kroske <evankroske@google.com> @@ -449,6 +463,7 @@ Han-Wen Nienhuys <hanwen@google.com> Hari haran <hariharan.uno@gmail.com> Hariharan Srinath <srinathh@gmail.com> Harley Laue <losinggeneration@gmail.com> +Harry Moreno <morenoh149@gmail.com> Harshavardhana <hrshvardhana@gmail.com> Håvard Haugen <havard.haugen@gmail.com> Hector Chu <hectorchu@gmail.com> @@ -470,6 +485,7 @@ Ian Gudger <ian@loosescre.ws> Ian Lance Taylor <iant@golang.org> Icarus Sparry <golang@icarus.freeuk.com> Idora Shinatose <idora.shinatose@gmail.com> +Igor Bernstein <igorbernstein@google.com> Igor Dolzhikov <bluesriverz@gmail.com> Ilya Tocar <ilya.tocar@intel.com> INADA Naoki <songofacandy@gmail.com> @@ -518,6 +534,7 @@ Jan Ziak <0xe2.0x9a.0x9b@gmail.com> Jani Monoses <jani.monoses@ubuntu.com> <jani.monoses@gmail.com> Jaroslavas Počepko <jp@webmaster.ms> Jason Barnett <jason.w.barnett@gmail.com> +Jason Buberel <jbuberel@google.com> Jason Del Ponte <delpontej@gmail.com> Jason Hall <jasonhall@google.com> Jason Smale <jsmale@zendesk.com> @@ -537,6 +554,7 @@ Jeremy Jackins <jeremyjackins@gmail.com> Jeremy Schlatter <jeremy.schlatter@gmail.com> Jeroen Bobbeldijk <jerbob92@gmail.com> Jess Frazelle <me@jessfraz.com> +Jesse Szwedko <jesse.szwedko@gmail.com> Jihyun Yu <yjh0502@gmail.com> Jim Cote <jfcote87@gmail.com> Jim Kingdon <jim@bolt.me> @@ -586,6 +604,8 @@ Jonathan Rudenberg <jonathan@titanous.com> Jonathan Wills <runningwild@gmail.com> Jongmin Kim <atomaths@gmail.com> Joonas Kuorilehto <joneskoo@derbian.fi> +Joop Kiefte <ikojba@gmail.com> <joop@kiefte.net> +Jordan Lewis <jordanthelewis@gmail.com> Jos Visser <josv@google.com> Jose Luis Vázquez González <josvazg@gmail.com> Joseph Bonneau <jcb@google.com> @@ -617,8 +637,10 @@ Kang Hu <hukangustc@gmail.com> Karan Dhiman <karandhi@ca.ibm.com> Kato Kazuyoshi <kato.kazuyoshi@gmail.com> Katrina Owen <katrina.owen@gmail.com> +Kaviraj Kanagaraj <kavirajkanagaraj@gmail.com> Kay Zhu <kayzhu@google.com> KB Sriram <kbsriram@google.com> +Keegan Carruthers-Smith <keegan.csmith@gmail.com> Kei Son <hey.calmdown@gmail.com> Keith Ball <inflatablewoman@gmail.com> Keith Randall <khr@golang.org> @@ -670,6 +692,7 @@ Luke Curley <qpingu@gmail.com> Luna Duclos <luna.duclos@palmstonegames.com> Luuk van Dijk <lvd@golang.org> <lvd@google.com> Lynn Boger <laboger@linux.vnet.ibm.com> +Maksym Trykur <maksym.trykur@gmail.com> Mal Curtis <mal@mal.co.nz> Manfred Touron <m@42.am> Manoj Dayaram <platform-dev@moovweb.com> <manoj.dayaram@moovweb.com> @@ -678,9 +701,11 @@ Manu S Ajith <neo@codingarena.in> Manuel Mendez <mmendez534@gmail.com> Marc Weistroff <marc@weistroff.net> Marc-Antoine Ruel <maruel@chromium.org> +Marcel Edmund Franke <marcel.edmund.franke@gmail.com> Marcel van Lohuizen <mpvl@golang.org> Marco Hennings <marco.hennings@freiheit.com> Marga Manterola <marga@google.com> +Marin Bašić <marin.basic02@gmail.com> Marius Nuennerich <mnu@google.com> Mark Bucciarelli <mkbucc@gmail.com> Mark Severson <miquella@gmail.com> @@ -695,6 +720,7 @@ Markus Zimmermann <zimmski@gmail.com> Martin Bertschler <mbertschler@gmail.com> Martin Garton <garton@gmail.com> Martin Hamrle <martin.hamrle@gmail.com> +Martin Kreichgauer <martinkr@google.com> Martin Möhrmann <moehrmann@google.com> <martisch@uos.de> Martin Neubauer <m.ne@gmx.net> Martin Olsson <martin@minimum.se> @@ -723,6 +749,8 @@ Matthew Denton <mdenton@skyportsystems.com> Matthew Holt <Matthew.Holt+git@gmail.com> Matthew Horsnell <matthew.horsnell@gmail.com> Matthieu Hauglustaine <matt.hauglustaine@gmail.com> +Matthieu Olivier <olivier.matthieu@gmail.com> +Max Riveiro <kavu13@gmail.com> Maxim Khitrov <max@mxcrypt.com> Maxim Pimenov <mpimenov@google.com> Maxim Ushakov <ushakov@google.com> @@ -806,6 +834,7 @@ Nicholas Waples <nwaples@gmail.com> Nick Cooper <nmvc@google.com> Nick Craig-Wood <nick@craig-wood.com> <nickcw@gmail.com> Nick Harper <nharper@google.com> +Nick Leli <nicholasleli@gmail.com> Nick Patavalis <nick.patavalis@gmail.com> Nick Petroni <npetroni@cs.umd.edu> Nicolas Kaiser <nikai@nikai.net> @@ -814,11 +843,13 @@ Nicolas S. Dade <nic.dade@gmail.com> Niels Widger <niels.widger@gmail.com> Nigel Kerr <nigel.kerr@gmail.com> Nigel Tao <nigeltao@golang.org> +Nik Nyby <nnyby@columbia.edu> Niko Dziemba <niko@dziemba.com> Nikolay Turpitko <nikolay@turpitko.com> Noah Campbell <noahcampbell@gmail.com> Nodir Turakulov <nodir@google.com> Norberto Lopes <nlopes.ml@gmail.com> +Odin Ugedal <odin@ugedal.com> Oleg Vakheta <helginet@gmail.com> Oleku Konko <oleku.konko@gmail.com> Oling Cat <olingcat@gmail.com> @@ -837,6 +868,7 @@ Pascal S. de Kloe <pascal@quies.net> Patrick Crosby <patrick@stathat.com> Patrick Gavlin <pgavlin@gmail.com> Patrick Higgins <patrick.allen.higgins@gmail.com> +Patrick Lee <pattyshack101@gmail.com> Patrick Mézard <patrick@mezard.eu> Patrick Mylund Nielsen <patrick@patrickmn.com> Patrick Riley <pfr@google.com> @@ -894,15 +926,19 @@ Quan Tran <qeed.quan@gmail.com> Quan Yong Zhai <qyzhai@gmail.com> Quentin Perez <qperez@ocs.online.net> Quentin Smith <quentin@golang.org> +Quinn Slack <sqs@sourcegraph.com> Quoc-Viet Nguyen <afelion@gmail.com> Radu Berinde <radu@cockroachlabs.com> +Rafal Jeczalik <rjeczalik@gmail.com> Rahul Chaudhry <rahulchaudhry@chromium.org> Raif S. Naffah <go@naffah-raif.name> Rajat Goel <rajat.goel2010@gmail.com> Ralph Corderoy <ralph@inputplus.co.uk> Ramesh Dharan <dharan@google.com> Raph Levien <raph@google.com> +Raphael Geronimi <raphael.geronimi@gmail.com> Raul Silvera <rsilvera@google.com> +Rebecca Stambler <rstambler@golang.org> Reinaldo de Souza Jr <juniorz@gmail.com> Rémy Oudompheng <oudomphe@phare.normalesup.org> <remyoudompheng@gmail.com> Rhys Hiltner <rhys@justin.tv> @@ -943,12 +979,14 @@ Rowan Worth <sqweek@gmail.com> Rui Ueyama <ruiu@google.com> Russ Cox <rsc@golang.org> Russell Haering <russellhaering@gmail.com> +Ryan Bagwell <ryanbagwell@outlook.com> Ryan Barrett <ryanb@google.com> Ryan Brown <ribrdb@google.com> Ryan Hitchman <hitchmanr@gmail.com> Ryan Lower <rpjlower@gmail.com> Ryan Seys <ryan@ryanseys.com> Ryan Slade <ryanslade@gmail.com> +Ryuzo Yamamoto <ryuzo.yamamoto@gmail.com> S.Çağlar Onur <caglar@10ur.org> Sai Cheemalapati <saicheems@google.com> Salmān Aljammāz <s@0x65.net> @@ -1012,6 +1050,7 @@ Stéphane Travostino <stephane.travostino@gmail.com> Stephen Ma <stephenm@golang.org> Stephen McQuay <stephen@mcquay.me> Stephen Weinberg <stephen@q5comm.com> +Steve Francia <spf@golang.org> Steve McCoy <mccoyst@gmail.com> Steve Newman <snewman@google.com> Steve Phillips <elimisteve@gmail.com> @@ -1029,6 +1068,7 @@ Tad Glines <tad.glines@gmail.com> Taj Khattra <taj.khattra@gmail.com> Takashi Matsuo <tmatsuo@google.com> Takeshi YAMANASHI <9.nashi@gmail.com> +Takuya Ueda <uedatakuya@gmail.com> Tal Shprecher <tshprecher@gmail.com> Tamir Duberstein <tamird@gmail.com> Tarmigan Casebolt <tarmigan@gmail.com> @@ -1044,6 +1084,7 @@ Thomas Desrosiers <thomasdesr@gmail.com> Thomas Habets <habets@google.com> Thomas Kappler <tkappler@gmail.com> Thorben Krueger <thorben.krueger@gmail.com> +Thordur Bjornsson <thorduri@secnorth.net> Tilman Dilo <tilman.dilo@gmail.com> Tim Cooijmans <timcooijmans@gmail.com> Tim Ebringer <tim.ebringer@gmail.com> @@ -1072,6 +1113,7 @@ Trevor Strohman <trevor.strohman@gmail.com> Trey Lawrence <lawrence.trey@gmail.com> Trey Tacon <ttacon@gmail.com> Tristan Amini <tamini01@ca.ibm.com> +Tristan Colgate <tcolgate@gmail.com> Tristan Ooohry <ooohry@gmail.com> Tudor Golubenco <tudor.g@gmail.com> Tuo Shan <sturbo89@gmail.com> <shantuo@google.com> @@ -1132,8 +1174,10 @@ Yusuke Kagiwada <block.rxckin.beats@gmail.com> Yuusei Kuwana <kuwana@kumama.org> Yuval Pavel Zholkover <paulzhol@gmail.com> Yves Junqueira <yvesj@google.com> <yves.junqueira@gmail.com> +Zac Bergquist <zbergquist99@gmail.com> Zev Goldstein <zev.goldstein@gmail.com> Zhongwei Yao <zhongwei.yao@arm.com> Ziad Hatahet <hatahet@gmail.com> Zorion Arrizabalaga <zorionk@gmail.com> +Фахриддин Балтаев <faxriddinjon@gmail.com> 申习之 <bronze1man@gmail.com> diff --git a/api/except.txt b/api/except.txt index 2062cbf0da..857ebb5d7d 100644 --- a/api/except.txt +++ b/api/except.txt @@ -338,3 +338,7 @@ pkg unicode, const Version = "6.2.0" pkg unicode, const Version = "6.3.0" pkg unicode, const Version = "7.0.0" pkg unicode, const Version = "8.0.0" +pkg syscall (openbsd-386), const SYS_KILL = 37 +pkg syscall (openbsd-386-cgo), const SYS_KILL = 37 +pkg syscall (openbsd-amd64), const SYS_KILL = 37 +pkg syscall (openbsd-amd64-cgo), const SYS_KILL = 37 diff --git a/api/go1.8.txt b/api/go1.8.txt index e9ddc28079..fca7e03c9f 100644 --- a/api/go1.8.txt +++ b/api/go1.8.txt @@ -73,10 +73,8 @@ pkg database/sql, const LevelSnapshot = 5 pkg database/sql, const LevelSnapshot IsolationLevel pkg database/sql, const LevelWriteCommitted = 3 pkg database/sql, const LevelWriteCommitted IsolationLevel -pkg database/sql/driver, func IsolationFromContext(context.Context) (IsolationLevel, bool) -pkg database/sql/driver, func ReadOnlyFromContext(context.Context) bool -pkg database/sql/driver, type ConnBeginContext interface { BeginContext } -pkg database/sql/driver, type ConnBeginContext interface, BeginContext(context.Context) (Tx, error) +pkg database/sql/driver, type ConnBeginTx interface { BeginTx } +pkg database/sql/driver, type ConnBeginTx interface, BeginTx(context.Context, TxOptions) (Tx, error) pkg database/sql/driver, type ConnPrepareContext interface { PrepareContext } pkg database/sql/driver, type ConnPrepareContext interface, PrepareContext(context.Context, string) (Stmt, error) pkg database/sql/driver, type ExecerContext interface { ExecContext } @@ -125,16 +123,17 @@ pkg database/sql/driver, type StmtExecContext interface { ExecContext } pkg database/sql/driver, type StmtExecContext interface, ExecContext(context.Context, []NamedValue) (Result, error) pkg database/sql/driver, type StmtQueryContext interface { QueryContext } pkg database/sql/driver, type StmtQueryContext interface, QueryContext(context.Context, []NamedValue) (Rows, error) -pkg database/sql, func IsolationContext(context.Context, IsolationLevel) context.Context +pkg database/sql/driver, type TxOptions struct +pkg database/sql/driver, type TxOptions struct, Isolation IsolationLevel +pkg database/sql/driver, type TxOptions struct, ReadOnly bool pkg database/sql, func Named(string, interface{}) NamedArg -pkg database/sql, func ReadOnlyContext(context.Context) context.Context pkg database/sql, method (*ColumnType) DatabaseTypeName() string pkg database/sql, method (*ColumnType) DecimalSize() (int64, int64, bool) pkg database/sql, method (*ColumnType) Length() (int64, bool) pkg database/sql, method (*ColumnType) Name() string pkg database/sql, method (*ColumnType) Nullable() (bool, bool) pkg database/sql, method (*ColumnType) ScanType() reflect.Type -pkg database/sql, method (*DB) BeginContext(context.Context) (*Tx, error) +pkg database/sql, method (*DB) BeginTx(context.Context, *TxOptions) (*Tx, error) pkg database/sql, method (*DB) ExecContext(context.Context, string, ...interface{}) (Result, error) pkg database/sql, method (*DB) PingContext(context.Context) error pkg database/sql, method (*DB) PrepareContext(context.Context, string) (*Stmt, error) @@ -155,7 +154,9 @@ pkg database/sql, type IsolationLevel int pkg database/sql, type NamedArg struct pkg database/sql, type NamedArg struct, Name string pkg database/sql, type NamedArg struct, Value interface{} -pkg debug/gosym, func PCValue([]uint8, uint64, int) int +pkg database/sql, type TxOptions struct +pkg database/sql, type TxOptions struct, Isolation IsolationLevel +pkg database/sql, type TxOptions struct, ReadOnly bool pkg debug/pe, method (*COFFSymbol) FullName(StringTable) (string, error) pkg debug/pe, method (StringTable) String(uint32) (string, error) pkg debug/pe, type File struct, COFFSymbols []COFFSymbol @@ -175,7 +176,6 @@ pkg expvar, method (*Float) Value() float64 pkg expvar, method (Func) Value() interface{} pkg expvar, method (*Int) Value() int64 pkg expvar, method (*String) Value() string -pkg go/build, type NoGoError struct, Ignored bool pkg go/doc, func IsPredeclared(string) bool pkg go/types, func Default(Type) Type pkg go/types, func IdenticalIgnoreTags(Type, Type) bool @@ -239,21 +239,23 @@ pkg plugin, type Symbol interface {} pkg reflect, func Swapper(interface{}) func(int, int) pkg runtime, func MutexProfile([]BlockProfileRecord) (int, bool) pkg runtime, func SetMutexProfileFraction(int) int +pkg runtime, type MemStats struct, NumForcedGC uint32 pkg sort, func Slice(interface{}, func(int, int) bool) pkg sort, func SliceIsSorted(interface{}, func(int, int) bool) bool pkg sort, func SliceStable(interface{}, func(int, int) bool) pkg syscall (linux-arm-cgo), func TimevalToNsec(Timeval) int64 pkg syscall (linux-arm), func TimevalToNsec(Timeval) int64 +pkg syscall (openbsd-386), const SYS_KILL = 122 +pkg syscall (openbsd-386-cgo), const SYS_KILL = 122 +pkg syscall (openbsd-amd64), const SYS_KILL = 122 +pkg syscall (openbsd-amd64-cgo), const SYS_KILL = 122 pkg syscall (windows-386), const ERROR_DIR_NOT_EMPTY = 145 pkg syscall (windows-386), const ERROR_DIR_NOT_EMPTY Errno pkg syscall (windows-amd64), const ERROR_DIR_NOT_EMPTY = 145 pkg syscall (windows-amd64), const ERROR_DIR_NOT_EMPTY Errno pkg testing, func CoverMode() string pkg testing, func MainStart(testDeps, []InternalTest, []InternalBenchmark, []InternalExample) *M -pkg testing, method (*B) Context() context.Context pkg testing, method (*B) Name() string -pkg testing, method (*T) Context() context.Context pkg testing, method (*T) Name() string -pkg testing, type TB interface, Context() context.Context pkg testing, type TB interface, Name() string pkg time, func Until(Time) Duration diff --git a/doc/asm.html b/doc/asm.html index 3e03c548fd..79dc7df322 100644 --- a/doc/asm.html +++ b/doc/asm.html @@ -838,6 +838,44 @@ It is a scaled mode as on the x86, but the only scale allowed is <code>1</code>. </ul> +<h3 id="mips">MIPS, MIPS64</h3> + +<p> +General purpose registers are named <code>R0</code> through <code>R31</code>, +floating point registers are <code>F0</code> through <code>F31</code>. +</p> + +<p> +<code>R30</code> is reserved to point to <code>g</code>. +<code>R23</code> is used as a temporary register. +</p> + +<p> +In a <code>TEXT</code> directive, the frame size <code>$-4</code> for MIPS or +<code>$-8</code> for MIPS64 instructs the linker not to save <code>LR</code>. +</p> + +<p> +<code>SP</code> refers to the virtual stack pointer. +For the hardware register, use <code>R29</code>. +</p> + +<p> +Addressing modes: +</p> + +<ul> + +<li> +<code>16(R1)</code>: The location at <code>R1</code> plus 16. +</li> + +<li> +<code>(R1)</code>: Alias for <code>0(R1)</code>. +</li> + +</ul> + <h3 id="unsupported_opcodes">Unsupported opcodes</h3> <p> diff --git a/doc/code.html b/doc/code.html index 9978b523b4..796431aa14 100644 --- a/doc/code.html +++ b/doc/code.html @@ -160,9 +160,13 @@ $ <b>export GOPATH=$(go env GOPATH)</b> </pre> <p> -To learn more about setting up the <code>GOPATH</code> environment variable, -please see -<a href="/cmd/go/#hdr-GOPATH_environment_variable"><code>'go help gopath'</code></a> +To learn more about the <code>GOPATH</code> environment variable, see +<a href="/cmd/go/#hdr-GOPATH_environment_variable"><code>'go help gopath'</code></a>. +</p> + +<p> +To use a custom workspace location, +<a href="https://golang.org/wiki/SettingGOPATH">set the <code>GOPATH</code> environment variable</a>. </p> <h3 id="ImportPaths">Import paths</h3> diff --git a/doc/conduct.html b/doc/conduct.html index c749266248..5b81681c10 100644 --- a/doc/conduct.html +++ b/doc/conduct.html @@ -67,7 +67,6 @@ official forums operated by the Go project (“Go spaces”): <li>The <a href="https://groups.google.com/group/golang-nuts">golang-nuts</a> and <a href="https://groups.google.com/group/golang-dev">golang-dev</a> mailing lists. <li>The #go-nuts IRC channel on Freenode. -<li>The <a href="https://reddit.com/r/golang">/r/golang subreddit</a>. </ul> <p> diff --git a/doc/devel/release.html b/doc/devel/release.html index 773f889e8b..51957dff28 100644 --- a/doc/devel/release.html +++ b/doc/devel/release.html @@ -50,11 +50,23 @@ See the <a href="https://github.com/golang/go/issues?q=milestone%3AGo1.7.1">Go </p> <p> -go1.7.2 (released 2016/10/17) includes fixes to the compiler, runtime, +go1.7.2 should not be used. It was tagged but not fully released. +The release was deferred due to a last minute bug report. +Use go1.7.3 instead, and refer to the summary of changes below. +</p> + +<p> +go1.7.3 (released 2016/10/19) includes fixes to the compiler, runtime, and the <code>crypto/cipher</code>, <code>crypto/tls</code>, <code>net/http</code>, and <code>strings</code> packages. -See the <a href="https://github.com/golang/go/issues?q=milestone%3AGo1.7.2">Go -1.7.2 milestone</a> on our issue tracker for details. +See the <a href="https://github.com/golang/go/issues?q=milestone%3AGo1.7.3">Go +1.7.3 milestone</a> on our issue tracker for details. +</p> + +<p> +go1.7.4 (released 2016/12/01) includes two security fixes. +See the <a href="https://github.com/golang/go/issues?q=milestone%3AGo1.7.4">Go +1.7.4 milestone</a> on our issue tracker for details. </p> <h2 id="go1.6">go1.6 (released 2016/02/17)</h2> @@ -88,6 +100,13 @@ See the <a href="https://github.com/golang/go/issues?q=milestone%3AGo1.6.3">Go 1.6.3 milestone</a> on our issue tracker for details. </p> +<p> +go1.6.4 (released 2016/12/01) includes two security fixes. +It contains the same fixes as Go 1.7.4 and was released at the same time. +See the <a href="https://github.com/golang/go/issues?q=milestone%3AGo1.7.4">Go +1.7.4 milestone</a> on our issue tracker for details. +</p> + <h2 id="go1.5">go1.5 (released 2015/08/19)</h2> <p> diff --git a/doc/devel/weekly.html b/doc/devel/weekly.html index 7166a76507..e17461db24 100644 --- a/doc/devel/weekly.html +++ b/doc/devel/weekly.html @@ -519,7 +519,7 @@ Other changes: fix FreeBSD signal handling around thread creation (thanks Devon H. O'Dell), goroutine profile, stack dumps, implement runtime.osyield on FreeBSD 386, amd64 (thanks Devon H. O'Dell), - permit default behaviour of SIGTSTP, SIGTTIN, SIGTTOU, + permit default behavior of SIGTSTP, SIGTTIN, SIGTTOU, release unused memory to the OS (thanks Sébastien Paolacci), remove an obsolete file (thanks Mikio Hara). * spec: make all comparison results untyped bool, @@ -4157,7 +4157,7 @@ Other changes in this release: * suffixarray: use binary search for both ends of Lookup (thanks Eric Eisner). * syscall: add missing network interface constants (thanks Mikio Hara). * template: treat map keys as zero, not non-existent (thanks Roger Peppe). -* time: allow cancelling of After events (thanks Roger Peppe), +* time: allow canceling of After events (thanks Roger Peppe), support Solaris zoneinfo directory. * token/position: added SetLinesForContent. * unicode: update to unicode 6.0.0. @@ -5696,7 +5696,7 @@ This release contains many changes: * cmath: new complex math library (thanks Charles L. Dorian). * docs: update to match current coding style (thanks Christopher Wedgwood). * exp/eval: fix example and add target to Makefile (thanks Evan Shaw). -* fmt: change behaviour of format verb %b to match %x when negative (thanks Andrei Vieru). +* fmt: change behavior of format verb %b to match %x when negative (thanks Andrei Vieru). * gc: compile s == "" as len(s) == 0, distinguish fatal compiler bug from error+exit, fix alignment on non-amd64, diff --git a/doc/go1.8.html b/doc/go1.8.html index 22176a2a92..2ac478632e 100644 --- a/doc/go1.8.html +++ b/doc/go1.8.html @@ -25,7 +25,7 @@ release notes. Go 1.8 is expected to be released in February 2017. <p> The latest Go release, version 1.8, arrives six months after <a href="go1.7">Go 1.7</a>. Most of its changes are in the implementation of the toolchain, runtime, and libraries. -There is one minor change to the language specification. +There are <a href="#language">two minor changes</a> to the language specification. As always, the release maintains the Go 1 <a href="/doc/go1compat.html">promise of compatibility</a>. We expect almost all Go programs to continue to compile and run as before. </p> @@ -44,8 +44,9 @@ and <a href="#sort_slice">simplifies sorting slices</a>. <h2 id="language">Changes to the language</h2> <p> - When explicitly converting a value from one struct type to another, as of Go 1. 8 the tags are ignored. - Thus two structs that differ only in their tags may be converted from one to the other: + When explicitly converting a value from one struct type to another, + as of Go 1.8 the tags are ignored. Thus two structs that differ + only in their tags may be converted from one to the other: </p> <pre> @@ -76,7 +77,9 @@ func example() { <p> Go now supports 32-bit MIPS on Linux for both big-endian (<code>linux/mips</code>) and little-endian machines -(<code>linux/mipsle</code>). +(<code>linux/mipsle</code>) that implement the MIPS32r1 instruction set with FPU +or kernel FPU emulation. Note that many common MIPS-based routers lack an FPU and +have firmware that doesn't enable kernel FPU emulation; Go won't run on such machines. </p> <p> @@ -84,6 +87,10 @@ On DragonFly BSD, Go now requires DragonFly 4.4.4 or later. <!-- CL 29491, CL 29 </p> <p> +On OpenBSD, Go now requires OpenBSD 5.9 or later. <!-- CL 34093 --> +</p> + +<p> The Plan 9 port's networking support is now much more complete and matches the behavior of Unix and Windows with respect to deadlines and cancelation. @@ -95,6 +102,18 @@ and cancelation. binaries on older OS X versions is untested. </p> +<p> + Go 1.8 will be the last release to support Linux on ARMv5E and ARMv6 processors: + Go 1.9 will likely require the ARMv6K (as found in the Raspberry Pi 1) or later. + To identify whether a Linux system is ARMv6K or later, run + “<code>go</code> <code>tool</code> <code>dist</code> <code>-check-armv6k</code>” + (to facilitate testing, it is also possible to just copy the <code>dist</code> command to the + system without installing a full copy of Go 1.8) + and if the program terminates with output "ARMv6K supported." then the system + implements ARMv6K or later. + Go on non-Linux ARM systems already requires ARMv6K or later. +</p> + <h3 id="known_issues">Known Issues</h3> @@ -102,8 +121,7 @@ and cancelation. There are some instabilities on FreeBSD and NetBSD that are known but not understood. These can lead to program crashes in rare cases. See -<a href="https://golang.org/issue/15658">issue 15658</a>, -<a href="https://golang.org/issue/16396">issue 16396</a>, and +<a href="https://golang.org/issue/15658">issue 15658</a> and <a href="https://golang.org/issue/16511">issue 16511</a>. Any help in solving these issues would be appreciated. </p> @@ -120,8 +138,9 @@ For 64-bit x86 systems, the following instructions have been added: <code>MOVSHDUP</code>, <code>MOVSLDUP</code>, <code>VMOVDDUP</code>, -<code>VMOVSHDUP</code>, -and <code>VMOVSLDUP</code>.</p> +<code>VMOVSHDUP</code>, and +<code>VMOVSLDUP</code>. +</p> <p> For 64-bit PPC systems, the common vector scalar instructions have been @@ -203,7 +222,7 @@ added: <code>XXSEL</code>, <code>XXSI</code>, <code>XXSLDWI</code>, -<code>XXSPLT</code>, and +<code>XXSPLT</code>, and <code>XXSPLTW</code>. </p> @@ -211,8 +230,8 @@ added: <p> <!-- CL 27324, CL 27325 --> The <code>yacc</code> tool (previously available by running -“<code>go</code> <code>tool</code> <code>yacc</code>”) -has been removed. As of Go 1.7 it was no longer used by the Go compiler. +“<code>go</code> <code>tool</code> <code>yacc</code>”) has been removed. +As of Go 1.7 it was no longer used by the Go compiler. It has moved to the “tools” repository and is now available at <code><a href="https://godoc.org/golang.org/x/tools/cmd/goyacc">golang.org/x/tools/cmd/goyacc</a></code>. </p> @@ -229,7 +248,7 @@ It has moved to the “tools” repository and is now available at <p> <!-- CL 33157 --> The <code>pprof</code> tool can now profile TLS servers - and skip certificate validation by using the "<code>https+insecure</code>" + and skip certificate validation by using the “<code>https+insecure</code>” URL scheme. </p> @@ -237,37 +256,32 @@ It has moved to the “tools” repository and is now available at The callgrind output now has instruction-level granularity. </p> -<p> - TODO: more. proto? standalone profiles with symbols? -<pre> -runtime/pprof: output CPU profiles in pprof protobuf format (CL 33071) -runtime/pprof: write profiles in protobuf format. (CL 32257) -</pre> -</p> - <h3 id="tool_trace">Trace</h3> -<p>TODO:</p> -<pre> -cmd/trace: add option to output pprof files (CL 23324) -cmd/trace: fix a runnable goroutine count bug (CL 25552) -cmd/trace: move process-wide GC events to their own row (CL 30017) -internal/trace: fix analysis of EvGoWaiting/EvGoInSyscall events (CL 25572) -cmd/trace: annotate different mark worker types (CL 30702) -</pre> +<p> <!-- CL 23324 --> + The <code>trace</code> tool has a new <code>-pprof</code> flag for + producing pprof-compatible blocking and latency profiles from an + execution trace. +</p> + +<p> <!-- CL 30017, CL 30702 --> + Garbage collection events are now shown more clearly in the + execution trace viewer. Garbage collection activity is shown on its + own row and GC helper goroutines are annotated with their roles. +</p> <h3 id="tool_vet">Vet</h3> <p>Vet is stricter in some ways and looser where it previously caused false positives.</p> -<p>Vet now checks copying of array of locks, +<p>Vet now checks for copying an array of locks, duplicate JSON and XML struct field tags, non-space-separated struct tags, deferred calls to HTTP <code>Response.Body.Close</code> - before checking errors, - indexed arguments in <code>Printf</code>, - and improves existing checks.</p> + before checking errors, and + indexed arguments in <code>Printf</code>. + It also improves existing checks.</p> </p> <h3 id="compiler">Compiler Toolchain</h3> @@ -286,14 +300,14 @@ and provides a better platform for optimizations such as bounds check elimination. The new back end reduces the CPU time required by <a href="https://golang.org/test/bench/go1/">our benchmark programs</a> by 20-30% -on 32-bit ARM systems. For 64-bit x86 systems, which already used the SSA backend in +on 32-bit ARM systems. For 64-bit x86 systems, which already used the SSA back end in Go 1.7, the gains are a more modest 0-10%. Other architectures will likely see improvements closer to the 32-bit ARM numbers. </p> <p> The temporary <code>-ssa=0</code> compiler flag introduced in Go 1.7 - to disable the new backend has been removed in Go 1.8. + to disable the new back end has been removed in Go 1.8. </p> <p> @@ -313,9 +327,21 @@ see improvements closer to the 32-bit ARM numbers. <h3 id="cmd_cgo">Cgo</h3> +<p> <!-- CL 31141 --> +The Go tool now remembers the value of the <code>CGO_ENABLED</code> environment +variable set during <code>make.bash</code> and applies it to all future compilations +by default to fix issue <a href="https://golang.org/issue/12808">#12808</a>. +When doing native compilation, it is rarely necessary to explicitly set +the <code>CGO_ENABLED</code> environment variable as <code>make.bash</code> +will detect the correct setting automatically. The main reason to explicitly +set the <code>CGO_ENABLED</code> environment variable is when your environment +supports cgo, but you explicitly do not want cgo support, in which case, set +<code>CGO_ENABLED=0</code> during <code>make.bash</code> or <code>all.bash</code>. +</p> + <p> <!-- CL 29991 --> The environment variable <code>PKG_CONFIG</code> may now be used to -set the program to run to handle <code>#cgo pkg-config</code> +set the program to run to handle <code>#cgo</code> <code>pkg-config</code> directives. The default is <code>pkg-config</code>, the program always used by earlier releases. This is intended to make it easier to cross-compile @@ -365,12 +391,21 @@ version of gccgo. <code>%USERPROFILE%/go</code> on Windows. </p> +<h3 id="go_get">Go get</h3> + +<p> <!-- CL 34818 --> + The “<code>go</code> <code>get</code>” command now always respects + HTTP proxy environment variables, regardless of whether + the <code style='white-space:nowrap'>-insecure</code> flag is used. In previous releases, the + <code style='white-space:nowrap'>-insecure</code> flag had the side effect of not using proxies. +</p> + <h3 id="go_bug">Go bug</h3> <p> The new - “<a href="/cmd/go/#hdr-Print_information_for_bug_reports"><code>go</code> - <code>bug</code></a>” command starts a bug report on GitHub, prefilled + “<a href="/cmd/go/#hdr-Print_information_for_bug_reports"><code>go</code> <code>bug</code></a>” + command starts a bug report on GitHub, prefilled with information about the current system. </p> @@ -378,9 +413,8 @@ version of gccgo. <p> <!-- CL 25419 --> The - “<a href="/cmd/go/#hdr-Show_documentation_for_package_or_symbol"><code>go</code> - <code>doc</code></a>” command - now groups constants and variables with their type, + “<a href="/cmd/go/#hdr-Show_documentation_for_package_or_symbol"><code>go</code> <code>doc</code></a>” + command now groups constants and variables with their type, following the behavior of <a href="/cmd/godoc/"><code>godoc</code></a>. </p> @@ -404,7 +438,7 @@ version of gccgo. plugins written in Go, and a new <a href="/pkg/plugin/"><code>plugin</code></a> package for loading such plugins at run time. Plugin support is only currently - available on Linux and macOS. + available on Linux. </p> <h2 id="runtime">Runtime</h2> @@ -429,10 +463,30 @@ version of gccgo. documentation</a> and its example for more details. </p> +<h3 id="mapiter">Concurrent Map Misuse</h3> + +<p> +In Go 1.6, the runtime +<a href="/doc/go1.6#runtime">added lightweight, +best-effort detection of concurrent misuse of maps</a>. This release +improves that detector with support for detecting programs that +concurrently write to and iterate over a map. +</p> +<p> +As always, if one goroutine is writing to a map, no other goroutine should be +reading (which includes iterating) or writing the map concurrently. +If the runtime detects this condition, it prints a diagnosis and crashes the program. +The best way to find out more about the problem is to run the program +under the +<a href="https://blog.golang.org/race-detector">race detector</a>, +which will more reliably identify the race +and give more detail. +</p> + <h3 id="memstats">MemStats Documentation</h3> <p> <!-- CL 28972 --> - The runtime's <a href="/pkg/runtime/#MemStats"><code>MemStats</code></a> + The <a href="/pkg/runtime/#MemStats"><code>runtime.MemStats</code></a> type has been more thoroughly documented. </p> @@ -470,7 +524,7 @@ There have been optimizations to implementations in the <a href="/pkg/strings/"><code>strings</code></a>, <a href="/pkg/syscall/"><code>syscall</code></a>, <a href="/pkg/text/template/"><code>text/template</code></a>, and -<a href="/pkg/unicode/utf8/"><code>unicode/utf8</code></a>, +<a href="/pkg/unicode/utf8/"><code>unicode/utf8</code></a> packages. </p> @@ -562,9 +616,6 @@ now implements the new takes a context argument.</li> <li>There have been <a href="#database_sql">significant additions</a> to the <a href="/pkg/database/sql/">database/sql</a> package with context support.</li> - <li>The new <a href="/pkg/testing/#T.Context"><code>T.Context</code></a> - method in the <a href="/pkg/testing/">testing</a> package now returns a context for - the active test or benchmark.</li> <li>All nine of the new <code>Lookup</code> methods on the new <a href="/pkg/net/#Resolver"><code>net.Resolver</code></a> now take a context.</li> @@ -578,7 +629,7 @@ now implements the new <p> Most users will want to use the new <code>-mutexprofile</code> - flag with <a href="/cmd/go/#hdr-Description_of_testing_flags"><code>go</code> <code>test</code></a>, + flag with “<a href="/cmd/go/#hdr-Description_of_testing_flags"><code>go</code> <code>test</code></a>”, and then use <a href="/cmd/pprof/">pprof</a> on the resultant file. </p> @@ -589,13 +640,20 @@ now implements the new <a href="/pkg/runtime/#SetMutexProfileFraction"><code>SetMutexProfileFraction</code></a>. </p> +<p> + A known limitation for Go 1.8 is that the profile only reports contention for + <a href="/pkg/sync/#Mutex"><code>sync.Mutex</code></a>, + not + <a href="/pkg/sync/#RWMutex"><code>sync.RWMutex</code></a>. +</p> + <h3 id="minor_library_changes">Minor changes to the library</h3> <p> As always, there are various minor changes and updates to the library, made with the Go 1 <a href="/doc/go1compat">promise of compatibility</a> -in mind. The follow sections list the user visible changes and additions. -Optimizations and bug fixes are not listed. +in mind. The following sections list the user visible changes and additions. +Optimizations and minor bug fixes are not listed. </p> <dl id="archive_tar"><dt><a href="/pkg/archive/tar/">archive/tar</a></dt> @@ -612,20 +670,6 @@ Optimizations and bug fixes are not listed. </dd> </dl> -<dl id="archive_zip"><dt><a href="/pkg/archive/zip/">archive/zip</a></dt> - <dd> - - <p> <!-- CL 18274 --> - The zip <code>Reader</code> now supports modification times in - the NTFS, UNIX, and Extended Time Stamp metadata fields. - <!-- CL 30811 --> - When writing zip files, the Extended Time Stamp field is written - for files with non-zero modification times. - </p> - - </dd> -</dl> - <dl id="compress_flate"><dt><a href="/pkg/compress/flate/">compress/flate</a></dt> <dd> @@ -633,11 +677,11 @@ Optimizations and bug fixes are not listed. There have been some minor fixes to the encoder to improve the compression ratio in certain situations. As a result, the exact encoded output of <code>DEFLATE</code> may be different from Go 1.7. Since - DEFLATE is the underlying compression of gzip, png, zlib, and zip, + <code>DEFLATE</code> is the underlying compression of gzip, png, zlib, and zip, those formats may have changed outputs. </p> - <p> + <p> <!-- CL 31174 --> The encoder, when operating in <a href="/pkg/compress/flate/#NoCompression"><code>NoCompression</code></a> mode, now produces a consistent output that is not dependent on @@ -751,14 +795,14 @@ Optimizations and bug fixes are not listed. X25519 and <!-- CL 30824, CL 30825 --> ChaCha20-Poly1305. <!-- CL 30957, CL 30958 --> ChaCha20-Poly1305 is now prioritized unless <!-- CL 32871 --> - AES-GCM when hardware support is present. + hardware support for AES-GCM is present. </p> <p> <!-- CL 27315 --> AES-128-CBC cipher suites with SHA-256 are also now supported. </p> - + </dd> </dl> @@ -799,87 +843,85 @@ crypto/x509: return error for missing SerialNumber (CL 27238) at <code>/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem</code> on Linux, to support RHEL and CentOS. </p> - + </dd> </dl> - + <dl id="database_sql"><dt><a href="/pkg/database/sql/">database/sql</a></dt> <dd> <p> - The package now supports <code>context.Context</code>. There are new methods - ending in <code>Context</code> such as - <a href="/pkg/database/sql/#DB.QueryContext"><code>DB.QueryContext</code></a> and - <a href="/pkg/database/sql/#DB.PrepareContext"><code>DB.PrepareContext</code></a> - that take context arguments. Using the new <code>Context</code> methods ensures that - connections are closed and returned to the connection pool when the - request is done; enables canceling in-progress queries - should the driver support that; and allows the database - pool to cancel waiting for the next available connection. + The package now supports <code>context.Context</code>. There are new methods + ending in <code>Context</code> such as + <a href="/pkg/database/sql/#DB.QueryContext"><code>DB.QueryContext</code></a> and + <a href="/pkg/database/sql/#DB.PrepareContext"><code>DB.PrepareContext</code></a> + that take context arguments. Using the new <code>Context</code> methods ensures that + connections are closed and returned to the connection pool when the + request is done; enables canceling in-progress queries + should the driver support that; and allows the database + pool to cancel waiting for the next available connection. </p> <p> The <a href="/pkg/database/sql#IsolationLevel"><code>IsolationLevel</code></a> - can now be set when starting a transaction by setting the isolation level - on the <code>Context</code> then passing that <code>Context</code> to - <a href="/pkg/database/sql#DB.BeginContext"><code>DB.BeginContext</code></a>. - An error will be returned if an isolation level is selected that the driver - does not support. A read-only attribute may also be set on the transaction - with <a href="/pkg/database/sql/#ReadOnlyContext"><code>ReadOnlyContext</code></a>. - </p> - <p> + can now be set when starting a transaction by setting the isolation level + on the <code>Context</code> then passing that <code>Context</code> to + <a href="/pkg/database/sql#DB.BeginContext"><code>DB.BeginContext</code></a>. + An error will be returned if an isolation level is selected that the driver + does not support. A read-only attribute may also be set on the transaction + with <a href="/pkg/database/sql/#ReadOnlyContext"><code>ReadOnlyContext</code></a>. + </p> + <p> Queries now expose the SQL column type information for drivers that support it. - Rows can return <a href="/pkg/database/sql#Rows.ColumnTypes"><code>ColumnTypes</code></a> - which can include SQL type information, column type lengths, and the Go type. - </p> - <p> - A <a href="/pkg/database/sql/#Rows"><code>Rows</code></a> - can now represent multiple result sets. After - <a href="/pkg/database/sql/#Rows.Next"><code>Rows.Next</code></a> returns false, - <a href="/pkg/database/sql/#Rows.NextResultSet"><code>Rows.NextResultSet</code></a> - may be called to advance to the next result set. The existing <code>Rows</code> - should continue to be used after it advances to the next result set. - </p> - <p> - <a href="/pkg/database/sql/#NamedParam"><code>NamedParam</code></a> may be used - as query arguments. The new function <a href="/pkg/database/sql/#Param"><code>Param</code></a> - helps create a <a href="/pkg/database/sql/#NamedParam"><code>NamedParam</code></a> - more succinctly. - <p> - If a driver supports the new - <a href="/pkg/database/sql/driver/#Pinger"><code>Pinger</code></a> - interface, the <code>DB</code>'s - <a href="/pkg/database/sql/#DB.Ping"><code>DB.Ping</code></a> - and - <a href="/pkg/database/sql/#DB.PingContext"><code>DB.PingContext</code></a> - methods will use that interface to check whether a - database connection is still valid. - </p> + Rows can return <a href="/pkg/database/sql#Rows.ColumnTypes"><code>ColumnTypes</code></a> + which can include SQL type information, column type lengths, and the Go type. + </p> + <p> + A <a href="/pkg/database/sql/#Rows"><code>Rows</code></a> + can now represent multiple result sets. After + <a href="/pkg/database/sql/#Rows.Next"><code>Rows.Next</code></a> returns false, + <a href="/pkg/database/sql/#Rows.NextResultSet"><code>Rows.NextResultSet</code></a> + may be called to advance to the next result set. The existing <code>Rows</code> + should continue to be used after it advances to the next result set. + </p> + <p> + <a href="/pkg/database/sql/#NamedArg"><code>NamedArg</code></a> may be used + as query arguments. The new function <a href="/pkg/database/sql/#Named"><code>Named</code></a> + helps create a <a href="/pkg/database/sql/#NamedArg"><code>NamedArg</code></a> + more succinctly. + <p> + If a driver supports the new + <a href="/pkg/database/sql/driver/#Pinger"><code>Pinger</code></a> + interface, the + <a href="/pkg/database/sql/#DB.Ping"><code>DB.Ping</code></a> + and + <a href="/pkg/database/sql/#DB.PingContext"><code>DB.PingContext</code></a> + methods will use that interface to check whether a + database connection is still valid. + </p> <p> - The new <code>Context</code> query methods work for all drivers, but - <code>Context</code> cancelation is not responsive unless the driver has been - updated to use them. The other features require driver support in - <a href="/pkg/database/sql/driver"><code>database/sql/driver</code></a>. - Driver authors should review the new interfaces. Users of existing - driver should review the driver documentation to see what - it supports and any system specific documentation on each feature. - </p> + The new <code>Context</code> query methods work for all drivers, but + <code>Context</code> cancelation is not responsive unless the driver has been + updated to use them. The other features require driver support in + <a href="/pkg/database/sql/driver"><code>database/sql/driver</code></a>. + Driver authors should review the new interfaces. Users of existing + driver should review the driver documentation to see what + it supports and any system specific documentation on each feature. + </p> </dd> </dl> <dl id="debug_pe"><dt><a href="/pkg/debug/pe/">debug/pe</a></dt> <dd> <p> <!-- CL 22720, CL 27212, CL 22181, CL 22332, CL 22336, Issue 15345 --> - The package has been fleshed out and is now used by <a href="/cmd/link/">the Go linker</a>. - New are - <a href="/pkg/debug/pe/#Reloc"><code>Reloc</code></a>, - <a href="/pkg/debug/pe/#Section"><code>Section</code></a>, - <a href="/pkg/debug/pe/#StringTable"><code>StringTable</code></a>, - the method - <a href="/pkg/debug/pe/#COFFSymbol.FullName"><code>COFFSymbol.FullName</code></a>, + The package has been extended and is now used by + <a href="/cmd/link/">the Go linker</a> to read <code>gcc</code>-generated object files. + The new + <a href="/pkg/debug/pe/#File.StringTable"><code>File.StringTable</code></a> and - <a href="/pkg/debug/pe/#File"><code>File</code></a> - fields - <a href="/pkg/debug/pe/#File.COFFSymbols"><code>COFFSymbols</code></a> and - <a href="/pkg/debug/pe/#File.StringTable"><code>StringTable</code></a>. + <a href="/pkg/debug/pe/#Section.Relocs"><code>Section.Relocs</code></a> + fields provide access to the COFF string table and COFF relocations. + The new + <a href="/pkg/debug/pe/#File.COFFSymbols"><code>File.COFFSymbols</code></a> + allows low-level access to the COFF symbol table. </p> </dd> </dl> @@ -933,9 +975,8 @@ crypto/x509: return error for missing SerialNumber (CL 27238) </p> <p> <!-- CL 30944 --> - In previous versions of Go, unmarshaling a JSON <code>null</code> into an - of <a href="/pkg/encoding/json/#Unmarshaler"><code>Unmarshaler</code></a> + <a href="/pkg/encoding/json/#Unmarshaler"><code>Unmarshaler</code></a> was considered a no-op; now the <code>Unmarshaler</code>'s <code>UnmarshalJSON</code> method is called with the JSON literal <code>null</code> and can define the semantics of that case. @@ -1065,7 +1106,7 @@ crypto/x509: return error for missing SerialNumber (CL 27238) <a href="/pkg/math/big/#Int.ModInverse"><code>Int.ModInverse</code></a> now supports negative numbers. </p> - + </dd> </dl> @@ -1092,9 +1133,9 @@ crypto/x509: return error for missing SerialNumber (CL 27238) <a href="/pkg/mime/#ParseMediaType"><code>ParseMediaType</code></a> now preserves unnecessary backslash escapes as literals, in order to support MSIE. - When MSIE sends a full file path (in "intranet mode"), it does not - escape backslashes: <code>"C:\dev\go\foo.txt"</code>, not - <code>"C:\\dev\\go\\foo.txt"</code>. + When MSIE sends a full file path (in “intranet mode”), it does not + escape backslashes: “<code>C:\dev\go\foo.txt</code>”, not + “<code>C:\\dev\\go\\foo.txt</code>”. If we see an unnecessary backslash escape, we now assume it is from MSIE and intended as a literal backslash. No known MIME generators emit unnecessary backslash escapes @@ -1126,7 +1167,7 @@ crypto/x509: return error for missing SerialNumber (CL 27238) <dl id="net"><dt><a href="/pkg/net/">net</a></dt> <dd> - + <p><!-- CL 30164, CL 33473 --> The <a href="/pkg/net/#Conn"><code>Conn</code></a> documentation has been updated to clarify expectations of an interface @@ -1147,8 +1188,8 @@ crypto/x509: return error for missing SerialNumber (CL 27238) </p> <p><!-- CL 29951 --> - The new <a href="/pkg/net/#Buffers"><code>Buffers</code></a> types permits - more efficiently writing to the network from multiple discontiguous buffers + The new <a href="/pkg/net/#Buffers"><code>Buffers</code></a> type permits + writing to the network more efficiently from multiple discontiguous buffers in memory. On certain machines, for certain types of connections, this is optimized into an OS-specific batch write operation (such as <code>writev</code>). </p> @@ -1165,8 +1206,8 @@ crypto/x509: return error for missing SerialNumber (CL 27238) </p> <p><!-- CL 29233, CL 24901 --> - The Go DNS resolver now supports <code>resolv.conf</code>'s "<code>rotate</code>" - and "<code>option ndots:0</code>" options. The "<code>ndots</code>" option is + The Go DNS resolver now supports <code>resolv.conf</code>'s “<code>rotate</code>” + and “<code>option</code> <code>ndots:0</code>” options. The “<code>ndots</code>” option is now respected in the same way as <code>libresolve</code>. </p> @@ -1193,7 +1234,7 @@ crypto/x509: return error for missing SerialNumber (CL 27238) <a href="/pkg/net/http/#ServeContent"><code>ServeContent</code></a> now support HTTP <code>If-Match</code> conditional requests, in addition to the previous <code>If-None-Match</code> - support. + support for ETags properly formatted according to RFC 7232, section 2.3. </li> </ul> @@ -1213,11 +1254,11 @@ crypto/x509: return error for missing SerialNumber (CL 27238) existing <a href="/pkg/net/http/#CloseNotifier"><code>CloseNotifier</code></a> support. This functionality requires that the underlying <a href="/pkg/net/#Conn"><code>net.Conn</code></a> implements - <a href="#net">recently-clarified interface documentation</a>. + <a href="#net">recently clarified interface documentation</a>. </li> <li><!-- CL 32479 --> - To serve trailers known after the header has been written, + To serve trailers produced after the header has already been written, see the new <a href="/pkg/net/http/#TrailerPrefix"><code>TrailerPrefix</code></a> mechanism. @@ -1244,7 +1285,7 @@ crypto/x509: return error for missing SerialNumber (CL 27238) <p>Client & Transport changes:</p> <ul> - <li><!-- CL 28930 --> + <li><!-- CL 28930, CL 31435 --> The <a href="/pkg/net/http/#Client"><code>Client</code></a> now copies most request headers on redirect. See <a href="/pkg/net/http/#Client">the documentation</a> @@ -1258,9 +1299,17 @@ crypto/x509: return error for missing SerialNumber (CL 27238) </li> <li><!-- CL 31733, CL 29852 --> - The <code>Client</code> now supports 307 and 308 redirects. - If the redirect requires resending the request body, - the request must have the new + The <code>Client</code> now supports 301, 307, and 308 redirects. + + For example, <code>Client.Post</code> now follows 301 + redirects, converting them to <code>GET</code> requests + without bodies, like it did for 302 and 303 redirect responses + previously. + + The <code>Client</code> now also follows 307 and 308 + redirects, preserving the original request method and body, if + any. If the redirect requires resending the request body, the + request must have the new <a href="/pkg/net/http/#Request"><code>Request.GetBody</code></a> field defined. <a href="pkg/net/http/#NewRequest"><code>NewRequest</code></a> @@ -1275,7 +1324,8 @@ crypto/x509: return error for missing SerialNumber (CL 27238) <li><!-- CL 27117 --> The <code>Transport</code> will now retry non-idempotent - requests if no bytes were written before a network failure. + requests if no bytes were written before a network failure + and the request has no body. </li> <li><!-- CL 32481 --> @@ -1288,9 +1338,25 @@ crypto/x509: return error for missing SerialNumber (CL 27238) <li> <!-- CL 28077 --> The <a href="/pkg/net/http/#DefaultTransport"><code>DefaultTransport.Dialer</code></a> now enables <code>DualStack</code> ("<a href="https://tools.ietf.org/html/rfc6555">Happy Eyeballs</a>") support, - to use IPv4 as a backup if it looks like IPv6 might be + allowing the use of IPv4 as a backup if it looks like IPv6 might be failing. </li> + + <li> <!-- CL 31726 --> + The <a href="/pkg/net/http/#Transport"><code>Transport</code></a> + no longer reads a byte of a non-nil + <a href="/pkg/net/http/#Request.Body"><code>Request.Body</code></a> + when the + <a href="/pkg/net/http/#Request.ContentLength"><code>Request.ContentLength</code></a> + is zero to determine whether the <code>ContentLength</code> + is actually zero or just undefined. + To explicitly signal that a body has zero length, + either set it to <code>nil</code>, or set it to the new value + <a href="/pkg/net/http/#NoBody"><code>NoBody</code></a>. + The new <code>NoBody</code> value is intended for use by <code>Request</code> + constructor functions; it is used by + <a href="/pkg/net/http/#NewRequest"><code>NewRequest</code></a>. + </li> </ul> </dd> @@ -1314,9 +1380,9 @@ crypto/x509: return error for missing SerialNumber (CL 27238) The <a href="/pkg/net/http/httputil/#ReverseProxy"><code>ReverseProxy</code></a> has a new optional hook, <a href="/pkg/net/http/httputil/#ReverseProxy.ModifyResponse"><code>ModifyResponse</code></a>, - for modifying the response from the backend before proxying it to the client. + for modifying the response from the back end before proxying it to the client. </p> - + </dd> </dl> @@ -1326,7 +1392,7 @@ crypto/x509: return error for missing SerialNumber (CL 27238) <p> <!-- CL 32176 --> Empty quoted strings are once again allowed in the name part of an address. That is, Go 1.4 and earlier accepted - <code>"" <gopher@example.com></code>, + <code>""</code> <code><gopher@example.com></code>, but Go 1.5 introduced a bug that rejected this address. The address is recognized again. </p> @@ -1341,7 +1407,7 @@ crypto/x509: return error for missing SerialNumber (CL 27238) allows parsing dates found in other header lines, such as the <code>Resent-Date:</code> header. </p> - + </dd> </dl> @@ -1349,10 +1415,10 @@ crypto/x509: return error for missing SerialNumber (CL 27238) <dd> <p> <!-- CL 33143 --> - If an implementation of - the <a href="/pkg/net/smtp/#Auth"><code>Auth</code></a> - interface's <code>Start</code> method returns an - empty <code>toServer</code> value, the package no longer sends + If an implementation of the + <a href="/pkg/net/smtp/#Auth"><code>Auth.Start</code></a> + method returns an empty <code>toServer</code> value, + the package no longer sends trailing whitespace in the SMTP <code>AUTH</code> command, which some servers rejected. </p> @@ -1363,14 +1429,17 @@ crypto/x509: return error for missing SerialNumber (CL 27238) <dl id="net_url"><dt><a href="/pkg/net/url/">net/url</a></dt> <dd> - <p> <!-- CL 31322 --> The new functions + <p> <!-- CL 31322 --> + The new functions <a href="/pkg/net/url/#PathEscape"><code>PathEscape</code></a> and <a href="/pkg/net/url/#PathUnescape"><code>PathUnescape</code></a> are similar to the query escaping and unescaping functions but - for path elements.</p> + for path elements. + </p> - <p> <!-- CL 28933 --> The new methods + <p> <!-- CL 28933 --> + The new methods <a href="/pkg/net/url/#URL.Hostname"><code>URL.Hostname</code></a> and <a href="/pkg/net/url/#URL.Port"><code>URL.Port</code></a> @@ -1378,7 +1447,8 @@ crypto/x509: return error for missing SerialNumber (CL 27238) correctly handling the case where the port may not be present. </p> - <p> <!-- CL 28343 --> The existing method + <p> <!-- CL 28343 --> + The existing method <a href="/pkg/net/url/#URL.ResolveReference"><code>URL.ResolveReference</code></a> now properly handles paths with escaped bytes without losing the escaping. @@ -1397,7 +1467,7 @@ crypto/x509: return error for missing SerialNumber (CL 27238) now rejects URLs like <code>this_that:other/thing</code> instead of interpreting them as relative paths (<code>this_that</code> is not a valid scheme). To force interpretation as a relative path, - such URLs should be prefixed with <code>"./"</code>. + such URLs should be prefixed with “<code>./</code>”. The <code>URL.String</code> method now inserts this prefix as needed. </p> @@ -1406,7 +1476,7 @@ crypto/x509: return error for missing SerialNumber (CL 27238) <dl id="os"><dt><a href="/pkg/os/">os</a></dt> <dd> - <p> + <p> <!-- CL 16551 --> The new function <a href="/pkg/os/#Executable"><code>Executable</code></a> returns the path name of the running executable. @@ -1427,12 +1497,12 @@ crypto/x509: return error for missing SerialNumber (CL 27238) existing empty directory. Previously it would fail when renaming to a non-empty directory but succeed when renaming to an empty directory. - This makes the behavior on Unix correspond to that on other systems. + This makes the behavior on Unix correspond to that of other systems. </p> <p> <!-- CL 32451 --> On Windows, long absolute paths are now transparently converted to - extended-length paths (paths that start with <code>\\?\</code>). + extended-length paths (paths that start with “<code>\\?\</code>”). This permits the package to work with files whose path names are longer than 260 characters. </p> @@ -1454,32 +1524,18 @@ crypto/x509: return error for missing SerialNumber (CL 27238) </dd> </dl> -<dl id="os_signal"><dt><a href="/pkg/os/signal/">os/signal</a></dt> - <dd> - <p> <!-- CL 32796 --> - In a Go library built with <code>-buildmode=c-archive</code> - or <code>c-shared</code>, when C code calls a Go function, - the <code>SIGPIPE</code> signal will be treated as usual for Go code. - In particular, when <code>SIGPIPE</code> is triggered by a write - to a closed Go network connection, it will not cause the program - to exit. - </p> - </dd> -</dl> - <dl id="path_filepath"><dt><a href="/pkg/path/filepath/">path/filepath</a></dt> <dd> <p> - <p>A number of bugs and corner cases on Windows were fixed: - <a href="/pkg/path/filepath/#Abs"><code>Abs</code></a> now calls <code>Clean</code> paths as documented, + A number of bugs and corner cases on Windows were fixed: + <a href="/pkg/path/filepath/#Abs"><code>Abs</code></a> now calls <code>Clean</code> as documented, <a href="/pkg/path/filepath/#Glob"><code>Glob</code></a> now matches - "<code>\\?\c:\*</code>", + “<code>\\?\c:\*</code>”, <a href="/pkg/path/filepath/#EvalSymlinks"><code>EvalSymlinks</code></a> now - correctly handles "<code>C:.</code>", and + correctly handles “<code>C:.</code>”, and <a href="/pkg/path/filepath/#Clean"><code>Clean</code></a> now properly - handles a leading "<code>..</code>" in the path. - <p> - + handles a leading “<code>..</code>” in the path. + </p> </dd> </dl> @@ -1580,14 +1636,6 @@ crypto/x509: return error for missing SerialNumber (CL 27238) test or benchmark. </p> - <p><!-- CL 31724 --> - The new method - <a href="/pkg/testing/#T.Context"><code>T.Context</code></a> - (and <code>B.Context</code>) returns - a <a href="/pkg/context/#Context"><code>Context</code></a> for - the current running test or benchmark. - </p> - <p><!-- CL 32483 --> The new function <a href="/pkg/testing/#CoverMode"><code>CoverMode</code></a> @@ -1600,15 +1648,15 @@ crypto/x509: return error for missing SerialNumber (CL 27238) Previously, individual test cases would appear to pass, and only the overall execution of the test binary would fail. </p> - + </dd> </dl> <dl id="unicode"><dt><a href="/pkg/unicode/">unicode</a></dt> <dd> <p><!-- CL 30935 --> - <code>SimpleFold</code> now returns its argument unchanged - if the provided input was an invalid rune. + <a href="/pkg/unicode/#SimpleFold"><code>SimpleFold</code></a> + now returns its argument unchanged if the provided input was an invalid rune. Previously, the implementation failed with an index bounds check panic. </p> </dd> diff --git a/doc/go1.8.txt b/doc/go1.8.txt index e66ad387f3..caa9a728f8 100644 --- a/doc/go1.8.txt +++ b/doc/go1.8.txt @@ -29,7 +29,6 @@ cmd/link: fix -buildmode=pie / -linkshared combination (CL 28996) cmd/link: for -buildmode=exe pass -no-pie to external linker (CL 33106) cmd/link: insert trampolines for too-far jumps on ARM (CL 29397) cmd/link: non-executable stack support for Solaris (CL 24142) -cmd/link: plugin support on darwin/amd64 (CL 29394) cmd/link: put text at address 0x1000000 on darwin/amd64 (CL 32185) cmd/link: remove the -shared flag (CL 28852) cmd/link: split large elf text sections on ppc64x (CL 27790) diff --git a/doc/help.html b/doc/help.html index 644819106d..62d9a4a6b6 100644 --- a/doc/help.html +++ b/doc/help.html @@ -11,6 +11,9 @@ <h3 id="mailinglist"><a href="https://groups.google.com/group/golang-nuts">Go Nuts Mailing List</a></h3> <p> +Get help from Go users, and share your work on the official mailing list. +</p> +<p> Search the <a href="https://groups.google.com/group/golang-nuts">golang-nuts</a> archives and consult the <a href="/doc/go_faq.html">FAQ</a> and <a href="//golang.org/wiki">wiki</a> before posting. @@ -18,12 +21,12 @@ archives and consult the <a href="/doc/go_faq.html">FAQ</a> and <h3 id="forum"><a href="https://forum.golangbridge.org/">Go Forum</a></h3> <p> -The <a href="https://forum.golangbridge.org/">Go Forum</a> is an alternate discussion +The <a href="https://forum.golangbridge.org/">Go Forum</a> is a discussion forum for Go programmers. </p> <h3 id="slack"><a href="https://blog.gopheracademy.com/gophers-slack-community/">Gopher Slack</a></h3> -<p>Get live support from the official Go slack channel.</p> +<p>Get live support from other users in the Go slack channel.</p> <h3 id="irc"><a href="irc:irc.freenode.net/go-nuts">Go IRC Channel</a></h3> <p>Get live support at <b>#go-nuts</b> on <b>irc.freenode.net</b>, the official diff --git a/doc/install-source.html b/doc/install-source.html index 4a25e37d22..4bf0ba35fb 100644 --- a/doc/install-source.html +++ b/doc/install-source.html @@ -33,7 +33,7 @@ compiler using the GCC back end, see </p> <p> -The Go compilers support seven instruction sets. +The Go compilers support eight instruction sets. There are important differences in the quality of the compilers for the different architectures. </p> @@ -55,7 +55,7 @@ architectures. <code>arm</code> (<code>ARM</code>) </dt> <dd> - Supports Linux, FreeBSD, NetBSD and Darwin binaries. Less widely used than the other ports. + Supports Linux, FreeBSD, NetBSD, OpenBSD and Darwin binaries. Less widely used than the other ports. </dd> <dt> <code>arm64</code> (<code>AArch64</code>) @@ -70,6 +70,12 @@ architectures. Supports Linux binaries. New in 1.5 and not as well exercised as other ports. </dd> <dt> + <code>mips, mipsle</code> (32-bit MIPS big- and little-endian) +</dt> +<dd> + Supports Linux binaries. New in 1.8 and not as well exercised as other ports. +</dd> +<dt> <code>mips64, mips64le</code> (64-bit MIPS big- and little-endian) </dt> <dd> @@ -212,7 +218,7 @@ To build without <code>cgo</code>, set the environment variable Change to the directory that will be its parent and make sure the <code>go</code> directory does not exist. Then clone the repository and check out the latest release tag -(<code class="versionTag">go1.7.2</code>, for example):</p> +(<code class="versionTag">go1.7.4</code>, for example):</p> <pre> $ git clone https://go.googlesource.com/go @@ -329,7 +335,7 @@ You just need to do a little more setup. </p> <p> -The <a href="/doc/code.html">How to Write Go Code</a> document +The <a href="/doc/code.html">How to Write Go Code</a> document provides <b>essential setup instructions</b> for using the Go tools. </p> @@ -355,7 +361,7 @@ $ go get golang.org/x/tools/cmd/godoc </pre> <p> -To install these tools, the <code>go</code> <code>get</code> command requires +To install these tools, the <code>go</code> <code>get</code> command requires that <a href="#git">Git</a> be installed locally. </p> @@ -400,7 +406,7 @@ New releases are announced on the <a href="//groups.google.com/group/golang-announce">golang-announce</a> mailing list. Each announcement mentions the latest release tag, for instance, -<code class="versionTag">go1.7.2</code>. +<code class="versionTag">go1.7.4</code>. </p> <p> @@ -443,7 +449,7 @@ The value assumed by installed binaries and scripts when <code>$GOROOT</code> is not set explicitly. It defaults to the value of <code>$GOROOT</code>. If you want to build the Go tree in one location -but move it elsewhere after the build, set +but move it elsewhere after the build, set <code>$GOROOT_FINAL</code> to the eventual location. </p> @@ -463,6 +469,7 @@ Choices for <code>$GOARCH</code> are <code>386</code> (32-bit x86), <code>arm</code> (32-bit ARM), <code>arm64</code> (64-bit ARM), <code>ppc64le</code> (PowerPC 64-bit, little-endian), <code>ppc64</code> (PowerPC 64-bit, big-endian), <code>mips64le</code> (MIPS 64-bit, little-endian), and <code>mips64</code> (MIPS 64-bit, big-endian). +<code>mipsle</code> (MIPS 32-bit, little-endian), and <code>mips</code> (MIPS 32-bit, big-endian). The valid combinations of <code>$GOOS</code> and <code>$GOARCH</code> are: <table cellpadding="0"> <tr> @@ -514,6 +521,12 @@ The valid combinations of <code>$GOOS</code> and <code>$GOARCH</code> are: <td></td><td><code>linux</code></td> <td><code>ppc64le</code></td> </tr> <tr> +<td></td><td><code>linux</code></td> <td><code>mips</code></td> +</tr> +<tr> +<td></td><td><code>linux</code></td> <td><code>mipsle</code></td> +</tr> +<tr> <td></td><td><code>linux</code></td> <td><code>mips64</code></td> </tr> <tr> @@ -566,7 +579,7 @@ architecture. Valid choices are the same as for <code>$GOOS</code> and <code>$GOARCH</code>, listed above. The specified values must be compatible with the local system. -For example, you should not set <code>$GOHOSTARCH</code> to +For example, you should not set <code>$GOHOSTARCH</code> to <code>arm</code> on an x86 system. </p> @@ -627,7 +640,7 @@ not <code>amd64</code>. <p> If you choose to override the defaults, set these variables in your shell profile (<code>$HOME/.bashrc</code>, -<code>$HOME/.profile</code>, or equivalent). The settings might look +<code>$HOME/.profile</code>, or equivalent). The settings might look something like this: </p> diff --git a/doc/install.html b/doc/install.html index ebe66c0205..2143d591cb 100644 --- a/doc/install.html +++ b/doc/install.html @@ -47,8 +47,8 @@ If your OS or architecture is not on the list, you may be able to <th align="center">Notes</th> </tr> <tr><td colspan="3"><hr></td></tr> -<tr><td>FreeBSD 8-STABLE or later</td> <td>amd64</td> <td>Debian GNU/kFreeBSD not supported</td></tr> -<tr><td>Linux 2.6.23 or later with glibc</td> <td>amd64, 386, arm</td> <td>CentOS/RHEL 5.x not supported</td></tr> +<tr><td>FreeBSD 8-STABLE or later</td> <td>amd64, 386</td> <td>Debian GNU/kFreeBSD not supported</td></tr> +<tr><td>Linux 2.6.23 or later with glibc</td> <td>amd64, 386, arm, s390x, ppc64le</td> <td>CentOS/RHEL 5.x not supported</td></tr> <tr><td>Mac OS X 10.8 or later</td> <td>amd64</td> <td>use the clang or gcc<sup>†</sup> that comes with Xcode<sup>‡</sup> for <code>cgo</code> support</td></tr> <tr><td>Windows XP or later</td> <td>amd64, 386</td> <td>use MinGW gcc<sup>†</sup>. No need for cygwin or msys.</td></tr> </table> @@ -246,12 +246,12 @@ Then build it with the <code>go</code> tool: </p> <pre class="testUnix"> -$ <b>cd $HOME/go/src/hello +$ <b>cd $HOME/go/src/hello</b> $ <b>go build</b> </pre> <pre class="testWindows" style="display: none"> -C:\> <b>cd %USERPROFILE%\go\src\hello<b> +C:\> <b>cd %USERPROFILE%\go\src\hello</b> C:\Users\Gopher\go\src\hello> <b>go build</b> </pre> @@ -313,16 +313,10 @@ environment variables under Windows</a>. <h2 id="help">Getting help</h2> <p> -For real-time help, ask the helpful gophers in <code>#go-nuts</code> on the -<a href="http://freenode.net/">Freenode</a> IRC server. + For help, see the <a href="/help/">list of Go mailing lists, forums, and places to chat</a>. </p> <p> -The official mailing list for discussion of the Go language is -<a href="//groups.google.com/group/golang-nuts">Go Nuts</a>. -</p> - -<p> -Report bugs using the -<a href="//golang.org/issue">Go issue tracker</a>. + Report bugs either by running “<b><code>go</code> <code>bug</code></b>”, or + manually at the <a href="https://golang.org/issue">Go issue tracker</a>. </p> diff --git a/lib/time/update.bash b/lib/time/update.bash index b70788e7b4..4297c5f545 100755 --- a/lib/time/update.bash +++ b/lib/time/update.bash @@ -8,8 +8,8 @@ # Consult http://www.iana.org/time-zones for the latest versions. # Versions to use. -CODE=2016i -DATA=2016i +CODE=2016j +DATA=2016j set -e rm -rf work diff --git a/lib/time/zoneinfo.zip b/lib/time/zoneinfo.zip Binary files differindex e12d6dccaf..d33dc1d273 100644 --- a/lib/time/zoneinfo.zip +++ b/lib/time/zoneinfo.zip diff --git a/misc/cgo/test/cgo_unix_test.go b/misc/cgo/test/cgo_unix_test.go index b3633b73f3..e3d5916649 100644 --- a/misc/cgo/test/cgo_unix_test.go +++ b/misc/cgo/test/cgo_unix_test.go @@ -10,3 +10,4 @@ import "testing" func TestSigaltstack(t *testing.T) { testSigaltstack(t) } func TestSigprocmask(t *testing.T) { testSigprocmask(t) } +func Test18146(t *testing.T) { test18146(t) } diff --git a/misc/cgo/test/issue17537.go b/misc/cgo/test/issue17537.go index debdbfe4c5..777104e512 100644 --- a/misc/cgo/test/issue17537.go +++ b/misc/cgo/test/issue17537.go @@ -23,6 +23,18 @@ int I17537(S17537 *p); const int F17537(const char **p) { return **p; } + +// Calling this function used to trigger an error from the C compiler +// (issue 18298). +void F18298(const void *const *p) { +} + +// Test that conversions between typedefs work as they used to. +typedef const void *T18298_1; +struct S18298 { int i; }; +typedef const struct S18298 *T18298_2; +void G18298(T18298_1 t) { +} */ import "C" @@ -39,4 +51,8 @@ func test17537(t *testing.T) { if got, want := C.F17537(&p), C.int(17); got != want { t.Errorf("got %d, want %d", got, want) } + + C.F18298(nil) + var v18298 C.T18298_2 + C.G18298(C.T18298_1(v18298)) } diff --git a/misc/cgo/test/issue18146.go b/misc/cgo/test/issue18146.go new file mode 100644 index 0000000000..ffb04e9037 --- /dev/null +++ b/misc/cgo/test/issue18146.go @@ -0,0 +1,128 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !windows + +// Issue 18146: pthread_create failure during syscall.Exec. + +package cgotest + +import "C" + +import ( + "bytes" + "crypto/md5" + "os" + "os/exec" + "runtime" + "syscall" + "testing" + "time" +) + +func test18146(t *testing.T) { + if runtime.GOOS == "darwin" { + t.Skipf("skipping flaky test on %s; see golang.org/issue/18202", runtime.GOOS) + } + + if runtime.GOARCH == "mips" || runtime.GOARCH == "mips64" { + t.Skipf("skipping on %s", runtime.GOARCH) + } + + attempts := 1000 + threads := 4 + + if testing.Short() { + attempts = 100 + } + + // Restrict the number of attempts based on RLIMIT_NPROC. + // Tediously, RLIMIT_NPROC was left out of the syscall package, + // probably because it is not in POSIX.1, so we define it here. + // It is not defined on Solaris. + var nproc int + setNproc := true + switch runtime.GOOS { + default: + setNproc = false + case "linux": + nproc = 6 + case "darwin", "dragonfly", "freebsd", "netbsd", "openbsd": + nproc = 7 + } + if setNproc { + var rlim syscall.Rlimit + if syscall.Getrlimit(nproc, &rlim) == nil { + max := int(rlim.Cur) / (threads + 5) + if attempts > max { + t.Logf("lowering attempts from %d to %d for RLIMIT_NPROC", attempts, max) + attempts = max + } + } + } + + if os.Getenv("test18146") == "exec" { + runtime.GOMAXPROCS(1) + for n := threads; n > 0; n-- { + go func() { + for { + _ = md5.Sum([]byte("Hello, !")) + } + }() + } + runtime.GOMAXPROCS(threads) + argv := append(os.Args, "-test.run=NoSuchTestExists") + if err := syscall.Exec(os.Args[0], argv, nil); err != nil { + t.Fatal(err) + } + } + + var cmds []*exec.Cmd + defer func() { + for _, cmd := range cmds { + cmd.Process.Kill() + } + }() + + args := append(append([]string(nil), os.Args[1:]...), "-test.run=Test18146") + for n := attempts; n > 0; n-- { + cmd := exec.Command(os.Args[0], args...) + cmd.Env = append(os.Environ(), "test18146=exec") + buf := bytes.NewBuffer(nil) + cmd.Stdout = buf + cmd.Stderr = buf + if err := cmd.Start(); err != nil { + // We are starting so many processes that on + // some systems (problem seen on Darwin, + // Dragonfly, OpenBSD) the fork call will fail + // with EAGAIN. + if pe, ok := err.(*os.PathError); ok { + err = pe.Err + } + if se, ok := err.(syscall.Errno); ok && (se == syscall.EAGAIN || se == syscall.EMFILE) { + time.Sleep(time.Millisecond) + continue + } + + t.Error(err) + return + } + cmds = append(cmds, cmd) + } + + failures := 0 + for _, cmd := range cmds { + err := cmd.Wait() + if err == nil { + continue + } + + t.Errorf("syscall.Exec failed: %v\n%s", err, cmd.Stdout) + failures++ + } + + if failures > 0 { + t.Logf("Failed %v of %v attempts.", failures, len(cmds)) + } +} diff --git a/misc/cgo/test/issue9400/asm_mipsx.s b/misc/cgo/test/issue9400/asm_mipsx.s new file mode 100644 index 0000000000..ddf33e9f8e --- /dev/null +++ b/misc/cgo/test/issue9400/asm_mipsx.s @@ -0,0 +1,31 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build mips mipsle +// +build !gccgo + +#include "textflag.h" + +TEXT ·RewindAndSetgid(SB),NOSPLIT,$-4-0 + // Rewind stack pointer so anything that happens on the stack + // will clobber the test pattern created by the caller + ADDU $(1024*8), R29 + + // Ask signaller to setgid + MOVW $1, R1 + SYNC + MOVW R1, ·Baton(SB) + SYNC + + // Wait for setgid completion +loop: + SYNC + MOVW ·Baton(SB), R1 + OR R2, R2, R2 // hint that we're in a spin loop + BNE R1, loop + SYNC + + // Restore stack + ADDU $(-1024*8), R29 + RET diff --git a/misc/cgo/testcarchive/carchive_test.go b/misc/cgo/testcarchive/carchive_test.go index 3c768a0ef3..4999929775 100644 --- a/misc/cgo/testcarchive/carchive_test.go +++ b/misc/cgo/testcarchive/carchive_test.go @@ -265,25 +265,6 @@ func TestSignalForwarding(t *testing.T) { t.Logf("%s", out) t.Errorf("got %v; expected SIGSEGV", ee) } - - // Test SIGPIPE forwarding - cmd = exec.Command(bin[0], append(bin[1:], "3")...) - - out, err = cmd.CombinedOutput() - - if err == nil { - t.Logf("%s", out) - t.Error("test program succeeded unexpectedly") - } else if ee, ok := err.(*exec.ExitError); !ok { - t.Logf("%s", out) - t.Errorf("error (%v) has type %T; expected exec.ExitError", err, err) - } else if ws, ok := ee.Sys().(syscall.WaitStatus); !ok { - t.Logf("%s", out) - t.Errorf("error.Sys (%v) has type %T; expected syscall.WaitStatus", ee.Sys(), ee.Sys()) - } else if !ws.Signaled() || ws.Signal() != syscall.SIGPIPE { - t.Logf("%s", out) - t.Errorf("got %v; expected SIGPIPE", ee) - } } func TestSignalForwardingExternal(t *testing.T) { diff --git a/misc/cgo/testcarchive/main2.c b/misc/cgo/testcarchive/main2.c index 55625c543a..774e014a16 100644 --- a/misc/cgo/testcarchive/main2.c +++ b/misc/cgo/testcarchive/main2.c @@ -17,7 +17,6 @@ #include <unistd.h> #include <sched.h> #include <time.h> -#include <errno.h> #include "libgo2.h" @@ -27,7 +26,6 @@ static void die(const char* msg) { } static volatile sig_atomic_t sigioSeen; -static volatile sig_atomic_t sigpipeSeen; // Use up some stack space. static void recur(int i, char *p) { @@ -40,11 +38,6 @@ static void recur(int i, char *p) { } // Signal handler that uses up more stack space than a goroutine will have. -static void pipeHandler(int signo, siginfo_t* info, void* ctxt) { - sigpipeSeen = 1; -} - -// Signal handler that uses up more stack space than a goroutine will have. static void ioHandler(int signo, siginfo_t* info, void* ctxt) { char a[1024]; @@ -113,10 +106,6 @@ static void init() { die("sigaction"); } - sa.sa_sigaction = pipeHandler; - if (sigaction(SIGPIPE, &sa, NULL) < 0) { - die("sigaction"); - } } int main(int argc, char** argv) { @@ -178,30 +167,7 @@ int main(int argc, char** argv) { nanosleep(&ts, NULL); i++; if (i > 5000) { - fprintf(stderr, "looping too long waiting for SIGIO\n"); - exit(EXIT_FAILURE); - } - } - - if (verbose) { - printf("provoking SIGPIPE\n"); - } - - GoRaiseSIGPIPE(); - - if (verbose) { - printf("waiting for sigpipeSeen\n"); - } - - // Wait until the signal has been delivered. - i = 0; - while (!sigpipeSeen) { - ts.tv_sec = 0; - ts.tv_nsec = 1000000; - nanosleep(&ts, NULL); - i++; - if (i > 1000) { - fprintf(stderr, "looping too long waiting for SIGPIPE\n"); + fprintf(stderr, "looping too long waiting for signal\n"); exit(EXIT_FAILURE); } } diff --git a/misc/cgo/testcarchive/main3.c b/misc/cgo/testcarchive/main3.c index 07d5d1e64e..0a6c0d3f74 100644 --- a/misc/cgo/testcarchive/main3.c +++ b/misc/cgo/testcarchive/main3.c @@ -35,13 +35,6 @@ int main(int argc, char** argv) { setvbuf(stdout, NULL, _IONBF, 0); if (verbose) { - printf("raising SIGPIPE\n"); - } - - // Test that the Go runtime handles SIGPIPE. - ProvokeSIGPIPE(); - - if (verbose) { printf("calling sigaction\n"); } diff --git a/misc/cgo/testcarchive/main5.c b/misc/cgo/testcarchive/main5.c index 2437bf07c5..9fadf0801e 100644 --- a/misc/cgo/testcarchive/main5.c +++ b/misc/cgo/testcarchive/main5.c @@ -68,24 +68,6 @@ int main(int argc, char** argv) { break; } - case 3: { - if (verbose) { - printf("attempting SIGPIPE\n"); - } - - int fd[2]; - if (pipe(fd) != 0) { - printf("pipe(2) failed\n"); - return 0; - } - // Close the reading end. - close(fd[0]); - // Expect that write(2) fails (EPIPE) - if (write(fd[1], "some data", 9) != -1) { - printf("write(2) unexpectedly succeeded\n"); - return 0; - } - } default: printf("Unknown test: %d\n", test); return 0; diff --git a/misc/cgo/testcarchive/src/libgo2/libgo2.go b/misc/cgo/testcarchive/src/libgo2/libgo2.go index 19c8e1a6dc..fbed493b93 100644 --- a/misc/cgo/testcarchive/src/libgo2/libgo2.go +++ b/misc/cgo/testcarchive/src/libgo2/libgo2.go @@ -4,30 +4,6 @@ package main -/* -#include <signal.h> -#include <unistd.h> -#include <stdlib.h> -#include <stdio.h> - -// Raise SIGPIPE. -static void CRaiseSIGPIPE() { - int fds[2]; - - if (pipe(fds) == -1) { - perror("pipe"); - exit(EXIT_FAILURE); - } - // Close the reader end - close(fds[0]); - // Write to the writer end to provoke a SIGPIPE - if (write(fds[1], "some data", 9) != -1) { - fprintf(stderr, "write to a closed pipe succeeded\n"); - exit(EXIT_FAILURE); - } - close(fds[1]); -} -*/ import "C" import ( @@ -70,11 +46,5 @@ func TestSEGV() { func Noop() { } -// Raise SIGPIPE. -//export GoRaiseSIGPIPE -func GoRaiseSIGPIPE() { - C.CRaiseSIGPIPE() -} - func main() { } diff --git a/misc/cgo/testcarchive/src/libgo3/libgo3.go b/misc/cgo/testcarchive/src/libgo3/libgo3.go index 19fcc7f346..94e5d21c14 100644 --- a/misc/cgo/testcarchive/src/libgo3/libgo3.go +++ b/misc/cgo/testcarchive/src/libgo3/libgo3.go @@ -40,17 +40,5 @@ func SawSIGIO() C.int { } } -// ProvokeSIGPIPE provokes a kernel-initiated SIGPIPE -//export ProvokeSIGPIPE -func ProvokeSIGPIPE() { - r, w, err := os.Pipe() - if err != nil { - panic(err) - } - r.Close() - defer w.Close() - w.Write([]byte("some data")) -} - func main() { } diff --git a/misc/cgo/testcshared/main2.c b/misc/cgo/testcshared/main2.c index 9752006f79..6e8bf141ca 100644 --- a/misc/cgo/testcshared/main2.c +++ b/misc/cgo/testcshared/main2.c @@ -21,7 +21,7 @@ int main(void) { // The descriptor will be initialized in a thread, so we have to // give a chance to get opened. - for (i = 0; i < 100; i++) { + for (i = 0; i < 1000; i++) { n = read(fd, buf, sizeof buf); if (n >= 0) break; diff --git a/misc/cgo/testplugin/src/plugin1/plugin1.go b/misc/cgo/testplugin/src/plugin1/plugin1.go index 7a62242134..edcef2c77e 100644 --- a/misc/cgo/testplugin/src/plugin1/plugin1.go +++ b/misc/cgo/testplugin/src/plugin1/plugin1.go @@ -9,7 +9,10 @@ import "C" import "common" -func F() int { return 3 } +func F() int { + _ = make([]byte, 1<<21) // trigger stack unwind, Issue #18190. + return 3 +} func ReadCommonX() int { return common.X diff --git a/misc/cgo/testplugin/src/plugin2/plugin2.go b/misc/cgo/testplugin/src/plugin2/plugin2.go index 6c23a5e633..9c507fc365 100644 --- a/misc/cgo/testplugin/src/plugin2/plugin2.go +++ b/misc/cgo/testplugin/src/plugin2/plugin2.go @@ -4,12 +4,21 @@ package main -// // No C code required. +//#include <errno.h> +//#include <string.h> import "C" -import "common" +// #include +// void cfunc() {} // uses cgo_topofstack + +import ( + "common" + "strings" +) func init() { + _ = strings.NewReplacer() // trigger stack unwind, Issue #18190. + C.strerror(C.EIO) // uses cgo_topofstack common.X = 2 } diff --git a/misc/cgo/testsanitizers/test.bash b/misc/cgo/testsanitizers/test.bash index 01cce956b8..dfc6d3819a 100755 --- a/misc/cgo/testsanitizers/test.bash +++ b/misc/cgo/testsanitizers/test.bash @@ -24,8 +24,14 @@ msan=yes TMPDIR=${TMPDIR:-/tmp} echo 'int main() { return 0; }' > ${TMPDIR}/testsanitizers$$.c -if $CC -fsanitize=memory -c ${TMPDIR}/testsanitizers$$.c -o ${TMPDIR}/testsanitizers$$.o 2>&1 | grep "unrecognized" >& /dev/null; then - echo "skipping msan tests: -fsanitize=memory not supported" +if $CC -fsanitize=memory -o ${TMPDIR}/testsanitizers$$ ${TMPDIR}/testsanitizers$$.c 2>&1 | grep "unrecognized" >& /dev/null; then + echo "skipping msan tests: $CC -fsanitize=memory not supported" + msan=no +elif ! test -x ${TMPDIR}/testsanitizers$$; then + echo "skipping msan tests: $CC -fsanitize-memory did not generate an executable" + msan=no +elif ! ${TMPDIR}/testsanitizers$$ >/dev/null 2>&1; then + echo "skipping msan tests: $CC -fsanitize-memory generates broken executable" msan=no fi rm -f ${TMPDIR}/testsanitizers$$.* @@ -145,6 +151,7 @@ if test "$tsan" = "yes"; then testtsan tsan3.go testtsan tsan4.go testtsan tsan8.go + testtsan tsan9.go # These tests are only reliable using clang or GCC version 7 or later. # Otherwise runtime/cgo/libcgo.h can't tell whether TSAN is in use. diff --git a/misc/cgo/testsanitizers/tsan9.go b/misc/cgo/testsanitizers/tsan9.go new file mode 100644 index 0000000000..f166d8b495 --- /dev/null +++ b/misc/cgo/testsanitizers/tsan9.go @@ -0,0 +1,67 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +// This program failed when run under the C/C++ ThreadSanitizer. The +// TSAN library was not keeping track of whether signals should be +// delivered on the alternate signal stack, and the Go signal handler +// was not preserving callee-saved registers from C callers. + +/* +#cgo CFLAGS: -g -fsanitize=thread +#cgo LDFLAGS: -g -fsanitize=thread + +#include <stdlib.h> +#include <sys/time.h> + +void spin() { + size_t n; + struct timeval tvstart, tvnow; + int diff; + void *prev = NULL, *cur; + + gettimeofday(&tvstart, NULL); + for (n = 0; n < 1<<20; n++) { + cur = malloc(n); + free(prev); + prev = cur; + + gettimeofday(&tvnow, NULL); + diff = (tvnow.tv_sec - tvstart.tv_sec) * 1000 * 1000 + (tvnow.tv_usec - tvstart.tv_usec); + + // Profile frequency is 100Hz so we should definitely + // get a signal in 50 milliseconds. + if (diff > 50 * 1000) { + break; + } + } + + free(prev); +} +*/ +import "C" + +import ( + "io/ioutil" + "runtime/pprof" + "time" +) + +func goSpin() { + start := time.Now() + for n := 0; n < 1<<20; n++ { + _ = make([]byte, n) + if time.Since(start) > 50*time.Millisecond { + break + } + } +} + +func main() { + pprof.StartCPUProfile(ioutil.Discard) + go C.spin() + goSpin() + pprof.StopCPUProfile() +} diff --git a/misc/cgo/testshared/src/exe/exe.go b/misc/cgo/testshared/src/exe/exe.go index 31fbedd31c..433727112b 100644 --- a/misc/cgo/testshared/src/exe/exe.go +++ b/misc/cgo/testshared/src/exe/exe.go @@ -7,6 +7,18 @@ import ( "runtime" ) +// Having a function declared in the main package triggered +// golang.org/issue/18250 +func DeclaredInMain() { +} + +type C struct { +} + +func F() *C { + return nil +} + func main() { defer depBase.ImplementedInAsm() // This code below causes various go.itab.* symbols to be generated in @@ -15,4 +27,9 @@ func main() { reflect.TypeOf(os.Stdout).Elem() runtime.GC() depBase.V = depBase.F() + 1 + + var c *C + if reflect.TypeOf(F).Out(0) != reflect.TypeOf(c) { + panic("bad reflection results, see golang.org/issue/18252") + } } diff --git a/misc/ios/clangwrap.sh b/misc/ios/clangwrap.sh index 9cad49fe7d..9141c8c447 100755 --- a/misc/ios/clangwrap.sh +++ b/misc/ios/clangwrap.sh @@ -17,4 +17,4 @@ else exit 1 fi -exec $CLANG -arch $CLANGARCH -isysroot $SDK_PATH "$@" +exec $CLANG -arch $CLANGARCH -isysroot $SDK_PATH -mios-version-min=6.0 "$@" diff --git a/src/archive/zip/reader.go b/src/archive/zip/reader.go index 9bbc9a9745..f6c3ead3be 100644 --- a/src/archive/zip/reader.go +++ b/src/archive/zip/reader.go @@ -13,7 +13,6 @@ import ( "hash/crc32" "io" "os" - "time" ) var ( @@ -290,16 +289,13 @@ func readDirectoryHeader(f *File, r io.Reader) error { // Other zip authors might not even follow the basic format, // and we'll just ignore the Extra content in that case. b := readBuf(f.Extra) - - Extras: for len(b) >= 4 { // need at least tag and size tag := b.uint16() size := b.uint16() if int(size) > len(b) { break } - switch tag { - case zip64ExtraId: + if tag == zip64ExtraId { // update directory values from the zip64 extra block. // They should only be consulted if the sizes read earlier // are maxed out. @@ -327,42 +323,7 @@ func readDirectoryHeader(f *File, r io.Reader) error { } f.headerOffset = int64(eb.uint64()) } - break Extras - - case ntfsExtraId: - if size == 32 { - eb := readBuf(b[:size]) - eb.uint32() // reserved - eb.uint16() // tag1 - size1 := eb.uint16() - if size1 == 24 { - sub := readBuf(eb[:size1]) - lo := sub.uint32() - hi := sub.uint32() - tick := (uint64(uint64(lo)|uint64(hi)<<32) - 116444736000000000) / 10000000 - f.SetModTime(time.Unix(int64(tick), 0)) - } - } - break Extras - - case unixExtraId: - if size >= 12 { - eb := readBuf(b[:size]) - eb.uint32() // AcTime - epoch := eb.uint32() // ModTime - f.SetModTime(time.Unix(int64(epoch), 0)) - break Extras - } - case exttsExtraId: - if size >= 3 { - eb := readBuf(b[:size]) - flags := eb.uint8() // Flags - epoch := eb.uint32() // AcTime/ModTime/CrTime - if flags&1 != 0 { - f.SetModTime(time.Unix(int64(epoch), 0)) - } - break Extras - } + break } b = b[size:] } @@ -547,12 +508,6 @@ func findSignatureInBlock(b []byte) int { type readBuf []byte -func (b *readBuf) uint8() uint8 { - v := uint8((*b)[0]) - *b = (*b)[1:] - return v -} - func (b *readBuf) uint16() uint16 { v := binary.LittleEndian.Uint16(*b) *b = (*b)[2:] diff --git a/src/archive/zip/reader_test.go b/src/archive/zip/reader_test.go index 576a1697a4..dfaae78436 100644 --- a/src/archive/zip/reader_test.go +++ b/src/archive/zip/reader_test.go @@ -65,13 +65,13 @@ var tests = []ZipTest{ { Name: "test.txt", Content: []byte("This is a test text file.\n"), - Mtime: "09-05-10 02:12:00", + Mtime: "09-05-10 12:12:02", Mode: 0644, }, { Name: "gophercolor16x16.png", File: "gophercolor16x16.png", - Mtime: "09-05-10 05:52:58", + Mtime: "09-05-10 15:52:58", Mode: 0644, }, }, @@ -83,13 +83,13 @@ var tests = []ZipTest{ { Name: "test.txt", Content: []byte("This is a test text file.\n"), - Mtime: "09-05-10 02:12:00", + Mtime: "09-05-10 12:12:02", Mode: 0644, }, { Name: "gophercolor16x16.png", File: "gophercolor16x16.png", - Mtime: "09-05-10 05:52:58", + Mtime: "09-05-10 15:52:58", Mode: 0644, }, }, @@ -145,17 +145,6 @@ var tests = []ZipTest{ File: crossPlatform, }, { - Name: "extra-timestamp.zip", - File: []ZipTestFile{ - { - Name: "hello.txt", - Content: []byte(""), - Mtime: "01-06-16 12:25:56", - Mode: 0666, - }, - }, - }, - { // created by Go, before we wrote the "optional" data // descriptor signatures (which are required by OS X) Name: "go-no-datadesc-sig.zip", @@ -163,13 +152,13 @@ var tests = []ZipTest{ { Name: "foo.txt", Content: []byte("foo\n"), - Mtime: "03-09-12 00:59:10", + Mtime: "03-08-12 16:59:10", Mode: 0644, }, { Name: "bar.txt", Content: []byte("bar\n"), - Mtime: "03-09-12 00:59:12", + Mtime: "03-08-12 16:59:12", Mode: 0644, }, }, @@ -216,13 +205,13 @@ var tests = []ZipTest{ { Name: "foo.txt", Content: []byte("foo\n"), - Mtime: "03-09-12 00:59:10", + Mtime: "03-08-12 16:59:10", Mode: 0644, }, { Name: "bar.txt", Content: []byte("bar\n"), - Mtime: "03-09-12 00:59:12", + Mtime: "03-08-12 16:59:12", Mode: 0644, }, }, @@ -236,14 +225,14 @@ var tests = []ZipTest{ { Name: "foo.txt", Content: []byte("foo\n"), - Mtime: "03-09-12 00:59:10", + Mtime: "03-08-12 16:59:10", Mode: 0644, ContentErr: ErrChecksum, }, { Name: "bar.txt", Content: []byte("bar\n"), - Mtime: "03-09-12 00:59:12", + Mtime: "03-08-12 16:59:12", Mode: 0644, }, }, diff --git a/src/archive/zip/struct.go b/src/archive/zip/struct.go index 287571ed3a..e92d02f8a2 100644 --- a/src/archive/zip/struct.go +++ b/src/archive/zip/struct.go @@ -63,9 +63,6 @@ const ( // extra header id's zip64ExtraId = 0x0001 // zip64 Extended Information Extra Field - ntfsExtraId = 0x000a // NTFS Extra Field - unixExtraId = 0x000d // UNIX Extra Field - exttsExtraId = 0x5455 // Extended Timestamp Extra Field ) // FileHeader describes a file within a zip file. diff --git a/src/archive/zip/testdata/extra-timestamp.zip b/src/archive/zip/testdata/extra-timestamp.zip Binary files differdeleted file mode 100644 index 819e22cb68..0000000000 --- a/src/archive/zip/testdata/extra-timestamp.zip +++ /dev/null diff --git a/src/archive/zip/writer.go b/src/archive/zip/writer.go index ea4559e698..8940e25560 100644 --- a/src/archive/zip/writer.go +++ b/src/archive/zip/writer.go @@ -103,18 +103,6 @@ func (w *Writer) Close() error { b.uint32(h.UncompressedSize) } - // use Extended Timestamp Extra Field. - if h.ModifiedTime != 0 || h.ModifiedDate != 0 { - mt := uint32(h.ModTime().Unix()) - var mbuf [9]byte // 2x uint16 + uint8 + uint32 - eb := writeBuf(mbuf[:]) - eb.uint16(exttsExtraId) - eb.uint16(5) // size = uint8 + uint32 - eb.uint8(1) // flags = modtime - eb.uint32(mt) // ModTime - h.Extra = append(h.Extra, mbuf[:]...) - } - b.uint16(uint16(len(h.Name))) b.uint16(uint16(len(h.Extra))) b.uint16(uint16(len(h.Comment))) @@ -397,11 +385,6 @@ func (w nopCloser) Close() error { type writeBuf []byte -func (b *writeBuf) uint8(v uint8) { - (*b)[0] = v - *b = (*b)[1:] -} - func (b *writeBuf) uint16(v uint16) { binary.LittleEndian.PutUint16(*b, v) *b = (*b)[2:] diff --git a/src/archive/zip/writer_test.go b/src/archive/zip/writer_test.go index f20daa0e3d..86841c755f 100644 --- a/src/archive/zip/writer_test.go +++ b/src/archive/zip/writer_test.go @@ -11,7 +11,6 @@ import ( "math/rand" "os" "testing" - "time" ) // TODO(adg): a more sophisticated test suite @@ -21,7 +20,6 @@ type WriteTest struct { Data []byte Method uint16 Mode os.FileMode - Mtime string } var writeTests = []WriteTest{ @@ -30,35 +28,30 @@ var writeTests = []WriteTest{ Data: []byte("Rabbits, guinea pigs, gophers, marsupial rats, and quolls."), Method: Store, Mode: 0666, - Mtime: "02-01-08 00:01:02", }, { Name: "bar", Data: nil, // large data set in the test Method: Deflate, Mode: 0644, - Mtime: "03-02-08 01:02:03", }, { Name: "setuid", Data: []byte("setuid file"), Method: Deflate, Mode: 0755 | os.ModeSetuid, - Mtime: "04-03-08 02:03:04", }, { Name: "setgid", Data: []byte("setgid file"), Method: Deflate, Mode: 0755 | os.ModeSetgid, - Mtime: "05-04-08 03:04:04", }, { Name: "symlink", Data: []byte("../link/target"), Method: Deflate, Mode: 0755 | os.ModeSymlink, - Mtime: "03-02-08 11:22:33", }, } @@ -155,11 +148,6 @@ func testCreate(t *testing.T, w *Writer, wt *WriteTest) { if wt.Mode != 0 { header.SetMode(wt.Mode) } - mtime, err := time.Parse("01-02-06 15:04:05", wt.Mtime) - if err != nil { - t.Fatal("time.Parse:", err) - } - header.SetModTime(mtime) f, err := w.CreateHeader(header) if err != nil { t.Fatal(err) @@ -190,21 +178,6 @@ func testReadFile(t *testing.T, f *File, wt *WriteTest) { if !bytes.Equal(b, wt.Data) { t.Errorf("File contents %q, want %q", b, wt.Data) } - - mtime, err := time.Parse("01-02-06 15:04:05", wt.Mtime) - if err != nil { - t.Fatal("time.Parse:", err) - } - - diff := mtime.Sub(f.ModTime()) - if diff < 0 { - diff = -diff - } - - // allow several time span - if diff > 5*time.Second { - t.Errorf("File modtime %v, want %v", mtime, f.ModTime()) - } } func BenchmarkCompressedZipGarbage(b *testing.B) { diff --git a/src/archive/zip/zip_test.go b/src/archive/zip/zip_test.go index 8801e90413..57edb2cabf 100644 --- a/src/archive/zip/zip_test.go +++ b/src/archive/zip/zip_test.go @@ -15,7 +15,6 @@ import ( "internal/testenv" "io" "io/ioutil" - "reflect" "sort" "strings" "testing" @@ -114,44 +113,6 @@ func TestFileHeaderRoundTrip64(t *testing.T) { testHeaderRoundTrip(fh, uint32max, fh.UncompressedSize64, t) } -func TestZeroFileRoundTrip(t *testing.T) { - var b bytes.Buffer - w := NewWriter(&b) - if _, err := w.Create(""); err != nil { - t.Fatal(err) - } - if err := w.Close(); err != nil { - t.Fatal(err) - } - r, err := NewReader(bytes.NewReader(b.Bytes()), int64(b.Len())) - if err != nil { - t.Fatal(err) - } - - // Verify that fields that should reasonably be the zero value stays - // as the zero value. - var want FileHeader - if len(r.File) != 1 { - t.Fatalf("len(r.File) = %d, want 1", len(r.File)) - } - fh := r.File[0].FileHeader - got := FileHeader{ - Name: fh.Name, - ModifiedTime: fh.ModifiedTime, - ModifiedDate: fh.ModifiedDate, - UncompressedSize: fh.UncompressedSize, - UncompressedSize64: fh.UncompressedSize64, - ExternalAttrs: fh.ExternalAttrs, - Comment: fh.Comment, - } - if len(fh.Extra) > 0 { - got.Extra = fh.Extra - } - if !reflect.DeepEqual(got, want) { - t.Errorf("FileHeader mismatch:\ngot %#v\nwant %#v", got, want) - } -} - type repeatedByte struct { off int64 b byte diff --git a/src/bytes/buffer.go b/src/bytes/buffer.go index 2ee3d738ef..196419dc3d 100644 --- a/src/bytes/buffer.go +++ b/src/bytes/buffer.go @@ -23,7 +23,7 @@ type Buffer struct { // The readOp constants describe the last action performed on // the buffer, so that UnreadRune and UnreadByte can check for -// invalid usage. opReadRuneX constants are choosen such that +// invalid usage. opReadRuneX constants are chosen such that // converted to int they correspond to the rune size that was read. type readOp int diff --git a/src/cmd/cgo/gcc.go b/src/cmd/cgo/gcc.go index 670a73f546..5ea2d941ca 100644 --- a/src/cmd/cgo/gcc.go +++ b/src/cmd/cgo/gcc.go @@ -1210,6 +1210,8 @@ func (p *Package) gccMachine() []string { return []string{"-m64"} case "mips64", "mips64le": return []string{"-mabi=64"} + case "mips", "mipsle": + return []string{"-mabi=32"} } return nil } @@ -1727,6 +1729,15 @@ func (c *typeConv) Type(dtype dwarf.Type, pos token.Pos) *Type { if _, ok := base(dt.Type).(*dwarf.VoidType); ok { t.Go = c.goVoidPtr t.C.Set("void*") + dq := dt.Type + for { + if d, ok := dq.(*dwarf.QualType); ok { + t.C.Set(d.Qual + " " + t.C.String()) + dq = d.Type + } else { + break + } + } break } diff --git a/src/cmd/compile/internal/gc/asm_test.go b/src/cmd/compile/internal/gc/asm_test.go index 2e5d7e7488..db800aba20 100644 --- a/src/cmd/compile/internal/gc/asm_test.go +++ b/src/cmd/compile/internal/gc/asm_test.go @@ -175,6 +175,14 @@ func f(b []byte, i int) uint64 { }, {"amd64", "linux", ` import "encoding/binary" +func f(b []byte, v uint64) { + binary.BigEndian.PutUint64(b, v) +} +`, + []string{"\tBSWAPQ\t"}, + }, + {"amd64", "linux", ` +import "encoding/binary" func f(b []byte) uint32 { return binary.BigEndian.Uint32(b) } @@ -189,6 +197,14 @@ func f(b []byte, i int) uint32 { `, []string{"\tBSWAPL\t"}, }, + {"amd64", "linux", ` +import "encoding/binary" +func f(b []byte, v uint32) { + binary.BigEndian.PutUint32(b, v) +} +`, + []string{"\tBSWAPL\t"}, + }, {"386", "linux", ` import "encoding/binary" func f(b []byte) uint32 { @@ -205,6 +221,19 @@ func f(b []byte, i int) uint32 { `, []string{"\tMOVL\t\\(.*\\)\\(.*\\*1\\),"}, }, + + // Structure zeroing. See issue #18370. + {"amd64", "linux", ` +type T struct { + a, b, c int +} +func f(t *T) { + *t = T{} +} +`, + []string{"\tMOVQ\t\\$0, \\(.*\\)", "\tMOVQ\t\\$0, 8\\(.*\\)", "\tMOVQ\t\\$0, 16\\(.*\\)"}, + }, + // TODO: add a test for *t = T{3,4,5} when we fix that. } // mergeEnvLists merges the two environment lists such that diff --git a/src/cmd/compile/internal/gc/bexport.go b/src/cmd/compile/internal/gc/bexport.go index 1da5b699a4..ffc5419708 100644 --- a/src/cmd/compile/internal/gc/bexport.go +++ b/src/cmd/compile/internal/gc/bexport.go @@ -1000,7 +1000,7 @@ func parName(f *Field, numbered bool) string { Fatalf("invalid symbol name: %s", name) } - // Functions that can be inlined use numbered parameters so we can distingish them + // Functions that can be inlined use numbered parameters so we can distinguish them // from other names in their context after inlining (i.e., the parameter numbering // is a form of parameter rewriting). See issue 4326 for an example and test case. if forceObjFileStability || numbered { diff --git a/src/cmd/compile/internal/gc/builtin.go b/src/cmd/compile/internal/gc/builtin.go index e02e2feb01..71b323f8a1 100644 --- a/src/cmd/compile/internal/gc/builtin.go +++ b/src/cmd/compile/internal/gc/builtin.go @@ -15,6 +15,7 @@ var runtimeDecls = [...]struct { {"panicwrap", funcTag, 7}, {"gopanic", funcTag, 9}, {"gorecover", funcTag, 12}, + {"goschedguarded", funcTag, 5}, {"printbool", funcTag, 14}, {"printfloat", funcTag, 16}, {"printint", funcTag, 18}, diff --git a/src/cmd/compile/internal/gc/builtin/runtime.go b/src/cmd/compile/internal/gc/builtin/runtime.go index 98e25fefb8..69511155f4 100644 --- a/src/cmd/compile/internal/gc/builtin/runtime.go +++ b/src/cmd/compile/internal/gc/builtin/runtime.go @@ -21,6 +21,7 @@ func panicwrap(string, string, string) func gopanic(interface{}) func gorecover(*int32) interface{} +func goschedguarded() func printbool(bool) func printfloat(float64) diff --git a/src/cmd/compile/internal/gc/inl.go b/src/cmd/compile/internal/gc/inl.go index d8f1f24536..7b2fcf89ba 100644 --- a/src/cmd/compile/internal/gc/inl.go +++ b/src/cmd/compile/internal/gc/inl.go @@ -107,6 +107,12 @@ func caninl(fn *Node) { return } + // If marked "go:cgo_unsafe_args", don't inline + if fn.Func.Pragma&CgoUnsafeArgs != 0 { + reason = "marked go:cgo_unsafe_args" + return + } + // If fn has no body (is defined outside of Go), cannot inline it. if fn.Nbody.Len() == 0 { reason = "no function body" diff --git a/src/cmd/compile/internal/gc/noder.go b/src/cmd/compile/internal/gc/noder.go index f9de48a08b..ca99adea27 100644 --- a/src/cmd/compile/internal/gc/noder.go +++ b/src/cmd/compile/internal/gc/noder.go @@ -1022,13 +1022,14 @@ func (p *noder) error(err error) { func (p *noder) pragma(pos, line int, text string) syntax.Pragma { switch { case strings.HasPrefix(text, "line "): - i := strings.IndexByte(text, ':') + // Want to use LastIndexByte below but it's not defined in Go1.4 and bootstrap fails. + i := strings.LastIndex(text, ":") // look from right (Windows filenames may contain ':') if i < 0 { break } n, err := strconv.Atoi(text[i+1:]) if err != nil { - // todo: make this an error instead? it is almost certainly a bug. + // TODO: make this an error instead? it is almost certainly a bug. break } if n > 1e8 { @@ -1054,6 +1055,7 @@ func (p *noder) pragma(pos, line int, text string) syntax.Pragma { lookup(f[1]).Linkname = f[2] case strings.HasPrefix(text, "go:cgo_"): + lineno = p.baseline + int32(line) - 1 // pragcgo may call yyerror pragcgobuf += pragcgo(text) fallthrough // because of //go:cgo_unsafe_args default: @@ -1061,6 +1063,7 @@ func (p *noder) pragma(pos, line int, text string) syntax.Pragma { if i := strings.Index(text, " "); i >= 0 { verb = verb[:i] } + lineno = p.baseline + int32(line) - 1 // pragmaValue may call yyerror return syntax.Pragma(pragmaValue(verb)) } diff --git a/src/cmd/compile/internal/gc/pgen.go b/src/cmd/compile/internal/gc/pgen.go index acea790498..643ba79d63 100644 --- a/src/cmd/compile/internal/gc/pgen.go +++ b/src/cmd/compile/internal/gc/pgen.go @@ -380,6 +380,9 @@ func compile(fn *Node) { if fn.Func.Wrapper { ptxt.From3.Offset |= obj.WRAPPER } + if fn.Func.NoFramePointer { + ptxt.From3.Offset |= obj.NOFRAME + } if fn.Func.Needctxt { ptxt.From3.Offset |= obj.NEEDCTXT } diff --git a/src/cmd/compile/internal/gc/reflect.go b/src/cmd/compile/internal/gc/reflect.go index 4f9d92ed8a..61ac67c0bc 100644 --- a/src/cmd/compile/internal/gc/reflect.go +++ b/src/cmd/compile/internal/gc/reflect.go @@ -998,7 +998,6 @@ func itabname(t, itype *Type) *Node { Fatalf("itabname(%v, %v)", t, itype) } s := Pkglookup(t.tconv(FmtLeft)+","+itype.tconv(FmtLeft), itabpkg) - Linksym(s).Set(obj.AttrLocal, true) if s.Def == nil { n := newname(s) n.Type = Types[TUINT8] @@ -1411,15 +1410,15 @@ func dumptypestructs() { // } o := dsymptr(i.sym, 0, dtypesym(i.itype), 0) o = dsymptr(i.sym, o, dtypesym(i.t), 0) - o += Widthptr + 8 // skip link/bad/unused fields + o += Widthptr + 8 // skip link/bad/inhash fields o += len(imethods(i.itype)) * Widthptr // skip fun method pointers // at runtime the itab will contain pointers to types, other itabs and // method functions. None are allocated on heap, so we can use obj.NOPTR. - ggloblsym(i.sym, int32(o), int16(obj.DUPOK|obj.NOPTR|obj.LOCAL)) + ggloblsym(i.sym, int32(o), int16(obj.DUPOK|obj.NOPTR)) ilink := Pkglookup(i.t.tconv(FmtLeft)+","+i.itype.tconv(FmtLeft), itablinkpkg) dsymptr(ilink, 0, i.sym, 0) - ggloblsym(ilink, int32(Widthptr), int16(obj.DUPOK|obj.RODATA|obj.LOCAL)) + ggloblsym(ilink, int32(Widthptr), int16(obj.DUPOK|obj.RODATA)) } // process ptabs diff --git a/src/cmd/compile/internal/gc/sinit.go b/src/cmd/compile/internal/gc/sinit.go index 1192f3fac9..6b3c426ca3 100644 --- a/src/cmd/compile/internal/gc/sinit.go +++ b/src/cmd/compile/internal/gc/sinit.go @@ -295,7 +295,9 @@ func staticcopy(l *Node, r *Node, out *[]*Node) bool { if staticcopy(l, r, out) { return true } - *out = append(*out, nod(OAS, l, r)) + // We may have skipped past one or more OCONVNOPs, so + // use conv to ensure r is assignable to l (#13263). + *out = append(*out, nod(OAS, l, conv(r, l.Type))) return true case OLITERAL: @@ -1076,6 +1078,8 @@ func anylit(n *Node, var_ *Node, init *Nodes) { var r *Node if n.Right != nil { + // n.Right is stack temporary used as backing store. + init.Append(nod(OAS, n.Right, nil)) // zero backing store, just in case (#18410) r = nod(OADDR, n.Right, nil) r = typecheck(r, Erv) } else { @@ -1193,7 +1197,7 @@ func getlit(lit *Node) int { return -1 } -// stataddr sets nam to the static address of n and reports whether it succeeeded. +// stataddr sets nam to the static address of n and reports whether it succeeded. func stataddr(nam *Node, n *Node) bool { if n == nil { return false diff --git a/src/cmd/compile/internal/gc/ssa.go b/src/cmd/compile/internal/gc/ssa.go index 55ee3c01dc..bf483f8416 100644 --- a/src/cmd/compile/internal/gc/ssa.go +++ b/src/cmd/compile/internal/gc/ssa.go @@ -64,6 +64,9 @@ func buildssa(fn *Node) *ssa.Func { s.config = initssa() s.f = s.config.NewFunc() s.f.Name = name + if fn.Func.Pragma&Nosplit != 0 { + s.f.NoSplit = true + } s.exitCode = fn.Func.Exit s.panics = map[funcLine]*ssa.Block{} s.config.DebugTest = s.config.DebugHashMatch("GOSSAHASH", name) diff --git a/src/cmd/compile/internal/gc/subr.go b/src/cmd/compile/internal/gc/subr.go index a53ba1fffc..9b9a3f1210 100644 --- a/src/cmd/compile/internal/gc/subr.go +++ b/src/cmd/compile/internal/gc/subr.go @@ -1810,6 +1810,8 @@ func genwrapper(rcvr *Type, method *Field, newnam *Sym, iface int) { n := nod(ORETJMP, nil, nil) n.Left = newname(methodsym(method.Sym, methodrcvr, 0)) fn.Nbody.Append(n) + // When tail-calling, we can't use a frame pointer. + fn.Func.NoFramePointer = true } else { fn.Func.Wrapper = true // ignore frame for panic+recover matching call := nod(OCALL, dot, nil) diff --git a/src/cmd/compile/internal/gc/syntax.go b/src/cmd/compile/internal/gc/syntax.go index 8b06d3aba8..8848bb5955 100644 --- a/src/cmd/compile/internal/gc/syntax.go +++ b/src/cmd/compile/internal/gc/syntax.go @@ -317,6 +317,7 @@ type Func struct { Needctxt bool // function uses context register (has closure variables) ReflectMethod bool // function calls reflect.Type.Method or MethodByName IsHiddenClosure bool + NoFramePointer bool // Must not use a frame pointer for this function } type Op uint8 diff --git a/src/cmd/compile/internal/gc/testdata/array.go b/src/cmd/compile/internal/gc/testdata/array.go index 0334339d43..6be8d9155b 100644 --- a/src/cmd/compile/internal/gc/testdata/array.go +++ b/src/cmd/compile/internal/gc/testdata/array.go @@ -106,7 +106,7 @@ func testSliceSetElement() { func testSlicePanic1() { defer func() { if r := recover(); r != nil { - println("paniced as expected") + println("panicked as expected") } }() @@ -119,7 +119,7 @@ func testSlicePanic1() { func testSlicePanic2() { defer func() { if r := recover(); r != nil { - println("paniced as expected") + println("panicked as expected") } }() diff --git a/src/cmd/compile/internal/gc/testdata/string.go b/src/cmd/compile/internal/gc/testdata/string.go index 897e874ee5..03053a6134 100644 --- a/src/cmd/compile/internal/gc/testdata/string.go +++ b/src/cmd/compile/internal/gc/testdata/string.go @@ -73,7 +73,7 @@ func testStructSlice() { func testStringSlicePanic() { defer func() { if r := recover(); r != nil { - println("paniced as expected") + println("panicked as expected") } }() @@ -148,7 +148,7 @@ func testInt64Index() { func testInt64IndexPanic() { defer func() { if r := recover(); r != nil { - println("paniced as expected") + println("panicked as expected") } }() @@ -161,7 +161,7 @@ func testInt64IndexPanic() { func testInt64SlicePanic() { defer func() { if r := recover(); r != nil { - println("paniced as expected") + println("panicked as expected") } }() diff --git a/src/cmd/compile/internal/gc/type.go b/src/cmd/compile/internal/gc/type.go index 29048f1a19..5f04f680c7 100644 --- a/src/cmd/compile/internal/gc/type.go +++ b/src/cmd/compile/internal/gc/type.go @@ -213,7 +213,7 @@ func (t *Type) FuncType() *FuncType { return t.Extra.(*FuncType) } -// InterMethType contains Type fields specific to interface method psuedo-types. +// InterMethType contains Type fields specific to interface method pseudo-types. type InterMethType struct { Nname *Node } diff --git a/src/cmd/compile/internal/gc/typecheck.go b/src/cmd/compile/internal/gc/typecheck.go index 23c60fa0d0..5ec1c9e2f2 100644 --- a/src/cmd/compile/internal/gc/typecheck.go +++ b/src/cmd/compile/internal/gc/typecheck.go @@ -859,7 +859,7 @@ OpSwitch: } if n.Type.Etype != TFUNC || !n.IsMethod() { - yyerror("type %v has no method %S", n.Left.Type, n.Right.Sym) + yyerror("type %v has no method %S", n.Left.Type, n.Sym) n.Type = nil return n } diff --git a/src/cmd/compile/internal/gc/walk.go b/src/cmd/compile/internal/gc/walk.go index 8248d503b3..efe2016e46 100644 --- a/src/cmd/compile/internal/gc/walk.go +++ b/src/cmd/compile/internal/gc/walk.go @@ -694,6 +694,10 @@ opswitch: break } + if !instrumenting && iszero(n.Right) && !needwritebarrier(n.Left, n.Right) { + break + } + switch n.Right.Op { default: n.Right = walkexpr(n.Right, init) diff --git a/src/cmd/compile/internal/ssa/checkbce.go b/src/cmd/compile/internal/ssa/checkbce.go index 820ea6e809..3b15d5a125 100644 --- a/src/cmd/compile/internal/ssa/checkbce.go +++ b/src/cmd/compile/internal/ssa/checkbce.go @@ -6,7 +6,7 @@ package ssa // checkbce prints all bounds checks that are present in the function. // Useful to find regressions. checkbce is only activated when with -// corresponsing debug options, so it's off by default. +// corresponding debug options, so it's off by default. // See test/checkbce.go func checkbce(f *Func) { if f.pass.debug <= 0 { diff --git a/src/cmd/compile/internal/ssa/compile.go b/src/cmd/compile/internal/ssa/compile.go index b9ec7eb6b7..5b461bac48 100644 --- a/src/cmd/compile/internal/ssa/compile.go +++ b/src/cmd/compile/internal/ssa/compile.go @@ -5,6 +5,7 @@ package ssa import ( + "cmd/internal/obj" "fmt" "log" "os" @@ -349,6 +350,8 @@ var passes = [...]pass{ {name: "writebarrier", fn: writebarrier, required: true}, // expand write barrier ops {name: "fuse", fn: fuse}, {name: "dse", fn: dse}, + {name: "insert resched checks", fn: insertLoopReschedChecks, + disabled: obj.Preemptibleloops_enabled == 0}, // insert resched checks in loops. {name: "tighten", fn: tighten}, // move values closer to their uses {name: "lower", fn: lower, required: true}, {name: "lowered cse", fn: cse}, @@ -378,7 +381,13 @@ type constraint struct { } var passOrder = [...]constraint{ - // prove reliese on common-subexpression elimination for maximum benefits. + // "insert resched checks" uses mem, better to clean out stores first. + {"dse", "insert resched checks"}, + // insert resched checks adds new blocks containing generic instructions + {"insert resched checks", "lower"}, + {"insert resched checks", "tighten"}, + + // prove relies on common-subexpression elimination for maximum benefits. {"generic cse", "prove"}, // deadcode after prove to eliminate all new dead blocks. {"prove", "generic deadcode"}, diff --git a/src/cmd/compile/internal/ssa/config.go b/src/cmd/compile/internal/ssa/config.go index 919386e889..4931da8d07 100644 --- a/src/cmd/compile/internal/ssa/config.go +++ b/src/cmd/compile/internal/ssa/config.go @@ -93,7 +93,7 @@ type Logger interface { // Warnl writes compiler messages in the form expected by "errorcheck" tests Warnl(line int32, fmt_ string, args ...interface{}) - // Fowards the Debug flags from gc + // Forwards the Debug flags from gc Debug_checknil() bool Debug_wb() bool } diff --git a/src/cmd/compile/internal/ssa/export_test.go b/src/cmd/compile/internal/ssa/export_test.go index 010c4d7680..3a9357dfae 100644 --- a/src/cmd/compile/internal/ssa/export_test.go +++ b/src/cmd/compile/internal/ssa/export_test.go @@ -6,6 +6,7 @@ package ssa import ( "cmd/internal/obj" + "cmd/internal/obj/x86" "testing" ) @@ -16,7 +17,7 @@ var Deadcode = deadcode var Copyelim = copyelim func testConfig(t testing.TB) *Config { - testCtxt := &obj.Link{} + testCtxt := &obj.Link{Arch: &x86.Linkamd64} return NewConfig("amd64", DummyFrontend{t}, testCtxt, true) } @@ -67,7 +68,7 @@ func (DummyFrontend) Line(line int32) string { func (DummyFrontend) AllocFrame(f *Func) { } func (DummyFrontend) Syslook(s string) interface{} { - return nil + return DummySym(s) } func (d DummyFrontend) Logf(msg string, args ...interface{}) { d.t.Logf(msg, args...) } @@ -98,3 +99,7 @@ func (d DummyFrontend) CanSSA(t Type) bool { // There are no un-SSAable types in dummy land. return true } + +type DummySym string + +func (s DummySym) String() string { return string(s) } diff --git a/src/cmd/compile/internal/ssa/func.go b/src/cmd/compile/internal/ssa/func.go index 7b2097bcae..df29aa3606 100644 --- a/src/cmd/compile/internal/ssa/func.go +++ b/src/cmd/compile/internal/ssa/func.go @@ -24,6 +24,7 @@ type Func struct { vid idAlloc // value ID allocator scheduled bool // Values in Blocks are in final order + NoSplit bool // true if function is marked as nosplit. Used by schedule check pass. // when register allocation is done, maps value ids to locations RegAlloc []Location diff --git a/src/cmd/compile/internal/ssa/gen/ARM64Ops.go b/src/cmd/compile/internal/ssa/gen/ARM64Ops.go index dce61e3b8a..e8d5be2582 100644 --- a/src/cmd/compile/internal/ssa/gen/ARM64Ops.go +++ b/src/cmd/compile/internal/ssa/gen/ARM64Ops.go @@ -12,7 +12,7 @@ import "strings" // - Integer types live in the low portion of registers. Upper portions are junk. // - Boolean types use the low-order byte of a register. 0=false, 1=true. // Upper bytes are junk. -// - *const instructions may use a constant larger than the instuction can encode. +// - *const instructions may use a constant larger than the instruction can encode. // In this case the assembler expands to multiple instructions and uses tmp // register (R27). diff --git a/src/cmd/compile/internal/ssa/gen/ARMOps.go b/src/cmd/compile/internal/ssa/gen/ARMOps.go index 5bf3c0091a..e296d0600d 100644 --- a/src/cmd/compile/internal/ssa/gen/ARMOps.go +++ b/src/cmd/compile/internal/ssa/gen/ARMOps.go @@ -12,7 +12,7 @@ import "strings" // - Integer types live in the low portion of registers. Upper portions are junk. // - Boolean types use the low-order byte of a register. 0=false, 1=true. // Upper bytes are junk. -// - *const instructions may use a constant larger than the instuction can encode. +// - *const instructions may use a constant larger than the instruction can encode. // In this case the assembler expands to multiple instructions and uses tmp // register (R11). diff --git a/src/cmd/compile/internal/ssa/gen/MIPS64Ops.go b/src/cmd/compile/internal/ssa/gen/MIPS64Ops.go index d7d7fece28..020d6930d7 100644 --- a/src/cmd/compile/internal/ssa/gen/MIPS64Ops.go +++ b/src/cmd/compile/internal/ssa/gen/MIPS64Ops.go @@ -12,7 +12,7 @@ import "strings" // - Integer types live in the low portion of registers. Upper portions are junk. // - Boolean types use the low-order byte of a register. 0=false, 1=true. // Upper bytes are junk. -// - *const instructions may use a constant larger than the instuction can encode. +// - *const instructions may use a constant larger than the instruction can encode. // In this case the assembler expands to multiple instructions and uses tmp // register (R23). diff --git a/src/cmd/compile/internal/ssa/gen/MIPSOps.go b/src/cmd/compile/internal/ssa/gen/MIPSOps.go index c803c49519..78b961ffb2 100644 --- a/src/cmd/compile/internal/ssa/gen/MIPSOps.go +++ b/src/cmd/compile/internal/ssa/gen/MIPSOps.go @@ -13,7 +13,7 @@ import "strings" // - Boolean types use the low-order byte of a register. 0=false, 1=true. // Upper bytes are junk. // - Unused portions of AuxInt are filled by sign-extending the used portion. -// - *const instructions may use a constant larger than the instuction can encode. +// - *const instructions may use a constant larger than the instruction can encode. // In this case the assembler expands to multiple instructions and uses tmp // register (R23). diff --git a/src/cmd/compile/internal/ssa/gen/PPC64Ops.go b/src/cmd/compile/internal/ssa/gen/PPC64Ops.go index d7a1363c0c..003479774a 100644 --- a/src/cmd/compile/internal/ssa/gen/PPC64Ops.go +++ b/src/cmd/compile/internal/ssa/gen/PPC64Ops.go @@ -12,7 +12,7 @@ import "strings" // - Less-than-64-bit integer types live in the low portion of registers. // For now, the upper portion is junk; sign/zero-extension might be optimized in the future, but not yet. // - Boolean types are zero or 1; stored in a byte, but loaded with AMOVBZ so the upper bytes of a register are zero. -// - *const instructions may use a constant larger than the instuction can encode. +// - *const instructions may use a constant larger than the instruction can encode. // In this case the assembler expands to multiple instructions and uses tmp // register (R31). diff --git a/src/cmd/compile/internal/ssa/loopreschedchecks.go b/src/cmd/compile/internal/ssa/loopreschedchecks.go new file mode 100644 index 0000000000..8f8055e302 --- /dev/null +++ b/src/cmd/compile/internal/ssa/loopreschedchecks.go @@ -0,0 +1,517 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ssa + +import "fmt" + +// an edgeMemCtr records a backedge, together with the memory and +// counter phi functions at the target of the backedge that must +// be updated when a rescheduling check replaces the backedge. +type edgeMemCtr struct { + e Edge + m *Value // phi for memory at dest of e + c *Value // phi for counter at dest of e +} + +// a rewriteTarget is a a value-argindex pair indicating +// where a rewrite is applied. Note that this is for values, +// not for block controls, because block controls are not targets +// for the rewrites performed in inserting rescheduling checks. +type rewriteTarget struct { + v *Value + i int +} + +type rewrite struct { + before, after *Value // before is the expected value before rewrite, after is the new value installed. + rewrites []rewriteTarget // all the targets for this rewrite. +} + +func (r *rewrite) String() string { + s := "\n\tbefore=" + r.before.String() + ", after=" + r.after.String() + for _, rw := range r.rewrites { + s += ", (i=" + fmt.Sprint(rw.i) + ", v=" + rw.v.LongString() + ")" + } + s += "\n" + return s +} + +const initialRescheduleCounterValue = 1021 // Largest 10-bit prime. 97 nSec loop bodies will check every 100 uSec. + +// insertLoopReschedChecks inserts rescheduling checks on loop backedges. +func insertLoopReschedChecks(f *Func) { + // TODO: when split information is recorded in export data, insert checks only on backedges that can be reached on a split-call-free path. + + // Loop reschedule checks decrement a per-function counter + // shared by all loops, and when the counter becomes non-positive + // a call is made to a rescheduling check in the runtime. + // + // Steps: + // 1. locate backedges. + // 2. Record memory definitions at block end so that + // the SSA graph for mem can be prperly modified. + // 3. Define a counter and record its future uses (at backedges) + // (Same process as 2, applied to a single definition of the counter. + // difference for mem is that there are zero-to-many existing mem + // definitions, versus exactly one for the new counter.) + // 4. Ensure that phi functions that will-be-needed for mem and counter + // are present in the graph, initially with trivial inputs. + // 5. Record all to-be-modified uses of mem and counter; + // apply modifications (split into two steps to simplify and + // avoided nagging order-dependences). + // 6. Rewrite backedges to include counter check, reschedule check, + // and modify destination phi function appropriately with new + // definitions for mem and counter. + + if f.NoSplit { // nosplit functions don't reschedule. + return + } + + backedges := backedges(f) + if len(backedges) == 0 { // no backedges means no rescheduling checks. + return + } + + lastMems := findLastMems(f) + + idom := f.Idom() + sdom := f.sdom() + + if f.pass.debug > 2 { + fmt.Printf("before %s = %s\n", f.Name, sdom.treestructure(f.Entry)) + } + + tofixBackedges := []edgeMemCtr{} + + for _, e := range backedges { // TODO: could filter here by calls in loops, if declared and inferred nosplit are recorded in export data. + tofixBackedges = append(tofixBackedges, edgeMemCtr{e, nil, nil}) + } + + // It's possible that there is no memory state (no global/pointer loads/stores or calls) + if lastMems[f.Entry.ID] == nil { + lastMems[f.Entry.ID] = f.Entry.NewValue0(f.Entry.Line, OpInitMem, TypeMem) + } + + memDefsAtBlockEnds := make([]*Value, f.NumBlocks()) // For each block, the mem def seen at its bottom. Could be from earlier block. + + // Propagate last mem definitions forward through successor blocks. + po := f.postorder() + for i := len(po) - 1; i >= 0; i-- { + b := po[i] + mem := lastMems[b.ID] + for j := 0; mem == nil; j++ { // if there's no def, then there's no phi, so the visible mem is identical in all predecessors. + // loop because there might be backedges that haven't been visited yet. + mem = memDefsAtBlockEnds[b.Preds[j].b.ID] + } + memDefsAtBlockEnds[b.ID] = mem + } + + // Set up counter. There are no phis etc pre-existing for it. + counter0 := f.Entry.NewValue0I(f.Entry.Line, OpConst32, f.Config.fe.TypeInt32(), initialRescheduleCounterValue) + ctrDefsAtBlockEnds := make([]*Value, f.NumBlocks()) // For each block, def visible at its end, if that def will be used. + + // There's a minor difference between memDefsAtBlockEnds and ctrDefsAtBlockEnds; + // because the counter only matter for loops and code that reaches them, it is nil for blocks where the ctr is no + // longer live. This will avoid creation of dead phi functions. This optimization is ignored for the mem variable + // because it is harder and also less likely to be helpful, though dead code elimination ought to clean this out anyhow. + + for _, emc := range tofixBackedges { + e := emc.e + // set initial uses of counter zero (note available-at-bottom and use are the same thing initially.) + // each back-edge will be rewritten to include a reschedule check, and that will use the counter. + src := e.b.Preds[e.i].b + ctrDefsAtBlockEnds[src.ID] = counter0 + } + + // Push uses towards root + for _, b := range f.postorder() { + bd := ctrDefsAtBlockEnds[b.ID] + if bd == nil { + continue + } + for _, e := range b.Preds { + p := e.b + if ctrDefsAtBlockEnds[p.ID] == nil { + ctrDefsAtBlockEnds[p.ID] = bd + } + } + } + + // Maps from block to newly-inserted phi function in block. + newmemphis := make(map[*Block]rewrite) + newctrphis := make(map[*Block]rewrite) + + // Insert phi functions as necessary for future changes to flow graph. + for i, emc := range tofixBackedges { + e := emc.e + h := e.b + + // find the phi function for the memory input at "h", if there is one. + var headerMemPhi *Value // look for header mem phi + + for _, v := range h.Values { + if v.Op == OpPhi && v.Type.IsMemory() { + headerMemPhi = v + } + } + + if headerMemPhi == nil { + // if the header is nil, make a trivial phi from the dominator + mem0 := memDefsAtBlockEnds[idom[h.ID].ID] + headerMemPhi = newPhiFor(h, mem0) + newmemphis[h] = rewrite{before: mem0, after: headerMemPhi} + addDFphis(mem0, h, h, f, memDefsAtBlockEnds, newmemphis) + + } + tofixBackedges[i].m = headerMemPhi + + var headerCtrPhi *Value + rw, ok := newctrphis[h] + if !ok { + headerCtrPhi = newPhiFor(h, counter0) + newctrphis[h] = rewrite{before: counter0, after: headerCtrPhi} + addDFphis(counter0, h, h, f, ctrDefsAtBlockEnds, newctrphis) + } else { + headerCtrPhi = rw.after + } + tofixBackedges[i].c = headerCtrPhi + } + + rewriteNewPhis(f.Entry, f.Entry, f, memDefsAtBlockEnds, newmemphis) + rewriteNewPhis(f.Entry, f.Entry, f, ctrDefsAtBlockEnds, newctrphis) + + if f.pass.debug > 0 { + for b, r := range newmemphis { + fmt.Printf("b=%s, rewrite=%s\n", b, r.String()) + } + + for b, r := range newctrphis { + fmt.Printf("b=%s, rewrite=%s\n", b, r.String()) + } + } + + // Apply collected rewrites. + for _, r := range newmemphis { + for _, rw := range r.rewrites { + rw.v.SetArg(rw.i, r.after) + } + } + + for _, r := range newctrphis { + for _, rw := range r.rewrites { + rw.v.SetArg(rw.i, r.after) + } + } + + zero := f.Entry.NewValue0I(f.Entry.Line, OpConst32, f.Config.fe.TypeInt32(), 0) + one := f.Entry.NewValue0I(f.Entry.Line, OpConst32, f.Config.fe.TypeInt32(), 1) + + // Rewrite backedges to include reschedule checks. + for _, emc := range tofixBackedges { + e := emc.e + headerMemPhi := emc.m + headerCtrPhi := emc.c + h := e.b + i := e.i + p := h.Preds[i] + bb := p.b + mem0 := headerMemPhi.Args[i] + ctr0 := headerCtrPhi.Args[i] + // bb e->p h, + // Because we're going to insert a rare-call, make sure the + // looping edge still looks likely. + likely := BranchLikely + if p.i != 0 { + likely = BranchUnlikely + } + bb.Likely = likely + + // rewrite edge to include reschedule check + // existing edges: + // + // bb.Succs[p.i] == Edge{h, i} + // h.Preds[i] == p == Edge{bb,p.i} + // + // new block(s): + // test: + // ctr1 := ctr0 - 1 + // if ctr1 <= 0 { goto sched } + // goto join + // sched: + // mem1 := call resched (mem0) + // goto join + // join: + // ctr2 := phi(ctr1, counter0) // counter0 is the constant + // mem2 := phi(mem0, mem1) + // goto h + // + // and correct arg i of headerMemPhi and headerCtrPhi + // + // EXCEPT: block containing only phi functions is bad + // for the register allocator. Therefore, there is no + // join, and instead branches targeting join instead target + // the header, and the other phi functions within header are + // adjusted for the additional input. + + test := f.NewBlock(BlockIf) + sched := f.NewBlock(BlockPlain) + + test.Line = bb.Line + sched.Line = bb.Line + + // ctr1 := ctr0 - 1 + // if ctr1 <= 0 { goto sched } + // goto header + ctr1 := test.NewValue2(bb.Line, OpSub32, f.Config.fe.TypeInt32(), ctr0, one) + cmp := test.NewValue2(bb.Line, OpLeq32, f.Config.fe.TypeBool(), ctr1, zero) + test.SetControl(cmp) + test.AddEdgeTo(sched) // if true + // if false -- rewrite edge to header. + // do NOT remove+add, because that will perturb all the other phi functions + // as well as messing up other edges to the header. + test.Succs = append(test.Succs, Edge{h, i}) + h.Preds[i] = Edge{test, 1} + headerMemPhi.SetArg(i, mem0) + headerCtrPhi.SetArg(i, ctr1) + + test.Likely = BranchUnlikely + + // sched: + // mem1 := call resched (mem0) + // goto header + resched := f.Config.fe.Syslook("goschedguarded") + mem1 := sched.NewValue1A(bb.Line, OpStaticCall, TypeMem, resched, mem0) + sched.AddEdgeTo(h) + headerMemPhi.AddArg(mem1) + headerCtrPhi.AddArg(counter0) + + bb.Succs[p.i] = Edge{test, 0} + test.Preds = append(test.Preds, Edge{bb, p.i}) + + // Must correct all the other phi functions in the header for new incoming edge. + // Except for mem and counter phis, it will be the same value seen on the original + // backedge at index i. + for _, v := range h.Values { + if v.Op == OpPhi && v != headerMemPhi && v != headerCtrPhi { + v.AddArg(v.Args[i]) + } + } + } + + f.invalidateCFG() + + if f.pass.debug > 2 { + sdom = newSparseTree(f, f.Idom()) + fmt.Printf("after %s = %s\n", f.Name, sdom.treestructure(f.Entry)) + } + + return +} + +// newPhiFor inserts a new Phi function into b, +// with all inputs set to v. +func newPhiFor(b *Block, v *Value) *Value { + phiV := b.NewValue0(b.Line, OpPhi, v.Type) + + for range b.Preds { + phiV.AddArg(v) + } + return phiV +} + +// rewriteNewPhis updates newphis[h] to record all places where the new phi function inserted +// in block h will replace a previous definition. Block b is the block currently being processed; +// if b has its own phi definition then it takes the place of h. +// defsForUses provides information about other definitions of the variable that are present +// (and if nil, indicates that the variable is no longer live) +func rewriteNewPhis(h, b *Block, f *Func, defsForUses []*Value, newphis map[*Block]rewrite) { + // If b is a block with a new phi, then a new rewrite applies below it in the dominator tree. + if _, ok := newphis[b]; ok { + h = b + } + change := newphis[h] + x := change.before + y := change.after + + // Apply rewrites to this block + if x != nil { // don't waste time on the common case of no definition. + p := &change.rewrites + for _, v := range b.Values { + if v == y { // don't rewrite self -- phi inputs are handled below. + continue + } + for i, w := range v.Args { + if w != x { + continue + } + *p = append(*p, rewriteTarget{v, i}) + } + } + + // Rewrite appropriate inputs of phis reached in successors + // in dominance frontier, self, and dominated. + // If the variable def reaching uses in b is itself defined in b, then the new phi function + // does not reach the successors of b. (This assumes a bit about the structure of the + // phi use-def graph, but it's true for memory and the inserted counter.) + if dfu := defsForUses[b.ID]; dfu != nil && dfu.Block != b { + for _, e := range b.Succs { + s := e.b + if sphi, ok := newphis[s]; ok { // saves time to find the phi this way. + *p = append(*p, rewriteTarget{sphi.after, e.i}) + continue + } + for _, v := range s.Values { + if v.Op == OpPhi && v.Args[e.i] == x { + *p = append(*p, rewriteTarget{v, e.i}) + break + } + } + } + } + newphis[h] = change + } + + sdom := f.sdom() + + for c := sdom[b.ID].child; c != nil; c = sdom[c.ID].sibling { + rewriteNewPhis(h, c, f, defsForUses, newphis) // TODO: convert to explicit stack from recursion. + } +} + +// addDFphis creates new trivial phis that are necessary to correctly reflect (within SSA) +// a new definition for variable "x" inserted at h (usually but not necessarily a phi). +// These new phis can only occur at the dominance frontier of h; block s is in the dominance +// frontier of h if h does not strictly dominate s and if s is a successor of a block b where +// either b = h or h strictly dominates b. +// These newly created phis are themselves new definitions that may require addition of their +// own trivial phi functions in their own dominance frontier, and this is handled recursively. +func addDFphis(x *Value, h, b *Block, f *Func, defForUses []*Value, newphis map[*Block]rewrite) { + oldv := defForUses[b.ID] + if oldv != x { // either a new definition replacing x, or nil if it is proven that there are no uses reachable from b + return + } + sdom := f.sdom() + idom := f.Idom() +outer: + for _, e := range b.Succs { + s := e.b + // check phi functions in the dominance frontier + if sdom.isAncestor(h, s) { + continue // h dominates s, successor of b, therefore s is not in the frontier. + } + if _, ok := newphis[s]; ok { + continue // successor s of b already has a new phi function, so there is no need to add another. + } + if x != nil { + for _, v := range s.Values { + if v.Op == OpPhi && v.Args[e.i] == x { + continue outer // successor s of b has an old phi function, so there is no need to add another. + } + } + } + + old := defForUses[idom[s.ID].ID] // new phi function is correct-but-redundant, combining value "old" on all inputs. + headerPhi := newPhiFor(s, old) + // the new phi will replace "old" in block s and all blocks dominated by s. + newphis[s] = rewrite{before: old, after: headerPhi} // record new phi, to have inputs labeled "old" rewritten to "headerPhi" + addDFphis(old, s, s, f, defForUses, newphis) // the new definition may also create new phi functions. + } + for c := sdom[b.ID].child; c != nil; c = sdom[c.ID].sibling { + addDFphis(x, h, c, f, defForUses, newphis) // TODO: convert to explicit stack from recursion. + } +} + +// findLastMems maps block ids to last memory-output op in a block, if any +func findLastMems(f *Func) []*Value { + + var stores []*Value + lastMems := make([]*Value, f.NumBlocks()) + storeUse := f.newSparseSet(f.NumValues()) + defer f.retSparseSet(storeUse) + for _, b := range f.Blocks { + // Find all the stores in this block. Categorize their uses: + // storeUse contains stores which are used by a subsequent store. + storeUse.clear() + stores = stores[:0] + var memPhi *Value + for _, v := range b.Values { + if v.Op == OpPhi { + if v.Type.IsMemory() { + memPhi = v + } + continue + } + if v.Type.IsMemory() { + stores = append(stores, v) + if v.Op == OpSelect1 { + // Use the arg of the tuple-generating op. + v = v.Args[0] + } + for _, a := range v.Args { + if a.Block == b && a.Type.IsMemory() { + storeUse.add(a.ID) + } + } + } + } + if len(stores) == 0 { + lastMems[b.ID] = memPhi + continue + } + + // find last store in the block + var last *Value + for _, v := range stores { + if storeUse.contains(v.ID) { + continue + } + if last != nil { + b.Fatalf("two final stores - simultaneous live stores %s %s", last, v) + } + last = v + } + if last == nil { + b.Fatalf("no last store found - cycle?") + } + lastMems[b.ID] = last + } + return lastMems +} + +type backedgesState struct { + b *Block + i int +} + +// backedges returns a slice of successor edges that are back +// edges. For reducible loops, edge.b is the header. +func backedges(f *Func) []Edge { + edges := []Edge{} + mark := make([]markKind, f.NumBlocks()) + stack := []backedgesState{} + + mark[f.Entry.ID] = notExplored + stack = append(stack, backedgesState{f.Entry, 0}) + + for len(stack) > 0 { + l := len(stack) + x := stack[l-1] + if x.i < len(x.b.Succs) { + e := x.b.Succs[x.i] + stack[l-1].i++ + s := e.b + if mark[s.ID] == notFound { + mark[s.ID] = notExplored + stack = append(stack, backedgesState{s, 0}) + } else if mark[s.ID] == notExplored { + edges = append(edges, e) + } + } else { + mark[x.b.ID] = done + stack = stack[0 : l-1] + } + } + return edges +} diff --git a/src/cmd/compile/internal/ssa/nilcheck.go b/src/cmd/compile/internal/ssa/nilcheck.go index eb2d297f80..9f58db664b 100644 --- a/src/cmd/compile/internal/ssa/nilcheck.go +++ b/src/cmd/compile/internal/ssa/nilcheck.go @@ -132,6 +132,8 @@ func nilcheckelim(f *Func) { } // All platforms are guaranteed to fault if we load/store to anything smaller than this address. +// +// This should agree with minLegalPointer in the runtime. const minZeroPage = 4096 // nilcheckelim2 eliminates unnecessary nil checks. diff --git a/src/cmd/compile/internal/ssa/op.go b/src/cmd/compile/internal/ssa/op.go index 315d7203d4..4c3164f231 100644 --- a/src/cmd/compile/internal/ssa/op.go +++ b/src/cmd/compile/internal/ssa/op.go @@ -21,7 +21,7 @@ type opInfo struct { name string reg regInfo auxType auxType - argLen int32 // the number of arugments, -1 if variable length + argLen int32 // the number of arguments, -1 if variable length asm obj.As generic bool // this is a generic (arch-independent) opcode rematerializeable bool // this op is rematerializeable diff --git a/src/cmd/compile/internal/ssa/regalloc.go b/src/cmd/compile/internal/ssa/regalloc.go index 2b66982340..7bf778609e 100644 --- a/src/cmd/compile/internal/ssa/regalloc.go +++ b/src/cmd/compile/internal/ssa/regalloc.go @@ -759,7 +759,7 @@ func (s *regAllocState) regalloc(f *Func) { liveSet.add(e.ID) } if v := b.Control; v != nil && s.values[v.ID].needReg { - s.addUse(v.ID, int32(len(b.Values)), b.Line) // psuedo-use by control value + s.addUse(v.ID, int32(len(b.Values)), b.Line) // pseudo-use by control value liveSet.add(v.ID) } for i := len(b.Values) - 1; i >= 0; i-- { diff --git a/src/cmd/compile/internal/ssa/sparsetree.go b/src/cmd/compile/internal/ssa/sparsetree.go index 7c82a60d0f..8e5b9f3e5b 100644 --- a/src/cmd/compile/internal/ssa/sparsetree.go +++ b/src/cmd/compile/internal/ssa/sparsetree.go @@ -4,7 +4,10 @@ package ssa -import "fmt" +import ( + "fmt" + "strings" +) type SparseTreeNode struct { child *Block @@ -67,6 +70,34 @@ func newSparseTree(f *Func, parentOf []*Block) SparseTree { return t } +// treestructure provides a string description of the dominator +// tree and flow structure of block b and all blocks that it +// dominates. +func (t SparseTree) treestructure(b *Block) string { + return t.treestructure1(b, 0) +} +func (t SparseTree) treestructure1(b *Block, i int) string { + s := "\n" + strings.Repeat("\t", i) + b.String() + "->[" + for i, e := range b.Succs { + if i > 0 { + s = s + "," + } + s = s + e.b.String() + } + s += "]" + if c0 := t[b.ID].child; c0 != nil { + s += "(" + for c := c0; c != nil; c = t[c.ID].sibling { + if c != c0 { + s += " " + } + s += t.treestructure1(c, i+1) + } + s += ")" + } + return s +} + // numberBlock assigns entry and exit numbers for b and b's // children in an in-order walk from a gappy sequence, where n // is the first number not yet assigned or reserved. N should diff --git a/src/cmd/compile/internal/ssa/type_test.go b/src/cmd/compile/internal/ssa/type_test.go index a76a0651bb..2f917288de 100644 --- a/src/cmd/compile/internal/ssa/type_test.go +++ b/src/cmd/compile/internal/ssa/type_test.go @@ -44,7 +44,7 @@ func (t *TypeImpl) IsVoid() bool { return false } func (t *TypeImpl) String() string { return t.Name } func (t *TypeImpl) SimpleString() string { return t.Name } func (t *TypeImpl) ElemType() Type { return t.Elem_ } -func (t *TypeImpl) PtrTo() Type { panic("not implemented") } +func (t *TypeImpl) PtrTo() Type { return TypeBytePtr } func (t *TypeImpl) NumFields() int { panic("not implemented") } func (t *TypeImpl) FieldType(i int) Type { panic("not implemented") } func (t *TypeImpl) FieldOff(i int) int64 { panic("not implemented") } diff --git a/src/cmd/compile/internal/ssa/writebarrier.go b/src/cmd/compile/internal/ssa/writebarrier.go index b914154b48..1eb4d7bb1a 100644 --- a/src/cmd/compile/internal/ssa/writebarrier.go +++ b/src/cmd/compile/internal/ssa/writebarrier.go @@ -78,7 +78,6 @@ func writebarrier(f *Func) { defer f.retSparseSet(wbs) } - mem := v.Args[2] line := v.Line // there may be a sequence of WB stores in the current block. find them. @@ -106,6 +105,20 @@ func writebarrier(f *Func) { } } + // find the memory before the WB stores + // this memory is not a WB store but it is used in a WB store. + var mem *Value + for _, w := range storeWBs { + a := w.Args[len(w.Args)-1] + if wbs.contains(a.ID) { + continue + } + if mem != nil { + b.Fatalf("two stores live simultaneously: %s, %s", mem, a) + } + mem = a + } + b.Values = append(b.Values[:i], others...) // move WB ops out of this block bThen := f.NewBlock(BlockPlain) @@ -177,20 +190,39 @@ func writebarrier(f *Func) { // which may be used in subsequent blocks. Other memories in the // sequence must be dead after this block since there can be only // one memory live. - v = storeWBs[len(storeWBs)-1] - bEnd.Values = append(bEnd.Values, v) - v.Block = bEnd - v.reset(OpPhi) - v.Type = TypeMem - v.AddArg(memThen) - v.AddArg(memElse) - for _, w := range storeWBs[:len(storeWBs)-1] { - for _, a := range w.Args { - a.Uses-- + last := storeWBs[0] + if len(storeWBs) > 1 { + // find the last store + last = nil + wbs.clear() // we reuse wbs to record WB stores that is used in another WB store + for _, w := range storeWBs { + wbs.add(w.Args[len(w.Args)-1].ID) + } + for _, w := range storeWBs { + if wbs.contains(w.ID) { + continue + } + if last != nil { + b.Fatalf("two stores live simultaneously: %s, %s", last, w) + } + last = w } } - for _, w := range storeWBs[:len(storeWBs)-1] { - f.freeValue(w) + bEnd.Values = append(bEnd.Values, last) + last.Block = bEnd + last.reset(OpPhi) + last.Type = TypeMem + last.AddArg(memThen) + last.AddArg(memElse) + for _, w := range storeWBs { + if w != last { + w.resetArgs() + } + } + for _, w := range storeWBs { + if w != last { + f.freeValue(w) + } } if f.Config.fe.Debug_wb() { diff --git a/src/cmd/compile/internal/ssa/writebarrier_test.go b/src/cmd/compile/internal/ssa/writebarrier_test.go new file mode 100644 index 0000000000..c2ba695971 --- /dev/null +++ b/src/cmd/compile/internal/ssa/writebarrier_test.go @@ -0,0 +1,29 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ssa + +import "testing" + +func TestWriteBarrierStoreOrder(t *testing.T) { + // Make sure writebarrier phase works even StoreWB ops are not in dependency order + c := testConfig(t) + ptrType := &TypeImpl{Size_: 8, Ptr: true, Name: "testptr"} // dummy for testing + fun := Fun(c, "entry", + Bloc("entry", + Valu("start", OpInitMem, TypeMem, 0, nil), + Valu("sb", OpSB, TypeInvalid, 0, nil), + Valu("sp", OpSP, TypeInvalid, 0, nil), + Valu("v", OpConstNil, ptrType, 0, nil), + Valu("addr1", OpAddr, ptrType, 0, nil, "sb"), + Valu("wb2", OpStoreWB, TypeMem, 8, nil, "addr1", "v", "wb1"), + Valu("wb1", OpStoreWB, TypeMem, 8, nil, "addr1", "v", "start"), // wb1 and wb2 are out of order + Goto("exit")), + Bloc("exit", + Exit("wb2"))) + + CheckFunc(fun.f) + writebarrier(fun.f) + CheckFunc(fun.f) +} diff --git a/src/cmd/compile/internal/syntax/parser.go b/src/cmd/compile/internal/syntax/parser.go index a2e307f46f..121dfb75e5 100644 --- a/src/cmd/compile/internal/syntax/parser.go +++ b/src/cmd/compile/internal/syntax/parser.go @@ -26,8 +26,6 @@ type parser struct { indent []byte // tracing support } -type parserError string // for error recovery if no error handler was installed - func (p *parser) init(src io.Reader, errh ErrorHandler, pragh PragmaHandler) { p.scanner.init(src, errh, pragh) diff --git a/src/cmd/cover/cover.go b/src/cmd/cover/cover.go index b7d9125d60..ee140702d3 100644 --- a/src/cmd/cover/cover.go +++ b/src/cmd/cover/cover.go @@ -151,11 +151,12 @@ type Block struct { // File is a wrapper for the state of a file used in the parser. // The basic parse tree walker is a method of this type. type File struct { - fset *token.FileSet - name string // Name of file. - astFile *ast.File - blocks []Block - atomicPkg string // Package name for "sync/atomic" in this file. + fset *token.FileSet + name string // Name of file. + astFile *ast.File + blocks []Block + atomicPkg string // Package name for "sync/atomic" in this file. + directives map[*ast.Comment]bool // Map of compiler directives to whether it's processed in ast.Visitor or not. } // Visit implements the ast.Visitor interface. @@ -247,8 +248,11 @@ func (f *File) Visit(node ast.Node) ast.Visitor { // to appear in syntactically incorrect places. //go: appears at the beginning of // the line and is syntactically safe. for _, c := range n.List { - if strings.HasPrefix(c.Text, "//go:") && f.fset.Position(c.Slash).Column == 1 { + if f.isDirective(c) { list = append(list, c) + + // Mark compiler directive as handled. + f.directives[c] = true } } n.List = list @@ -360,17 +364,27 @@ func annotate(name string) { if err != nil { log.Fatalf("cover: %s: %s", name, err) } - // Remove comments. Or else they interfere with new AST. - parsedFile.Comments = nil file := &File{ - fset: fset, - name: name, - astFile: parsedFile, + fset: fset, + name: name, + astFile: parsedFile, + directives: map[*ast.Comment]bool{}, } if *mode == "atomic" { file.atomicPkg = file.addImport(atomicPackagePath) } + + for _, cg := range parsedFile.Comments { + for _, c := range cg.List { + if file.isDirective(c) { + file.directives[c] = false + } + } + } + // Remove comments. Or else they interfere with new AST. + parsedFile.Comments = nil + ast.Walk(file, file.astFile) fd := os.Stdout if *output != "" { @@ -381,6 +395,17 @@ func annotate(name string) { } } fd.Write(initialComments(content)) // Retain '// +build' directives. + + // Retain compiler directives that are not processed in ast.Visitor. + // Some compiler directives like "go:linkname" and "go:cgo_" + // can be not attached to anything in the tree and hence will not be printed by printer. + // So, we have to explicitly print them here. + for cd, handled := range file.directives { + if !handled { + fmt.Fprintln(fd, cd.Text) + } + } + file.print(fd) // After printing the source tree, add some declarations for the counters etc. // We could do this by adding to the tree, but it's easier just to print the text. @@ -391,6 +416,11 @@ func (f *File) print(w io.Writer) { printer.Fprint(w, f.fset, f.astFile) } +// isDirective reports whether a comment is a compiler directive. +func (f *File) isDirective(c *ast.Comment) bool { + return strings.HasPrefix(c.Text, "//go:") && f.fset.Position(c.Slash).Column == 1 +} + // intLiteral returns an ast.BasicLit representing the integer value. func (f *File) intLiteral(i int) *ast.BasicLit { node := &ast.BasicLit{ diff --git a/src/cmd/cover/cover_test.go b/src/cmd/cover/cover_test.go index 50a7ce829f..81ac8ae467 100644 --- a/src/cmd/cover/cover_test.go +++ b/src/cmd/cover/cover_test.go @@ -90,6 +90,11 @@ func TestCover(t *testing.T) { if got, err := regexp.MatchString(".*\n//go:nosplit\nfunc someFunction().*", string(file)); err != nil || !got { t.Errorf("misplaced compiler directive: got=(%v, %v); want=(true; nil)", got, err) } + // "go:linkname" compiler directive should be present. + if got, err := regexp.MatchString(`.*go\:linkname some\_name some\_name.*`, string(file)); err != nil || !got { + t.Errorf("'go:linkname' compiler directive not found: got=(%v, %v); want=(true; nil)", got, err) + } + // No other comments should be present in generated code. c := ".*// This comment shouldn't appear in generated go code.*" if got, err := regexp.MatchString(c, string(file)); err != nil || got { diff --git a/src/cmd/cover/testdata/test.go b/src/cmd/cover/testdata/test.go index 61b40eaa74..5effa2d7e9 100644 --- a/src/cmd/cover/testdata/test.go +++ b/src/cmd/cover/testdata/test.go @@ -10,6 +10,10 @@ package main +import _ "unsafe" // for go:linkname + +//go:linkname some_name some_name + const anything = 1e9 // Just some unlikely value that means "we got here, don't care how often" func testAll() { diff --git a/src/cmd/dist/build.go b/src/cmd/dist/build.go index 6fb7884560..4d0b1a0b41 100644 --- a/src/cmd/dist/build.go +++ b/src/cmd/dist/build.go @@ -1107,8 +1107,8 @@ var cgoEnabled = map[string]bool{ "linux/arm64": true, "linux/ppc64": false, "linux/ppc64le": true, - "linux/mips": false, - "linux/mipsle": false, + "linux/mips": true, + "linux/mipsle": true, "linux/mips64": true, "linux/mips64le": true, "linux/s390x": true, diff --git a/src/cmd/dist/test.go b/src/cmd/dist/test.go index 508863f275..7d5f79f339 100644 --- a/src/cmd/dist/test.go +++ b/src/cmd/dist/test.go @@ -15,6 +15,7 @@ import ( "os/exec" "path/filepath" "regexp" + "runtime" "strconv" "strings" "sync" @@ -152,8 +153,11 @@ func (t *tester) run() { } t.timeoutScale = 1 - if t.goarch == "arm" || t.goos == "windows" { + switch t.goarch { + case "arm": t.timeoutScale = 2 + case "mips", "mipsle", "mips64", "mips64le": + t.timeoutScale = 4 } if s := os.Getenv("GO_TEST_TIMEOUT_SCALE"); s != "" { t.timeoutScale, err = strconv.Atoi(s) @@ -328,6 +332,10 @@ func (t *tester) registerRaceBenchTest(pkg string) { }) } +// stdOutErrAreTerminals is defined in test_linux.go, to report +// whether stdout & stderr are terminals. +var stdOutErrAreTerminals func() bool + func (t *tester) registerTests() { if strings.HasSuffix(os.Getenv("GO_BUILDER_NAME"), "-vetall") { // Run vet over std and cmd and call it quits. @@ -344,6 +352,27 @@ func (t *tester) registerTests() { return } + // This test needs its stdout/stderr to be terminals, so we don't run it from cmd/go's tests. + // See issue 18153. + if runtime.GOOS == "linux" { + t.tests = append(t.tests, distTest{ + name: "cmd_go_test_terminal", + heading: "cmd/go terminal test", + fn: func(dt *distTest) error { + t.runPending(dt) + if !stdOutErrAreTerminals() { + fmt.Println("skipping terminal test; stdout/stderr not terminals") + return nil + } + cmd := exec.Command("go", "test") + cmd.Dir = filepath.Join(os.Getenv("GOROOT"), "src/cmd/go/testdata/testterminal18153") + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + return cmd.Run() + }, + }) + } + // Fast path to avoid the ~1 second of `go list std cmd` when // the caller lists specific tests to run. (as the continuous // build coordinator does). @@ -675,7 +704,7 @@ func (t *tester) extLink() bool { "darwin-arm", "darwin-arm64", "dragonfly-386", "dragonfly-amd64", "freebsd-386", "freebsd-amd64", "freebsd-arm", - "linux-386", "linux-amd64", "linux-arm", "linux-arm64", "linux-ppc64le", "linux-mips64", "linux-mips64le", + "linux-386", "linux-amd64", "linux-arm", "linux-arm64", "linux-ppc64le", "linux-mips64", "linux-mips64le", "linux-mips", "linux-mipsle", "linux-s390x", "netbsd-386", "netbsd-amd64", "openbsd-386", "openbsd-amd64", "windows-386", "windows-amd64": @@ -712,7 +741,7 @@ func (t *tester) internalLink() bool { // Internally linking cgo is incomplete on some architectures. // https://golang.org/issue/10373 // https://golang.org/issue/14449 - if t.goarch == "arm64" || t.goarch == "mips64" || t.goarch == "mips64le" { + if t.goarch == "arm64" || t.goarch == "mips64" || t.goarch == "mips64le" || t.goarch == "mips" || t.goarch == "mipsle" { return false } return true @@ -757,8 +786,7 @@ func (t *tester) supportedBuildmode(mode string) bool { // linux-arm64 is missing because it causes the external linker // to crash, see https://golang.org/issue/17138 switch pair { - case "linux-386", "linux-amd64", "linux-arm", - "darwin-amd64": + case "linux-386", "linux-amd64", "linux-arm": return true } return false diff --git a/src/cmd/dist/test_linux.go b/src/cmd/dist/test_linux.go new file mode 100644 index 0000000000..b6d0aedbbf --- /dev/null +++ b/src/cmd/dist/test_linux.go @@ -0,0 +1,27 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build linux + +package main + +import ( + "syscall" + "unsafe" +) + +const ioctlReadTermios = syscall.TCGETS + +// isTerminal reports whether fd is a terminal. +func isTerminal(fd uintptr) bool { + var termios syscall.Termios + _, _, err := syscall.Syscall6(syscall.SYS_IOCTL, fd, ioctlReadTermios, uintptr(unsafe.Pointer(&termios)), 0, 0, 0) + return err == 0 +} + +func init() { + stdOutErrAreTerminals = func() bool { + return isTerminal(1) && isTerminal(2) + } +} diff --git a/src/cmd/dist/util.go b/src/cmd/dist/util.go index e2f22dfe44..511978f2f5 100644 --- a/src/cmd/dist/util.go +++ b/src/cmd/dist/util.go @@ -393,6 +393,14 @@ func errprintf(format string, args ...interface{}) { func main() { os.Setenv("TERM", "dumb") // disable escape codes in clang errors + // provide -check-armv6k first, before checking for $GOROOT so that + // it is possible to run this check without having $GOROOT available. + if len(os.Args) > 1 && os.Args[1] == "-check-armv6k" { + useARMv6K() // might fail with SIGILL + println("ARMv6K supported.") + os.Exit(0) + } + slash = string(filepath.Separator) gohostos = runtime.GOOS diff --git a/src/cmd/dist/util_gc.go b/src/cmd/dist/util_gc.go index 6e099e5f9c..698beef704 100644 --- a/src/cmd/dist/util_gc.go +++ b/src/cmd/dist/util_gc.go @@ -25,3 +25,8 @@ func useVFPv1() // useVFPv3 tries to execute one VFPv3 instruction on ARM. // It will crash the current process if VFPv3 is missing. func useVFPv3() + +// useARMv6K tries to run ARMv6K instructions on ARM. +// It will crash the current process if it doesn't implement +// ARMv6K or above. +func useARMv6K() diff --git a/src/cmd/dist/util_gccgo.go b/src/cmd/dist/util_gccgo.go index 5edb4734f9..f9f01dc048 100644 --- a/src/cmd/dist/util_gccgo.go +++ b/src/cmd/dist/util_gccgo.go @@ -22,3 +22,5 @@ func cansse2() bool { return C.supports_sse2() != 0 } func useVFPv1() {} func useVFPv3() {} + +func useARMv6K() {} diff --git a/src/cmd/dist/vfp_arm.s b/src/cmd/dist/vfp_arm.s index c42b593aec..d571f8b82a 100644 --- a/src/cmd/dist/vfp_arm.s +++ b/src/cmd/dist/vfp_arm.s @@ -15,3 +15,12 @@ TEXT ·useVFPv1(SB),NOSPLIT,$0 TEXT ·useVFPv3(SB),NOSPLIT,$0 WORD $0xeeb70b00 // vmov.f64 d0, #112 RET + +// try to run ARMv6K (or above) "ldrexd" instruction +TEXT ·useARMv6K(SB),NOSPLIT,$32 + MOVW R13, R2 + BIC $15, R13 + WORD $0xe1bd0f9f // ldrexd r0, r1, [sp] + WORD $0xf57ff01f // clrex + MOVW R2, R13 + RET diff --git a/src/cmd/dist/vfp_default.s b/src/cmd/dist/vfp_default.s index 95ccbe3e44..84829beeff 100644 --- a/src/cmd/dist/vfp_default.s +++ b/src/cmd/dist/vfp_default.s @@ -11,3 +11,6 @@ TEXT ·useVFPv1(SB),NOSPLIT,$0 TEXT ·useVFPv3(SB),NOSPLIT,$0 RET + +TEXT ·useARMv6K(SB),NOSPLIT,$0 + RET diff --git a/src/cmd/go/alldocs.go b/src/cmd/go/alldocs.go index b4807420b0..e93fd6ebed 100644 --- a/src/cmd/go/alldocs.go +++ b/src/cmd/go/alldocs.go @@ -935,6 +935,8 @@ // unless that directory holds a Go distribution. // Run "go env GOPATH" to see the current GOPATH. // +// See https://golang.org/wiki/SettingGOPATH to set a custom GOPATH. +// // Each directory listed in GOPATH must have a prescribed structure: // // The src directory holds source code. The path below src @@ -1474,7 +1476,7 @@ // text from Log and Logf calls even if the test succeeds. // // The following flags are also recognized by 'go test' and can be used to -// profile the tests during execution:: +// profile the tests during execution: // // -benchmem // Print memory allocation statistics for benchmarks. @@ -1517,7 +1519,7 @@ // Writes test binary as -c would. // // -mutexprofilefraction n -// Sample 1 in n stack traces of goroutines holding a +// Sample 1 in n stack traces of goroutines holding a // contended mutex. // // -outputdir directory @@ -1606,7 +1608,8 @@ // is compared exactly against the comment (see examples below). If the last // comment begins with "Unordered output:" then the output is compared to the // comment, however the order of the lines is ignored. An example with no such -// comment, or with no text after "Output:" is compiled but not executed. +// comment is compiled but not executed. An example with no text after +// "Output:" is compiled, executed, and expected to produce no output. // // Godoc displays the body of ExampleXXX to demonstrate the use // of the function, constant, or variable XXX. An example of a method M with diff --git a/src/cmd/go/bug.go b/src/cmd/go/bug.go index 2977c94c14..cbd258b80b 100644 --- a/src/cmd/go/bug.go +++ b/src/cmd/go/bug.go @@ -42,7 +42,11 @@ func runBug(cmd *Command, args []string) { env := newEnv env = append(env, extraEnvVars()...) for _, e := range env { - fmt.Fprintf(&buf, "%s=\"%s\"\n", e.name, e.value) + // Hide the TERM environment variable from "go bug". + // See issue #18128 + if e.name != "TERM" { + fmt.Fprintf(&buf, "%s=\"%s\"\n", e.name, e.value) + } } printGoDetails(&buf) printOSDetails(&buf) diff --git a/src/cmd/go/build.go b/src/cmd/go/build.go index 684d033d3a..98a650918a 100644 --- a/src/cmd/go/build.go +++ b/src/cmd/go/build.go @@ -413,8 +413,7 @@ func buildModeInit() { } else { switch platform { case "linux/amd64", "linux/arm", "linux/arm64", "linux/386", - "android/amd64", "android/arm", "android/arm64", "android/386", - "darwin/amd64": + "android/amd64", "android/arm", "android/arm64", "android/386": default: fatalf("-buildmode=plugin not supported on %s\n", platform) } @@ -2406,8 +2405,7 @@ func (gcToolchain) gc(b *builder, p *Package, archive, obj string, asmhdr bool, func (gcToolchain) asm(b *builder, p *Package, obj string, sfiles []string) ([]string, error) { // Add -I pkg/GOOS_GOARCH so #include "textflag.h" works in .s files. inc := filepath.Join(goroot, "pkg", "include") - ofile := obj + "asm.o" - args := []interface{}{buildToolExec, tool("asm"), "-o", ofile, "-trimpath", b.work, "-I", obj, "-I", inc, "-D", "GOOS_" + goos, "-D", "GOARCH_" + goarch, buildAsmflags} + args := []interface{}{buildToolExec, tool("asm"), "-trimpath", b.work, "-I", obj, "-I", inc, "-D", "GOOS_" + goos, "-D", "GOARCH_" + goarch, buildAsmflags} if p.ImportPath == "runtime" && goarch == "386" { for _, arg := range buildAsmflags { if arg == "-dynlink" { @@ -2415,13 +2413,16 @@ func (gcToolchain) asm(b *builder, p *Package, obj string, sfiles []string) ([]s } } } + var ofiles []string for _, sfile := range sfiles { - args = append(args, mkAbs(p.Dir, sfile)) - } - if err := b.run(p.Dir, p.ImportPath, nil, args...); err != nil { - return nil, err + ofile := obj + sfile[:len(sfile)-len(".s")] + ".o" + ofiles = append(ofiles, ofile) + a := append(args, "-o", ofile, mkAbs(p.Dir, sfile)) + if err := b.run(p.Dir, p.ImportPath, nil, a...); err != nil { + return nil, err + } } - return []string{ofile}, nil + return ofiles, nil } // toolVerify checks that the command line args writes the same output file @@ -3218,6 +3219,8 @@ func (b *builder) gccArchArgs() []string { return []string{"-m64", "-march=z196"} case "mips64", "mips64le": return []string{"-mabi=64"} + case "mips", "mipsle": + return []string{"-mabi=32", "-march=mips32"} } return nil } @@ -3773,7 +3776,11 @@ func instrumentInit() { return } if buildRace && buildMSan { - fmt.Fprintf(os.Stderr, "go %s: may not use -race and -msan simultaneously", flag.Args()[0]) + fmt.Fprintf(os.Stderr, "go %s: may not use -race and -msan simultaneously\n", flag.Args()[0]) + os.Exit(2) + } + if buildMSan && (goos != "linux" || goarch != "amd64") { + fmt.Fprintf(os.Stderr, "-msan is not supported on %s/%s\n", goos, goarch) os.Exit(2) } if goarch != "amd64" || goos != "linux" && goos != "freebsd" && goos != "darwin" && goos != "windows" { diff --git a/src/cmd/go/get.go b/src/cmd/go/get.go index 82408d6a39..1d7677c615 100644 --- a/src/cmd/go/get.go +++ b/src/cmd/go/get.go @@ -205,6 +205,10 @@ var downloadRootCache = map[string]bool{} // download runs the download half of the get command // for the package named by the argument. func download(arg string, parent *Package, stk *importStack, mode int) { + if mode&useVendor != 0 { + // Caller is responsible for expanding vendor paths. + panic("internal error: download mode has useVendor set") + } load := func(path string, mode int) *Package { if parent == nil { return loadPackage(path, stk) @@ -315,32 +319,42 @@ func download(arg string, parent *Package, stk *importStack, mode int) { } // Process dependencies, now that we know what they are. - for _, path := range p.Imports { + imports := p.Imports + if mode&getTestDeps != 0 { + // Process test dependencies when -t is specified. + // (But don't get test dependencies for test dependencies: + // we always pass mode 0 to the recursive calls below.) + imports = stringList(imports, p.TestImports, p.XTestImports) + } + for i, path := range imports { if path == "C" { continue } - // Don't get test dependencies recursively. - // Imports is already vendor-expanded. - download(path, p, stk, 0) - } - if mode&getTestDeps != 0 { - // Process test dependencies when -t is specified. - // (Don't get test dependencies for test dependencies.) - // We pass useVendor here because p.load does not - // vendor-expand TestImports and XTestImports. - // The call to loadImport inside download needs to do that. - for _, path := range p.TestImports { - if path == "C" { - continue - } - download(path, p, stk, useVendor) + // Fail fast on import naming full vendor path. + // Otherwise expand path as needed for test imports. + // Note that p.Imports can have additional entries beyond p.build.Imports. + orig := path + if i < len(p.build.Imports) { + orig = p.build.Imports[i] } - for _, path := range p.XTestImports { - if path == "C" { - continue + if j, ok := findVendor(orig); ok { + stk.push(path) + err := &PackageError{ + ImportStack: stk.copy(), + Err: "must be imported as " + path[j+len("vendor/"):], } - download(path, p, stk, useVendor) + stk.pop() + errorf("%s", err) + continue + } + // If this is a test import, apply vendor lookup now. + // We cannot pass useVendor to download, because + // download does caching based on the value of path, + // so it must be the fully qualified path already. + if i >= len(p.Imports) { + path = vendoredImportPath(p, path) } + download(path, p, stk, 0) } if isWildcard { diff --git a/src/cmd/go/go_test.go b/src/cmd/go/go_test.go index 5731066fd6..5727eb094e 100644 --- a/src/cmd/go/go_test.go +++ b/src/cmd/go/go_test.go @@ -1700,6 +1700,8 @@ func TestMissingGOPATHEnvShowsDefault(t *testing.T) { // Test go get missing GOPATH causes go get to warn if directory doesn't exist. func TestMissingGOPATHGetWarnsIfNotExists(t *testing.T) { + testenv.MustHaveExternalNetwork(t) + if _, err := exec.LookPath("git"); err != nil { t.Skip("skipping because git binary not found") } @@ -1727,6 +1729,8 @@ func TestMissingGOPATHGetWarnsIfNotExists(t *testing.T) { // Test go get missing GOPATH causes no warning if directory exists. func TestMissingGOPATHGetDoesntWarnIfExists(t *testing.T) { + testenv.MustHaveExternalNetwork(t) + if _, err := exec.LookPath("git"); err != nil { t.Skip("skipping because git binary not found") } @@ -1757,6 +1761,8 @@ func TestMissingGOPATHGetDoesntWarnIfExists(t *testing.T) { // Test go get missing GOPATH fails if pointed file is not a directory. func TestMissingGOPATHGetFailsIfItsNotDirectory(t *testing.T) { + testenv.MustHaveExternalNetwork(t) + tg := testgo(t) defer tg.cleanup() @@ -1879,6 +1885,26 @@ func TestGoTestCpuprofileDashOControlsBinaryLocation(t *testing.T) { tg.wantExecutable("myerrors.test"+exeSuffix, "go test -cpuprofile -o myerrors.test did not create myerrors.test") } +func TestGoTestMutexprofileLeavesBinaryBehind(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + // TODO: tg.parallel() + tg.makeTempdir() + tg.cd(tg.path(".")) + tg.run("test", "-mutexprofile", "errors.prof", "errors") + tg.wantExecutable("errors.test"+exeSuffix, "go test -mutexprofile did not create errors.test") +} + +func TestGoTestMutexprofileDashOControlsBinaryLocation(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + // TODO: tg.parallel() + tg.makeTempdir() + tg.cd(tg.path(".")) + tg.run("test", "-mutexprofile", "errors.prof", "-o", "myerrors.test"+exeSuffix, "errors") + tg.wantExecutable("myerrors.test"+exeSuffix, "go test -mutexprofile -o myerrors.test did not create myerrors.test") +} + func TestGoTestDashCDashOControlsBinaryLocation(t *testing.T) { tg := testgo(t) defer tg.cleanup() @@ -2067,9 +2093,7 @@ func TestCaseCollisions(t *testing.T) { // Issue 8181. func TestGoGetDashTIssue8181(t *testing.T) { - if testing.Short() { - t.Skip("skipping test that uses network in short mode") - } + testenv.MustHaveExternalNetwork(t) tg := testgo(t) defer tg.cleanup() @@ -2083,9 +2107,7 @@ func TestGoGetDashTIssue8181(t *testing.T) { func TestIssue11307(t *testing.T) { // go get -u was not working except in checkout directory - if testing.Short() { - t.Skip("skipping test that uses network in short mode") - } + testenv.MustHaveExternalNetwork(t) tg := testgo(t) defer tg.cleanup() @@ -2245,6 +2267,28 @@ func TestCoverageImportMainLoop(t *testing.T) { tg.grepStderr("not an importable package", "did not detect import main") } +func TestTestEmpty(t *testing.T) { + if !canRace { + t.Skip("no race detector") + } + + wd, _ := os.Getwd() + testdata := filepath.Join(wd, "testdata") + + for _, dir := range []string{"pkg", "test", "xtest", "pkgtest", "pkgxtest", "pkgtestxtest", "testxtest"} { + t.Run(dir, func(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + tg.setenv("GOPATH", testdata) + tg.cd(filepath.Join(testdata, "src/empty/"+dir)) + tg.run("test", "-cover", "-coverpkg=.", "-race") + }) + if testing.Short() { + break + } + } +} + func TestBuildDryRunWithCgo(t *testing.T) { if !canCgo { t.Skip("skipping because cgo not enabled") @@ -3355,9 +3399,11 @@ func TestCgoConsistentResults(t *testing.T) { if !canCgo { t.Skip("skipping because cgo not enabled") } - if runtime.GOOS == "solaris" { - // See https://golang.org/issue/13247 - t.Skip("skipping because Solaris builds are known to be inconsistent; see #13247") + switch runtime.GOOS { + case "freebsd": + testenv.SkipFlaky(t, 15405) + case "solaris": + testenv.SkipFlaky(t, 13247) } tg := testgo(t) @@ -3716,3 +3762,28 @@ func TestLinkXImportPathEscape(t *testing.T) { tg.t.Fatal(`incorrect output: expected "linkXworked\n"`) } } + +// Issue 18044. +func TestLdBindNow(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + tg.parallel() + tg.setenv("LD_BIND_NOW", "1") + tg.run("help") +} + +// Issue 18225. +// This is really a cmd/asm issue but this is a convenient place to test it. +func TestConcurrentAsm(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + tg.parallel() + asm := `DATA ·constants<>+0x0(SB)/8,$0 +GLOBL ·constants<>(SB),8,$8 +` + tg.tempFile("go/src/p/a.s", asm) + tg.tempFile("go/src/p/b.s", asm) + tg.tempFile("go/src/p/p.go", `package p`) + tg.setenv("GOPATH", tg.path("go")) + tg.run("build", "p") +} diff --git a/src/cmd/go/help.go b/src/cmd/go/help.go index fb69d8ec54..0c663ad463 100644 --- a/src/cmd/go/help.go +++ b/src/cmd/go/help.go @@ -295,6 +295,8 @@ to a subdirectory named "go" in the user's home directory unless that directory holds a Go distribution. Run "go env GOPATH" to see the current GOPATH. +See https://golang.org/wiki/SettingGOPATH to set a custom GOPATH. + Each directory listed in GOPATH must have a prescribed structure: The src directory holds source code. The path below src diff --git a/src/cmd/go/http.go b/src/cmd/go/http.go index 1dc2c12c64..dcb4e9fea5 100644 --- a/src/cmd/go/http.go +++ b/src/cmd/go/http.go @@ -33,6 +33,7 @@ var httpClient = http.DefaultClient var impatientInsecureHTTPClient = &http.Client{ Timeout: 5 * time.Second, Transport: &http.Transport{ + Proxy: http.ProxyFromEnvironment, TLSClientConfig: &tls.Config{ InsecureSkipVerify: true, }, diff --git a/src/cmd/go/pkg.go b/src/cmd/go/pkg.go index 852a1a0db9..d69fa5118f 100644 --- a/src/cmd/go/pkg.go +++ b/src/cmd/go/pkg.go @@ -178,7 +178,9 @@ func (p *Package) copyBuild(pp *build.Package) { p.CgoCXXFLAGS = pp.CgoCXXFLAGS p.CgoLDFLAGS = pp.CgoLDFLAGS p.CgoPkgConfig = pp.CgoPkgConfig - p.Imports = pp.Imports + // We modify p.Imports in place, so make copy now. + p.Imports = make([]string, len(pp.Imports)) + copy(p.Imports, pp.Imports) p.TestGoFiles = pp.TestGoFiles p.TestImports = pp.TestImports p.XTestGoFiles = pp.XTestGoFiles @@ -953,6 +955,10 @@ func (p *Package) load(stk *importStack, bp *build.Package, err error) *Package if p.Name == "main" && goarch == "arm" { importPaths = append(importPaths, "math") } + // In coverage atomic mode everything depends on sync/atomic. + if testCoverMode == "atomic" && (!p.Standard || (p.ImportPath != "runtime/cgo" && p.ImportPath != "runtime/race" && p.ImportPath != "sync/atomic")) { + importPaths = append(importPaths, "sync/atomic") + } } // Runtime and its internal packages depend on runtime/internal/sys, diff --git a/src/cmd/go/test.go b/src/cmd/go/test.go index 95914d5f58..cdb167de75 100644 --- a/src/cmd/go/test.go +++ b/src/cmd/go/test.go @@ -13,7 +13,6 @@ import ( "go/doc" "go/parser" "go/token" - "io" "os" "os/exec" "path" @@ -201,7 +200,7 @@ const testFlag2 = ` text from Log and Logf calls even if the test succeeds. The following flags are also recognized by 'go test' and can be used to -profile the tests during execution:: +profile the tests during execution: -benchmem Print memory allocation statistics for benchmarks. @@ -244,7 +243,7 @@ profile the tests during execution:: Writes test binary as -c would. -mutexprofilefraction n - Sample 1 in n stack traces of goroutines holding a + Sample 1 in n stack traces of goroutines holding a contended mutex. -outputdir directory @@ -335,7 +334,8 @@ If the last comment in the function starts with "Output:" then the output is compared exactly against the comment (see examples below). If the last comment begins with "Unordered output:" then the output is compared to the comment, however the order of the lines is ignored. An example with no such -comment, or with no text after "Output:" is compiled but not executed. +comment is compiled but not executed. An example with no text after +"Output:" is compiled, executed, and expected to produce no output. Godoc displays the body of ExampleXXX to demonstrate the use of the function, constant, or variable XXX. An example of a method M with @@ -1122,12 +1122,8 @@ func (b *builder) runTest(a *action) error { cmd.Env = envForDir(cmd.Dir, origEnv) var buf bytes.Buffer if testStreamOutput { - // The only way to keep the ordering of the messages and still - // intercept its contents. os/exec will share the same Pipe for - // both Stdout and Stderr when running the test program. - mw := io.MultiWriter(os.Stdout, &buf) - cmd.Stdout = mw - cmd.Stderr = mw + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr } else { cmd.Stdout = &buf cmd.Stderr = &buf @@ -1192,7 +1188,7 @@ func (b *builder) runTest(a *action) error { t := fmt.Sprintf("%.3fs", time.Since(t0).Seconds()) if err == nil { norun := "" - if testShowPass && !testStreamOutput { + if testShowPass { a.testOutput.Write(out) } if bytes.HasPrefix(out, noTestsToRun[1:]) || bytes.Contains(out, noTestsToRun) { @@ -1204,9 +1200,7 @@ func (b *builder) runTest(a *action) error { setExitStatus(1) if len(out) > 0 { - if !testStreamOutput { - a.testOutput.Write(out) - } + a.testOutput.Write(out) // assume printing the test binary's exit status is superfluous } else { fmt.Fprintf(a.testOutput, "%s\n", err) diff --git a/src/cmd/go/testdata/src/empty/pkg/pkg.go b/src/cmd/go/testdata/src/empty/pkg/pkg.go new file mode 100644 index 0000000000..c89cd18d0f --- /dev/null +++ b/src/cmd/go/testdata/src/empty/pkg/pkg.go @@ -0,0 +1 @@ +package p diff --git a/src/cmd/go/testdata/src/empty/pkgtest/pkg.go b/src/cmd/go/testdata/src/empty/pkgtest/pkg.go new file mode 100644 index 0000000000..c89cd18d0f --- /dev/null +++ b/src/cmd/go/testdata/src/empty/pkgtest/pkg.go @@ -0,0 +1 @@ +package p diff --git a/src/cmd/go/testdata/src/empty/pkgtest/test_test.go b/src/cmd/go/testdata/src/empty/pkgtest/test_test.go new file mode 100644 index 0000000000..c89cd18d0f --- /dev/null +++ b/src/cmd/go/testdata/src/empty/pkgtest/test_test.go @@ -0,0 +1 @@ +package p diff --git a/src/cmd/go/testdata/src/empty/pkgtestxtest/pkg.go b/src/cmd/go/testdata/src/empty/pkgtestxtest/pkg.go new file mode 100644 index 0000000000..c89cd18d0f --- /dev/null +++ b/src/cmd/go/testdata/src/empty/pkgtestxtest/pkg.go @@ -0,0 +1 @@ +package p diff --git a/src/cmd/go/testdata/src/empty/pkgtestxtest/test_test.go b/src/cmd/go/testdata/src/empty/pkgtestxtest/test_test.go new file mode 100644 index 0000000000..c89cd18d0f --- /dev/null +++ b/src/cmd/go/testdata/src/empty/pkgtestxtest/test_test.go @@ -0,0 +1 @@ +package p diff --git a/src/cmd/go/testdata/src/empty/pkgtestxtest/xtest_test.go b/src/cmd/go/testdata/src/empty/pkgtestxtest/xtest_test.go new file mode 100644 index 0000000000..9b64e8e1a2 --- /dev/null +++ b/src/cmd/go/testdata/src/empty/pkgtestxtest/xtest_test.go @@ -0,0 +1 @@ +package p_test diff --git a/src/cmd/go/testdata/src/empty/pkgxtest/pkg.go b/src/cmd/go/testdata/src/empty/pkgxtest/pkg.go new file mode 100644 index 0000000000..c89cd18d0f --- /dev/null +++ b/src/cmd/go/testdata/src/empty/pkgxtest/pkg.go @@ -0,0 +1 @@ +package p diff --git a/src/cmd/go/testdata/src/empty/pkgxtest/xtest_test.go b/src/cmd/go/testdata/src/empty/pkgxtest/xtest_test.go new file mode 100644 index 0000000000..9b64e8e1a2 --- /dev/null +++ b/src/cmd/go/testdata/src/empty/pkgxtest/xtest_test.go @@ -0,0 +1 @@ +package p_test diff --git a/src/cmd/go/testdata/src/empty/test/test_test.go b/src/cmd/go/testdata/src/empty/test/test_test.go new file mode 100644 index 0000000000..c89cd18d0f --- /dev/null +++ b/src/cmd/go/testdata/src/empty/test/test_test.go @@ -0,0 +1 @@ +package p diff --git a/src/cmd/go/testdata/src/empty/testxtest/test_test.go b/src/cmd/go/testdata/src/empty/testxtest/test_test.go new file mode 100644 index 0000000000..c89cd18d0f --- /dev/null +++ b/src/cmd/go/testdata/src/empty/testxtest/test_test.go @@ -0,0 +1 @@ +package p diff --git a/src/cmd/go/testdata/src/empty/testxtest/xtest_test.go b/src/cmd/go/testdata/src/empty/testxtest/xtest_test.go new file mode 100644 index 0000000000..9b64e8e1a2 --- /dev/null +++ b/src/cmd/go/testdata/src/empty/testxtest/xtest_test.go @@ -0,0 +1 @@ +package p_test diff --git a/src/cmd/go/testdata/src/empty/xtest/xtest_test.go b/src/cmd/go/testdata/src/empty/xtest/xtest_test.go new file mode 100644 index 0000000000..9b64e8e1a2 --- /dev/null +++ b/src/cmd/go/testdata/src/empty/xtest/xtest_test.go @@ -0,0 +1 @@ +package p_test diff --git a/src/cmd/go/testdata/testterminal18153/terminal_test.go b/src/cmd/go/testdata/testterminal18153/terminal_test.go new file mode 100644 index 0000000000..d662e55ee5 --- /dev/null +++ b/src/cmd/go/testdata/testterminal18153/terminal_test.go @@ -0,0 +1,39 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build linux + +// This test is run by src/cmd/dist/test.go (cmd_go_test_terminal), +// and not by cmd/go's tests. This is because this test requires that +// that it be called with its stdout and stderr being a terminal. +// dist doesn't run `cmd/go test` against this test directory if +// dist's stdout/stderr aren't terminals. +// +// See issue 18153. + +package p + +import ( + "syscall" + "testing" + "unsafe" +) + +const ioctlReadTermios = syscall.TCGETS + +// isTerminal reports whether fd is a terminal. +func isTerminal(fd uintptr) bool { + var termios syscall.Termios + _, _, err := syscall.Syscall6(syscall.SYS_IOCTL, fd, ioctlReadTermios, uintptr(unsafe.Pointer(&termios)), 0, 0, 0) + return err == 0 +} + +func TestIsTerminal(t *testing.T) { + if !isTerminal(1) { + t.Errorf("stdout is not a terminal") + } + if !isTerminal(2) { + t.Errorf("stderr is not a terminal") + } +} diff --git a/src/cmd/go/testflag.go b/src/cmd/go/testflag.go index cf4d2b47ff..fa53bfcdf0 100644 --- a/src/cmd/go/testflag.go +++ b/src/cmd/go/testflag.go @@ -151,10 +151,10 @@ func testFlags(args []string) (packageNames, passToTest []string) { testBench = true case "timeout": testTimeout = value - case "blockprofile", "cpuprofile", "memprofile": + case "blockprofile", "cpuprofile", "memprofile", "mutexprofile": testProfile = true testNeedBinary = true - case "mutexprofile", "trace": + case "trace": testProfile = true case "coverpkg": testCover = true diff --git a/src/cmd/go/vendor_test.go b/src/cmd/go/vendor_test.go index 226b5377b9..deec02e341 100644 --- a/src/cmd/go/vendor_test.go +++ b/src/cmd/go/vendor_test.go @@ -188,6 +188,42 @@ func TestVendorGetUpdate(t *testing.T) { tg.run("get", "-u", "github.com/rsc/go-get-issue-11864") } +func TestVendorGetU(t *testing.T) { + testenv.MustHaveExternalNetwork(t) + + tg := testgo(t) + defer tg.cleanup() + tg.makeTempdir() + tg.setenv("GOPATH", tg.path(".")) + tg.run("get", "-u", "github.com/rsc/go-get-issue-11864") +} + +func TestVendorGetTU(t *testing.T) { + testenv.MustHaveExternalNetwork(t) + + tg := testgo(t) + defer tg.cleanup() + tg.makeTempdir() + tg.setenv("GOPATH", tg.path(".")) + tg.run("get", "-t", "-u", "github.com/rsc/go-get-issue-11864/...") +} + +func TestVendorGetBadVendor(t *testing.T) { + testenv.MustHaveExternalNetwork(t) + + for _, suffix := range []string{"bad/imp", "bad/imp2", "bad/imp3", "..."} { + t.Run(suffix, func(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + tg.makeTempdir() + tg.setenv("GOPATH", tg.path(".")) + tg.runFail("get", "-t", "-u", "github.com/rsc/go-get-issue-18219/"+suffix) + tg.grepStderr("must be imported as", "did not find error about vendor import") + tg.mustNotExist(tg.path("src/github.com/rsc/vendor")) + }) + } +} + func TestGetSubmodules(t *testing.T) { testenv.MustHaveExternalNetwork(t) diff --git a/src/cmd/internal/obj/go.go b/src/cmd/internal/obj/go.go index 1852dc74f6..732ce19634 100644 --- a/src/cmd/internal/obj/go.go +++ b/src/cmd/internal/obj/go.go @@ -13,8 +13,9 @@ import ( // go-specific code shared across loaders (5l, 6l, 8l). var ( - framepointer_enabled int - Fieldtrack_enabled int + framepointer_enabled int + Fieldtrack_enabled int + Preemptibleloops_enabled int ) // Toolchain experiments. @@ -27,6 +28,7 @@ var exper = []struct { }{ {"fieldtrack", &Fieldtrack_enabled}, {"framepointer", &framepointer_enabled}, + {"preemptibleloops", &Preemptibleloops_enabled}, } func addexp(s string) { diff --git a/src/cmd/internal/obj/obj.go b/src/cmd/internal/obj/line.go index 566263d3d7..566263d3d7 100644 --- a/src/cmd/internal/obj/obj.go +++ b/src/cmd/internal/obj/line.go diff --git a/src/cmd/internal/obj/mips/obj0.go b/src/cmd/internal/obj/mips/obj0.go index 221fd428a3..12cfb31377 100644 --- a/src/cmd/internal/obj/mips/obj0.go +++ b/src/cmd/internal/obj/mips/obj0.go @@ -32,6 +32,7 @@ package mips import ( "cmd/internal/obj" "cmd/internal/sys" + "encoding/binary" "fmt" "math" ) @@ -533,6 +534,43 @@ func preprocess(ctxt *obj.Link, cursym *obj.LSym) { } } + if ctxt.Mode&Mips32 != 0 { + // rewrite MOVD into two MOVF in 32-bit mode to avoid unaligned memory access + for p = cursym.Text; p != nil; p = p1 { + p1 = p.Link + + if p.As != AMOVD { + continue + } + if p.From.Type != obj.TYPE_MEM && p.To.Type != obj.TYPE_MEM { + continue + } + + p.As = AMOVF + q = ctxt.NewProg() + *q = *p + q.Link = p.Link + p.Link = q + p1 = q.Link + + var regOff int16 + if ctxt.Arch.ByteOrder == binary.BigEndian { + regOff = 1 // load odd register first + } + if p.From.Type == obj.TYPE_MEM { + reg := REG_F0 + (p.To.Reg-REG_F0)&^1 + p.To.Reg = reg + regOff + q.To.Reg = reg + 1 - regOff + q.From.Offset += 4 + } else if p.To.Type == obj.TYPE_MEM { + reg := REG_F0 + (p.From.Reg-REG_F0)&^1 + p.From.Reg = reg + regOff + q.From.Reg = reg + 1 - regOff + q.To.Offset += 4 + } + } + } + if nosched { // if we don't do instruction scheduling, simply add // NOP after each branch instruction. diff --git a/src/cmd/internal/obj/pcln.go b/src/cmd/internal/obj/pcln.go index d9893e42cd..e4a724ac27 100644 --- a/src/cmd/internal/obj/pcln.go +++ b/src/cmd/internal/obj/pcln.go @@ -6,9 +6,8 @@ package obj import "log" -func addvarint(ctxt *Link, d *Pcdata, val uint32) { - var v uint32 - for v = val; v >= 0x80; v >>= 7 { +func addvarint(d *Pcdata, v uint32) { + for ; v >= 0x80; v >>= 7 { d.P = append(d.P, uint8(v|0x80)) } d.P = append(d.P, uint8(v)) @@ -98,7 +97,7 @@ func funcpctab(ctxt *Link, dst *Pcdata, func_ *LSym, desc string, valfunc func(* } if started != 0 { - addvarint(ctxt, dst, uint32((p.Pc-pc)/int64(ctxt.Arch.MinLC))) + addvarint(dst, uint32((p.Pc-pc)/int64(ctxt.Arch.MinLC))) pc = p.Pc } @@ -108,7 +107,7 @@ func funcpctab(ctxt *Link, dst *Pcdata, func_ *LSym, desc string, valfunc func(* } else { delta <<= 1 } - addvarint(ctxt, dst, delta) + addvarint(dst, delta) oldval = val started = 1 val = valfunc(ctxt, func_, val, p, 1, arg) @@ -118,8 +117,8 @@ func funcpctab(ctxt *Link, dst *Pcdata, func_ *LSym, desc string, valfunc func(* if ctxt.Debugpcln != 0 { ctxt.Logf("%6x done\n", uint64(func_.Text.Pc+func_.Size)) } - addvarint(ctxt, dst, uint32((func_.Size-pc)/int64(ctxt.Arch.MinLC))) - addvarint(ctxt, dst, 0) // terminator + addvarint(dst, uint32((func_.Size-pc)/int64(ctxt.Arch.MinLC))) + addvarint(dst, 0) // terminator } if ctxt.Debugpcln != 0 { diff --git a/src/cmd/internal/obj/reloctype_string.go b/src/cmd/internal/obj/reloctype_string.go index 6de617cd78..09c1312df5 100644 --- a/src/cmd/internal/obj/reloctype_string.go +++ b/src/cmd/internal/obj/reloctype_string.go @@ -4,9 +4,9 @@ package obj import "fmt" -const _RelocType_name = "R_ADDRR_ADDRPOWERR_ADDRARM64R_ADDRMIPSR_ADDROFFR_SIZER_CALLR_CALLARMR_CALLARM64R_CALLINDR_CALLPOWERR_CALLMIPSR_CONSTR_PCRELR_TLS_LER_TLS_IER_GOTOFFR_PLT0R_PLT1R_PLT2R_USEFIELDR_USETYPER_METHODOFFR_POWER_TOCR_GOTPCRELR_JMPMIPSR_DWARFREFR_ARM64_TLS_LER_ARM64_TLS_IER_ARM64_GOTPCRELR_POWER_TLS_LER_POWER_TLS_IER_POWER_TLSR_ADDRPOWER_DSR_ADDRPOWER_GOTR_ADDRPOWER_PCRELR_ADDRPOWER_TOCRELR_ADDRPOWER_TOCREL_DSR_PCRELDBLR_ADDRMIPSUR_ADDRMIPSTLS" +const _RelocType_name = "R_ADDRR_ADDRPOWERR_ADDRARM64R_ADDRMIPSR_ADDROFFR_WEAKADDROFFR_SIZER_CALLR_CALLARMR_CALLARM64R_CALLINDR_CALLPOWERR_CALLMIPSR_CONSTR_PCRELR_TLS_LER_TLS_IER_GOTOFFR_PLT0R_PLT1R_PLT2R_USEFIELDR_USETYPER_METHODOFFR_POWER_TOCR_GOTPCRELR_JMPMIPSR_DWARFREFR_ARM64_TLS_LER_ARM64_TLS_IER_ARM64_GOTPCRELR_POWER_TLS_LER_POWER_TLS_IER_POWER_TLSR_ADDRPOWER_DSR_ADDRPOWER_GOTR_ADDRPOWER_PCRELR_ADDRPOWER_TOCRELR_ADDRPOWER_TOCREL_DSR_PCRELDBLR_ADDRMIPSUR_ADDRMIPSTLS" -var _RelocType_index = [...]uint16{0, 6, 17, 28, 38, 47, 53, 59, 68, 79, 88, 99, 109, 116, 123, 131, 139, 147, 153, 159, 165, 175, 184, 195, 206, 216, 225, 235, 249, 263, 279, 293, 307, 318, 332, 347, 364, 382, 403, 413, 424, 437} +var _RelocType_index = [...]uint16{0, 6, 17, 28, 38, 47, 60, 66, 72, 81, 92, 101, 112, 122, 129, 136, 144, 152, 160, 166, 172, 178, 188, 197, 208, 219, 229, 238, 248, 262, 276, 292, 306, 320, 331, 345, 360, 377, 395, 416, 426, 437, 450} func (i RelocType) String() string { i -= 1 diff --git a/src/cmd/internal/obj/x86/obj6.go b/src/cmd/internal/obj/x86/obj6.go index 102d8c3c4f..eb6f867ca7 100644 --- a/src/cmd/internal/obj/x86/obj6.go +++ b/src/cmd/internal/obj/x86/obj6.go @@ -632,11 +632,27 @@ func preprocess(ctxt *obj.Link, cursym *obj.LSym) { autoffset = 0 } + hasCall := false + for q := p; q != nil; q = q.Link { + if q.As == obj.ACALL || q.As == obj.ADUFFCOPY || q.As == obj.ADUFFZERO { + hasCall = true + break + } + } + var bpsize int - if p.Mode == 64 && ctxt.Framepointer_enabled && autoffset > 0 && p.From3.Offset&obj.NOFRAME == 0 { - // Make room for to save a base pointer. If autoffset == 0, - // this might do something special like a tail jump to - // another function, so in that case we omit this. + if p.Mode == 64 && ctxt.Framepointer_enabled && + p.From3.Offset&obj.NOFRAME == 0 && // (1) below + !(autoffset == 0 && p.From3.Offset&obj.NOSPLIT != 0) && // (2) below + !(autoffset == 0 && !hasCall) { // (3) below + // Make room to save a base pointer. + // There are 2 cases we must avoid: + // 1) If noframe is set (which we do for functions which tail call). + // 2) Scary runtime internals which would be all messed up by frame pointers. + // We detect these using a heuristic: frameless nosplit functions. + // TODO: Maybe someday we label them all with NOFRAME and get rid of this heuristic. + // For performance, we also want to avoid: + // 3) Frameless leaf functions bpsize = ctxt.Arch.PtrSize autoffset += int32(bpsize) p.To.Offset += int64(bpsize) diff --git a/src/cmd/internal/objfile/goobj.go b/src/cmd/internal/objfile/goobj.go index 230137e0f5..c04987cc82 100644 --- a/src/cmd/internal/objfile/goobj.go +++ b/src/cmd/internal/objfile/goobj.go @@ -114,14 +114,14 @@ func (f *goobjFile) PCToLine(pc uint64) (string, int, *gosym.Func) { if err != nil { return "", 0, nil } - fileID := gosym.PCValue(pcfile, pc-uint64(s.Data.Offset), arch.MinLC) + fileID := int(pcValue(pcfile, pc-uint64(s.Data.Offset), arch)) fileName := s.Func.File[fileID] pcline := make([]byte, s.Func.PCLine.Size) _, err = f.f.ReadAt(pcline, s.Func.PCLine.Offset) if err != nil { return "", 0, nil } - line := gosym.PCValue(pcline, pc-uint64(s.Data.Offset), arch.MinLC) + line := int(pcValue(pcline, pc-uint64(s.Data.Offset), arch)) // Note: we provide only the name in the Func structure. // We could provide more if needed. return fileName, line, &gosym.Func{Sym: &gosym.Sym{Name: s.Name}} @@ -129,6 +129,53 @@ func (f *goobjFile) PCToLine(pc uint64) (string, int, *gosym.Func) { return "", 0, nil } +// pcValue looks up the given PC in a pc value table. target is the +// offset of the pc from the entry point. +func pcValue(tab []byte, target uint64, arch *sys.Arch) int32 { + val := int32(-1) + var pc uint64 + for step(&tab, &pc, &val, pc == 0, arch) { + if target < pc { + return val + } + } + return -1 +} + +// step advances to the next pc, value pair in the encoded table. +func step(p *[]byte, pc *uint64, val *int32, first bool, arch *sys.Arch) bool { + uvdelta := readvarint(p) + if uvdelta == 0 && !first { + return false + } + if uvdelta&1 != 0 { + uvdelta = ^(uvdelta >> 1) + } else { + uvdelta >>= 1 + } + vdelta := int32(uvdelta) + pcdelta := readvarint(p) * uint32(arch.MinLC) + *pc += uint64(pcdelta) + *val += vdelta + return true +} + +// readvarint reads, removes, and returns a varint from *p. +func readvarint(p *[]byte) uint32 { + var v, shift uint32 + s := *p + for shift = 0; ; shift += 7 { + b := s[0] + s = s[1:] + v |= (uint32(b) & 0x7F) << shift + if b&0x80 == 0 { + break + } + } + *p = s + return v +} + // We treat the whole object file as the text section. func (f *goobjFile) text() (textStart uint64, text []byte, err error) { var info os.FileInfo diff --git a/src/cmd/link/internal/arm/obj.go b/src/cmd/link/internal/arm/obj.go index 4feaa096a7..05b90d20f2 100644 --- a/src/cmd/link/internal/arm/obj.go +++ b/src/cmd/link/internal/arm/obj.go @@ -104,7 +104,7 @@ func archinit(ctxt *ld.Link) { *ld.FlagDataAddr = 0 } if *ld.FlagRound == -1 { - *ld.FlagRound = 4096 + *ld.FlagRound = 0x10000 } case obj.Hnacl: diff --git a/src/cmd/link/internal/ld/config.go b/src/cmd/link/internal/ld/config.go index c9ee8847ad..7d00ff1675 100644 --- a/src/cmd/link/internal/ld/config.go +++ b/src/cmd/link/internal/ld/config.go @@ -184,7 +184,7 @@ func mustLinkExternal(ctxt *Link) (res bool, reason string) { // Internally linking cgo is incomplete on some architectures. // https://golang.org/issue/10373 // https://golang.org/issue/14449 - if iscgo && SysArch.InFamily(sys.ARM64, sys.MIPS64) { + if iscgo && SysArch.InFamily(sys.ARM64, sys.MIPS64, sys.MIPS) { return true, obj.GOARCH + " does not support internal cgo" } diff --git a/src/cmd/link/internal/ld/data.go b/src/cmd/link/internal/ld/data.go index eaf6aa2080..aca8973a85 100644 --- a/src/cmd/link/internal/ld/data.go +++ b/src/cmd/link/internal/ld/data.go @@ -1250,7 +1250,7 @@ func (p *GCProg) AddSym(s *Symbol) { } // dataSortKey is used to sort a slice of data symbol *Symbol pointers. -// The sort keys are kept inline to improve cache behaviour while sorting. +// The sort keys are kept inline to improve cache behavior while sorting. type dataSortKey struct { size int64 name string @@ -1976,6 +1976,13 @@ func dodataSect(ctxt *Link, symn obj.SymKind, syms []*Symbol) (result []*Symbol, copy(syms[first+2:], syms[first+1:second]) syms[first+0] = rel syms[first+1] = plt + + // Make sure alignment doesn't introduce a gap. + // Setting the alignment explicitly prevents + // symalign from basing it on the size and + // getting it wrong. + rel.Align = int32(SysArch.RegSize) + plt.Align = int32(SysArch.RegSize) } } @@ -2107,6 +2114,7 @@ func assignAddress(ctxt *Link, sect *Section, n int, sym *Symbol, va uint64) (*S // Create new section, set the starting Vaddr sect = addsection(&Segtext, ".text", 05) sect.Vaddr = va + sym.Sect = sect // Create a symbol for the start of the secondary text sections ctxt.Syms.Lookup(fmt.Sprintf("runtime.text.%d", n), 0).Sect = sect diff --git a/src/cmd/link/internal/ld/elf.go b/src/cmd/link/internal/ld/elf.go index b4d5aae27a..1d8a5dd35e 100644 --- a/src/cmd/link/internal/ld/elf.go +++ b/src/cmd/link/internal/ld/elf.go @@ -10,6 +10,7 @@ import ( "crypto/sha1" "encoding/binary" "encoding/hex" + "io" "path/filepath" "sort" "strings" @@ -961,7 +962,7 @@ func Elfinit(ctxt *Link) { ehdr.flags = 0x5000002 // has entry point, Version5 EABI } } else if SysArch.Family == sys.MIPS { - ehdr.flags = 0x50000000 /* MIPS 32 */ + ehdr.flags = 0x50001004 /* MIPS 32 CPIC O32*/ } fallthrough default: @@ -2130,7 +2131,7 @@ func (ctxt *Link) doelf() { sort.Sort(byPkg(ctxt.Library)) h := sha1.New() for _, l := range ctxt.Library { - h.Write(l.hash) + io.WriteString(h, l.hash) } addgonote(ctxt, ".note.go.abihash", ELF_NOTE_GOABIHASH_TAG, h.Sum([]byte{})) addgonote(ctxt, ".note.go.pkg-list", ELF_NOTE_GOPKGLIST_TAG, pkglistfornote) @@ -2776,7 +2777,7 @@ func Elfadddynsym(ctxt *Link, s *Symbol) { /* type */ t := STB_GLOBAL << 4 - // TODO(mwhudson): presumably the behaviour should actually be the same on both arm and 386. + // TODO(mwhudson): presumably the behavior should actually be the same on both arm and 386. if SysArch.Family == sys.I386 && s.Attr.CgoExport() && s.Type&obj.SMASK == obj.STEXT { t |= STT_FUNC } else if SysArch.Family == sys.ARM && s.Attr.CgoExportDynamic() && s.Type&obj.SMASK == obj.STEXT { diff --git a/src/cmd/link/internal/ld/lib.go b/src/cmd/link/internal/ld/lib.go index fb321905e1..74d79d394c 100644 --- a/src/cmd/link/internal/ld/lib.go +++ b/src/cmd/link/internal/ld/lib.go @@ -39,6 +39,7 @@ import ( "crypto/sha1" "debug/elf" "encoding/binary" + "encoding/hex" "fmt" "io" "io/ioutil" @@ -603,6 +604,16 @@ func (ctxt *Link) loadlib() { } } + // If package versioning is required, generate a hash of the + // the packages used in the link. + if Buildmode == BuildmodeShared || Buildmode == BuildmodePlugin || ctxt.Syms.ROLookup("plugin.Open", 0) != nil { + for i = 0; i < len(ctxt.Library); i++ { + if ctxt.Library[i].Shlib == "" { + genhash(ctxt, ctxt.Library[i]) + } + } + } + if SysArch == sys.Arch386 { if (Buildmode == BuildmodeCArchive && Iself) || Buildmode == BuildmodeCShared || Buildmode == BuildmodePIE || ctxt.DynlinkingGo() { got := ctxt.Syms.Lookup("_GLOBAL_OFFSET_TABLE_", 0) @@ -678,6 +689,29 @@ func nextar(bp *bio.Reader, off int64, a *ArHdr) int64 { return arsize + SAR_HDR } +func genhash(ctxt *Link, lib *Library) { + f, err := bio.Open(lib.File) + if err != nil { + Errorf(nil, "cannot open file %s for hash generation: %v", lib.File, err) + return + } + defer f.Close() + + var arhdr ArHdr + l := nextar(f, int64(len(ARMAG)), &arhdr) + if l <= 0 { + Errorf(nil, "%s: short read on archive file symbol header", lib.File) + return + } + + h := sha1.New() + if _, err := io.CopyN(h, f, atolwhex(arhdr.size)); err != nil { + Errorf(nil, "bad read of %s for hash generation: %v", lib.File, err) + return + } + lib.hash = hex.EncodeToString(h.Sum(nil)) +} + func objfile(ctxt *Link, lib *Library) { pkg := pathtoprefix(lib.Pkg) @@ -720,17 +754,6 @@ func objfile(ctxt *Link, lib *Library) { goto out } - if Buildmode == BuildmodeShared || Buildmode == BuildmodePlugin || ctxt.Syms.ROLookup("plugin.Open", 0) != nil { - before := f.Offset() - pkgdefBytes := make([]byte, atolwhex(arhdr.size)) - if _, err := io.ReadFull(f, pkgdefBytes); err != nil { - Errorf(nil, "%s: short read on archive file symbol header: %v", lib.File, err) - } - hash := sha1.Sum(pkgdefBytes) - lib.hash = hash[:] - f.Seek(before, 0) - } - off += l ldpkg(ctxt, f, pkg, atolwhex(arhdr.size), lib.File, Pkgdef) @@ -1234,6 +1257,8 @@ func hostlinkArchArgs() []string { // nothing needed case sys.MIPS64: return []string{"-mabi=64"} + case sys.MIPS: + return []string{"-mabi=32"} } return nil } diff --git a/src/cmd/link/internal/ld/link.go b/src/cmd/link/internal/ld/link.go index ab7e49b51f..ffe0873fc8 100644 --- a/src/cmd/link/internal/ld/link.go +++ b/src/cmd/link/internal/ld/link.go @@ -223,7 +223,7 @@ type Library struct { File string Pkg string Shlib string - hash []byte + hash string imports []*Library textp []*Symbol // text symbols defined in this library dupTextSyms []*Symbol // dupok text symbols defined in this library diff --git a/src/cmd/link/internal/ld/macho.go b/src/cmd/link/internal/ld/macho.go index c88af64a3a..f3687daa91 100644 --- a/src/cmd/link/internal/ld/macho.go +++ b/src/cmd/link/internal/ld/macho.go @@ -684,6 +684,29 @@ func machosymorder(ctxt *Link) { } } +// machoShouldExport reports whether a symbol needs to be exported. +// +// When dynamically linking, all non-local variables and plugin-exported +// symbols need to be exported. +func machoShouldExport(ctxt *Link, s *Symbol) bool { + if !ctxt.DynlinkingGo() || s.Attr.Local() { + return false + } + if Buildmode == BuildmodePlugin && strings.HasPrefix(s.Extname, *flagPluginPath) { + return true + } + if strings.HasPrefix(s.Name, "type.") && !strings.HasPrefix(s.Name, "type..") { + // reduce runtime typemap pressure, but do not + // export alg functions (type..*), as these + // appear in pclntable. + return true + } + if strings.HasPrefix(s.Name, "go.link.pkghash") { + return true + } + return s.Type >= obj.SELFSECT // only writable sections +} + func machosymtab(ctxt *Link) { symtab := ctxt.Syms.Lookup(".machosymtab", 0) symstr := ctxt.Syms.Lookup(".machosymstr", 0) @@ -692,13 +715,17 @@ func machosymtab(ctxt *Link) { s := sortsym[i] Adduint32(ctxt, symtab, uint32(symstr.Size)) + export := machoShouldExport(ctxt, s) + // In normal buildmodes, only add _ to C symbols, as // Go symbols have dot in the name. // - // When dynamically linking, prefix all non-local - // symbols with _ as dlsym on darwin requires it to - // resolve any symbol. - if !strings.Contains(s.Extname, ".") || (ctxt.DynlinkingGo() && !s.Attr.Local()) { + // Do not export C symbols in plugins, as runtime C + // symbols like crosscall2 are in pclntab and end up + // pointing at the host binary, breaking unwinding. + // See Issue #18190. + cexport := !strings.Contains(s.Extname, ".") && (Buildmode != BuildmodePlugin || onlycsymbol(s)) + if cexport || export { Adduint8(ctxt, symstr, '_') } @@ -711,7 +738,7 @@ func machosymtab(ctxt *Link) { Adduint16(ctxt, symtab, 0) // desc adduintxx(ctxt, symtab, 0, SysArch.PtrSize) // no value } else { - if s.Attr.CgoExport() || (ctxt.DynlinkingGo() && !s.Attr.Local()) { + if s.Attr.CgoExport() || export { Adduint8(ctxt, symtab, 0x0f) } else { Adduint8(ctxt, symtab, 0x0e) diff --git a/src/cmd/link/internal/ld/pcln.go b/src/cmd/link/internal/ld/pcln.go index 5a6c425f3e..1ebd7de662 100644 --- a/src/cmd/link/internal/ld/pcln.go +++ b/src/cmd/link/internal/ld/pcln.go @@ -154,10 +154,26 @@ func renumberfiles(ctxt *Link, files []*Symbol, d *Pcdata) { *d = out } +// onlycsymbol reports whether this is a cgo symbol provided by the +// runtime and only used from C code. +func onlycsymbol(s *Symbol) bool { + switch s.Name { + case "_cgo_topofstack", "_cgo_panic", "crosscall2": + return true + } + return false +} + func container(s *Symbol) int { + if s == nil { + return 0 + } + if Buildmode == BuildmodePlugin && onlycsymbol(s) { + return 1 + } // We want to generate func table entries only for the "lowest level" symbols, // not containers of subsymbols. - if s != nil && s.Type&obj.SCONTAINER != 0 { + if s.Type&obj.SCONTAINER != 0 { return 1 } return 0 @@ -232,6 +248,9 @@ func (ctxt *Link) pclntab() { setaddr(ctxt, ftab, 8+int64(SysArch.PtrSize)+int64(nfunc)*2*int64(SysArch.PtrSize), s) setuintxx(ctxt, ftab, 8+int64(SysArch.PtrSize)+int64(nfunc)*2*int64(SysArch.PtrSize)+int64(SysArch.PtrSize), uint64(funcstart), int64(SysArch.PtrSize)) + // Write runtime._func. Keep in sync with ../../../../runtime/runtime2.go:/_func + // and package debug/gosym. + // fixed size of struct, checked below off := funcstart diff --git a/src/cmd/link/internal/ld/symtab.go b/src/cmd/link/internal/ld/symtab.go index 98ce3ad79b..dd0e5407e8 100644 --- a/src/cmd/link/internal/ld/symtab.go +++ b/src/cmd/link/internal/ld/symtab.go @@ -602,8 +602,7 @@ func (ctxt *Link) symtab() { adduint(ctxt, moduledata, uint64(nitablinks)) adduint(ctxt, moduledata, uint64(nitablinks)) // The ptab slice - if ptab := ctxt.Syms.ROLookup("go.plugin.tabs", 0); ptab != nil { - ptab.Attr |= AttrReachable + if ptab := ctxt.Syms.ROLookup("go.plugin.tabs", 0); ptab != nil && ptab.Attr.Reachable() { ptab.Attr |= AttrLocal ptab.Type = obj.SRODATA diff --git a/src/cmd/link/internal/mips/asm.go b/src/cmd/link/internal/mips/asm.go index b2c72893a1..a591b06dfa 100644 --- a/src/cmd/link/internal/mips/asm.go +++ b/src/cmd/link/internal/mips/asm.go @@ -47,7 +47,33 @@ func adddynrel(ctxt *ld.Link, s *ld.Symbol, r *ld.Reloc) bool { } func elfreloc1(ctxt *ld.Link, r *ld.Reloc, sectoff int64) int { - return -1 + ld.Thearch.Lput(uint32(sectoff)) + + elfsym := r.Xsym.ElfsymForReloc() + switch r.Type { + default: + return -1 + + case obj.R_ADDR: + if r.Siz != 4 { + return -1 + } + ld.Thearch.Lput(ld.R_MIPS_32 | uint32(elfsym)<<8) + + case obj.R_ADDRMIPS: + ld.Thearch.Lput(ld.R_MIPS_LO16 | uint32(elfsym)<<8) + + case obj.R_ADDRMIPSU: + ld.Thearch.Lput(ld.R_MIPS_HI16 | uint32(elfsym)<<8) + + case obj.R_ADDRMIPSTLS: + ld.Thearch.Lput(ld.R_MIPS_TLS_TPREL_LO16 | uint32(elfsym)<<8) + + case obj.R_CALLMIPS, obj.R_JMPMIPS: + ld.Thearch.Lput(ld.R_MIPS_26 | uint32(elfsym)<<8) + } + + return 0 } func elfsetupplt(ctxt *ld.Link) { @@ -58,9 +84,50 @@ func machoreloc1(s *ld.Symbol, r *ld.Reloc, sectoff int64) int { return -1 } +func applyrel(r *ld.Reloc, s *ld.Symbol, val *int64, t int64) { + o := ld.SysArch.ByteOrder.Uint32(s.P[r.Off:]) + switch r.Type { + case obj.R_ADDRMIPS, obj.R_ADDRMIPSTLS: + *val = int64(o&0xffff0000 | uint32(t)&0xffff) + case obj.R_ADDRMIPSU: + *val = int64(o&0xffff0000 | uint32((t+(1<<15))>>16)&0xffff) + case obj.R_CALLMIPS, obj.R_JMPMIPS: + *val = int64(o&0xfc000000 | uint32(t>>2)&^0xfc000000) + } +} + func archreloc(ctxt *ld.Link, r *ld.Reloc, s *ld.Symbol, val *int64) int { if ld.Linkmode == ld.LinkExternal { - return -1 + switch r.Type { + default: + return -1 + + case obj.R_ADDRMIPS, obj.R_ADDRMIPSU: + + r.Done = 0 + + // set up addend for eventual relocation via outer symbol. + rs := r.Sym + r.Xadd = r.Add + for rs.Outer != nil { + r.Xadd += ld.Symaddr(rs) - ld.Symaddr(rs.Outer) + rs = rs.Outer + } + + if rs.Type != obj.SHOSTOBJ && rs.Type != obj.SDYNIMPORT && rs.Sect == nil { + ld.Errorf(s, "missing section for %s", rs.Name) + } + r.Xsym = rs + applyrel(r, s, val, r.Xadd) + return 0 + + case obj.R_ADDRMIPSTLS, obj.R_CALLMIPS, obj.R_JMPMIPS: + r.Done = 0 + r.Xsym = r.Sym + r.Xadd = r.Add + applyrel(r, s, val, r.Add) + return 0 + } } switch r.Type { @@ -72,23 +139,33 @@ func archreloc(ctxt *ld.Link, r *ld.Reloc, s *ld.Symbol, val *int64) int { *val = ld.Symaddr(r.Sym) + r.Add - ld.Symaddr(ctxt.Syms.Lookup(".got", 0)) return 0 - case obj.R_ADDRMIPS, - obj.R_ADDRMIPSU: + case obj.R_ADDRMIPS, obj.R_ADDRMIPSU: t := ld.Symaddr(r.Sym) + r.Add - o1 := ld.SysArch.ByteOrder.Uint32(s.P[r.Off:]) - if r.Type == obj.R_ADDRMIPS { - *val = int64(o1&0xffff0000 | uint32(t)&0xffff) - } else { - *val = int64(o1&0xffff0000 | uint32((t+1<<15)>>16)&0xffff) - } + applyrel(r, s, val, t) return 0 - case obj.R_CALLMIPS, - obj.R_JMPMIPS: - // Low 26 bits = (S + A) >> 2 + case obj.R_CALLMIPS, obj.R_JMPMIPS: t := ld.Symaddr(r.Sym) + r.Add - o1 := ld.SysArch.ByteOrder.Uint32(s.P[r.Off:]) - *val = int64(o1&0xfc000000 | uint32(t>>2)&^0xfc000000) + + if t&3 != 0 { + ld.Errorf(s, "direct call is not aligned: %s %x", r.Sym.Name, t) + } + + // check if target address is in the same 256 MB region as the next instruction + if (s.Value+int64(r.Off)+4)&0xf0000000 != (t & 0xf0000000) { + ld.Errorf(s, "direct call too far: %s %x", r.Sym.Name, t) + } + + applyrel(r, s, val, t) + return 0 + + case obj.R_ADDRMIPSTLS: + // thread pointer is at 0x7000 offset from the start of TLS data area + t := ld.Symaddr(r.Sym) + r.Add - 0x7000 + if t < -32768 || t >= 32678 { + ld.Errorf(s, "TLS offset out of range %d", t) + } + applyrel(r, s, val, t) return 0 } diff --git a/src/cmd/objdump/objdump_test.go b/src/cmd/objdump/objdump_test.go index 30b964d883..10fc716463 100644 --- a/src/cmd/objdump/objdump_test.go +++ b/src/cmd/objdump/objdump_test.go @@ -166,7 +166,7 @@ func TestDisasmExtld(t *testing.T) { t.Skipf("skipping on %s, no support for external linking, issue 9038", runtime.GOARCH) case "arm64": t.Skipf("skipping on %s, issue 10106", runtime.GOARCH) - case "mips64", "mips64le": + case "mips64", "mips64le", "mips", "mipsle": t.Skipf("skipping on %s, issue 12559 and 12560", runtime.GOARCH) case "s390x": t.Skipf("skipping on %s, issue 15255", runtime.GOARCH) diff --git a/src/cmd/pprof/internal/driver/driver.go b/src/cmd/pprof/internal/driver/driver.go index 931985a7f2..0f1ed6eece 100644 --- a/src/cmd/pprof/internal/driver/driver.go +++ b/src/cmd/pprof/internal/driver/driver.go @@ -780,14 +780,14 @@ func processFlags(p *profile.Profile, ui plugin.UI, f *flags) error { var err error si, sm := *f.flagSampleIndex, *f.flagMean || *f.flagMeanDelay - si, err = sampleIndex(p, &f.flagTotalDelay, si, 1, "delay", "-total_delay", err) - si, err = sampleIndex(p, &f.flagMeanDelay, si, 1, "delay", "-mean_delay", err) - si, err = sampleIndex(p, &f.flagContentions, si, 0, "contentions", "-contentions", err) + si, err = sampleIndex(p, &f.flagTotalDelay, si, "delay", "-total_delay", err) + si, err = sampleIndex(p, &f.flagMeanDelay, si, "delay", "-mean_delay", err) + si, err = sampleIndex(p, &f.flagContentions, si, "contentions", "-contentions", err) - si, err = sampleIndex(p, &f.flagInUseSpace, si, 1, "inuse_space", "-inuse_space", err) - si, err = sampleIndex(p, &f.flagInUseObjects, si, 0, "inuse_objects", "-inuse_objects", err) - si, err = sampleIndex(p, &f.flagAllocSpace, si, 1, "alloc_space", "-alloc_space", err) - si, err = sampleIndex(p, &f.flagAllocObjects, si, 0, "alloc_objects", "-alloc_objects", err) + si, err = sampleIndex(p, &f.flagInUseSpace, si, "inuse_space", "-inuse_space", err) + si, err = sampleIndex(p, &f.flagInUseObjects, si, "inuse_objects", "-inuse_objects", err) + si, err = sampleIndex(p, &f.flagAllocSpace, si, "alloc_space", "-alloc_space", err) + si, err = sampleIndex(p, &f.flagAllocObjects, si, "alloc_objects", "-alloc_objects", err) if si == -1 { // Use last value if none is requested. @@ -806,7 +806,6 @@ func processFlags(p *profile.Profile, ui plugin.UI, f *flags) error { func sampleIndex(p *profile.Profile, flag **bool, sampleIndex int, - newSampleIndex int, sampleType, option string, err error) (int, error) { if err != nil || !**flag { @@ -816,11 +815,12 @@ func sampleIndex(p *profile.Profile, flag **bool, if sampleIndex != -1 { return 0, fmt.Errorf("set at most one sample value selection option") } - if newSampleIndex >= len(p.SampleType) || - p.SampleType[newSampleIndex].Type != sampleType { - return 0, fmt.Errorf("option %s not valid for this profile", option) + for index, s := range p.SampleType { + if sampleType == s.Type { + return index, nil + } } - return newSampleIndex, nil + return 0, fmt.Errorf("option %s not valid for this profile", option) } func countFlags(bs []*bool) int { diff --git a/src/cmd/pprof/internal/report/report.go b/src/cmd/pprof/internal/report/report.go index 14875c16db..f897c9086f 100644 --- a/src/cmd/pprof/internal/report/report.go +++ b/src/cmd/pprof/internal/report/report.go @@ -123,7 +123,7 @@ func symbolsFromBinaries(prof *profile.Profile, g graph, rx *regexp.Regexp, addr // Walk all mappings looking for matching functions with samples. var objSyms []*objSymbol for _, m := range prof.Mapping { - if !hasSamples[filepath.Base(m.File)] { + if !hasSamples[m.File] { if address == nil || !(m.Start <= *address && *address <= m.Limit) { continue } diff --git a/src/cmd/pprof/internal/report/source.go b/src/cmd/pprof/internal/report/source.go index 7ab7e3861f..458985d1fa 100644 --- a/src/cmd/pprof/internal/report/source.go +++ b/src/cmd/pprof/internal/report/source.go @@ -229,7 +229,7 @@ func assemblyPerSourceLine(objSyms []*objSymbol, rs nodes, src string, obj plugi func findMatchingSymbol(objSyms []*objSymbol, ns nodes) *objSymbol { for _, n := range ns { for _, o := range objSyms { - if filepath.Base(o.sym.File) == n.info.objfile && + if o.sym.File == n.info.objfile && o.sym.Start <= n.info.address-o.base && n.info.address-o.base <= o.sym.End { return o diff --git a/src/cmd/pprof/internal/svg/svgpan.go b/src/cmd/pprof/internal/svg/svgpan.go index 4975b103e3..d8f12afea4 100644 --- a/src/cmd/pprof/internal/svg/svgpan.go +++ b/src/cmd/pprof/internal/svg/svgpan.go @@ -17,7 +17,7 @@ const svgPanJS = ` * - Mouse zooming (using the wheel) * - Object dragging * - * You can configure the behaviour of the pan/zoom/drag with the variables + * You can configure the behavior of the pan/zoom/drag with the variables * listed in the CONFIGURATION section of this file. * * Known issues: diff --git a/src/cmd/vet/cgo.go b/src/cmd/vet/cgo.go index d233e9a960..984911c489 100644 --- a/src/cmd/vet/cgo.go +++ b/src/cmd/vet/cgo.go @@ -44,7 +44,7 @@ func checkCgoCall(f *File, node ast.Node) { } for _, arg := range x.Args { - if !typeOKForCgoCall(cgoBaseType(f, arg)) { + if !typeOKForCgoCall(cgoBaseType(f, arg), make(map[types.Type]bool)) { f.Badf(arg.Pos(), "possibly passing Go type with embedded pointer to C") } @@ -53,7 +53,7 @@ func checkCgoCall(f *File, node ast.Node) { arg = conv.Args[0] } if u, ok := arg.(*ast.UnaryExpr); ok && u.Op == token.AND { - if !typeOKForCgoCall(cgoBaseType(f, u.X)) { + if !typeOKForCgoCall(cgoBaseType(f, u.X), make(map[types.Type]bool)) { f.Badf(arg.Pos(), "possibly passing Go type with embedded pointer to C") } } @@ -110,23 +110,24 @@ func cgoBaseType(f *File, arg ast.Expr) types.Type { return f.pkg.types[arg].Type } -// typeOKForCgoCall returns true if the type of arg is OK to pass to a +// typeOKForCgoCall reports whether the type of arg is OK to pass to a // C function using cgo. This is not true for Go types with embedded -// pointers. -func typeOKForCgoCall(t types.Type) bool { - if t == nil { +// pointers. m is used to avoid infinite recursion on recursive types. +func typeOKForCgoCall(t types.Type, m map[types.Type]bool) bool { + if t == nil || m[t] { return true } + m[t] = true switch t := t.Underlying().(type) { case *types.Chan, *types.Map, *types.Signature, *types.Slice: return false case *types.Pointer: - return typeOKForCgoCall(t.Elem()) + return typeOKForCgoCall(t.Elem(), m) case *types.Array: - return typeOKForCgoCall(t.Elem()) + return typeOKForCgoCall(t.Elem(), m) case *types.Struct: for i := 0; i < t.NumFields(); i++ { - if !typeOKForCgoCall(t.Field(i).Type()) { + if !typeOKForCgoCall(t.Field(i).Type(), m) { return false } } diff --git a/src/cmd/vet/copylock.go b/src/cmd/vet/copylock.go index 31c1257a47..27eb5d4651 100644 --- a/src/cmd/vet/copylock.go +++ b/src/cmd/vet/copylock.go @@ -93,13 +93,15 @@ func checkCopyLocksReturnStmt(f *File, rs *ast.ReturnStmt) { // checkCopyLocksCallExpr detects lock copy in the arguments to a function call func checkCopyLocksCallExpr(f *File, ce *ast.CallExpr) { - if id, ok := ce.Fun.(*ast.Ident); ok && id.Name == "new" && f.pkg.types[id].IsBuiltin() { - // Skip 'new(Type)' for built-in 'new' - return + if id, ok := ce.Fun.(*ast.Ident); ok && f.pkg.types[id].IsBuiltin() { + switch id.Name { + case "new", "len", "cap": + return + } } for _, x := range ce.Args { if path := lockPathRhs(f, x); path != nil { - f.Badf(x.Pos(), "function call copies lock value: %v", path) + f.Badf(x.Pos(), "call of %s copies lock value: %v", f.gofmt(ce.Fun), path) } } } diff --git a/src/cmd/vet/structtag.go b/src/cmd/vet/structtag.go index 814bbda594..872fde79ce 100644 --- a/src/cmd/vet/structtag.go +++ b/src/cmd/vet/structtag.go @@ -54,14 +54,37 @@ func checkCanonicalFieldTag(f *File, field *ast.Field, seen *map[[2]string]token if val == "" || val == "-" || val[0] == ',' { continue } + if key == "xml" && len(field.Names) > 0 && field.Names[0].Name == "XMLName" { + // XMLName defines the XML element name of the struct being + // checked. That name cannot collide with element or attribute + // names defined on other fields of the struct. Vet does not have a + // check for untagged fields of type struct defining their own name + // by containing a field named XMLName; see issue 18256. + continue + } if i := strings.Index(val, ","); i >= 0 { + if key == "xml" { + // Use a separate namespace for XML attributes. + for _, opt := range strings.Split(val[i:], ",") { + if opt == "attr" { + key += " attribute" // Key is part of the error message. + break + } + } + } val = val[:i] } if *seen == nil { *seen = map[[2]string]token.Pos{} } if pos, ok := (*seen)[[2]string{key, val}]; ok { - f.Badf(field.Pos(), "struct field %s repeats %s tag %q also at %s", field.Names[0].Name, key, val, f.loc(pos)) + var name string + if len(field.Names) > 0 { + name = field.Names[0].Name + } else { + name = field.Type.(*ast.Ident).Name + } + f.Badf(field.Pos(), "struct field %s repeats %s tag %q also at %s", name, key, val, f.loc(pos)) } else { (*seen)[[2]string{key, val}] = field.Pos() } diff --git a/src/cmd/vet/testdata/cgo/cgo.go b/src/cmd/vet/testdata/cgo/cgo.go index 25d395b1ea..d0df7cf678 100644 --- a/src/cmd/vet/testdata/cgo/cgo.go +++ b/src/cmd/vet/testdata/cgo/cgo.go @@ -52,5 +52,8 @@ func CgoTests() { C.f(*(*unsafe.Pointer)(unsafe.Pointer(&st2))) C.f(unsafe.Pointer(&st2)) + type cgoStruct struct{ p *cgoStruct } + C.f(unsafe.Pointer(&cgoStruct{})) + C.CBytes([]byte("hello")) } diff --git a/src/cmd/vet/testdata/copylock.go b/src/cmd/vet/testdata/copylock.go index 35ed766f1d..6fabbc337a 100644 --- a/src/cmd/vet/testdata/copylock.go +++ b/src/cmd/vet/testdata/copylock.go @@ -67,7 +67,7 @@ func BadFunc() { // override 'new' keyword new := func(interface{}) {} - new(t) // ERROR "function call copies lock value: testdata.Tlock contains sync.Once contains sync.Mutex" + new(t) // ERROR "call of new copies lock value: testdata.Tlock contains sync.Once contains sync.Mutex" // copy of array of locks var muA [5]sync.Mutex @@ -88,6 +88,20 @@ func BadFunc() { fmuSlice := fmuA[:] // OK } +func LenAndCapOnLockArrays() { + var a [5]sync.Mutex + aLen := len(a) // OK + aCap := cap(a) // OK + + // override 'len' and 'cap' keywords + + len := func(interface{}) {} + len(a) // ERROR "call of len copies lock value: sync.Mutex" + + cap := func(interface{}) {} + cap(a) // ERROR "call of cap copies lock value: sync.Mutex" +} + // SyncTypesCheck checks copying of sync.* types except sync.Mutex func SyncTypesCheck() { // sync.RWMutex copying diff --git a/src/cmd/vet/testdata/copylock_func.go b/src/cmd/vet/testdata/copylock_func.go index bfafa124fa..d51ff27cda 100644 --- a/src/cmd/vet/testdata/copylock_func.go +++ b/src/cmd/vet/testdata/copylock_func.go @@ -86,8 +86,10 @@ func FuncCallInterfaceArg(f func(a int, b interface{})) { f(1, "foo") f(2, &t) f(3, &sync.Mutex{}) - f(4, m) // ERROR "function call copies lock value: sync.Mutex" - f(5, t) // ERROR "function call copies lock value: struct{lock sync.Mutex} contains sync.Mutex" + f(4, m) // ERROR "call of f copies lock value: sync.Mutex" + f(5, t) // ERROR "call of f copies lock value: struct{lock sync.Mutex} contains sync.Mutex" + var fntab []func(t) + fntab[0](t) // ERROR "call of fntab.0. copies lock value: struct{lock sync.Mutex} contains sync.Mutex" } // Returning lock via interface value diff --git a/src/cmd/vet/testdata/structtag.go b/src/cmd/vet/testdata/structtag.go index cba990fccd..363aa898bf 100644 --- a/src/cmd/vet/testdata/structtag.go +++ b/src/cmd/vet/testdata/structtag.go @@ -6,6 +6,8 @@ package testdata +import "encoding/xml" + type StructTagTest struct { A int "hello" // ERROR "not compatible with reflect.StructTag.Get: bad syntax for struct tag pair" B int "\tx:\"y\"" // ERROR "not compatible with reflect.StructTag.Get: bad syntax for struct tag key" @@ -37,30 +39,44 @@ type JSONEmbeddedField struct { unexp `is:"embedded,notexported" json:"unexp"` // OK for now, see issue 7363 } +type AnonymousJSON struct{} +type AnonymousXML struct{} + type DuplicateJSONFields struct { JSON int `json:"a"` - DuplicateJSON int `json:"a"` // ERROR "struct field DuplicateJSON repeats json tag .a. also at testdata/structtag.go:41" + DuplicateJSON int `json:"a"` // ERROR "struct field DuplicateJSON repeats json tag .a. also at testdata/structtag.go:46" IgnoredJSON int `json:"-"` OtherIgnoredJSON int `json:"-"` OmitJSON int `json:",omitempty"` OtherOmitJSON int `json:",omitempty"` - DuplicateOmitJSON int `json:"a,omitempty"` // ERROR "struct field DuplicateOmitJSON repeats json tag .a. also at testdata/structtag.go:41" + DuplicateOmitJSON int `json:"a,omitempty"` // ERROR "struct field DuplicateOmitJSON repeats json tag .a. also at testdata/structtag.go:46" NonJSON int `foo:"a"` DuplicateNonJSON int `foo:"a"` Embedded struct { DuplicateJSON int `json:"a"` // OK because its not in the same struct type } + AnonymousJSON `json:"a"` // ERROR "struct field AnonymousJSON repeats json tag .a. also at testdata/structtag.go:46" XML int `xml:"a"` - DuplicateXML int `xml:"a"` // ERROR "struct field DuplicateXML repeats xml tag .a. also at testdata/structtag.go:54" + DuplicateXML int `xml:"a"` // ERROR "struct field DuplicateXML repeats xml tag .a. also at testdata/structtag.go:60" IgnoredXML int `xml:"-"` OtherIgnoredXML int `xml:"-"` OmitXML int `xml:",omitempty"` OtherOmitXML int `xml:",omitempty"` - DuplicateOmitXML int `xml:"a,omitempty"` // ERROR "struct field DuplicateOmitXML repeats xml tag .a. also at testdata/structtag.go:54" + DuplicateOmitXML int `xml:"a,omitempty"` // ERROR "struct field DuplicateOmitXML repeats xml tag .a. also at testdata/structtag.go:60" NonXML int `foo:"a"` DuplicateNonXML int `foo:"a"` Embedded struct { DuplicateXML int `xml:"a"` // OK because its not in the same struct type } + AnonymousXML `xml:"a"` // ERROR "struct field AnonymousXML repeats xml tag .a. also at testdata/structtag.go:60" + Attribute struct { + XMLName xml.Name `xml:"b"` + NoDup int `xml:"b"` // OK because XMLName above affects enclosing struct. + Attr int `xml:"b,attr"` // OK because <b b="0"><b>0</b></b> is valid. + DupAttr int `xml:"b,attr"` // ERROR "struct field DupAttr repeats xml attribute tag .b. also at testdata/structtag.go:76" + DupOmitAttr int `xml:"b,omitempty,attr"` // ERROR "struct field DupOmitAttr repeats xml attribute tag .b. also at testdata/structtag.go:76" + + AnonymousXML `xml:"b,attr"` // ERROR "struct field AnonymousXML repeats xml attribute tag .b. also at testdata/structtag.go:76" + } } diff --git a/src/cmd/vet/vet_test.go b/src/cmd/vet/vet_test.go index 725f013a7d..b3d5c663a7 100644 --- a/src/cmd/vet/vet_test.go +++ b/src/cmd/vet/vet_test.go @@ -143,6 +143,7 @@ func TestVetDirs(t *testing.T) { "divergent", "buildtag", "incomplete", // incomplete examples + "cgo", } { dir := dir t.Run(dir, func(t *testing.T) { diff --git a/src/context/context_test.go b/src/context/context_test.go index 2d604a04d3..6efc06cbe5 100644 --- a/src/context/context_test.go +++ b/src/context/context_test.go @@ -595,14 +595,14 @@ func XTestCancelRemoves(t testingT) { _, cancel := WithCancel(ctx) checkChildren("with WithCancel child ", ctx, 1) cancel() - checkChildren("after cancelling WithCancel child", ctx, 0) + checkChildren("after canceling WithCancel child", ctx, 0) ctx, _ = WithCancel(Background()) checkChildren("after creation", ctx, 0) _, cancel = WithTimeout(ctx, 60*time.Minute) checkChildren("with WithTimeout child ", ctx, 1) cancel() - checkChildren("after cancelling WithTimeout child", ctx, 0) + checkChildren("after canceling WithTimeout child", ctx, 0) } func XTestWithCancelCanceledParent(t testingT) { diff --git a/src/crypto/aes/cipher_s390x.go b/src/crypto/aes/cipher_s390x.go index 6030c25ee3..28d0ac90cf 100644 --- a/src/crypto/aes/cipher_s390x.go +++ b/src/crypto/aes/cipher_s390x.go @@ -27,7 +27,7 @@ type aesCipherAsm struct { // cryptBlocks invokes the cipher message (KM) instruction with // the given function code. This is equivalent to AES in ECB // mode. The length must be a multiple of BlockSize (16). -//go:noesape +//go:noescape func cryptBlocks(c code, key, dst, src *byte, length int) var useAsm = cipherhw.AESGCMSupport() diff --git a/src/crypto/aes/const.go b/src/crypto/aes/const.go index aee73a7c52..cbac5ff0ea 100644 --- a/src/crypto/aes/const.go +++ b/src/crypto/aes/const.go @@ -4,6 +4,13 @@ // Package aes implements AES encryption (formerly Rijndael), as defined in // U.S. Federal Information Processing Standards Publication 197. +// +// The AES operations in this package are not implemented using constant-time algorithms. +// An exception is when running on systems with enabled hardware support for AES +// that makes these operations constant-time. Examples include amd64 systems using AES-NI +// extensions and s390x systems using Message-Security-Assist extensions. +// On such systems, when the result of NewCipher is passed to cipher.NewGCM, +// the GHASH operation used by GCM is also constant-time. package aes // This file contains AES constants - 8720 bytes of initialized data. diff --git a/src/crypto/aes/gcm_s390x.go b/src/crypto/aes/gcm_s390x.go index 9eaaf7c21e..438310d3de 100644 --- a/src/crypto/aes/gcm_s390x.go +++ b/src/crypto/aes/gcm_s390x.go @@ -257,7 +257,7 @@ func (g *gcmAsm) Open(dst, nonce, ciphertext, data []byte) ([]byte, error) { if subtle.ConstantTimeCompare(expectedTag[:], tag) != 1 { // The AESNI code decrypts and authenticates concurrently, and // so overwrites dst in the event of a tag mismatch. That - // behaviour is mimicked here in order to be consistent across + // behavior is mimicked here in order to be consistent across // platforms. for i := range out { out[i] = 0 diff --git a/src/crypto/cipher/gcm.go b/src/crypto/cipher/gcm.go index cfc5769a80..62085aac0f 100644 --- a/src/crypto/cipher/gcm.go +++ b/src/crypto/cipher/gcm.go @@ -74,6 +74,10 @@ type gcm struct { // NewGCM returns the given 128-bit, block cipher wrapped in Galois Counter Mode // with the standard nonce length. +// +// In general, the GHASH operation performed by this implementation of GCM is not constant-time. +// An exception is when the underlying Block was created by aes.NewCipher +// on systems with hardware support for AES. See the crypto/aes package documentation for details. func NewGCM(cipher Block) (AEAD, error) { return NewGCMWithNonceSize(cipher, gcmStandardNonceSize) } @@ -184,7 +188,7 @@ func (g *gcm) Open(dst, nonce, ciphertext, data []byte) ([]byte, error) { if subtle.ConstantTimeCompare(expectedTag[:], tag) != 1 { // The AESNI code decrypts and authenticates concurrently, and // so overwrites dst in the event of a tag mismatch. That - // behaviour is mimicked here in order to be consistent across + // behavior is mimicked here in order to be consistent across // platforms. for i := range out { out[i] = 0 diff --git a/src/crypto/dsa/dsa.go b/src/crypto/dsa/dsa.go index e9b6a0c253..bc0c3e3462 100644 --- a/src/crypto/dsa/dsa.go +++ b/src/crypto/dsa/dsa.go @@ -3,6 +3,8 @@ // license that can be found in the LICENSE file. // Package dsa implements the Digital Signature Algorithm, as defined in FIPS 186-3. +// +// The DSA operations in this package are not implemented using constant-time algorithms. package dsa import ( @@ -189,17 +191,21 @@ func fermatInverse(k, P *big.Int) *big.Int { // Note that FIPS 186-3 section 4.6 specifies that the hash should be truncated // to the byte-length of the subgroup. This function does not perform that // truncation itself. +// +// Be aware that calling Sign with an attacker-controlled PrivateKey may +// require an arbitrary amount of CPU. func Sign(rand io.Reader, priv *PrivateKey, hash []byte) (r, s *big.Int, err error) { // FIPS 186-3, section 4.6 n := priv.Q.BitLen() - if n&7 != 0 { + if priv.Q.Sign() <= 0 || priv.P.Sign() <= 0 || priv.G.Sign() <= 0 || priv.X.Sign() <= 0 || n&7 != 0 { err = ErrInvalidPublicKey return } n >>= 3 - for { + var attempts int + for attempts = 10; attempts > 0; attempts-- { k := new(big.Int) buf := make([]byte, n) for { @@ -208,6 +214,10 @@ func Sign(rand io.Reader, priv *PrivateKey, hash []byte) (r, s *big.Int, err err return } k.SetBytes(buf) + // priv.Q must be >= 128 because the test above + // requires it to be > 0 and that + // ceil(log_2(Q)) mod 8 = 0 + // Thus this loop will quickly terminate. if k.Sign() > 0 && k.Cmp(priv.Q) < 0 { break } @@ -235,6 +245,12 @@ func Sign(rand io.Reader, priv *PrivateKey, hash []byte) (r, s *big.Int, err err } } + // Only degenerate private keys will require more than a handful of + // attempts. + if attempts == 0 { + return nil, nil, ErrInvalidPublicKey + } + return } diff --git a/src/crypto/dsa/dsa_test.go b/src/crypto/dsa/dsa_test.go index 568416d0df..b89aeaebea 100644 --- a/src/crypto/dsa/dsa_test.go +++ b/src/crypto/dsa/dsa_test.go @@ -73,6 +73,14 @@ func TestParameterGeneration(t *testing.T) { testParameterGeneration(t, L3072N256, 3072, 256) } +func fromHex(s string) *big.Int { + result, ok := new(big.Int).SetString(s, 16) + if !ok { + panic(s) + } + return result +} + func TestSignAndVerify(t *testing.T) { var priv PrivateKey priv.P, _ = new(big.Int).SetString("A9B5B793FB4785793D246BAE77E8FF63CA52F442DA763C440259919FE1BC1D6065A9350637A04F75A2F039401D49F08E066C4D275A5A65DA5684BC563C14289D7AB8A67163BFBF79D85972619AD2CFF55AB0EE77A9002B0EF96293BDD0F42685EBB2C66C327079F6C98000FBCB79AACDE1BC6F9D5C7B1A97E3D9D54ED7951FEF", 16) @@ -83,3 +91,33 @@ func TestSignAndVerify(t *testing.T) { testSignAndVerify(t, 0, &priv) } + +func TestSigningWithDegenerateKeys(t *testing.T) { + // Signing with degenerate private keys should not cause an infinite + // loop. + badKeys := []struct{ + p, q, g, y, x string + }{ + {"00", "01", "00", "00", "00"}, + {"01", "ff", "00", "00", "00"}, + } + + for i, test := range badKeys { + priv := PrivateKey{ + PublicKey: PublicKey{ + Parameters: Parameters { + P: fromHex(test.p), + Q: fromHex(test.q), + G: fromHex(test.g), + }, + Y: fromHex(test.y), + }, + X: fromHex(test.x), + } + + hashed := []byte("testing") + if _, _, err := Sign(rand.Reader, &priv, hashed); err == nil { + t.Errorf("#%d: unexpected success", i) + } + } +} diff --git a/src/crypto/elliptic/elliptic.go b/src/crypto/elliptic/elliptic.go index c02df45d10..d3527243e7 100644 --- a/src/crypto/elliptic/elliptic.go +++ b/src/crypto/elliptic/elliptic.go @@ -367,18 +367,24 @@ func initP521() { } // P256 returns a Curve which implements P-256 (see FIPS 186-3, section D.2.3) +// +// The cryptographic operations are implemented using constant-time algorithms. func P256() Curve { initonce.Do(initAll) return p256 } // P384 returns a Curve which implements P-384 (see FIPS 186-3, section D.2.4) +// +// The cryptographic operations do not use constant-time algorithms. func P384() Curve { initonce.Do(initAll) return p384 } // P521 returns a Curve which implements P-521 (see FIPS 186-3, section D.2.5) +// +// The cryptographic operations do not use constant-time algorithms. func P521() Curve { initonce.Do(initAll) return p521 diff --git a/src/crypto/elliptic/p224.go b/src/crypto/elliptic/p224.go index de266ca77a..22d0e2429c 100644 --- a/src/crypto/elliptic/p224.go +++ b/src/crypto/elliptic/p224.go @@ -35,7 +35,9 @@ func initP224() { p224FromBig(&p224.b, p224.B) } -// P224 returns a Curve which implements P-224 (see FIPS 186-3, section D.2.2) +// P224 returns a Curve which implements P-224 (see FIPS 186-3, section D.2.2). +// +// The cryptographic operations are implemented using constant-time algorithms. func P224() Curve { initonce.Do(initAll) return p224 diff --git a/src/crypto/rsa/rsa.go b/src/crypto/rsa/rsa.go index f809a9b9bc..1de4fcb473 100644 --- a/src/crypto/rsa/rsa.go +++ b/src/crypto/rsa/rsa.go @@ -18,6 +18,8 @@ // with v1.5/OAEP and signing/verifying with v1.5/PSS. If one needs to abstract // over the public-key primitive, the PrivateKey struct implements the // Decrypter and Signer interfaces from the crypto package. +// +// The RSA operations in this package are not implemented using constant-time algorithms. package rsa import ( diff --git a/src/crypto/sha1/sha1block_amd64.s b/src/crypto/sha1/sha1block_amd64.s index 0cdb43b422..77c8ec3906 100644 --- a/src/crypto/sha1/sha1block_amd64.s +++ b/src/crypto/sha1/sha1block_amd64.s @@ -225,7 +225,7 @@ end: RET -// This is the implementation using AVX2. It is based on: +// This is the implementation using AVX2, BMI1 and BMI2. It is based on: // "SHA-1 implementation with Intel(R) AVX2 instruction set extensions" // From http://software.intel.com/en-us/articles // (look for improving-the-performance-of-the-secure-hash-algorithm-1) @@ -1459,15 +1459,19 @@ TEXT ·blockAVX2(SB),$1408-32 // func checkAVX2() bool -// returns whether AVX2 is supported +// returns whether AVX2, BMI1 and BMI2 are supported TEXT ·checkAVX2(SB),NOSPLIT,$0 - CMPB runtime·support_avx2(SB), $1 - JE has - MOVB $0, ret+0(FP) - RET -has: + CMPB runtime·support_avx2(SB), $0 + JE noavx2 + CMPB runtime·support_bmi1(SB), $0 // check for ANDNL instruction + JE noavx2 + CMPB runtime·support_bmi2(SB), $0 // check for RORXL instruction + JE noavx2 MOVB $1, ret+0(FP) RET +noavx2: + MOVB $0, ret+0(FP) + RET DATA K_XMM_AR<>+0x00(SB)/4,$0x5a827999 diff --git a/src/crypto/sha256/sha256block_amd64.s b/src/crypto/sha256/sha256block_amd64.s index edf7ad1a3b..e9705b94b1 100644 --- a/src/crypto/sha256/sha256block_amd64.s +++ b/src/crypto/sha256/sha256block_amd64.s @@ -559,8 +559,11 @@ ADDL y3, h // h = t1 + S0 + MAJ // -- TEXT ·block(SB), 0, $536-32 - CMPB runtime·support_avx2(SB), $1 + CMPB runtime·support_avx2(SB), $0 + JE noavx2bmi2 + CMPB runtime·support_bmi2(SB), $1 // check for RORXL instruction JE avx2 +noavx2bmi2: MOVQ p_base+8(FP), SI MOVQ p_len+16(FP), DX diff --git a/src/crypto/tls/conn.go b/src/crypto/tls/conn.go index 4b2702a716..03895a723f 100644 --- a/src/crypto/tls/conn.go +++ b/src/crypto/tls/conn.go @@ -37,7 +37,7 @@ type Conn struct { vers uint16 // TLS version haveVers bool // version has been negotiated config *Config // configuration passed to constructor - // handshakeComplete is true if the connection is currently transfering + // handshakeComplete is true if the connection is currently transferring // application data (i.e. is not currently processing a handshake). handshakeComplete bool // handshakes counts the number of handshakes performed on the diff --git a/src/crypto/x509/root_cgo_darwin.go b/src/crypto/x509/root_cgo_darwin.go index ea86b60e11..8e80533590 100644 --- a/src/crypto/x509/root_cgo_darwin.go +++ b/src/crypto/x509/root_cgo_darwin.go @@ -73,10 +73,11 @@ int useOldCode() { // // On success it returns 0 and fills pemRoots with a CFDataRef that contains the extracted root // certificates of the system. On failure, the function returns -1. +// Additionally, it fills untrustedPemRoots with certs that must be removed from pemRoots. // -// Note: The CFDataRef returned in pemRoots must be released (using CFRelease) after -// we've consumed its content. -int FetchPEMRoots(CFDataRef *pemRoots) { +// Note: The CFDataRef returned in pemRoots and untrustedPemRoots must +// be released (using CFRelease) after we've consumed its content. +int FetchPEMRoots(CFDataRef *pemRoots, CFDataRef *untrustedPemRoots) { if (useOldCode()) { return FetchPEMRoots_MountainLion(pemRoots); } @@ -93,23 +94,69 @@ int FetchPEMRoots(CFDataRef *pemRoots) { return -1; } + // kSecTrustSettingsResult is defined as CFSTR("kSecTrustSettingsResult"), + // but the Go linker's internal linking mode can't handle CFSTR relocations. + // Create our own dynamic string instead and release it below. + CFStringRef policy = CFStringCreateWithCString(NULL, "kSecTrustSettingsResult", kCFStringEncodingUTF8); + CFMutableDataRef combinedData = CFDataCreateMutable(kCFAllocatorDefault, 0); + CFMutableDataRef combinedUntrustedData = CFDataCreateMutable(kCFAllocatorDefault, 0); for (int i = 0; i < numDomains; i++) { CFArrayRef certs = NULL; - // Only get certificates from domain that are trusted OSStatus err = SecTrustSettingsCopyCertificates(domains[i], &certs); if (err != noErr) { continue; } - int numCerts = CFArrayGetCount(certs); + CFIndex numCerts = CFArrayGetCount(certs); for (int j = 0; j < numCerts; j++) { CFDataRef data = NULL; CFErrorRef errRef = NULL; + CFArrayRef trustSettings = NULL; SecCertificateRef cert = (SecCertificateRef)CFArrayGetValueAtIndex(certs, j); if (cert == NULL) { continue; } + // We only want trusted certs. + int untrusted = 0; + if (i != 0) { + // Certs found in the system domain are always trusted. If the user + // configures "Never Trust" on such a cert, it will also be found in the + // admin or user domain, causing it to be added to untrustedPemRoots. The + // Go code will then clean this up. + + // Trust may be stored in any of the domains. According to Apple's + // SecTrustServer.c, "user trust settings overrule admin trust settings", + // so take the last trust settings array we find. + // Skip the system domain since it is always trusted. + for (int k = 1; k < numDomains; k++) { + CFArrayRef domainTrustSettings = NULL; + err = SecTrustSettingsCopyTrustSettings(cert, domains[k], &domainTrustSettings); + if (err == errSecSuccess && domainTrustSettings != NULL) { + if (trustSettings) { + CFRelease(trustSettings); + } + trustSettings = domainTrustSettings; + } + } + if (trustSettings == NULL) { + // "this certificate must be verified to a known trusted certificate"; aka not a root. + continue; + } + for (CFIndex k = 0; k < CFArrayGetCount(trustSettings); k++) { + CFNumberRef cfNum; + CFDictionaryRef tSetting = (CFDictionaryRef)CFArrayGetValueAtIndex(trustSettings, k); + if (CFDictionaryGetValueIfPresent(tSetting, policy, (const void**)&cfNum)){ + SInt32 result = 0; + CFNumberGetValue(cfNum, kCFNumberSInt32Type, &result); + // TODO: The rest of the dictionary specifies conditions for evaluation. + if (result == kSecTrustSettingsResultDeny) { + untrusted = 1; + } + } + } + CFRelease(trustSettings); + } // We only want to add Root CAs, so make sure Subject and Issuer Name match CFDataRef subjectName = SecCertificateCopyNormalizedSubjectContent(cert, &errRef); if (errRef != NULL) { @@ -138,13 +185,16 @@ int FetchPEMRoots(CFDataRef *pemRoots) { } if (data != NULL) { - CFDataAppendBytes(combinedData, CFDataGetBytePtr(data), CFDataGetLength(data)); + CFMutableDataRef appendTo = untrusted ? combinedUntrustedData : combinedData; + CFDataAppendBytes(appendTo, CFDataGetBytePtr(data), CFDataGetLength(data)); CFRelease(data); } } CFRelease(certs); } + CFRelease(policy); *pemRoots = combinedData; + *untrustedPemRoots = combinedUntrustedData; return 0; } */ @@ -158,7 +208,8 @@ func loadSystemRoots() (*CertPool, error) { roots := NewCertPool() var data C.CFDataRef = nil - err := C.FetchPEMRoots(&data) + var untrustedData C.CFDataRef = nil + err := C.FetchPEMRoots(&data, &untrustedData) if err == -1 { // TODO: better error message return nil, errors.New("crypto/x509: failed to load darwin system roots with cgo") @@ -167,5 +218,19 @@ func loadSystemRoots() (*CertPool, error) { defer C.CFRelease(C.CFTypeRef(data)) buf := C.GoBytes(unsafe.Pointer(C.CFDataGetBytePtr(data)), C.int(C.CFDataGetLength(data))) roots.AppendCertsFromPEM(buf) - return roots, nil + if untrustedData == nil { + return roots, nil + } + defer C.CFRelease(C.CFTypeRef(untrustedData)) + buf = C.GoBytes(unsafe.Pointer(C.CFDataGetBytePtr(untrustedData)), C.int(C.CFDataGetLength(untrustedData))) + untrustedRoots := NewCertPool() + untrustedRoots.AppendCertsFromPEM(buf) + + trustedRoots := NewCertPool() + for _, c := range roots.certs { + if !untrustedRoots.contains(c) { + trustedRoots.AddCert(c) + } + } + return trustedRoots, nil } diff --git a/src/crypto/x509/root_darwin.go b/src/crypto/x509/root_darwin.go index 78de56c221..66cdb5ea26 100644 --- a/src/crypto/x509/root_darwin.go +++ b/src/crypto/x509/root_darwin.go @@ -6,20 +6,239 @@ package x509 -import "os/exec" +import ( + "bufio" + "bytes" + "crypto/sha1" + "encoding/pem" + "fmt" + "io" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" +) + +var debugExecDarwinRoots = strings.Contains(os.Getenv("GODEBUG"), "x509roots=1") func (c *Certificate) systemVerify(opts *VerifyOptions) (chains [][]*Certificate, err error) { return nil, nil } +// This code is only used when compiling without cgo. +// It is here, instead of root_nocgo_darwin.go, so that tests can check it +// even if the tests are run with cgo enabled. +// The linker will not include these unused functions in binaries built with cgo enabled. + +// execSecurityRoots finds the macOS list of trusted root certificates +// using only command-line tools. This is our fallback path when cgo isn't available. +// +// The strategy is as follows: +// +// 1. Run "security trust-settings-export" and "security +// trust-settings-export -d" to discover the set of certs with some +// user-tweaked trust policy. We're too lazy to parse the XML (at +// least at this stage of Go 1.8) to understand what the trust +// policy actually is. We just learn that there is _some_ policy. +// +// 2. Run "security find-certificate" to dump the list of system root +// CAs in PEM format. +// +// 3. For each dumped cert, conditionally verify it with "security +// verify-cert" if that cert was in the set discovered in Step 1. +// Without the Step 1 optimization, running "security verify-cert" +// 150-200 times takes 3.5 seconds. With the optimization, the +// whole process takes about 180 milliseconds with 1 untrusted root +// CA. (Compared to 110ms in the cgo path) func execSecurityRoots() (*CertPool, error) { + hasPolicy, err := getCertsWithTrustPolicy() + if err != nil { + return nil, err + } + if debugExecDarwinRoots { + println(fmt.Sprintf("crypto/x509: %d certs have a trust policy", len(hasPolicy))) + } + cmd := exec.Command("/usr/bin/security", "find-certificate", "-a", "-p", "/System/Library/Keychains/SystemRootCertificates.keychain") data, err := cmd.Output() if err != nil { return nil, err } - roots := NewCertPool() - roots.AppendCertsFromPEM(data) + var ( + mu sync.Mutex + roots = NewCertPool() + numVerified int // number of execs of 'security verify-cert', for debug stats + ) + + blockCh := make(chan *pem.Block) + var wg sync.WaitGroup + + // Using 4 goroutines to pipe into verify-cert seems to be + // about the best we can do. The verify-cert binary seems to + // just RPC to another server with coarse locking anyway, so + // running 16 at a time for instance doesn't help at all. Due + // to the "if hasPolicy" check below, though, we will rarely + // (or never) call verify-cert on stock macOS systems, though. + // The hope is that we only call verify-cert when the user has + // tweaked their trust policy. These 4 goroutines are only + // defensive in the pathological case of many trust edits. + for i := 0; i < 4; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for block := range blockCh { + cert, err := ParseCertificate(block.Bytes) + if err != nil { + continue + } + sha1CapHex := fmt.Sprintf("%X", sha1.Sum(block.Bytes)) + + valid := true + verifyChecks := 0 + if hasPolicy[sha1CapHex] { + verifyChecks++ + if !verifyCertWithSystem(block, cert) { + valid = false + } + } + + mu.Lock() + numVerified += verifyChecks + if valid { + roots.AddCert(cert) + } + mu.Unlock() + } + }() + } + for len(data) > 0 { + var block *pem.Block + block, data = pem.Decode(data) + if block == nil { + break + } + if block.Type != "CERTIFICATE" || len(block.Headers) != 0 { + continue + } + blockCh <- block + } + close(blockCh) + wg.Wait() + + if debugExecDarwinRoots { + mu.Lock() + defer mu.Unlock() + println(fmt.Sprintf("crypto/x509: ran security verify-cert %d times", numVerified)) + } + return roots, nil } + +func verifyCertWithSystem(block *pem.Block, cert *Certificate) bool { + data := pem.EncodeToMemory(block) + + f, err := ioutil.TempFile("", "cert") + if err != nil { + fmt.Fprintf(os.Stderr, "can't create temporary file for cert: %v", err) + return false + } + defer os.Remove(f.Name()) + if _, err := f.Write(data); err != nil { + fmt.Fprintf(os.Stderr, "can't write temporary file for cert: %v", err) + return false + } + if err := f.Close(); err != nil { + fmt.Fprintf(os.Stderr, "can't write temporary file for cert: %v", err) + return false + } + cmd := exec.Command("/usr/bin/security", "verify-cert", "-c", f.Name(), "-l", "-L") + var stderr bytes.Buffer + if debugExecDarwinRoots { + cmd.Stderr = &stderr + } + if err := cmd.Run(); err != nil { + if debugExecDarwinRoots { + println(fmt.Sprintf("crypto/x509: verify-cert rejected %s: %q", cert.Subject.CommonName, bytes.TrimSpace(stderr.Bytes()))) + } + return false + } + if debugExecDarwinRoots { + println(fmt.Sprintf("crypto/x509: verify-cert approved %s", cert.Subject.CommonName)) + } + return true +} + +// getCertsWithTrustPolicy returns the set of certs that have a +// possibly-altered trust policy. The keys of the map are capitalized +// sha1 hex of the raw cert. +// They are the certs that should be checked against `security +// verify-cert` to see whether the user altered the default trust +// settings. This code is only used for cgo-disabled builds. +func getCertsWithTrustPolicy() (map[string]bool, error) { + set := map[string]bool{} + td, err := ioutil.TempDir("", "x509trustpolicy") + if err != nil { + return nil, err + } + defer os.RemoveAll(td) + run := func(file string, args ...string) error { + file = filepath.Join(td, file) + args = append(args, file) + cmd := exec.Command("/usr/bin/security", args...) + var stderr bytes.Buffer + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + // If there are no trust settings, the + // `security trust-settings-export` command + // fails with: + // exit status 1, SecTrustSettingsCreateExternalRepresentation: No Trust Settings were found. + // Rather than match on English substrings that are probably + // localized on macOS, just interpret any failure to mean that + // there are no trust settings. + if debugExecDarwinRoots { + println(fmt.Sprintf("crypto/x509: exec %q: %v, %s", cmd.Args, err, stderr.Bytes())) + } + return nil + } + + f, err := os.Open(file) + if err != nil { + return err + } + defer f.Close() + + // Gather all the runs of 40 capitalized hex characters. + br := bufio.NewReader(f) + var hexBuf bytes.Buffer + for { + b, err := br.ReadByte() + isHex := ('A' <= b && b <= 'F') || ('0' <= b && b <= '9') + if isHex { + hexBuf.WriteByte(b) + } else { + if hexBuf.Len() == 40 { + set[hexBuf.String()] = true + } + hexBuf.Reset() + } + if err == io.EOF { + break + } + if err != nil { + return err + } + } + + return nil + } + if err := run("user", "trust-settings-export"); err != nil { + return nil, fmt.Errorf("dump-trust-settings (user): %v", err) + } + if err := run("admin", "trust-settings-export", "-d"); err != nil { + return nil, fmt.Errorf("dump-trust-settings (admin): %v", err) + } + return set, nil +} diff --git a/src/crypto/x509/root_darwin_test.go b/src/crypto/x509/root_darwin_test.go index 8b6b1516ae..2784ce2f0f 100644 --- a/src/crypto/x509/root_darwin_test.go +++ b/src/crypto/x509/root_darwin_test.go @@ -7,6 +7,7 @@ package x509 import ( "runtime" "testing" + "time" ) func TestSystemRoots(t *testing.T) { @@ -15,21 +16,33 @@ func TestSystemRoots(t *testing.T) { t.Skipf("skipping on %s/%s, no system root", runtime.GOOS, runtime.GOARCH) } - sysRoots := systemRootsPool() // actual system roots + t0 := time.Now() + sysRoots := systemRootsPool() // actual system roots + sysRootsDuration := time.Since(t0) + + t1 := time.Now() execRoots, err := execSecurityRoots() // non-cgo roots + execSysRootsDuration := time.Since(t1) if err != nil { t.Fatalf("failed to read system roots: %v", err) } + t.Logf(" cgo sys roots: %v", sysRootsDuration) + t.Logf("non-cgo sys roots: %v", execSysRootsDuration) + for _, tt := range []*CertPool{sysRoots, execRoots} { if tt == nil { t.Fatal("no system roots") } - // On Mavericks, there are 212 bundled certs; require only - // 150 here, since this is just a sanity check, and the - // exact number will vary over time. - if want, have := 150, len(tt.certs); have < want { + // On Mavericks, there are 212 bundled certs, at least + // there was at one point in time on one machine. + // (Maybe it was a corp laptop with extra certs?) + // Other OS X users report + // 135, 142, 145... Let's try requiring at least 100, + // since this is just a sanity check. + t.Logf("got %d roots", len(tt.certs)) + if want, have := 100, len(tt.certs); have < want { t.Fatalf("want at least %d system roots, have %d", want, have) } } diff --git a/src/crypto/x509/verify.go b/src/crypto/x509/verify.go index 0d3de30bec..29345a1755 100644 --- a/src/crypto/x509/verify.go +++ b/src/crypto/x509/verify.go @@ -153,7 +153,7 @@ type VerifyOptions struct { CurrentTime time.Time // if zero, the current time is used // KeyUsage specifies which Extended Key Usage values are acceptable. // An empty list means ExtKeyUsageServerAuth. Key usage is considered a - // constraint down the chain which mirrors Windows CryptoAPI behaviour, + // constraint down the chain which mirrors Windows CryptoAPI behavior, // but not the spec. To accept any key usage, include ExtKeyUsageAny. KeyUsages []ExtKeyUsage } @@ -262,7 +262,7 @@ func (c *Certificate) isValid(certType int, currentChain []*Certificate, opts *V // WARNING: this doesn't do any revocation checking. func (c *Certificate) Verify(opts VerifyOptions) (chains [][]*Certificate, err error) { // Platform-specific verification needs the ASN.1 contents so - // this makes the behaviour consistent across platforms. + // this makes the behavior consistent across platforms. if len(c.Raw) == 0 { return nil, errNotParsed } diff --git a/src/crypto/x509/x509.go b/src/crypto/x509/x509.go index d9077db653..949ce01856 100644 --- a/src/crypto/x509/x509.go +++ b/src/crypto/x509/x509.go @@ -1850,13 +1850,20 @@ func (c *Certificate) CreateCRL(rand io.Reader, priv interface{}, revokedCerts [ return nil, err } + // Force revocation times to UTC per RFC 5280. + revokedCertsUTC := make([]pkix.RevokedCertificate, len(revokedCerts)) + for i, rc := range revokedCerts { + rc.RevocationTime = rc.RevocationTime.UTC() + revokedCertsUTC[i] = rc + } + tbsCertList := pkix.TBSCertificateList{ Version: 1, Signature: signatureAlgorithm, Issuer: c.Subject.ToRDNSequence(), ThisUpdate: now.UTC(), NextUpdate: expiry.UTC(), - RevokedCertificates: revokedCerts, + RevokedCertificates: revokedCertsUTC, } // Authority Key Id diff --git a/src/crypto/x509/x509_test.go b/src/crypto/x509/x509_test.go index 354545ccbc..aa30d85b7d 100644 --- a/src/crypto/x509/x509_test.go +++ b/src/crypto/x509/x509_test.go @@ -850,17 +850,31 @@ func TestCRLCreation(t *testing.T) { block, _ = pem.Decode([]byte(pemCertificate)) cert, _ := ParseCertificate(block.Bytes) - now := time.Unix(1000, 0) + loc := time.FixedZone("Oz/Atlantis", int((2 * time.Hour).Seconds())) + + now := time.Unix(1000, 0).In(loc) + nowUTC := now.UTC() expiry := time.Unix(10000, 0) revokedCerts := []pkix.RevokedCertificate{ { SerialNumber: big.NewInt(1), + RevocationTime: nowUTC, + }, + { + SerialNumber: big.NewInt(42), + // RevocationTime should be converted to UTC before marshaling. RevocationTime: now, }, + } + expectedCerts := []pkix.RevokedCertificate{ + { + SerialNumber: big.NewInt(1), + RevocationTime: nowUTC, + }, { SerialNumber: big.NewInt(42), - RevocationTime: now, + RevocationTime: nowUTC, }, } @@ -869,10 +883,14 @@ func TestCRLCreation(t *testing.T) { t.Errorf("error creating CRL: %s", err) } - _, err = ParseDERCRL(crlBytes) + parsedCRL, err := ParseDERCRL(crlBytes) if err != nil { t.Errorf("error reparsing CRL: %s", err) } + if !reflect.DeepEqual(parsedCRL.TBSCertList.RevokedCertificates, expectedCerts) { + t.Errorf("RevokedCertificates mismatch: got %v; want %v.", + parsedCRL.TBSCertList.RevokedCertificates, expectedCerts) + } } func fromBase64(in string) []byte { diff --git a/src/database/sql/convert.go b/src/database/sql/convert.go index 4b4dfc40d7..ea2f377810 100644 --- a/src/database/sql/convert.go +++ b/src/database/sql/convert.go @@ -13,6 +13,8 @@ import ( "reflect" "strconv" "time" + "unicode" + "unicode/utf8" ) var errNilPtr = errors.New("destination pointer is nil") // embedded in descriptive error @@ -24,6 +26,17 @@ func describeNamedValue(nv *driver.NamedValue) string { return fmt.Sprintf("with name %q", nv.Name) } +func validateNamedValueName(name string) error { + if len(name) == 0 { + return nil + } + r, _ := utf8.DecodeRuneInString(name) + if unicode.IsLetter(r) { + return nil + } + return fmt.Errorf("name %q does not begin with a letter", name) +} + // driverArgs converts arguments from callers of Stmt.Exec and // Stmt.Query into driver Values. // @@ -43,6 +56,9 @@ func driverArgs(ds *driverStmt, args []interface{}) ([]driver.NamedValue, error) nv := &nvargs[n] nv.Ordinal = n + 1 if np, ok := arg.(NamedArg); ok { + if err := validateNamedValueName(np.Name); err != nil { + return nil, err + } arg = np.Value nvargs[n].Name = np.Name } @@ -60,6 +76,9 @@ func driverArgs(ds *driverStmt, args []interface{}) ([]driver.NamedValue, error) nv := &nvargs[n] nv.Ordinal = n + 1 if np, ok := arg.(NamedArg); ok { + if err := validateNamedValueName(np.Name); err != nil { + return nil, err + } arg = np.Value nv.Name = np.Name } diff --git a/src/database/sql/ctxutil.go b/src/database/sql/ctxutil.go index 7c05ce2448..1071446227 100644 --- a/src/database/sql/ctxutil.go +++ b/src/database/sql/ctxutil.go @@ -111,25 +111,32 @@ func ctxDriverStmtQuery(ctx context.Context, si driver.Stmt, nvdargs []driver.Na var errLevelNotSupported = errors.New("sql: selected isolation level is not supported") -func ctxDriverBegin(ctx context.Context, ci driver.Conn) (driver.Tx, error) { - if ciCtx, is := ci.(driver.ConnBeginContext); is { - return ciCtx.BeginContext(ctx) +func ctxDriverBegin(ctx context.Context, opts *TxOptions, ci driver.Conn) (driver.Tx, error) { + if ciCtx, is := ci.(driver.ConnBeginTx); is { + dopts := driver.TxOptions{} + if opts != nil { + dopts.Isolation = driver.IsolationLevel(opts.Isolation) + dopts.ReadOnly = opts.ReadOnly + } + return ciCtx.BeginTx(ctx, dopts) } if ctx.Done() == context.Background().Done() { return ci.Begin() } - // Check the transaction level in ctx. If set and non-default - // then return an error here as the BeginContext driver value is not supported. - if level, ok := driver.IsolationFromContext(ctx); ok && level != driver.IsolationLevel(LevelDefault) { - return nil, errors.New("sql: driver does not support non-default isolation level") - } + if opts != nil { + // Check the transaction level. If the transaction level is non-default + // then return an error here as the BeginTx driver value is not supported. + if opts.Isolation != LevelDefault { + return nil, errors.New("sql: driver does not support non-default isolation level") + } - // Check for a read-only parameter in ctx. If a read-only transaction is - // requested return an error as the BeginContext driver value is not supported. - if ro := driver.ReadOnlyFromContext(ctx); ro { - return nil, errors.New("sql: driver does not support read-only transactions") + // If a read-only transaction is requested return an error as the + // BeginTx driver value is not supported. + if opts.ReadOnly { + return nil, errors.New("sql: driver does not support read-only transactions") + } } txi, err := ci.Begin() diff --git a/src/database/sql/driver/driver.go b/src/database/sql/driver/driver.go index c8cbbf0696..d66196fd48 100644 --- a/src/database/sql/driver/driver.go +++ b/src/database/sql/driver/driver.go @@ -10,7 +10,6 @@ package driver import ( "context" - "database/sql/internal" "errors" "reflect" ) @@ -27,13 +26,18 @@ import ( type Value interface{} // NamedValue holds both the value name and value. -// The Ordinal is the position of the parameter starting from one and is always set. -// If the Name is not empty it should be used for the parameter identifier and -// not the ordinal position. type NamedValue struct { - Name string + // If the Name is not empty it should be used for the parameter identifier and + // not the ordinal position. + // + // Name will not have a symbol prefix. + Name string + + // Ordinal position of the parameter starting from one and is always set. Ordinal int - Value Value + + // Value is the parameter value. + Value Value } // Driver is the interface that must be implemented by a database @@ -152,7 +156,7 @@ type Conn interface { // Begin starts and returns a new transaction. // - // Deprecated: Drivers should implement ConnBeginContext instead (or additionally). + // Deprecated: Drivers should implement ConnBeginTx instead (or additionally). Begin() (Tx, error) } @@ -164,41 +168,35 @@ type ConnPrepareContext interface { PrepareContext(ctx context.Context, query string) (Stmt, error) } -// IsolationLevel is the transaction isolation level stored in Context. +// IsolationLevel is the transaction isolation level stored in TxOptions. // // This type should be considered identical to sql.IsolationLevel along // with any values defined on it. type IsolationLevel int -// IsolationFromContext extracts the isolation level from a Context. -func IsolationFromContext(ctx context.Context) (level IsolationLevel, ok bool) { - level, ok = ctx.Value(internal.IsolationLevelKey{}).(IsolationLevel) - return level, ok -} - -// ReadOnlyFromContext extracts the read-only property from a Context. -// When readonly is true the transaction must be set to read-only -// or return an error. -func ReadOnlyFromContext(ctx context.Context) (readonly bool) { - readonly, _ = ctx.Value(internal.ReadOnlyKey{}).(bool) - return readonly +// TxOptions holds the transaction options. +// +// This type should be considered identical to sql.TxOptions. +type TxOptions struct { + Isolation IsolationLevel + ReadOnly bool } -// ConnBeginContext enhances the Conn interface with context. -type ConnBeginContext interface { - // BeginContext starts and returns a new transaction. +// ConnBeginTx enhances the Conn interface with context and TxOptions. +type ConnBeginTx interface { + // BeginTx starts and returns a new transaction. // If the context is canceled by the user the sql package will // call Tx.Rollback before discarding and closing the connection. // - // This must call IsolationFromContext to determine if there is a set - // isolation level. If the driver does not support setting the isolation - // level and one is set or if there is a set isolation level - // but the set level is not supported, an error must be returned. + // This must check opts.Isolation to determine if there is a set + // isolation level. If the driver does not support a non-default + // level and one is set or if there is a non-default isolation level + // that is not supported, an error must be returned. // - // This must also call ReadOnlyFromContext to determine if the read-only + // This must also check opts.ReadOnly to determine if the read-only // value is true to either set the read-only transaction property if supported // or return an error if it is not supported. - BeginContext(ctx context.Context) (Tx, error) + BeginTx(ctx context.Context, opts TxOptions) (Tx, error) } // Result is the result of a query execution. diff --git a/src/database/sql/fakedb_test.go b/src/database/sql/fakedb_test.go index 416b97d501..4b15f5bec7 100644 --- a/src/database/sql/fakedb_test.go +++ b/src/database/sql/fakedb_test.go @@ -713,7 +713,7 @@ func (s *fakeStmt) execInsert(args []driver.NamedValue, doInsert bool) (driver.R } else { // Assign value from argument placeholder name. for _, a := range args { - if a.Name == strvalue { + if a.Name == strvalue[1:] { val = a.Value break } @@ -818,7 +818,7 @@ func (s *fakeStmt) QueryContext(ctx context.Context, args []driver.NamedValue) ( } else { // Assign arg value from placeholder name. for _, a := range args { - if a.Name == wcol.Placeholder { + if a.Name == wcol.Placeholder[1:] { argValue = a.Value break } diff --git a/src/database/sql/internal/types.go b/src/database/sql/internal/types.go deleted file mode 100644 index 1895144cb2..0000000000 --- a/src/database/sql/internal/types.go +++ /dev/null @@ -1,11 +0,0 @@ -// Copyright 2016 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package internal - -// Context keys that set transaction properties for sql.BeginContext. -type ( - IsolationLevelKey struct{} // context value is driver.IsolationLevel - ReadOnlyKey struct{} // context value is bool -) diff --git a/src/database/sql/sql.go b/src/database/sql/sql.go index a620707b2d..0fa7c34a13 100644 --- a/src/database/sql/sql.go +++ b/src/database/sql/sql.go @@ -8,14 +8,16 @@ // The sql package must be used in conjunction with a database driver. // See https://golang.org/s/sqldrivers for a list of drivers. // -// For more usage examples, see the wiki page at +// Drivers that do not support context cancelation will not return until +// after the query is completed. +// +// For usage examples, see the wiki page at // https://golang.org/s/sqlwiki. package sql import ( "context" "database/sql/driver" - "database/sql/internal" "errors" "fmt" "io" @@ -69,17 +71,26 @@ func Drivers() []string { return list } -// A NamedArg used as an argument to Query or Exec -// binds to the corresponding named parameter in the SQL statement. +// A NamedArg is a named argument. NamedArg values may be used as +// arguments to Query or Exec and bind to the corresponding named +// parameter in the SQL statement. +// +// For a more concise way to create NamedArg values, see +// the Named function. type NamedArg struct { _Named_Fields_Required struct{} - // Name of the parameter placeholder. If empty the ordinal position in the - // argument list will be used. + // Name is the name of the parameter placeholder. + // + // If empty, the ordinal position in the argument list will be + // used. + // + // Name must omit any symbol prefix. Name string - // Value of the parameter. It may be assigned the same value types as - // the query arguments. + // Value is the value of the parameter. + // It may be assigned the same value types as the query + // arguments. Value interface{} } @@ -103,12 +114,10 @@ func Named(name string, value interface{}) NamedArg { return NamedArg{Name: name, Value: value} } -// IsolationLevel is the transaction isolation level stored in Context. -// The IsolationLevel is set with IsolationContext and the context -// should be passed to BeginContext. +// IsolationLevel is the transaction isolation level used in TxOptions. type IsolationLevel int -// Various isolation levels that drivers may support in BeginContext. +// Various isolation levels that drivers may support in BeginTx. // If a driver does not support a given isolation level an error may be returned. // // See https://en.wikipedia.org/wiki/Isolation_(database_systems)#Isolation_levels. @@ -123,18 +132,12 @@ const ( LevelLinearizable ) -// IsolationContext returns a new Context that carries the provided isolation level. -// The context must contain the isolation level before beginning the transaction -// with BeginContext. -func IsolationContext(ctx context.Context, level IsolationLevel) context.Context { - return context.WithValue(ctx, internal.IsolationLevelKey{}, driver.IsolationLevel(level)) -} - -// ReadOnlyWithContext returns a new Context that carries the provided -// read-only transaction property. The context must contain the read-only property -// before beginning the transaction with BeginContext. -func ReadOnlyContext(ctx context.Context) context.Context { - return context.WithValue(ctx, internal.ReadOnlyKey{}, true) +// TxOptions holds the transaction options to be used in DB.BeginTx. +type TxOptions struct { + // Isolation is the transaction isolation level. + // If zero, the driver or database's default level is used. + Isolation IsolationLevel + ReadOnly bool } // RawBytes is a byte slice that holds a reference to memory owned by @@ -1299,28 +1302,27 @@ func (db *DB) QueryRow(query string, args ...interface{}) *Row { return db.QueryRowContext(context.Background(), query, args...) } -// BeginContext starts a transaction. +// BeginTx starts a transaction. // // The provided context is used until the transaction is committed or rolled back. // If the context is canceled, the sql package will roll back // the transaction. Tx.Commit will return an error if the context provided to -// BeginContext is canceled. +// BeginTx is canceled. // -// An isolation level may be set by setting the value in the context -// before calling this. If a non-default isolation level is used -// that the driver doesn't support an error will be returned. Different drivers -// may have slightly different meanings for the same isolation level. -func (db *DB) BeginContext(ctx context.Context) (*Tx, error) { +// The provided TxOptions is optional and may be nil if defaults should be used. +// If a non-default isolation level is used that the driver doesn't support, +// an error will be returned. +func (db *DB) BeginTx(ctx context.Context, opts *TxOptions) (*Tx, error) { var tx *Tx var err error for i := 0; i < maxBadConnRetries; i++ { - tx, err = db.begin(ctx, cachedOrNewConn) + tx, err = db.begin(ctx, opts, cachedOrNewConn) if err != driver.ErrBadConn { break } } if err == driver.ErrBadConn { - return db.begin(ctx, alwaysNewConn) + return db.begin(ctx, opts, alwaysNewConn) } return tx, err } @@ -1328,17 +1330,17 @@ func (db *DB) BeginContext(ctx context.Context) (*Tx, error) { // Begin starts a transaction. The default isolation level is dependent on // the driver. func (db *DB) Begin() (*Tx, error) { - return db.BeginContext(context.Background()) + return db.BeginTx(context.Background(), nil) } -func (db *DB) begin(ctx context.Context, strategy connReuseStrategy) (tx *Tx, err error) { +func (db *DB) begin(ctx context.Context, opts *TxOptions, strategy connReuseStrategy) (tx *Tx, err error) { dc, err := db.conn(ctx, strategy) if err != nil { return nil, err } var txi driver.Tx withLock(dc, func() { - txi, err = ctxDriverBegin(ctx, dc.ci) + txi, err = ctxDriverBegin(ctx, opts, dc.ci) }) if err != nil { db.putConn(dc, err) @@ -1419,10 +1421,9 @@ func (tx *Tx) isDone() bool { // that has already been committed or rolled back. var ErrTxDone = errors.New("sql: Transaction has already been committed or rolled back") +// close returns the connection to the pool and +// must only be called by Tx.rollback or Tx.Commit. func (tx *Tx) close(err error) { - if !atomic.CompareAndSwapInt32(&tx.done, 0, 1) { - panic("double close") // internal error - } tx.db.putConn(tx.dc, err) tx.cancel() tx.dc = nil @@ -1447,14 +1448,14 @@ func (tx *Tx) closePrepared() { // Commit commits the transaction. func (tx *Tx) Commit() error { + if !atomic.CompareAndSwapInt32(&tx.done, 0, 1) { + return ErrTxDone + } select { default: case <-tx.ctx.Done(): return tx.ctx.Err() } - if tx.isDone() { - return ErrTxDone - } var err error withLock(tx.dc, func() { err = tx.txi.Commit() @@ -1469,7 +1470,7 @@ func (tx *Tx) Commit() error { // rollback aborts the transaction and optionally forces the pool to discard // the connection. func (tx *Tx) rollback(discardConn bool) error { - if tx.isDone() { + if !atomic.CompareAndSwapInt32(&tx.done, 0, 1) { return ErrTxDone } var err error @@ -2085,7 +2086,7 @@ func (rs *Rows) Next() bool { } // The driver is at the end of the current result set. // Test to see if there is another result set after the current one. - // Only close Rows if there is no futher result sets to read. + // Only close Rows if there is no further result sets to read. if !nextResultSet.HasNextResultSet() { rs.Close() } diff --git a/src/database/sql/sql_test.go b/src/database/sql/sql_test.go index 27fb765cde..63e1292cb1 100644 --- a/src/database/sql/sql_test.go +++ b/src/database/sql/sql_test.go @@ -375,7 +375,7 @@ func TestTxContextWait(t *testing.T) { ctx, _ := context.WithTimeout(context.Background(), time.Millisecond*15) - tx, err := db.BeginContext(ctx) + tx, err := db.BeginTx(ctx, nil) if err != nil { t.Fatal(err) } @@ -486,8 +486,8 @@ func TestQueryNamedArg(t *testing.T) { rows, err := db.Query( // Ensure the name and age parameters only match on placeholder name, not position. "SELECT|people|age,name|name=?name,age=?age", - Named("?age", 2), - Named("?name", "Bob"), + Named("age", 2), + Named("name", "Bob"), ) if err != nil { t.Fatalf("Query: %v", err) @@ -683,6 +683,37 @@ func TestQueryRow(t *testing.T) { } } +func TestTxRollbackCommitErr(t *testing.T) { + db := newTestDB(t, "people") + defer closeDB(t, db) + + tx, err := db.Begin() + if err != nil { + t.Fatal(err) + } + err = tx.Rollback() + if err != nil { + t.Errorf("expected nil error from Rollback; got %v", err) + } + err = tx.Commit() + if err != ErrTxDone { + t.Errorf("expected %q from Commit; got %q", ErrTxDone, err) + } + + tx, err = db.Begin() + if err != nil { + t.Fatal(err) + } + err = tx.Commit() + if err != nil { + t.Errorf("expected nil error from Commit; got %v", err) + } + err = tx.Rollback() + if err != ErrTxDone { + t.Errorf("expected %q from Rollback; got %q", ErrTxDone, err) + } +} + func TestStatementErrorAfterClose(t *testing.T) { db := newTestDB(t, "people") defer closeDB(t, db) @@ -2576,6 +2607,54 @@ func TestIssue6081(t *testing.T) { } } +// TestIssue18429 attempts to stress rolling back the transaction from a +// context cancel while simultaneously calling Tx.Rollback. Rolling back from a +// context happens concurrently so tx.rollback and tx.Commit must guard against +// double entry. +// +// In the test, a context is canceled while the query is in process so +// the internal rollback will run concurrently with the explicitly called +// Tx.Rollback. +func TestIssue18429(t *testing.T) { + db := newTestDB(t, "people") + defer closeDB(t, db) + + ctx := context.Background() + sem := make(chan bool, 20) + var wg sync.WaitGroup + + const milliWait = 30 + + for i := 0; i < 100; i++ { + sem <- true + wg.Add(1) + go func() { + defer func() { + <-sem + wg.Done() + }() + qwait := (time.Duration(rand.Intn(milliWait)) * time.Millisecond).String() + + ctx, cancel := context.WithTimeout(ctx, time.Duration(rand.Intn(milliWait))*time.Millisecond) + defer cancel() + + tx, err := db.BeginTx(ctx, nil) + if err != nil { + return + } + rows, err := tx.QueryContext(ctx, "WAIT|"+qwait+"|SELECT|people|name|") + if rows != nil { + rows.Close() + } + // This call will race with the context cancel rollback to complete + // if the rollback itself isn't guarded. + tx.Rollback() + }() + } + wg.Wait() + time.Sleep(milliWait * 3 * time.Millisecond) +} + func TestConcurrency(t *testing.T) { doConcurrentTest(t, new(concurrentDBQueryTest)) doConcurrentTest(t, new(concurrentDBExecTest)) diff --git a/src/debug/gosym/pclntab.go b/src/debug/gosym/pclntab.go index e94ed19d7d..ba1cf8b699 100644 --- a/src/debug/gosym/pclntab.go +++ b/src/debug/gosym/pclntab.go @@ -291,13 +291,6 @@ func (t *LineTable) step(p *[]byte, pc *uint64, val *int32, first bool) bool { return true } -// PCValue looks up the given PC in a pc value table. target is the -// offset of the pc from the entry point. -func PCValue(tab []byte, target uint64, quantum int) int { - t := LineTable{Data: tab, quantum: uint32(quantum)} - return int(t.pcvalue(0, 0, target)) -} - // pcvalue reports the value associated with the target pc. // off is the offset to the beginning of the pc-value table, // and entry is the start PC for the corresponding function. diff --git a/src/encoding/asn1/marshal.go b/src/encoding/asn1/marshal.go index 76d0b0c825..225fd0849c 100644 --- a/src/encoding/asn1/marshal.go +++ b/src/encoding/asn1/marshal.go @@ -535,7 +535,7 @@ func makeField(v reflect.Value, params fieldParameters) (e encoder, err error) { // If no default value is given then the zero value for the type is // assumed to be the default value. This isn't obviously the correct - // behaviour, but it's what Go has traditionally done. + // behavior, but it's what Go has traditionally done. if params.optional && params.defaultValue == nil { if reflect.DeepEqual(v.Interface(), reflect.Zero(v.Type()).Interface()) { return bytesEncoder(nil), nil diff --git a/src/fmt/fmt_test.go b/src/fmt/fmt_test.go index 6f8c1550a0..b7089be1a1 100644 --- a/src/fmt/fmt_test.go +++ b/src/fmt/fmt_test.go @@ -1561,18 +1561,23 @@ func TestWidthAndPrecision(t *testing.T) { } } -// Panic is a type that panics in String. -type Panic struct { +// PanicS is a type that panics in String. +type PanicS struct { message interface{} } // Value receiver. -func (p Panic) GoString() string { +func (p PanicS) String() string { panic(p.message) } +// PanicGo is a type that panics in GoString. +type PanicGo struct { + message interface{} +} + // Value receiver. -func (p Panic) String() string { +func (p PanicGo) GoString() string { panic(p.message) } @@ -1592,13 +1597,15 @@ var panictests = []struct { out string }{ // String - {"%s", (*Panic)(nil), "<nil>"}, // nil pointer special case - {"%s", Panic{io.ErrUnexpectedEOF}, "%!s(PANIC=unexpected EOF)"}, - {"%s", Panic{3}, "%!s(PANIC=3)"}, + {"%s", (*PanicS)(nil), "<nil>"}, // nil pointer special case + {"%s", PanicS{io.ErrUnexpectedEOF}, "%!s(PANIC=unexpected EOF)"}, + {"%s", PanicS{3}, "%!s(PANIC=3)"}, // GoString - {"%#v", (*Panic)(nil), "<nil>"}, // nil pointer special case - {"%#v", Panic{io.ErrUnexpectedEOF}, "%!v(PANIC=unexpected EOF)"}, - {"%#v", Panic{3}, "%!v(PANIC=3)"}, + {"%#v", (*PanicGo)(nil), "<nil>"}, // nil pointer special case + {"%#v", PanicGo{io.ErrUnexpectedEOF}, "%!v(PANIC=unexpected EOF)"}, + {"%#v", PanicGo{3}, "%!v(PANIC=3)"}, + // Issue 18282. catchPanic should not clear fmtFlags permanently. + {"%#v", []interface{}{PanicGo{3}, PanicGo{3}}, "[]interface {}{%!v(PANIC=3), %!v(PANIC=3)}"}, // Format {"%s", (*PanicF)(nil), "<nil>"}, // nil pointer special case {"%s", PanicF{io.ErrUnexpectedEOF}, "%!s(PANIC=unexpected EOF)"}, diff --git a/src/fmt/print.go b/src/fmt/print.go index 75301a238e..a7ef2e5ac2 100644 --- a/src/fmt/print.go +++ b/src/fmt/print.go @@ -535,7 +535,11 @@ func (p *pp) catchPanic(arg interface{}, verb rune) { // Nested panics; the recursion in printArg cannot succeed. panic(err) } - p.fmt.clearflags() // We are done, and for this output we want default behavior. + + oldFlags := p.fmt.fmtFlags + // For this output we want default behavior. + p.fmt.clearflags() + p.buf.WriteString(percentBangString) p.buf.WriteRune(verb) p.buf.WriteString(panicString) @@ -543,6 +547,8 @@ func (p *pp) catchPanic(arg interface{}, verb rune) { p.printArg(err, 'v') p.panicking = false p.buf.WriteByte(')') + + p.fmt.fmtFlags = oldFlags } } @@ -813,16 +819,15 @@ func (p *pp) printValue(value reflect.Value, verb rune, depth int) { if f.Kind() == reflect.Slice && f.IsNil() { p.buf.WriteString(nilParenString) return - } else { - p.buf.WriteByte('{') - for i := 0; i < f.Len(); i++ { - if i > 0 { - p.buf.WriteString(commaSpaceString) - } - p.printValue(f.Index(i), verb, depth+1) + } + p.buf.WriteByte('{') + for i := 0; i < f.Len(); i++ { + if i > 0 { + p.buf.WriteString(commaSpaceString) } - p.buf.WriteByte('}') + p.printValue(f.Index(i), verb, depth+1) } + p.buf.WriteByte('}') } else { p.buf.WriteByte('[') for i := 0; i < f.Len(); i++ { diff --git a/src/go/build/build.go b/src/go/build/build.go index f6aabcb3af..da12d50bb1 100644 --- a/src/go/build/build.go +++ b/src/go/build/build.go @@ -76,8 +76,9 @@ type Context struct { // If IsDir is nil, Import calls os.Stat and uses the result's IsDir method. IsDir func(path string) bool - // HasSubdir reports whether dir is a subdirectory of - // (perhaps multiple levels below) root. + // HasSubdir reports whether dir is lexically a subdirectory of + // root, perhaps multiple levels below. It does not try to check + // whether dir exists. // If so, HasSubdir sets rel to a slash-separated path that // can be joined to root to produce a path equivalent to dir. // If HasSubdir is nil, Import uses an implementation built on @@ -438,16 +439,11 @@ func (ctxt *Context) ImportDir(dir string, mode ImportMode) (*Package, error) { // containing no buildable Go source files. (It may still contain // test files, files hidden by build tags, and so on.) type NoGoError struct { - Dir string - Ignored bool // whether any Go files were ignored due to build tags + Dir string } func (e *NoGoError) Error() string { - msg := "no buildable Go source files in " + e.Dir - if e.Ignored { - msg += " (.go files ignored due to build tags)" - } - return msg + return "no buildable Go source files in " + e.Dir } // MultiplePackageError describes a directory containing @@ -879,7 +875,7 @@ Found: return p, badGoError } if len(p.GoFiles)+len(p.CgoFiles)+len(p.TestGoFiles)+len(p.XTestGoFiles) == 0 { - return p, &NoGoError{Dir: p.Dir, Ignored: len(p.IgnoredGoFiles) > 0} + return p, &NoGoError{p.Dir} } for tag := range allTags { diff --git a/src/go/build/build_test.go b/src/go/build/build_test.go index 8ca8e5e04d..a9972416ef 100644 --- a/src/go/build/build_test.go +++ b/src/go/build/build_test.go @@ -93,17 +93,6 @@ func TestEmptyFolderImport(t *testing.T) { } } -func TestIgnoredGoFilesImport(t *testing.T) { - _, err := Import(".", "testdata/ignored", 0) - e, ok := err.(*NoGoError) - if !ok { - t.Fatal(`Import("testdata/ignored") did not return NoGoError.`) - } - if !e.Ignored { - t.Fatal(`Import("testdata/ignored") should have ignored Go files.`) - } -} - func TestMultiplePackageImport(t *testing.T) { _, err := Import(".", "testdata/multi", 0) mpe, ok := err.(*MultiplePackageError) diff --git a/src/go/build/deps_test.go b/src/go/build/deps_test.go index e6f228852b..147eaf6aba 100644 --- a/src/go/build/deps_test.go +++ b/src/go/build/deps_test.go @@ -179,7 +179,7 @@ var pkgDeps = map[string][]string{ "runtime/trace": {"L0"}, "text/tabwriter": {"L2"}, - "testing": {"L2", "context", "flag", "fmt", "internal/race", "os", "runtime/debug", "runtime/pprof", "runtime/trace", "time"}, + "testing": {"L2", "flag", "fmt", "internal/race", "os", "runtime/debug", "runtime/pprof", "runtime/trace", "time"}, "testing/iotest": {"L2", "log"}, "testing/quick": {"L2", "flag", "fmt", "reflect"}, "internal/testenv": {"L2", "OS", "flag", "testing", "syscall"}, diff --git a/src/go/build/testdata/ignored/ignored.go b/src/go/build/testdata/ignored/ignored.go deleted file mode 100644 index 48a2ae88f4..0000000000 --- a/src/go/build/testdata/ignored/ignored.go +++ /dev/null @@ -1,3 +0,0 @@ -// +build alwaysignore - -package ignored diff --git a/src/go/internal/gccgoimporter/importer_test.go b/src/go/internal/gccgoimporter/importer_test.go index 58abbba94e..2b454701be 100644 --- a/src/go/internal/gccgoimporter/importer_test.go +++ b/src/go/internal/gccgoimporter/importer_test.go @@ -96,8 +96,11 @@ var importerTests = [...]importerTest{ {pkgpath: "complexnums", name: "PN", want: "const PN untyped complex", wantval: "(1 + -1i)"}, {pkgpath: "complexnums", name: "PP", want: "const PP untyped complex", wantval: "(1 + 1i)"}, {pkgpath: "conversions", name: "Bits", want: "const Bits Units", wantval: `"bits"`}, - // TODO: enable this entry once bug has been tracked down - //{pkgpath: "imports", wantinits: []string{"imports..import", "fmt..import", "math..import"}}, + {pkgpath: "time", name: "Duration", want: "type Duration int64"}, + {pkgpath: "time", name: "Nanosecond", want: "const Nanosecond Duration", wantval: "1"}, + {pkgpath: "unicode", name: "IsUpper", want: "func IsUpper(r rune) bool"}, + {pkgpath: "unicode", name: "MaxRune", want: "const MaxRune untyped rune", wantval: "1114111"}, + {pkgpath: "imports", wantinits: []string{"imports..import", "fmt..import", "math..import"}}, } func TestGoxImporter(t *testing.T) { diff --git a/src/go/internal/gccgoimporter/parser.go b/src/go/internal/gccgoimporter/parser.go index 7312cb4879..3b97c96d43 100644 --- a/src/go/internal/gccgoimporter/parser.go +++ b/src/go/internal/gccgoimporter/parser.go @@ -711,7 +711,10 @@ func (p *parser) parseType(pkg *types.Package) (t types.Type) { func (p *parser) parsePackageInit() PackageInit { name := p.parseUnquotedString() initfunc := p.parseUnquotedString() - priority := int(p.parseInt()) + priority := -1 + if p.version == "v1" { + priority = int(p.parseInt()) + } return PackageInit{Name: name, InitFunc: initfunc, Priority: priority} } @@ -766,6 +769,15 @@ func (p *parser) parseInitDataDirective() { } p.expect(';') + case "init_graph": + p.next() + // The graph data is thrown away for now. + for p.tok != ';' && p.tok != scanner.EOF { + p.parseInt() + p.parseInt() + } + p.expect(';') + case "checksum": // Don't let the scanner try to parse the checksum as a number. defer func(mode uint) { @@ -797,7 +809,7 @@ func (p *parser) parseDirective() { } switch p.lit { - case "v1", "v2", "priority", "init", "checksum": + case "v1", "v2", "priority", "init", "init_graph", "checksum": p.parseInitDataDirective() case "package": diff --git a/src/go/internal/gccgoimporter/testdata/time.gox b/src/go/internal/gccgoimporter/testdata/time.gox Binary files differnew file mode 100644 index 0000000000..80c2dbcb47 --- /dev/null +++ b/src/go/internal/gccgoimporter/testdata/time.gox diff --git a/src/go/internal/gccgoimporter/testdata/unicode.gox b/src/go/internal/gccgoimporter/testdata/unicode.gox Binary files differnew file mode 100644 index 0000000000..e70e539655 --- /dev/null +++ b/src/go/internal/gccgoimporter/testdata/unicode.gox diff --git a/src/go/parser/interface.go b/src/go/parser/interface.go index bff79cab46..724d8658a7 100644 --- a/src/go/parser/interface.go +++ b/src/go/parser/interface.go @@ -173,7 +173,7 @@ func ParseDir(fset *token.FileSet, path string, filter func(os.FileInfo) bool, m } // ParseExprFrom is a convenience function for parsing an expression. -// The arguments have the same meaning as for Parse, but the source must +// The arguments have the same meaning as for ParseFile, but the source must // be a valid Go (type or value) expression. Specifically, fset must not // be nil. // diff --git a/src/go/types/api.go b/src/go/types/api.go index 44949895a7..5b911cb96c 100644 --- a/src/go/types/api.go +++ b/src/go/types/api.go @@ -5,7 +5,7 @@ // Package types declares the data types and implements // the algorithms for type-checking of Go packages. Use // Config.Check to invoke the type checker for a package. -// Alternatively, create a new type checked with NewChecker +// Alternatively, create a new type checker with NewChecker // and invoke it incrementally by calling Checker.Files. // // Type-checking consists of several interdependent phases: diff --git a/src/go/types/stdlib_test.go b/src/go/types/stdlib_test.go index 1c6d7b5299..06d2c93dda 100644 --- a/src/go/types/stdlib_test.go +++ b/src/go/types/stdlib_test.go @@ -157,6 +157,7 @@ func TestStdFixed(t *testing.T) { "issue11362.go", // canonical import path check "issue15002.go", // uses Mmap; testTestDir should consult build tags "issue16369.go", // go/types handles this correctly - not an issue + "issue18459.go", // go/types doesn't check validity of //go:xxx directives ) } diff --git a/src/html/template/js.go b/src/html/template/js.go index 8f1185c81e..6434fa3be6 100644 --- a/src/html/template/js.go +++ b/src/html/template/js.go @@ -368,9 +368,10 @@ func isJSIdentPart(r rune) bool { // It is used to determine whether a script tag with a type attribute is a javascript container. func isJSType(mimeType string) bool { // per - // http://www.w3.org/TR/html5/scripting-1.html#attr-script-type + // https://www.w3.org/TR/html5/scripting-1.html#attr-script-type // https://tools.ietf.org/html/rfc7231#section-3.1.1 - // http://tools.ietf.org/html/rfc4329#section-3 + // https://tools.ietf.org/html/rfc4329#section-3 + // https://www.ietf.org/rfc/rfc4627.txt // discard parameters if i := strings.Index(mimeType, ";"); i >= 0 { @@ -381,6 +382,7 @@ func isJSType(mimeType string) bool { case "application/ecmascript", "application/javascript", + "application/json", "application/x-ecmascript", "application/x-javascript", "text/ecmascript", diff --git a/src/html/template/js_test.go b/src/html/template/js_test.go index 58fc37ae3a..7484f60b54 100644 --- a/src/html/template/js_test.go +++ b/src/html/template/js_test.go @@ -341,6 +341,7 @@ func TestIsJsMimeType(t *testing.T) { {"application/javascript;version=1.8;foo=bar", true}, {"application/javascript/version=1.8", false}, {"text/javascript", true}, + {"application/json", true}, } for _, test := range tests { diff --git a/src/io/io.go b/src/io/io.go index 3cab7288c9..9e4b86594d 100644 --- a/src/io/io.go +++ b/src/io/io.go @@ -420,6 +420,7 @@ func LimitReader(r Reader, n int64) Reader { return &LimitedReader{r, n} } // A LimitedReader reads from R but limits the amount of // data returned to just N bytes. Each call to Read // updates N to reflect the new amount remaining. +// Read returns EOF when N <= 0 or when the underlying R returns EOF. type LimitedReader struct { R Reader // underlying reader N int64 // max bytes remaining diff --git a/src/io/multi.go b/src/io/multi.go index 46e45a60e8..d784846862 100644 --- a/src/io/multi.go +++ b/src/io/multi.go @@ -4,13 +4,19 @@ package io +type eofReader struct{} + +func (eofReader) Read([]byte) (int, error) { + return 0, EOF +} + type multiReader struct { readers []Reader } func (mr *multiReader) Read(p []byte) (n int, err error) { for len(mr.readers) > 0 { - // Optimization to flatten nested multiReaders (Issue 13558) + // Optimization to flatten nested multiReaders (Issue 13558). if len(mr.readers) == 1 { if r, ok := mr.readers[0].(*multiReader); ok { mr.readers = r.readers @@ -19,7 +25,9 @@ func (mr *multiReader) Read(p []byte) (n int, err error) { } n, err = mr.readers[0].Read(p) if err == EOF { - mr.readers[0] = nil // permit earlier GC + // Use eofReader instead of nil to avoid nil panic + // after performing flatten (Issue 18232). + mr.readers[0] = eofReader{} // permit earlier GC mr.readers = mr.readers[1:] } if n > 0 || err != EOF { diff --git a/src/io/multi_test.go b/src/io/multi_test.go index 16e351a879..1a6292fa8a 100644 --- a/src/io/multi_test.go +++ b/src/io/multi_test.go @@ -264,3 +264,27 @@ func TestMultiReaderFreesExhaustedReaders(t *testing.T) { t.Fatalf(`ReadFull = %d (%q), %v; want 2, "ar", nil`, n, buf[:n], err) } } + +func TestInterleavedMultiReader(t *testing.T) { + r1 := strings.NewReader("123") + r2 := strings.NewReader("45678") + + mr1 := MultiReader(r1, r2) + mr2 := MultiReader(mr1) + + buf := make([]byte, 4) + + // Have mr2 use mr1's []Readers. + // Consume r1 (and clear it for GC to handle) and consume part of r2. + n, err := ReadFull(mr2, buf) + if got := string(buf[:n]); got != "1234" || err != nil { + t.Errorf(`ReadFull(mr2) = (%q, %v), want ("1234", nil)`, got, err) + } + + // Consume the rest of r2 via mr1. + // This should not panic even though mr2 cleared r1. + n, err = ReadFull(mr1, buf) + if got := string(buf[:n]); got != "5678" || err != nil { + t.Errorf(`ReadFull(mr1) = (%q, %v), want ("5678", nil)`, got, err) + } +} diff --git a/src/io/pipe.go b/src/io/pipe.go index 6145872391..b6e7755f64 100644 --- a/src/io/pipe.go +++ b/src/io/pipe.go @@ -85,6 +85,7 @@ func (p *pipe) write(b []byte) (n int, err error) { } if p.werr != nil { err = ErrClosedPipe + break } p.wwait.Wait() } diff --git a/src/io/pipe_test.go b/src/io/pipe_test.go index b16e653069..95930e86a4 100644 --- a/src/io/pipe_test.go +++ b/src/io/pipe_test.go @@ -247,6 +247,18 @@ func TestPipeWriteClose(t *testing.T) { } } +// Test close on Write side during Write. +func TestPipeWriteClose2(t *testing.T) { + c := make(chan int, 1) + _, w := Pipe() + go delayClose(t, w, c, pipeTest{}) + n, err := w.Write(make([]byte, 64)) + <-c + if n != 0 || err != ErrClosedPipe { + t.Errorf("write to closed pipe: %v, %v want %v, %v", n, err, 0, ErrClosedPipe) + } +} + func TestWriteEmpty(t *testing.T) { r, w := Pipe() go func() { diff --git a/src/math/big/int.go b/src/math/big/int.go index a2c1b580f5..1d8dabce12 100644 --- a/src/math/big/int.go +++ b/src/math/big/int.go @@ -404,8 +404,11 @@ func (x *Int) BitLen() int { // Exp sets z = x**y mod |m| (i.e. the sign of m is ignored), and returns z. // If y <= 0, the result is 1 mod |m|; if m == nil or m == 0, z = x**y. -// See Knuth, volume 2, section 4.6.3. +// +// Modular exponentation of inputs of a particular size is not a +// cryptographically constant-time operation. func (z *Int) Exp(x, y, m *Int) *Int { + // See Knuth, volume 2, section 4.6.3. var yWords nat if !y.neg { yWords = y.abs diff --git a/src/net/addrselect.go b/src/net/addrselect.go index 0b9d160fd4..1ab9fc5326 100644 --- a/src/net/addrselect.go +++ b/src/net/addrselect.go @@ -188,33 +188,17 @@ func (s *byRFC6724) Less(i, j int) bool { // Rule 9: Use longest matching prefix. // When DA and DB belong to the same address family (both are IPv6 or - // both are IPv4): If CommonPrefixLen(Source(DA), DA) > + // both are IPv4 [but see below]): If CommonPrefixLen(Source(DA), DA) > // CommonPrefixLen(Source(DB), DB), then prefer DA. Similarly, if // CommonPrefixLen(Source(DA), DA) < CommonPrefixLen(Source(DB), DB), // then prefer DB. - da4 := DA.To4() != nil - db4 := DB.To4() != nil - if da4 == db4 { + // + // However, applying this rule to IPv4 addresses causes + // problems (see issues 13283 and 18518), so limit to IPv6. + if DA.To4() == nil && DB.To4() == nil { commonA := commonPrefixLen(SourceDA, DA) commonB := commonPrefixLen(SourceDB, DB) - // CommonPrefixLen doesn't really make sense for IPv4, and even - // causes problems for common load balancing practices - // (e.g., https://golang.org/issue/13283). Glibc instead only - // uses CommonPrefixLen for IPv4 when the source and destination - // addresses are on the same subnet, but that requires extra - // work to find the netmask for our source addresses. As a - // simpler heuristic, we limit its use to when the source and - // destination belong to the same special purpose block. - if da4 { - if !sameIPv4SpecialPurposeBlock(SourceDA, DA) { - commonA = 0 - } - if !sameIPv4SpecialPurposeBlock(SourceDB, DB) { - commonB = 0 - } - } - if commonA > commonB { return preferDA } @@ -404,28 +388,3 @@ func commonPrefixLen(a, b IP) (cpl int) { } return } - -// sameIPv4SpecialPurposeBlock reports whether a and b belong to the same -// address block reserved by the IANA IPv4 Special-Purpose Address Registry: -// http://www.iana.org/assignments/iana-ipv4-special-registry/iana-ipv4-special-registry.xhtml -func sameIPv4SpecialPurposeBlock(a, b IP) bool { - a, b = a.To4(), b.To4() - if a == nil || b == nil || a[0] != b[0] { - return false - } - // IANA defines more special-purpose blocks, but these are the only - // ones likely to be relevant to typical Go systems. - switch a[0] { - case 10: // 10.0.0.0/8: Private-Use - return true - case 127: // 127.0.0.0/8: Loopback - return true - case 169: // 169.254.0.0/16: Link Local - return a[1] == 254 && b[1] == 254 - case 172: // 172.16.0.0/12: Private-Use - return a[1]&0xf0 == 16 && b[1]&0xf0 == 16 - case 192: // 192.168.0.0/16: Private-Use - return a[1] == 168 && b[1] == 168 - } - return false -} diff --git a/src/net/addrselect_test.go b/src/net/addrselect_test.go index 80aa4eb195..d6e0e63c3b 100644 --- a/src/net/addrselect_test.go +++ b/src/net/addrselect_test.go @@ -117,27 +117,6 @@ func TestSortByRFC6724(t *testing.T) { }, reverse: false, }, - - // Prefer longer common prefixes, but only for IPv4 address - // pairs in the same special-purpose block. - { - in: []IPAddr{ - {IP: ParseIP("1.2.3.4")}, - {IP: ParseIP("10.55.0.1")}, - {IP: ParseIP("10.66.0.1")}, - }, - srcs: []IP{ - ParseIP("1.2.3.5"), - ParseIP("10.66.1.2"), - ParseIP("10.66.1.2"), - }, - want: []IPAddr{ - {IP: ParseIP("10.66.0.1")}, - {IP: ParseIP("10.55.0.1")}, - {IP: ParseIP("1.2.3.4")}, - }, - reverse: true, - }, } for i, tt := range tests { inCopy := make([]IPAddr, len(tt.in)) @@ -268,67 +247,3 @@ func TestRFC6724CommonPrefixLength(t *testing.T) { } } - -func mustParseCIDRs(t *testing.T, blocks ...string) []*IPNet { - res := make([]*IPNet, len(blocks)) - for i, block := range blocks { - var err error - _, res[i], err = ParseCIDR(block) - if err != nil { - t.Fatalf("ParseCIDR(%s) failed: %v", block, err) - } - } - return res -} - -func TestSameIPv4SpecialPurposeBlock(t *testing.T) { - blocks := mustParseCIDRs(t, - "10.0.0.0/8", - "127.0.0.0/8", - "169.254.0.0/16", - "172.16.0.0/12", - "192.168.0.0/16", - ) - - addrs := []struct { - ip IP - block int // index or -1 - }{ - {IP{1, 2, 3, 4}, -1}, - {IP{2, 3, 4, 5}, -1}, - {IP{10, 2, 3, 4}, 0}, - {IP{10, 6, 7, 8}, 0}, - {IP{127, 0, 0, 1}, 1}, - {IP{127, 255, 255, 255}, 1}, - {IP{169, 254, 77, 99}, 2}, - {IP{169, 254, 44, 22}, 2}, - {IP{169, 255, 0, 1}, -1}, - {IP{172, 15, 5, 6}, -1}, - {IP{172, 16, 32, 41}, 3}, - {IP{172, 31, 128, 9}, 3}, - {IP{172, 32, 88, 100}, -1}, - {IP{192, 168, 1, 1}, 4}, - {IP{192, 168, 128, 42}, 4}, - {IP{192, 169, 1, 1}, -1}, - } - - for i, addr := range addrs { - for j, block := range blocks { - got := block.Contains(addr.ip) - want := addr.block == j - if got != want { - t.Errorf("%d/%d. %s.Contains(%s): got %v, want %v", i, j, block, addr.ip, got, want) - } - } - } - - for i, addr1 := range addrs { - for j, addr2 := range addrs { - got := sameIPv4SpecialPurposeBlock(addr1.ip, addr2.ip) - want := addr1.block >= 0 && addr1.block == addr2.block - if got != want { - t.Errorf("%d/%d. sameIPv4SpecialPurposeBlock(%s, %s): got %v, want %v", i, j, addr1.ip, addr2.ip, got, want) - } - } - } -} diff --git a/src/net/dial.go b/src/net/dial.go index 5db3585894..50bba5a49e 100644 --- a/src/net/dial.go +++ b/src/net/dial.go @@ -163,7 +163,7 @@ func parseNetwork(ctx context.Context, net string) (afnet string, proto int, err return "", 0, UnknownNetworkError(net) } -// resolverAddrList resolves addr using hint and returns a list of +// resolveAddrList resolves addr using hint and returns a list of // addresses. The result contains at least one address when error is // nil. func (r *Resolver) resolveAddrList(ctx context.Context, op, network, addr string, hint Addr) (addrList, error) { @@ -265,6 +265,9 @@ func (r *Resolver) resolveAddrList(ctx context.Context, op, network, addr string // Dial("ip6:ipv6-icmp", "2001:db8::1") // // For Unix networks, the address must be a file system path. +// +// If the host is resolved to multiple addresses, +// Dial will try each address in order until one succeeds. func Dial(network, address string) (Conn, error) { var d Dialer return d.Dial(network, address) @@ -299,6 +302,14 @@ func (d *Dialer) Dial(network, address string) (Conn, error) { // connected, any expiration of the context will not affect the // connection. // +// When using TCP, and the host in the address parameter resolves to multiple +// network addresses, any dial timeout (from d.Timeout or ctx) is spread +// over each consecutive dial, such that each is given an appropriate +// fraction of the time to connect. +// For example, if a host has 4 IP addresses and the timeout is 1 minute, +// the connect to each single address will be given 15 seconds to complete +// before trying the next one. +// // See func Dial for a description of the network and address // parameters. func (d *Dialer) DialContext(ctx context.Context, network, address string) (Conn, error) { diff --git a/src/net/dial_test.go b/src/net/dial_test.go index 6ba8e950b8..9919d72ce3 100644 --- a/src/net/dial_test.go +++ b/src/net/dial_test.go @@ -142,6 +142,8 @@ func TestDialerDualStackFDLeak(t *testing.T) { t.Skipf("%s does not have full support of socktest", runtime.GOOS) case "windows": t.Skipf("not implemented a way to cancel dial racers in TCP SYN-SENT state on %s", runtime.GOOS) + case "openbsd": + testenv.SkipFlaky(t, 15157) } if !supportsIPv4 || !supportsIPv6 { t.Skip("both IPv4 and IPv6 are required") diff --git a/src/net/dnsclient_unix.go b/src/net/dnsclient_unix.go index 2980302849..4dd4e16b0f 100644 --- a/src/net/dnsclient_unix.go +++ b/src/net/dnsclient_unix.go @@ -444,7 +444,7 @@ func goLookupHostOrder(ctx context.Context, name string, order hostLookupOrder) return } } - ips, err := goLookupIPOrder(ctx, name, order) + ips, _, err := goLookupIPCNAMEOrder(ctx, name, order) if err != nil { return } @@ -472,27 +472,28 @@ func goLookupIPFiles(name string) (addrs []IPAddr) { // The libc versions are in cgo_*.go. func goLookupIP(ctx context.Context, host string) (addrs []IPAddr, err error) { order := systemConf().hostLookupOrder(host) - return goLookupIPOrder(ctx, host, order) + addrs, _, err = goLookupIPCNAMEOrder(ctx, host, order) + return } -func goLookupIPOrder(ctx context.Context, name string, order hostLookupOrder) (addrs []IPAddr, err error) { +func goLookupIPCNAMEOrder(ctx context.Context, name string, order hostLookupOrder) (addrs []IPAddr, cname string, err error) { if order == hostLookupFilesDNS || order == hostLookupFiles { addrs = goLookupIPFiles(name) if len(addrs) > 0 || order == hostLookupFiles { - return addrs, nil + return addrs, name, nil } } if !isDomainName(name) { // See comment in func lookup above about use of errNoSuchHost. - return nil, &DNSError{Err: errNoSuchHost.Error(), Name: name} + return nil, "", &DNSError{Err: errNoSuchHost.Error(), Name: name} } resolvConf.tryUpdate("/etc/resolv.conf") resolvConf.mu.RLock() conf := resolvConf.dnsConfig resolvConf.mu.RUnlock() type racer struct { - fqdn string - rrs []dnsRR + cname string + rrs []dnsRR error } lane := make(chan racer, 1) @@ -501,20 +502,23 @@ func goLookupIPOrder(ctx context.Context, name string, order hostLookupOrder) (a for _, fqdn := range conf.nameList(name) { for _, qtype := range qtypes { go func(qtype uint16) { - _, rrs, err := tryOneName(ctx, conf, fqdn, qtype) - lane <- racer{fqdn, rrs, err} + cname, rrs, err := tryOneName(ctx, conf, fqdn, qtype) + lane <- racer{cname, rrs, err} }(qtype) } for range qtypes { racer := <-lane if racer.error != nil { // Prefer error for original name. - if lastErr == nil || racer.fqdn == name+"." { + if lastErr == nil || fqdn == name+"." { lastErr = racer.error } continue } addrs = append(addrs, addrRecordList(racer.rrs)...) + if cname == "" { + cname = racer.cname + } } if len(addrs) > 0 { break @@ -532,24 +536,16 @@ func goLookupIPOrder(ctx context.Context, name string, order hostLookupOrder) (a addrs = goLookupIPFiles(name) } if len(addrs) == 0 && lastErr != nil { - return nil, lastErr + return nil, "", lastErr } } - return addrs, nil + return addrs, cname, nil } -// goLookupCNAME is the native Go implementation of LookupCNAME. -// Used only if cgoLookupCNAME refuses to handle the request -// (that is, only if cgoLookupCNAME is the stub in cgo_stub.go). -// Normally we let cgo use the C library resolver instead of -// depending on our lookup code, so that Go and C get the same -// answers. -func goLookupCNAME(ctx context.Context, name string) (cname string, err error) { - _, rrs, err := lookup(ctx, name, dnsTypeCNAME) - if err != nil { - return - } - cname = rrs[0].(*dnsRR_CNAME).Cname +// goLookupCNAME is the native Go (non-cgo) implementation of LookupCNAME. +func goLookupCNAME(ctx context.Context, host string) (cname string, err error) { + order := systemConf().hostLookupOrder(host) + _, cname, err = goLookupIPCNAMEOrder(ctx, host, order) return } diff --git a/src/net/dnsclient_unix_test.go b/src/net/dnsclient_unix_test.go index 7dc364de50..85267bbddc 100644 --- a/src/net/dnsclient_unix_test.go +++ b/src/net/dnsclient_unix_test.go @@ -455,14 +455,14 @@ func TestGoLookupIPOrderFallbackToFile(t *testing.T) { name := fmt.Sprintf("order %v", order) // First ensure that we get an error when contacting a non-existent host. - _, err := goLookupIPOrder(context.Background(), "notarealhost", order) + _, _, err := goLookupIPCNAMEOrder(context.Background(), "notarealhost", order) if err == nil { t.Errorf("%s: expected error while looking up name not in hosts file", name) continue } // Now check that we get an address when the name appears in the hosts file. - addrs, err := goLookupIPOrder(context.Background(), "thor", order) // entry is in "testdata/hosts" + addrs, _, err := goLookupIPCNAMEOrder(context.Background(), "thor", order) // entry is in "testdata/hosts" if err != nil { t.Errorf("%s: expected to successfully lookup host entry", name) continue @@ -744,8 +744,11 @@ func TestRetryTimeout(t *testing.T) { } defer conf.teardown() - if err := conf.writeAndUpdate([]string{"nameserver 192.0.2.1", // the one that will timeout - "nameserver 192.0.2.2"}); err != nil { + testConf := []string{ + "nameserver 192.0.2.1", // the one that will timeout + "nameserver 192.0.2.2", + } + if err := conf.writeAndUpdate(testConf); err != nil { t.Fatal(err) } @@ -771,28 +774,10 @@ func TestRetryTimeout(t *testing.T) { t.Error("deadline didn't change") } - r := &dnsMsg{ - dnsMsgHdr: dnsMsgHdr{ - id: q.id, - response: true, - recursion_available: true, - }, - question: q.question, - answer: []dnsRR{ - &dnsRR_CNAME{ - Hdr: dnsRR_Header{ - Name: q.question[0].Name, - Rrtype: dnsTypeCNAME, - Class: dnsClassINET, - }, - Cname: "golang.org", - }, - }, - } - return r, nil + return mockTXTResponse(q), nil } - _, err = goLookupCNAME(context.Background(), "www.golang.org") + _, err = LookupTXT("www.golang.org") if err != nil { t.Fatal(err) } @@ -838,36 +823,40 @@ func testRotate(t *testing.T, rotate bool, nameservers, wantServers []string) { var usedServers []string d.rh = func(s string, q *dnsMsg, _ time.Time) (*dnsMsg, error) { usedServers = append(usedServers, s) - - r := &dnsMsg{ - dnsMsgHdr: dnsMsgHdr{ - id: q.id, - response: true, - recursion_available: true, - }, - question: q.question, - answer: []dnsRR{ - &dnsRR_CNAME{ - Hdr: dnsRR_Header{ - Name: q.question[0].Name, - Rrtype: dnsTypeCNAME, - Class: dnsClassINET, - }, - Cname: "golang.org", - }, - }, - } - return r, nil + return mockTXTResponse(q), nil } // len(nameservers) + 1 to allow rotation to get back to start for i := 0; i < len(nameservers)+1; i++ { - if _, err := goLookupCNAME(context.Background(), "www.golang.org"); err != nil { + if _, err := LookupTXT("www.golang.org"); err != nil { t.Fatal(err) } } if !reflect.DeepEqual(usedServers, wantServers) { - t.Fatalf("rotate=%t got used servers:\n%v\nwant:\n%v", rotate, usedServers, wantServers) + t.Errorf("rotate=%t got used servers:\n%v\nwant:\n%v", rotate, usedServers, wantServers) } } + +func mockTXTResponse(q *dnsMsg) *dnsMsg { + r := &dnsMsg{ + dnsMsgHdr: dnsMsgHdr{ + id: q.id, + response: true, + recursion_available: true, + }, + question: q.question, + answer: []dnsRR{ + &dnsRR_TXT{ + Hdr: dnsRR_Header{ + Name: q.question[0].Name, + Rrtype: dnsTypeTXT, + Class: dnsClassINET, + }, + Txt: "ok", + }, + }, + } + + return r +} diff --git a/src/net/http/client.go b/src/net/http/client.go index fe2b0196ef..d368bae861 100644 --- a/src/net/http/client.go +++ b/src/net/http/client.go @@ -163,22 +163,23 @@ func refererForURL(lastReq, newReq *url.URL) string { return referer } -func (c *Client) send(req *Request, deadline time.Time) (*Response, error) { +// didTimeout is non-nil only if err != nil. +func (c *Client) send(req *Request, deadline time.Time) (resp *Response, didTimeout func() bool, err error) { if c.Jar != nil { for _, cookie := range c.Jar.Cookies(req.URL) { req.AddCookie(cookie) } } - resp, err := send(req, c.transport(), deadline) + resp, didTimeout, err = send(req, c.transport(), deadline) if err != nil { - return nil, err + return nil, didTimeout, err } if c.Jar != nil { if rc := resp.Cookies(); len(rc) > 0 { c.Jar.SetCookies(req.URL, rc) } } - return resp, nil + return resp, nil, nil } func (c *Client) deadline() time.Time { @@ -197,22 +198,22 @@ func (c *Client) transport() RoundTripper { // send issues an HTTP request. // Caller should close resp.Body when done reading from it. -func send(ireq *Request, rt RoundTripper, deadline time.Time) (*Response, error) { +func send(ireq *Request, rt RoundTripper, deadline time.Time) (resp *Response, didTimeout func() bool, err error) { req := ireq // req is either the original request, or a modified fork if rt == nil { req.closeBody() - return nil, errors.New("http: no Client.Transport or DefaultTransport") + return nil, alwaysFalse, errors.New("http: no Client.Transport or DefaultTransport") } if req.URL == nil { req.closeBody() - return nil, errors.New("http: nil Request.URL") + return nil, alwaysFalse, errors.New("http: nil Request.URL") } if req.RequestURI != "" { req.closeBody() - return nil, errors.New("http: Request.RequestURI can't be set in client requests.") + return nil, alwaysFalse, errors.New("http: Request.RequestURI can't be set in client requests.") } // forkReq forks req into a shallow clone of ireq the first @@ -245,7 +246,7 @@ func send(ireq *Request, rt RoundTripper, deadline time.Time) (*Response, error) } stopTimer, didTimeout := setRequestCancel(req, rt, deadline) - resp, err := rt.RoundTrip(req) + resp, err = rt.RoundTrip(req) if err != nil { stopTimer() if resp != nil { @@ -259,7 +260,7 @@ func send(ireq *Request, rt RoundTripper, deadline time.Time) (*Response, error) err = errors.New("http: server gave HTTP response to HTTPS client") } } - return nil, err + return nil, didTimeout, err } if !deadline.IsZero() { resp.Body = &cancelTimerBody{ @@ -268,12 +269,17 @@ func send(ireq *Request, rt RoundTripper, deadline time.Time) (*Response, error) reqDidTimeout: didTimeout, } } - return resp, nil + return resp, nil, nil } // setRequestCancel sets the Cancel field of req, if deadline is // non-zero. The RoundTripper's type is used to determine whether the legacy // CancelRequest behavior should be used. +// +// As background, there are three ways to cancel a request: +// First was Transport.CancelRequest. (deprecated) +// Second was Request.Cancel (this mechanism). +// Third was Request.Context. func setRequestCancel(req *Request, rt RoundTripper, deadline time.Time) (stopTimer func(), didTimeout func() bool) { if deadline.IsZero() { return nop, alwaysFalse @@ -285,7 +291,7 @@ func setRequestCancel(req *Request, rt RoundTripper, deadline time.Time) (stopTi req.Cancel = cancel doCancel := func() { - // The new way: + // The newer way (the second way in the func comment): close(cancel) // The legacy compatibility way, used only @@ -407,19 +413,26 @@ func (c *Client) checkRedirect(req *Request, via []*Request) error { // redirectBehavior describes what should happen when the // client encounters a 3xx status code from the server -func redirectBehavior(reqMethod string, resp *Response, via []*Request) (redirectMethod string, shouldRedirect bool) { +func redirectBehavior(reqMethod string, resp *Response, ireq *Request) (redirectMethod string, shouldRedirect bool) { switch resp.StatusCode { case 301, 302, 303: - redirectMethod = "GET" + redirectMethod = reqMethod shouldRedirect = true + + // RFC 2616 allowed automatic redirection only with GET and + // HEAD requests. RFC 7231 lifts this restriction, but we still + // restrict other methods to GET to maintain compatibility. + // See Issue 18570. + if reqMethod != "GET" && reqMethod != "HEAD" { + redirectMethod = "GET" + } case 307, 308: redirectMethod = reqMethod shouldRedirect = true // Treat 307 and 308 specially, since they're new in // Go 1.8, and they also require re-sending the request body. - loc := resp.Header.Get("Location") - if loc == "" { + if resp.Header.Get("Location") == "" { // 308s have been observed in the wild being served // without Location headers. Since Go 1.7 and earlier // didn't follow these codes, just stop here instead @@ -428,7 +441,6 @@ func redirectBehavior(reqMethod string, resp *Response, via []*Request) (redirec shouldRedirect = false break } - ireq := via[0] if ireq.GetBody == nil && ireq.outgoingLength() != 0 { // We had a request body, and 307/308 require // re-sending it, but GetBody is not defined. So just @@ -437,7 +449,6 @@ func redirectBehavior(reqMethod string, resp *Response, via []*Request) (redirec shouldRedirect = false } } - return redirectMethod, shouldRedirect } @@ -464,6 +475,16 @@ func redirectBehavior(reqMethod string, resp *Response, via []*Request) (redirec // the returned Response.Body is already closed. // // Generally Get, Post, or PostForm will be used instead of Do. +// +// If the server replies with a redirect, the Client first uses the +// CheckRedirect function to determine whether the redirect should be +// followed. If permitted, a 301, 302, or 303 redirect causes +// subsequent requests to use HTTP method GET +// (or HEAD if the original request was HEAD), with no body. +// A 307 or 308 redirect preserves the original HTTP method and body, +// provided that the Request.GetBody function is defined. +// The NewRequest function automatically sets GetBody for common +// standard library body types. func (c *Client) Do(req *Request) (*Response, error) { if req.URL == nil { req.closeBody() @@ -565,8 +586,9 @@ func (c *Client) Do(req *Request) (*Response, error) { reqs = append(reqs, req) var err error - if resp, err = c.send(req, deadline); err != nil { - if !deadline.IsZero() && !time.Now().Before(deadline) { + var didTimeout func() bool + if resp, didTimeout, err = c.send(req, deadline); err != nil { + if !deadline.IsZero() && didTimeout() { err = &httpError{ err: err.Error() + " (Client.Timeout exceeded while awaiting headers)", timeout: true, @@ -576,7 +598,7 @@ func (c *Client) Do(req *Request) (*Response, error) { } var shouldRedirect bool - redirectMethod, shouldRedirect = redirectBehavior(req.Method, resp, reqs) + redirectMethod, shouldRedirect = redirectBehavior(req.Method, resp, reqs[0]) if !shouldRedirect { return resp, nil } @@ -666,6 +688,9 @@ func defaultCheckRedirect(req *Request, via []*Request) error { // Post is a wrapper around DefaultClient.Post. // // To set custom headers, use NewRequest and DefaultClient.Do. +// +// See the Client.Do method documentation for details on how redirects +// are handled. func Post(url string, contentType string, body io.Reader) (resp *Response, err error) { return DefaultClient.Post(url, contentType, body) } @@ -678,6 +703,9 @@ func Post(url string, contentType string, body io.Reader) (resp *Response, err e // request. // // To set custom headers, use NewRequest and Client.Do. +// +// See the Client.Do method documentation for details on how redirects +// are handled. func (c *Client) Post(url string, contentType string, body io.Reader) (resp *Response, err error) { req, err := NewRequest("POST", url, body) if err != nil { @@ -697,6 +725,9 @@ func (c *Client) Post(url string, contentType string, body io.Reader) (resp *Res // Caller should close resp.Body when done reading from it. // // PostForm is a wrapper around DefaultClient.PostForm. +// +// See the Client.Do method documentation for details on how redirects +// are handled. func PostForm(url string, data url.Values) (resp *Response, err error) { return DefaultClient.PostForm(url, data) } @@ -709,6 +740,9 @@ func PostForm(url string, data url.Values) (resp *Response, err error) { // // When err is nil, resp always contains a non-nil resp.Body. // Caller should close resp.Body when done reading from it. +// +// See the Client.Do method documentation for details on how redirects +// are handled. func (c *Client) PostForm(url string, data url.Values) (resp *Response, err error) { return c.Post(url, "application/x-www-form-urlencoded", strings.NewReader(data.Encode())) } diff --git a/src/net/http/client_test.go b/src/net/http/client_test.go index a5f58cb5cb..eaf2cdca8e 100644 --- a/src/net/http/client_test.go +++ b/src/net/http/client_test.go @@ -26,6 +26,7 @@ import ( "strconv" "strings" "sync" + "sync/atomic" "testing" "time" ) @@ -1664,9 +1665,9 @@ func TestClientRedirectTypes(t *testing.T) { 3: {method: "POST", serverStatus: 307, wantMethod: "POST"}, 4: {method: "POST", serverStatus: 308, wantMethod: "POST"}, - 5: {method: "HEAD", serverStatus: 301, wantMethod: "GET"}, - 6: {method: "HEAD", serverStatus: 302, wantMethod: "GET"}, - 7: {method: "HEAD", serverStatus: 303, wantMethod: "GET"}, + 5: {method: "HEAD", serverStatus: 301, wantMethod: "HEAD"}, + 6: {method: "HEAD", serverStatus: 302, wantMethod: "HEAD"}, + 7: {method: "HEAD", serverStatus: 303, wantMethod: "HEAD"}, 8: {method: "HEAD", serverStatus: 307, wantMethod: "HEAD"}, 9: {method: "HEAD", serverStatus: 308, wantMethod: "HEAD"}, @@ -1738,3 +1739,76 @@ func TestClientRedirectTypes(t *testing.T) { res.Body.Close() } } + +// issue18239Body is an io.ReadCloser for TestTransportBodyReadError. +// Its Read returns readErr and increments *readCalls atomically. +// Its Close returns nil and increments *closeCalls atomically. +type issue18239Body struct { + readCalls *int32 + closeCalls *int32 + readErr error +} + +func (b issue18239Body) Read([]byte) (int, error) { + atomic.AddInt32(b.readCalls, 1) + return 0, b.readErr +} + +func (b issue18239Body) Close() error { + atomic.AddInt32(b.closeCalls, 1) + return nil +} + +// Issue 18239: make sure the Transport doesn't retry requests with bodies. +// (Especially if Request.GetBody is not defined.) +func TestTransportBodyReadError(t *testing.T) { + setParallel(t) + defer afterTest(t) + ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { + if r.URL.Path == "/ping" { + return + } + buf := make([]byte, 1) + n, err := r.Body.Read(buf) + w.Header().Set("X-Body-Read", fmt.Sprintf("%v, %v", n, err)) + })) + defer ts.Close() + tr := &Transport{} + defer tr.CloseIdleConnections() + c := &Client{Transport: tr} + + // Do one initial successful request to create an idle TCP connection + // for the subsequent request to reuse. (The Transport only retries + // requests on reused connections.) + res, err := c.Get(ts.URL + "/ping") + if err != nil { + t.Fatal(err) + } + res.Body.Close() + + var readCallsAtomic int32 + var closeCallsAtomic int32 // atomic + someErr := errors.New("some body read error") + body := issue18239Body{&readCallsAtomic, &closeCallsAtomic, someErr} + + req, err := NewRequest("POST", ts.URL, body) + if err != nil { + t.Fatal(err) + } + _, err = tr.RoundTrip(req) + if err != someErr { + t.Errorf("Got error: %v; want Request.Body read error: %v", err, someErr) + } + + // And verify that our Body wasn't used multiple times, which + // would indicate retries. (as it buggily was during part of + // Go 1.8's dev cycle) + readCalls := atomic.LoadInt32(&readCallsAtomic) + closeCalls := atomic.LoadInt32(&closeCallsAtomic) + if readCalls != 1 { + t.Errorf("read calls = %d; want 1", readCalls) + } + if closeCalls != 1 { + t.Errorf("close calls = %d; want 1", closeCalls) + } +} diff --git a/src/net/http/h2_bundle.go b/src/net/http/h2_bundle.go index bb7f05df2e..4536b2ff5d 100644 --- a/src/net/http/h2_bundle.go +++ b/src/net/http/h2_bundle.go @@ -855,10 +855,12 @@ type http2Framer struct { // If the limit is hit, MetaHeadersFrame.Truncated is set true. MaxHeaderListSize uint32 - logReads bool + logReads, logWrites bool - debugFramer *http2Framer // only use for logging written writes - debugFramerBuf *bytes.Buffer + debugFramer *http2Framer // only use for logging written writes + debugFramerBuf *bytes.Buffer + debugReadLoggerf func(string, ...interface{}) + debugWriteLoggerf func(string, ...interface{}) } func (fr *http2Framer) maxHeaderListSize() uint32 { @@ -892,7 +894,7 @@ func (f *http2Framer) endWrite() error { byte(length>>16), byte(length>>8), byte(length)) - if http2logFrameWrites { + if f.logWrites { f.logWrite() } @@ -914,10 +916,10 @@ func (f *http2Framer) logWrite() { f.debugFramerBuf.Write(f.wbuf) fr, err := f.debugFramer.ReadFrame() if err != nil { - log.Printf("http2: Framer %p: failed to decode just-written frame", f) + f.debugWriteLoggerf("http2: Framer %p: failed to decode just-written frame", f) return } - log.Printf("http2: Framer %p: wrote %v", f, http2summarizeFrame(fr)) + f.debugWriteLoggerf("http2: Framer %p: wrote %v", f, http2summarizeFrame(fr)) } func (f *http2Framer) writeByte(v byte) { f.wbuf = append(f.wbuf, v) } @@ -938,9 +940,12 @@ const ( // NewFramer returns a Framer that writes frames to w and reads them from r. func http2NewFramer(w io.Writer, r io.Reader) *http2Framer { fr := &http2Framer{ - w: w, - r: r, - logReads: http2logFrameReads, + w: w, + r: r, + logReads: http2logFrameReads, + logWrites: http2logFrameWrites, + debugReadLoggerf: log.Printf, + debugWriteLoggerf: log.Printf, } fr.getReadBuf = func(size uint32) []byte { if cap(fr.readBuf) >= int(size) { @@ -1022,7 +1027,7 @@ func (fr *http2Framer) ReadFrame() (http2Frame, error) { return nil, err } if fr.logReads { - log.Printf("http2: Framer %p: read %v", fr, http2summarizeFrame(f)) + fr.debugReadLoggerf("http2: Framer %p: read %v", fr, http2summarizeFrame(f)) } if fh.Type == http2FrameHeaders && fr.ReadMetaHeaders != nil { return fr.readMetaFrame(f.(*http2HeadersFrame)) @@ -1922,8 +1927,8 @@ func (fr *http2Framer) readMetaFrame(hf *http2HeadersFrame) (*http2MetaHeadersFr hdec.SetEmitEnabled(true) hdec.SetMaxStringLength(fr.maxHeaderStringLen()) hdec.SetEmitFunc(func(hf hpack.HeaderField) { - if http2VerboseLogs && http2logFrameReads { - log.Printf("http2: decoded hpack field %+v", hf) + if http2VerboseLogs && fr.logReads { + fr.debugReadLoggerf("http2: decoded hpack field %+v", hf) } if !httplex.ValidHeaderFieldValue(hf.Value) { invalid = http2headerFieldValueError(hf.Value) @@ -2188,6 +2193,14 @@ func http2shouldLogPanic(panicValue interface{}) bool { return panicValue != nil && panicValue != ErrAbortHandler } +func http2reqGetBody(req *Request) func() (io.ReadCloser, error) { + return req.GetBody +} + +func http2reqBodyIsNoBody(body io.ReadCloser) bool { + return body == NoBody +} + var http2DebugGoroutines = os.Getenv("DEBUG_HTTP2_GOROUTINES") == "1" type http2goroutineLock uint64 @@ -3128,6 +3141,10 @@ func (s *http2Server) ServeConn(c net.Conn, opts *http2ServeConnOpts) { pushEnabled: true, } + if sc.hs.WriteTimeout != 0 { + sc.conn.SetWriteDeadline(time.Time{}) + } + if s.NewWriteScheduler != nil { sc.writeSched = s.NewWriteScheduler() } else { @@ -3247,6 +3264,11 @@ func (sc *http2serverConn) maxHeaderListSize() uint32 { return uint32(n + typicalHeaders*perFieldOverhead) } +func (sc *http2serverConn) curOpenStreams() uint32 { + sc.serveG.check() + return sc.curClientStreams + sc.curPushedStreams +} + // stream represents a stream. This is the minimal metadata needed by // the serve goroutine. Most of the actual stream state is owned by // the http.Handler's goroutine in the responseWriter. Because the @@ -3272,8 +3294,7 @@ type http2stream struct { numTrailerValues int64 weight uint8 state http2streamState - sentReset bool // only true once detached from streams map - gotReset bool // only true once detacted from streams map + resetQueued bool // RST_STREAM queued for write; set by sc.resetStream gotTrailerHeader bool // HEADER frame for trailers was seen wroteHeaders bool // whether we wrote headers (not status 100) reqBuf []byte // if non-nil, body pipe buffer to return later at EOF @@ -3560,7 +3581,7 @@ func (sc *http2serverConn) serve() { fn(loopNum) } - if sc.inGoAway && sc.curClientStreams == 0 && !sc.needToSendGoAway && !sc.writingFrame { + if sc.inGoAway && sc.curOpenStreams() == 0 && !sc.needToSendGoAway && !sc.writingFrame { return } } @@ -3669,13 +3690,25 @@ func (sc *http2serverConn) writeFrameFromHandler(wr http2FrameWriteRequest) erro func (sc *http2serverConn) writeFrame(wr http2FrameWriteRequest) { sc.serveG.check() + // If true, wr will not be written and wr.done will not be signaled. var ignoreWrite bool + if wr.StreamID() != 0 { + _, isReset := wr.write.(http2StreamError) + if state, _ := sc.state(wr.StreamID()); state == http2stateClosed && !isReset { + ignoreWrite = true + } + } + switch wr.write.(type) { case *http2writeResHeaders: wr.stream.wroteHeaders = true case http2write100ContinueHeadersFrame: if wr.stream.wroteHeaders { + + if wr.done != nil { + panic("wr.done != nil for write100ContinueHeadersFrame") + } ignoreWrite = true } } @@ -3699,14 +3732,14 @@ func (sc *http2serverConn) startFrameWrite(wr http2FrameWriteRequest) { if st != nil { switch st.state { case http2stateHalfClosedLocal: - panic("internal error: attempt to send frame on half-closed-local stream") - case http2stateClosed: - if st.sentReset || st.gotReset { + switch wr.write.(type) { + case http2StreamError, http2handlerPanicRST, http2writeWindowUpdate: - sc.scheduleFrameWrite() - return + default: + panic(fmt.Sprintf("internal error: attempt to send frame on a half-closed-local stream: %v", wr)) } - panic(fmt.Sprintf("internal error: attempt to send a write %v on a closed stream", wr)) + case http2stateClosed: + panic(fmt.Sprintf("internal error: attempt to send frame on a closed stream: %v", wr)) } } if wpp, ok := wr.write.(*http2writePushPromise); ok { @@ -3714,9 +3747,7 @@ func (sc *http2serverConn) startFrameWrite(wr http2FrameWriteRequest) { wpp.promisedID, err = wpp.allocatePromisedID() if err != nil { sc.writingFrameAsync = false - if wr.done != nil { - wr.done <- err - } + wr.replyToWriter(err) return } } @@ -3749,24 +3780,9 @@ func (sc *http2serverConn) wroteFrame(res http2frameWriteResult) { sc.writingFrameAsync = false wr := res.wr - st := wr.stream - - closeStream := http2endsStream(wr.write) - if _, ok := wr.write.(http2handlerPanicRST); ok { - sc.closeStream(st, http2errHandlerPanicked) - } - - if ch := wr.done; ch != nil { - select { - case ch <- res.err: - default: - panic(fmt.Sprintf("unbuffered done channel passed in for type %T", wr.write)) - } - } - wr.write = nil - - if closeStream { + if http2writeEndsStream(wr.write) { + st := wr.stream if st == nil { panic("internal error: expecting non-nil stream") } @@ -3774,13 +3790,24 @@ func (sc *http2serverConn) wroteFrame(res http2frameWriteResult) { case http2stateOpen: st.state = http2stateHalfClosedLocal - errCancel := http2streamError(st.id, http2ErrCodeCancel) - sc.resetStream(errCancel) + sc.resetStream(http2streamError(st.id, http2ErrCodeCancel)) case http2stateHalfClosedRemote: sc.closeStream(st, http2errHandlerComplete) } + } else { + switch v := wr.write.(type) { + case http2StreamError: + + if st, ok := sc.streams[v.StreamID]; ok { + sc.closeStream(st, v) + } + case http2handlerPanicRST: + sc.closeStream(wr.stream, http2errHandlerPanicked) + } } + wr.replyToWriter(res.err) + sc.scheduleFrameWrite() } @@ -3877,8 +3904,7 @@ func (sc *http2serverConn) resetStream(se http2StreamError) { sc.serveG.check() sc.writeFrame(http2FrameWriteRequest{write: se}) if st, ok := sc.streams[se.StreamID]; ok { - st.sentReset = true - sc.closeStream(st, se) + st.resetQueued = true } } @@ -4017,7 +4043,6 @@ func (sc *http2serverConn) processResetStream(f *http2RSTStreamFrame) error { return http2ConnectionError(http2ErrCodeProtocol) } if st != nil { - st.gotReset = true st.cancelCtx() sc.closeStream(st, http2streamError(f.StreamID, f.ErrCode)) } @@ -4132,7 +4157,7 @@ func (sc *http2serverConn) processData(f *http2DataFrame) error { return http2ConnectionError(http2ErrCodeProtocol) } - if st == nil || state != http2stateOpen || st.gotTrailerHeader { + if st == nil || state != http2stateOpen || st.gotTrailerHeader || st.resetQueued { if sc.inflow.available() < int32(f.Length) { return http2streamError(id, http2ErrCodeFlowControl) @@ -4141,6 +4166,10 @@ func (sc *http2serverConn) processData(f *http2DataFrame) error { sc.inflow.take(int32(f.Length)) sc.sendWindowUpdate(nil, int(f.Length)) + if st != nil && st.resetQueued { + + return nil + } return http2streamError(id, http2ErrCodeStreamClosed) } if st.body == nil { @@ -4238,6 +4267,10 @@ func (sc *http2serverConn) processHeaders(f *http2MetaHeadersFrame) error { } if st := sc.streams[f.StreamID]; st != nil { + if st.resetQueued { + + return nil + } return st.processTrailerHeaders(f) } @@ -4373,7 +4406,7 @@ func (sc *http2serverConn) newStream(id, pusherID uint32, state http2streamState } else { sc.curClientStreams++ } - if sc.curClientStreams+sc.curPushedStreams == 1 { + if sc.curOpenStreams() == 1 { sc.setConnState(StateActive) } @@ -5114,7 +5147,7 @@ func (w *http2responseWriter) push(target string, opts http2pushOptions) error { } for k := range opts.Header { if strings.HasPrefix(k, ":") { - return fmt.Errorf("promised request headers cannot include psuedo header %q", k) + return fmt.Errorf("promised request headers cannot include pseudo header %q", k) } switch strings.ToLower(k) { @@ -5203,7 +5236,7 @@ func (sc *http2serverConn) startPush(msg http2startPushRequest) { scheme: msg.url.Scheme, authority: msg.url.Host, path: msg.url.RequestURI(), - header: msg.header, + header: http2cloneHeader(msg.header), }) if err != nil { @@ -5510,6 +5543,7 @@ type http2clientStream struct { ID uint32 resc chan http2resAndError bufPipe http2pipe // buffered pipe with the flow-controlled response payload + startedWrite bool // started request body write; guarded by cc.mu requestedGzip bool on100 func() // optional code to run if get a 100 continue response @@ -5633,6 +5667,10 @@ func http2authorityAddr(scheme string, authority string) (addr string) { if a, err := idna.ToASCII(host); err == nil { host = a } + + if strings.HasPrefix(host, "[") && strings.HasSuffix(host, "]") { + return host + ":" + port + } return net.JoinHostPort(host, port) } @@ -5651,8 +5689,10 @@ func (t *http2Transport) RoundTripOpt(req *Request, opt http2RoundTripOpt) (*Res } http2traceGotConn(req, cc) res, err := cc.RoundTrip(req) - if http2shouldRetryRequest(req, err) { - continue + if err != nil { + if req, err = http2shouldRetryRequest(req, err); err == nil { + continue + } } if err != nil { t.vlogf("RoundTrip failure: %v", err) @@ -5674,11 +5714,39 @@ func (t *http2Transport) CloseIdleConnections() { var ( http2errClientConnClosed = errors.New("http2: client conn is closed") http2errClientConnUnusable = errors.New("http2: client conn not usable") + + http2errClientConnGotGoAway = errors.New("http2: Transport received Server's graceful shutdown GOAWAY") + http2errClientConnGotGoAwayAfterSomeReqBody = errors.New("http2: Transport received Server's graceful shutdown GOAWAY; some request body already written") ) -func http2shouldRetryRequest(req *Request, err error) bool { +// shouldRetryRequest is called by RoundTrip when a request fails to get +// response headers. It is always called with a non-nil error. +// It returns either a request to retry (either the same request, or a +// modified clone), or an error if the request can't be replayed. +func http2shouldRetryRequest(req *Request, err error) (*Request, error) { + switch err { + default: + return nil, err + case http2errClientConnUnusable, http2errClientConnGotGoAway: + return req, nil + case http2errClientConnGotGoAwayAfterSomeReqBody: - return err == http2errClientConnUnusable + if req.Body == nil || http2reqBodyIsNoBody(req.Body) { + return req, nil + } + + getBody := http2reqGetBody(req) + if getBody == nil { + return nil, errors.New("http2: Transport: peer server initiated graceful shutdown after some of Request.Body was written; define Request.GetBody to avoid this error") + } + body, err := getBody() + if err != nil { + return nil, err + } + newReq := *req + newReq.Body = body + return &newReq, nil + } } func (t *http2Transport) dialClientConn(addr string, singleUse bool) (*http2ClientConn, error) { @@ -5826,6 +5894,15 @@ func (cc *http2ClientConn) setGoAway(f *http2GoAwayFrame) { if old != nil && old.ErrCode != http2ErrCodeNo { cc.goAway.ErrCode = old.ErrCode } + last := f.LastStreamID + for streamID, cs := range cc.streams { + if streamID > last { + select { + case cs.resc <- http2resAndError{err: http2errClientConnGotGoAway}: + default: + } + } + } } func (cc *http2ClientConn) CanTakeNewRequest() bool { @@ -6059,6 +6136,13 @@ func (cc *http2ClientConn) RoundTrip(req *Request) (*Response, error) { cs.abortRequestBodyWrite(http2errStopReqBodyWrite) } if re.err != nil { + if re.err == http2errClientConnGotGoAway { + cc.mu.Lock() + if cs.startedWrite { + re.err = http2errClientConnGotGoAwayAfterSomeReqBody + } + cc.mu.Unlock() + } cc.forgetStreamID(cs.ID) return nil, re.err } @@ -7225,6 +7309,9 @@ func (t *http2Transport) getBodyWriterState(cs *http2clientStream, body io.Reade resc := make(chan error, 1) s.resc = resc s.fn = func() { + cs.cc.mu.Lock() + cs.startedWrite = true + cs.cc.mu.Unlock() resc <- cs.writeRequestBody(body, cs.req.Body) } s.delay = t.expectContinueTimeout() @@ -7313,9 +7400,10 @@ type http2writeContext interface { HeaderEncoder() (*hpack.Encoder, *bytes.Buffer) } -// endsStream reports whether the given frame writer w will locally -// close the stream. -func http2endsStream(w http2writeFramer) bool { +// writeEndsStream reports whether w writes a frame that will transition +// the stream to a half-closed local state. This returns false for RST_STREAM, +// which closes the entire stream (not just the local half). +func http2writeEndsStream(w http2writeFramer) bool { switch v := w.(type) { case *http2writeData: return v.endStream @@ -7323,7 +7411,7 @@ func http2endsStream(w http2writeFramer) bool { return v.endStream case nil: - panic("endsStream called on nil writeFramer") + panic("writeEndsStream called on nil writeFramer") } return false } @@ -7644,7 +7732,9 @@ type http2WriteScheduler interface { // https://tools.ietf.org/html/rfc7540#section-5.1 AdjustStream(streamID uint32, priority http2PriorityParam) - // Push queues a frame in the scheduler. + // Push queues a frame in the scheduler. In most cases, this will not be + // called with wr.StreamID()!=0 unless that stream is currently open. The one + // exception is RST_STREAM frames, which may be sent on idle or closed streams. Push(wr http2FrameWriteRequest) // Pop dequeues the next frame to write. Returns false if no frames can @@ -7767,6 +7857,20 @@ func (wr http2FrameWriteRequest) String() string { return fmt.Sprintf("[FrameWriteRequest stream=%d, ch=%v, writer=%v]", wr.StreamID(), wr.done != nil, des) } +// replyToWriter sends err to wr.done and panics if the send must block +// This does nothing if wr.done is nil. +func (wr *http2FrameWriteRequest) replyToWriter(err error) { + if wr.done == nil { + return + } + select { + case wr.done <- err: + default: + panic(fmt.Sprintf("unbuffered done channel passed in for type %T", wr.write)) + } + wr.write = nil +} + // writeQueue is used by implementations of WriteScheduler. type http2writeQueue struct { s []http2FrameWriteRequest @@ -8183,7 +8287,11 @@ func (ws *http2priorityWriteScheduler) Push(wr http2FrameWriteRequest) { } else { n = ws.nodes[id] if n == nil { - panic("add on non-open stream") + + if wr.DataSize() > 0 { + panic("add DATA on non-open stream") + } + n = &ws.root } } n.q.push(wr) diff --git a/src/net/http/httptrace/trace.go b/src/net/http/httptrace/trace.go index 3b7417911f..ea7b38c8fc 100644 --- a/src/net/http/httptrace/trace.go +++ b/src/net/http/httptrace/trace.go @@ -146,7 +146,8 @@ type ClientTrace struct { Wait100Continue func() // WroteRequest is called with the result of writing the - // request and any body. + // request and any body. It may be called multiple times + // in the case of retried requests. WroteRequest func(WroteRequestInfo) } diff --git a/src/net/http/httputil/dump.go b/src/net/http/httputil/dump.go index 1511681632..7104c37454 100644 --- a/src/net/http/httputil/dump.go +++ b/src/net/http/httputil/dump.go @@ -18,11 +18,16 @@ import ( "time" ) -// One of the copies, say from b to r2, could be avoided by using a more -// elaborate trick where the other copy is made during Request/Response.Write. -// This would complicate things too much, given that these functions are for -// debugging only. +// drainBody reads all of b to memory and then returns two equivalent +// ReadClosers yielding the same bytes. +// +// It returns an error if the initial slurp of all bytes fails. It does not attempt +// to make the returned ReadClosers have identical error-matching behavior. func drainBody(b io.ReadCloser) (r1, r2 io.ReadCloser, err error) { + if b == http.NoBody { + // No copying needed. Preserve the magic sentinel meaning of NoBody. + return http.NoBody, http.NoBody, nil + } var buf bytes.Buffer if _, err = buf.ReadFrom(b); err != nil { return nil, b, err diff --git a/src/net/http/httputil/dump_test.go b/src/net/http/httputil/dump_test.go index 2e980d39f8..f881020fef 100644 --- a/src/net/http/httputil/dump_test.go +++ b/src/net/http/httputil/dump_test.go @@ -184,6 +184,18 @@ var dumpTests = []dumpTest{ WantDump: "POST /v2/api/?login HTTP/1.1\r\n" + "Host: passport.myhost.com\r\n\r\n", }, + + // Issue 18506: make drainBody recognize NoBody. Otherwise + // this was turning into a chunked request. + { + Req: *mustNewRequest("POST", "http://example.com/foo", http.NoBody), + + WantDumpOut: "POST /foo HTTP/1.1\r\n" + + "Host: example.com\r\n" + + "User-Agent: Go-http-client/1.1\r\n" + + "Content-Length: 0\r\n" + + "Accept-Encoding: gzip\r\n\r\n", + }, } func TestDumpRequest(t *testing.T) { diff --git a/src/net/http/httputil/reverseproxy.go b/src/net/http/httputil/reverseproxy.go index 7867505708..79c8fe2770 100644 --- a/src/net/http/httputil/reverseproxy.go +++ b/src/net/http/httputil/reverseproxy.go @@ -30,6 +30,8 @@ type ReverseProxy struct { // the request into a new request to be sent // using Transport. Its response is then copied // back to the original client unmodified. + // Director must not access the provided Request + // after returning. Director func(*http.Request) // The transport used to perform proxy requests. diff --git a/src/net/http/request.go b/src/net/http/request.go index fd9ea5494e..fb6bb0aab5 100644 --- a/src/net/http/request.go +++ b/src/net/http/request.go @@ -171,7 +171,7 @@ type Request struct { Body io.ReadCloser // GetBody defines an optional func to return a new copy of - // Body. It used for client requests when a redirect requires + // Body. It is used for client requests when a redirect requires // reading the body more than once. Use of GetBody still // requires setting Body. // @@ -313,8 +313,8 @@ type Request struct { // For outgoing client requests, the context controls cancelation. // // For incoming server requests, the context is canceled when the -// ServeHTTP method returns. For its associated values, see -// ServerContextKey and LocalAddrContextKey. +// client's connection closes, the request is canceled (with HTTP/2), +// or when the ServeHTTP method returns. func (r *Request) Context() context.Context { if r.ctx != nil { return r.ctx @@ -341,6 +341,18 @@ func (r *Request) ProtoAtLeast(major, minor int) bool { r.ProtoMajor == major && r.ProtoMinor >= minor } +// protoAtLeastOutgoing is like ProtoAtLeast, but is for outgoing +// requests (see issue 18407) where these fields aren't supposed to +// matter. As a minor fix for Go 1.8, at least treat (0, 0) as +// matching HTTP/1.1 or HTTP/1.0. Only HTTP/1.1 is used. +// TODO(bradfitz): ideally remove this whole method. It shouldn't be used. +func (r *Request) protoAtLeastOutgoing(major, minor int) bool { + if r.ProtoMajor == 0 && r.ProtoMinor == 0 && major == 1 && minor <= 1 { + return true + } + return r.ProtoAtLeast(major, minor) +} + // UserAgent returns the client's User-Agent, if sent in the request. func (r *Request) UserAgent() string { return r.Header.Get("User-Agent") @@ -600,6 +612,12 @@ func (req *Request) write(w io.Writer, usingProxy bool, extraHeaders Header, wai } } + if bw, ok := w.(*bufio.Writer); ok && tw.FlushHeaders { + if err := bw.Flush(); err != nil { + return err + } + } + // Write body and trailer err = tw.WriteBody(w) if err != nil { @@ -731,6 +749,12 @@ func validMethod(method string) bool { // net/http/httptest package, use ReadRequest, or manually update the // Request fields. See the Request type's documentation for the // difference between inbound and outbound request fields. +// +// If body is of type *bytes.Buffer, *bytes.Reader, or +// *strings.Reader, the returned request's ContentLength is set to its +// exact value (instead of -1), GetBody is populated (so 307 and 308 +// redirects can replay the body), and Body is set to NoBody if the +// ContentLength is 0. func NewRequest(method, urlStr string, body io.Reader) (*Request, error) { if method == "" { // We document that "" means "GET" for Request.Method, and people have @@ -785,7 +809,11 @@ func NewRequest(method, urlStr string, body io.Reader) (*Request, error) { return ioutil.NopCloser(&r), nil } default: - req.ContentLength = -1 // unknown + // This is where we'd set it to -1 (at least + // if body != NoBody) to mean unknown, but + // that broke people during the Go 1.8 testing + // period. People depend on it being 0 I + // guess. Maybe retry later. See Issue 18117. } // For client requests, Request.ContentLength of 0 // means either actually 0, or unknown. The only way @@ -795,7 +823,7 @@ func NewRequest(method, urlStr string, body io.Reader) (*Request, error) { // so we use a well-known ReadCloser variable instead // and have the http package also treat that sentinel // variable to mean explicitly zero. - if req.ContentLength == 0 { + if req.GetBody != nil && req.ContentLength == 0 { req.Body = NoBody req.GetBody = func() (io.ReadCloser, error) { return NoBody, nil } } @@ -1289,3 +1317,18 @@ func (r *Request) outgoingLength() int64 { } return -1 } + +// requestMethodUsuallyLacksBody reports whether the given request +// method is one that typically does not involve a request body. +// This is used by the Transport (via +// transferWriter.shouldSendChunkedRequestBody) to determine whether +// we try to test-read a byte from a non-nil Request.Body when +// Request.outgoingLength() returns -1. See the comments in +// shouldSendChunkedRequestBody. +func requestMethodUsuallyLacksBody(method string) bool { + switch method { + case "GET", "HEAD", "DELETE", "OPTIONS", "PROPFIND", "SEARCH": + return true + } + return false +} diff --git a/src/net/http/request_test.go b/src/net/http/request_test.go index 3c965c1e8a..e6748375b5 100644 --- a/src/net/http/request_test.go +++ b/src/net/http/request_test.go @@ -498,10 +498,14 @@ func TestNewRequestContentLength(t *testing.T) { {bytes.NewBuffer([]byte("1234")), 4}, {strings.NewReader("12345"), 5}, {strings.NewReader(""), 0}, - // Not detected: - {struct{ io.Reader }{strings.NewReader("xyz")}, -1}, - {io.NewSectionReader(strings.NewReader("x"), 0, 6), -1}, - {readByte(io.NewSectionReader(strings.NewReader("xy"), 0, 6)), -1}, + {NoBody, 0}, + + // Not detected. During Go 1.8 we tried to make these set to -1, but + // due to Issue 18117, we keep these returning 0, even though they're + // unknown. + {struct{ io.Reader }{strings.NewReader("xyz")}, 0}, + {io.NewSectionReader(strings.NewReader("x"), 0, 6), 0}, + {readByte(io.NewSectionReader(strings.NewReader("xy"), 0, 6)), 0}, } for i, tt := range tests { req, err := NewRequest("POST", "http://localhost/", tt.r) @@ -511,9 +515,6 @@ func TestNewRequestContentLength(t *testing.T) { if req.ContentLength != tt.want { t.Errorf("test[%d]: ContentLength(%T) = %d; want %d", i, tt.r, req.ContentLength, tt.want) } - if (req.ContentLength == 0) != (req.Body == NoBody) { - t.Errorf("test[%d]: ContentLength = %d but Body non-nil is %v", i, req.ContentLength, req.Body != nil) - } } } diff --git a/src/net/http/requestwrite_test.go b/src/net/http/requestwrite_test.go index d13e37aba0..eb65b9f736 100644 --- a/src/net/http/requestwrite_test.go +++ b/src/net/http/requestwrite_test.go @@ -5,14 +5,17 @@ package http import ( + "bufio" "bytes" "errors" "fmt" "io" "io/ioutil" + "net" "net/url" "strings" "testing" + "time" ) type reqWriteTest struct { @@ -566,6 +569,138 @@ func TestRequestWrite(t *testing.T) { } } +func TestRequestWriteTransport(t *testing.T) { + t.Parallel() + + matchSubstr := func(substr string) func(string) error { + return func(written string) error { + if !strings.Contains(written, substr) { + return fmt.Errorf("expected substring %q in request: %s", substr, written) + } + return nil + } + } + + noContentLengthOrTransferEncoding := func(req string) error { + if strings.Contains(req, "Content-Length: ") { + return fmt.Errorf("unexpected Content-Length in request: %s", req) + } + if strings.Contains(req, "Transfer-Encoding: ") { + return fmt.Errorf("unexpected Transfer-Encoding in request: %s", req) + } + return nil + } + + all := func(checks ...func(string) error) func(string) error { + return func(req string) error { + for _, c := range checks { + if err := c(req); err != nil { + return err + } + } + return nil + } + } + + type testCase struct { + method string + clen int64 // ContentLength + body io.ReadCloser + want func(string) error + + // optional: + init func(*testCase) + afterReqRead func() + } + + tests := []testCase{ + { + method: "GET", + want: noContentLengthOrTransferEncoding, + }, + { + method: "GET", + body: ioutil.NopCloser(strings.NewReader("")), + want: noContentLengthOrTransferEncoding, + }, + { + method: "GET", + clen: -1, + body: ioutil.NopCloser(strings.NewReader("")), + want: noContentLengthOrTransferEncoding, + }, + // A GET with a body, with explicit content length: + { + method: "GET", + clen: 7, + body: ioutil.NopCloser(strings.NewReader("foobody")), + want: all(matchSubstr("Content-Length: 7"), + matchSubstr("foobody")), + }, + // A GET with a body, sniffing the leading "f" from "foobody". + { + method: "GET", + clen: -1, + body: ioutil.NopCloser(strings.NewReader("foobody")), + want: all(matchSubstr("Transfer-Encoding: chunked"), + matchSubstr("\r\n1\r\nf\r\n"), + matchSubstr("oobody")), + }, + // But a POST request is expected to have a body, so + // no sniffing happens: + { + method: "POST", + clen: -1, + body: ioutil.NopCloser(strings.NewReader("foobody")), + want: all(matchSubstr("Transfer-Encoding: chunked"), + matchSubstr("foobody")), + }, + { + method: "POST", + clen: -1, + body: ioutil.NopCloser(strings.NewReader("")), + want: all(matchSubstr("Transfer-Encoding: chunked")), + }, + // Verify that a blocking Request.Body doesn't block forever. + { + method: "GET", + clen: -1, + init: func(tt *testCase) { + pr, pw := io.Pipe() + tt.afterReqRead = func() { + pw.Close() + } + tt.body = ioutil.NopCloser(pr) + }, + want: matchSubstr("Transfer-Encoding: chunked"), + }, + } + + for i, tt := range tests { + if tt.init != nil { + tt.init(&tt) + } + req := &Request{ + Method: tt.method, + URL: &url.URL{ + Scheme: "http", + Host: "example.com", + }, + Header: make(Header), + ContentLength: tt.clen, + Body: tt.body, + } + got, err := dumpRequestOut(req, tt.afterReqRead) + if err != nil { + t.Errorf("test[%d]: %v", i, err) + continue + } + if err := tt.want(string(got)); err != nil { + t.Errorf("test[%d]: %v", i, err) + } + } +} + type closeChecker struct { io.Reader closed bool @@ -581,12 +716,14 @@ func (rc *closeChecker) Close() error { // inside a NopCloser, and that it serializes it correctly. func TestRequestWriteClosesBody(t *testing.T) { rc := &closeChecker{Reader: strings.NewReader("my body")} - req, _ := NewRequest("POST", "http://foo.com/", rc) - if req.ContentLength != -1 { - t.Errorf("got req.ContentLength %d, want -1", req.ContentLength) + req, err := NewRequest("POST", "http://foo.com/", rc) + if err != nil { + t.Fatal(err) } buf := new(bytes.Buffer) - req.Write(buf) + if err := req.Write(buf); err != nil { + t.Error(err) + } if !rc.closed { t.Error("body not closed after write") } @@ -670,3 +807,76 @@ func TestRequestWriteError(t *testing.T) { t.Fatalf("writeCalls constant is outdated in test") } } + +// dumpRequestOut is a modified copy of net/http/httputil.DumpRequestOut. +// Unlike the original, this version doesn't mutate the req.Body and +// try to restore it. It always dumps the whole body. +// And it doesn't support https. +func dumpRequestOut(req *Request, onReadHeaders func()) ([]byte, error) { + + // Use the actual Transport code to record what we would send + // on the wire, but not using TCP. Use a Transport with a + // custom dialer that returns a fake net.Conn that waits + // for the full input (and recording it), and then responds + // with a dummy response. + var buf bytes.Buffer // records the output + pr, pw := io.Pipe() + defer pr.Close() + defer pw.Close() + dr := &delegateReader{c: make(chan io.Reader)} + + t := &Transport{ + Dial: func(net, addr string) (net.Conn, error) { + return &dumpConn{io.MultiWriter(&buf, pw), dr}, nil + }, + } + defer t.CloseIdleConnections() + + // Wait for the request before replying with a dummy response: + go func() { + req, err := ReadRequest(bufio.NewReader(pr)) + if err == nil { + if onReadHeaders != nil { + onReadHeaders() + } + // Ensure all the body is read; otherwise + // we'll get a partial dump. + io.Copy(ioutil.Discard, req.Body) + req.Body.Close() + } + dr.c <- strings.NewReader("HTTP/1.1 204 No Content\r\nConnection: close\r\n\r\n") + }() + + _, err := t.RoundTrip(req) + if err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +// delegateReader is a reader that delegates to another reader, +// once it arrives on a channel. +type delegateReader struct { + c chan io.Reader + r io.Reader // nil until received from c +} + +func (r *delegateReader) Read(p []byte) (int, error) { + if r.r == nil { + r.r = <-r.c + } + return r.r.Read(p) +} + +// dumpConn is a net.Conn that writes to Writer and reads from Reader. +type dumpConn struct { + io.Writer + io.Reader +} + +func (c *dumpConn) Close() error { return nil } +func (c *dumpConn) LocalAddr() net.Addr { return nil } +func (c *dumpConn) RemoteAddr() net.Addr { return nil } +func (c *dumpConn) SetDeadline(t time.Time) error { return nil } +func (c *dumpConn) SetReadDeadline(t time.Time) error { return nil } +func (c *dumpConn) SetWriteDeadline(t time.Time) error { return nil } diff --git a/src/net/http/serve_test.go b/src/net/http/serve_test.go index 593b1f3cdd..072da2552b 100644 --- a/src/net/http/serve_test.go +++ b/src/net/http/serve_test.go @@ -481,11 +481,11 @@ func TestServerTimeouts(t *testing.T) { if err != nil { t.Fatalf("http Get #1: %v", err) } - got, _ := ioutil.ReadAll(r.Body) + got, err := ioutil.ReadAll(r.Body) expected := "req=1" - if string(got) != expected { - t.Errorf("Unexpected response for request #1; got %q; expected %q", - string(got), expected) + if string(got) != expected || err != nil { + t.Errorf("Unexpected response for request #1; got %q ,%v; expected %q, nil", + string(got), err, expected) } // Slow client that should timeout. @@ -496,6 +496,7 @@ func TestServerTimeouts(t *testing.T) { } buf := make([]byte, 1) n, err := conn.Read(buf) + conn.Close() latency := time.Since(t1) if n != 0 || err != io.EOF { t.Errorf("Read = %v, %v, wanted %v, %v", n, err, 0, io.EOF) @@ -507,14 +508,14 @@ func TestServerTimeouts(t *testing.T) { // Hit the HTTP server successfully again, verifying that the // previous slow connection didn't run our handler. (that we // get "req=2", not "req=3") - r, err = Get(ts.URL) + r, err = c.Get(ts.URL) if err != nil { t.Fatalf("http Get #2: %v", err) } - got, _ = ioutil.ReadAll(r.Body) + got, err = ioutil.ReadAll(r.Body) expected = "req=2" - if string(got) != expected { - t.Errorf("Get #2 got %q, want %q", string(got), expected) + if string(got) != expected || err != nil { + t.Errorf("Get #2 got %q, %v, want %q, nil", string(got), err, expected) } if !testing.Short() { @@ -534,6 +535,56 @@ func TestServerTimeouts(t *testing.T) { } } +// Test that the HTTP/2 server handles Server.WriteTimeout (Issue 18437) +func TestHTTP2WriteDeadlineExtendedOnNewRequest(t *testing.T) { + if testing.Short() { + t.Skip("skipping in short mode") + } + setParallel(t) + defer afterTest(t) + ts := httptest.NewUnstartedServer(HandlerFunc(func(res ResponseWriter, req *Request) {})) + ts.Config.WriteTimeout = 250 * time.Millisecond + ts.TLS = &tls.Config{NextProtos: []string{"h2"}} + ts.StartTLS() + defer ts.Close() + + tr := newTLSTransport(t, ts) + defer tr.CloseIdleConnections() + if err := ExportHttp2ConfigureTransport(tr); err != nil { + t.Fatal(err) + } + c := &Client{Transport: tr} + + for i := 1; i <= 3; i++ { + req, err := NewRequest("GET", ts.URL, nil) + if err != nil { + t.Fatal(err) + } + + // fail test if no response after 1 second + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) + defer cancel() + req = req.WithContext(ctx) + + r, err := c.Do(req) + select { + case <-ctx.Done(): + if ctx.Err() == context.DeadlineExceeded { + t.Fatalf("http2 Get #%d response timed out", i) + } + default: + } + if err != nil { + t.Fatalf("http2 Get #%d: %v", i, err) + } + r.Body.Close() + if r.ProtoMajor != 2 { + t.Fatalf("http2 Get expected HTTP/2.0, got %q", r.Proto) + } + time.Sleep(ts.Config.WriteTimeout / 2) + } +} + // golang.org/issue/4741 -- setting only a write timeout that triggers // shouldn't cause a handler to block forever on reads (next HTTP // request) that will never happen. @@ -5038,3 +5089,87 @@ func testServerKeepAlivesEnabled(t *testing.T, h2 bool) { t.Fatalf("test server has active conns") } } + +// Issue 18447: test that the Server's ReadTimeout is stopped while +// the server's doing its 1-byte background read between requests, +// waiting for the connection to maybe close. +func TestServerCancelsReadTimeoutWhenIdle(t *testing.T) { + setParallel(t) + defer afterTest(t) + const timeout = 250 * time.Millisecond + ts := httptest.NewUnstartedServer(HandlerFunc(func(w ResponseWriter, r *Request) { + select { + case <-time.After(2 * timeout): + fmt.Fprint(w, "ok") + case <-r.Context().Done(): + fmt.Fprint(w, r.Context().Err()) + } + })) + ts.Config.ReadTimeout = timeout + ts.Start() + defer ts.Close() + + tr := &Transport{} + defer tr.CloseIdleConnections() + c := &Client{Transport: tr} + + res, err := c.Get(ts.URL) + if err != nil { + t.Fatal(err) + } + slurp, err := ioutil.ReadAll(res.Body) + res.Body.Close() + if err != nil { + t.Fatal(err) + } + if string(slurp) != "ok" { + t.Fatalf("Got: %q, want ok", slurp) + } +} + +// Issue 18535: test that the Server doesn't try to do a background +// read if it's already done one. +func TestServerDuplicateBackgroundRead(t *testing.T) { + setParallel(t) + defer afterTest(t) + + const goroutines = 5 + const requests = 2000 + + hts := httptest.NewServer(HandlerFunc(NotFound)) + defer hts.Close() + + reqBytes := []byte("GET / HTTP/1.1\r\nHost: e.com\r\n\r\n") + + var wg sync.WaitGroup + for i := 0; i < goroutines; i++ { + wg.Add(1) + go func() { + defer wg.Done() + cn, err := net.Dial("tcp", hts.Listener.Addr().String()) + if err != nil { + t.Error(err) + return + } + defer cn.Close() + + wg.Add(1) + go func() { + defer wg.Done() + io.Copy(ioutil.Discard, cn) + }() + + for j := 0; j < requests; j++ { + if t.Failed() { + return + } + _, err := cn.Write(reqBytes) + if err != nil { + t.Error(err) + return + } + } + }() + } + wg.Wait() +} diff --git a/src/net/http/server.go b/src/net/http/server.go index 6df9c260e4..96236489bd 100644 --- a/src/net/http/server.go +++ b/src/net/http/server.go @@ -636,7 +636,11 @@ func (cr *connReader) startBackgroundRead() { if cr.inRead { panic("invalid concurrent Body.Read call") } + if cr.hasByte { + return + } cr.inRead = true + cr.conn.rwc.SetReadDeadline(time.Time{}) go cr.backgroundRead() } diff --git a/src/net/http/transfer.go b/src/net/http/transfer.go index beafb7ac97..4f47637aa7 100644 --- a/src/net/http/transfer.go +++ b/src/net/http/transfer.go @@ -17,6 +17,7 @@ import ( "strconv" "strings" "sync" + "time" "golang_org/x/net/lex/httplex" ) @@ -33,6 +34,23 @@ func (r errorReader) Read(p []byte) (n int, err error) { return 0, r.err } +type byteReader struct { + b byte + done bool +} + +func (br *byteReader) Read(p []byte) (n int, err error) { + if br.done { + return 0, io.EOF + } + if len(p) == 0 { + return 0, nil + } + br.done = true + p[0] = br.b + return 1, io.EOF +} + // transferWriter inspects the fields of a user-supplied Request or Response, // sanitizes them without changing the user object and provides methods for // writing the respective header, body and trailer in wire format. @@ -46,6 +64,9 @@ type transferWriter struct { TransferEncoding []string Trailer Header IsResponse bool + + FlushHeaders bool // flush headers to network before body + ByteReadCh chan readResult // non-nil if probeRequestBody called } func newTransferWriter(r interface{}) (t *transferWriter, err error) { @@ -62,14 +83,11 @@ func newTransferWriter(r interface{}) (t *transferWriter, err error) { t.Close = rr.Close t.TransferEncoding = rr.TransferEncoding t.Trailer = rr.Trailer - atLeastHTTP11 = rr.ProtoAtLeast(1, 1) - + atLeastHTTP11 = rr.protoAtLeastOutgoing(1, 1) t.Body = rr.Body + t.BodyCloser = rr.Body t.ContentLength = rr.outgoingLength() - if t.Body != nil { - t.BodyCloser = rr.Body - } - if t.ContentLength < 0 && len(t.TransferEncoding) == 0 && atLeastHTTP11 { + if t.ContentLength < 0 && len(t.TransferEncoding) == 0 && atLeastHTTP11 && t.shouldSendChunkedRequestBody() { t.TransferEncoding = []string{"chunked"} } case *Response: @@ -84,7 +102,7 @@ func newTransferWriter(r interface{}) (t *transferWriter, err error) { t.TransferEncoding = rr.TransferEncoding t.Trailer = rr.Trailer atLeastHTTP11 = rr.ProtoAtLeast(1, 1) - t.ResponseToHEAD = noBodyExpected(t.Method) + t.ResponseToHEAD = noResponseBodyExpected(t.Method) } // Sanitize Body,ContentLength,TransferEncoding @@ -112,7 +130,100 @@ func newTransferWriter(r interface{}) (t *transferWriter, err error) { return t, nil } -func noBodyExpected(requestMethod string) bool { +// shouldSendChunkedRequestBody reports whether we should try to send a +// chunked request body to the server. In particular, the case we really +// want to prevent is sending a GET or other typically-bodyless request to a +// server with a chunked body when the body has zero bytes, since GETs with +// bodies (while acceptable according to specs), even zero-byte chunked +// bodies, are approximately never seen in the wild and confuse most +// servers. See Issue 18257, as one example. +// +// The only reason we'd send such a request is if the user set the Body to a +// non-nil value (say, ioutil.NopCloser(bytes.NewReader(nil))) and didn't +// set ContentLength, or NewRequest set it to -1 (unknown), so then we assume +// there's bytes to send. +// +// This code tries to read a byte from the Request.Body in such cases to see +// whether the body actually has content (super rare) or is actually just +// a non-nil content-less ReadCloser (the more common case). In that more +// common case, we act as if their Body were nil instead, and don't send +// a body. +func (t *transferWriter) shouldSendChunkedRequestBody() bool { + // Note that t.ContentLength is the corrected content length + // from rr.outgoingLength, so 0 actually means zero, not unknown. + if t.ContentLength >= 0 || t.Body == nil { // redundant checks; caller did them + return false + } + if requestMethodUsuallyLacksBody(t.Method) { + // Only probe the Request.Body for GET/HEAD/DELETE/etc + // requests, because it's only those types of requests + // that confuse servers. + t.probeRequestBody() // adjusts t.Body, t.ContentLength + return t.Body != nil + } + // For all other request types (PUT, POST, PATCH, or anything + // made-up we've never heard of), assume it's normal and the server + // can deal with a chunked request body. Maybe we'll adjust this + // later. + return true +} + +// probeRequestBody reads a byte from t.Body to see whether it's empty +// (returns io.EOF right away). +// +// But because we've had problems with this blocking users in the past +// (issue 17480) when the body is a pipe (perhaps waiting on the response +// headers before the pipe is fed data), we need to be careful and bound how +// long we wait for it. This delay will only affect users if all the following +// are true: +// * the request body blocks +// * the content length is not set (or set to -1) +// * the method doesn't usually have a body (GET, HEAD, DELETE, ...) +// * there is no transfer-encoding=chunked already set. +// In other words, this delay will not normally affect anybody, and there +// are workarounds if it does. +func (t *transferWriter) probeRequestBody() { + t.ByteReadCh = make(chan readResult, 1) + go func(body io.Reader) { + var buf [1]byte + var rres readResult + rres.n, rres.err = body.Read(buf[:]) + if rres.n == 1 { + rres.b = buf[0] + } + t.ByteReadCh <- rres + }(t.Body) + timer := time.NewTimer(200 * time.Millisecond) + select { + case rres := <-t.ByteReadCh: + timer.Stop() + if rres.n == 0 && rres.err == io.EOF { + // It was empty. + t.Body = nil + t.ContentLength = 0 + } else if rres.n == 1 { + if rres.err != nil { + t.Body = io.MultiReader(&byteReader{b: rres.b}, errorReader{rres.err}) + } else { + t.Body = io.MultiReader(&byteReader{b: rres.b}, t.Body) + } + } else if rres.err != nil { + t.Body = errorReader{rres.err} + } + case <-timer.C: + // Too slow. Don't wait. Read it later, and keep + // assuming that this is ContentLength == -1 + // (unknown), which means we'll send a + // "Transfer-Encoding: chunked" header. + t.Body = io.MultiReader(finishAsyncByteRead{t}, t.Body) + // Request that Request.Write flush the headers to the + // network before writing the body, since our body may not + // become readable until it's seen the response headers. + t.FlushHeaders = true + } +} + +func noResponseBodyExpected(requestMethod string) bool { return requestMethod == "HEAD" } @@ -216,7 +327,9 @@ func (t *transferWriter) WriteBody(w io.Writer) error { if err != nil { return err } - if err = t.BodyCloser.Close(); err != nil { + } + if t.BodyCloser != nil { + if err := t.BodyCloser.Close(); err != nil { return err } } @@ -366,7 +479,7 @@ func readTransfer(msg interface{}, r *bufio.Reader) (err error) { // or close connection when finished, since multipart is not supported yet switch { case chunked(t.TransferEncoding): - if noBodyExpected(t.RequestMethod) { + if noResponseBodyExpected(t.RequestMethod) { t.Body = NoBody } else { t.Body = &body{src: internal.NewChunkedReader(r), hdr: msg, r: r, closing: t.Close} @@ -498,7 +611,7 @@ func fixLength(isResponse bool, status int, requestMethod string, header Header, } // Logic based on response type or status - if noBodyExpected(requestMethod) { + if noResponseBodyExpected(requestMethod) { // For HTTP requests, as part of hardening against request // smuggling (RFC 7230), don't allow a Content-Length header for // methods which don't permit bodies. As an exception, allow @@ -861,3 +974,21 @@ func parseContentLength(cl string) (int64, error) { return n, nil } + +// finishAsyncByteRead finishes reading the 1-byte sniff +// from the ContentLength==0, Body!=nil case. +type finishAsyncByteRead struct { + tw *transferWriter +} + +func (fr finishAsyncByteRead) Read(p []byte) (n int, err error) { + if len(p) == 0 { + return + } + rres := <-fr.tw.ByteReadCh + n, err = rres.n, rres.err + if n == 1 { + p[0] = rres.b + } + return +} diff --git a/src/net/http/transport.go b/src/net/http/transport.go index e484548773..571943d6e5 100644 --- a/src/net/http/transport.go +++ b/src/net/http/transport.go @@ -923,6 +923,9 @@ func (t *Transport) getConn(treq *transportRequest, cm connectMethod) (*persistC // value. select { case <-req.Cancel: + // It was an error due to cancelation, so prioritize that + // error value. (Issue 16049) + return nil, errRequestCanceledConn case <-req.Context().Done(): return nil, req.Context().Err() case err := <-cancelc: @@ -935,9 +938,6 @@ func (t *Transport) getConn(treq *transportRequest, cm connectMethod) (*persistC // return the original error message: return nil, v.err } - // It was an error due to cancelation, so prioritize that - // error value. (Issue 16049) - return nil, errRequestCanceledConn case pc := <-idleConnCh: // Another request finished first and its net.Conn // became available before our dial. Or somebody @@ -1384,7 +1384,7 @@ func (pc *persistConn) closeConnIfStillIdle() { // // The startBytesWritten value should be the value of pc.nwrite before the roundTrip // started writing the request. -func (pc *persistConn) mapRoundTripErrorFromReadLoop(startBytesWritten int64, err error) (out error) { +func (pc *persistConn) mapRoundTripErrorFromReadLoop(req *Request, startBytesWritten int64, err error) (out error) { if err == nil { return nil } @@ -1399,7 +1399,7 @@ func (pc *persistConn) mapRoundTripErrorFromReadLoop(startBytesWritten int64, er } if pc.isBroken() { <-pc.writeLoopDone - if pc.nwrite == startBytesWritten { + if pc.nwrite == startBytesWritten && req.outgoingLength() == 0 { return nothingWrittenError{err} } } @@ -1410,7 +1410,7 @@ func (pc *persistConn) mapRoundTripErrorFromReadLoop(startBytesWritten int64, er // up to Transport.RoundTrip method when persistConn.roundTrip sees // its pc.closech channel close, indicating the persistConn is dead. // (after closech is closed, pc.closed is valid). -func (pc *persistConn) mapRoundTripErrorAfterClosed(startBytesWritten int64) error { +func (pc *persistConn) mapRoundTripErrorAfterClosed(req *Request, startBytesWritten int64) error { if err := pc.canceled(); err != nil { return err } @@ -1428,7 +1428,7 @@ func (pc *persistConn) mapRoundTripErrorAfterClosed(startBytesWritten int64) err // see if we actually managed to write anything. If not, we // can retry the request. <-pc.writeLoopDone - if pc.nwrite == startBytesWritten { + if pc.nwrite == startBytesWritten && req.outgoingLength() == 0 { return nothingWrittenError{err} } @@ -1710,7 +1710,7 @@ func (pc *persistConn) writeLoop() { } if err != nil { wr.req.Request.closeBody() - if pc.nwrite == startBytesWritten { + if pc.nwrite == startBytesWritten && wr.req.outgoingLength() == 0 { err = nothingWrittenError{err} } } @@ -1911,14 +1911,14 @@ WaitResponse: respHeaderTimer = timer.C } case <-pc.closech: - re = responseAndError{err: pc.mapRoundTripErrorAfterClosed(startBytesWritten)} + re = responseAndError{err: pc.mapRoundTripErrorAfterClosed(req.Request, startBytesWritten)} break WaitResponse case <-respHeaderTimer: pc.close(errTimeout) re = responseAndError{err: errTimeout} break WaitResponse case re = <-resc: - re.err = pc.mapRoundTripErrorFromReadLoop(startBytesWritten, re.err) + re.err = pc.mapRoundTripErrorFromReadLoop(req.Request, startBytesWritten, re.err) break WaitResponse case <-cancelChan: pc.t.CancelRequest(req.Request) diff --git a/src/net/http/transport_test.go b/src/net/http/transport_test.go index 5a402657cc..d5ddf6a123 100644 --- a/src/net/http/transport_test.go +++ b/src/net/http/transport_test.go @@ -1083,8 +1083,10 @@ func waitNumGoroutine(nmax int) int { func TestTransportPersistConnLeak(t *testing.T) { // Not parallel: counts goroutines defer afterTest(t) - gotReqCh := make(chan bool) - unblockCh := make(chan bool) + + const numReq = 25 + gotReqCh := make(chan bool, numReq) + unblockCh := make(chan bool, numReq) ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { gotReqCh <- true <-unblockCh @@ -1098,14 +1100,15 @@ func TestTransportPersistConnLeak(t *testing.T) { n0 := runtime.NumGoroutine() - const numReq = 25 - didReqCh := make(chan bool) + didReqCh := make(chan bool, numReq) + failed := make(chan bool, numReq) for i := 0; i < numReq; i++ { go func() { res, err := c.Get(ts.URL) didReqCh <- true if err != nil { t.Errorf("client fetch error: %v", err) + failed <- true return } res.Body.Close() @@ -1114,7 +1117,13 @@ func TestTransportPersistConnLeak(t *testing.T) { // Wait for all goroutines to be stuck in the Handler. for i := 0; i < numReq; i++ { - <-gotReqCh + select { + case <-gotReqCh: + // ok + case <-failed: + close(unblockCh) + return + } } nhigh := runtime.NumGoroutine() diff --git a/src/net/interface.go b/src/net/interface.go index 301a5cfd22..b3297f249d 100644 --- a/src/net/interface.go +++ b/src/net/interface.go @@ -172,6 +172,9 @@ func InterfaceByName(name string) (*Interface, error) { // An ipv6ZoneCache represents a cache holding partial network // interface information. It is used for reducing the cost of IPv6 // addressing scope zone resolution. +// +// Multiple names sharing the index are managed by first-come +// first-served basis for consistency. type ipv6ZoneCache struct { sync.RWMutex // guard the following lastFetched time.Time // last time routing information was fetched @@ -202,7 +205,9 @@ func (zc *ipv6ZoneCache) update(ift []Interface) { zc.toName = make(map[int]string, len(ift)) for _, ifi := range ift { zc.toIndex[ifi.Name] = ifi.Index - zc.toName[ifi.Index] = ifi.Name + if _, ok := zc.toName[ifi.Index]; !ok { + zc.toName[ifi.Index] = ifi.Name + } } } diff --git a/src/net/ip.go b/src/net/ip.go index c5b454d3bd..db3364c1b3 100644 --- a/src/net/ip.go +++ b/src/net/ip.go @@ -90,7 +90,7 @@ func CIDRMask(ones, bits int) IPMask { // Well-known IPv4 addresses var ( - IPv4bcast = IPv4(255, 255, 255, 255) // broadcast + IPv4bcast = IPv4(255, 255, 255, 255) // limited broadcast IPv4allsys = IPv4(224, 0, 0, 1) // all systems IPv4allrouter = IPv4(224, 0, 0, 2) // all routers IPv4zero = IPv4(0, 0, 0, 0) // all zeros @@ -153,6 +153,12 @@ func (ip IP) IsLinkLocalUnicast() bool { // IsGlobalUnicast reports whether ip is a global unicast // address. +// +// The identification of global unicast addresses uses address type +// identification as defined in RFC 1122, RFC 4632 and RFC 4291 with +// the exception of IPv4 directed broadcast addresses. +// It returns true even if ip is in IPv4 private address space or +// local IPv6 unicast address space. func (ip IP) IsGlobalUnicast() bool { return (len(ip) == IPv4len || len(ip) == IPv6len) && !ip.Equal(IPv4bcast) && @@ -654,13 +660,14 @@ func ParseIP(s string) IP { return nil } -// ParseCIDR parses s as a CIDR notation IP address and mask, +// ParseCIDR parses s as a CIDR notation IP address and prefix length, // like "192.0.2.0/24" or "2001:db8::/32", as defined in // RFC 4632 and RFC 4291. // -// It returns the IP address and the network implied by the IP -// and mask. For example, ParseCIDR("198.51.100.1/24") returns -// the IP address 198.51.100.1 and the network 198.51.100.0/24. +// It returns the IP address and the network implied by the IP and +// prefix length. +// For example, ParseCIDR("192.0.2.1/24") returns the IP address +// 198.0.2.1 and the network 198.0.2.0/24. func ParseCIDR(s string) (IP, *IPNet, error) { i := byteIndex(s, '/') if i < 0 { diff --git a/src/net/iprawsock.go b/src/net/iprawsock.go index b3cc03e00d..d994fc67c6 100644 --- a/src/net/iprawsock.go +++ b/src/net/iprawsock.go @@ -9,6 +9,18 @@ import ( "syscall" ) +// BUG(mikio): On every POSIX platform, reads from the "ip4" network +// using the ReadFrom or ReadFromIP method might not return a complete +// IPv4 packet, including its header, even if there is space +// available. This can occur even in cases where Read or ReadMsgIP +// could return a complete packet. For this reason, it is recommended +// that you do not use these methods if it is important to receive a +// full packet. +// +// The Go 1 compatibility guidelines make it impossible for us to +// change the behavior of these methods; use Read or ReadMsgIP +// instead. + // BUG(mikio): On NaCl, Plan 9 and Windows, the ReadMsgIP and // WriteMsgIP methods of IPConn are not implemented. diff --git a/src/net/iprawsock_posix.go b/src/net/iprawsock_posix.go index d5e229fb9c..8f4b702e48 100644 --- a/src/net/iprawsock_posix.go +++ b/src/net/iprawsock_posix.go @@ -11,18 +11,6 @@ import ( "syscall" ) -// BUG(mikio): On every POSIX platform, reads from the "ip4" network -// using the ReadFrom or ReadFromIP method might not return a complete -// IPv4 packet, including its header, even if there is space -// available. This can occur even in cases where Read or ReadMsgIP -// could return a complete packet. For this reason, it is recommended -// that you do not uses these methods if it is important to receive a -// full packet. -// -// The Go 1 compatibility guidelines make it impossible for us to -// change the behavior of these methods; use Read or ReadMsgIP -// instead. - func sockaddrToIP(sa syscall.Sockaddr) Addr { switch sa := sa.(type) { case *syscall.SockaddrInet4: diff --git a/src/net/iprawsock_test.go b/src/net/iprawsock_test.go index 29cd4b6fd0..5d33b26a91 100644 --- a/src/net/iprawsock_test.go +++ b/src/net/iprawsock_test.go @@ -43,6 +43,13 @@ var resolveIPAddrTests = []resolveIPAddrTest{ {"l2tp", "127.0.0.1", nil, UnknownNetworkError("l2tp")}, {"l2tp:gre", "127.0.0.1", nil, UnknownNetworkError("l2tp:gre")}, {"tcp", "1.2.3.4:123", nil, UnknownNetworkError("tcp")}, + + {"ip4", "2001:db8::1", nil, &AddrError{Err: errNoSuitableAddress.Error(), Addr: "2001:db8::1"}}, + {"ip4:icmp", "2001:db8::1", nil, &AddrError{Err: errNoSuitableAddress.Error(), Addr: "2001:db8::1"}}, + {"ip6", "127.0.0.1", nil, &AddrError{Err: errNoSuitableAddress.Error(), Addr: "127.0.0.1"}}, + {"ip6", "::ffff:127.0.0.1", nil, &AddrError{Err: errNoSuitableAddress.Error(), Addr: "::ffff:127.0.0.1"}}, + {"ip6:ipv6-icmp", "127.0.0.1", nil, &AddrError{Err: errNoSuitableAddress.Error(), Addr: "127.0.0.1"}}, + {"ip6:ipv6-icmp", "::ffff:127.0.0.1", nil, &AddrError{Err: errNoSuitableAddress.Error(), Addr: "::ffff:127.0.0.1"}}, } func TestResolveIPAddr(t *testing.T) { @@ -54,21 +61,17 @@ func TestResolveIPAddr(t *testing.T) { defer func() { testHookLookupIP = origTestHookLookupIP }() testHookLookupIP = lookupLocalhost - for i, tt := range resolveIPAddrTests { + for _, tt := range resolveIPAddrTests { addr, err := ResolveIPAddr(tt.network, tt.litAddrOrName) - if err != tt.err { - t.Errorf("#%d: %v", i, err) - } else if !reflect.DeepEqual(addr, tt.addr) { - t.Errorf("#%d: got %#v; want %#v", i, addr, tt.addr) - } - if err != nil { + if !reflect.DeepEqual(addr, tt.addr) || !reflect.DeepEqual(err, tt.err) { + t.Errorf("ResolveIPAddr(%q, %q) = %#v, %v, want %#v, %v", tt.network, tt.litAddrOrName, addr, err, tt.addr, tt.err) continue } - rtaddr, err := ResolveIPAddr(addr.Network(), addr.String()) - if err != nil { - t.Errorf("#%d: %v", i, err) - } else if !reflect.DeepEqual(rtaddr, addr) { - t.Errorf("#%d: got %#v; want %#v", i, rtaddr, addr) + if err == nil { + addr2, err := ResolveIPAddr(addr.Network(), addr.String()) + if !reflect.DeepEqual(addr2, tt.addr) || err != tt.err { + t.Errorf("(%q, %q): ResolveIPAddr(%q, %q) = %#v, %v, want %#v, %v", tt.network, tt.litAddrOrName, addr.Network(), addr.String(), addr2, err, tt.addr, tt.err) + } } } } diff --git a/src/net/ipsock.go b/src/net/ipsock.go index c91e2017d4..f1394a7ed8 100644 --- a/src/net/ipsock.go +++ b/src/net/ipsock.go @@ -10,6 +10,13 @@ import ( "context" ) +// BUG(rsc,mikio): On DragonFly BSD and OpenBSD, listening on the +// "tcp" and "udp" networks does not listen for both IPv4 and IPv6 +// connections. This is due to the fact that IPv4 traffic will not be +// routed to an IPv6 socket - two separate sockets are required if +// both address families are to be supported. +// See inet6(4) for details. + var ( // supportsIPv4 reports whether the platform supports IPv4 // networking functionality. diff --git a/src/net/ipsock_posix.go b/src/net/ipsock_posix.go index f4fab3f9aa..ff280c3e4e 100644 --- a/src/net/ipsock_posix.go +++ b/src/net/ipsock_posix.go @@ -12,13 +12,6 @@ import ( "syscall" ) -// BUG(rsc,mikio): On DragonFly BSD and OpenBSD, listening on the -// "tcp" and "udp" networks does not listen for both IPv4 and IPv6 -// connections. This is due to the fact that IPv4 traffic will not be -// routed to an IPv6 socket - two separate sockets are required if -// both address families are to be supported. -// See inet6(4) for details. - func probeIPv4Stack() bool { s, err := socketFunc(syscall.AF_INET, syscall.SOCK_STREAM, syscall.IPPROTO_TCP) switch err { diff --git a/src/net/lookup.go b/src/net/lookup.go index 8b5cab0894..cc2013e432 100644 --- a/src/net/lookup.go +++ b/src/net/lookup.go @@ -233,20 +233,32 @@ func (r *Resolver) LookupPort(ctx context.Context, network, service string) (por return port, nil } -// LookupCNAME returns the canonical DNS host for the given name. +// LookupCNAME returns the canonical name for the given host. // Callers that do not care about the canonical name can call // LookupHost or LookupIP directly; both take care of resolving // the canonical name as part of the lookup. -func LookupCNAME(name string) (cname string, err error) { - return DefaultResolver.lookupCNAME(context.Background(), name) +// +// A canonical name is the final name after following zero +// or more CNAME records. +// LookupCNAME does not return an error if host does not +// contain DNS "CNAME" records, as long as host resolves to +// address records. +func LookupCNAME(host string) (cname string, err error) { + return DefaultResolver.lookupCNAME(context.Background(), host) } -// LookupCNAME returns the canonical DNS host for the given name. +// LookupCNAME returns the canonical name for the given host. // Callers that do not care about the canonical name can call // LookupHost or LookupIP directly; both take care of resolving // the canonical name as part of the lookup. -func (r *Resolver) LookupCNAME(ctx context.Context, name string) (cname string, err error) { - return r.lookupCNAME(ctx, name) +// +// A canonical name is the final name after following zero +// or more CNAME records. +// LookupCNAME does not return an error if host does not +// contain DNS "CNAME" records, as long as host resolves to +// address records. +func (r *Resolver) LookupCNAME(ctx context.Context, host string) (cname string, err error) { + return r.lookupCNAME(ctx, host) } // LookupSRV tries to resolve an SRV query of the given service, diff --git a/src/net/lookup_plan9.go b/src/net/lookup_plan9.go index 11f2349afe..f81e220fc8 100644 --- a/src/net/lookup_plan9.go +++ b/src/net/lookup_plan9.go @@ -198,6 +198,10 @@ func (*Resolver) lookupPort(ctx context.Context, network, service string) (port func (*Resolver) lookupCNAME(ctx context.Context, name string) (cname string, err error) { lines, err := queryDNS(ctx, name, "cname") if err != nil { + if stringsHasSuffix(err.Error(), "dns failure") { + cname = name + "." + err = nil + } return } if len(lines) > 0 { diff --git a/src/net/lookup_test.go b/src/net/lookup_test.go index 656bebb9b8..36db56acd0 100644 --- a/src/net/lookup_test.go +++ b/src/net/lookup_test.go @@ -243,14 +243,15 @@ func TestLookupIPv6LinkLocalAddr(t *testing.T) { } } -var lookupIANACNAMETests = []struct { +var lookupCNAMETests = []struct { name, cname string }{ {"www.iana.org", "icann.org."}, {"www.iana.org.", "icann.org."}, + {"www.google.com", "google.com."}, } -func TestLookupIANACNAME(t *testing.T) { +func TestLookupCNAME(t *testing.T) { if testenv.Builder() == "" { testenv.MustHaveExternalNetwork(t) } @@ -259,7 +260,7 @@ func TestLookupIANACNAME(t *testing.T) { t.Skip("IPv4 is required") } - for _, tt := range lookupIANACNAMETests { + for _, tt := range lookupCNAMETests { cname, err := LookupCNAME(tt.name) if err != nil { t.Fatal(err) diff --git a/src/net/lookup_unix.go b/src/net/lookup_unix.go index 35f253c1da..be2ced9c39 100644 --- a/src/net/lookup_unix.go +++ b/src/net/lookup_unix.go @@ -72,17 +72,20 @@ func (r *Resolver) lookupIP(ctx context.Context, host string) (addrs []IPAddr, e // cgo not available (or netgo); fall back to Go's DNS resolver order = hostLookupFilesDNS } - return goLookupIPOrder(ctx, host, order) + addrs, _, err = goLookupIPCNAMEOrder(ctx, host, order) + return } func (r *Resolver) lookupPort(ctx context.Context, network, service string) (int, error) { - // TODO: use the context if there ever becomes a need. Related - // is issue 15321. But port lookup generally just involves - // local files, and the os package has no context support. The - // files might be on a remote filesystem, though. This should - // probably race goroutines if ctx != context.Background(). if !r.PreferGo && systemConf().canUseCgo() { if port, err, ok := cgoLookupPort(ctx, network, service); ok { + if err != nil { + // Issue 18213: if cgo fails, first check to see whether we + // have the answer baked-in to the net package. + if port, err := goLookupPort(network, service); err == nil { + return port, nil + } + } return port, err } } diff --git a/src/net/lookup_windows_test.go b/src/net/lookup_windows_test.go index bc9ffe15a4..cebb2d0558 100644 --- a/src/net/lookup_windows_test.go +++ b/src/net/lookup_windows_test.go @@ -24,7 +24,7 @@ func toJson(v interface{}) string { return string(data) } -func TestLookupMX(t *testing.T) { +func TestNSLookupMX(t *testing.T) { testenv.MustHaveExternalNetwork(t) for _, server := range nslookupTestServers { @@ -49,7 +49,7 @@ func TestLookupMX(t *testing.T) { } } -func TestLookupCNAME(t *testing.T) { +func TestNSLookupCNAME(t *testing.T) { testenv.MustHaveExternalNetwork(t) for _, server := range nslookupTestServers { @@ -72,7 +72,7 @@ func TestLookupCNAME(t *testing.T) { } } -func TestLookupNS(t *testing.T) { +func TestNSLookupNS(t *testing.T) { testenv.MustHaveExternalNetwork(t) for _, server := range nslookupTestServers { @@ -98,7 +98,7 @@ func TestLookupNS(t *testing.T) { } } -func TestLookupTXT(t *testing.T) { +func TestNSLookupTXT(t *testing.T) { testenv.MustHaveExternalNetwork(t) for _, server := range nslookupTestServers { diff --git a/src/net/port_unix.go b/src/net/port_unix.go index 4e0478194e..868d1e4784 100644 --- a/src/net/port_unix.go +++ b/src/net/port_unix.go @@ -10,12 +10,11 @@ package net import "sync" -var servicesError error var onceReadServices sync.Once func readServices() { - var file *file - if file, servicesError = open("/etc/services"); servicesError != nil { + file, err := open("/etc/services") + if err != nil { return } for line, ok := file.readLine(); ok; line, ok = file.readLine() { diff --git a/src/net/tcpsock_test.go b/src/net/tcpsock_test.go index 573e834911..54bf0cfccc 100644 --- a/src/net/tcpsock_test.go +++ b/src/net/tcpsock_test.go @@ -317,10 +317,11 @@ var resolveTCPAddrTests = []resolveTCPAddrTest{ {"tcp", "[2001:db8::1]:http", &TCPAddr{IP: ParseIP("2001:db8::1"), Port: 80}, nil}, {"tcp4", "127.0.0.1:http", &TCPAddr{IP: ParseIP("127.0.0.1"), Port: 80}, nil}, {"tcp4", "[::ffff:127.0.0.1]:http", &TCPAddr{IP: ParseIP("127.0.0.1"), Port: 80}, nil}, + {"tcp6", "[2001:db8::1]:http", &TCPAddr{IP: ParseIP("2001:db8::1"), Port: 80}, nil}, + {"tcp4", "[2001:db8::1]:http", nil, &AddrError{Err: errNoSuitableAddress.Error(), Addr: "2001:db8::1"}}, {"tcp6", "127.0.0.1:http", nil, &AddrError{Err: errNoSuitableAddress.Error(), Addr: "127.0.0.1"}}, {"tcp6", "[::ffff:127.0.0.1]:http", nil, &AddrError{Err: errNoSuitableAddress.Error(), Addr: "::ffff:127.0.0.1"}}, - {"tcp6", "[2001:db8::1]:http", &TCPAddr{IP: ParseIP("2001:db8::1"), Port: 80}, nil}, } func TestResolveTCPAddr(t *testing.T) { @@ -331,13 +332,13 @@ func TestResolveTCPAddr(t *testing.T) { for _, tt := range resolveTCPAddrTests { addr, err := ResolveTCPAddr(tt.network, tt.litAddrOrName) if !reflect.DeepEqual(addr, tt.addr) || !reflect.DeepEqual(err, tt.err) { - t.Errorf("ResolveTCPAddr(%q, %q) = %v, %v, want %v, %v", tt.network, tt.litAddrOrName, addr, err, tt.addr, tt.err) + t.Errorf("ResolveTCPAddr(%q, %q) = %#v, %v, want %#v, %v", tt.network, tt.litAddrOrName, addr, err, tt.addr, tt.err) continue } if err == nil { addr2, err := ResolveTCPAddr(addr.Network(), addr.String()) if !reflect.DeepEqual(addr2, tt.addr) || err != tt.err { - t.Errorf("(%q, %q): ResolveTCPAddr(%q, %q) = %v, %v, want %v, %v", tt.network, tt.litAddrOrName, addr.Network(), addr.String(), addr2, err, tt.addr, tt.err) + t.Errorf("(%q, %q): ResolveTCPAddr(%q, %q) = %#v, %v, want %#v, %v", tt.network, tt.litAddrOrName, addr.Network(), addr.String(), addr2, err, tt.addr, tt.err) } } } diff --git a/src/net/udpsock_test.go b/src/net/udpsock_test.go index 29d769c5a5..708cc10120 100644 --- a/src/net/udpsock_test.go +++ b/src/net/udpsock_test.go @@ -72,6 +72,17 @@ var resolveUDPAddrTests = []resolveUDPAddrTest{ {"udp", ":12345", &UDPAddr{Port: 12345}, nil}, {"http", "127.0.0.1:0", nil, UnknownNetworkError("http")}, + + {"udp", "127.0.0.1:domain", &UDPAddr{IP: ParseIP("127.0.0.1"), Port: 53}, nil}, + {"udp", "[::ffff:127.0.0.1]:domain", &UDPAddr{IP: ParseIP("::ffff:127.0.0.1"), Port: 53}, nil}, + {"udp", "[2001:db8::1]:domain", &UDPAddr{IP: ParseIP("2001:db8::1"), Port: 53}, nil}, + {"udp4", "127.0.0.1:domain", &UDPAddr{IP: ParseIP("127.0.0.1"), Port: 53}, nil}, + {"udp4", "[::ffff:127.0.0.1]:domain", &UDPAddr{IP: ParseIP("127.0.0.1"), Port: 53}, nil}, + {"udp6", "[2001:db8::1]:domain", &UDPAddr{IP: ParseIP("2001:db8::1"), Port: 53}, nil}, + + {"udp4", "[2001:db8::1]:domain", nil, &AddrError{Err: errNoSuitableAddress.Error(), Addr: "2001:db8::1"}}, + {"udp6", "127.0.0.1:domain", nil, &AddrError{Err: errNoSuitableAddress.Error(), Addr: "127.0.0.1"}}, + {"udp6", "[::ffff:127.0.0.1]:domain", nil, &AddrError{Err: errNoSuitableAddress.Error(), Addr: "::ffff:127.0.0.1"}}, } func TestResolveUDPAddr(t *testing.T) { @@ -79,21 +90,17 @@ func TestResolveUDPAddr(t *testing.T) { defer func() { testHookLookupIP = origTestHookLookupIP }() testHookLookupIP = lookupLocalhost - for i, tt := range resolveUDPAddrTests { + for _, tt := range resolveUDPAddrTests { addr, err := ResolveUDPAddr(tt.network, tt.litAddrOrName) - if err != tt.err { - t.Errorf("#%d: %v", i, err) - } else if !reflect.DeepEqual(addr, tt.addr) { - t.Errorf("#%d: got %#v; want %#v", i, addr, tt.addr) - } - if err != nil { + if !reflect.DeepEqual(addr, tt.addr) || !reflect.DeepEqual(err, tt.err) { + t.Errorf("ResolveUDPAddr(%q, %q) = %#v, %v, want %#v, %v", tt.network, tt.litAddrOrName, addr, err, tt.addr, tt.err) continue } - rtaddr, err := ResolveUDPAddr(addr.Network(), addr.String()) - if err != nil { - t.Errorf("#%d: %v", i, err) - } else if !reflect.DeepEqual(rtaddr, addr) { - t.Errorf("#%d: got %#v; want %#v", i, rtaddr, addr) + if err == nil { + addr2, err := ResolveUDPAddr(addr.Network(), addr.String()) + if !reflect.DeepEqual(addr2, tt.addr) || err != tt.err { + t.Errorf("(%q, %q): ResolveUDPAddr(%q, %q) = %#v, %v, want %#v, %v", tt.network, tt.litAddrOrName, addr.Network(), addr.String(), addr2, err, tt.addr, tt.err) + } } } } diff --git a/src/net/writev_test.go b/src/net/writev_test.go index 4d2fc39506..7160d28c3a 100644 --- a/src/net/writev_test.go +++ b/src/net/writev_test.go @@ -151,7 +151,7 @@ func testBuffer_writeTo(t *testing.T, chunks int, useCopy bool) { var wantSum int switch runtime.GOOS { - case "darwin", "dragonfly", "freebsd", "linux", "netbsd", "openbsd": + case "android", "darwin", "dragonfly", "freebsd", "linux", "netbsd", "openbsd": var wantMinCalls int wantSum = want.Len() v := chunks diff --git a/src/os/exec/exec_test.go b/src/os/exec/exec_test.go index d3ac7ab4b9..34337450a0 100644 --- a/src/os/exec/exec_test.go +++ b/src/os/exec/exec_test.go @@ -247,6 +247,11 @@ func TestStdinClose(t *testing.T) { } // Issue 17647. +// It used to be the case that TestStdinClose, above, would fail when +// run under the race detector. This test is a variant of TestStdinClose +// that also used to fail when run under the race detector. +// This test is run by cmd/dist under the race detector to verify that +// the race detector no longer reports any problems. func TestStdinCloseRace(t *testing.T) { cmd := helperCommand(t, "stdinClose") stdin, err := cmd.StdinPipe() @@ -262,7 +267,12 @@ func TestStdinCloseRace(t *testing.T) { } }() go func() { - io.Copy(stdin, strings.NewReader(stdinCloseTestString)) + // Send the wrong string, so that the child fails even + // if the other goroutine doesn't manage to kill it first. + // This test is to check that the race detector does not + // falsely report an error, so it doesn't matter how the + // child process fails. + io.Copy(stdin, strings.NewReader("unexpected string")) if err := stdin.Close(); err != nil { t.Errorf("stdin.Close: %v", err) } @@ -978,7 +988,7 @@ func TestContextCancel(t *testing.T) { break } if time.Since(start) > time.Second { - t.Fatal("cancelling context did not stop program") + t.Fatal("canceling context did not stop program") } time.Sleep(time.Millisecond) } diff --git a/src/os/file.go b/src/os/file.go index de245c5479..d45a00b123 100644 --- a/src/os/file.go +++ b/src/os/file.go @@ -260,7 +260,7 @@ func Create(name string) (*File, error) { var lstat = Lstat // Rename renames (moves) oldpath to newpath. -// If newpath already exists, Rename replaces it. +// If newpath already exists and is not a directory, Rename replaces it. // OS-specific restrictions may apply when oldpath and newpath are in different directories. // If there is an error, it will be of type *LinkError. func Rename(oldpath, newpath string) error { diff --git a/src/os/os_test.go b/src/os/os_test.go index b1e20b7839..b7300cd38c 100644 --- a/src/os/os_test.go +++ b/src/os/os_test.go @@ -1708,51 +1708,62 @@ func TestLongPath(t *testing.T) { t.Fatalf("RemoveAll failed: %v", err) } }(tmpdir) + + // Test the boundary of 247 and fewer bytes (normal) and 248 and more bytes (adjusted). + sizes := []int{247, 248, 249, 400} for len(tmpdir) < 400 { tmpdir += "/dir3456789" } - if err := MkdirAll(tmpdir, 0755); err != nil { - t.Fatalf("MkdirAll failed: %v", err) - } - data := []byte("hello world\n") - if err := ioutil.WriteFile(tmpdir+"/foo.txt", data, 0644); err != nil { - t.Fatalf("ioutil.WriteFile() failed: %v", err) - } - if err := Rename(tmpdir+"/foo.txt", tmpdir+"/bar.txt"); err != nil { - t.Fatalf("Rename failed: %v", err) - } - mtime := time.Now().Truncate(time.Minute) - if err := Chtimes(tmpdir+"/bar.txt", mtime, mtime); err != nil { - t.Fatalf("Chtimes failed: %v", err) - } - names := []string{"bar.txt"} - if testenv.HasSymlink() { - if err := Symlink(tmpdir+"/bar.txt", tmpdir+"/symlink.txt"); err != nil { - t.Fatalf("Symlink failed: %v", err) - } - names = append(names, "symlink.txt") - } - if testenv.HasLink() { - if err := Link(tmpdir+"/bar.txt", tmpdir+"/link.txt"); err != nil { - t.Fatalf("Link failed: %v", err) - } - names = append(names, "link.txt") - } - for _, wantSize := range []int64{int64(len(data)), 0} { - for _, name := range names { - path := tmpdir + "/" + name - dir, err := Stat(path) - if err != nil { - t.Fatalf("Stat(%q) failed: %v", path, err) + for _, sz := range sizes { + t.Run(fmt.Sprintf("length=%d", sz), func(t *testing.T) { + sizedTempDir := tmpdir[:sz-1] + "x" // Ensure it does not end with a slash. + + // The various sized runs are for this call to trigger the boundary + // condition. + if err := MkdirAll(sizedTempDir, 0755); err != nil { + t.Fatalf("MkdirAll failed: %v", err) } - filesize := size(path, t) - if dir.Size() != filesize || filesize != wantSize { - t.Errorf("Size(%q) is %d, len(ReadFile()) is %d, want %d", path, dir.Size(), filesize, wantSize) + data := []byte("hello world\n") + if err := ioutil.WriteFile(sizedTempDir+"/foo.txt", data, 0644); err != nil { + t.Fatalf("ioutil.WriteFile() failed: %v", err) } - } - if err := Truncate(tmpdir+"/bar.txt", 0); err != nil { - t.Fatalf("Truncate failed: %v", err) - } + if err := Rename(sizedTempDir+"/foo.txt", sizedTempDir+"/bar.txt"); err != nil { + t.Fatalf("Rename failed: %v", err) + } + mtime := time.Now().Truncate(time.Minute) + if err := Chtimes(sizedTempDir+"/bar.txt", mtime, mtime); err != nil { + t.Fatalf("Chtimes failed: %v", err) + } + names := []string{"bar.txt"} + if testenv.HasSymlink() { + if err := Symlink(sizedTempDir+"/bar.txt", sizedTempDir+"/symlink.txt"); err != nil { + t.Fatalf("Symlink failed: %v", err) + } + names = append(names, "symlink.txt") + } + if testenv.HasLink() { + if err := Link(sizedTempDir+"/bar.txt", sizedTempDir+"/link.txt"); err != nil { + t.Fatalf("Link failed: %v", err) + } + names = append(names, "link.txt") + } + for _, wantSize := range []int64{int64(len(data)), 0} { + for _, name := range names { + path := sizedTempDir + "/" + name + dir, err := Stat(path) + if err != nil { + t.Fatalf("Stat(%q) failed: %v", path, err) + } + filesize := size(path, t) + if dir.Size() != filesize || filesize != wantSize { + t.Errorf("Size(%q) is %d, len(ReadFile()) is %d, want %d", path, dir.Size(), filesize, wantSize) + } + } + if err := Truncate(sizedTempDir+"/bar.txt", 0); err != nil { + t.Fatalf("Truncate failed: %v", err) + } + } + }) } } diff --git a/src/os/path_windows.go b/src/os/path_windows.go index ccac1c0b64..101b026dc9 100644 --- a/src/os/path_windows.go +++ b/src/os/path_windows.go @@ -139,13 +139,16 @@ func dirname(path string) string { func fixLongPath(path string) string { // Do nothing (and don't allocate) if the path is "short". // Empirically (at least on the Windows Server 2013 builder), - // the kernel is arbitrarily okay with <= 248 bytes. That + // the kernel is arbitrarily okay with < 248 bytes. That // matches what the docs above say: // "When using an API to create a directory, the specified // path cannot be so long that you cannot append an 8.3 file // name (that is, the directory name cannot exceed MAX_PATH // minus 12)." Since MAX_PATH is 260, 260 - 12 = 248. - if len(path) <= 248 { + // + // The MSDN docs appear to say that a normal path that is 248 bytes long + // will work; empirically the path must be less then 248 bytes long. + if len(path) < 248 { // Don't fix. (This is how Go 1.7 and earlier worked, // not automatically generating the \\?\ form) return path diff --git a/src/os/signal/doc.go b/src/os/signal/doc.go index 16f49c7ab8..73b01a2764 100644 --- a/src/os/signal/doc.go +++ b/src/os/signal/doc.go @@ -181,11 +181,10 @@ If the Go runtime sees an existing signal handler for the SIGCANCEL or SIGSETXID signals (which are used only on GNU/Linux), it will turn on the SA_ONSTACK flag and otherwise keep the signal handler. -For the synchronous signals and SIGPIPE, the Go runtime will install a -signal handler. It will save any existing signal handler. If a -synchronous signal arrives while executing non-Go code, the Go runtime -will invoke the existing signal handler instead of the Go signal -handler. +For the synchronous signals, the Go runtime will install a signal +handler. It will save any existing signal handler. If a synchronous +signal arrives while executing non-Go code, the Go runtime will invoke +the existing signal handler instead of the Go signal handler. Go code built with -buildmode=c-archive or -buildmode=c-shared will not install any other signal handlers by default. If there is an diff --git a/src/os/user/user.go b/src/os/user/user.go index 7b44397afb..ad61992ad3 100644 --- a/src/os/user/user.go +++ b/src/os/user/user.go @@ -15,31 +15,39 @@ var ( ) // User represents a user account. -// -// On POSIX systems Uid and Gid contain a decimal number -// representing uid and gid. On windows Uid and Gid -// contain security identifier (SID) in a string format. -// On Plan 9, Uid, Gid, Username, and Name will be the -// contents of /dev/user. type User struct { - Uid string // user ID - Gid string // primary group ID + // Uid is the user ID. + // On POSIX systems, this is a decimal number representing the uid. + // On Windows, this is a security identifier (SID) in a string format. + // On Plan 9, this is the contents of /dev/user. + Uid string + // Gid is the primary group ID. + // On POSIX systems, this is a decimal number representing the gid. + // On Windows, this is a SID in a string format. + // On Plan 9, this is the contents of /dev/user. + Gid string + // Username is the login name. Username string - Name string - HomeDir string + // Name is the user's real or display name. + // It might be blank. + // On POSIX systems, this is the first (or only) entry in the GECOS field + // list. + // On Windows, this is the user's display name. + // On Plan 9, this is the contents of /dev/user. + Name string + // HomeDir is the path to the user's home directory (if they have one). + HomeDir string } // Group represents a grouping of users. // -// On POSIX systems Gid contains a decimal number -// representing the group ID. +// On POSIX systems Gid contains a decimal number representing the group ID. type Group struct { Gid string // group ID Name string // group name } -// UnknownUserIdError is returned by LookupId when -// a user cannot be found. +// UnknownUserIdError is returned by LookupId when a user cannot be found. type UnknownUserIdError int func (e UnknownUserIdError) Error() string { diff --git a/src/path/filepath/path_plan9.go b/src/path/filepath/path_plan9.go index 60d46d9d42..ec792fc831 100644 --- a/src/path/filepath/path_plan9.go +++ b/src/path/filepath/path_plan9.go @@ -18,6 +18,9 @@ func volumeNameLen(path string) int { } // HasPrefix exists for historical compatibility and should not be used. +// +// Deprecated: HasPrefix does not respect path boundaries and +// does not ignore case when required. func HasPrefix(p, prefix string) bool { return strings.HasPrefix(p, prefix) } diff --git a/src/path/filepath/path_unix.go b/src/path/filepath/path_unix.go index dddcac0a5c..d77ff24cdc 100644 --- a/src/path/filepath/path_unix.go +++ b/src/path/filepath/path_unix.go @@ -21,7 +21,8 @@ func volumeNameLen(path string) int { // HasPrefix exists for historical compatibility and should not be used. // -// Deprecated: Use strings.HasPrefix instead. +// Deprecated: HasPrefix does not respect path boundaries and +// does not ignore case when required. func HasPrefix(p, prefix string) bool { return strings.HasPrefix(p, prefix) } diff --git a/src/path/filepath/path_windows.go b/src/path/filepath/path_windows.go index 359703de26..0d8b62015c 100644 --- a/src/path/filepath/path_windows.go +++ b/src/path/filepath/path_windows.go @@ -65,6 +65,9 @@ func volumeNameLen(path string) int { } // HasPrefix exists for historical compatibility and should not be used. +// +// Deprecated: HasPrefix does not respect path boundaries and +// does not ignore case when required. func HasPrefix(p, prefix string) bool { if strings.HasPrefix(p, prefix) { return true diff --git a/src/path/filepath/symlink_windows.go b/src/path/filepath/symlink_windows.go index bb05aabc92..f771fe3a8a 100644 --- a/src/path/filepath/symlink_windows.go +++ b/src/path/filepath/symlink_windows.go @@ -47,7 +47,7 @@ func baseIsDotDot(path string) bool { return path[i+1:] == ".." } -// toNorm returns the normalized path that is guranteed to be unique. +// toNorm returns the normalized path that is guaranteed to be unique. // It should accept the following formats: // * UNC paths (e.g \\server\share\foo\bar) // * absolute paths (e.g C:\foo\bar) diff --git a/src/plugin/plugin.go b/src/plugin/plugin.go index 5c822bd9ba..b86099a4f6 100644 --- a/src/plugin/plugin.go +++ b/src/plugin/plugin.go @@ -4,7 +4,7 @@ // Package plugin implements loading and symbol resolution of Go plugins. // -// Currently plugins only work on Linux and Darwin. +// Currently plugins only work on Linux. // // A plugin is a Go main package with exported functions and variables that // has been built with: diff --git a/src/reflect/example_test.go b/src/reflect/example_test.go index 9e2b9b3e97..f959b95846 100644 --- a/src/reflect/example_test.go +++ b/src/reflect/example_test.go @@ -5,6 +5,8 @@ package reflect_test import ( + "bytes" + "encoding/json" "fmt" "io" "os" @@ -107,3 +109,42 @@ func ExampleTypeOf() { // Output: // true } + +func ExampleStructOf() { + typ := reflect.StructOf([]reflect.StructField{ + { + Name: "Height", + Type: reflect.TypeOf(float64(0)), + Tag: `json:"height"`, + }, + { + Name: "Age", + Type: reflect.TypeOf(int(0)), + Tag: `json:"age"`, + }, + }) + + v := reflect.New(typ).Elem() + v.Field(0).SetFloat(0.4) + v.Field(1).SetInt(2) + s := v.Addr().Interface() + + w := new(bytes.Buffer) + if err := json.NewEncoder(w).Encode(s); err != nil { + panic(err) + } + + fmt.Printf("value: %+v\n", s) + fmt.Printf("json: %s", w.Bytes()) + + r := bytes.NewReader([]byte(`{"height":1.5,"age":10}`)) + if err := json.NewDecoder(r).Decode(s); err != nil { + panic(err) + } + fmt.Printf("value: %+v\n", s) + + // Output: + // value: &{Height:0.4 Age:2} + // json: {"height":0.4,"age":2} + // value: &{Height:1.5 Age:10} +} diff --git a/src/reflect/makefunc.go b/src/reflect/makefunc.go index a7efeb8262..885966db6f 100644 --- a/src/reflect/makefunc.go +++ b/src/reflect/makefunc.go @@ -12,9 +12,12 @@ import ( // makeFuncImpl is the closure value implementing the function // returned by MakeFunc. +// The first two words of this type must be kept in sync with +// methodValue and runtime.reflectMethodValue. +// Any changes should be reflected in all three. type makeFuncImpl struct { code uintptr - stack *bitVector // stack bitmap for args - offset known to runtime + stack *bitVector typ *funcType fn func([]Value) []Value } @@ -70,11 +73,12 @@ func MakeFunc(typ Type, fn func(args []Value) (results []Value)) Value { // word in the passed-in argument frame. func makeFuncStub() -// This type is partially duplicated as runtime.reflectMethodValue. -// Any changes should be reflected in both. +// The first two words of this type must be kept in sync with +// makeFuncImpl and runtime.reflectMethodValue. +// Any changes should be reflected in all three. type methodValue struct { fn uintptr - stack *bitVector // stack bitmap for args - offset known to runtime + stack *bitVector method int rcvr Value } diff --git a/src/runtime/HACKING.md b/src/runtime/HACKING.md index 88fb708c7e..ea7c5c128d 100644 --- a/src/runtime/HACKING.md +++ b/src/runtime/HACKING.md @@ -1,6 +1,139 @@ -This is a very incomplete and probably out-of-date guide to -programming in the Go runtime and how it differs from writing normal -Go. +This is a living document and at times it will be out of date. It is +intended to articulate how programming in the Go runtime differs from +writing normal Go. It focuses on pervasive concepts rather than +details of particular interfaces. + +Scheduler structures +==================== + +The scheduler manages three types of resources that pervade the +runtime: Gs, Ms, and Ps. It's important to understand these even if +you're not working on the scheduler. + +Gs, Ms, Ps +---------- + +A "G" is simply a goroutine. It's represented by type `g`. When a +goroutine exits, its `g` object is returned to a pool of free `g`s and +can later be reused for some other goroutine. + +An "M" is an OS thread that can be executing user Go code, runtime +code, a system call, or be idle. It's represented by type `m`. There +can be any number of Ms at a time since any number of threads may be +blocked in system calls. + +Finally, a "P" represents the resources required to execute user Go +code, such as scheduler and memory allocator state. It's represented +by type `p`. There are exactly `GOMAXPROCS` Ps. A P can be thought of +like a CPU in the OS scheduler and the contents of the `p` type like +per-CPU state. This is a good place to put state that needs to be +sharded for efficiency, but doesn't need to be per-thread or +per-goroutine. + +The scheduler's job is to match up a G (the code to execute), an M +(where to execute it), and a P (the rights and resources to execute +it). When an M stops executing user Go code, for example by entering a +system call, it returns its P to the idle P pool. In order to resume +executing user Go code, for example on return from a system call, it +must acquire a P from the idle pool. + +All `g`, `m`, and `p` objects are heap allocated, but are never freed, +so their memory remains type stable. As a result, the runtime can +avoid write barriers in the depths of the scheduler. + +User stacks and system stacks +----------------------------- + +Every non-dead G has a *user stack* associated with it, which is what +user Go code executes on. User stacks start small (e.g., 2K) and grow +or shrink dynamically. + +Every M has a *system stack* associated with it (also known as the M's +"g0" stack because it's implemented as a stub G) and, on Unix +platforms, a *signal stack* (also known as the M's "gsignal" stack). +System and signal stacks cannot grow, but are large enough to execute +runtime and cgo code (8K in a pure Go binary; system-allocated in a +cgo binary). + +Runtime code often temporarily switches to the system stack using +`systemstack`, `mcall`, or `asmcgocall` to perform tasks that must not +be preempted, that must not grow the user stack, or that switch user +goroutines. Code running on the system stack is implicitly +non-preemptible and the garbage collector does not scan system stacks. +While running on the system stack, the current user stack is not used +for execution. + +`getg()` and `getg().m.curg` +---------------------------- + +To get the current user `g`, use `getg().m.curg`. + +`getg()` alone returns the current `g`, but when executing on the +system or signal stacks, this will return the current M's "g0" or +"gsignal", respectively. This is usually not what you want. + +To determine if you're running on the user stack or the system stack, +use `getg() == getg().m.curg`. + +Error handling and reporting +============================ + +Errors that can reasonably be recovered from in user code should use +`panic` like usual. However, there are some situations where `panic` +will cause an immediate fatal error, such as when called on the system +stack or when called during `mallocgc`. + +Most errors in the runtime are not recoverable. For these, use +`throw`, which dumps the traceback and immediately terminates the +process. In general, `throw` should be passed a string constant to +avoid allocating in perilous situations. By convention, additional +details are printed before `throw` using `print` or `println` and the +messages are prefixed with "runtime:". + +For runtime error debugging, it's useful to run with +`GOTRACEBACK=system` or `GOTRACEBACK=crash`. + +Synchronization +=============== + +The runtime has multiple synchronization mechanisms. They differ in +semantics and, in particular, in whether they interact with the +goroutine scheduler or the OS scheduler. + +The simplest is `mutex`, which is manipulated using `lock` and +`unlock`. This should be used to protect shared structures for short +periods. Blocking on a `mutex` directly blocks the M, without +interacting with the Go scheduler. This means it is safe to use from +the lowest levels of the runtime, but also prevents any associated G +and P from being rescheduled. + +For one-shot notifications, use `note`, which provides `notesleep` and +`notewakeup`. Unlike traditional UNIX `sleep`/`wakeup`, `note`s are +race-free, so `notesleep` returns immediately if the `notewakeup` has +already happened. A `note` can be reset after use with `noteclear`, +which must not race with a sleep or wakeup. Like `mutex`, blocking on +a `note` blocks the M. However, there are different ways to sleep on a +`note`:`notesleep` also prevents rescheduling of any associated G and +P, while `notetsleepg` acts like a blocking system call that allows +the P to be reused to run another G. This is still less efficient than +blocking the G directly since it consumes an M. + +To interact directly with the goroutine scheduler, use `gopark` and +`goready`. `gopark` parks the current goroutine—putting it in the +"waiting" state and removing it from the scheduler's run queue—and +schedules another goroutine on the current M/P. `goready` puts a +parked goroutine back in the "runnable" state and adds it to the run +queue. + +In summary, + +<table> +<tr><th></th><th colspan="3">Blocks</th></tr> +<tr><th>Interface</th><th>G</th><th>M</th><th>P</th></tr> +<tr><td>mutex</td><td>Y</td><td>Y</td><td>Y</td></tr> +<tr><td>note</td><td>Y</td><td>Y</td><td>Y/N</td></tr> +<tr><td>park</td><td>Y</td><td>N</td><td>N</td></tr> +</table> Unmanaged memory ================ diff --git a/src/runtime/asm_amd64.s b/src/runtime/asm_amd64.s index 9ffd297d84..cb428d6de3 100644 --- a/src/runtime/asm_amd64.s +++ b/src/runtime/asm_amd64.s @@ -75,11 +75,24 @@ no7: TESTL $(1<<5), runtime·cpuid_ebx7(SB) // check for AVX2 bit JEQ noavx2 MOVB $1, runtime·support_avx2(SB) - JMP nocpuinfo + JMP testbmi1 noavx: MOVB $0, runtime·support_avx(SB) noavx2: MOVB $0, runtime·support_avx2(SB) +testbmi1: + // Detect BMI1 and BMI2 extensions as per + // 5.1.16.1 Detection of VEX-encoded GPR Instructions, + // LZCNT and TZCNT, PREFETCHW chapter of [1] + MOVB $0, runtime·support_bmi1(SB) + TESTL $(1<<3), runtime·cpuid_ebx7(SB) // check for BMI1 bit + JEQ testbmi2 + MOVB $1, runtime·support_bmi1(SB) +testbmi2: + MOVB $0, runtime·support_bmi2(SB) + TESTL $(1<<8), runtime·cpuid_ebx7(SB) // check for BMI2 bit + JEQ nocpuinfo + MOVB $1, runtime·support_bmi2(SB) nocpuinfo: // if there is an _cgo_init, call it. @@ -742,7 +755,7 @@ havem: MOVQ (g_sched+gobuf_pc)(SI), BX MOVQ BX, -8(DI) // Compute the size of the frame, including return PC and, if - // GOEXPERIMENT=framepointer, the saved based pointer + // GOEXPERIMENT=framepointer, the saved base pointer MOVQ ctxt+24(FP), BX LEAQ fv+0(FP), AX SUBQ SP, AX diff --git a/src/runtime/asm_mipsx.s b/src/runtime/asm_mipsx.s index cd855c7d34..73da768897 100644 --- a/src/runtime/asm_mipsx.s +++ b/src/runtime/asm_mipsx.s @@ -12,11 +12,11 @@ #define REGCTXT R22 TEXT runtime·rt0_go(SB),NOSPLIT,$0 - // R29 = stack; R1 = argc; R2 = argv + // R29 = stack; R4 = argc; R5 = argv ADDU $-12, R29 - MOVW R1, 4(R29) // argc - MOVW R2, 8(R29) // argv + MOVW R4, 4(R29) // argc + MOVW R5, 8(R29) // argv // create istack out of the given (operating system) stack. // _cgo_init may update stackguard. @@ -28,7 +28,16 @@ TEXT runtime·rt0_go(SB),NOSPLIT,$0 MOVW R1, (g_stack+stack_lo)(g) MOVW R29, (g_stack+stack_hi)(g) -// TODO(mips32): cgo + // if there is a _cgo_init, call it using the gcc ABI. + MOVW _cgo_init(SB), R25 + BEQ R25, nocgo + ADDU $-16, R29 + MOVW R0, R7 // arg 3: not used + MOVW R0, R6 // arg 2: not used + MOVW $setg_gcc<>(SB), R5 // arg 1: setg + MOVW g, R4 // arg 0: G + JAL (R25) + ADDU $16, R29 nocgo: // update stackguard after _cgo_init @@ -434,7 +443,7 @@ TEXT runtime·jmpdefer(SB),NOSPLIT,$0-8 JMP (R4) // Save state of caller into g->sched. Smashes R1. -TEXT gosave<>(SB),NOSPLIT,$0 +TEXT gosave<>(SB),NOSPLIT,$-4 MOVW R31, (g_sched+gobuf_pc)(g) MOVW R29, (g_sched+gobuf_sp)(g) MOVW R0, (g_sched+gobuf_lr)(g) @@ -449,22 +458,168 @@ TEXT gosave<>(SB),NOSPLIT,$0 // Call fn(arg) on the scheduler stack, // aligned appropriately for the gcc ABI. // See cgocall.go for more details. -// Not implemented. TEXT ·asmcgocall(SB),NOSPLIT,$0-12 - UNDEF + MOVW fn+0(FP), R25 + MOVW arg+4(FP), R4 + + MOVW R29, R3 // save original stack pointer + MOVW g, R2 + + // Figure out if we need to switch to m->g0 stack. + // We get called to create new OS threads too, and those + // come in on the m->g0 stack already. + MOVW g_m(g), R5 + MOVW m_g0(R5), R6 + BEQ R6, g, g0 + + JAL gosave<>(SB) + MOVW R6, g + JAL runtime·save_g(SB) + MOVW (g_sched+gobuf_sp)(g), R29 + + // Now on a scheduling stack (a pthread-created stack). +g0: + // Save room for two of our pointers and O32 frame. + ADDU $-24, R29 + AND $~7, R29 // O32 ABI expects 8-byte aligned stack on function entry + MOVW R2, 16(R29) // save old g on stack + MOVW (g_stack+stack_hi)(R2), R2 + SUBU R3, R2 + MOVW R2, 20(R29) // save depth in old g stack (can't just save SP, as stack might be copied during a callback) + JAL (R25) + + // Restore g, stack pointer. R2 is return value. + MOVW 16(R29), g + JAL runtime·save_g(SB) + MOVW (g_stack+stack_hi)(g), R5 + MOVW 20(R29), R6 + SUBU R6, R5 + MOVW R5, R29 + + MOVW R2, ret+8(FP) + RET // cgocallback(void (*fn)(void*), void *frame, uintptr framesize) // Turn the fn into a Go func (by taking its address) and call // cgocallback_gofunc. -// Not implemented. -TEXT runtime·cgocallback(SB),NOSPLIT,$0-16 - UNDEF +TEXT runtime·cgocallback(SB),NOSPLIT,$16-16 + MOVW $fn+0(FP), R1 + MOVW R1, 4(R29) + MOVW frame+4(FP), R1 + MOVW R1, 8(R29) + MOVW framesize+8(FP), R1 + MOVW R1, 12(R29) + MOVW ctxt+12(FP), R1 + MOVW R1, 16(R29) + MOVW $runtime·cgocallback_gofunc(SB), R1 + JAL (R1) + RET -// cgocallback_gofunc(FuncVal*, void *frame, uintptr framesize) +// cgocallback_gofunc(FuncVal*, void *frame, uintptr framesize, uintptr ctxt) // See cgocall.go for more details. -// Not implemented. -TEXT ·cgocallback_gofunc(SB),NOSPLIT,$0-16 - UNDEF +TEXT ·cgocallback_gofunc(SB),NOSPLIT,$8-16 + NO_LOCAL_POINTERS + + // Load m and g from thread-local storage. + MOVB runtime·iscgo(SB), R1 + BEQ R1, nocgo + JAL runtime·load_g(SB) +nocgo: + + // If g is nil, Go did not create the current thread. + // Call needm to obtain one for temporary use. + // In this case, we're running on the thread stack, so there's + // lots of space, but the linker doesn't know. Hide the call from + // the linker analysis by using an indirect call. + BEQ g, needm + + MOVW g_m(g), R3 + MOVW R3, savedm-4(SP) + JMP havem + +needm: + MOVW g, savedm-4(SP) // g is zero, so is m. + MOVW $runtime·needm(SB), R4 + JAL (R4) + + // Set m->sched.sp = SP, so that if a panic happens + // during the function we are about to execute, it will + // have a valid SP to run on the g0 stack. + // The next few lines (after the havem label) + // will save this SP onto the stack and then write + // the same SP back to m->sched.sp. That seems redundant, + // but if an unrecovered panic happens, unwindm will + // restore the g->sched.sp from the stack location + // and then systemstack will try to use it. If we don't set it here, + // that restored SP will be uninitialized (typically 0) and + // will not be usable. + MOVW g_m(g), R3 + MOVW m_g0(R3), R1 + MOVW R29, (g_sched+gobuf_sp)(R1) + +havem: + // Now there's a valid m, and we're running on its m->g0. + // Save current m->g0->sched.sp on stack and then set it to SP. + // Save current sp in m->g0->sched.sp in preparation for + // switch back to m->curg stack. + // NOTE: unwindm knows that the saved g->sched.sp is at 4(R29) aka savedsp-8(SP). + MOVW m_g0(R3), R1 + MOVW (g_sched+gobuf_sp)(R1), R2 + MOVW R2, savedsp-8(SP) + MOVW R29, (g_sched+gobuf_sp)(R1) + + // Switch to m->curg stack and call runtime.cgocallbackg. + // Because we are taking over the execution of m->curg + // but *not* resuming what had been running, we need to + // save that information (m->curg->sched) so we can restore it. + // We can restore m->curg->sched.sp easily, because calling + // runtime.cgocallbackg leaves SP unchanged upon return. + // To save m->curg->sched.pc, we push it onto the stack. + // This has the added benefit that it looks to the traceback + // routine like cgocallbackg is going to return to that + // PC (because the frame we allocate below has the same + // size as cgocallback_gofunc's frame declared above) + // so that the traceback will seamlessly trace back into + // the earlier calls. + // + // In the new goroutine, -4(SP) is unused (where SP refers to + // m->curg's SP while we're setting it up, before we've adjusted it). + MOVW m_curg(R3), g + JAL runtime·save_g(SB) + MOVW (g_sched+gobuf_sp)(g), R2 // prepare stack as R2 + MOVW (g_sched+gobuf_pc)(g), R4 + MOVW R4, -12(R2) + MOVW ctxt+12(FP), R1 + MOVW R1, -8(R2) + MOVW $-12(R2), R29 + JAL runtime·cgocallbackg(SB) + + // Restore g->sched (== m->curg->sched) from saved values. + MOVW 0(R29), R4 + MOVW R4, (g_sched+gobuf_pc)(g) + MOVW $12(R29), R2 + MOVW R2, (g_sched+gobuf_sp)(g) + + // Switch back to m->g0's stack and restore m->g0->sched.sp. + // (Unlike m->curg, the g0 goroutine never uses sched.pc, + // so we do not have to restore it.) + MOVW g_m(g), R3 + MOVW m_g0(R3), g + JAL runtime·save_g(SB) + MOVW (g_sched+gobuf_sp)(g), R29 + MOVW savedsp-8(SP), R2 + MOVW R2, (g_sched+gobuf_sp)(g) + + // If the m on entry was nil, we called needm above to borrow an m + // for the duration of the call. Since the call is over, return it with dropm. + MOVW savedm-4(SP), R3 + BNE R3, droppedm + MOVW $runtime·dropm(SB), R4 + JAL (R4) +droppedm: + + // Done! + RET // void setg(G*); set g. for use by needm. // This only happens if iscgo, so jump straight to save_g @@ -475,9 +630,10 @@ TEXT runtime·setg(SB),NOSPLIT,$0-4 // void setg_gcc(G*); set g in C TLS. // Must obey the gcc calling convention. -// Not implemented. TEXT setg_gcc<>(SB),NOSPLIT,$0 - UNDEF + MOVW R4, g + JAL runtime·save_g(SB) + RET TEXT runtime·getcallerpc(SB),NOSPLIT,$4-8 MOVW 8(R29), R1 // LR saved by caller @@ -764,9 +920,23 @@ TEXT runtime·return0(SB),NOSPLIT,$0 // Called from cgo wrappers, this function returns g->m->curg.stack.hi. // Must obey the gcc calling convention. -// Not implemented. TEXT _cgo_topofstack(SB),NOSPLIT,$-4 - UNDEF + // g (R30), R3 and REGTMP (R23) might be clobbered by load_g. R30 and R23 + // are callee-save in the gcc calling convention, so save them. + MOVW R23, R8 + MOVW g, R9 + MOVW R31, R10 // this call frame does not save LR + + JAL runtime·load_g(SB) + MOVW g_m(g), R1 + MOVW m_curg(R1), R1 + MOVW (g_stack+stack_hi)(R1), R2 // return value in R2 + + MOVW R8, R23 + MOVW R9, g + MOVW R10, R31 + + RET // The top-most function running on a goroutine // returns to goexit+PCQuantum. diff --git a/src/runtime/cgo/asm_mipsx.s b/src/runtime/cgo/asm_mipsx.s new file mode 100644 index 0000000000..dd16af6fbe --- /dev/null +++ b/src/runtime/cgo/asm_mipsx.s @@ -0,0 +1,67 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build mips mipsle + +#include "textflag.h" + +/* + * void crosscall2(void (*fn)(void*, int32, uintptr), void*, int32, uintptr) + * Save registers and call fn with two arguments. + */ +TEXT crosscall2(SB),NOSPLIT,$-4 + /* + * We still need to save all callee save register as before, and then + * push 3 args for fn (R5, R6, R7). + * Also note that at procedure entry in gc world, 4(R29) will be the + * first arg. + */ + + // Space for 9 caller-saved GPR + LR + 6 caller-saved FPR. + // O32 ABI allows us to smash 16 bytes argument area of caller frame. + SUBU $(4*14+8*6-16), R29 + MOVW R5, (4*1)(R29) + MOVW R6, (4*2)(R29) + MOVW R7, (4*3)(R29) + MOVW R16, (4*4)(R29) + MOVW R17, (4*5)(R29) + MOVW R18, (4*6)(R29) + MOVW R19, (4*7)(R29) + MOVW R20, (4*8)(R29) + MOVW R21, (4*9)(R29) + MOVW R22, (4*10)(R29) + MOVW R23, (4*11)(R29) + MOVW g, (4*12)(R29) + MOVW R31, (4*13)(R29) + + MOVD F20, (4*14)(R29) + MOVD F22, (4*14+8*1)(R29) + MOVD F24, (4*14+8*2)(R29) + MOVD F26, (4*14+8*3)(R29) + MOVD F28, (4*14+8*4)(R29) + MOVD F30, (4*14+8*5)(R29) + + JAL runtime·load_g(SB) + JAL (R4) + + MOVW (4*4)(R29), R16 + MOVW (4*5)(R29), R17 + MOVW (4*6)(R29), R18 + MOVW (4*7)(R29), R19 + MOVW (4*8)(R29), R20 + MOVW (4*9)(R29), R21 + MOVW (4*10)(R29), R22 + MOVW (4*11)(R29), R23 + MOVW (4*12)(R29), g + MOVW (4*13)(R29), R31 + + MOVD (4*14)(R29), F20 + MOVD (4*14+8*1)(R29), F22 + MOVD (4*14+8*2)(R29), F24 + MOVD (4*14+8*3)(R29), F26 + MOVD (4*14+8*4)(R29), F28 + MOVD (4*14+8*5)(R29), F30 + + ADDU $(4*14+8*6-16), R29 + RET diff --git a/src/runtime/cgo/gcc_darwin_386.c b/src/runtime/cgo/gcc_darwin_386.c index effbcdfd4b..83092dbeac 100644 --- a/src/runtime/cgo/gcc_darwin_386.c +++ b/src/runtime/cgo/gcc_darwin_386.c @@ -6,6 +6,7 @@ #include <pthread.h> #include <signal.h> #include "libcgo.h" +#include "libcgo_unix.h" static void* threadentry(void*); static pthread_key_t k1; @@ -123,7 +124,7 @@ _cgo_sys_thread_start(ThreadStart *ts) pthread_attr_getstacksize(&attr, &size); // Leave stacklo=0 and set stackhi=size; mstack will do the rest. ts->g->stackhi = size; - err = pthread_create(&p, &attr, threadentry, ts); + err = _cgo_try_pthread_create(&p, &attr, threadentry, ts); pthread_sigmask(SIG_SETMASK, &oset, nil); diff --git a/src/runtime/cgo/gcc_darwin_amd64.c b/src/runtime/cgo/gcc_darwin_amd64.c index 15396b0d25..93a6b8e3ed 100644 --- a/src/runtime/cgo/gcc_darwin_amd64.c +++ b/src/runtime/cgo/gcc_darwin_amd64.c @@ -6,6 +6,7 @@ #include <pthread.h> #include <signal.h> #include "libcgo.h" +#include "libcgo_unix.h" static void* threadentry(void*); static pthread_key_t k1; @@ -94,7 +95,7 @@ _cgo_sys_thread_start(ThreadStart *ts) pthread_attr_getstacksize(&attr, &size); // Leave stacklo=0 and set stackhi=size; mstack will do the rest. ts->g->stackhi = size; - err = pthread_create(&p, &attr, threadentry, ts); + err = _cgo_try_pthread_create(&p, &attr, threadentry, ts); pthread_sigmask(SIG_SETMASK, &oset, nil); diff --git a/src/runtime/cgo/gcc_darwin_arm.c b/src/runtime/cgo/gcc_darwin_arm.c index dbf88c34ac..b3f8046011 100644 --- a/src/runtime/cgo/gcc_darwin_arm.c +++ b/src/runtime/cgo/gcc_darwin_arm.c @@ -10,6 +10,7 @@ #include <unistd.h> #include "libcgo.h" +#include "libcgo_unix.h" #include <CoreFoundation/CFBundle.h> #include <CoreFoundation/CFString.h> @@ -65,7 +66,7 @@ _cgo_sys_thread_start(ThreadStart *ts) pthread_attr_getstacksize(&attr, &size); // Leave stacklo=0 and set stackhi=size; mstack will do the rest. ts->g->stackhi = size; - err = pthread_create(&p, &attr, threadentry, ts); + err = _cgo_try_pthread_create(&p, &attr, threadentry, ts); pthread_sigmask(SIG_SETMASK, &oset, nil); diff --git a/src/runtime/cgo/gcc_darwin_arm64.c b/src/runtime/cgo/gcc_darwin_arm64.c index a9eb4f2cd2..039dcc02bd 100644 --- a/src/runtime/cgo/gcc_darwin_arm64.c +++ b/src/runtime/cgo/gcc_darwin_arm64.c @@ -11,6 +11,7 @@ #include <stdlib.h> #include "libcgo.h" +#include "libcgo_unix.h" #include <CoreFoundation/CFBundle.h> #include <CoreFoundation/CFString.h> @@ -67,7 +68,7 @@ _cgo_sys_thread_start(ThreadStart *ts) pthread_attr_getstacksize(&attr, &size); // Leave stacklo=0 and set stackhi=size; mstack will do the rest. ts->g->stackhi = size; - err = pthread_create(&p, &attr, threadentry, ts); + err = _cgo_try_pthread_create(&p, &attr, threadentry, ts); pthread_sigmask(SIG_SETMASK, &oset, nil); diff --git a/src/runtime/cgo/gcc_dragonfly_amd64.c b/src/runtime/cgo/gcc_dragonfly_amd64.c index e532ad69d6..bdfbf6b561 100644 --- a/src/runtime/cgo/gcc_dragonfly_amd64.c +++ b/src/runtime/cgo/gcc_dragonfly_amd64.c @@ -8,6 +8,7 @@ #include <signal.h> #include <string.h> #include "libcgo.h" +#include "libcgo_unix.h" static void* threadentry(void*); static void (*setg_gcc)(void*); @@ -42,7 +43,7 @@ _cgo_sys_thread_start(ThreadStart *ts) // Leave stacklo=0 and set stackhi=size; mstack will do the rest. ts->g->stackhi = size; - err = pthread_create(&p, &attr, threadentry, ts); + err = _cgo_try_pthread_create(&p, &attr, threadentry, ts); pthread_sigmask(SIG_SETMASK, &oset, nil); diff --git a/src/runtime/cgo/gcc_freebsd_386.c b/src/runtime/cgo/gcc_freebsd_386.c index d288666a3d..c6d4f258c0 100644 --- a/src/runtime/cgo/gcc_freebsd_386.c +++ b/src/runtime/cgo/gcc_freebsd_386.c @@ -8,6 +8,7 @@ #include <signal.h> #include <string.h> #include "libcgo.h" +#include "libcgo_unix.h" static void* threadentry(void*); static void (*setg_gcc)(void*); @@ -42,7 +43,7 @@ _cgo_sys_thread_start(ThreadStart *ts) pthread_attr_getstacksize(&attr, &size); // Leave stacklo=0 and set stackhi=size; mstack will do the rest. ts->g->stackhi = size; - err = pthread_create(&p, &attr, threadentry, ts); + err = _cgo_try_pthread_create(&p, &attr, threadentry, ts); pthread_sigmask(SIG_SETMASK, &oset, nil); diff --git a/src/runtime/cgo/gcc_freebsd_amd64.c b/src/runtime/cgo/gcc_freebsd_amd64.c index e532ad69d6..bdfbf6b561 100644 --- a/src/runtime/cgo/gcc_freebsd_amd64.c +++ b/src/runtime/cgo/gcc_freebsd_amd64.c @@ -8,6 +8,7 @@ #include <signal.h> #include <string.h> #include "libcgo.h" +#include "libcgo_unix.h" static void* threadentry(void*); static void (*setg_gcc)(void*); @@ -42,7 +43,7 @@ _cgo_sys_thread_start(ThreadStart *ts) // Leave stacklo=0 and set stackhi=size; mstack will do the rest. ts->g->stackhi = size; - err = pthread_create(&p, &attr, threadentry, ts); + err = _cgo_try_pthread_create(&p, &attr, threadentry, ts); pthread_sigmask(SIG_SETMASK, &oset, nil); diff --git a/src/runtime/cgo/gcc_freebsd_arm.c b/src/runtime/cgo/gcc_freebsd_arm.c index c4e7574326..746ca89322 100644 --- a/src/runtime/cgo/gcc_freebsd_arm.c +++ b/src/runtime/cgo/gcc_freebsd_arm.c @@ -9,6 +9,7 @@ #include <signal.h> #include <string.h> #include "libcgo.h" +#include "libcgo_unix.h" #ifdef ARM_TP_ADDRESS // ARM_TP_ADDRESS is (ARM_VECTORS_HIGH + 0x1000) or 0xffff1000 @@ -58,7 +59,7 @@ _cgo_sys_thread_start(ThreadStart *ts) pthread_attr_getstacksize(&attr, &size); // Leave stacklo=0 and set stackhi=size; mstack will do the rest. ts->g->stackhi = size; - err = pthread_create(&p, &attr, threadentry, ts); + err = _cgo_try_pthread_create(&p, &attr, threadentry, ts); pthread_sigmask(SIG_SETMASK, &oset, nil); diff --git a/src/runtime/cgo/gcc_libinit.c b/src/runtime/cgo/gcc_libinit.c index 0bdf40a4ca..f6fbaa3f01 100644 --- a/src/runtime/cgo/gcc_libinit.c +++ b/src/runtime/cgo/gcc_libinit.c @@ -6,10 +6,13 @@ // +build darwin dragonfly freebsd linux netbsd solaris #include <pthread.h> +#include <errno.h> #include <stdio.h> #include <stdlib.h> #include <string.h> // strerror +#include <time.h> #include "libcgo.h" +#include "libcgo_unix.h" static pthread_cond_t runtime_init_cond = PTHREAD_COND_INITIALIZER; static pthread_mutex_t runtime_init_mu = PTHREAD_MUTEX_INITIALIZER; @@ -21,7 +24,7 @@ static void (*cgo_context_function)(struct context_arg*); void x_cgo_sys_thread_create(void* (*func)(void*), void* arg) { pthread_t p; - int err = pthread_create(&p, NULL, func, arg); + int err = _cgo_try_pthread_create(&p, NULL, func, arg); if (err != 0) { fprintf(stderr, "pthread_create failed: %s", strerror(err)); abort(); @@ -84,3 +87,23 @@ void (*(_cgo_get_context_function(void)))(struct context_arg*) { pthread_mutex_unlock(&runtime_init_mu); return ret; } + +// _cgo_try_pthread_create retries pthread_create if it fails with +// EAGAIN. +int +_cgo_try_pthread_create(pthread_t* thread, const pthread_attr_t* attr, void* (*pfn)(void*), void* arg) { + int tries; + int err; + struct timespec ts; + + for (tries = 0; tries < 20; tries++) { + err = pthread_create(thread, attr, pfn, arg); + if (err != EAGAIN) { + return err; + } + ts.tv_sec = 0; + ts.tv_nsec = (tries + 1) * 1000 * 1000; // Milliseconds. + nanosleep(&ts, nil); + } + return EAGAIN; +} diff --git a/src/runtime/cgo/gcc_libinit_openbsd.c b/src/runtime/cgo/gcc_libinit_openbsd.c index 626bf8adca..c8308e54c3 100644 --- a/src/runtime/cgo/gcc_libinit_openbsd.c +++ b/src/runtime/cgo/gcc_libinit_openbsd.c @@ -2,6 +2,9 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +#include <sys/types.h> +#include <errno.h> +#include <pthread.h> #include <stdio.h> #include <stdlib.h> #include "libcgo.h" @@ -48,3 +51,24 @@ void x_cgo_set_context_function(void (*context)(struct context_arg*)) { void (*(_cgo_get_context_function(void)))(struct context_arg*) { return cgo_context_function; } + +// _cgo_try_pthread_create retries sys_pthread_create if it fails with +// EAGAIN. +int +_cgo_openbsd_try_pthread_create(int (*sys_pthread_create)(pthread_t*, const pthread_attr_t*, void* (*)(void*), void*), + pthread_t* thread, const pthread_attr_t* attr, void* (*pfn)(void*), void* arg) { + int tries; + int err; + struct timespec ts; + + for (tries = 0; tries < 100; tries++) { + err = sys_pthread_create(thread, attr, pfn, arg); + if (err != EAGAIN) { + return err; + } + ts.tv_sec = 0; + ts.tv_nsec = (tries + 1) * 1000 * 1000; // Milliseconds. + nanosleep(&ts, nil); + } + return EAGAIN; +} diff --git a/src/runtime/cgo/gcc_linux_386.c b/src/runtime/cgo/gcc_linux_386.c index 30fe92bfea..457a2c7e3a 100644 --- a/src/runtime/cgo/gcc_linux_386.c +++ b/src/runtime/cgo/gcc_linux_386.c @@ -6,6 +6,7 @@ #include <string.h> #include <signal.h> #include "libcgo.h" +#include "libcgo_unix.h" static void *threadentry(void*); static void (*setg_gcc)(void*); @@ -53,7 +54,7 @@ _cgo_sys_thread_start(ThreadStart *ts) pthread_attr_getstacksize(&attr, &size); // Leave stacklo=0 and set stackhi=size; mstack will do the rest. ts->g->stackhi = size; - err = pthread_create(&p, &attr, threadentry, ts); + err = _cgo_try_pthread_create(&p, &attr, threadentry, ts); pthread_sigmask(SIG_SETMASK, &oset, nil); diff --git a/src/runtime/cgo/gcc_linux_amd64.c b/src/runtime/cgo/gcc_linux_amd64.c index 0c34c66592..5d8ff10140 100644 --- a/src/runtime/cgo/gcc_linux_amd64.c +++ b/src/runtime/cgo/gcc_linux_amd64.c @@ -8,6 +8,7 @@ #include <signal.h> #include <stdlib.h> #include "libcgo.h" +#include "libcgo_unix.h" static void* threadentry(void*); static void (*setg_gcc)(void*); @@ -70,7 +71,7 @@ _cgo_sys_thread_start(ThreadStart *ts) pthread_attr_getstacksize(&attr, &size); // Leave stacklo=0 and set stackhi=size; mstack will do the rest. ts->g->stackhi = size; - err = pthread_create(&p, &attr, threadentry, ts); + err = _cgo_try_pthread_create(&p, &attr, threadentry, ts); pthread_sigmask(SIG_SETMASK, &oset, nil); diff --git a/src/runtime/cgo/gcc_linux_arm.c b/src/runtime/cgo/gcc_linux_arm.c index 945c3f19e4..31ced5e03c 100644 --- a/src/runtime/cgo/gcc_linux_arm.c +++ b/src/runtime/cgo/gcc_linux_arm.c @@ -6,6 +6,7 @@ #include <string.h> #include <signal.h> #include "libcgo.h" +#include "libcgo_unix.h" static void *threadentry(void*); @@ -33,7 +34,7 @@ _cgo_sys_thread_start(ThreadStart *ts) pthread_attr_getstacksize(&attr, &size); // Leave stacklo=0 and set stackhi=size; mstack will do the rest. ts->g->stackhi = size; - err = pthread_create(&p, &attr, threadentry, ts); + err = _cgo_try_pthread_create(&p, &attr, threadentry, ts); pthread_sigmask(SIG_SETMASK, &oset, nil); diff --git a/src/runtime/cgo/gcc_linux_arm64.c b/src/runtime/cgo/gcc_linux_arm64.c index ca9ba0ba6e..35b8e27967 100644 --- a/src/runtime/cgo/gcc_linux_arm64.c +++ b/src/runtime/cgo/gcc_linux_arm64.c @@ -6,6 +6,7 @@ #include <string.h> #include <signal.h> #include "libcgo.h" +#include "libcgo_unix.h" static void *threadentry(void*); @@ -33,7 +34,7 @@ _cgo_sys_thread_start(ThreadStart *ts) pthread_attr_getstacksize(&attr, &size); // Leave stacklo=0 and set stackhi=size; mstack will do the rest. ts->g->stackhi = size; - err = pthread_create(&p, &attr, threadentry, ts); + err = _cgo_try_pthread_create(&p, &attr, threadentry, ts); pthread_sigmask(SIG_SETMASK, &oset, nil); diff --git a/src/runtime/cgo/gcc_linux_mips64x.c b/src/runtime/cgo/gcc_linux_mips64x.c index 8a95629f56..e0ce08f4e5 100644 --- a/src/runtime/cgo/gcc_linux_mips64x.c +++ b/src/runtime/cgo/gcc_linux_mips64x.c @@ -10,6 +10,7 @@ #include <string.h> #include <signal.h> #include "libcgo.h" +#include "libcgo_unix.h" static void *threadentry(void*); @@ -37,7 +38,7 @@ _cgo_sys_thread_start(ThreadStart *ts) pthread_attr_getstacksize(&attr, &size); // Leave stacklo=0 and set stackhi=size; mstack will do the rest. ts->g->stackhi = size; - err = pthread_create(&p, &attr, threadentry, ts); + err = _cgo_try_pthread_create(&p, &attr, threadentry, ts); pthread_sigmask(SIG_SETMASK, &oset, nil); diff --git a/src/runtime/cgo/gcc_linux_mipsx.c b/src/runtime/cgo/gcc_linux_mipsx.c new file mode 100644 index 0000000000..7ed9d87575 --- /dev/null +++ b/src/runtime/cgo/gcc_linux_mipsx.c @@ -0,0 +1,80 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build cgo +// +build linux +// +build mips mipsle + +#include <pthread.h> +#include <string.h> +#include <signal.h> +#include "libcgo.h" +#include "libcgo_unix.h" + +static void *threadentry(void*); + +void (*x_cgo_inittls)(void **tlsg, void **tlsbase); +void (*setg_gcc)(void*); + +void +_cgo_sys_thread_start(ThreadStart *ts) +{ + pthread_attr_t attr; + sigset_t ign, oset; + pthread_t p; + size_t size; + int err; + + sigfillset(&ign); + pthread_sigmask(SIG_SETMASK, &ign, &oset); + + // Not sure why the memset is necessary here, + // but without it, we get a bogus stack size + // out of pthread_attr_getstacksize. C'est la Linux. + memset(&attr, 0, sizeof attr); + pthread_attr_init(&attr); + size = 0; + pthread_attr_getstacksize(&attr, &size); + // Leave stacklo=0 and set stackhi=size; mstack will do the rest. + ts->g->stackhi = size; + err = _cgo_try_pthread_create(&p, &attr, threadentry, ts); + + pthread_sigmask(SIG_SETMASK, &oset, nil); + + if (err != 0) { + fatalf("pthread_create failed: %s", strerror(err)); + } +} + +extern void crosscall1(void (*fn)(void), void (*setg_gcc)(void*), void *g); +static void* +threadentry(void *v) +{ + ThreadStart ts; + + ts = *(ThreadStart*)v; + free(v); + + crosscall1(ts.fn, setg_gcc, (void*)ts.g); + return nil; +} + +void +x_cgo_init(G *g, void (*setg)(void*), void **tlsg, void **tlsbase) +{ + pthread_attr_t attr; + size_t size; + + setg_gcc = setg; + + memset(&attr, 0, sizeof attr); + pthread_attr_init(&attr); + pthread_attr_getstacksize(&attr, &size); + g->stacklo = (uintptr)&attr - size + 4096; + pthread_attr_destroy(&attr); + + if (x_cgo_inittls) { + x_cgo_inittls(tlsg, tlsbase); + } +} diff --git a/src/runtime/cgo/gcc_linux_ppc64x.c b/src/runtime/cgo/gcc_linux_ppc64x.c index fb19805bda..fcf77cfe47 100644 --- a/src/runtime/cgo/gcc_linux_ppc64x.c +++ b/src/runtime/cgo/gcc_linux_ppc64x.c @@ -8,6 +8,7 @@ #include <string.h> #include <signal.h> #include "libcgo.h" +#include "libcgo_unix.h" static void *threadentry(void*); @@ -43,7 +44,7 @@ _cgo_sys_thread_start(ThreadStart *ts) pthread_attr_getstacksize(&attr, &size); // Leave stacklo=0 and set stackhi=size; mstack will do the rest. ts->g->stackhi = size; - err = pthread_create(&p, &attr, threadentry, ts); + err = _cgo_try_pthread_create(&p, &attr, threadentry, ts); pthread_sigmask(SIG_SETMASK, &oset, nil); diff --git a/src/runtime/cgo/gcc_linux_s390x.c b/src/runtime/cgo/gcc_linux_s390x.c index 81e3b339b0..cdc9c23f49 100644 --- a/src/runtime/cgo/gcc_linux_s390x.c +++ b/src/runtime/cgo/gcc_linux_s390x.c @@ -6,6 +6,7 @@ #include <string.h> #include <signal.h> #include "libcgo.h" +#include "libcgo_unix.h" static void *threadentry(void*); @@ -41,7 +42,7 @@ _cgo_sys_thread_start(ThreadStart *ts) pthread_attr_getstacksize(&attr, &size); // Leave stacklo=0 and set stackhi=size; mstack will do the rest. ts->g->stackhi = size; - err = pthread_create(&p, &attr, threadentry, ts); + err = _cgo_try_pthread_create(&p, &attr, threadentry, ts); pthread_sigmask(SIG_SETMASK, &oset, nil); diff --git a/src/runtime/cgo/gcc_mipsx.S b/src/runtime/cgo/gcc_mipsx.S new file mode 100644 index 0000000000..c51c36a9b7 --- /dev/null +++ b/src/runtime/cgo/gcc_mipsx.S @@ -0,0 +1,68 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build mips mipsle + +/* + * void crosscall1(void (*fn)(void), void (*setg_gcc)(void *g), void *g) + * + * Calling into the gc tool chain, where all registers are caller save. + * Called from standard MIPS O32 ABI, where $16-$23, $30, and $f20-$f31 + * are callee-save, so they must be saved explicitly, along with $31 (LR). + */ +.globl crosscall1 +.set noat +crosscall1: + addiu $29, $29, -88 + + sw $31, 0($29) + sw $16, 4($29) + sw $17, 8($29) + sw $18, 12($29) + sw $19, 16($29) + sw $20, 20($29) + sw $21, 24($29) + sw $22, 28($29) + sw $23, 32($29) + sw $30, 36($29) + + sdc1 $f20, 40($29) + sdc1 $f22, 48($29) + sdc1 $f24, 56($29) + sdc1 $f26, 64($29) + sdc1 $f28, 72($29) + sdc1 $f30, 80($29) + + + move $20, $4 // save R4 + move $4, $6 + jalr $5 // call setg_gcc + jalr $20 // call fn + + lw $16, 4($29) + lw $17, 8($29) + lw $18, 12($29) + lw $19, 16($29) + lw $20, 20($29) + lw $21, 24($29) + lw $22, 28($29) + lw $23, 32($29) + lw $30, 36($29) + ldc1 $f20, 40($29) + ldc1 $f22, 48($29) + ldc1 $f24, 56($29) + ldc1 $f26, 64($29) + ldc1 $f28, 72($29) + ldc1 $f30, 80($29) + + lw $31, 0($29) + + addiu $29, $29, 88 + jr $31 + +.set at + +#ifdef __ELF__ +.section .note.GNU-stack,"",%progbits +#endif diff --git a/src/runtime/cgo/gcc_netbsd_386.c b/src/runtime/cgo/gcc_netbsd_386.c index 99558ea140..fb317c1c68 100644 --- a/src/runtime/cgo/gcc_netbsd_386.c +++ b/src/runtime/cgo/gcc_netbsd_386.c @@ -7,6 +7,7 @@ #include <signal.h> #include <string.h> #include "libcgo.h" +#include "libcgo_unix.h" static void* threadentry(void*); static void (*setg_gcc)(void*); @@ -41,7 +42,7 @@ _cgo_sys_thread_start(ThreadStart *ts) pthread_attr_getstacksize(&attr, &size); // Leave stacklo=0 and set stackhi=size; mstack will do the rest. ts->g->stackhi = size; - err = pthread_create(&p, &attr, threadentry, ts); + err = _cgo_try_pthread_create(&p, &attr, threadentry, ts); pthread_sigmask(SIG_SETMASK, &oset, nil); diff --git a/src/runtime/cgo/gcc_netbsd_amd64.c b/src/runtime/cgo/gcc_netbsd_amd64.c index f5c8b1e74f..77a553f5fa 100644 --- a/src/runtime/cgo/gcc_netbsd_amd64.c +++ b/src/runtime/cgo/gcc_netbsd_amd64.c @@ -7,6 +7,7 @@ #include <signal.h> #include <string.h> #include "libcgo.h" +#include "libcgo_unix.h" static void* threadentry(void*); static void (*setg_gcc)(void*); @@ -42,7 +43,7 @@ _cgo_sys_thread_start(ThreadStart *ts) // Leave stacklo=0 and set stackhi=size; mstack will do the rest. ts->g->stackhi = size; - err = pthread_create(&p, &attr, threadentry, ts); + err = _cgo_try_pthread_create(&p, &attr, threadentry, ts); pthread_sigmask(SIG_SETMASK, &oset, nil); diff --git a/src/runtime/cgo/gcc_netbsd_arm.c b/src/runtime/cgo/gcc_netbsd_arm.c index 97ce908485..672f49c3d8 100644 --- a/src/runtime/cgo/gcc_netbsd_arm.c +++ b/src/runtime/cgo/gcc_netbsd_arm.c @@ -7,6 +7,7 @@ #include <signal.h> #include <string.h> #include "libcgo.h" +#include "libcgo_unix.h" static void *threadentry(void*); @@ -42,7 +43,7 @@ _cgo_sys_thread_start(ThreadStart *ts) pthread_attr_getstacksize(&attr, &size); // Leave stacklo=0 and set stackhi=size; mstack will do the rest. ts->g->stackhi = size; - err = pthread_create(&p, &attr, threadentry, ts); + err = _cgo_try_pthread_create(&p, &attr, threadentry, ts); pthread_sigmask(SIG_SETMASK, &oset, nil); diff --git a/src/runtime/cgo/gcc_openbsd_386.c b/src/runtime/cgo/gcc_openbsd_386.c index 1bc61ff708..0cac047ad3 100644 --- a/src/runtime/cgo/gcc_openbsd_386.c +++ b/src/runtime/cgo/gcc_openbsd_386.c @@ -9,6 +9,7 @@ #include <signal.h> #include <string.h> #include "libcgo.h" +#include "libcgo_unix.h" static void* threadentry(void*); static void (*setg_gcc)(void*); @@ -170,7 +171,7 @@ _cgo_sys_thread_start(ThreadStart *ts) // Leave stacklo=0 and set stackhi=size; mstack will do the rest. ts->g->stackhi = size; - err = sys_pthread_create(&p, &attr, threadentry, ts); + err = _cgo_openbsd_try_pthread_create(sys_pthread_create, &p, &attr, threadentry, ts); pthread_sigmask(SIG_SETMASK, &oset, nil); diff --git a/src/runtime/cgo/gcc_openbsd_amd64.c b/src/runtime/cgo/gcc_openbsd_amd64.c index 4d4d14314c..86a9185a37 100644 --- a/src/runtime/cgo/gcc_openbsd_amd64.c +++ b/src/runtime/cgo/gcc_openbsd_amd64.c @@ -9,6 +9,7 @@ #include <signal.h> #include <string.h> #include "libcgo.h" +#include "libcgo_unix.h" static void* threadentry(void*); static void (*setg_gcc)(void*); @@ -170,7 +171,7 @@ _cgo_sys_thread_start(ThreadStart *ts) // Leave stacklo=0 and set stackhi=size; mstack will do the rest. ts->g->stackhi = size; - err = sys_pthread_create(&p, &attr, threadentry, ts); + err = _cgo_openbsd_try_pthread_create(sys_pthread_create, &p, &attr, threadentry, ts); pthread_sigmask(SIG_SETMASK, &oset, nil); diff --git a/src/runtime/cgo/gcc_signal_darwin_armx.c b/src/runtime/cgo/gcc_signal_darwin_armx.c index 02c54d80a2..a2d520bce8 100644 --- a/src/runtime/cgo/gcc_signal_darwin_armx.c +++ b/src/runtime/cgo/gcc_signal_darwin_armx.c @@ -37,6 +37,7 @@ #include <mach/thread_status.h> #include "libcgo.h" +#include "libcgo_unix.h" uintptr_t x_cgo_panicmem; @@ -201,7 +202,7 @@ darwin_arm_init_mach_exception_handler() uintptr_t port_set = (uintptr_t)mach_exception_handler_port_set; pthread_attr_init(&attr); pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); - ret = pthread_create(&thr, &attr, mach_exception_handler, (void*)port_set); + ret = _cgo_try_pthread_create(&thr, &attr, mach_exception_handler, (void*)port_set); pthread_sigmask(SIG_SETMASK, &oset, nil); diff --git a/src/runtime/cgo/gcc_solaris_amd64.c b/src/runtime/cgo/gcc_solaris_amd64.c index 98a1a8be53..079bd12898 100644 --- a/src/runtime/cgo/gcc_solaris_amd64.c +++ b/src/runtime/cgo/gcc_solaris_amd64.c @@ -7,6 +7,7 @@ #include <signal.h> #include <ucontext.h> #include "libcgo.h" +#include "libcgo_unix.h" static void* threadentry(void*); static void (*setg_gcc)(void*); @@ -53,7 +54,7 @@ _cgo_sys_thread_start(ThreadStart *ts) ts->g->stackhi = size; } pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); - err = pthread_create(&p, &attr, threadentry, ts); + err = _cgo_try_pthread_create(&p, &attr, threadentry, ts); pthread_sigmask(SIG_SETMASK, &oset, nil); diff --git a/src/runtime/cgo/libcgo_unix.h b/src/runtime/cgo/libcgo_unix.h new file mode 100644 index 0000000000..a56a366f23 --- /dev/null +++ b/src/runtime/cgo/libcgo_unix.h @@ -0,0 +1,15 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* + * Call pthread_create, retrying on EAGAIN. + */ +extern int _cgo_try_pthread_create(pthread_t*, const pthread_attr_t*, void* (*)(void*), void*); + +/* + * Same as _cgo_try_pthread_create, but passing on the pthread_create function. + * Only defined on OpenBSD. + */ +extern int _cgo_openbsd_try_pthread_create(int (*)(pthread_t*, const pthread_attr_t*, void *(*pfn)(void*), void*), + pthread_t*, const pthread_attr_t*, void* (*)(void*), void* arg); diff --git a/src/runtime/cgocall.go b/src/runtime/cgocall.go index 007406b426..69e29ef976 100644 --- a/src/runtime/cgocall.go +++ b/src/runtime/cgocall.go @@ -286,6 +286,10 @@ func cgocallbackg1(ctxt uintptr) { // On mips64x, stack frame is two words and there's a saved LR between // SP and the stack frame and between the stack frame and the arguments. cb = (*args)(unsafe.Pointer(sp + 4*sys.PtrSize)) + case "mips", "mipsle": + // On mipsx, stack frame is two words and there's a saved LR between + // SP and the stack frame and between the stack frame and the arguments. + cb = (*args)(unsafe.Pointer(sp + 4*sys.PtrSize)) } // Invoke callback. @@ -323,7 +327,7 @@ func unwindm(restore *bool) { switch GOARCH { default: throw("unwindm not implemented") - case "386", "amd64", "arm", "ppc64", "ppc64le", "mips64", "mips64le", "s390x": + case "386", "amd64", "arm", "ppc64", "ppc64le", "mips64", "mips64le", "s390x", "mips", "mipsle": sched.sp = *(*uintptr)(unsafe.Pointer(sched.sp + sys.MinFrameSize)) case "arm64": sched.sp = *(*uintptr)(unsafe.Pointer(sched.sp + 16)) diff --git a/src/runtime/crash_cgo_test.go b/src/runtime/crash_cgo_test.go index 7014f119ad..347b820eb5 100644 --- a/src/runtime/crash_cgo_test.go +++ b/src/runtime/crash_cgo_test.go @@ -49,8 +49,6 @@ func TestCgoCallbackGC(t *testing.T) { switch runtime.GOOS { case "plan9", "windows": t.Skipf("no pthreads on %s", runtime.GOOS) - case "freebsd": - testenv.SkipFlaky(t, 16396) } if testing.Short() { switch { diff --git a/src/runtime/fastlog2.go b/src/runtime/fastlog2.go index b22e8259ad..5f3fb53423 100644 --- a/src/runtime/fastlog2.go +++ b/src/runtime/fastlog2.go @@ -8,7 +8,7 @@ import "unsafe" // fastlog2 implements a fast approximation to the base 2 log of a // float64. This is used to compute a geometric distribution for heap -// sampling, without introducing dependences into package math. This +// sampling, without introducing dependencies into package math. This // uses a very rough approximation using the float64 exponent and the // first 25 bits of the mantissa. The top 5 bits of the mantissa are // used to load limits from a table of constants and the rest are used @@ -29,5 +29,5 @@ func fastlog2(x float64) float64 { } // float64bits returns the IEEE 754 binary representation of f. -// Taken from math.Float64bits to avoid dependences into package math. +// Taken from math.Float64bits to avoid dependencies into package math. func float64bits(f float64) uint64 { return *(*uint64)(unsafe.Pointer(&f)) } diff --git a/src/runtime/iface.go b/src/runtime/iface.go index c932e149dd..46010d58fc 100644 --- a/src/runtime/iface.go +++ b/src/runtime/iface.go @@ -138,11 +138,8 @@ func additab(m *itab, locked, canfail bool) { throw("invalid itab locking") } h := itabhash(inter, typ) - if m == hash[h] { - println("duplicate itab for", typ.string(), "and", inter.typ.string()) - throw("duplicate itabs") - } m.link = hash[h] + m.inhash = 1 atomicstorep(unsafe.Pointer(&hash[h]), unsafe.Pointer(m)) } @@ -150,7 +147,14 @@ func itabsinit() { lock(&ifaceLock) for _, md := range activeModules() { for _, i := range md.itablinks { - additab(i, true, false) + // itablinks is a slice of pointers to the itabs used in this + // module. A given itab may be used in more than one module + // and thanks to the way global symbol resolution works, the + // pointed-to itab may already have been inserted into the + // global 'hash'. + if i.inhash == 0 { + additab(i, true, false) + } } } unlock(&ifaceLock) @@ -179,7 +183,7 @@ func panicnildottype(want *_type) { // The conv and assert functions below do very similar things. // The convXXX functions are guaranteed by the compiler to succeed. -// The assertXXX functions may fail (either panicing or returning false, +// The assertXXX functions may fail (either panicking or returning false, // depending on whether they are 1-result or 2-result). // The convXXX functions succeed on a nil input, whereas the assertXXX // functions fail on a nil input. diff --git a/src/runtime/malloc.go b/src/runtime/malloc.go index 56585de86a..837f3ea801 100644 --- a/src/runtime/malloc.go +++ b/src/runtime/malloc.go @@ -157,6 +157,13 @@ const ( _MaxGcproc = 32 _MaxArena32 = 1<<32 - 1 + + // minLegalPointer is the smallest possible legal pointer. + // This is the smallest possible architectural page size, + // since we assume that the first page is never mapped. + // + // This should agree with minZeroPage in the compiler. + minLegalPointer uintptr = 4096 ) // physPageSize is the size in bytes of the OS's physical pages. diff --git a/src/runtime/malloc_test.go b/src/runtime/malloc_test.go index 767b51f453..0cf9cfbf42 100644 --- a/src/runtime/malloc_test.go +++ b/src/runtime/malloc_test.go @@ -13,6 +13,9 @@ import ( ) func TestMemStats(t *testing.T) { + // Make sure there's at least one forced GC. + GC() + // Test that MemStats has sane values. st := new(MemStats) ReadMemStats(st) @@ -24,7 +27,7 @@ func TestMemStats(t *testing.T) { st.HeapInuse == 0 || st.HeapObjects == 0 || st.StackInuse == 0 || st.StackSys == 0 || st.MSpanInuse == 0 || st.MSpanSys == 0 || st.MCacheInuse == 0 || st.MCacheSys == 0 || st.BuckHashSys == 0 || st.GCSys == 0 || st.OtherSys == 0 || - st.NextGC == 0 { + st.NextGC == 0 || st.NumForcedGC == 0 { t.Fatalf("Zero value: %+v", *st) } @@ -33,7 +36,7 @@ func TestMemStats(t *testing.T) { st.HeapIdle > 1e10 || st.HeapInuse > 1e10 || st.HeapObjects > 1e10 || st.StackInuse > 1e10 || st.StackSys > 1e10 || st.MSpanInuse > 1e10 || st.MSpanSys > 1e10 || st.MCacheInuse > 1e10 || st.MCacheSys > 1e10 || st.BuckHashSys > 1e10 || st.GCSys > 1e10 || st.OtherSys > 1e10 || - st.NextGC > 1e10 || st.NumGC > 1e9 || st.PauseTotalNs > 1e11 { + st.NextGC > 1e10 || st.NumGC > 1e9 || st.NumForcedGC > 1e9 || st.PauseTotalNs > 1e11 { t.Fatalf("Insanely high value (overflow?): %+v", *st) } @@ -72,6 +75,10 @@ func TestMemStats(t *testing.T) { t.Fatalf("PauseTotalNs(%d) < sum PauseNs(%d)", st.PauseTotalNs, pauseTotal) } } + + if st.NumForcedGC > st.NumGC { + t.Fatalf("NumForcedGC(%d) > NumGC(%d)", st.NumForcedGC, st.NumGC) + } } func TestStringConcatenationAllocs(t *testing.T) { diff --git a/src/runtime/mgc.go b/src/runtime/mgc.go index f1227c14fd..0117044baf 100644 --- a/src/runtime/mgc.go +++ b/src/runtime/mgc.go @@ -2,9 +2,6 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// TODO(rsc): The code having to do with the heap bitmap needs very serious cleanup. -// It has gotten completely out of control. - // Garbage collector (GC). // // The GC runs concurrently with mutator threads, is type accurate (aka precise), allows multiple @@ -24,67 +21,73 @@ // Hudson, R., and Moss, J.E.B. Copying Garbage Collection without stopping the world. // Concurrency and Computation: Practice and Experience 15(3-5), 2003. // -// TODO(austin): The rest of this comment is woefully out of date and -// needs to be rewritten. There is no distinct scan phase any more and -// we allocate black during GC. +// 1. GC performs sweep termination. +// +// a. Stop the world. This causes all Ps to reach a GC safe-point. +// +// b. Sweep any unswept spans. There will only be unswept spans if +// this GC cycle was forced before the expected time. +// +// 2. GC performs the "mark 1" sub-phase. In this sub-phase, Ps are +// allowed to locally cache parts of the work queue. +// +// a. Prepare for the mark phase by setting gcphase to _GCmark +// (from _GCoff), enabling the write barrier, enabling mutator +// assists, and enqueueing root mark jobs. No objects may be +// scanned until all Ps have enabled the write barrier, which is +// accomplished using STW. +// +// b. Start the world. From this point, GC work is done by mark +// workers started by the scheduler and by assists performed as +// part of allocation. The write barrier shades both the +// overwritten pointer and the new pointer value for any pointer +// writes (see mbarrier.go for details). Newly allocated objects +// are immediately marked black. +// +// c. GC performs root marking jobs. This includes scanning all +// stacks, shading all globals, and shading any heap pointers in +// off-heap runtime data structures. Scanning a stack stops a +// goroutine, shades any pointers found on its stack, and then +// resumes the goroutine. +// +// d. GC drains the work queue of grey objects, scanning each grey +// object to black and shading all pointers found in the object +// (which in turn may add those pointers to the work queue). +// +// 3. Once the global work queue is empty (but local work queue caches +// may still contain work), GC performs the "mark 2" sub-phase. +// +// a. GC stops all workers, disables local work queue caches, +// flushes each P's local work queue cache to the global work queue +// cache, and reenables workers. +// +// b. GC again drains the work queue, as in 2d above. +// +// 4. Once the work queue is empty, GC performs mark termination. // -// 0. Set phase = GCscan from GCoff. -// 1. Wait for all P's to acknowledge phase change. -// At this point all goroutines have passed through a GC safepoint and -// know we are in the GCscan phase. -// 2. GC scans all goroutine stacks, mark and enqueues all encountered pointers -// (marking avoids most duplicate enqueuing but races may produce benign duplication). -// Preempted goroutines are scanned before P schedules next goroutine. -// 3. Set phase = GCmark. -// 4. Wait for all P's to acknowledge phase change. -// 5. Now write barrier marks and enqueues black, grey, or white to white pointers. -// Malloc still allocates white (non-marked) objects. -// 6. Meanwhile GC transitively walks the heap marking reachable objects. -// 7. When GC finishes marking heap, it preempts P's one-by-one and -// retakes partial wbufs (filled by write barrier or during a stack scan of the goroutine -// currently scheduled on the P). -// 8. Once the GC has exhausted all available marking work it sets phase = marktermination. -// 9. Wait for all P's to acknowledge phase change. -// 10. Malloc now allocates black objects, so number of unmarked reachable objects -// monotonically decreases. -// 11. GC preempts P's one-by-one taking partial wbufs and marks all unmarked yet -// reachable objects. -// 12. When GC completes a full cycle over P's and discovers no new grey -// objects, (which means all reachable objects are marked) set phase = GCoff. -// 13. Wait for all P's to acknowledge phase change. -// 14. Now malloc allocates white (but sweeps spans before use). -// Write barrier becomes nop. -// 15. GC does background sweeping, see description below. -// 16. When sufficient allocation has taken place replay the sequence starting at 0 above, -// see discussion of GC rate below. - -// Changing phases. -// Phases are changed by setting the gcphase to the next phase and possibly calling ackgcphase. -// All phase action must be benign in the presence of a change. -// Starting with GCoff -// GCoff to GCscan -// GSscan scans stacks and globals greying them and never marks an object black. -// Once all the P's are aware of the new phase they will scan gs on preemption. -// This means that the scanning of preempted gs can't start until all the Ps -// have acknowledged. -// When a stack is scanned, this phase also installs stack barriers to -// track how much of the stack has been active. -// This transition enables write barriers because stack barriers -// assume that writes to higher frames will be tracked by write -// barriers. Technically this only needs write barriers for writes -// to stack slots, but we enable write barriers in general. -// GCscan to GCmark -// In GCmark, work buffers are drained until there are no more -// pointers to scan. -// No scanning of objects (making them black) can happen until all -// Ps have enabled the write barrier, but that already happened in -// the transition to GCscan. -// GCmark to GCmarktermination -// The only change here is that we start allocating black so the Ps must acknowledge -// the change before we begin the termination algorithm -// GCmarktermination to GSsweep -// Object currently on the freelist must be marked black for this to work. -// Are things on the free lists black or white? How does the sweep phase work? +// a. Stop the world. +// +// b. Set gcphase to _GCmarktermination, and disable workers and +// assists. +// +// c. Drain any remaining work from the work queue (typically there +// will be none). +// +// d. Perform other housekeeping like flushing mcaches. +// +// 5. GC performs the sweep phase. +// +// a. Prepare for the sweep phase by setting gcphase to _GCoff, +// setting up sweep state and disabling the write barrier. +// +// b. Start the world. From this point on, newly allocated objects +// are white, and allocating sweeps spans before use if necessary. +// +// c. GC does concurrent sweeping in the background and in response +// to allocation. See description below. +// +// 6. When sufficient allocation has taken place, replay the sequence +// starting with 1 above. See discussion of GC rate below. // Concurrent sweep. // @@ -903,7 +906,7 @@ type gcMode int const ( gcBackgroundMode gcMode = iota // concurrent GC and sweep gcForceMode // stop-the-world GC now, concurrent sweep - gcForceBlockMode // stop-the-world GC now and STW sweep + gcForceBlockMode // stop-the-world GC now and STW sweep (forced by user) ) // gcShouldStart returns true if the exit condition for the _GCoff @@ -967,6 +970,9 @@ func gcStart(mode gcMode, forceTrigger bool) { } } + // For stats, check if this GC was forced by the user. + forced := mode != gcBackgroundMode + // In gcstoptheworld debug mode, upgrade the mode accordingly. // We do this after re-checking the transition condition so // that multiple goroutines that detect the heap trigger don't @@ -1071,6 +1077,10 @@ func gcStart(mode gcMode, forceTrigger bool) { work.tMark, work.tMarkTerm = t, t work.heapGoal = work.heap0 + if forced { + memstats.numforcedgc++ + } + // Perform mark termination. This will restart the world. gcMarkTermination() } diff --git a/src/runtime/mgcsweep.go b/src/runtime/mgcsweep.go index 5ec4b0f7c3..722a8a577e 100644 --- a/src/runtime/mgcsweep.go +++ b/src/runtime/mgcsweep.go @@ -405,7 +405,10 @@ func reimburseSweepCredit(unusableBytes uintptr) { // Nobody cares about the credit. Avoid the atomic. return } - if int64(atomic.Xadd64(&mheap_.spanBytesAlloc, -int64(unusableBytes))) < 0 { + nval := atomic.Xadd64(&mheap_.spanBytesAlloc, -int64(unusableBytes)) + if int64(nval) < 0 { + // Debugging for #18043. + print("runtime: bad spanBytesAlloc=", nval, " (was ", nval+uint64(unusableBytes), ") unusableBytes=", unusableBytes, " sweepPagesPerByte=", mheap_.sweepPagesPerByte, "\n") throw("spanBytesAlloc underflow") } } diff --git a/src/runtime/mksizeclasses.go b/src/runtime/mksizeclasses.go index 587d3c77a1..0f897ba8e6 100644 --- a/src/runtime/mksizeclasses.go +++ b/src/runtime/mksizeclasses.go @@ -54,6 +54,8 @@ func main() { fmt.Fprintln(&b, "package runtime") classes := makeClasses() + printComment(&b, classes) + printClasses(&b, classes) out, err := format.Source(b.Bytes()) @@ -239,6 +241,20 @@ nextk: } } +func printComment(w io.Writer, classes []class) { + fmt.Fprintf(w, "// %-5s %-9s %-10s %-7s %-11s\n", "class", "bytes/obj", "bytes/span", "objects", "waste bytes") + for i, c := range classes { + if i == 0 { + continue + } + spanSize := c.npages * pageSize + objects := spanSize / c.size + waste := spanSize - c.size*(spanSize/c.size) + fmt.Fprintf(w, "// %5d %9d %10d %7d %11d\n", i, c.size, spanSize, objects, waste) + } + fmt.Fprintf(w, "\n") +} + func printClasses(w io.Writer, classes []class) { fmt.Fprintln(w, "const (") fmt.Fprintf(w, "_MaxSmallSize = %d\n", maxSmallSize) diff --git a/src/runtime/mstats.go b/src/runtime/mstats.go index 8ea7b6deae..aaf16ac524 100644 --- a/src/runtime/mstats.go +++ b/src/runtime/mstats.go @@ -77,6 +77,7 @@ type mstats struct { pause_ns [256]uint64 // circular buffer of recent gc pause lengths pause_end [256]uint64 // circular buffer of recent gc end times (nanoseconds since 1970) numgc uint32 + numforcedgc uint32 // number of user-forced GCs gc_cpu_fraction float64 // fraction of CPU time used by GC enablegc bool debuggc bool @@ -100,8 +101,6 @@ type mstats struct { // must be complete. gc_trigger uint64 - _ uint32 // force 8-byte alignment of heap_live and prevent an alignment check crash on MIPS32. - // heap_live is the number of bytes considered live by the GC. // That is: retained by the most recent GC plus allocated // since then. heap_live <= heap_alloc, since heap_alloc @@ -175,6 +174,7 @@ type MemStats struct { Lookups uint64 // Mallocs is the cumulative count of heap objects allocated. + // The number of live objects is Mallocs - Frees. Mallocs uint64 // Frees is the cumulative count of heap objects freed. @@ -365,6 +365,10 @@ type MemStats struct { // NumGC is the number of completed GC cycles. NumGC uint32 + // NumForcedGC is the number of GC cycles that were forced by + // the application calling the GC function. + NumForcedGC uint32 + // GCCPUFraction is the fraction of this program's available // CPU time used by the GC since the program started. // @@ -394,9 +398,19 @@ type MemStats struct { // // This does not report allocations larger than BySize[60].Size. BySize [61]struct { - Size uint32 + // Size is the maximum byte size of an object in this + // size class. + Size uint32 + + // Mallocs is the cumulative count of heap objects + // allocated in this size class. The cumulative bytes + // of allocation is Size*Mallocs. The number of live + // objects in this size class is Mallocs - Frees. Mallocs uint64 - Frees uint64 + + // Frees is the cumulative count of heap objects freed + // in this size class. + Frees uint64 } } diff --git a/src/runtime/os_linux.go b/src/runtime/os_linux.go index 320c1281c2..a6efc0e3d1 100644 --- a/src/runtime/os_linux.go +++ b/src/runtime/os_linux.go @@ -91,6 +91,9 @@ func getproccount() int32 { const maxCPUs = 64 * 1024 var buf [maxCPUs / (sys.PtrSize * 8)]uintptr r := sched_getaffinity(0, unsafe.Sizeof(buf), &buf[0]) + if r < 0 { + return 1 + } n := int32(0) for _, v := range buf[:r/sys.PtrSize] { for v != 0 { @@ -208,6 +211,26 @@ func sysargs(argc int32, argv **byte) { // Fall back to /proc/self/auxv. fd := open(&procAuxv[0], 0 /* O_RDONLY */, 0) if fd < 0 { + // On Android, /proc/self/auxv might be unreadable (issue 9229), so we fallback to + // try using mincore to detect the physical page size. + // mincore should return EINVAL when address is not a multiple of system page size. + const size = 256 << 10 // size of memory region to allocate + p := mmap(nil, size, _PROT_READ|_PROT_WRITE, _MAP_ANON|_MAP_PRIVATE, -1, 0) + if uintptr(p) < 4096 { + return + } + var n uintptr + for n = 4 << 10; n < size; n <<= 1 { + err := mincore(unsafe.Pointer(uintptr(p)+n), 1, &addrspace_vec[0]) + if err == 0 { + physPageSize = n + break + } + } + if physPageSize == 0 { + physPageSize = size + } + munmap(p, size) return } var buf [128]uintptr diff --git a/src/runtime/os_linux_arm.go b/src/runtime/os_linux_arm.go index 2b0834a5aa..896ec15e6a 100644 --- a/src/runtime/os_linux_arm.go +++ b/src/runtime/os_linux_arm.go @@ -18,6 +18,12 @@ var armArch uint8 = 6 // we default to ARMv6 var hwcap uint32 // set by setup_auxv func checkgoarm() { + // On Android, /proc/self/auxv might be unreadable and hwcap won't + // reflect the CPU capabilities. Assume that every Android arm device + // has the necessary floating point hardware available. + if GOOS == "android" { + return + } if goarm > 5 && hwcap&_HWCAP_VFP == 0 { print("runtime: this CPU has no floating point hardware, so it cannot run\n") print("this GOARM=", goarm, " binary. Recompile using GOARM=5.\n") diff --git a/src/runtime/os_windows.go b/src/runtime/os_windows.go index 0db57f8c5b..75b8acdceb 100644 --- a/src/runtime/os_windows.go +++ b/src/runtime/os_windows.go @@ -332,8 +332,12 @@ func goenvs() { stdcall1(_FreeEnvironmentStringsW, uintptr(strings)) } +// exiting is set to non-zero when the process is exiting. +var exiting uint32 + //go:nosplit func exit(code int32) { + atomic.Store(&exiting, 1) stdcall1(_ExitProcess, uintptr(code)) } @@ -510,7 +514,7 @@ func semacreate(mp *m) { // May run with m.p==nil, so write barriers are not allowed. This // function is called by newosproc0, so it is also required to // operate without stack guards. -//go:nowritebarrierc +//go:nowritebarrierrec //go:nosplit func newosproc(mp *m, stk unsafe.Pointer) { const _STACK_SIZE_PARAM_IS_A_RESERVATION = 0x00010000 @@ -519,6 +523,14 @@ func newosproc(mp *m, stk unsafe.Pointer) { _STACK_SIZE_PARAM_IS_A_RESERVATION, 0) if thandle == 0 { + if atomic.Load(&exiting) != 0 { + // CreateThread may fail if called + // concurrently with ExitProcess. If this + // happens, just freeze this thread and let + // the process exit. See issue #18253. + lock(&deadlock) + lock(&deadlock) + } print("runtime: failed to create new OS thread (have ", mcount(), " already; errno=", getlasterror(), ")\n") throw("runtime.newosproc") } @@ -527,7 +539,7 @@ func newosproc(mp *m, stk unsafe.Pointer) { // Used by the C library build mode. On Linux this function would allocate a // stack, but that's not necessary for Windows. No stack guards are present // and the GC has not been initialized, so write barriers will fail. -//go:nowritebarrierc +//go:nowritebarrierrec //go:nosplit func newosproc0(mp *m, stk unsafe.Pointer) { newosproc(mp, stk) diff --git a/src/runtime/panic.go b/src/runtime/panic.go index 73924365c3..876bca7fd4 100644 --- a/src/runtime/panic.go +++ b/src/runtime/panic.go @@ -646,7 +646,7 @@ func startpanic_m() { freezetheworld() return case 1: - // Something failed while panicing, probably the print of the + // Something failed while panicking, probably the print of the // argument to panic(). Just print a stack trace and exit. _g_.m.dying = 2 print("panic during panic\n") diff --git a/src/runtime/plugin.go b/src/runtime/plugin.go index 845bf76e92..80869e1b1c 100644 --- a/src/runtime/plugin.go +++ b/src/runtime/plugin.go @@ -51,6 +51,9 @@ func plugin_lastmoduleinit() (path string, syms map[string]interface{}, mismatch modulesinit() typelinksinit() + pluginftabverify(md) + moduledataverify1(md) + lock(&ifaceLock) for _, i := range md.itablinks { additab(i, true, false) @@ -82,6 +85,35 @@ func plugin_lastmoduleinit() (path string, syms map[string]interface{}, mismatch return md.pluginpath, syms, "" } +func pluginftabverify(md *moduledata) { + badtable := false + for i := 0; i < len(md.ftab); i++ { + entry := md.ftab[i].entry + if md.minpc <= entry && entry <= md.maxpc { + continue + } + + f := (*_func)(unsafe.Pointer(&md.pclntable[md.ftab[i].funcoff])) + name := funcname(f) + + // A common bug is f.entry has a relocation to a duplicate + // function symbol, meaning if we search for its PC we get + // a valid entry with a name that is useful for debugging. + name2 := "none" + entry2 := uintptr(0) + f2 := findfunc(entry) + if f2 != nil { + name2 = funcname(f2) + entry2 = f2.entry + } + badtable = true + println("ftab entry outside pc range: ", hex(entry), "/", hex(entry2), ": ", name, "/", name2) + } + if badtable { + throw("runtime: plugin has bad symbol table") + } +} + // inRange reports whether v0 or v1 are in the range [r0, r1]. func inRange(r0, r1, v0, v1 uintptr) bool { return (v0 >= r0 && v0 <= r1) || (v1 >= r0 && v1 <= r1) diff --git a/src/runtime/pprof/pprof.go b/src/runtime/pprof/pprof.go index aed5b8da9c..871fba0e5f 100644 --- a/src/runtime/pprof/pprof.go +++ b/src/runtime/pprof/pprof.go @@ -386,12 +386,22 @@ func printCountProfile(w io.Writer, debug int, name string, p countProfile) erro Sample: make([]*profile.Sample, 0, len(keys)), SampleType: []*profile.ValueType{{Type: name, Unit: "count"}}, } + locMap := make(map[uintptr]*profile.Location) for _, k := range keys { stk := p.Stack(index[k]) c := count[k] locs := make([]*profile.Location, len(stk)) for i, addr := range stk { - locs[i] = &profile.Location{Address: uint64(addr) - 1} + loc := locMap[addr] + if loc == nil { + loc = &profile.Location{ + ID: uint64(len(locMap) + 1), + Address: uint64(addr - 1), + } + prof.Location = append(prof.Location, loc) + locMap[addr] = loc + } + locs[i] = loc } prof.Sample = append(prof.Sample, &profile.Sample{ Location: locs, diff --git a/src/runtime/pprof/pprof_test.go b/src/runtime/pprof/pprof_test.go index fd06607805..8372283589 100644 --- a/src/runtime/pprof/pprof_test.go +++ b/src/runtime/pprof/pprof_test.go @@ -204,7 +204,11 @@ func profileOk(t *testing.T, need []string, prof bytes.Buffer, duration time.Dur } // Check that we got a reasonable number of samples. - if ideal := uintptr(duration * 100 / time.Second); samples == 0 || samples < ideal/4 { + // We used to always require at least ideal/4 samples, + // but that is too hard to guarantee on a loaded system. + // Now we accept 10 or more samples, which we take to be + // enough to show that at least some profiling is occurring. + if ideal := uintptr(duration * 100 / time.Second); samples == 0 || (samples < ideal/4 && samples < 10) { t.Logf("too few samples; got %d, want at least %d, ideally %d", samples, ideal/4, ideal) ok = false } diff --git a/src/runtime/proc.go b/src/runtime/proc.go index cad1b1c0f4..f41672de73 100644 --- a/src/runtime/proc.go +++ b/src/runtime/proc.go @@ -240,6 +240,16 @@ func Gosched() { mcall(gosched_m) } +var alwaysFalse bool + +// goschedguarded does nothing, but is written in a way that guarantees a preemption check in its prologue. +// Calls to this function are inserted by the compiler in otherwise uninterruptible loops (see insertLoopReschedChecks). +func goschedguarded() { + if alwaysFalse { + goschedguarded() + } +} + // Puts the current goroutine into a waiting state and calls unlockf. // If unlockf returns false, the goroutine is resumed. // unlockf must not access this G's stack, as it may be moved between @@ -632,10 +642,15 @@ func helpgc(nproc int32) { // sched.stopwait to in order to request that all Gs permanently stop. const freezeStopWait = 0x7fffffff +// freezing is set to non-zero if the runtime is trying to freeze the +// world. +var freezing uint32 + // Similar to stopTheWorld but best-effort and can be called several times. // There is no reverse operation, used during crashing. // This function must not lock any mutexes. func freezetheworld() { + atomic.Store(&freezing, 1) // stopwait and preemption requests can be lost // due to races with concurrently executing threads, // so try several times @@ -1018,15 +1033,30 @@ func stopTheWorldWithSema() { preemptall() } } + + // sanity checks + bad := "" if sched.stopwait != 0 { - throw("stopTheWorld: not stopped") - } - for i := 0; i < int(gomaxprocs); i++ { - p := allp[i] - if p.status != _Pgcstop { - throw("stopTheWorld: not stopped") + bad = "stopTheWorld: not stopped (stopwait != 0)" + } else { + for i := 0; i < int(gomaxprocs); i++ { + p := allp[i] + if p.status != _Pgcstop { + bad = "stopTheWorld: not stopped (status != _Pgcstop)" + } } } + if atomic.Load(&freezing) != 0 { + // Some other thread is panicking. This can cause the + // sanity checks above to fail if the panic happens in + // the signal handler on a stopped thread. Either way, + // we should halt this thread. + lock(&deadlock) + lock(&deadlock) + } + if bad != "" { + throw(bad) + } } func mhelpgc() { diff --git a/src/runtime/rt0_linux_mipsx.s b/src/runtime/rt0_linux_mipsx.s index 5e8c5c3161..9a2e561246 100644 --- a/src/runtime/rt0_linux_mipsx.s +++ b/src/runtime/rt0_linux_mipsx.s @@ -18,10 +18,11 @@ TEXT _main<>(SB),NOSPLIT,$-4 // argv as argc string pointers followed by a NULL, envv as a // sequence of string pointers followed by a NULL, and auxv. // There is no TLS base pointer. - MOVW 0(R29), R1 // argc - ADD $4, R29, R2 // argv + MOVW 0(R29), R4 // argc + ADD $4, R29, R5 // argv JMP main(SB) TEXT main(SB),NOSPLIT,$-4 - MOVW $runtime·rt0_go(SB), R4 - JMP (R4) + // In external linking, libc jumps to main with argc in R4, argv in R5 + MOVW $runtime·rt0_go(SB), R1 + JMP (R1) diff --git a/src/runtime/runtime-gdb_test.go b/src/runtime/runtime-gdb_test.go index 94ba879ed1..c2844375f7 100644 --- a/src/runtime/runtime-gdb_test.go +++ b/src/runtime/runtime-gdb_test.go @@ -85,6 +85,10 @@ func main() { ` func TestGdbPython(t *testing.T) { + if runtime.GOARCH == "mips64" { + testenv.SkipFlaky(t, 18173) + } + t.Parallel() checkGdbEnvironment(t) checkGdbVersion(t) @@ -220,13 +224,16 @@ func main() { // TestGdbBacktrace tests that gdb can unwind the stack correctly // using only the DWARF debug info. func TestGdbBacktrace(t *testing.T) { - t.Parallel() - checkGdbEnvironment(t) - checkGdbVersion(t) - if runtime.GOOS == "netbsd" { testenv.SkipFlaky(t, 15603) } + if runtime.GOARCH == "mips64" { + testenv.SkipFlaky(t, 18173) + } + + t.Parallel() + checkGdbEnvironment(t) + checkGdbVersion(t) dir, err := ioutil.TempDir("", "go-build") if err != nil { @@ -294,6 +301,10 @@ func main() { // TestGdbAutotmpTypes ensures that types of autotmp variables appear in .debug_info // See bug #17830. func TestGdbAutotmpTypes(t *testing.T) { + if runtime.GOARCH == "mips64" { + testenv.SkipFlaky(t, 18173) + } + t.Parallel() checkGdbEnvironment(t) checkGdbVersion(t) diff --git a/src/runtime/runtime2.go b/src/runtime/runtime2.go index 696ea81e00..1ceab0ad8c 100644 --- a/src/runtime/runtime2.go +++ b/src/runtime/runtime2.go @@ -615,7 +615,7 @@ const ( // Layout of in-memory per-function information prepared by linker // See https://golang.org/s/go12symtab. -// Keep in sync with linker +// Keep in sync with linker (../cmd/link/internal/ld/pcln.go:/pclntab) // and with package debug/gosym and with symtab.go in package runtime. type _func struct { entry uintptr // start pc @@ -640,7 +640,7 @@ type itab struct { _type *_type link *itab bad int32 - unused int32 + inhash int32 // has this itab been added to hash? fun [1]uintptr // variable sized } @@ -745,6 +745,8 @@ var ( lfenceBeforeRdtsc bool support_avx bool support_avx2 bool + support_bmi1 bool + support_bmi2 bool goarm uint8 // set by cmd/link on arm systems framepointer_enabled bool // set by cmd/link diff --git a/src/runtime/runtime_test.go b/src/runtime/runtime_test.go index cd078c7eac..9febbe621d 100644 --- a/src/runtime/runtime_test.go +++ b/src/runtime/runtime_test.go @@ -8,6 +8,7 @@ import ( "io" . "runtime" "runtime/debug" + "strings" "testing" "unsafe" ) @@ -329,3 +330,11 @@ func TestGoroutineProfileTrivial(t *testing.T) { } } } + +func TestVersion(t *testing.T) { + // Test that version does not contain \r or \n. + vers := Version() + if strings.Contains(vers, "\r") || strings.Contains(vers, "\n") { + t.Fatalf("cr/nl in version: %q", vers) + } +} diff --git a/src/runtime/select.go b/src/runtime/select.go index 03e9e4a30a..0d846b1470 100644 --- a/src/runtime/select.go +++ b/src/runtime/select.go @@ -422,8 +422,62 @@ loop: gp.param = nil gopark(selparkcommit, nil, "select", traceEvGoBlockSelect, 2) - // someone woke us up - sellock(scases, lockorder) + // While we were asleep, some goroutine came along and completed + // one of the cases in the select and woke us up (called ready). + // As part of that process, the goroutine did a cas on done above + // (aka *sg.selectdone for all queued sg) to win the right to + // complete the select. Now done = 1. + // + // If we copy (grow) our own stack, we will update the + // selectdone pointers inside the gp.waiting sudog list to point + // at the new stack. Another goroutine attempting to + // complete one of our (still linked in) select cases might + // see the new selectdone pointer (pointing at the new stack) + // before the new stack has real data; if the new stack has done = 0 + // (before the old values are copied over), the goroutine might + // do a cas via sg.selectdone and incorrectly believe that it has + // won the right to complete the select, executing a second + // communication and attempting to wake us (call ready) again. + // + // Then things break. + // + // The best break is that the goroutine doing ready sees the + // _Gcopystack status and throws, as in #17007. + // A worse break would be for us to continue on, start running real code, + // block in a semaphore acquisition (sema.go), and have the other + // goroutine wake us up without having really acquired the semaphore. + // That would result in the goroutine spuriously running and then + // queue up another spurious wakeup when the semaphore really is ready. + // In general the situation can cascade until something notices the + // problem and causes a crash. + // + // A stack shrink does not have this problem, because it locks + // all the channels that are involved first, blocking out the + // possibility of a cas on selectdone. + // + // A stack growth before gopark above does not have this + // problem, because we hold those channel locks (released by + // selparkcommit). + // + // A stack growth after sellock below does not have this + // problem, because again we hold those channel locks. + // + // The only problem is a stack growth during sellock. + // To keep that from happening, run sellock on the system stack. + // + // It might be that we could avoid this if copystack copied the + // stack before calling adjustsudogs. In that case, + // syncadjustsudogs would need to recopy the tiny part that + // it copies today, resulting in a little bit of extra copying. + // + // An even better fix, not for the week before a release candidate, + // would be to put space in every sudog and make selectdone + // point at (say) the space in the first sudog. + + systemstack(func() { + sellock(scases, lockorder) + }) + sg = (*sudog)(gp.param) gp.param = nil @@ -464,8 +518,15 @@ loop: } if cas == nil { - // This can happen if we were woken up by a close(). - // TODO: figure that out explicitly so we don't need this loop. + // We can wake up with gp.param == nil (so cas == nil) + // when a channel involved in the select has been closed. + // It is easiest to loop and re-run the operation; + // we'll see that it's now closed. + // Maybe some day we can signal the close explicitly, + // but we'd have to distinguish close-on-reader from close-on-writer. + // It's easiest not to duplicate the code and just recheck above. + // We know that something closed, and things never un-close, + // so we won't block again. goto loop } diff --git a/src/runtime/signal_unix.go b/src/runtime/signal_unix.go index 78381e58d7..49c7579f27 100644 --- a/src/runtime/signal_unix.go +++ b/src/runtime/signal_unix.go @@ -111,8 +111,8 @@ func sigInstallGoHandler(sig uint32) bool { } // When built using c-archive or c-shared, only install signal - // handlers for synchronous signals and SIGPIPE. - if (isarchive || islibrary) && t.flags&_SigPanic == 0 && sig != _SIGPIPE { + // handlers for synchronous signals. + if (isarchive || islibrary) && t.flags&_SigPanic == 0 { return false } @@ -212,25 +212,43 @@ func sigtrampgo(sig uint32, info *siginfo, ctx unsafe.Pointer) { } // If some non-Go code called sigaltstack, adjust. + setStack := false + var gsignalStack gsignalStack sp := uintptr(unsafe.Pointer(&sig)) if sp < g.m.gsignal.stack.lo || sp >= g.m.gsignal.stack.hi { - var st stackt - sigaltstack(nil, &st) - if st.ss_flags&_SS_DISABLE != 0 { - setg(nil) - needm(0) - noSignalStack(sig) - dropm() - } - stsp := uintptr(unsafe.Pointer(st.ss_sp)) - if sp < stsp || sp >= stsp+st.ss_size { - setg(nil) - needm(0) - sigNotOnStack(sig) - dropm() + if sp >= g.m.g0.stack.lo && sp < g.m.g0.stack.hi { + // The signal was delivered on the g0 stack. + // This can happen when linked with C code + // using the thread sanitizer, which collects + // signals then delivers them itself by calling + // the signal handler directly when C code, + // including C code called via cgo, calls a + // TSAN-intercepted function such as malloc. + st := stackt{ss_size: g.m.g0.stack.hi - g.m.g0.stack.lo} + setSignalstackSP(&st, g.m.g0.stack.lo) + setGsignalStack(&st, &gsignalStack) + g.m.gsignal.stktopsp = getcallersp(unsafe.Pointer(&sig)) + setStack = true + } else { + var st stackt + sigaltstack(nil, &st) + if st.ss_flags&_SS_DISABLE != 0 { + setg(nil) + needm(0) + noSignalStack(sig) + dropm() + } + stsp := uintptr(unsafe.Pointer(st.ss_sp)) + if sp < stsp || sp >= stsp+st.ss_size { + setg(nil) + needm(0) + sigNotOnStack(sig) + dropm() + } + setGsignalStack(&st, &gsignalStack) + g.m.gsignal.stktopsp = getcallersp(unsafe.Pointer(&sig)) + setStack = true } - setGsignalStack(&st) - g.m.gsignal.stktopsp = getcallersp(unsafe.Pointer(&sig)) } setg(g.m.gsignal) @@ -238,6 +256,9 @@ func sigtrampgo(sig uint32, info *siginfo, ctx unsafe.Pointer) { c.fixsigcode(sig) sighandler(sig, info, ctx, g) setg(g) + if setStack { + restoreGsignalStack(&gsignalStack) + } } // sigpanic turns a synchronous signal into a run-time panic. @@ -497,15 +518,9 @@ func sigfwdgo(sig uint32, info *siginfo, ctx unsafe.Pointer) bool { return true } + // Only forward synchronous signals. c := &sigctxt{info, ctx} - // Only forward signals from the kernel. - // On Linux and Darwin there is no way to distinguish a SIGPIPE raised by a write - // to a closed socket or pipe from a SIGPIPE raised by kill or pthread_kill - // so we'll treat every SIGPIPE as kernel-generated. - userSig := c.sigcode() == _SI_USER && - (sig != _SIGPIPE || GOOS != "linux" && GOOS != "android" && GOOS != "darwin") - // Only forward synchronous signals and SIGPIPE. - if userSig || flags&_SigPanic == 0 && sig != _SIGPIPE { + if c.sigcode() == _SI_USER || flags&_SigPanic == 0 { return false } // Determine if the signal occurred inside Go code. We test that: @@ -591,7 +606,7 @@ func minitSignalStack() { signalstack(&_g_.m.gsignal.stack) _g_.m.newSigstack = true } else { - setGsignalStack(&st) + setGsignalStack(&st, nil) _g_.m.newSigstack = false } } @@ -624,14 +639,32 @@ func unminitSignals() { } } +// gsignalStack saves the fields of the gsignal stack changed by +// setGsignalStack. +type gsignalStack struct { + stack stack + stackguard0 uintptr + stackguard1 uintptr + stackAlloc uintptr + stktopsp uintptr +} + // setGsignalStack sets the gsignal stack of the current m to an // alternate signal stack returned from the sigaltstack system call. +// It saves the old values in *old for use by restoreGsignalStack. // This is used when handling a signal if non-Go code has set the // alternate signal stack. //go:nosplit //go:nowritebarrierrec -func setGsignalStack(st *stackt) { +func setGsignalStack(st *stackt, old *gsignalStack) { g := getg() + if old != nil { + old.stack = g.m.gsignal.stack + old.stackguard0 = g.m.gsignal.stackguard0 + old.stackguard1 = g.m.gsignal.stackguard1 + old.stackAlloc = g.m.gsignal.stackAlloc + old.stktopsp = g.m.gsignal.stktopsp + } stsp := uintptr(unsafe.Pointer(st.ss_sp)) g.m.gsignal.stack.lo = stsp g.m.gsignal.stack.hi = stsp + st.ss_size @@ -640,6 +673,19 @@ func setGsignalStack(st *stackt) { g.m.gsignal.stackAlloc = st.ss_size } +// restoreGsignalStack restores the gsignal stack to the value it had +// before entering the signal handler. +//go:nosplit +//go:nowritebarrierrec +func restoreGsignalStack(st *gsignalStack) { + gp := getg().m.gsignal + gp.stack = st.stack + gp.stackguard0 = st.stackguard0 + gp.stackguard1 = st.stackguard1 + gp.stackAlloc = st.stackAlloc + gp.stktopsp = st.stktopsp +} + // signalstack sets the current thread's alternate signal stack to s. //go:nosplit func signalstack(s *stack) { diff --git a/src/runtime/sizeclasses.go b/src/runtime/sizeclasses.go index ec30d15d36..e616e95148 100644 --- a/src/runtime/sizeclasses.go +++ b/src/runtime/sizeclasses.go @@ -3,6 +3,74 @@ package runtime +// class bytes/obj bytes/span objects waste bytes +// 1 8 8192 1024 0 +// 2 16 8192 512 0 +// 3 32 8192 256 0 +// 4 48 8192 170 32 +// 5 64 8192 128 0 +// 6 80 8192 102 32 +// 7 96 8192 85 32 +// 8 112 8192 73 16 +// 9 128 8192 64 0 +// 10 144 8192 56 128 +// 11 160 8192 51 32 +// 12 176 8192 46 96 +// 13 192 8192 42 128 +// 14 208 8192 39 80 +// 15 224 8192 36 128 +// 16 240 8192 34 32 +// 17 256 8192 32 0 +// 18 288 8192 28 128 +// 19 320 8192 25 192 +// 20 352 8192 23 96 +// 21 384 8192 21 128 +// 22 416 8192 19 288 +// 23 448 8192 18 128 +// 24 480 8192 17 32 +// 25 512 8192 16 0 +// 26 576 8192 14 128 +// 27 640 8192 12 512 +// 28 704 8192 11 448 +// 29 768 8192 10 512 +// 30 896 8192 9 128 +// 31 1024 8192 8 0 +// 32 1152 8192 7 128 +// 33 1280 8192 6 512 +// 34 1408 16384 11 896 +// 35 1536 8192 5 512 +// 36 1792 16384 9 256 +// 37 2048 8192 4 0 +// 38 2304 16384 7 256 +// 39 2688 8192 3 128 +// 40 3072 24576 8 0 +// 41 3200 16384 5 384 +// 42 3456 24576 7 384 +// 43 4096 8192 2 0 +// 44 4864 24576 5 256 +// 45 5376 16384 3 256 +// 46 6144 24576 4 0 +// 47 6528 32768 5 128 +// 48 6784 40960 6 256 +// 49 6912 49152 7 768 +// 50 8192 8192 1 0 +// 51 9472 57344 6 512 +// 52 9728 49152 5 512 +// 53 10240 40960 4 0 +// 54 10880 32768 3 128 +// 55 12288 24576 2 0 +// 56 13568 40960 3 256 +// 57 14336 57344 4 0 +// 58 16384 16384 1 0 +// 59 18432 73728 4 0 +// 60 19072 57344 3 128 +// 61 20480 40960 2 0 +// 62 21760 65536 3 256 +// 63 24576 24576 1 0 +// 64 27264 81920 3 128 +// 65 28672 57344 2 0 +// 66 32768 32768 1 0 + const ( _MaxSmallSize = 32768 smallSizeDiv = 8 diff --git a/src/runtime/stack.go b/src/runtime/stack.go index ea9a69aa1e..0f1a5c1c55 100644 --- a/src/runtime/stack.go +++ b/src/runtime/stack.go @@ -123,6 +123,9 @@ const ( stackPoisonCopy = 0 // fill stack that should not be accessed with garbage, to detect bad dereferences during copy stackCache = 1 + + // check the BP links during traceback. + debugCheckBP = false ) const ( @@ -598,7 +601,7 @@ func adjustpointers(scanp unsafe.Pointer, cbv *bitvector, adjinfo *adjustinfo, f pp := (*uintptr)(add(scanp, i*sys.PtrSize)) retry: p := *pp - if f != nil && 0 < p && p < _PageSize && debug.invalidptr != 0 { + if f != nil && 0 < p && p < minLegalPointer && debug.invalidptr != 0 { // Looks like a junk value in a pointer slot. // Live analysis wrong? getg().m.traceback = 2 @@ -688,6 +691,16 @@ func adjustframe(frame *stkframe, arg unsafe.Pointer) bool { if stackDebug >= 3 { print(" saved bp\n") } + if debugCheckBP { + // Frame pointers should always point to the next higher frame on + // the Go stack (or be nil, for the top frame on the stack). + bp := *(*uintptr)(unsafe.Pointer(frame.varp)) + if bp != 0 && (bp < adjinfo.old.lo || bp >= adjinfo.old.hi) { + println("runtime: found invalid frame pointer") + print("bp=", hex(bp), " min=", hex(adjinfo.old.lo), " max=", hex(adjinfo.old.hi), "\n") + throw("bad frame pointer") + } + } adjustpointer(adjinfo, unsafe.Pointer(frame.varp)) } @@ -719,6 +732,18 @@ func adjustframe(frame *stkframe, arg unsafe.Pointer) bool { func adjustctxt(gp *g, adjinfo *adjustinfo) { adjustpointer(adjinfo, unsafe.Pointer(&gp.sched.ctxt)) + if !framepointer_enabled { + return + } + if debugCheckBP { + bp := gp.sched.bp + if bp != 0 && (bp < adjinfo.old.lo || bp >= adjinfo.old.hi) { + println("runtime: found invalid top frame pointer") + print("bp=", hex(bp), " min=", hex(adjinfo.old.lo), " max=", hex(adjinfo.old.hi), "\n") + throw("bad top frame pointer") + } + } + adjustpointer(adjinfo, unsafe.Pointer(&gp.sched.bp)) } func adjustdefers(gp *g, adjinfo *adjustinfo) { diff --git a/src/runtime/symtab.go b/src/runtime/symtab.go index 8a5b0df310..f52190661c 100644 --- a/src/runtime/symtab.go +++ b/src/runtime/symtab.go @@ -666,7 +666,7 @@ func readvarint(p []byte) (newp []byte, val uint32) { type stackmap struct { n int32 // number of bitmaps nbit int32 // number of bits in each bitmap - bytedata [1]byte // bitmaps, each starting on a 32-bit boundary + bytedata [1]byte // bitmaps, each starting on a byte boundary } //go:nowritebarrier diff --git a/src/runtime/sys_dragonfly_amd64.s b/src/runtime/sys_dragonfly_amd64.s index 88c7f9dd8a..b950b69fe0 100644 --- a/src/runtime/sys_dragonfly_amd64.s +++ b/src/runtime/sys_dragonfly_amd64.s @@ -200,11 +200,28 @@ TEXT runtime·sigfwd(SB),NOSPLIT,$0-32 POPQ BP RET -TEXT runtime·sigtramp(SB),NOSPLIT,$24 - MOVQ DI, 0(SP) - MOVQ SI, 8(SP) - MOVQ DX, 16(SP) +TEXT runtime·sigtramp(SB),NOSPLIT,$72 + // Save callee-saved C registers, since the caller may be a C signal handler. + MOVQ BX, bx-8(SP) + MOVQ BP, bp-16(SP) // save in case GOEXPERIMENT=noframepointer is set + MOVQ R12, r12-24(SP) + MOVQ R13, r13-32(SP) + MOVQ R14, r14-40(SP) + MOVQ R15, r15-48(SP) + // We don't save mxcsr or the x87 control word because sigtrampgo doesn't + // modify them. + + MOVQ DX, ctx-56(SP) + MOVQ SI, info-64(SP) + MOVQ DI, signum-72(SP) CALL runtime·sigtrampgo(SB) + + MOVQ r15-48(SP), R15 + MOVQ r14-40(SP), R14 + MOVQ r13-32(SP), R13 + MOVQ r12-24(SP), R12 + MOVQ bp-16(SP), BP + MOVQ bx-8(SP), BX RET TEXT runtime·mmap(SB),NOSPLIT,$0 diff --git a/src/runtime/sys_freebsd_amd64.s b/src/runtime/sys_freebsd_amd64.s index 19007dc401..158a60dec2 100644 --- a/src/runtime/sys_freebsd_amd64.s +++ b/src/runtime/sys_freebsd_amd64.s @@ -196,11 +196,28 @@ TEXT runtime·sigfwd(SB),NOSPLIT,$0-32 POPQ BP RET -TEXT runtime·sigtramp(SB),NOSPLIT,$24 - MOVQ DI, 0(SP) - MOVQ SI, 8(SP) - MOVQ DX, 16(SP) +TEXT runtime·sigtramp(SB),NOSPLIT,$72 + // Save callee-saved C registers, since the caller may be a C signal handler. + MOVQ BX, bx-8(SP) + MOVQ BP, bp-16(SP) // save in case GOEXPERIMENT=noframepointer is set + MOVQ R12, r12-24(SP) + MOVQ R13, r13-32(SP) + MOVQ R14, r14-40(SP) + MOVQ R15, r15-48(SP) + // We don't save mxcsr or the x87 control word because sigtrampgo doesn't + // modify them. + + MOVQ DX, ctx-56(SP) + MOVQ SI, info-64(SP) + MOVQ DI, signum-72(SP) CALL runtime·sigtrampgo(SB) + + MOVQ r15-48(SP), R15 + MOVQ r14-40(SP), R14 + MOVQ r13-32(SP), R13 + MOVQ r12-24(SP), R12 + MOVQ bp-16(SP), BP + MOVQ bx-8(SP), BX RET TEXT runtime·mmap(SB),NOSPLIT,$0 diff --git a/src/runtime/sys_linux_386.s b/src/runtime/sys_linux_386.s index 1d798c741e..45320c068a 100644 --- a/src/runtime/sys_linux_386.s +++ b/src/runtime/sys_linux_386.s @@ -228,7 +228,15 @@ TEXT runtime·sigfwd(SB),NOSPLIT,$12-16 MOVL AX, SP RET -TEXT runtime·sigtramp(SB),NOSPLIT,$12 +TEXT runtime·sigtramp(SB),NOSPLIT,$28 + // Save callee-saved C registers, since the caller may be a C signal handler. + MOVL BX, bx-4(SP) + MOVL BP, bp-8(SP) + MOVL SI, si-12(SP) + MOVL DI, di-16(SP) + // We don't save mxcsr or the x87 control word because sigtrampgo doesn't + // modify them. + MOVL sig+0(FP), BX MOVL BX, 0(SP) MOVL info+4(FP), BX @@ -236,6 +244,11 @@ TEXT runtime·sigtramp(SB),NOSPLIT,$12 MOVL ctx+8(FP), BX MOVL BX, 8(SP) CALL runtime·sigtrampgo(SB) + + MOVL di-16(SP), DI + MOVL si-12(SP), SI + MOVL bp-8(SP), BP + MOVL bx-4(SP), BX RET TEXT runtime·cgoSigtramp(SB),NOSPLIT,$0 diff --git a/src/runtime/sys_linux_amd64.s b/src/runtime/sys_linux_amd64.s index 832b98b674..839df16406 100644 --- a/src/runtime/sys_linux_amd64.s +++ b/src/runtime/sys_linux_amd64.s @@ -244,12 +244,29 @@ TEXT runtime·sigfwd(SB),NOSPLIT,$0-32 POPQ BP RET -TEXT runtime·sigtramp(SB),NOSPLIT,$24 - MOVQ DI, 0(SP) // signum - MOVQ SI, 8(SP) // info - MOVQ DX, 16(SP) // ctx +TEXT runtime·sigtramp(SB),NOSPLIT,$72 + // Save callee-saved C registers, since the caller may be a C signal handler. + MOVQ BX, bx-8(SP) + MOVQ BP, bp-16(SP) // save in case GOEXPERIMENT=noframepointer is set + MOVQ R12, r12-24(SP) + MOVQ R13, r13-32(SP) + MOVQ R14, r14-40(SP) + MOVQ R15, r15-48(SP) + // We don't save mxcsr or the x87 control word because sigtrampgo doesn't + // modify them. + + MOVQ DX, ctx-56(SP) + MOVQ SI, info-64(SP) + MOVQ DI, signum-72(SP) MOVQ $runtime·sigtrampgo(SB), AX CALL AX + + MOVQ r15-48(SP), R15 + MOVQ r14-40(SP), R14 + MOVQ r13-32(SP), R13 + MOVQ r12-24(SP), R12 + MOVQ bp-16(SP), BP + MOVQ bx-8(SP), BX RET // Used instead of sigtramp in programs that use cgo. diff --git a/src/runtime/sys_linux_mipsx.s b/src/runtime/sys_linux_mipsx.s index 6f089f5932..73ce06114c 100644 --- a/src/runtime/sys_linux_mipsx.s +++ b/src/runtime/sys_linux_mipsx.s @@ -249,7 +249,7 @@ TEXT runtime·sigfwd(SB),NOSPLIT,$0-16 MOVW fn+0(FP), R25 MOVW R29, R22 SUBU $16, R29 - AND $0x7, R29 // shadow space for 4 args aligned to 8 bytes as per O32 ABI + AND $~7, R29 // shadow space for 4 args aligned to 8 bytes as per O32 ABI JAL (R25) MOVW R22, R29 RET diff --git a/src/runtime/sys_netbsd_386.s b/src/runtime/sys_netbsd_386.s index 50d35e5b5c..8c4f004a4b 100644 --- a/src/runtime/sys_netbsd_386.s +++ b/src/runtime/sys_netbsd_386.s @@ -232,7 +232,15 @@ TEXT runtime·sigfwd(SB),NOSPLIT,$12-16 MOVL AX, SP RET -TEXT runtime·sigtramp(SB),NOSPLIT,$12 +TEXT runtime·sigtramp(SB),NOSPLIT,$28 + // Save callee-saved C registers, since the caller may be a C signal handler. + MOVL BX, bx-4(SP) + MOVL BP, bp-8(SP) + MOVL SI, si-12(SP) + MOVL DI, di-16(SP) + // We don't save mxcsr or the x87 control word because sigtrampgo doesn't + // modify them. + MOVL signo+0(FP), BX MOVL BX, 0(SP) MOVL info+4(FP), BX @@ -240,6 +248,11 @@ TEXT runtime·sigtramp(SB),NOSPLIT,$12 MOVL context+8(FP), BX MOVL BX, 8(SP) CALL runtime·sigtrampgo(SB) + + MOVL di-16(SP), DI + MOVL si-12(SP), SI + MOVL bp-8(SP), BP + MOVL bx-4(SP), BX RET // int32 lwp_create(void *context, uintptr flags, void *lwpid); diff --git a/src/runtime/sys_netbsd_amd64.s b/src/runtime/sys_netbsd_amd64.s index 2c50adb123..7c7771bcba 100644 --- a/src/runtime/sys_netbsd_amd64.s +++ b/src/runtime/sys_netbsd_amd64.s @@ -250,13 +250,28 @@ TEXT runtime·sigfwd(SB),NOSPLIT,$0-32 POPQ BP RET -TEXT runtime·sigtramp(SB),NOSPLIT,$32 - MOVQ DI, 0(SP) // signum - MOVQ SI, 8(SP) // info - MOVQ DX, 16(SP) // ctx - MOVQ R15, 24(SP) // for sigreturn +TEXT runtime·sigtramp(SB),NOSPLIT,$72 + // Save callee-saved C registers, since the caller may be a C signal handler. + MOVQ BX, bx-8(SP) + MOVQ BP, bp-16(SP) // save in case GOEXPERIMENT=noframepointer is set + MOVQ R12, r12-24(SP) + MOVQ R13, r13-32(SP) + MOVQ R14, r14-40(SP) + MOVQ R15, r15-48(SP) + // We don't save mxcsr or the x87 control word because sigtrampgo doesn't + // modify them. + + MOVQ DX, ctx-56(SP) + MOVQ SI, info-64(SP) + MOVQ DI, signum-72(SP) CALL runtime·sigtrampgo(SB) - MOVQ 24(SP), R15 + + MOVQ r15-48(SP), R15 + MOVQ r14-40(SP), R14 + MOVQ r13-32(SP), R13 + MOVQ r12-24(SP), R12 + MOVQ bp-16(SP), BP + MOVQ bx-8(SP), BX RET TEXT runtime·mmap(SB),NOSPLIT,$0 diff --git a/src/runtime/sys_openbsd_386.s b/src/runtime/sys_openbsd_386.s index e969395088..76d22b0131 100644 --- a/src/runtime/sys_openbsd_386.s +++ b/src/runtime/sys_openbsd_386.s @@ -79,14 +79,15 @@ TEXT runtime·usleep(SB),NOSPLIT,$24 INT $0x80 RET -TEXT runtime·raise(SB),NOSPLIT,$12 +TEXT runtime·raise(SB),NOSPLIT,$16 MOVL $299, AX // sys_getthrid INT $0x80 MOVL $0, 0(SP) - MOVL AX, 4(SP) // arg 1 - pid + MOVL AX, 4(SP) // arg 1 - tid MOVL sig+0(FP), AX MOVL AX, 8(SP) // arg 2 - signum - MOVL $37, AX // sys_kill + MOVL $0, 12(SP) // arg 3 - tcb + MOVL $119, AX // sys_thrkill INT $0x80 RET @@ -97,7 +98,7 @@ TEXT runtime·raiseproc(SB),NOSPLIT,$12 MOVL AX, 4(SP) // arg 1 - pid MOVL sig+0(FP), AX MOVL AX, 8(SP) // arg 2 - signum - MOVL $37, AX // sys_kill + MOVL $122, AX // sys_kill INT $0x80 RET @@ -212,7 +213,15 @@ TEXT runtime·sigfwd(SB),NOSPLIT,$12-16 MOVL AX, SP RET -TEXT runtime·sigtramp(SB),NOSPLIT,$12 +TEXT runtime·sigtramp(SB),NOSPLIT,$28 + // Save callee-saved C registers, since the caller may be a C signal handler. + MOVL BX, bx-4(SP) + MOVL BP, bp-8(SP) + MOVL SI, si-12(SP) + MOVL DI, di-16(SP) + // We don't save mxcsr or the x87 control word because sigtrampgo doesn't + // modify them. + MOVL signo+0(FP), BX MOVL BX, 0(SP) MOVL info+4(FP), BX @@ -220,6 +229,11 @@ TEXT runtime·sigtramp(SB),NOSPLIT,$12 MOVL context+8(FP), BX MOVL BX, 8(SP) CALL runtime·sigtrampgo(SB) + + MOVL di-16(SP), DI + MOVL si-12(SP), SI + MOVL bp-8(SP), BP + MOVL bx-4(SP), BX RET // int32 tfork(void *param, uintptr psize, M *mp, G *gp, void (*fn)(void)); diff --git a/src/runtime/sys_openbsd_amd64.s b/src/runtime/sys_openbsd_amd64.s index 01d6bd8e85..cf7a3fb7a9 100644 --- a/src/runtime/sys_openbsd_amd64.s +++ b/src/runtime/sys_openbsd_amd64.s @@ -156,9 +156,10 @@ TEXT runtime·usleep(SB),NOSPLIT,$16 TEXT runtime·raise(SB),NOSPLIT,$16 MOVL $299, AX // sys_getthrid SYSCALL - MOVQ AX, DI // arg 1 - pid + MOVQ AX, DI // arg 1 - tid MOVL sig+0(FP), SI // arg 2 - signum - MOVL $37, AX // sys_kill + MOVQ $0, DX // arg 3 - tcb + MOVL $119, AX // sys_thrkill SYSCALL RET @@ -167,7 +168,7 @@ TEXT runtime·raiseproc(SB),NOSPLIT,$16 SYSCALL MOVQ AX, DI // arg 1 - pid MOVL sig+0(FP), SI // arg 2 - signum - MOVL $37, AX // sys_kill + MOVL $122, AX // sys_kill SYSCALL RET @@ -241,11 +242,28 @@ TEXT runtime·sigfwd(SB),NOSPLIT,$0-32 POPQ BP RET -TEXT runtime·sigtramp(SB),NOSPLIT,$24 - MOVQ DI, 0(SP) - MOVQ SI, 8(SP) - MOVQ DX, 16(SP) +TEXT runtime·sigtramp(SB),NOSPLIT,$72 + // Save callee-saved C registers, since the caller may be a C signal handler. + MOVQ BX, bx-8(SP) + MOVQ BP, bp-16(SP) // save in case GOEXPERIMENT=noframepointer is set + MOVQ R12, r12-24(SP) + MOVQ R13, r13-32(SP) + MOVQ R14, r14-40(SP) + MOVQ R15, r15-48(SP) + // We don't save mxcsr or the x87 control word because sigtrampgo doesn't + // modify them. + + MOVQ DX, ctx-56(SP) + MOVQ SI, info-64(SP) + MOVQ DI, signum-72(SP) CALL runtime·sigtrampgo(SB) + + MOVQ r15-48(SP), R15 + MOVQ r14-40(SP), R14 + MOVQ r13-32(SP), R13 + MOVQ r12-24(SP), R12 + MOVQ bp-16(SP), BP + MOVQ bx-8(SP), BX RET TEXT runtime·mmap(SB),NOSPLIT,$0 diff --git a/src/runtime/sys_openbsd_arm.s b/src/runtime/sys_openbsd_arm.s index e0f775d0eb..f573a028a0 100644 --- a/src/runtime/sys_openbsd_arm.s +++ b/src/runtime/sys_openbsd_arm.s @@ -87,9 +87,10 @@ TEXT runtime·usleep(SB),NOSPLIT,$16 TEXT runtime·raise(SB),NOSPLIT,$12 MOVW $0x12B, R12 SWI $0 // sys_getthrid - // arg 1 - pid, already in R0 + // arg 1 - tid, already in R0 MOVW sig+0(FP), R1 // arg 2 - signum - MOVW $37, R12 // sys_kill + MOVW $0, R2 // arg 3 - tcb + MOVW $119, R12 // sys_thrkill SWI $0 RET @@ -98,7 +99,7 @@ TEXT runtime·raiseproc(SB),NOSPLIT,$12 SWI $0 // sys_getpid // arg 1 - pid, already in R0 MOVW sig+0(FP), R1 // arg 2 - signum - MOVW $37, R12 // sys_kill + MOVW $122, R12 // sys_kill SWI $0 RET diff --git a/src/runtime/tls_mipsx.s b/src/runtime/tls_mipsx.s index 95fbc32a7c..a2c01d2167 100644 --- a/src/runtime/tls_mipsx.s +++ b/src/runtime/tls_mipsx.s @@ -10,12 +10,20 @@ #include "textflag.h" // If !iscgo, this is a no-op. +// NOTE: gogo asumes load_g only clobers g (R30) and REGTMP (R23) TEXT runtime·save_g(SB),NOSPLIT,$-4-0 MOVB runtime·iscgo(SB), R23 BEQ R23, nocgo - UNDEF + + MOVW R3, R23 + MOVW g, runtime·tls_g(SB) // TLS relocation clobbers R3 + MOVW R23, R3 + nocgo: RET TEXT runtime·load_g(SB),NOSPLIT,$-4-0 + MOVW runtime·tls_g(SB), g // TLS relocation clobbers R3 RET + +GLOBL runtime·tls_g(SB), TLSBSS, $4 diff --git a/src/runtime/traceback.go b/src/runtime/traceback.go index 0049e82d63..180489fb2c 100644 --- a/src/runtime/traceback.go +++ b/src/runtime/traceback.go @@ -546,7 +546,8 @@ func gentraceback(pc0, sp0, lr0 uintptr, gp *g, skip int, pcbuf *uintptr, max in return n } -// reflectMethodValue is a partial duplicate of reflect.methodValue. +// reflectMethodValue is a partial duplicate of reflect.makeFuncImpl +// and reflect.methodValue. type reflectMethodValue struct { fn uintptr stack *bitvector // args bitmap diff --git a/src/sort/example_test.go b/src/sort/example_test.go index f7372bec37..980c0d0368 100644 --- a/src/sort/example_test.go +++ b/src/sort/example_test.go @@ -22,3 +22,22 @@ func ExampleReverse() { fmt.Println(s) // Output: [6 5 4 3 2 1] } + +func ExampleSlice() { + people := []struct { + Name string + Age int + }{ + {"Gopher", 7}, + {"Alice", 55}, + {"Vera", 24}, + {"Bob", 75}, + } + sort.Slice(people, func(i, j int) bool { return people[i].Name < people[j].Name }) + fmt.Println("By name:", people) + + sort.Slice(people, func(i, j int) bool { return people[i].Age < people[j].Age }) + fmt.Println("By age:", people) + // Output: By name: [{Alice 55} {Bob 75} {Gopher 7} {Vera 24}] + // By age: [{Gopher 7} {Vera 24} {Alice 55} {Bob 75}] +} diff --git a/src/syscall/zsysnum_openbsd_386.go b/src/syscall/zsysnum_openbsd_386.go index c19f6de649..c172534aeb 100644 --- a/src/syscall/zsysnum_openbsd_386.go +++ b/src/syscall/zsysnum_openbsd_386.go @@ -41,7 +41,6 @@ const ( SYS_CHFLAGS = 34 // { int sys_chflags(const char *path, u_int flags); } SYS_FCHFLAGS = 35 // { int sys_fchflags(int fd, u_int flags); } SYS_SYNC = 36 // { void sys_sync(void); } - SYS_KILL = 37 // { int sys_kill(int pid, int signum); } SYS_STAT = 38 // { int sys_stat(const char *path, struct stat *ub); } SYS_GETPPID = 39 // { pid_t sys_getppid(void); } SYS_LSTAT = 40 // { int sys_lstat(const char *path, struct stat *ub); } @@ -113,6 +112,7 @@ const ( SYS_READV = 120 // { ssize_t sys_readv(int fd, \ SYS_WRITEV = 121 // { ssize_t sys_writev(int fd, \ SYS_FCHOWN = 123 // { int sys_fchown(int fd, uid_t uid, gid_t gid); } + SYS_KILL = 122 // { int sys_kill(int pid, int signum); } SYS_FCHMOD = 124 // { int sys_fchmod(int fd, mode_t mode); } SYS_SETREUID = 126 // { int sys_setreuid(uid_t ruid, uid_t euid); } SYS_SETREGID = 127 // { int sys_setregid(gid_t rgid, gid_t egid); } diff --git a/src/syscall/zsysnum_openbsd_amd64.go b/src/syscall/zsysnum_openbsd_amd64.go index 86e04cd47e..356c187375 100644 --- a/src/syscall/zsysnum_openbsd_amd64.go +++ b/src/syscall/zsysnum_openbsd_amd64.go @@ -41,7 +41,6 @@ const ( SYS_CHFLAGS = 34 // { int sys_chflags(const char *path, u_int flags); } SYS_FCHFLAGS = 35 // { int sys_fchflags(int fd, u_int flags); } SYS_SYNC = 36 // { void sys_sync(void); } - SYS_KILL = 37 // { int sys_kill(int pid, int signum); } SYS_STAT = 38 // { int sys_stat(const char *path, struct stat *ub); } SYS_GETPPID = 39 // { pid_t sys_getppid(void); } SYS_LSTAT = 40 // { int sys_lstat(const char *path, struct stat *ub); } @@ -112,6 +111,7 @@ const ( SYS_GETSOCKOPT = 118 // { int sys_getsockopt(int s, int level, int name, \ SYS_READV = 120 // { ssize_t sys_readv(int fd, \ SYS_WRITEV = 121 // { ssize_t sys_writev(int fd, \ + SYS_KILL = 122 // { int sys_kill(int pid, int signum); } SYS_FCHOWN = 123 // { int sys_fchown(int fd, uid_t uid, gid_t gid); } SYS_FCHMOD = 124 // { int sys_fchmod(int fd, mode_t mode); } SYS_SETREUID = 126 // { int sys_setreuid(uid_t ruid, uid_t euid); } diff --git a/src/syscall/zsysnum_openbsd_arm.go b/src/syscall/zsysnum_openbsd_arm.go index 38b43caba6..3e6b984a85 100644 --- a/src/syscall/zsysnum_openbsd_arm.go +++ b/src/syscall/zsysnum_openbsd_arm.go @@ -42,7 +42,6 @@ const ( SYS_CHFLAGS = 34 // { int sys_chflags(const char *path, u_int flags); } SYS_FCHFLAGS = 35 // { int sys_fchflags(int fd, u_int flags); } SYS_SYNC = 36 // { void sys_sync(void); } - SYS_KILL = 37 // { int sys_kill(int pid, int signum); } SYS_STAT = 38 // { int sys_stat(const char *path, struct stat *ub); } SYS_GETPPID = 39 // { pid_t sys_getppid(void); } SYS_LSTAT = 40 // { int sys_lstat(const char *path, struct stat *ub); } @@ -118,6 +117,7 @@ const ( SYS_GETSOCKOPT = 118 // { int sys_getsockopt(int s, int level, int name, \ SYS_READV = 120 // { ssize_t sys_readv(int fd, \ SYS_WRITEV = 121 // { ssize_t sys_writev(int fd, \ + SYS_KILL = 122 // { int sys_kill(int pid, int signum); } SYS_FCHOWN = 123 // { int sys_fchown(int fd, uid_t uid, gid_t gid); } SYS_FCHMOD = 124 // { int sys_fchmod(int fd, mode_t mode); } SYS_SETREUID = 126 // { int sys_setreuid(uid_t ruid, uid_t euid); } diff --git a/src/testing/benchmark.go b/src/testing/benchmark.go index b1c6d2eff0..c033ce5fec 100644 --- a/src/testing/benchmark.go +++ b/src/testing/benchmark.go @@ -5,7 +5,6 @@ package testing import ( - "context" "flag" "fmt" "internal/race" @@ -128,9 +127,6 @@ func (b *B) nsPerOp() int64 { // runN runs a single benchmark for the specified number of iterations. func (b *B) runN(n int) { - b.ctx, b.cancel = context.WithCancel(b.parentContext()) - defer b.cancel() - benchmarkLock.Lock() defer benchmarkLock.Unlock() // Try to get a comparable environment for each run diff --git a/src/testing/sub_test.go b/src/testing/sub_test.go index 563e8656c6..8d5d9206f0 100644 --- a/src/testing/sub_test.go +++ b/src/testing/sub_test.go @@ -6,7 +6,6 @@ package testing import ( "bytes" - "context" "regexp" "strings" "sync/atomic" @@ -278,33 +277,28 @@ func TestTRun(t *T) { ok: true, maxPar: 4, f: func(t *T) { - // t.Parallel doesn't work in the pseudo-T we start with: - // it leaks a goroutine. - // Call t.Run to get a real one. - t.Run("X", func(t *T) { - t.Parallel() - for i := 0; i < 12; i++ { - t.Run("a", func(t *T) { - t.Parallel() - time.Sleep(time.Nanosecond) - for i := 0; i < 12; i++ { - t.Run("b", func(t *T) { - time.Sleep(time.Nanosecond) - for i := 0; i < 12; i++ { - t.Run("c", func(t *T) { - t.Parallel() - time.Sleep(time.Nanosecond) - t.Run("d1", func(t *T) {}) - t.Run("d2", func(t *T) {}) - t.Run("d3", func(t *T) {}) - t.Run("d4", func(t *T) {}) - }) - } - }) - } - }) - } - }) + t.Parallel() + for i := 0; i < 12; i++ { + t.Run("a", func(t *T) { + t.Parallel() + time.Sleep(time.Nanosecond) + for i := 0; i < 12; i++ { + t.Run("b", func(t *T) { + time.Sleep(time.Nanosecond) + for i := 0; i < 12; i++ { + t.Run("c", func(t *T) { + t.Parallel() + time.Sleep(time.Nanosecond) + t.Run("d1", func(t *T) {}) + t.Run("d2", func(t *T) {}) + t.Run("d3", func(t *T) {}) + t.Run("d4", func(t *T) {}) + }) + } + }) + } + }) + } }, }, { desc: "skip output", @@ -347,7 +341,6 @@ func TestTRun(t *T) { }, context: ctx, } - root.ctx, root.cancel = context.WithCancel(context.Background()) ok := root.Run(tc.desc, tc.f) ctx.release() @@ -364,7 +357,7 @@ func TestTRun(t *T) { want := strings.TrimSpace(tc.output) re := makeRegexp(want) if ok, err := regexp.MatchString(re, got); !ok || err != nil { - t.Errorf("%s:ouput:\ngot:\n%s\nwant:\n%s", tc.desc, got, want) + t.Errorf("%s:output:\ngot:\n%s\nwant:\n%s", tc.desc, got, want) } } } @@ -505,7 +498,7 @@ func TestBRun(t *T) { want := strings.TrimSpace(tc.output) re := makeRegexp(want) if ok, err := regexp.MatchString(re, got); !ok || err != nil { - t.Errorf("%s:ouput:\ngot:\n%s\nwant:\n%s", tc.desc, got, want) + t.Errorf("%s:output:\ngot:\n%s\nwant:\n%s", tc.desc, got, want) } } } diff --git a/src/testing/testing.go b/src/testing/testing.go index f08c5c6b8e..c972b2737f 100644 --- a/src/testing/testing.go +++ b/src/testing/testing.go @@ -204,7 +204,6 @@ package testing import ( "bytes" - "context" "errors" "flag" "fmt" @@ -262,14 +261,12 @@ type common struct { mu sync.RWMutex // guards output, failed, and done. output []byte // Output generated by test or benchmark. w io.Writer // For flushToParent. - ctx context.Context - cancel context.CancelFunc - chatty bool // A copy of the chatty flag. - ran bool // Test or benchmark (or one of its subtests) was executed. - failed bool // Test or benchmark has failed. - skipped bool // Test of benchmark has been skipped. - finished bool // Test function has completed. - done bool // Test is finished and all subtests have completed. + chatty bool // A copy of the chatty flag. + ran bool // Test or benchmark (or one of its subtests) was executed. + failed bool // Test or benchmark has failed. + skipped bool // Test of benchmark has been skipped. + finished bool // Test function has completed. + done bool // Test is finished and all subtests have completed. hasSub bool raceErrors int // number of races detected during test @@ -283,13 +280,6 @@ type common struct { sub []*T // Queue of subtests to be run in parallel. } -func (c *common) parentContext() context.Context { - if c == nil || c.parent == nil || c.parent.ctx == nil { - return context.Background() - } - return c.parent.ctx -} - // Short reports whether the -test.short flag is set. func Short() bool { return *short @@ -386,7 +376,6 @@ func fmtDuration(d time.Duration) string { // TB is the interface common to T and B. type TB interface { - Context() context.Context Error(args ...interface{}) Errorf(format string, args ...interface{}) Fail() @@ -434,15 +423,6 @@ func (c *common) Name() string { return c.name } -// Context returns the context for the current test or benchmark. -// The context is cancelled when the test or benchmark finishes. -// A goroutine started during a test or benchmark can wait for the -// context's Done channel to become readable as a signal that the -// test or benchmark is over, so that the goroutine can exit. -func (c *common) Context() context.Context { - return c.ctx -} - func (c *common) setRan() { if c.parent != nil { c.parent.setRan() @@ -619,9 +599,6 @@ type InternalTest struct { } func tRunner(t *T, fn func(t *T)) { - t.ctx, t.cancel = context.WithCancel(t.parentContext()) - defer t.cancel() - // When this goroutine is done, either because fn(t) // returned normally or because a test failure triggered // a call to runtime.Goexit, record the duration and send @@ -918,11 +895,11 @@ func (m *M) before() { if *cpuProfile != "" { f, err := os.Create(toOutputDir(*cpuProfile)) if err != nil { - fmt.Fprintf(os.Stderr, "testing: %s", err) + fmt.Fprintf(os.Stderr, "testing: %s\n", err) return } if err := m.deps.StartCPUProfile(f); err != nil { - fmt.Fprintf(os.Stderr, "testing: can't start cpu profile: %s", err) + fmt.Fprintf(os.Stderr, "testing: can't start cpu profile: %s\n", err) f.Close() return } @@ -931,11 +908,11 @@ func (m *M) before() { if *traceFile != "" { f, err := os.Create(toOutputDir(*traceFile)) if err != nil { - fmt.Fprintf(os.Stderr, "testing: %s", err) + fmt.Fprintf(os.Stderr, "testing: %s\n", err) return } if err := trace.Start(f); err != nil { - fmt.Fprintf(os.Stderr, "testing: can't start tracing: %s", err) + fmt.Fprintf(os.Stderr, "testing: can't start tracing: %s\n", err) f.Close() return } diff --git a/src/testing/testing_test.go b/src/testing/testing_test.go index 9954f9af8c..45e44683b4 100644 --- a/src/testing/testing_test.go +++ b/src/testing/testing_test.go @@ -5,42 +5,14 @@ package testing_test import ( - "fmt" "os" - "runtime" "testing" - "time" ) -func TestMain(m *testing.M) { - g0 := runtime.NumGoroutine() - - code := m.Run() - if code != 0 { - os.Exit(code) - } +// This is exactly what a test would do without a TestMain. +// It's here only so that there is at least one package in the +// standard library with a TestMain, so that code is executed. - // Check that there are no goroutines left behind. - t0 := time.Now() - stacks := make([]byte, 1<<20) - for { - g1 := runtime.NumGoroutine() - if g1 == g0 { - return - } - stacks = stacks[:runtime.Stack(stacks, true)] - time.Sleep(50 * time.Millisecond) - if time.Since(t0) > 2*time.Second { - fmt.Fprintf(os.Stderr, "Unexpected leftover goroutines detected: %v -> %v\n%s\n", g0, g1, stacks) - os.Exit(1) - } - } -} - -func TestContextCancel(t *testing.T) { - ctx := t.Context() - // Tests we don't leak this goroutine: - go func() { - <-ctx.Done() - }() +func TestMain(m *testing.M) { + os.Exit(m.Run()) } diff --git a/src/time/format.go b/src/time/format.go index 3fbfa734d0..b903e1485c 100644 --- a/src/time/format.go +++ b/src/time/format.go @@ -1101,8 +1101,9 @@ func parseTimeZone(value string) (length int, ok bool) { if value[4] == 'T' { return 5, true } - case 4: // Must end in T to match. - if value[3] == 'T' { + case 4: + // Must end in T, except one special case. + if value[3] == 'T' || value[:4] == "WITA" { return 4, true } case 3: diff --git a/src/time/format_test.go b/src/time/format_test.go index aa4434a09c..219c2caee8 100644 --- a/src/time/format_test.go +++ b/src/time/format_test.go @@ -405,6 +405,7 @@ var parseTimeZoneTests = []ParseTimeZoneTest{ {"ESAST hi", 5, true}, {"ESASTT hi", 0, false}, // run of upper-case letters too long. {"ESATY hi", 0, false}, // five letters must end in T. + {"WITA hi", 4, true}, // Issue #18251 } func TestParseTimeZone(t *testing.T) { diff --git a/src/vendor/golang_org/x/crypto/chacha20poly1305/chacha20poly1305_amd64.go b/src/vendor/golang_org/x/crypto/chacha20poly1305/chacha20poly1305_amd64.go index f0d34856b2..4755033212 100644 --- a/src/vendor/golang_org/x/crypto/chacha20poly1305/chacha20poly1305_amd64.go +++ b/src/vendor/golang_org/x/crypto/chacha20poly1305/chacha20poly1305_amd64.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build amd64,go1.7 +// +build go1.7,amd64,!gccgo,!appengine package chacha20poly1305 diff --git a/src/vendor/golang_org/x/crypto/chacha20poly1305/chacha20poly1305_amd64.s b/src/vendor/golang_org/x/crypto/chacha20poly1305/chacha20poly1305_amd64.s index ac9584481d..39c58b44a3 100644 --- a/src/vendor/golang_org/x/crypto/chacha20poly1305/chacha20poly1305_amd64.s +++ b/src/vendor/golang_org/x/crypto/chacha20poly1305/chacha20poly1305_amd64.s @@ -4,7 +4,7 @@ // This file was originally from https://golang.org/cl/24717 by Vlad Krasnov of CloudFlare. -// +build go1.7 +// +build go1.7,amd64,!gccgo,!appengine #include "textflag.h" // General register allocation @@ -209,7 +209,7 @@ GLOBL ·andMask<>(SB), (NOPTR+RODATA), $240 #define polyMul polyMulStage1; polyMulStage2; polyMulStage3; polyMulReduceStage #define polyMulAVX2 polyMulStage1_AVX2; polyMulStage2_AVX2; polyMulStage3_AVX2; polyMulReduceStage // ---------------------------------------------------------------------------- -TEXT polyHashADInternal(SB), NOSPLIT, $0 +TEXT polyHashADInternal<>(SB), NOSPLIT, $0 // adp points to beginning of additional data // itr2 holds ad length XORQ acc0, acc0 @@ -278,8 +278,15 @@ TEXT ·chacha20Poly1305Open(SB), 0, $288-97 MOVQ ad+72(FP), adp // Check for AVX2 support - CMPB runtime·support_avx2(SB), $1 - JE chacha20Poly1305Open_AVX2 + CMPB runtime·support_avx2(SB), $0 + JE noavx2bmi2Open + + // Check BMI2 bit for MULXQ. + // runtime·cpuid_ebx7 is always available here + // because it passed avx2 check + TESTL $(1<<8), runtime·cpuid_ebx7(SB) + JNE chacha20Poly1305Open_AVX2 +noavx2bmi2Open: // Special optimization, for very short buffers CMPQ inl, $128 @@ -315,7 +322,7 @@ openSSEPreparePolyKey: // Hash AAD MOVQ ad_len+80(FP), itr2 - CALL polyHashADInternal(SB) + CALL polyHashADInternal<>(SB) openSSEMainLoop: CMPQ inl, $256 @@ -476,7 +483,7 @@ openSSE128InnerCipherLoop: // Hash MOVQ ad_len+80(FP), itr2 - CALL polyHashADInternal(SB) + CALL polyHashADInternal<>(SB) openSSE128Open: CMPQ inl, $16 @@ -822,7 +829,7 @@ openAVX2PreparePolyKey: // Hash AD + first 64 bytes MOVQ ad_len+80(FP), itr2 - CALL polyHashADInternal(SB) + CALL polyHashADInternal<>(SB) XORQ itr1, itr1 openAVX2InitialHash64: @@ -1014,7 +1021,7 @@ openAVX2192InnerCipherLoop: openAVX2ShortOpen: // Hash MOVQ ad_len+80(FP), itr2 - CALL polyHashADInternal(SB) + CALL polyHashADInternal<>(SB) openAVX2ShortOpenLoop: CMPQ inl, $32 @@ -1485,8 +1492,15 @@ TEXT ·chacha20Poly1305Seal(SB), 0, $288-96 MOVQ ad+72(FP), adp // Check for AVX2 support - CMPB runtime·support_avx2(SB), $1 - JE chacha20Poly1305Seal_AVX2 + CMPB runtime·support_avx2(SB), $0 + JE noavx2bmi2Seal + + // Check BMI2 bit for MULXQ. + // runtime·cpuid_ebx7 is always available here + // because it passed avx2 check + TESTL $(1<<8), runtime·cpuid_ebx7(SB) + JNE chacha20Poly1305Seal_AVX2 +noavx2bmi2Seal: // Special optimization, for very short buffers CMPQ inl, $128 @@ -1547,7 +1561,7 @@ sealSSEIntroLoop: // Hash AAD MOVQ ad_len+80(FP), itr2 - CALL polyHashADInternal(SB) + CALL polyHashADInternal<>(SB) MOVOU (0*16)(inp), A0; MOVOU (1*16)(inp), B0; MOVOU (2*16)(inp), C0; MOVOU (3*16)(inp), D0 PXOR A0, A1; PXOR B0, B1; PXOR C0, C1; PXOR D0, D1 @@ -1691,7 +1705,7 @@ sealSSETail64: MOVO D1, ctr0Store sealSSETail64LoopA: - // Perform ChaCha rounds, while hashing the prevsiosly encrpyted ciphertext + // Perform ChaCha rounds, while hashing the previously encrypted ciphertext polyAdd(0(oup)) polyMul LEAQ 16(oup), oup @@ -1725,7 +1739,7 @@ sealSSETail128: MOVO A0, A1; MOVO B0, B1; MOVO C0, C1; MOVO D0, D1; PADDL ·sseIncMask<>(SB), D1; MOVO D1, ctr1Store sealSSETail128LoopA: - // Perform ChaCha rounds, while hashing the prevsiosly encrpyted ciphertext + // Perform ChaCha rounds, while hashing the previously encrypted ciphertext polyAdd(0(oup)) polyMul LEAQ 16(oup), oup @@ -1771,7 +1785,7 @@ sealSSETail192: MOVO A1, A2; MOVO B1, B2; MOVO C1, C2; MOVO D1, D2; PADDL ·sseIncMask<>(SB), D2; MOVO D2, ctr2Store sealSSETail192LoopA: - // Perform ChaCha rounds, while hashing the prevsiosly encrpyted ciphertext + // Perform ChaCha rounds, while hashing the previously encrypted ciphertext polyAdd(0(oup)) polyMul LEAQ 16(oup), oup @@ -1852,7 +1866,7 @@ sealSSE128InnerCipherLoop: // Hash MOVQ ad_len+80(FP), itr2 - CALL polyHashADInternal(SB) + CALL polyHashADInternal<>(SB) XORQ itr1, itr1 sealSSE128SealHash: @@ -2027,7 +2041,7 @@ sealAVX2IntroLoop: // Hash AD MOVQ ad_len+80(FP), itr2 - CALL polyHashADInternal(SB) + CALL polyHashADInternal<>(SB) // Can store at least 320 bytes VPXOR (0*32)(inp), AA0, AA0 @@ -2290,7 +2304,7 @@ sealAVX2192InnerCipherLoop: sealAVX2ShortSeal: // Hash aad MOVQ ad_len+80(FP), itr2 - CALL polyHashADInternal(SB) + CALL polyHashADInternal<>(SB) XORQ itr1, itr1 sealAVX2SealHash: diff --git a/src/vendor/golang_org/x/crypto/chacha20poly1305/chacha20poly1305_noasm.go b/src/vendor/golang_org/x/crypto/chacha20poly1305/chacha20poly1305_noasm.go index 1d4dcd33fd..4c2eb703c3 100644 --- a/src/vendor/golang_org/x/crypto/chacha20poly1305/chacha20poly1305_noasm.go +++ b/src/vendor/golang_org/x/crypto/chacha20poly1305/chacha20poly1305_noasm.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build !amd64 !go1.7 +// +build !amd64 !go1.7 gccgo appengine package chacha20poly1305 diff --git a/test/bench/go1/fasta_test.go b/test/bench/go1/fasta_test.go index 99d8c9754b..af4fbac274 100644 --- a/test/bench/go1/fasta_test.go +++ b/test/bench/go1/fasta_test.go @@ -12,10 +12,10 @@ var fastabytes = makefasta() func makefasta() []byte { var n int = 25e6 - if runtime.GOARCH == "arm" { + if runtime.GOARCH == "arm" || runtime.GOARCH == "mips" || runtime.GOARCH == "mips64" { // TODO(dfc) remove this limitation after precise gc. // A value of 25e6 consumes 465mb of heap on 32bit - // platforms, which is too much for most ARM systems. + // platforms, which is too much for some systems. // A value of 25e5 produces a memory layout that // confuses the gc on 32bit platforms. So 25e4 it is. n = 25e4 diff --git a/test/fixedbugs/bug500.go b/test/fixedbugs/bug500.go new file mode 100644 index 0000000000..2dd5df13b0 --- /dev/null +++ b/test/fixedbugs/bug500.go @@ -0,0 +1,41 @@ +// run + +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Gccgo generated incorrect GC info when a global variable was +// initialized to a slice of a value containing pointers. The initial +// backing array for the slice was allocated in the .data section, +// which is fine, but the backing array was not registered as a GC +// root. + +package main + +import ( + "runtime" +) + +type s struct { + str string +} + +var a = []struct { + str string +}{ + {""}, +} + +var b = "b" +var c = "c" + +func init() { + a[0].str = b + c +} + +func main() { + runtime.GC() + if a[0].str != b + c { + panic(a[0].str) + } +} diff --git a/test/fixedbugs/bug501.go b/test/fixedbugs/bug501.go new file mode 100644 index 0000000000..8e951b18c8 --- /dev/null +++ b/test/fixedbugs/bug501.go @@ -0,0 +1,24 @@ +// run + +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Gccgo got a compiler crash compiling the addition of more than five +// strings with mixed constants and variables. + +package main + +func F(s string) (string, error) { + return s, nil +} + +func G(a, b, c string) (string, error) { + return F("a" + a + "b" + b + "c" + c) +} + +func main() { + if got, _ := G("x", "y", "z"); got != "axbycz" { + panic(got) + } +} diff --git a/test/fixedbugs/gcc78763.go b/test/fixedbugs/gcc78763.go new file mode 100644 index 0000000000..3e3412753e --- /dev/null +++ b/test/fixedbugs/gcc78763.go @@ -0,0 +1,19 @@ +// compile + +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// The gccgo compiler crashed while compiling this code. +// https://gcc.gnu.org/PR78763. + +package p + +import "unsafe" + +func F() int { + if unsafe.Sizeof(0) == 8 { + return 8 + } + return 0 +} diff --git a/test/fixedbugs/issue10607.go b/test/fixedbugs/issue10607.go index 52fb51ad39..8831547da8 100644 --- a/test/fixedbugs/issue10607.go +++ b/test/fixedbugs/issue10607.go @@ -1,4 +1,4 @@ -// +build linux,!mips,!mipsle,!ppc64 +// +build linux,!ppc64 // run // Copyright 2015 The Go Authors. All rights reserved. diff --git a/test/fixedbugs/issue10958.go b/test/fixedbugs/issue10958.go new file mode 100644 index 0000000000..86d2057622 --- /dev/null +++ b/test/fixedbugs/issue10958.go @@ -0,0 +1,95 @@ +// +build !nacl,disabled +// buildrun -t 10 -gcflags=-d=ssa/insert_resched_checks/on,ssa/check/on + +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Test is disabled because it flakes when run in all.bash +// on some platforms, but is useful standalone to verify +// that rescheduling checks are working (and we may wish +// to investigate the flake, since it suggests that the +// loop rescheduling check may not work right on those +// platforms). + +// This checks to see that call-free infinite loops do not +// block garbage collection. IF YOU RUN IT STANDALONE without +// -gcflags=-d=ssa/insert_resched_checks/on in a not-experimental +// build, it should hang. + +package main + +import ( + "runtime" +) + +var someglobal1 int +var someglobal2 int +var someglobal3 int + +//go:noinline +func f() {} + +func standinacorner1() { + for someglobal1&1 == 0 { + someglobal1++ + someglobal1++ + } +} + +func standinacorner2(i int) { + // contains an irreducible loop containing changes to memory + if i != 0 { + goto midloop + } + +loop: + if someglobal2&1 != 0 { + goto done + } + someglobal2++ +midloop: + someglobal2++ + goto loop + +done: + return +} + +func standinacorner3() { + for someglobal3&1 == 0 { + if someglobal3&2 != 0 { + for someglobal3&3 == 2 { + someglobal3++ + someglobal3++ + someglobal3++ + someglobal3++ + } + } + someglobal3++ + someglobal3++ + someglobal3++ + someglobal3++ + } +} + +func main() { + go standinacorner1() + go standinacorner2(0) + go standinacorner3() + // println("About to stand in a corner1") + for someglobal1 == 0 { + runtime.Gosched() + } + // println("About to stand in a corner2") + for someglobal2 == 0 { + runtime.Gosched() + } + // println("About to stand in a corner3") + for someglobal3 == 0 { + runtime.Gosched() + } + // println("About to GC") + runtime.GC() + // println("Success") +} diff --git a/test/fixedbugs/issue11656.go b/test/fixedbugs/issue11656.go index e0ef097600..c04a66202b 100644 --- a/test/fixedbugs/issue11656.go +++ b/test/fixedbugs/issue11656.go @@ -61,9 +61,9 @@ func f(n int) { binary.BigEndian.PutUint32(ill, 0x7fe00008) // trap case "ppc64le": binary.LittleEndian.PutUint32(ill, 0x7fe00008) // trap - case "mips64": + case "mips", "mips64": binary.BigEndian.PutUint32(ill, 0x00000034) // trap - case "mips64le": + case "mipsle", "mips64le": binary.LittleEndian.PutUint32(ill, 0x00000034) // trap case "s390x": binary.BigEndian.PutUint32(ill, 0) // undefined instruction diff --git a/test/fixedbugs/issue13263.go b/test/fixedbugs/issue13263.go new file mode 100644 index 0000000000..1933f2b5c5 --- /dev/null +++ b/test/fixedbugs/issue13263.go @@ -0,0 +1,15 @@ +// compile + +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package b + +var ( + x uint + y = x + z = uintptr(y) + a = uint32(y) + b = uint64(y) +) diff --git a/test/fixedbugs/issue16130.go b/test/fixedbugs/issue16130.go index 19c8264c6f..c4e3ffd33d 100644 --- a/test/fixedbugs/issue16130.go +++ b/test/fixedbugs/issue16130.go @@ -5,7 +5,7 @@ // license that can be found in the LICENSE file. // Test that an interface conversion error panics with an "interface -// conversion" run-time error. It was (incorrectly) panicing with a +// conversion" run-time error. It was (incorrectly) panicking with a // "nil pointer dereference." package main diff --git a/test/fixedbugs/issue18149.go b/test/fixedbugs/issue18149.go new file mode 100644 index 0000000000..112cd52530 --- /dev/null +++ b/test/fixedbugs/issue18149.go @@ -0,0 +1,33 @@ +// run + +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Verify that //line directives with filenames +// containing ':' (Windows) are correctly parsed. +// (For a related issue, see test/fixedbugs/bug305.go) + +package main + +import ( + "fmt" + "runtime" +) + +func check(file string, line int) { + _, f, l, ok := runtime.Caller(1) + if !ok { + panic("runtime.Caller(1) failed") + } + if f != file || l != line { + panic(fmt.Sprintf("got %s:%d; want %s:%d", f, l, file, line)) + } +} + +func main() { +//line /foo/bar.go:123 + check(`/foo/bar.go`, 123) +//line c:/foo/bar.go:987 + check(`c:/foo/bar.go`, 987) +} diff --git a/test/fixedbugs/issue18392.go b/test/fixedbugs/issue18392.go new file mode 100644 index 0000000000..ad64238983 --- /dev/null +++ b/test/fixedbugs/issue18392.go @@ -0,0 +1,11 @@ +// errorcheck + +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package p + +type A interface { + Fn(A.Fn) // ERROR "type A has no method A.Fn" +} diff --git a/test/fixedbugs/issue18410.go b/test/fixedbugs/issue18410.go new file mode 100644 index 0000000000..e9c6f862eb --- /dev/null +++ b/test/fixedbugs/issue18410.go @@ -0,0 +1,40 @@ +// run + +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This checks partially initialized structure literals +// used to create value.method functions have their +// non-initialized fields properly zeroed/nil'd + +package main + +type X struct { + A, B, C *int +} + +//go:noinline +func (t X) Print() { + if t.B != nil { + panic("t.B must be nil") + } +} + +//go:noinline +func caller(f func()) { + f() +} + +//go:noinline +func test() { + var i, j int + x := X{A: &i, C: &j} + caller(func() { X{A: &i, C: &j}.Print() }) + caller(X{A: &i, C: &j}.Print) + caller(x.Print) +} + +func main() { + test() +} diff --git a/test/fixedbugs/issue18459.go b/test/fixedbugs/issue18459.go new file mode 100644 index 0000000000..ac07661d63 --- /dev/null +++ b/test/fixedbugs/issue18459.go @@ -0,0 +1,13 @@ +// errorcheck + +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Verify that we have a line number for this error. + +package main + +//go:nowritebarrier // ERROR "go:nowritebarrier only allowed in runtime" +func main() { +} diff --git a/test/fixedbugs/issue6772.go b/test/fixedbugs/issue6772.go new file mode 100644 index 0000000000..4d0001c870 --- /dev/null +++ b/test/fixedbugs/issue6772.go @@ -0,0 +1,21 @@ +// errorcheck + +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package p + +func f1() { + for a, a := range []int{1, 2, 3} { // ERROR "a repeated on left side of :=" + println(a) + } +} + +func f2() { + var a int + for a, a := range []int{1, 2, 3} { // ERROR "a repeated on left side of :=" + println(a) + } + println(a) +} diff --git a/test/live.go b/test/live.go index 4fb231cfef..b23e1509e0 100644 --- a/test/live.go +++ b/test/live.go @@ -1,6 +1,7 @@ -// errorcheckwithauto -0 -l -live -wb=0 +// errorcheckwithauto -0 -l -live -wb=0 -d=ssa/insert_resched_checks/off // +build !ppc64,!ppc64le // ppc64 needs a better tighten pass to make f18 pass +// rescheduling checks need to be turned off because there are some live variables across the inserted check call // Copyright 2014 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style diff --git a/test/nosplit.go b/test/nosplit.go index 5f4e62f5b1..e0d531c116 100644 --- a/test/nosplit.go +++ b/test/nosplit.go @@ -261,6 +261,8 @@ TestCases: var buf bytes.Buffer ptrSize := 4 switch goarch { + case "mips", "mipsle": + fmt.Fprintf(&buf, "#define CALL JAL\n#define REGISTER (R0)\n") case "mips64", "mips64le": ptrSize = 8 fmt.Fprintf(&buf, "#define CALL JAL\n#define REGISTER (R0)\n") diff --git a/test/opt_branchlikely.go b/test/opt_branchlikely.go index 5781253e3e..84de32179f 100644 --- a/test/opt_branchlikely.go +++ b/test/opt_branchlikely.go @@ -1,5 +1,6 @@ // +build amd64 -// errorcheck -0 -d=ssa/likelyadjust/debug=1 +// errorcheck -0 -d=ssa/likelyadjust/debug=1,ssa/insert_resched_checks/off +// rescheduling check insertion is turend off because the inserted conditional branches perturb the errorcheck // Copyright 2016 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style diff --git a/test/run.go b/test/run.go index 0dee6b5caa..19ca328765 100644 --- a/test/run.go +++ b/test/run.go @@ -463,6 +463,7 @@ func (t *test) run() { } var args, flags []string + var tim int wantError := false wantAuto := false singlefilepkgs := false @@ -478,7 +479,7 @@ func (t *test) run() { action = "rundir" case "cmpout": action = "run" // the run case already looks for <dir>/<test>.out files - case "compile", "compiledir", "build", "run", "runoutput", "rundir": + case "compile", "compiledir", "build", "run", "buildrun", "runoutput", "rundir": // nothing to do case "errorcheckandrundir": wantError = false // should be no error if also will run @@ -505,6 +506,14 @@ func (t *test) run() { wantError = false case "-s": singlefilepkgs = true + case "-t": // timeout in seconds + args = args[1:] + var err error + tim, err = strconv.Atoi(args[0]) + if err != nil { + t.err = fmt.Errorf("need number of seconds for -t timeout, got %s instead", args[0]) + } + default: flags = append(flags, args[0]) } @@ -539,7 +548,31 @@ func (t *test) run() { } else { cmd.Env = os.Environ() } - err := cmd.Run() + + var err error + + if tim != 0 { + err = cmd.Start() + // This command-timeout code adapted from cmd/go/test.go + if err == nil { + tick := time.NewTimer(time.Duration(tim) * time.Second) + done := make(chan error) + go func() { + done <- cmd.Wait() + }() + select { + case err = <-done: + // ok + case <-tick.C: + cmd.Process.Kill() + err = <-done + // err = errors.New("Test timeout") + } + tick.Stop() + } + } else { + err = cmd.Run() + } if err != nil { err = fmt.Errorf("%s\n%s", err, buf.Bytes()) } @@ -671,6 +704,32 @@ func (t *test) run() { t.err = err } + case "buildrun": // build binary, then run binary, instead of go run. Useful for timeout tests where failure mode is infinite loop. + // TODO: not supported on NaCl + useTmp = true + cmd := []string{"go", "build", "-o", "a.exe"} + if *linkshared { + cmd = append(cmd, "-linkshared") + } + longdirgofile := filepath.Join(filepath.Join(cwd, t.dir), t.gofile) + cmd = append(cmd, flags...) + cmd = append(cmd, longdirgofile) + out, err := runcmd(cmd...) + if err != nil { + t.err = err + return + } + cmd = []string{"./a.exe"} + out, err = runcmd(append(cmd, args...)...) + if err != nil { + t.err = err + return + } + + if strings.Replace(string(out), "\r\n", "\n", -1) != t.expectedOutput() { + t.err = fmt.Errorf("incorrect output\n%s", out) + } + case "run": useTmp = false cmd := []string{"go", "run"} diff --git a/test/zerodivide.go b/test/zerodivide.go index 9ab2713535..214d481164 100644 --- a/test/zerodivide.go +++ b/test/zerodivide.go @@ -28,6 +28,8 @@ var ( i32, j32, k32 int32 = 0, 0, 1 i64, j64, k64 int64 = 0, 0, 1 + bb = []int16{2, 0} + u, v, w uint = 0, 0, 1 u8, v8, w8 uint8 = 0, 0, 1 u16, v16, w16 uint16 = 0, 0, 1 @@ -124,6 +126,10 @@ var errorTests = []ErrorTest{ ErrorTest{"int32 1/0", func() { use(k32 / j32) }, "divide"}, ErrorTest{"int64 1/0", func() { use(k64 / j64) }, "divide"}, + // From issue 5790, we should ensure that _ assignments + // still evaluate and generate zerodivide panics. + ErrorTest{"int16 _ = bb[0]/bb[1]", func() { _ = bb[0] / bb[1] }, "divide"}, + ErrorTest{"uint 0/0", func() { use(u / v) }, "divide"}, ErrorTest{"uint8 0/0", func() { use(u8 / v8) }, "divide"}, ErrorTest{"uint16 0/0", func() { use(u16 / v16) }, "divide"}, @@ -195,9 +201,6 @@ func alike(a, b float64) bool { func main() { bad := false for _, t := range errorTests { - if t.err != "" { - continue - } err := error_(t.fn) switch { case t.err == "" && err == "": |