diff options
| author | Filippo Valsorda <filippo@golang.org> | 2018-07-02 18:35:13 -0400 |
|---|---|---|
| committer | Filippo Valsorda <filippo@golang.org> | 2018-07-02 18:35:13 -0400 |
| commit | 35ba5284935c4ea67060b0a6ed15142c415e256d (patch) | |
| tree | f323f61f58ed137f23e6b2aa35ad633c3e96962a | |
| parent | 497161dfd93ba9c6126288d4f4f8f4b0633c8d7c (diff) | |
| parent | fe8a0d12b14108cbe2408b417afcaab722b0727c (diff) | |
| download | go-35ba5284935c4ea67060b0a6ed15142c415e256d.tar.gz go-35ba5284935c4ea67060b0a6ed15142c415e256d.zip | |
[dev.boringcrypto.go1.10] all: merge go1.10.3 into dev.boringcrypto.go1.10
Change-Id: I22bd10efc446cab59f492382e885881273ff5783
106 files changed, 2947 insertions, 1300 deletions
diff --git a/.github/PULL_REQUEST_TEMPLATE b/.github/PULL_REQUEST_TEMPLATE deleted file mode 100644 index 00014e9b86..0000000000 --- a/.github/PULL_REQUEST_TEMPLATE +++ /dev/null @@ -1,7 +0,0 @@ -Please do not send pull requests to the golang/* repositories. - -We do, however, take contributions gladly. - -See https://golang.org/doc/contribute.html - -Thanks! diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4f2c4d4b02..7c1dd54b30 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -30,11 +30,6 @@ For change proposals, see [Proposing Changes To Go](https://github.com/golang/pr Please read the [Contribution Guidelines](https://golang.org/doc/contribute.html) before sending patches. -**We do not accept GitHub pull requests** -(we use [an instance](https://go-review.googlesource.com/) of the -[Gerrit](https://www.gerritcodereview.com/) code review system instead). -Also, please do not post patches on the issue tracker. - Unless otherwise noted, the Go source files are distributed under the BSD-style license found in the LICENSE file. @@ -36,10 +36,9 @@ Go is the work of hundreds of contributors. We appreciate your help! To contribute, please read the contribution guidelines: https://golang.org/doc/contribute.html -Note that the Go project does not use GitHub pull requests, and that -we use the issue tracker for bug reports and proposals only. See -https://golang.org/wiki/Questions for a list of places to ask -questions about the Go language. +Note that the Go project uses the issue tracker for bug reports and +proposals only. See https://golang.org/wiki/Questions for a list of +places to ask questions about the Go language. [rf]: https://reneefrench.blogspot.com/ [cc3-by]: https://creativecommons.org/licenses/by/3.0/ diff --git a/doc/articles/wiki/index.html b/doc/articles/wiki/index.html index 3e0d532d7f..b7ab2cc622 100644 --- a/doc/articles/wiki/index.html +++ b/doc/articles/wiki/index.html @@ -269,6 +269,12 @@ view a wiki page. It will handle URLs prefixed with "/view/". {{code "doc/articles/wiki/part2.go" `/^func viewHandler/` `/^}/`}} <p> +Again, note the use of <code>_</code> to ignore the <code>error</code> +return value from <code>loadPage</code>. This is done here for simplicity +and generally considered bad practice. We will attend to this later. +</p> + +<p> First, this function extracts the page title from <code>r.URL.Path</code>, the path component of the request URL. The <code>Path</code> is re-sliced with <code>[len("/view/"):]</code> to drop @@ -283,12 +289,6 @@ HTML, and writes it to <code>w</code>, the <code>http.ResponseWriter</code>. </p> <p> -Again, note the use of <code>_</code> to ignore the <code>error</code> -return value from <code>loadPage</code>. This is done here for simplicity -and generally considered bad practice. We will attend to this later. -</p> - -<p> To use this handler, we rewrite our <code>main</code> function to initialize <code>http</code> using the <code>viewHandler</code> to handle any requests under the path <code>/view/</code>. diff --git a/doc/asm.html b/doc/asm.html index 3a05d46aeb..f2f8fad576 100644 --- a/doc/asm.html +++ b/doc/asm.html @@ -739,6 +739,13 @@ The ARM64 port is in an experimental state. </p> <p> +<code>R18</code> is the "platform register", reserved on the Apple platform. +<code>R27</code> and <code>R28</code> are reserved by the compiler and linker. +<code>R29</code> is the frame pointer. +<code>R30</code> is the link register. +</p> + +<p> Instruction modifiers are appended to the instruction following a period. The only modifiers are <code>P</code> (postincrement) and <code>W</code> (preincrement): @@ -752,11 +759,61 @@ Addressing modes: <ul> <li> -<code>(R5, R6)</code>: Register pair for <code>LDP</code>/<code>STP</code>. +<code>R0->16</code> +<br> +<code>R0>>16</code> +<br> +<code>R0<<16</code> +<br> +<code>R0@>16</code>: +These are the same as on the 32-bit ARM. +</li> + +<li> +<code>$(8<<12)</code>: +Left shift the immediate value <code>8</code> by <code>12</code> bits. +</li> + +<li> +<code>8(R0)</code>: +Add the value of <code>R0</code> and <code>8</code>. +</li> + +<li> +<code>(R2)(R0)</code>: +The location at <code>R0</code> plus <code>R2</code>. +</li> + +<li> +<code>R0.UXTB</code> +<br> +<code>R0.UXTB<<imm</code>: +<code>UXTB</code>: extract an 8-bit value from the low-order bits of <code>R0</code> and zero-extend it to the size of <code>R0</code>. +<code>R0.UXTB<<imm</code>: left shift the result of <code>R0.UXTB</code> by <code>imm</code> bits. +The <code>imm</code> value can be 0, 1, 2, 3, or 4. +The other extensions include <code>UXTH</code> (16-bit), <code>UXTW</code> (32-bit), and <code>UXTX</code> (64-bit). +</li> + +<li> +<code>R0.SXTB</code> +<br> +<code>R0.SXTB<<imm</code>: +<code>SXTB</code>: extract an 8-bit value from the low-order bits of <code>R0</code> and sign-extend it to the size of <code>R0</code>. +<code>R0.SXTB<<imm</code>: left shift the result of <code>R0.SXTB</code> by <code>imm</code> bits. +The <code>imm</code> value can be 0, 1, 2, 3, or 4. +The other extensions include <code>SXTH</code> (16-bit), <code>SXTW</code> (32-bit), and <code>SXTX</code> (64-bit). +</li> + +<li> +<code>(R5, R6)</code>: Register pair for <code>LDAXP</code>/<code>LDP</code>/<code>LDXP</code>/<code>STLXP</code>/<code>STP</code>/<code>STP</code>. </li> </ul> +<p> +Reference: <a href="/pkg/cmd/internal/obj/arm64">Go ARM64 Assembly Instructions Reference Manual</a> +</p> + <h3 id="ppc64">64-bit PowerPC, a.k.a. ppc64</h3> <p> @@ -882,6 +939,12 @@ The value of <code>GOMIPS</code> environment variable (<code>hardfloat</code> or <code>GOMIPS_hardfloat</code> or <code>GOMIPS_softfloat</code>. </p> +<p> +The value of <code>GOMIPS64</code> environment variable (<code>hardfloat</code> or +<code>softfloat</code>) is made available to assembly code by predefining either +<code>GOMIPS64_hardfloat</code> or <code>GOMIPS64_softfloat</code>. +</p> + <h3 id="unsupported_opcodes">Unsupported opcodes</h3> <p> diff --git a/doc/conduct.html b/doc/conduct.html index bf52ef9fd5..f5f2fe148b 100644 --- a/doc/conduct.html +++ b/doc/conduct.html @@ -13,15 +13,14 @@ ul ul { } </style> -<h2 id="about">About the Code of Conduct</h2> - -<h3 id="why">Why have a Code of Conduct?</h3> +<h2 id="about">About</h2> <p> Online communities include people from many different backgrounds. The Go contributors are committed to providing a friendly, safe and welcoming -environment for all, regardless of age, disability, gender, nationality, -ethnicity, religion, sexuality, or similar personal characteristic. +environment for all, regardless of gender identity and expression, sexual orientation, +disabilities, neurodiversity, physical appearance, body size, ethnicity, nationality, +race, age, religion, or similar personal characteristics. </p> <p> @@ -44,35 +43,9 @@ contributors and users from all backgrounds. </p> <p> -With that said, a healthy community must allow for disagreement and debate. -The Code of Conduct is not a mechanism for people to silence others with whom -they disagree. -</p> - -<h3 id="spaces">Where does the Code of Conduct apply?</h3> - -<p> -If you participate in or contribute to the Go ecosystem in any way, -you are encouraged to follow the Code of Conduct while doing so. -</p> - -<p> -Explicit enforcement of the Code of Conduct applies to the -official forums operated by the Go project (“Go spaces”): -</p> - -<ul> - <li>The official <a href="https://github.com/golang/">GitHub projects</a> - and <a href="https://go-review.googlesource.com/">code reviews</a>. - <li>The <a href="https://groups.google.com/group/golang-nuts">golang-nuts</a> and - <a href="https://groups.google.com/group/golang-dev">golang-dev</a> mailing lists. -<li>The #go-nuts IRC channel on Freenode. -</ul> - -<p> -Other Go groups (such as conferences, meetups, and other unofficial forums) are -encouraged to adopt this Code of Conduct. Those groups must provide their own -moderators and/or working group (see below). +We believe that healthy debate and disagreement are essential to a healthy project and community. +However, it is never ok to be disrespectful. +We value diverse opinions, but we value respectful behavior more. </p> <h2 id="values">Gopher values</h2> @@ -129,118 +102,104 @@ Even if the intent was to provoke, do not rise to it. It is the responsibility of <i>all parties</i> to de-escalate conflict when it arises. </p> -<h2 id="unwelcome_behavior">Unwelcome behavior</h2> - -<p> -These actions are explicitly forbidden in Go spaces: -</p> - -<ul> -<li>Insulting, demeaning, hateful, or threatening remarks. -<li>Discrimination based on age, disability, gender, nationality, race, - religion, sexuality, or similar personal characteristic. -<li>Bullying or systematic harassment. -<li>Unwelcome sexual advances. -<li>Incitement to any of these. -</ul> - -<h2 id="moderation">Moderation</h2> - -<p> -The Go spaces are not free speech venues; they are for discussion about Go. -Each of these spaces have their own moderators. -</p> - -<p> -When using the official Go spaces you should act in the spirit of the “Gopher -values”. -If a reported conflict cannot be resolved amicably, the CoC Working Group -may make a recommendation to the relevant forum moderators. -</p> +<h2 id="code">Code of Conduct</h2> -<p> -CoC Working Group members and forum moderators are held to a higher standard than other community members. -If a working group member or moderator creates an inappropriate situation, they -should expect less leeway than others, and should expect to be removed from -their position if they cannot adhere to the CoC. -</p> +<h3 id="our-pledge">Our Pledge</h3> -<p> -Complaints about working group member or moderator actions must be handled -using the reporting process below. -</p> +<p>In the interest of fostering an open and welcoming environment, we as +contributors and maintainers pledge to making participation in our project and +our community a harassment-free experience for everyone, regardless of age, body +size, disability, ethnicity, gender identity and expression, level of +experience, education, socio-economic status, nationality, personal appearance, +race, religion, or sexual identity and orientation.</p> -<h2 id="reporting">Reporting issues</h2> +<h3 id="our-standards">Our Standards</h3> -<p> -The Code of Conduct Working Group is a group of people that represent the Go -community. They are responsible for handling conduct-related issues. -Their purpose is to de-escalate conflicts and try to resolve issues to the -satisfaction of all parties. They are: -</p> +<p>Examples of behavior that contributes to creating a positive environment +include:</p> <ul> - <li>Aditya Mukerjee <dev@chimeracoder.net> - <li>Andrew Gerrand <adg@golang.org> - <li>Peggy Li <peggyli.224@gmail.com> - <li>Steve Francia <steve.francia@gmail.com> - <li>Verónica López <gveronicalg@gmail.com> +<li>Using welcoming and inclusive language</li> +<li>Being respectful of differing viewpoints and experiences</li> +<li>Gracefully accepting constructive criticism</li> +<li>Focusing on what is best for the community</li> +<li>Showing empathy towards other community members</li> </ul> -<p> -If you encounter a conduct-related issue, you should report it to the -Working Group using the process described below. -<b>Do not</b> post about the issue publicly or try to rally sentiment against a -particular individual or group. -</p> +<p>Examples of unacceptable behavior by participants include:</p> <ul> -<li>Mail <a href="mailto:conduct@golang.org">conduct@golang.org</a>. - <ul> - <li>Your message will reach the Working Group. - <li>Reports are confidential within the Working Group. - <li>You may contact a member of the group directly if you do not feel - comfortable contacting the group as a whole. That member will then raise - the issue with the Working Group as a whole, preserving the privacy of the - reporter (if desired). - <li>If your report concerns a member of the Working Group they will be recused - from Working Group discussions of the report. - <li>The Working Group will strive to handle reports with discretion and - sensitivity, to protect the privacy of the involved parties, - and to avoid conflicts of interest. - </ul> -<li>You should receive a response within 48 hours (likely sooner). - (Should you choose to contact a single Working Group member, - it may take longer to receive a response.) -<li>The Working Group will meet to review the incident and determine what happened. - <ul> - <li>With the permission of person reporting the incident, the Working Group - may reach out to other community members for more context. - </ul> -<li>The Working Group will reach a decision as to how to act. These may include: - <ul> - <li>Nothing. - <li>Passing the report along to the offender. - <li>A recommendation of action to the relevant forum moderators. - </ul> -<li>The Working Group will reach out to the original reporter to let them know - the decision. -<li>Appeals to the decision may be made to the Working Group, - or to any of its members directly. +<li>The use of sexualized language or imagery and unwelcome sexual attention or +advances</li> +<li>Trolling, insulting/derogatory comments, and personal or political attacks</li> +<li>Public or private harassment</li> +<li>Publishing others’ private information, such as a physical or electronic +address, without explicit permission</li> +<li>Other conduct which could reasonably be considered inappropriate in a +professional setting</li> </ul> -<p> -<b>Note that the goal of the Code of Conduct and the Working Group is to resolve -conflicts in the most harmonious way possible.</b> -We hope that in most cases issues may be resolved through polite discussion and -mutual agreement. -</p> - -<p> -Changes to the Code of Conduct (including to the members of the Working Group) -should be proposed using the -<a href="https://golang.org/s/proposal-process">change proposal process</a>. -</p> +<h3 id="our-responsibilities">Our Responsibilities</h3> + +<p>Project maintainers are responsible for clarifying the standards of acceptable +behavior and are expected to take appropriate and fair corrective action in +response to any instances of unacceptable behavior.</p> + +<p>Project maintainers have the right and responsibility to remove, edit, or reject +comments, commits, code, wiki edits, issues, and other contributions that are +not aligned to this Code of Conduct, or to ban temporarily or permanently any +contributor for other behaviors that they deem inappropriate, threatening, +offensive, or harmful.</p> + +<h3 id="scope">Scope</h3> + +<p>This Code of Conduct applies both within project spaces and in public spaces +when an individual is representing the project or its community. Examples of +representing a project or community include using an official project e-mail +address, posting via an official social media account, or acting as an appointed +representative at an online or offline event. Representation of a project may be +further defined and clarified by project maintainers.</p> + +<p>This Code of Conduct also applies outside the project spaces when the Project +Steward has a reasonable belief that an individual’s behavior may have a +negative impact on the project or its community.</p> + +<h3 id="conflict-resolution"></a>Conflict Resolution</h3> + +<p>We do not believe that all conflict is bad; healthy debate and disagreement +often yield positive results. However, it is never okay to be disrespectful or +to engage in behavior that violates the project’s code of conduct.</p> + +<p>If you see someone violating the code of conduct, you are encouraged to address +the behavior directly with those involved. Many issues can be resolved quickly +and easily, and this gives people more control over the outcome of their +dispute. If you are unable to resolve the matter for any reason, or if the +behavior is threatening or harassing, report it. We are dedicated to providing +an environment where participants feel welcome and safe.</p> + +<p id="reporting">Reports should be directed to Cassandra Salisbury, the +Go Project Steward, at <i>conduct@golang.org</i>. +It is the Project Steward’s duty to +receive and address reported violations of the code of conduct. They will then +work with a committee consisting of representatives from the Open Source +Programs Office and the Google Open Source Strategy team. If for any reason you +are uncomfortable reaching out the Project Steward, please email +the Google Open Source Programs Office at <i>opensource@google.com</i>.</p> + +<p>We will investigate every complaint, but you may not receive a direct response. +We will use our discretion in determining when and how to follow up on reported +incidents, which may range from not taking action to permanent expulsion from +the project and project-sponsored spaces. We will notify the accused of the +report and provide them an opportunity to discuss it before any action is taken. +The identity of the reporter will be omitted from the details of the report +supplied to the accused. In potentially harmful situations, such as ongoing +harassment or threats to anyone’s safety, we may take action without notice.</p> + +<h3 id="attribution">Attribution</h3> + +<p>This Code of Conduct is adapted from the Contributor Covenant, version 1.4, +available at +<a href="https://www.contributor-covenant.org/version/1/4/code-of-conduct.html">https://www.contributor-covenant.org/version/1/4/code-of-conduct.html</a></p> <h2 id="summary">Summary</h2> @@ -250,10 +209,3 @@ should be proposed using the <li>Don’t be destructive or inflammatory. <li>If you encounter an issue, please mail <a href="mailto:conduct@golang.org">conduct@golang.org</a>. </ul> - -<h3 id="acknowledgements">Acknowledgements</h3> - -<p> -Parts of this document were derived from the Code of Conduct documents of the -Django, FreeBSD, and Rust projects. -</p> diff --git a/doc/contribute.html b/doc/contribute.html index e5312becbb..d802bd72a1 100644 --- a/doc/contribute.html +++ b/doc/contribute.html @@ -3,39 +3,65 @@ }--> <p> -The Go project welcomes all contributors. The process of contributing -to the Go project may be different than many projects you are used to. -This document is intended as a guide to help you through the contribution -process. This guide assumes you have a basic understanding of Git and Go. +The Go project welcomes all contributors. </p> <p> -(Note that the <code>gccgo</code> frontend lives elsewhere; -see <a href="gccgo_contribute.html">Contributing to gccgo</a>.) +This document is a guide to help you through the process +of contributing to the Go project, which is a little different +from that used by other open source projects. +We assume you have a basic understanding of Git and Go. </p> <p> -Sensitive security-related issues should be reported to <a href="mailto:security@golang.org">security@golang.org</a>. +In addition to the information here, the Go community maintains a +<a href="https://golang.org/wiki/CodeReview">CodeReview</a> wiki page. +Feel free to contribute to the wiki as you learn the review process. </p> -<h1 id="contributor">Becoming a contributor</h1> +<p> +Note that the <code>gccgo</code> front end lives elsewhere; +see <a href="gccgo_contribute.html">Contributing to gccgo</a>. +</p> + +<h2 id="contributor">Becoming a contributor</h2> + +<h3>Overview</h3> <p> -Before you can contribute to the Go project you need to setup a few prerequisites. -The Go project uses <a href="https://www.gerritcodereview.com/">Gerrit</a>, an open -source online tool, to perform all code reviews. -Gerrit uses your email address as a unique identifier. -The Go project contributing flow is currently configured to work only with Google Accounts. -You must go through the following process <em>prior to contributing</em>. -You only need to do this once per Google Account. +The first step is registering as a Go contributor and configuring your environment. +Here is a checklist of the required steps to follow: </p> -<h2 id="go-contrib-init">Automatically set up & diagnose your development environment</h2> +<ul> +<li> +<b>Step 0</b>: Decide on a single Google Account you will be using to contribute to Go. +Use that account for all the following steps and make sure that <code>git</code> +is configured to create commits with that account's e-mail address. +</li> +<li> +<b>Step 1</b>: <a href="https://cla.developers.google.com/clas">Sign and submit</a> a +CLA (Contributor License Agreement). +</li> +<li> +<b>Step 2</b>: Configure authentication credentials for the Go Git repository. +Visit <a href="https://go.googlesource.com/">go.googlesource.com</a>, click +on "Generate Password" (top right), and follow the instructions. +</li> +<li> +<b>Step 3</b>: Register for Gerrit, the code review tool used by the Go team, +by <a href="https://go-review.googlesource.com/login/">visiting this page</a>. +The CLA and the registration need to be done only once for your account. +</li> +<li> +<b>Step 4</b>: Install <code>git-codereview</code> by running +<code>go get -u golang.org/x/review/git-codereview</code> +</li> +</ul> + <p> - The <code>go-contrib-init</code> tool configures and debugs your Go - development environment, automatically performing many of the steps - on this page, or telling you what you need to do next. If you wish - to use it, run: +If you prefer, there is an automated tool that walks through these steps. +Just run: </p> <pre> @@ -45,66 +71,61 @@ $ go-contrib-init </pre> <p> - The tool will either set things up, tell you that everything is - configured, or tell you what steps you need to do manually. +The rest of this chapter elaborates on these instructions. +If you have completed the steps above (either manually or through the tool), jump to +<a href="#making_a_change">Making a change</a>. </p> -<h2 id="auth">Configure Git to use Gerrit</h2> -<p> -You'll need a web browser and a command line terminal. -You should already have Git installed. -</p> +<h3 id="google_account">Step 0: Select a Google Account</h3> <p> -Gerrit uses Google Accounts for authentication. -If you don't have a Google Account, you can create an account which -<a href="https://www.google.com/accounts/NewAccount">includes -a new Gmail email account</a> or create an account associated -<a href="https://accounts.google.com/SignUpWithoutGmail">with your existing -email address</a>. +A contribution to Go is made through a Google account with a specific +e-mail address. +Make sure to use the same account throughout the process and +for all your subsequent contributions. +You may need to decide whether to use a personal address or a corporate address. +The choice will depend on who +will own the copyright for the code that you will be writing +and submitting. +You might want to discuss this topic with your employer before deciding which +account to use. </p> -<h3>Step 1: Sign in to googlesource and generate a password</h3> - <p> -Visit <a href="https://go.googlesource.com">go.googlesource.com</a> -and click on "Generate Password" in the page's top right menu bar. -You will be redirected to accounts.google.com to sign in. +Google accounts can either be Gmail e-mail accounts, G-Suite organization accounts, or +accounts associated with an external e-mail address. +For instance, if you need to use +an existing corporate e-mail that is not managed through G-Suite, you can create +an account associated +<a href="https://accounts.google.com/SignUpWithoutGmail">with your existing +e-mail address</a>. </p> -<h3>Step 2: Run the provided script</h3> <p> -After signing in, you are taken to a page on go.googlesource.com with the title "Configure Git". -This page contains a personalized script which when run locally will configure git -to have your unique authentication key. -This key is paired with one generated server side similar to how ssh keys work. +You also need to make sure that your Git tool is configured to create commits +using your chosen e-mail address. +You can either configure Git globally +(as a default for all projects), or locally (for a single specific project). +You can check the current configuration with this command: </p> -<p> -Copy and run this script locally in your command line terminal. -(On a Windows computer using cmd you should instead follow the instructions -in the yellow box to run the command. If you are using git-bash use the same -script as *nix.) -</p> +<pre> +$ git config --global user.email # check current global config +$ git config user.email # check current local config +</pre> <p> -Your secret authentication token is now in a <code>.gitcookies</code> file -and Git is configured to use this file. +To change the configured address: </p> -<h3 id="gerrit">Step 3: Register with Gerrit</h3> +<pre> +$ git config --global user.email name@example.com # change global config +$ git config user.email name@example.com # change local config +</pre> -<p> -Now that you have your authentication token, you need to register your -account with Gerrit. -To do this, visit <a href="https://go-review.googlesource.com/login/"> -go-review.googlesource.com/login/</a>. -Sign in using the same Google Account you used above. -</p> -<h2 id="cla">Contributor License Agreement</h2> +<h3 id="cla">Step 1: Contributor License Agreement</h3> -<h3 id="which_cla">Which CLA</h3> <p> Before sending your first change to the Go project you must have completed one of the following two CLAs. @@ -126,42 +147,77 @@ contributor license agreement</a>.<br> </ul> <p> -<i>If the copyright holder for your contribution has already completed the +You can check your currently signed agreements and sign new ones at +the <a href="https://cla.developers.google.com/clas?pli=1&authuser=1">Google Developers +Contributor License Agreements</a> website. +If the copyright holder for your contribution has already completed the agreement in connection with another Google open source project, -it does not need to be completed again.</i> +it does not need to be completed again. +</p> + +<p> +If the copyright holder for the code you are submitting changes—for example, +if you start contributing code on behalf of a new company—please send mail +to the <a href="mailto:golang-dev@googlegroups.com"><code>golang-dev</code> +mailing list</a>. +This will let us know the situation so we can make sure an appropriate agreement is +completed and update the <code>AUTHORS</code> file. </p> -<h3 id="signing_cla">Completing the CLA</h3> + +<h3 id="config_git_auth">Step 2: Configure git authentication</h3> <p> -You can see your currently signed agreements and sign new ones through the Gerrit -interface. -To do this, <a href="https://go-review.googlesource.com/login/">Log into Gerrit</a>, -then visit the <a href="https://go-review.googlesource.com/settings/agreements">Agreements</a> -page. -If you do not have a signed agreement listed there, you can create one -by clicking "New Contributor Agreement" and following the steps. +Go development happens on +<a href="https://go.googlesource.com">go.googlesource.com</a>, +a Git server hosted by Google. +Authentication on the web server is made through your Google account, but +you also need to configure <code>git</code> on your computer to access it. +Follow this steps: </p> +<ol> +<li> +Visit <a href="https://go.googlesource.com">go.googlesource.com</a> +and click on "Generate Password" in the page's top right menu bar. +You will be redirected to accounts.google.com to sign in. +</li> +<li> +After signing in, you will be taken to a page with the title "Configure Git". +This page contains a personalized script that when run locally will configure Git +to hold your unique authentication key. +This key is paired with one that is generated and stored on the server, +analogous to how SSH keys work. +</li> +<li> +Copy and run this script locally in your command line terminal to store your +secret authentication token in a <code>.gitcookies</code> file. +If you are using a Windows computer and running <code>cmd</code>, +you should instead follow the instructions in the yellow box to run the command; +otherwise run the regular script. +</li> +</ol> + +<h3 id="auth">Step 3: Create a Gerrit account </h3> + <p> -If the copyright holder for the code you are submitting changes — for example, -if you start contributing code on behalf of a new company — please send email -to golang-dev and let us know, so that we can make sure an appropriate agreement is -completed and update the <code>AUTHORS</code> file. +Gerrit is an open-source tool used by Go maintainers to discuss and review +code submissions. +</p> + +<p> +To register your account, visit <a href="https://go-review.googlesource.com/login/"> +go-review.googlesource.com/login/</a> and sign in once using the same Google Account you used above. </p> -<span id="Code_review"></span> -<h1 id="prepare_dev_env">Preparing a Development Environment for Contributing</h1> +<h3 id="git-codereview_install">Step 4: Install the git-codereview command</h3> -<h2 id="git-codereview">Setting up Git for submission to Gerrit</h2> <p> Changes to Go must be reviewed before they are accepted, no matter who makes the change. -A custom git command called <code>git-codereview</code>, discussed below, -helps manage the code review process through a Google-hosted -<a href="https://go-review.googlesource.com/">instance</a> of Gerrit. +A custom <code>git</code> command called <code>git-codereview</code> +simplifies sending changes to Gerrit. </p> -<h3 id="git-codereview_install">Install the git-codereview command</h3> <p> Install the <code>git-codereview</code> command by running, </p> @@ -172,7 +228,8 @@ $ go get -u golang.org/x/review/git-codereview <p> Make sure <code>git-codereview</code> is installed in your shell path, so that the -<code>git</code> command can find it. Check that +<code>git</code> command can find it. +Check that </p> <pre> @@ -185,439 +242,686 @@ prints help text, not an error. <p> On Windows, when using git-bash you must make sure that -<code>git-codereview.exe</code> is in your git exec-path. +<code>git-codereview.exe</code> is in your <code>git</code> exec-path. Run <code>git --exec-path</code> to discover the right location then create a -symbolic link or simply copy the executable from $GOPATH/bin to this directory. +symbolic link or just copy the executable from $GOPATH/bin to this directory. +</p> + + +<h2 id="before_contributing">Before contributing code</h2> + +<p> +The project welcomes submissions but to make sure things are well +coordinated we ask that everyone to discuss any significant changes to the +Go repositories before starting work. +Best practice is to connect your work to the issue tracker, +either by <a href="https://golang.org/issue/new">filing a new issue</a> +or by claiming an <a href="https://golang.org/issues">existing issue</a>. </p> +<h3>Check the issue tracker</h3> + <p> -<b>Note to Git aficionados:</b> -The <code>git-codereview</code> command is not required to -upload and manage Gerrit code reviews. -For those who prefer plain Git, the text below gives the Git equivalent of -each git-codereview command. +Whether you already know what contribution to make, or you are searching for +an idea, the <a href="https://github.com/golang/go/issues">issue tracker</a> is +always the first place to go. +Issues are triaged to categorize them and manage the workflow. </p> <p> -If you do use plain Git, note that you still need the commit hooks that the -git-codereview command configures; those hooks add a Gerrit -<code>Change-Id</code> line to the commit message and check that all Go source -files have been formatted with gofmt. -Even if you intend to use plain Git for -daily work, install the hooks in a new Git checkout by running -<code>git-codereview</code> <code>hooks</code>. +Most issues will be marked with one of the following workflow labels: </p> -<h3 id="git-config">Set up git aliases</h3> +<ul> + <li> + <b>NeedsInvestigation</b>: The issue is not fully understood + and requires analysis to understand the root cause. + </li> + <li> + <b>NeedsDecision</b>: the issue is relatively well understood, but the + Go team hasn't yet decided the best way to address it. + It would be better to wait for a decision before writing code. + If you are interested on working on an issue in this state, + feel free to "ping" maintainers in the issue's comments + if some time has passed without a decision. + </li> + <li> + <b>NeedsFix</b>: the issue is fully understood and code can be written + to fix it. + </li> +</ul> + +<h3 id="design">Open an issue for any new problem</h3> <p> -The <code>git-codereview</code> command can be run directly from the shell -by typing, for instance, +Excluding very trivial changes, all contributions should be connected +to an existing issue. +Feel free to open one and discuss your plans. +This process gives everyone a chance to validate the design, +helps prevent duplication of effort, +and ensures that the idea fits inside the goals for the language and tools. +It also checks that the design is sound before code is written; +the code review tool is not the place for high-level discussions. </p> -<pre> -$ git codereview sync -</pre> +<p> +When planning work, please note that the Go project follows a <a +href="https://golang.org/wiki/Go-Release-Cycle">six-month development cycle</a>. +The latter half of each cycle is a three-month feature freeze during +which only bug fixes and documentation updates are accepted. +New contributions can be +sent during a feature freeze but will not be accepted until the freeze is over. +</p> <p> -but it is more convenient to set up aliases for <code>git-codereview</code>'s own -subcommands, so that the above becomes, +Changes in general other than bug and documentation fixes +must go through the +<a href="https://golang.org/s/proposal-process">change proposal process</a> +before they can be accepted. </p> -<pre> -$ git sync -</pre> +<p> +Sensitive security-related issues (only!) should be reported to <a href="mailto:security@golang.org">security@golang.org</a>. +</p> + +<h2 id="sending_a_change_github">Sending a change via GitHub</h2> <p> -The <code>git-codereview</code> subcommands have been chosen to be distinct from -Git's own, so it's safe to do so. +First-time contributors that are already familiar with the +<a href="https://guides.github.com/introduction/flow/">GitHub flow</a> +are encouraged to use the same process for Go contributions. +Even though Go +maintainers use Gerrit for code review, a bot called Gopherbot has been created to sync +GitHub pull requests to Gerrit. +</p> + +<p> +Open a pull request as you normally would. +Gopherbot will automatically +sync the code and post a link to Gerrit. +When somebody comments on the +change, it will be posted in the pull request, so you will also get a notification. </p> <p> -The aliases are optional, but in the rest of this document we will assume -they are installed. -To install them, copy this text into your Git configuration file -(usually <code>.gitconfig</code> in your home directory): +Some things to keep in mind: </p> +<ul> +<li> +To update the pull request with new code, just push it to the branch; you can either +add more commits, or rebase and force-push (both styles are accepted). +</li> +<li> +If the request is accepted, all commits will be squashed, and the final +commit description will be composed by concatenating the pull request's +title and description. +The individual commits' descriptions will be discarded. +See <a href="#commit_messages">Writing good commit messages</a> for some +suggestions. +</li> +<li> +Gopherbot is unable to sync line-by-line codereview into GitHub: only the +contents of the overall comment on the request will be synced. +Remember you can always visit Gerrit to see the fine-grained review. +</li> +</ul> + +<h2 id="sending_a_change_gerrit">Sending a change via Gerrit</h2> + +<p> +It is not possible to fully sync Gerrit and GitHub, at least at the moment, +so we recommend learning Gerrit. +It's different but powerful and familiarity +with help you understand the flow. +</p> + +<h3>Overview</h3> + +<p> +This is an overview of the overall process: +</p> + +<ul> +<li> +<b>Step 1:</b> Clone the Go source code from GitHub or go.googlesource.com +and make sure it's stable by compiling and testing it once: <pre> -[alias] - change = codereview change - gofmt = codereview gofmt - mail = codereview mail - pending = codereview pending - submit = codereview submit - sync = codereview sync +$ git clone https://github.com/golang/go # or https://go.googlesource.com/go +$ cd go/src +$ ./all.bash # compile and test </pre> +</li> -<span id="help"></span> -<h3 id="understanding_git-codereview">Understanding the git-codereview command</h3> +<li> +<b>Step 2:</b> Prepare changes in a new branch, created from the master branch. +To commit the changes, use <code>git</code> <code>codereview</code> <code>change</code>; that +will create or amend a single commit in the branch. +<pre> +$ git checkout -b mybranch +$ [edit files...] +$ git add [files...] +$ git codereview change # create commit in the branch +$ [edit again...] +$ git add [files...] +$ git codereview change # amend the existing commit with new changes +$ [etc.] +</pre> +</li> -<p>After installing the <code>git-codereview</code> command, you can run</p> +<li> +<b>Step 3:</b> Test your changes, re-running <code>all.bash</code>. +<pre> +$ ./all.bash # recompile and test +</pre> +</li> +<li> +<b>Step 4:</b> Send the changes for review to Gerrit using <code>git</code> +<code>codereview</code> <code>mail</code>(which doesn't use e-mail, despite the name). <pre> -$ git codereview help +$ git codereview mail # send changes to Gerrit +</pre> +</li> + +<li> +<b>Step 5:</b> After a review, apply changes to the same single commit +and mail them to Gerrit again: +<pre> +$ [edit files...] +$ git add [files...] +$ git codereview change # update same commit +$ git codereview mail # send to Gerrit again </pre> +</li> +</ul> <p> -to learn more about its commands. -You can also read the <a href="https://godoc.org/golang.org/x/review/git-codereview">command documentation</a>. +The rest of this section describes these steps in more detail. </p> -<h1 id="making_a_contribution">Making a Contribution</h1> - -<h2 id="Design">Discuss your design</h2> +<h3 id="checkout_go">Step 1: Clone the Go source code</h3> <p> -The project welcomes submissions but please let everyone know what -you're working on if you want to change or add to the Go repositories. +In addition to a recent Go installation, you need to have a local copy of the source +checked out from the correct repository. +You can check out the Go source repo onto your local file system anywhere +you want as long as it's outside your <code>GOPATH</code>. +Either clone from +<code>go.googlesource.com</code> or from GitHub: </p> +<pre> +$ git clone https://github.com/golang/go # or https://go.googlesource.com/go +$ cd go +</pre> + +<h3 id="make_branch">Step 2: Prepare changes in a new branch</h3> + <p> -Before undertaking to write something new for the Go project, -please <a href="https://golang.org/issue/new">file an issue</a> -(or claim an <a href="https://golang.org/issues">existing issue</a>). -Significant changes must go through the -<a href="https://golang.org/s/proposal-process">change proposal process</a> -before they can be accepted. +Each Go change must be made in a separate branch, created from the master branch. +You can use +the normal <code>git</code> commands to create a branch and add changes to the +staging area: </p> +<pre> +$ git checkout -b mybranch +$ [edit files...] +$ git add [files...] +</pre> + <p> -This process gives everyone a chance to validate the design, -helps prevent duplication of effort, -and ensures that the idea fits inside the goals for the language and tools. -It also checks that the design is sound before code is written; -the code review tool is not the place for high-level discussions. +To commit changes, instead of <code>git commit</code>, use <code>git codereview change</code>. </p> +<pre> +$ git codereview change +(open $EDITOR) +</pre> + <p> -When planning work, please note that the Go project follows a <a -href="https://golang.org/wiki/Go-Release-Cycle">six-month development cycle</a>. -The latter half of each cycle is a three-month feature freeze during -which only bug fixes and doc updates are accepted. New contributions can be -sent during a feature freeze but will not be accepted until the freeze thaws. +You can edit the commit description in your favorite editor as usual. +The <code>git</code> <code>codereview</code> <code>change</code> command +will automatically add a unique Change-Id line near the bottom. +That line is used by Gerrit to match successive uploads of the same change. +Do not edit or delete it. +A Change-Id looks like this: </p> -<h3 id="scratch">Not sure what change to make?</h3> +<pre> +Change-Id: I2fbdbffb3aab626c4b6f56348861b7909e3e8990 +</pre> <p> -If you want to become familiar with Gerrit and the contribution process, -but aren't sure what you'd like to contribute just yet, you can use the <a -href="https://go.googlesource.com/scratch">scratch repository</a> to practice -making a change. +The tool also checks that you've +run <code>go</code> <code>fmt</code> over the source code, and that +the commit message follows the <a href="#commit_messages">suggested format</a>. </p> -<h2 id="making_a_change">Making a change</h2> +<p> +If you need to edit the files again, you can stage the new changes and +re-run <code>git</code> <code>codereview</code> <code>change</code>: each subsequent +run will amend the existing commit while preserving the Change-Id. +</p> -<h3 id="checkout_go">Getting Go Source</h3> <p> -First you need to have a local copy of the source checked out from the correct -repository. -As Go builds Go you will also likely need to have a working version -of Go installed (some documentation changes may not need this). -This should be a recent version of Go and can be obtained via any package or -binary distribution or you can build it from source. +Make sure that you always keep a single commit in each branch. +If you add more +commits by mistake, you can use <code>git</code> <code>rebase</code> to +<a href="https://stackoverflow.com/questions/31668794/squash-all-your-commits-in-one-before-a-pull-request-in-github">squash them together</a> +into a single one. </p> + +<h3 id="testing">Step 3: Test your changes</h3> + <p> -You should checkout the Go source repo anywhere you want as long as it's -outside of your $GOPATH. -Go to a directory where you want the source to appear and run the following -command in a terminal. +You've <a href="code.html">written and tested your code</a>, but +before sending code out for review, run <i>all the tests for the whole +tree</i> to make sure the changes don't break other packages or programs: </p> <pre> -$ git clone https://go.googlesource.com/go -$ cd go +$ cd go/src +$ ./all.bash </pre> -<h3 id="master">Contributing to the main Go tree</h3> +<p> +(To build under Windows use <code>all.bat</code>; this also requires +setting the environment variable <code>GOROOT_BOOTSTRAP</code> to the +directory holding the Go tree for the bootstrap compiler.) +</p> <p> -Most Go installations use a release branch, but new changes should -only be made based on the master branch. <br> -(They may be applied later to a release branch as part of the release process, -but most contributors won't do this themselves.) -Before making a change, make sure you start on the master branch: +After running for a while and printing a lot of testing output, the command should finish +by printing, </p> <pre> -$ git checkout master -$ git sync +ALL TESTS PASSED </pre> <p> -(In Git terms, <code>git</code> <code>sync</code> runs -<code>git</code> <code>pull</code> <code>-r</code>.) +You can use <code>make.bash</code> instead of <code>all.bash</code> +to just build the compiler and standard packages without running the test suite. +Once the <code>go</code> tool is built, it will be installed as <code>bin/go</code> +under the directory in which you cloned the Go repository, and you can +run it directly from there. +See also +the section on how to <a href="#quick_test">test your changes quickly</a>. </p> -<h3 id="subrepos">Contributing to subrepositories (golang.org/x/...)</h3> +<h3 id="mail">Step 4: Send changes for review</h3> <p> -If you are contributing a change to a subrepository, obtain the -Go package using <code>go get</code>. For example, to contribute -to <code>golang.org/x/oauth2</code>, check out the code by running: +Once the change is ready and tested over the whole tree, send it for review. +This is done with the <code>mail</code> sub-command which, despite its name, doesn't +directly mail anything; it just sends the change to Gerrit: </p> <pre> -$ go get -d golang.org/x/oauth2/... +$ git codereview mail </pre> <p> -Then, change your directory to the package's source directory -(<code>$GOPATH/src/golang.org/x/oauth2</code>). +Gerrit assigns your change a number and URL, which <code>git</code> <code>codereview</code> <code>mail</code> will print, something like: </p> -<h3 id="change">Make your changes</h3> +<pre> +remote: New Changes: +remote: https://go-review.googlesource.com/99999 math: improved Sin, Cos and Tan precision for very large arguments +</pre> + +<p> +If you get an error instead, check the +<a href="#troubleshooting_mail">Troubleshooting mail errors</a> section. +</p> <p> -The entire checked-out tree is editable. -Make your changes as you see fit ensuring that you create appropriate -tests along with your changes. Test your changes as you go. +If your change relates to an open GitHub issue and you have followed the <a href="#commit_messages"> +suggested commit message format</a>, the issue will be updated in a few minutes by a bot, +linking your Gerrit change to it in the comments. </p> -<h3 id="copyright">Copyright</h3> + +<h3 id="revise">Step 5: Revise changes after a review</h3> <p> -Files in the Go repository don't list author names, both to avoid clutter -and to avoid having to keep the lists up to date. -Instead, your name will appear in the -<a href="https://golang.org/change">change log</a> and in the <a -href="/CONTRIBUTORS"><code>CONTRIBUTORS</code></a> file and perhaps the <a -href="/AUTHORS"><code>AUTHORS</code></a> file. -These files are automatically generated from the commit logs periodically. -The <a href="/AUTHORS"><code>AUTHORS</code></a> file defines who “The Go -Authors”—the copyright holders—are. +Go maintainers will review your code on Gerrit, and you will get notifications via e-mail. +You can see the review on Gerrit and comment on them there. +You can also reply +<a href="https://gerrit-review.googlesource.com/Documentation/intro-user.html#reply-by-email">using e-mail</a> +if you prefer. </p> -<p>New files that you contribute should use the standard copyright header:</p> +<p> +If you need to revise your change after the review, edit the files in correct branch, +add them to the Git staging area, and then amend the commit with +<code>git</code> <code>codereview</code> <code>change</code>: +</p> <pre> -// Copyright 2018 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. +$ git codereview change # amend current commit +(open $EDITOR) +$ git codereview mail # send new changes to Gerrit </pre> <p> -Files in the repository are copyright the year they are added. -Do not update the copyright year on files that you change. +If you don't need to change the commit description, just save and exit from the editor. +Remember not to touch the special Change-Id line. </p> -<h3 id="commit_changes">Commit your changes</h3> - <p> -Once you have edited files, you must tell Git that they have been modified. -You must also tell Git about any files that are added, removed, or renamed files. -These operations are done with the usual Git commands, -<code>git</code> <code>add</code>, -<code>git</code> <code>rm</code>, -and -<code>git</code> <code>mv</code>. +Again, make sure that you always keep a single commit in each branch. +If you add more +commits by mistake, you can use <code>git rebase</code> to +<a href="https://stackoverflow.com/questions/31668794/squash-all-your-commits-in-one-before-a-pull-request-in-github">squash them together</a> +into a single one. </p> +<h2 id="commit_messages">Good commit messages</h2> + <p> -Once you have the changes queued up, you will want to commit them. -In the Go contribution workflow this is done with a <code>git</code> -<code>change</code> command, which creates a local branch and commits the changes -directly to that local branch. +Commit messages in Go follow a specific set of conventions, +which we discuss in this section. </p> <p> -The workflow described here assumes a single change per branch. -It is also possible to prepare a sequence of (usually related) changes in a single branch. -See the <a href="https://golang.org/x/review/git-codereview">git-codereview documentation</a> for details. +Here is an example of a good one: </p> <pre> -$ git change <i><branch></i> +math: improve Sin, Cos and Tan precision for very large arguments + +The existing implementation has poor numerical properties for +large arguments, so use the McGillicutty algorithm to improve +accuracy above 1e10. + +The algorithm is described at http://wikipedia.org/wiki/McGillicutty_Algorithm + +Fixes #159 </pre> +<h3>First line</h3> + <p> -The name <i><branch></i> is an arbitrary one you choose to identify the -local branch containing your changes and will not be used elsewhere. -This is an offline operation and nothing will be sent to the server yet. +The first line of the change description is conventionally a short one-line +summary of the change, prefixed by the primary affected package. </p> <p> -(In Git terms, <code>git</code> <code>change</code> <code><branch></code> -runs <code>git</code> <code>checkout</code> <code>-b</code> <code>branch</code>, -then <code>git</code> <code>branch</code> <code>--set-upstream-to</code> <code>origin/master</code>, -then <code>git</code> <code>commit</code>.) +A rule of thumb is that it should be written so to complete the sentence +"This change modifies Go to _____." +That means it does not start with a capital letter, is not a complete sentence, +and actually summarizes the result of the change. </p> <p> -As the <code>git</code> <code>commit</code> is the final step, Git will open an -editor to ask for a commit message. (It uses the editor named by -the <code>$EDITOR</code> environment variable, -<code>vi</code> by default.) - -The file will look like: +Follow the first line by a blank line. </p> -<pre> - -# Please enter the commit message for your changes. Lines starting -# with '#' will be ignored, and an empty message aborts the commit. -# On branch foo -# Changes not staged for commit: -# modified: editedfile.go -# -</pre> +<h3>Main content</h3> <p> -At the beginning of this file is a blank line; replace it -with a thorough description of your change. -The first line of the change description is conventionally a one-line -summary of the change, prefixed by the primary affected package, -and is used as the subject for code review email. -It should complete the sentence "This change modifies Go to _____." The rest of the description elaborates and should provide context for the change and explain what it does. Write in complete sentences with correct punctuation, just like for your comments in Go. -If there is a helpful reference, mention it here. -If you've fixed an issue, reference it by number with a # before it. +Don't use HTML, Markdown, or any other markup language. </p> <p> -After editing, the template might now read: +Add any relevant information, such as benchmark data if the change +afects performance. +The <a href="https://godoc.org/golang.org/x/tools/cmd/benchcmp">benchcmp</a> +tool is conventionally used to format +benchmark data for change descriptions. </p> -<pre> -math: improve Sin, Cos and Tan precision for very large arguments +<h3>Referencing issues</h3> -The existing implementation has poor numerical properties for -large arguments, so use the McGillicutty algorithm to improve -accuracy above 1e10. +<p> +The special notation "Fixes #12345" associates the change with issue 12345 in the +<a href="https://golang.org/issue/12345">Go issue tracker</a>. +When this change is eventually applied, the issue +tracker will automatically mark the issue as fixed. +</p> -The algorithm is described at http://wikipedia.org/wiki/McGillicutty_Algorithm +<p> +If the change is a partial step towards the resolution of the issue, +uses the notation "Updates #12345". +This will leave a comment in the issue +linking back to the change in Gerrit, but it will not close the issue +when the change is applied. +</p> -Fixes #159 +<p> +If you are sending a change against a subrepository, you must use +the fully-qualified syntax supported by GitHub to make sure the change is +linked to the issue in the main repository, not the subrepository. +All issues are tracked in the main repository's issue tracker. +The correct form is "Fixes golang/go#159". +</p> -# Please enter the commit message for your changes. Lines starting -# with '#' will be ignored, and an empty message aborts the commit. -# On branch foo -# Changes not staged for commit: -# modified: editedfile.go -# -</pre> + +<h2 id="review">The review process</h2> <p> -The commented section of the file lists all the modified files in your client. -It is best to keep unrelated changes in different commits, -so if you see a file listed that should not be included, abort -the command and move that file to a different branch. +This section explains the review process in detail and how to approach +reviews after a change has been mailed. </p> + +<h3 id="mistakes">Common beginner mistakes</h3> + <p> -The special notation "Fixes #159" associates the change with issue 159 in the -<a href="https://golang.org/issue/159">Go issue tracker</a>. -When this change is eventually applied, the issue -tracker will automatically mark the issue as fixed. -(There are several such conventions, described in detail in the -<a href="https://help.github.com/articles/closing-issues-via-commit-messages/">GitHub Issue Tracker documentation</a>.) +When a change is sent to Gerrit, it is usually triaged within a few days. +A maintainer will have a look and provide some initial review that for first-time +contributors usually focuses on basic cosmetics and common mistakes. +These include things like: </p> +<ul> +<li> +Commit message not following the <a href="#commit_messages">suggested +format</a>. +</li> + +<li> +The lack of a linked GitHub issue. +The vast majority of changes +require a linked issue that describes the bug or the feature that the change +fixes or implements, and consensus should have been reached on the tracker +before proceeding with it. +Gerrit reviews do not discuss the merit of the change, +just its implementation. +<br> +Only trivial or cosmetic changes will be accepted without an associated issue. +</li> + +<li> +Change sent during the freeze phase of the development cycle, when the tree +is closed for general changes. +In this case, +a maintainer might review the code with a line such as <code>R=go1.12</code>, +which means that it will be reviewed later when the tree opens for a new +development window. +You can add <code>R=go1.XX</code> as a comment yourself +if you know that it's not the correct time frame for the change. +</li> +</ul> + +<h3 id="trybots">Trybots</h3> + <p> -Once you have finished writing the commit message, -save the file and exit the editor. +After an initial reading of your change, maintainers will trigger trybots, +a cluster of servers that will run the full test suite on several different +architectures. +Most trybots complete in a few minutes, at which point a link will +be posted in Gerrit where you can see the results. </p> <p> -You must have the $EDITOR environment variable set properly and working properly (exiting cleanly) -for this operation to succeed. -If you run into any issues at this step, it's likely your editor isn't exiting cleanly. -Try setting a different editor in your $EDITOR environment variable. +If the trybot run fails, follow the link and check the full logs of the +platforms on which the tests failed. +Try to understand what broke, update your patch to fix it, and upload again. +Maintainers will trigger a new trybot run to see +if the problem was fixed. </p> <p> -If you wish to do more editing, re-stage your changes using -<code>git</code> <code>add</code>, and then run +Sometimes, the tree can be broken on some platforms for a few hours; if +the failure reported by the trybot doesn't seem related to your patch, go to the +<a href="https://build.golang.org">Build Dashboard</a> and check if the same +failure appears in other recent commits on the same platform. +In this case, +feel free to write a comment in Gerrit to mention that the failure is +unrelated to your change, to help maintainers understand the situation. </p> -<pre> -$ git change -</pre> +<h3 id="reviews">Reviews</h3> <p> -to update the change description and incorporate the staged changes. -The change description contains a <code>Change-Id</code> line near the bottom, -added by a Git commit hook during the initial -<code>git</code> <code>change</code>. -That line is used by Gerrit to match successive uploads of the same change. -Do not edit or delete it. +The Go community values very thorough reviews. +Think of each review comment like a ticket: you are expected to somehow "close" it +by acting on it, either by implementing the suggestion or convincing the +reviewer otherwise. +</p> + +<p> +After you update the change, go through the review comments and make sure +to reply to every one. +You can click the "Done" button to reply +indicating that you've implemented the reviewer's suggestion; otherwise, +click on "Reply" and explain why you have not, or what you have done instead. </p> <p> -(In Git terms, <code>git</code> <code>change</code> with no branch name -runs <code>git</code> <code>commit</code> <code>--amend</code>.) +It is perfectly normal for changes to go through several round of reviews, +with one or more reviewers making new comments every time +and then waiting for an updated change before reviewing again. +This cycle happens even for experienced contributors, so +don't be discouraged by it. </p> -<h3 id="Testing">Testing</h3> +<h3 id="votes">Voting conventions</h3> <p> -You've <a href="code.html">written and tested your code</a>, but -before sending code out for review, run all the tests for the whole -tree to make sure the changes don't break other packages or programs: +As they near a decision, reviewers will make a "vote" on your change. +The Gerrit voting system involves an integer in the range -2 to +2: </p> -<pre> -$ cd go/src -$ ./all.bash -</pre> +<ul> + <li> + <b>+2</b> The change is approved for being merged. + Only Go maintainers can cast a +2 vote. + </li> + <li> + <b>+1</b> The change looks good, but either the reviewer is requesting + minor changes before approving it, or they are not a maintainer and cannot + approve it, but would like to encourage an approval. + </li> + <li> + <b>-1</b> The change is not good the way it is but might be fixable. + A -1 vote will always have a comment explaining why the change is unacceptable. + </li> + <li> + <b>-2</b> The change is blocked by a maintainer and cannot be approved. + Again, there will be a comment explaining the decision. + </li> +</ul> + +<h3 id="submit">Submitting an approved change</h3> <p> -(To build under Windows use <code>all.bat</code>.) +After the code has been +2'ed, an approver will +apply it to the master branch using the Gerrit user interface. +This is called "submitting the change". </p> <p> -After running for a while, the command should print +The two steps (approving and submitting) are separate because in some cases maintainers +may want to approve it but not to submit it right away (for instance, +the tree could be temporarily frozen). </p> -<pre> -"ALL TESTS PASSED". -</pre> +<p> +Submitting a change checks it into the repository. +The change description will include a link to the code review, +which will be updated with a link to the change +in the repository. +Since the method used to integrate the changes is Git's "Cherry Pick", +the commit hashes in the repository will be changed by +the submit operation. +</p> + +<p> +If your change has been approved for a few days without being +submitted, feel free to write a comment in Gerrit requesting +submission. +</p> -<h3 id="mail">Send the change for review</h3> + +<h3 id="more_information">More information</h3> <p> -Once the change is ready, send it for review. -This is similar to a <code>git push</code> in a GitHub style workflow. -This is done via the mail alias setup earlier which despite its name, doesn't -directly mail anything, it simply sends the change to Gerrit via git push. +In addition to the information here, the Go community maintains a <a +href="https://golang.org/wiki/CodeReview">CodeReview</a> wiki page. +Feel free to contribute to this page as you learn more about the review process. </p> -<pre> -$ git mail -</pre> + + +<h2 id="advanced_topics">Miscellaneous topics</h2> <p> -(In Git terms, <code>git</code> <code>mail</code> pushes the local committed -changes to Gerrit using <code>git</code> <code>push</code> <code>origin</code> -<code>HEAD:refs/for/master</code>.) +This section collects a number of other comments that are +outside the issue/edit/code review/submit process itself. </p> + +<h3 id="copyright">Copyright headers</h3> + <p> -If your change relates to an open issue, please add a comment to the issue -announcing your proposed fix, including a link to your change. +Files in the Go repository don't list author names, both to avoid clutter +and to avoid having to keep the lists up to date. +Instead, your name will appear in the +<a href="https://golang.org/change">change log</a> and in the <a +href="/CONTRIBUTORS"><code>CONTRIBUTORS</code></a> file and perhaps the <a +href="/AUTHORS"><code>AUTHORS</code></a> file. +These files are automatically generated from the commit logs periodically. +The <a href="/AUTHORS"><code>AUTHORS</code></a> file defines who “The Go +Authors”—the copyright holders—are. </p> <p> -The code review server assigns your change an issue number and URL, -which <code>git</code> <code>mail</code> will print, something like: +New files that you contribute should use the standard copyright header: </p> <pre> -remote: New Changes: -remote: https://go-review.googlesource.com/99999 math: improved Sin, Cos and Tan precision for very large arguments +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. </pre> -<h3>Troubleshooting</h3> +<p> +(Use the current year if you're reading this in 2019 or beyond.) +Files in the repository are copyrighted the year they are added. +Do not update the copyright year on files that you change. +</p> + + + + +<h3 id="troubleshooting_mail">Troubleshooting mail errors</h3> <p> -The most common way that the <code>git mail</code> command fails is because the -email address used has not gone through the setup above. +The most common way that the <code>git</code> <code>codereview</code> <code>mail</code> +command fails is because the e-mail address in the commit does not match the one +that you used during <a href="#google_account">the registration process</a>. + <br> If you see something like... </p> @@ -631,13 +935,9 @@ remote: ERROR: does not match your user account. </pre> <p> -You need to either add the email address listed to the CLA or set this repo to use -another email address already approved. -</p> - -<p> -First let's change the email address for this repo so this doesn't happen again. -You can change your email address for this repo with the following command: +you need to configure Git for this repository to use the +e-mail address that you registered with. +To change the e-mail address for this doesn't happen again, run: </p> <pre> @@ -645,8 +945,7 @@ $ git config user.email email@address.com </pre> <p> -Then change the previous commit to use this alternative email address. -You can do that with: +Then change the commit to use this alternative e-mail address with this command: </p> <pre> @@ -654,244 +953,239 @@ $ git commit --amend --author="Author Name <email@address.com>" </pre> <p> -Finally try to resend with: +Then retry by running: </p> <pre> -$ git mail +$ git codereview mail </pre> -<h3 id="cc">Specifying a reviewer / CCing others</h3> -<p> -Unless explicitly told otherwise, such as in the discussion leading -up to sending in the change, it's better not to specify a reviewer. -All changes are automatically CC'ed to the -<a href="https://groups.google.com/group/golang-codereviews">golang-codereviews@googlegroups.com</a> -mailing list. If this is your first ever change, there may be a moderation -delay before it appears on the mailing list, to prevent spam. -</p> +<h3 id="quick_test">Quickly testing your changes</h3> <p> -You can specify a reviewer or CC interested parties -using the <code>-r</code> or <code>-cc</code> options. -Both accept a comma-separated list of email addresses: +Running <code>all.bash</code> for every single change to the code tree +is burdensome. +Even though it is strongly suggested to run it before +sending a change, during the normal development cycle you may want +to compile and test only the package you are developing. </p> +<ul> +<li> +In general, you can run <code>make.bash</code> instead of <code>all.bash</code> +to only rebuild the Go tool chain without running the whole test suite. +Or you +can run <code>run.bash</code> to only run the whole test suite without rebuilding +the tool chain. +You can think of <code>all.bash</code> as <code>make.bash</code> +followed by <code>run.bash</code>. +</li> + +<li> +In this section, we'll call the directory into which you cloned the Go repository <code>$GODIR</code>. +The <code>go</code> tool built by <code>$GODIR/make.bash</code>will be installed +in <code>$GODIR/bin/go</code>and you +can invoke it to test your code. +For instance, if you +have modified the compiler and you want to test how it affects the +test suite of your own project, just run <code>go</code> <code>test</code> +using it: + <pre> -$ git mail -r joe@golang.org -cc mabel@example.com,math-nuts@swtch.com +$ cd <MYPROJECTDIR> +$ $GODIR/bin/go test </pre> +</li> -<h2 id="review">Going through the review process</h2> - -<p> -Running <code>git</code> <code>mail</code> will send an email to you and the -reviewers asking them to visit the issue's URL and make comments on the change. -When done, the reviewer adds comments through the Gerrit user interface -and clicks "Reply" to send comments back. -You will receive a mail notification when this happens. -You may reply through the web interface or -<a href="https://gerrit-review.googlesource.com/Documentation/intro-user.html#reply-by-email">via email</a>. -</p> +<li> +If you're changing the standard library, you probably don't need to rebuild +the compiler: you can just run the tests for the package you've changed. +You can do that either with the Go version you normally use, or +with the Go compiler built from your clone (which is +sometimes required because the standard library code you're modifying +might require a newer version than the stable one you have installed). -<h3 id="revise">Revise and resend</h3> +<pre> +$ cd $GODIR/src/hash/sha1 +$ [make changes...] +$ $GODIR/bin/go test . +</pre> +</li> -<p> -The Go contribution workflow is optimized for iterative revisions based on -feedback. -It is rare that an initial contribution will be ready to be applied as is. -As you revise your contribution and resend Gerrit will retain a history of -all the changes and comments made in the single URL. -</p> +<li> +If you're modifying the compiler itself, you can just recompile +the <code>compile</code> tool (which is the internal binary invoked +by <code>go</code> <code>build</code> to compile each single package). +After that, you will want to test it by compiling or running something. -<p> -You may respond to review comments through the web interface or -<a href="https://gerrit-review.googlesource.com/Documentation/intro-user.html#reply-by-email">via email</a>. -</p> +<pre> +$ cd $GODIR/src +$ [make changes...] +$ $GODIR/bin/go install cmd/compile +$ $GODIR/bin/go build [something...] # test the new compiler +$ $GODIR/bin/go run [something...] # test the new compiler +$ $GODIR/bin/go test [something...] # test the new compiler +</pre> -<p> -When you have revised the code and are ready for another round of review, -stage those changes and use <code>git</code> <code>change</code> to update the -commit. -To send the updated change for another round of review, -run <code>git</code> <code>mail</code> again. -</p> +The same applies to other internal tools of the Go tool chain, +such as <code>asm</code>, <code>cover</code>, <code>link</code>, and so on. +Just recompile and install the tool using <code>go</code> +<code>install</code> <code>cmd/<TOOL></code> and then use +the built Go binary to test it. +</li> -<p> -The reviewer can comment on the new copy, and the process repeats. -The reviewer approves the change by giving it a positive score -(+1 or +2) and replying <code>LGTM</code>: looks good to me. -</p> +<li> +In addition to the standard per-package tests, there is a top-level +test suite in <code>$GODIR/test</code> that contains +several black-box and regression tests. +The test suite is run +by <code>all.bash</code> but you can also run it manually: -<p> -You can see a list of your pending changes by running <code>git</code> -<code>pending</code>, and switch between change branches with <code>git</code> -<code>change</code> <code><i><branch></i></code>. -</p> +<pre> +$ cd $GODIR/test +$ $GODIR/bin/go run run.go +</pre> +</ul> -<h3 id="sync">Synchronize your client</h3> +<h3 id="subrepos">Contributing to subrepositories (golang.org/x/...)</h3> <p> -While you were working, others might have submitted changes to the repository. -To update your local branch, run +If you are contributing a change to a subrepository, obtain the +Go package using <code>go get</code>. +For example, to contribute +to <code>golang.org/x/oauth2</code>, check out the code by running: </p> <pre> -$ git sync +$ go get -d golang.org/x/oauth2/... </pre> <p> -(In git terms, <code>git</code> <code>sync</code> runs -<code>git</code> <code>pull</code> <code>-r</code>.) +Then, change your directory to the package's source directory +(<code>$GOPATH/src/golang.org/x/oauth2</code>), and follow the +normal contribution flow. </p> -<h3 id="resolving_conflicts">Resolving Conflicts</h3> + +<h3 id="cc">Specifying a reviewer / CCing others</h3> <p> -If files you were editing have changed, Git does its best to merge the -remote changes into your local changes. -It may leave some files to merge by hand. +Unless explicitly told otherwise, such as in the discussion leading +up to sending in the change, it's better not to specify a reviewer. +All changes are automatically CC'ed to the +<a href="https://groups.google.com/group/golang-codereviews">golang-codereviews@googlegroups.com</a> +mailing list. +If this is your first ever change, there may be a moderation +delay before it appears on the mailing list, to prevent spam. </p> <p> -For example, suppose you have edited <code>sin.go</code> but -someone else has committed an independent change. -When you run <code>git</code> <code>sync</code>, -you will get the (scary-looking) output: +You can specify a reviewer or CC interested parties +using the <code>-r</code> or <code>-cc</code> options. +Both accept a comma-separated list of e-mail addresses: +</p> <pre> -$ git sync -Failed to merge in the changes. -Patch failed at 0023 math: improved Sin, Cos and Tan precision for very large arguments -The copy of the patch that failed is found in: - /home/you/repo/.git/rebase-apply/patch - -When you have resolved this problem, run "git rebase --continue". -If you prefer to skip this patch, run "git rebase --skip" instead. -To check out the original branch and stop rebasing, run "git rebase --abort". +$ git codereview mail -r joe@golang.org -cc mabel@example.com,math-nuts@swtch.com </pre> + +<h3 id="sync">Synchronize your client</h3> + <p> -If this happens, run +While you were working, others might have submitted changes to the repository. +To update your local branch, run </p> <pre> -$ git status +$ git codereview sync </pre> <p> -to see which files failed to merge. -The output will look something like this: +(Under the covers this runs +<code>git</code> <code>pull</code> <code>-r</code>.) </p> -<pre> -rebase in progress; onto a24c3eb -You are currently rebasing branch 'mcgillicutty' on 'a24c3eb'. - (fix conflicts and then run "git rebase --continue") - (use "git rebase --skip" to skip this patch) - (use "git rebase --abort" to check out the original branch) - -Unmerged paths: - (use "git reset HEAD <file>..." to unstage) - (use "git add <file>..." to mark resolution) - <i>both modified: sin.go</i> -</pre> +<h3 id="download">Reviewing code by others</h3> <p> -The only important part in that transcript is the italicized "both modified" -line: Git failed to merge your changes with the conflicting change. -When this happens, Git leaves both sets of edits in the file, -with conflicts marked by <code><<<<<<<</code> and -<code>>>>>>>></code>. -It is now your job to edit the file to combine them. -Continuing the example, searching for those strings in <code>sin.go</code> -might turn up: +As part of the review process reviewers can propose changes directly (in the +GitHub workflow this would be someone else attaching commits to a pull request). + +You can import these changes proposed by someone else into your local Git repository. +On the Gerrit review page, click the "Download ▼" link in the upper right +corner, copy the "Checkout" command and run it from your local Git repo. +It will look something like this: </p> <pre> - arg = scale(arg) -<<<<<<< HEAD - if arg < 1e9 { -======= - if arg < 1e10 { ->>>>>>> mcgillicutty - largeReduce(arg) +$ git fetch https://go.googlesource.com/review refs/changes/21/13245/1 && git checkout FETCH_HEAD </pre> <p> -Git doesn't show it, but suppose the original text that both edits -started with was 1e8; you changed it to 1e10 and the other change to 1e9, -so the correct answer might now be 1e10. -First, edit the section to remove the markers and leave the correct code: +To revert, change back to the branch you were working in. </p> -<pre> - arg = scale(arg) - if arg < 1e10 { - largeReduce(arg) -</pre> + +<h3 id="git-config">Set up git aliases</h3> <p> -Then tell Git that the conflict is resolved by running +The <code>git-codereview</code> command can be run directly from the shell +by typing, for instance, </p> <pre> -$ git add sin.go +$ git codereview sync </pre> <p> -If you had been editing the file, say for debugging, but do not -care to preserve your changes, you can run -<code>git</code> <code>reset</code> <code>HEAD</code> <code>sin.go</code> -to abandon your changes. -Then run <code>git</code> <code>rebase</code> <code>--continue</code> to -restore the change commit. +but it is more convenient to set up aliases for <code>git-codereview</code>'s own +subcommands, so that the above becomes, </p> -<h3 id="download">Reviewing code by others</h3> +<pre> +$ git sync +</pre> <p> -As part of the review process reviewers can propose changes directly (in the -GitHub workflow this would be someone else attaching commits to a pull request). - -You can import these changes proposed by someone else into your local Git repository. -On the Gerrit review page, click the "Download ▼" link in the upper right -corner, copy the "Checkout" command and run it from your local Git repo. It -should look something like this: +The <code>git-codereview</code> subcommands have been chosen to be distinct from +Git's own, so it's safe to define these aliases. +To install them, copy this text into your +Git configuration file (usually <code>.gitconfig</code> in your home directory): </p> <pre> -$ git fetch https://go.googlesource.com/review refs/changes/21/1221/1 && git checkout FETCH_HEAD +[alias] + change = codereview change + gofmt = codereview gofmt + mail = codereview mail + pending = codereview pending + submit = codereview submit + sync = codereview sync </pre> -<p> -To revert, change back to the branch you were working in. -</p> -<h2 id="submit">Apply the change to the master branch</h2> +<h3 id="multiple_changes">Sending multiple dependent changes</h3> <p> -After the code has been <code>LGTM</code>'ed, an approver may -apply it to the master branch using the Gerrit UI. -There is a "Submit" button on the web page for the change -that appears once the change is approved (marked +2). +Advanced users may want to stack up related commits in a single branch. +Gerrit allows for changes to be dependent on each other, forming such a dependency chain. +Each change will need to be approved and submitted separately but the dependency +will be visible to reviewers. </p> <p> -This checks the change into the repository. -The change description will include a link to the code review, -and the code review will be updated with a link to the change -in the repository. -Since the method used to integrate the changes is "Cherry Pick", -the commit hashes in the repository will be changed by -the "Submit" operation. +To send out a group of dependent changes, keep each change as a different commit under +the same branch, and then run: </p> -<h2 id="more">More information</h2> +<pre> +$ git codereview mail HEAD +</pre> <p> -In addition to the information here, the Go community maintains a <a -href="https://golang.org/wiki/CodeReview">CodeReview</a> wiki page. -Feel free to contribute to this page as you learn the review process. +Make sure to explicitly specify <code>HEAD</code>, which is usually not required when sending +single changes. </p> diff --git a/doc/devel/release.html b/doc/devel/release.html index 24fe583062..584340b005 100644 --- a/doc/devel/release.html +++ b/doc/devel/release.html @@ -16,11 +16,45 @@ git checkout <i>release-branch</i> <p> Each major Go release is supported until there are two newer major releases. -For example, Go 1.8 is supported until Go 1.10 is released, -and Go 1.9 is supported until Go 1.11 is released. +For example, Go 1.5 was supported until the Go 1.7 release, and Go 1.6 was +supported until the Go 1.8 release. We fix critical problems, including <a href="/security">critical security problems</a>, in supported releases as needed by issuing minor revisions -(for example, Go 1.9.1, Go 1.9.2, and so on). +(for example, Go 1.6.1, Go 1.6.2, and so on). +</p> + +<h2 id="go1.10">go1.10 (released 2018/02/16)</h2> + +<p> +Go 1.10 is a major release of Go. +Read the <a href="/doc/go1.10">Go 1.10 Release Notes</a> for more information. +</p> + +<h3 id="go1.10.minor">Minor revisions</h3> + +<p> +go1.10.1 (released 2018/03/28) includes fixes to the compiler, runtime, and the +<code>archive/zip</code>, <code>crypto/tls</code>, <code>crypto/x509</code>, +<code>encoding/json</code>, <code>net</code>, <code>net/http</code>, and +<code>net/http/pprof</code> packages. +See the <a href="https://github.com/golang/go/issues?q=milestone%3AGo1.10.1">Go +1.10.1 milestone</a> on our issue tracker for details. +</p> + +<p> +go1.10.2 (released 2018/05/01) includes fixes to the compiler, linker, and go +command. +See the <a href="https://github.com/golang/go/issues?q=milestone%3AGo1.10.2">Go +1.10.2 milestone</a> on our issue tracker for details. +</p> + +<p> +go1.10.3 (released 2018/06/05) includes fixes to the go command, and the +<code>crypto/tls</code>, <code>crypto/x509</code>, and <code>strings</code> packages. +In particular, it adds <a href="https://go.googlesource.com/go/+/d4e21288e444d3ffd30d1a0737f15ea3fc3b8ad9"> +minimal support to the go command for the vgo transition</a>. +See the <a href="https://github.com/golang/go/issues?q=milestone%3AGo1.10.3">Go +1.10.3 milestone</a> on our issue tracker for details. </p> <h2 id="go1.9">go1.9 (released 2017/08/24)</h2> @@ -57,6 +91,35 @@ See the <a href="https://github.com/golang/go/issues?q=milestone%3AGo1.9.3">Go 1.9.3 milestone</a> on our issue tracker for details. </p> +<p> +go1.9.4 (released 2018/02/07) includes a security fix to “go get”. +See the <a href="https://github.com/golang/go/issues?q=milestone%3AGo1.9.4">Go +1.9.4</a> milestone on our issue tracker for details. +</p> + +<p> +go1.9.5 (released 2018/03/28) includes fixes to the compiler, go command, and +<code>net/http/pprof</code> package. +See the <a href="https://github.com/golang/go/issues?q=milestone%3AGo1.9.5">Go +1.9.5 milestone</a> on our issue tracker for details. +</p> + +<p> +go1.9.6 (released 2018/05/01) includes fixes to the compiler and go command. +See the <a href="https://github.com/golang/go/issues?q=milestone%3AGo1.9.6">Go +1.9.6 milestone</a> on our issue tracker for details. +</p> + +<p> +go1.9.7 (released 2018/06/05) includes fixes to the go command, and the +<code>crypto/x509</code>, and <code>strings</code> packages. +In particular, it adds <a href="https://go.googlesource.com/go/+/d4e21288e444d3ffd30d1a0737f15ea3fc3b8ad9"> +minimal support to the go command for the vgo transition</a>. +See the <a href="https://github.com/golang/go/issues?q=milestone%3AGo1.9.7">Go +1.9.7 milestone</a> on our issue tracker for details. +</p> + + <h2 id="go1.8">go1.8 (released 2017/02/16)</h2> <p> @@ -108,12 +171,19 @@ See the <a href="https://github.com/golang/go/issues?q=milestone%3AGo1.8.5">Go </p> <p> -go1.8.6 (released 2018/01/22) includes the the same fix in <code>math/big</code> +go1.8.6 (released 2018/01/22) includes the same fix in <code>math/big</code> as Go 1.9.3 and was released at the same time. See the <a href="https://github.com/golang/go/issues?q=milestone%3AGo1.8.6">Go 1.8.6 milestone</a> on our issue tracker for details. </p> +<p> +go1.8.7 (released 2018/02/07) includes a security fix to “go get”. +It contains the same fix as Go 1.9.4 and was released at the same time. +See the <a href="https://github.com/golang/go/issues?q=milestone%3AGo1.8.7">Go +1.8.7</a> milestone on our issue tracker for details. +</p> + <h2 id="go1.7">go1.7 (released 2016/08/15)</h2> <p> diff --git a/doc/diagnostics.html b/doc/diagnostics.html index 0ed0e81b92..35aae156e8 100644 --- a/doc/diagnostics.html +++ b/doc/diagnostics.html @@ -222,7 +222,7 @@ an execution tracer to trace the runtime events within an interval. <p>Tracing enables us to:</p> <ul> -<li>Instrument and profile application latency in a Go process.</li> +<li>Instrument and analyze application latency in a Go process.</li> <li>Measure the cost of specific calls in a long chain of calls.</li> <li>Figure out the utilization and performance improvements. Bottlenecks are not always obvious without tracing data.</li> diff --git a/doc/effective_go.html b/doc/effective_go.html index 61de824fcd..89c1d08782 100644 --- a/doc/effective_go.html +++ b/doc/effective_go.html @@ -3588,8 +3588,7 @@ That's left as an exercise for the reader. <p> Let's finish with a complete Go program, a web server. This one is actually a kind of web re-server. -Google provides a service at -<a href="http://chart.apis.google.com">http://chart.apis.google.com</a> +Google provides a service at <code>chart.apis.google.com</code> that does automatic formatting of data into charts and graphs. It's hard to use interactively, though, because you need to put the data into the URL as a query. diff --git a/doc/go1.10.html b/doc/go1.10.html index 5885176f46..2974fef9a3 100644 --- a/doc/go1.10.html +++ b/doc/go1.10.html @@ -15,12 +15,7 @@ Do not send CLs removing the interior tags from such phrases. ul li { margin: 0.5em 0; } </style> -<h2 id="introduction">DRAFT RELEASE NOTES - Introduction to Go 1.10</h2> - -<p><strong> - Go 1.10 is not yet released. These are work-in-progress - release notes. Go 1.10 is expected to be released in February 2018. -</strong></p> +<h2 id="introduction">Introduction to Go 1.10</h2> <p> The latest Go release, version 1.10, arrives six months after <a href="go1.9">Go 1.9</a>. @@ -35,6 +30,10 @@ adds <a href="#test">caching of successful test results</a>, runs <a href="#test-vet">vet automatically during tests</a>, and permits <a href="#cgo">passing string values directly between Go and C using cgo</a>. +A new <a href="#cgo">compiler option whitelist</a> may cause +unexpected <a href="https://golang.org/s/invalidflag"><code>invalid +flag</code></a> errors in code that built successfully with older +releases. </p> <h2 id="language">Changes to the language</h2> @@ -44,9 +43,9 @@ There are no significant changes to the language specification. </p> <p><!-- CL 60230 --> -A corner case involving shifts by untyped constants has been clarified, +A corner case involving shifts of untyped constants has been clarified, and as a result the compilers have been updated to allow the index expression -<code>x[1.0</code> <code><<</code> <code>s]</code> where <code>s</code> is an untyped constant; +<code>x[1.0</code> <code><<</code> <code>s]</code> where <code>s</code> is an unsigned integer; the <a href="/pkg/go/types/">go/types</a> package already did. </p> @@ -267,6 +266,18 @@ and the <a href="/cmd/test2json/">test2json documentation</a>. <h3 id="cgo">Cgo</h3> <p> +Options specified by cgo using <code>#cgo CFLAGS</code> and the like +are now checked against a whitelist of permitted options. +This closes a security hole in which a downloaded package uses +compiler options like +<span style="white-space: nowrap"><code>-fplugin</code></span> +to run arbitrary code on the machine where it is being built. +This can cause a build error such as <code>invalid flag in #cgo CFLAGS</code>. +For more background, and how to handle this error, see +<a href="https://golang.org/s/invalidflag">https://golang.org/s/invalidflag</a>. +</p> + +<p> Cgo now implements a C typedef like “<code>typedef</code> <code>X</code> <code>Y</code>” using a Go type alias, so that Go code may use the types <code>C.X</code> and <code>C.Y</code> interchangeably. It also now supports the use of niladic function-like macros. @@ -785,7 +796,8 @@ Parsed certificates also now report URI names and IP, email, and URI constraints <a href="/pkg/crypto/x509/#Certificate"><code>Certificate</code></a> fields <code>URIs</code>, <code>PermittedIPRanges</code>, <code>ExcludedIPRanges</code>, <code>PermittedEmailAddresses</code>, <code>ExcludedEmailAddresses</code>, -<code>PermittedURIDomains</code>, and <code>ExcludedURIDomains</code>. +<code>PermittedURIDomains</code>, and <code>ExcludedURIDomains</code>. Certificates with +invalid values for those fields are now rejected. </p> <p> @@ -1213,6 +1225,10 @@ The content-serving handlers also now omit the <code>Content-Type</code> header if passed an invalid (non-3-digit) status code. </p> <p> +<!-- CL 46631 --> +The <code>Server</code> will no longer add an implicit Content-Type when a <code>Handler</code> does not write any output. +</p> +<p> <a href="/pkg/net/http/#Redirect"><code>Redirect</code></a> now sets the <code>Content-Type</code> header before writing its HTTP response. </p> </dl> diff --git a/doc/go_faq.html b/doc/go_faq.html index f273688705..cc81e49a9b 100644 --- a/doc/go_faq.html +++ b/doc/go_faq.html @@ -45,7 +45,7 @@ analysis easy and avoids much of the overhead of C-style include files and libraries. <li> Go's type system has no hierarchy, so no time is spent defining the -relationships between types. Also, although Go has static types the language +relationships between types. Also, although Go has static types, the language attempts to make types feel lighter weight than in typical OO languages. <li> Go is fully garbage-collected and provides fundamental support for @@ -1097,24 +1097,27 @@ The <code>go get</code> command therefore uses HTTPS for safety. </p> <p> -If you use <code>git</code> and prefer to push changes through SSH using your existing key -it's easy to work around this. For GitHub, try one of these solutions: +<code>Git</code> can be configured to authenticate over HTTPS or to use SSH in place of HTTPS. +To authenticate over HTTPS, you can add a line +to the <code>$HOME/.netrc</code> file that git consults: </p> -<ul> -<li>Manually clone the repository in the expected package directory: <pre> -$ cd src/github.com/username -$ git clone git@github.com:username/package.git +machine github.com login <i>USERNAME</i> password <i>APIKEY</i> </pre> -</li> -<li>Force <code>git push</code> to use the <code>SSH</code> protocol by appending -these two lines to <code>~/.gitconfig</code>: +<p> +For GitHub accounts, the password can be a +<a href="https://help.github.com/articles/creating-a-personal-access-token-for-the-command-line/">personal access token</a>. +</p> + +<p> +<code>Git</code> can also be configured to use SSH in place of HTTPS for URLs matching a given prefix. +For example, to use SSH for all GitHub access, +add these lines to your <code>~/.gitconfig</code>: +</p> <pre> -[url "git@github.com:"] - pushInsteadOf = https://github.com/ +[url "ssh://git@github.com/"] + insteadOf = https://github.com/ </pre> -</li> -</ul> <h3 id="get_version"> How should I manage package versions using "go get"?</h3> @@ -1852,18 +1855,19 @@ Why is my trivial program such a large binary?</h3> <p> The linker in the <code>gc</code> toolchain -creates statically-linked binaries by default. All Go binaries therefore include the Go +creates statically-linked binaries by default. +All Go binaries therefore include the Go run-time, along with the run-time type information necessary to support dynamic type checks, reflection, and even panic-time stack traces. </p> <p> -A simple C "hello, world" program compiled and linked statically using gcc -on Linux is around 750 kB, -including an implementation of <code>printf</code>. -An equivalent Go program using <code>fmt.Printf</code> -is around 1.5 MB, but -that includes more powerful run-time support and type information. +A simple C "hello, world" program compiled and linked statically using +gcc on Linux is around 750 kB, including an implementation of +<code>printf</code>. +An equivalent Go program using +<code>fmt.Printf</code> weighs a couple of megabytes, but that includes +more powerful run-time support and type and debugging information. </p> <h3 id="unused_variables_and_imports"> @@ -1931,6 +1935,26 @@ eliminating the unused imports issue in practice. This program is easily connected to most editors to run automatically when a Go source file is written. </p> +<h3 id="virus"> +Why does my virus-scanning software think my Go distribution or compiled binary is infected?</h3> + +<p> +This is a common occurrence, especially on Windows machines, and is almost always a false positive. +Commercial virus scanning programs are often confused by the structure of Go binaries, which +they don't see as often as those compiled from other languages. +</p> + +<p> +If you've just installed the Go distribution and the system reports it is infected, that's certainly a mistake. +To be really thorough, you can verify the download by comparing the checksum with those on the +<a href="https://golang.org/dl/">downloads page</a>. +</p> + +<p> +In any case, if you believe the report is in error, please report a bug to the supplier of your virus scanner. +Maybe in time virus scanners can learn to understand Go programs. +</p> + <h2 id="Performance">Performance</h2> <h3 id="Why_does_Go_perform_badly_on_benchmark_x"> diff --git a/doc/go_spec.html b/doc/go_spec.html index 9a166ccdf4..f1300c105a 100644 --- a/doc/go_spec.html +++ b/doc/go_spec.html @@ -1,6 +1,6 @@ <!--{ "Title": "The Go Programming Language Specification", - "Subtitle": "Version of February 1, 2018", + "Subtitle": "Version of May 9, 2018", "Path": "/ref/spec" }--> @@ -3051,7 +3051,6 @@ used in an <a href="#Assignments">assignment</a> or initialization of the specia v, ok = a[x] v, ok := a[x] var v, ok = a[x] -var v, ok T = a[x] </pre> <p> @@ -4161,11 +4160,6 @@ operands and are evaluated at compile time. Untyped boolean, numeric, and string constants may be used as operands wherever it is legal to use an operand of boolean, numeric, or string type, respectively. -Except for shift operations, if the operands of a binary operation are -different kinds of untyped constants, the operation and, for non-boolean operations, the result use -the kind that appears later in this list: integer, rune, floating-point, complex. -For example, an untyped integer constant divided by an -untyped complex constant yields an untyped complex constant. </p> <p> @@ -4175,9 +4169,17 @@ an untyped boolean constant. If the left operand of a constant result is an integer constant; otherwise it is a constant of the same type as the left operand, which must be of <a href="#Numeric_types">integer type</a>. -Applying all other operators to untyped constants results in an untyped -constant of the same kind (that is, a boolean, integer, floating-point, -complex, or string constant). +</p> + +<p> +Any other operation on untyped constants results in an untyped constant of the +same kind; that is, a boolean, integer, floating-point, complex, or string +constant. +If the untyped operands of a binary operation (other than a shift) are of +different kinds, the result is of the operand's kind that appears later in this +list: integer, rune, floating-point, complex. +For example, an untyped integer constant divided by an +untyped complex constant yields an untyped complex constant. </p> <pre> diff --git a/doc/install-source.html b/doc/install-source.html index 4ed9487504..1928b0ba9b 100644 --- a/doc/install-source.html +++ b/doc/install-source.html @@ -307,7 +307,7 @@ package main import "fmt" func main() { - fmt.Printf("hello, world\n") + fmt.Printf("hello, world\n") } </pre> diff --git a/doc/install.html b/doc/install.html index f17dce5bdd..ee1516ac47 100644 --- a/doc/install.html +++ b/doc/install.html @@ -106,6 +106,14 @@ variable. You can do this by adding this line to your <code>/etc/profile</code> export PATH=$PATH:/usr/local/go/bin </pre> +<p> +<b>Note</b>: changes made to a <code>profile</code> file may not apply until the +next time you log into your computer. +To apply the changes immediately, just run the shell commands directly +or execute them from the profile using a command such as +<code>source $HOME/.profile</code>. +</p> + <h4 id="tarball_non_standard">Installing to a custom location</h4> <p> @@ -236,7 +244,7 @@ package main import "fmt" func main() { - fmt.Printf("hello, world\n") + fmt.Printf("hello, world\n") } </pre> @@ -278,7 +286,7 @@ If you see the "hello, world" message then your Go installation is working. <p> You can run <code>go</code> <code>install</code> to install the binary into your workspace's <code>bin</code> directory -or <code>go</code> <code>clean</code> to remove it. +or <code>go</code> <code>clean</code> <code>-i</code> to remove it. </p> <p> diff --git a/doc/progs/error2.go b/doc/progs/error2.go index 2b0e0c3563..086b6710d3 100644 --- a/doc/progs/error2.go +++ b/doc/progs/error2.go @@ -20,11 +20,11 @@ func viewRecord(w http.ResponseWriter, r *http.Request) { key := datastore.NewKey(c, "Record", r.FormValue("id"), 0, nil) record := new(Record) if err := datastore.Get(c, key, record); err != nil { - http.Error(w, err.Error(), 500) + http.Error(w, err.Error(), http.StatusInternalServerError) return } if err := viewTemplate.Execute(w, record); err != nil { - http.Error(w, err.Error(), 500) + http.Error(w, err.Error(), http.StatusInternalServerError) } } diff --git a/doc/progs/error3.go b/doc/progs/error3.go index e4e57e077b..d9e56b5d64 100644 --- a/doc/progs/error3.go +++ b/doc/progs/error3.go @@ -33,7 +33,7 @@ type appHandler func(http.ResponseWriter, *http.Request) error func (fn appHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { if err := fn(w, r); err != nil { - http.Error(w, err.Error(), 500) + http.Error(w, err.Error(), http.StatusInternalServerError) } } diff --git a/doc/root.html b/doc/root.html index a5119a9ff8..545b28d2d5 100644 --- a/doc/root.html +++ b/doc/root.html @@ -74,7 +74,7 @@ Linux, Mac OS X, Windows, and more. <div class="left"> <div id="video"> <div class="rootHeading">Featured video</div> - <iframe width="415" height="241" src="//www.youtube.com/embed/ytEkHepK08c" frameborder="0" allowfullscreen></iframe> + <div class="js-frontpage-video" style="--aspect-ratio-padding: 58.07%;"><iframe width="415" height="241" src="//www.youtube.com/embed/ytEkHepK08c" frameborder="0" allowfullscreen></iframe></div> </div> </div> @@ -153,6 +153,10 @@ Linux, Mac OS X, Windows, and more. ]; var v = videos[Math.floor(Math.random()*videos.length)]; $('#video iframe').attr('height', v.h).attr('src', v.s); + // Compute the aspect ratio (as a percentage) of the video + // using the fixed width 415 and the height of the current video, v.h. + var ar = 100*v.h/415; + $('.js-frontpage-video').attr('style', '--aspect-ratio-padding: ' + ar + '%;'); }); {{end}} diff --git a/misc/cgo/testcshared/main2.c b/misc/cgo/testcshared/main2.c index 6e8bf141ca..1a3f1d7545 100644 --- a/misc/cgo/testcshared/main2.c +++ b/misc/cgo/testcshared/main2.c @@ -9,7 +9,7 @@ #include <time.h> #include <unistd.h> -#define fd (100) +#define fd (30) // Tests libgo2.so, which does not export any functions. // Read a string from the file descriptor and print it. diff --git a/misc/cgo/testcshared/src/libgo2/libgo2.go b/misc/cgo/testcshared/src/libgo2/libgo2.go index 1b69d8f09f..e57c93b77d 100644 --- a/misc/cgo/testcshared/src/libgo2/libgo2.go +++ b/misc/cgo/testcshared/src/libgo2/libgo2.go @@ -21,7 +21,7 @@ import ( // that the C code can also use. const ( - fd = 100 + fd = 30 ) func init() { diff --git a/misc/cgo/testplugin/src/issue24351/main.go b/misc/cgo/testplugin/src/issue24351/main.go new file mode 100644 index 0000000000..4107adff7b --- /dev/null +++ b/misc/cgo/testplugin/src/issue24351/main.go @@ -0,0 +1,21 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import "plugin" + +func main() { + p, err := plugin.Open("issue24351.so") + if err != nil { + panic(err) + } + f, err := p.Lookup("B") + if err != nil { + panic(err) + } + c := make(chan bool) + f.(func(chan bool))(c) + <-c +} diff --git a/misc/cgo/testplugin/src/issue24351/plugin.go b/misc/cgo/testplugin/src/issue24351/plugin.go new file mode 100644 index 0000000000..db17e0a609 --- /dev/null +++ b/misc/cgo/testplugin/src/issue24351/plugin.go @@ -0,0 +1,14 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import "fmt" + +func B(c chan bool) { + go func() { + fmt.Println(1.5) + c <- true + }() +} diff --git a/misc/cgo/testplugin/test.bash b/misc/cgo/testplugin/test.bash index 18e3803bf4..df38204a4e 100755 --- a/misc/cgo/testplugin/test.bash +++ b/misc/cgo/testplugin/test.bash @@ -85,3 +85,8 @@ GOPATH=$(pwd) go build -gcflags "$GO_GCFLAGS" -o issue22175 src/issue22175/main. GOPATH=$(pwd) go build -gcflags "$GO_GCFLAGS" -buildmode=plugin -o issue.22295.so issue22295.pkg GOPATH=$(pwd) go build -gcflags "$GO_GCFLAGS" -o issue22295 src/issue22295.pkg/main.go ./issue22295 + +# Test for issue 24351 +GOPATH=$(pwd) go build -gcflags "$GO_GCFLAGS" -buildmode=plugin -o issue24351.so src/issue24351/plugin.go +GOPATH=$(pwd) go build -gcflags "$GO_GCFLAGS" -o issue24351 src/issue24351/main.go +./issue24351 diff --git a/misc/cgo/testshared/shared_test.go b/misc/cgo/testshared/shared_test.go index cf049ec35b..a296005780 100644 --- a/misc/cgo/testshared/shared_test.go +++ b/misc/cgo/testshared/shared_test.go @@ -790,6 +790,7 @@ func TestRebuilding(t *testing.T) { // If the .a file is newer than the .so, the .so is rebuilt (but not the .a) t.Run("newarchive", func(t *testing.T) { resetFileStamps() + AssertNotRebuilt(t, "new .a file before build", filepath.Join(gopathInstallDir, "depBase.a")) goCmd(t, "list", "-linkshared", "-f={{.ImportPath}} {{.Stale}} {{.StaleReason}} {{.Target}}", "depBase") AssertNotRebuilt(t, "new .a file before build", filepath.Join(gopathInstallDir, "depBase.a")) cleanup := touch(t, filepath.Join(gopathInstallDir, "depBase.a")) diff --git a/src/archive/zip/reader.go b/src/archive/zip/reader.go index 1563e74dfc..2444106ba6 100644 --- a/src/archive/zip/reader.go +++ b/src/archive/zip/reader.go @@ -366,7 +366,7 @@ parseExtras: epoch := time.Date(1601, time.January, 1, 0, 0, 0, 0, time.UTC) modified = time.Unix(epoch.Unix()+secs, nsecs) } - case unixExtraID: + case unixExtraID, infoZipUnixExtraID: if len(fieldBuf) < 8 { continue parseExtras } @@ -379,12 +379,6 @@ parseExtras: } ts := int64(fieldBuf.uint32()) // ModTime since Unix epoch modified = time.Unix(ts, 0) - case infoZipUnixExtraID: - if len(fieldBuf) < 4 { - continue parseExtras - } - ts := int64(fieldBuf.uint32()) // ModTime since Unix epoch - modified = time.Unix(ts, 0) } } diff --git a/src/archive/zip/reader_test.go b/src/archive/zip/reader_test.go index 0d9040f767..1e58b26b6e 100644 --- a/src/archive/zip/reader_test.go +++ b/src/archive/zip/reader_test.go @@ -414,7 +414,7 @@ var tests = []ZipTest{ Name: "test.txt", Content: []byte{}, Size: 1<<32 - 1, - Modified: time.Date(2017, 10, 31, 21, 17, 27, 0, timeZone(-7*time.Hour)), + Modified: time.Date(2017, 10, 31, 21, 11, 57, 0, timeZone(-7*time.Hour)), Mode: 0644, }, }, diff --git a/src/cmd/compile/internal/gc/noder.go b/src/cmd/compile/internal/gc/noder.go index fff04bcbef..6f32e52ec8 100644 --- a/src/cmd/compile/internal/gc/noder.go +++ b/src/cmd/compile/internal/gc/noder.go @@ -1346,8 +1346,22 @@ func (p *noder) pragma(pos src.Pos, text string) syntax.Pragma { } p.linknames = append(p.linknames, linkname{pos, f[1], f[2]}) + case strings.HasPrefix(text, "go:cgo_import_dynamic "): + // This is permitted for general use because Solaris + // code relies on it in golang.org/x/sys/unix and others. + fields := pragmaFields(text) + if len(fields) >= 4 { + lib := strings.Trim(fields[3], `"`) + if lib != "" && !safeArg(lib) && !isCgoGeneratedFile(pos) { + p.error(syntax.Error{Pos: pos, Msg: fmt.Sprintf("invalid library name %q in cgo_import_dynamic directive", lib)}) + } + p.pragcgobuf += p.pragcgo(pos, text) + return pragmaValue("go:cgo_import_dynamic") + } + fallthrough case strings.HasPrefix(text, "go:cgo_"): - // For security, we disallow //go:cgo_* directives outside cgo-generated files. + // For security, we disallow //go:cgo_* directives other + // than cgo_import_dynamic outside cgo-generated files. // Exception: they are allowed in the standard library, for runtime and syscall. if !isCgoGeneratedFile(pos) && !compiling_std { p.error(syntax.Error{Pos: pos, Msg: fmt.Sprintf("//%s only allowed in cgo-generated code", text)}) @@ -1383,6 +1397,18 @@ func isCgoGeneratedFile(pos src.Pos) bool { return strings.HasPrefix(filepath.Base(filepath.Clean(pos.AbsFilename())), "_cgo_") } +// safeArg reports whether arg is a "safe" command-line argument, +// meaning that when it appears in a command-line, it probably +// doesn't have some special meaning other than its own name. +// This is copied from SafeArg in cmd/go/internal/load/pkg.go. +func safeArg(name string) bool { + if name == "" { + return false + } + c := name[0] + return '0' <= c && c <= '9' || 'A' <= c && c <= 'Z' || 'a' <= c && c <= 'z' || c == '.' || c == '_' || c == '/' || c >= utf8.RuneSelf +} + func mkname(sym *types.Sym) *Node { n := oldname(sym) if n.Name != nil && n.Name.Pack != nil { diff --git a/src/cmd/compile/internal/gc/walk.go b/src/cmd/compile/internal/gc/walk.go index f48513dc73..ca3025bbaa 100644 --- a/src/cmd/compile/internal/gc/walk.go +++ b/src/cmd/compile/internal/gc/walk.go @@ -1261,6 +1261,13 @@ opswitch: } if cs != nil { cmp := Op(n.Etype) + // Our comparison below assumes that the non-constant string + // is on the left hand side, so rewrite "" cmp x to x cmp "". + // See issue 24817. + if Isconst(n.Left, CTSTR) { + cmp = brrev(cmp) + } + // maxRewriteLen was chosen empirically. // It is the value that minimizes cmd/go file size // across most architectures. diff --git a/src/cmd/compile/internal/ssa/gen/AMD64.rules b/src/cmd/compile/internal/ssa/gen/AMD64.rules index 90ff89c635..db7c1a447b 100644 --- a/src/cmd/compile/internal/ssa/gen/AMD64.rules +++ b/src/cmd/compile/internal/ssa/gen/AMD64.rules @@ -1647,9 +1647,9 @@ (SUBQconst (MOVQconst [d]) [c]) -> (MOVQconst [d-c]) (SUBQconst (SUBQconst x [d]) [c]) && is32Bit(-c-d) -> (ADDQconst [-c-d] x) (SARQconst [c] (MOVQconst [d])) -> (MOVQconst [d>>uint64(c)]) -(SARLconst [c] (MOVQconst [d])) -> (MOVQconst [d>>uint64(c)]) -(SARWconst [c] (MOVQconst [d])) -> (MOVQconst [d>>uint64(c)]) -(SARBconst [c] (MOVQconst [d])) -> (MOVQconst [d>>uint64(c)]) +(SARLconst [c] (MOVQconst [d])) -> (MOVQconst [int64(int32(d))>>uint64(c)]) +(SARWconst [c] (MOVQconst [d])) -> (MOVQconst [int64(int16(d))>>uint64(c)]) +(SARBconst [c] (MOVQconst [d])) -> (MOVQconst [int64(int8(d))>>uint64(c)]) (NEGQ (MOVQconst [c])) -> (MOVQconst [-c]) (NEGL (MOVLconst [c])) -> (MOVLconst [int64(int32(-c))]) (MULQconst [c] (MOVQconst [d])) -> (MOVQconst [c*d]) diff --git a/src/cmd/compile/internal/ssa/gen/AMD64Ops.go b/src/cmd/compile/internal/ssa/gen/AMD64Ops.go index 0c3b2efa30..b7c0ce9f08 100644 --- a/src/cmd/compile/internal/ssa/gen/AMD64Ops.go +++ b/src/cmd/compile/internal/ssa/gen/AMD64Ops.go @@ -270,22 +270,22 @@ func init() { // Note: x86 is weird, the 16 and 8 byte shifts still use all 5 bits of shift amount! {name: "SHRQ", argLength: 2, reg: gp21shift, asm: "SHRQ", resultInArg0: true, clobberFlags: true}, // unsigned arg0 >> arg1, shift amount is mod 64 - {name: "SHRL", argLength: 2, reg: gp21shift, asm: "SHRL", resultInArg0: true, clobberFlags: true}, // unsigned arg0 >> arg1, shift amount is mod 32 - {name: "SHRW", argLength: 2, reg: gp21shift, asm: "SHRW", resultInArg0: true, clobberFlags: true}, // unsigned arg0 >> arg1, shift amount is mod 32 - {name: "SHRB", argLength: 2, reg: gp21shift, asm: "SHRB", resultInArg0: true, clobberFlags: true}, // unsigned arg0 >> arg1, shift amount is mod 32 + {name: "SHRL", argLength: 2, reg: gp21shift, asm: "SHRL", resultInArg0: true, clobberFlags: true}, // unsigned uint32(arg0) >> arg1, shift amount is mod 32 + {name: "SHRW", argLength: 2, reg: gp21shift, asm: "SHRW", resultInArg0: true, clobberFlags: true}, // unsigned uint16(arg0) >> arg1, shift amount is mod 32 + {name: "SHRB", argLength: 2, reg: gp21shift, asm: "SHRB", resultInArg0: true, clobberFlags: true}, // unsigned uint8(arg0) >> arg1, shift amount is mod 32 {name: "SHRQconst", argLength: 1, reg: gp11, asm: "SHRQ", aux: "Int8", resultInArg0: true, clobberFlags: true}, // unsigned arg0 >> auxint, shift amount 0-63 - {name: "SHRLconst", argLength: 1, reg: gp11, asm: "SHRL", aux: "Int8", resultInArg0: true, clobberFlags: true}, // unsigned arg0 >> auxint, shift amount 0-31 - {name: "SHRWconst", argLength: 1, reg: gp11, asm: "SHRW", aux: "Int8", resultInArg0: true, clobberFlags: true}, // unsigned arg0 >> auxint, shift amount 0-15 - {name: "SHRBconst", argLength: 1, reg: gp11, asm: "SHRB", aux: "Int8", resultInArg0: true, clobberFlags: true}, // unsigned arg0 >> auxint, shift amount 0-7 + {name: "SHRLconst", argLength: 1, reg: gp11, asm: "SHRL", aux: "Int8", resultInArg0: true, clobberFlags: true}, // unsigned uint32(arg0) >> auxint, shift amount 0-31 + {name: "SHRWconst", argLength: 1, reg: gp11, asm: "SHRW", aux: "Int8", resultInArg0: true, clobberFlags: true}, // unsigned uint16(arg0) >> auxint, shift amount 0-15 + {name: "SHRBconst", argLength: 1, reg: gp11, asm: "SHRB", aux: "Int8", resultInArg0: true, clobberFlags: true}, // unsigned uint8(arg0) >> auxint, shift amount 0-7 {name: "SARQ", argLength: 2, reg: gp21shift, asm: "SARQ", resultInArg0: true, clobberFlags: true}, // signed arg0 >> arg1, shift amount is mod 64 - {name: "SARL", argLength: 2, reg: gp21shift, asm: "SARL", resultInArg0: true, clobberFlags: true}, // signed arg0 >> arg1, shift amount is mod 32 - {name: "SARW", argLength: 2, reg: gp21shift, asm: "SARW", resultInArg0: true, clobberFlags: true}, // signed arg0 >> arg1, shift amount is mod 32 - {name: "SARB", argLength: 2, reg: gp21shift, asm: "SARB", resultInArg0: true, clobberFlags: true}, // signed arg0 >> arg1, shift amount is mod 32 + {name: "SARL", argLength: 2, reg: gp21shift, asm: "SARL", resultInArg0: true, clobberFlags: true}, // signed int32(arg0) >> arg1, shift amount is mod 32 + {name: "SARW", argLength: 2, reg: gp21shift, asm: "SARW", resultInArg0: true, clobberFlags: true}, // signed int16(arg0) >> arg1, shift amount is mod 32 + {name: "SARB", argLength: 2, reg: gp21shift, asm: "SARB", resultInArg0: true, clobberFlags: true}, // signed int8(arg0) >> arg1, shift amount is mod 32 {name: "SARQconst", argLength: 1, reg: gp11, asm: "SARQ", aux: "Int8", resultInArg0: true, clobberFlags: true}, // signed arg0 >> auxint, shift amount 0-63 - {name: "SARLconst", argLength: 1, reg: gp11, asm: "SARL", aux: "Int8", resultInArg0: true, clobberFlags: true}, // signed arg0 >> auxint, shift amount 0-31 - {name: "SARWconst", argLength: 1, reg: gp11, asm: "SARW", aux: "Int8", resultInArg0: true, clobberFlags: true}, // signed arg0 >> auxint, shift amount 0-15 - {name: "SARBconst", argLength: 1, reg: gp11, asm: "SARB", aux: "Int8", resultInArg0: true, clobberFlags: true}, // signed arg0 >> auxint, shift amount 0-7 + {name: "SARLconst", argLength: 1, reg: gp11, asm: "SARL", aux: "Int8", resultInArg0: true, clobberFlags: true}, // signed int32(arg0) >> auxint, shift amount 0-31 + {name: "SARWconst", argLength: 1, reg: gp11, asm: "SARW", aux: "Int8", resultInArg0: true, clobberFlags: true}, // signed int16(arg0) >> auxint, shift amount 0-15 + {name: "SARBconst", argLength: 1, reg: gp11, asm: "SARB", aux: "Int8", resultInArg0: true, clobberFlags: true}, // signed int8(arg0) >> auxint, shift amount 0-7 {name: "ROLQ", argLength: 2, reg: gp21shift, asm: "ROLQ", resultInArg0: true, clobberFlags: true}, // arg0 rotate left arg1 bits. {name: "ROLL", argLength: 2, reg: gp21shift, asm: "ROLL", resultInArg0: true, clobberFlags: true}, // arg0 rotate left arg1 bits. diff --git a/src/cmd/compile/internal/ssa/gen/S390X.rules b/src/cmd/compile/internal/ssa/gen/S390X.rules index 6b997bd46d..366fcc2b2a 100644 --- a/src/cmd/compile/internal/ssa/gen/S390X.rules +++ b/src/cmd/compile/internal/ssa/gen/S390X.rules @@ -1088,7 +1088,7 @@ (SUBconst (MOVDconst [d]) [c]) -> (MOVDconst [d-c]) (SUBconst (SUBconst x [d]) [c]) && is32Bit(-c-d) -> (ADDconst [-c-d] x) (SRADconst [c] (MOVDconst [d])) -> (MOVDconst [d>>uint64(c)]) -(SRAWconst [c] (MOVDconst [d])) -> (MOVDconst [d>>uint64(c)]) +(SRAWconst [c] (MOVDconst [d])) -> (MOVDconst [int64(int32(d))>>uint64(c)]) (NEG (MOVDconst [c])) -> (MOVDconst [-c]) (NEGW (MOVDconst [c])) -> (MOVDconst [int64(int32(-c))]) (MULLDconst [c] (MOVDconst [d])) -> (MOVDconst [c*d]) diff --git a/src/cmd/compile/internal/ssa/gen/S390XOps.go b/src/cmd/compile/internal/ssa/gen/S390XOps.go index ab781cee90..e458e08a1a 100644 --- a/src/cmd/compile/internal/ssa/gen/S390XOps.go +++ b/src/cmd/compile/internal/ssa/gen/S390XOps.go @@ -309,15 +309,15 @@ func init() { {name: "SLWconst", argLength: 1, reg: gp11, asm: "SLW", aux: "Int8"}, // arg0 << auxint, shift amount 0-31 {name: "SRD", argLength: 2, reg: sh21, asm: "SRD"}, // unsigned arg0 >> arg1, shift amount is mod 64 - {name: "SRW", argLength: 2, reg: sh21, asm: "SRW"}, // unsigned arg0 >> arg1, shift amount is mod 32 + {name: "SRW", argLength: 2, reg: sh21, asm: "SRW"}, // unsigned uint32(arg0) >> arg1, shift amount is mod 32 {name: "SRDconst", argLength: 1, reg: gp11, asm: "SRD", aux: "Int8"}, // unsigned arg0 >> auxint, shift amount 0-63 - {name: "SRWconst", argLength: 1, reg: gp11, asm: "SRW", aux: "Int8"}, // unsigned arg0 >> auxint, shift amount 0-31 + {name: "SRWconst", argLength: 1, reg: gp11, asm: "SRW", aux: "Int8"}, // unsigned uint32(arg0) >> auxint, shift amount 0-31 // Arithmetic shifts clobber flags. {name: "SRAD", argLength: 2, reg: sh21, asm: "SRAD", clobberFlags: true}, // signed arg0 >> arg1, shift amount is mod 64 - {name: "SRAW", argLength: 2, reg: sh21, asm: "SRAW", clobberFlags: true}, // signed arg0 >> arg1, shift amount is mod 32 + {name: "SRAW", argLength: 2, reg: sh21, asm: "SRAW", clobberFlags: true}, // signed int32(arg0) >> arg1, shift amount is mod 32 {name: "SRADconst", argLength: 1, reg: gp11, asm: "SRAD", aux: "Int8", clobberFlags: true}, // signed arg0 >> auxint, shift amount 0-63 - {name: "SRAWconst", argLength: 1, reg: gp11, asm: "SRAW", aux: "Int8", clobberFlags: true}, // signed arg0 >> auxint, shift amount 0-31 + {name: "SRAWconst", argLength: 1, reg: gp11, asm: "SRAW", aux: "Int8", clobberFlags: true}, // signed int32(arg0) >> auxint, shift amount 0-31 {name: "RLLGconst", argLength: 1, reg: gp11, asm: "RLLG", aux: "Int8"}, // arg0 rotate left auxint, rotate amount 0-63 {name: "RLLconst", argLength: 1, reg: gp11, asm: "RLL", aux: "Int8"}, // arg0 rotate left auxint, rotate amount 0-31 @@ -485,8 +485,8 @@ func init() { // Atomic adds. // *(arg0+auxint+aux) += arg1. arg2=mem. // Returns a tuple of <old contents of *(arg0+auxint+aux), memory>. - {name: "LAA", argLength: 3, reg: gpstorelaa, asm: "LAA", typ: "(UInt32,Mem)", aux: "SymOff", faultOnNilArg0: true, hasSideEffects: true, symEffect: "RdWr"}, - {name: "LAAG", argLength: 3, reg: gpstorelaa, asm: "LAAG", typ: "(UInt64,Mem)", aux: "SymOff", faultOnNilArg0: true, hasSideEffects: true, symEffect: "RdWr"}, + {name: "LAA", argLength: 3, reg: gpstorelaa, asm: "LAA", typ: "(UInt32,Mem)", aux: "SymOff", clobberFlags: true, faultOnNilArg0: true, hasSideEffects: true, symEffect: "RdWr"}, + {name: "LAAG", argLength: 3, reg: gpstorelaa, asm: "LAAG", typ: "(UInt64,Mem)", aux: "SymOff", clobberFlags: true, faultOnNilArg0: true, hasSideEffects: true, symEffect: "RdWr"}, {name: "AddTupleFirst32", argLength: 2}, // arg1=tuple <x,y>. Returns <x+arg0,y>. {name: "AddTupleFirst64", argLength: 2}, // arg1=tuple <x,y>. Returns <x+arg0,y>. diff --git a/src/cmd/compile/internal/ssa/opGen.go b/src/cmd/compile/internal/ssa/opGen.go index 0beabb0b84..d50561cdf0 100644 --- a/src/cmd/compile/internal/ssa/opGen.go +++ b/src/cmd/compile/internal/ssa/opGen.go @@ -22080,6 +22080,7 @@ var opcodeTable = [...]opInfo{ name: "LAA", auxType: auxSymOff, argLen: 3, + clobberFlags: true, faultOnNilArg0: true, hasSideEffects: true, symEffect: SymRdWr, @@ -22098,6 +22099,7 @@ var opcodeTable = [...]opInfo{ name: "LAAG", auxType: auxSymOff, argLen: 3, + clobberFlags: true, faultOnNilArg0: true, hasSideEffects: true, symEffect: SymRdWr, diff --git a/src/cmd/compile/internal/ssa/rewriteAMD64.go b/src/cmd/compile/internal/ssa/rewriteAMD64.go index 607090ac27..0b2b321d25 100644 --- a/src/cmd/compile/internal/ssa/rewriteAMD64.go +++ b/src/cmd/compile/internal/ssa/rewriteAMD64.go @@ -34892,7 +34892,7 @@ func rewriteValueAMD64_OpAMD64SARBconst_0(v *Value) bool { } // match: (SARBconst [c] (MOVQconst [d])) // cond: - // result: (MOVQconst [d>>uint64(c)]) + // result: (MOVQconst [int64(int8(d))>>uint64(c)]) for { c := v.AuxInt v_0 := v.Args[0] @@ -34901,7 +34901,7 @@ func rewriteValueAMD64_OpAMD64SARBconst_0(v *Value) bool { } d := v_0.AuxInt v.reset(OpAMD64MOVQconst) - v.AuxInt = d >> uint64(c) + v.AuxInt = int64(int8(d)) >> uint64(c) return true } return false @@ -35147,7 +35147,7 @@ func rewriteValueAMD64_OpAMD64SARLconst_0(v *Value) bool { } // match: (SARLconst [c] (MOVQconst [d])) // cond: - // result: (MOVQconst [d>>uint64(c)]) + // result: (MOVQconst [int64(int32(d))>>uint64(c)]) for { c := v.AuxInt v_0 := v.Args[0] @@ -35156,7 +35156,7 @@ func rewriteValueAMD64_OpAMD64SARLconst_0(v *Value) bool { } d := v_0.AuxInt v.reset(OpAMD64MOVQconst) - v.AuxInt = d >> uint64(c) + v.AuxInt = int64(int32(d)) >> uint64(c) return true } return false @@ -35467,7 +35467,7 @@ func rewriteValueAMD64_OpAMD64SARWconst_0(v *Value) bool { } // match: (SARWconst [c] (MOVQconst [d])) // cond: - // result: (MOVQconst [d>>uint64(c)]) + // result: (MOVQconst [int64(int16(d))>>uint64(c)]) for { c := v.AuxInt v_0 := v.Args[0] @@ -35476,7 +35476,7 @@ func rewriteValueAMD64_OpAMD64SARWconst_0(v *Value) bool { } d := v_0.AuxInt v.reset(OpAMD64MOVQconst) - v.AuxInt = d >> uint64(c) + v.AuxInt = int64(int16(d)) >> uint64(c) return true } return false diff --git a/src/cmd/compile/internal/ssa/rewriteS390X.go b/src/cmd/compile/internal/ssa/rewriteS390X.go index fe9b2bd001..5f25e9d279 100644 --- a/src/cmd/compile/internal/ssa/rewriteS390X.go +++ b/src/cmd/compile/internal/ssa/rewriteS390X.go @@ -37197,7 +37197,7 @@ func rewriteValueS390X_OpS390XSRAW_0(v *Value) bool { func rewriteValueS390X_OpS390XSRAWconst_0(v *Value) bool { // match: (SRAWconst [c] (MOVDconst [d])) // cond: - // result: (MOVDconst [d>>uint64(c)]) + // result: (MOVDconst [int64(int32(d))>>uint64(c)]) for { c := v.AuxInt v_0 := v.Args[0] @@ -37206,7 +37206,7 @@ func rewriteValueS390X_OpS390XSRAWconst_0(v *Value) bool { } d := v_0.AuxInt v.reset(OpS390XMOVDconst) - v.AuxInt = d >> uint64(c) + v.AuxInt = int64(int32(d)) >> uint64(c) return true } return false diff --git a/src/cmd/cover/cover.go b/src/cmd/cover/cover.go index 500027ee0d..f496f4cff6 100644 --- a/src/cmd/cover/cover.go +++ b/src/cmd/cover/cover.go @@ -238,23 +238,28 @@ func (f *File) Visit(node ast.Node) ast.Visitor { // if y { // } // } - f.edit.Insert(f.offset(n.Body.End()), "else{") elseOffset := f.findText(n.Body.End(), "else") if elseOffset < 0 { panic("lost else") } - f.edit.Delete(elseOffset, elseOffset+4) + f.edit.Insert(elseOffset+4, "{") f.edit.Insert(f.offset(n.Else.End()), "}") + + // We just created a block, now walk it. + // Adjust the position of the new block to start after + // the "else". That will cause it to follow the "{" + // we inserted above. + pos := f.fset.File(n.Body.End()).Pos(elseOffset + 4) switch stmt := n.Else.(type) { case *ast.IfStmt: block := &ast.BlockStmt{ - Lbrace: n.Body.End(), // Start at end of the "if" block so the covered part looks like it starts at the "else". + Lbrace: pos, List: []ast.Stmt{stmt}, Rbrace: stmt.End(), } n.Else = block case *ast.BlockStmt: - stmt.Lbrace = n.Body.End() // Start at end of the "if" block so the covered part looks like it starts at the "else". + stmt.Lbrace = pos default: panic("unexpected node type in if") } diff --git a/src/cmd/cover/cover_test.go b/src/cmd/cover/cover_test.go index 79ddf4f465..f20fbb4b71 100644 --- a/src/cmd/cover/cover_test.go +++ b/src/cmd/cover/cover_test.go @@ -59,6 +59,17 @@ func TestCover(t *testing.T) { for i, line := range lines { lines[i] = bytes.Replace(line, []byte("LINE"), []byte(fmt.Sprint(i+1)), -1) } + + // Add a function that is not gofmt'ed. This used to cause a crash. + // We don't put it in test.go because then we would have to gofmt it. + // Issue 23927. + lines = append(lines, []byte("func unFormatted() {"), + []byte("\tif true {"), + []byte("\t}else{"), + []byte("\t}"), + []byte("}")) + lines = append(lines, []byte("func unFormatted2(b bool) {if b{}else{}}")) + if err := ioutil.WriteFile(coverInput, bytes.Join(lines, []byte("\n")), 0666); err != nil { t.Fatal(err) } @@ -246,6 +257,7 @@ func TestCoverFunc(t *testing.T) { } func run(c *exec.Cmd, t *testing.T) { + t.Helper() c.Stdout = os.Stdout c.Stderr = os.Stderr err := c.Run() diff --git a/src/cmd/fix/cftype.go b/src/cmd/fix/cftype.go index df1cc18f9e..b47b06682a 100644 --- a/src/cmd/fix/cftype.go +++ b/src/cmd/fix/cftype.go @@ -119,7 +119,7 @@ func typefix(f *ast.File, badType func(string) bool) bool { if !ok { return } - t := s.X.(*ast.SelectorExpr) + t, ok := s.X.(*ast.SelectorExpr) if !ok { return } diff --git a/src/cmd/go/go_test.go b/src/cmd/go/go_test.go index 0cd4d85639..80833d1b0a 100644 --- a/src/cmd/go/go_test.go +++ b/src/cmd/go/go_test.go @@ -3233,6 +3233,20 @@ func TestGoVetWithOnlyTestFiles(t *testing.T) { tg.run("vet", "p") } +// Issue 24193. +func TestVetWithOnlyCgoFiles(t *testing.T) { + if !canCgo { + t.Skip("skipping because cgo not enabled") + } + + tg := testgo(t) + defer tg.cleanup() + tg.parallel() + tg.tempFile("src/p/p.go", "package p; import \"C\"; func F() {}") + tg.setenv("GOPATH", tg.path(".")) + tg.run("vet", "p") +} + // Issue 9767, 19769. func TestGoGetDotSlashDownload(t *testing.T) { testenv.MustHaveExternalNetwork(t) @@ -5068,6 +5082,28 @@ func TestCacheOutput(t *testing.T) { } } +func TestCacheListStale(t *testing.T) { + tooSlow(t) + if strings.Contains(os.Getenv("GODEBUG"), "gocacheverify") { + t.Skip("GODEBUG gocacheverify") + } + tg := testgo(t) + defer tg.cleanup() + tg.parallel() + tg.makeTempdir() + tg.setenv("GOCACHE", tg.path("cache")) + tg.tempFile("gopath/src/p/p.go", "package p; import _ \"q\"; func F(){}\n") + tg.tempFile("gopath/src/q/q.go", "package q; func F(){}\n") + tg.tempFile("gopath/src/m/m.go", "package main; import _ \"q\"; func main(){}\n") + + tg.setenv("GOPATH", tg.path("gopath")) + tg.run("install", "p", "m") + tg.run("list", "-f={{.ImportPath}} {{.Stale}}", "m", "q", "p") + tg.grepStdout("^m false", "m should not be stale") + tg.grepStdout("^q true", "q should be stale") + tg.grepStdout("^p false", "p should not be stale") +} + func TestCacheCoverage(t *testing.T) { tooSlow(t) @@ -5748,6 +5784,21 @@ func TestAtomicCoverpkgAll(t *testing.T) { } } +// Issue 23882. +func TestCoverpkgAllRuntime(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + tg.parallel() + + tg.tempFile("src/x/x.go", `package x; import _ "runtime"; func F() {}`) + tg.tempFile("src/x/x_test.go", `package x; import "testing"; func TestF(t *testing.T) { F() }`) + tg.setenv("GOPATH", tg.path(".")) + tg.run("test", "-coverpkg=all", "x") + if canRace { + tg.run("test", "-coverpkg=all", "-race", "x") + } +} + func TestBadCommandLines(t *testing.T) { tg := testgo(t) defer tg.cleanup() @@ -5896,3 +5947,36 @@ func TestBadCgoDirectives(t *testing.T) { tg.run("build", "-n", "x") tg.grepStderr("-D@foo", "did not find -D@foo in commands") } + +func TestTwoPkgConfigs(t *testing.T) { + if !canCgo { + t.Skip("no cgo") + } + if runtime.GOOS == "windows" || runtime.GOOS == "plan9" { + t.Skipf("no shell scripts on %s", runtime.GOOS) + } + tg := testgo(t) + defer tg.cleanup() + tg.parallel() + tg.tempFile("src/x/a.go", `package x + // #cgo pkg-config: --static a + import "C" + `) + tg.tempFile("src/x/b.go", `package x + // #cgo pkg-config: --static a + import "C" + `) + tg.tempFile("pkg-config.sh", `#!/bin/sh +echo $* >>`+tg.path("pkg-config.out")) + tg.must(os.Chmod(tg.path("pkg-config.sh"), 0755)) + tg.setenv("GOPATH", tg.path(".")) + tg.setenv("PKG_CONFIG", tg.path("pkg-config.sh")) + tg.run("build", "x") + out, err := ioutil.ReadFile(tg.path("pkg-config.out")) + tg.must(err) + out = bytes.TrimSpace(out) + want := "--cflags --static --static -- a a\n--libs --static --static -- a a" + if !bytes.Equal(out, []byte(want)) { + t.Errorf("got %q want %q", out, want) + } +} diff --git a/src/cmd/go/internal/get/discovery.go b/src/cmd/go/internal/get/discovery.go index b2918dbb4f..97aa1d7e8d 100644 --- a/src/cmd/go/internal/get/discovery.go +++ b/src/cmd/go/internal/get/discovery.go @@ -55,6 +55,13 @@ func parseMetaGoImports(r io.Reader) (imports []metaImport, err error) { continue } if f := strings.Fields(attrValue(e.Attr, "content")); len(f) == 3 { + // Ignore VCS type "mod", which is new Go modules. + // This code is for old go get and must ignore the new mod lines. + // Otherwise matchGoImport will complain about two + // different metaImport lines for the same Prefix. + if f[1] == "mod" { + continue + } imports = append(imports, metaImport{ Prefix: f[0], VCS: f[1], diff --git a/src/cmd/go/internal/get/get.go b/src/cmd/go/internal/get/get.go index 733116eca0..5bfeac387c 100644 --- a/src/cmd/go/internal/get/get.go +++ b/src/cmd/go/internal/get/get.go @@ -209,7 +209,7 @@ var downloadRootCache = map[string]bool{} // download runs the download half of the get command // for the package named by the argument. func download(arg string, parent *load.Package, stk *load.ImportStack, mode int) { - if mode&load.UseVendor != 0 { + if mode&load.ResolveImport != 0 { // Caller is responsible for expanding vendor paths. panic("internal error: download mode has useVendor set") } @@ -217,7 +217,7 @@ func download(arg string, parent *load.Package, stk *load.ImportStack, mode int) if parent == nil { return load.LoadPackage(path, stk) } - return load.LoadImport(path, parent.Dir, parent, stk, nil, mode) + return load.LoadImport(path, parent.Dir, parent, stk, nil, mode|load.ResolveModule) } p := load1(arg, mode) @@ -346,12 +346,12 @@ func download(arg string, parent *load.Package, stk *load.ImportStack, mode int) base.Errorf("%s", err) continue } - // If this is a test import, apply vendor lookup now. - // We cannot pass useVendor to download, because + // If this is a test import, apply module and vendor lookup now. + // We cannot pass ResolveImport to download, because // download does caching based on the value of path, // so it must be the fully qualified path already. if i >= len(p.Imports) { - path = load.VendoredImportPath(p, path) + path = load.ResolveImportPath(p, path) } download(path, p, stk, 0) } diff --git a/src/cmd/go/internal/get/pkg_test.go b/src/cmd/go/internal/get/pkg_test.go index b8937a57ec..1179d86693 100644 --- a/src/cmd/go/internal/get/pkg_test.go +++ b/src/cmd/go/internal/get/pkg_test.go @@ -48,6 +48,20 @@ var parseMetaGoImportsTests = []struct { }, }, { + `<meta name="go-import" content="foo/bar git https://github.com/rsc/foo/bar"> + <meta name="go-import" content="foo/bar mod http://github.com/rsc/baz/quux">`, + []metaImport{ + {"foo/bar", "git", "https://github.com/rsc/foo/bar"}, + }, + }, + { + `<meta name="go-import" content="foo/bar mod http://github.com/rsc/baz/quux"> + <meta name="go-import" content="foo/bar git https://github.com/rsc/foo/bar">`, + []metaImport{ + {"foo/bar", "git", "https://github.com/rsc/foo/bar"}, + }, + }, + { `<head> <meta name="go-import" content="foo/bar git https://github.com/rsc/foo/bar"> </head>`, diff --git a/src/cmd/go/internal/get/vcs.go b/src/cmd/go/internal/get/vcs.go index 26693b13a9..0b2a04e04f 100644 --- a/src/cmd/go/internal/get/vcs.go +++ b/src/cmd/go/internal/get/vcs.go @@ -809,8 +809,8 @@ func repoRootForImportDynamic(importPath string, security web.SecurityMode) (*re } } - if !strings.Contains(mmi.RepoRoot, "://") { - return nil, fmt.Errorf("%s: invalid repo root %q; no scheme", urlStr, mmi.RepoRoot) + if err := validateRepoRootScheme(mmi.RepoRoot); err != nil { + return nil, fmt.Errorf("%s: invalid repo root %q: %v", urlStr, mmi.RepoRoot, err) } rr := &repoRoot{ vcs: vcsByCmd(mmi.VCS), @@ -824,6 +824,36 @@ func repoRootForImportDynamic(importPath string, security web.SecurityMode) (*re return rr, nil } +// validateRepoRootScheme returns an error if repoRoot does not seem +// to have a valid URL scheme. At this point we permit things that +// aren't valid URLs, although later, if not using -insecure, we will +// restrict repoRoots to be valid URLs. This is only because we've +// historically permitted them, and people may depend on that. +func validateRepoRootScheme(repoRoot string) error { + end := strings.Index(repoRoot, "://") + if end <= 0 { + return errors.New("no scheme") + } + + // RFC 3986 section 3.1. + for i := 0; i < end; i++ { + c := repoRoot[i] + switch { + case 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z': + // OK. + case '0' <= c && c <= '9' || c == '+' || c == '-' || c == '.': + // OK except at start. + if i == 0 { + return errors.New("invalid scheme") + } + default: + return errors.New("invalid scheme") + } + } + + return nil +} + var fetchGroup singleflight.Group var ( fetchCacheMu sync.Mutex diff --git a/src/cmd/go/internal/get/vcs_test.go b/src/cmd/go/internal/get/vcs_test.go index e29338aec1..a6f8642026 100644 --- a/src/cmd/go/internal/get/vcs_test.go +++ b/src/cmd/go/internal/get/vcs_test.go @@ -408,3 +408,46 @@ func TestMatchGoImport(t *testing.T) { } } } + +func TestValidateRepoRootScheme(t *testing.T) { + tests := []struct { + root string + err string + }{ + { + root: "", + err: "no scheme", + }, + { + root: "http://", + err: "", + }, + { + root: "a://", + err: "", + }, + { + root: "a#://", + err: "invalid scheme", + }, + { + root: "-config://", + err: "invalid scheme", + }, + } + + for _, test := range tests { + err := validateRepoRootScheme(test.root) + if err == nil { + if test.err != "" { + t.Errorf("validateRepoRootScheme(%q) = nil, want %q", test.root, test.err) + } + } else if test.err == "" { + if err != nil { + t.Errorf("validateRepoRootScheme(%q) = %q, want nil", test.root, test.err) + } + } else if err.Error() != test.err { + t.Errorf("validateRepoRootScheme(%q) = %q, want %q", test.root, err, test.err) + } + } +} diff --git a/src/cmd/go/internal/list/list.go b/src/cmd/go/internal/list/list.go index 7435273000..16e7f70d73 100644 --- a/src/cmd/go/internal/list/list.go +++ b/src/cmd/go/internal/list/list.go @@ -218,8 +218,8 @@ func runList(cmd *base.Command, args []string) { for _, pkg := range pkgs { // Show vendor-expanded paths in listing - pkg.TestImports = pkg.Vendored(pkg.TestImports) - pkg.XTestImports = pkg.Vendored(pkg.XTestImports) + pkg.TestImports = pkg.Resolve(pkg.TestImports) + pkg.XTestImports = pkg.Resolve(pkg.XTestImports) do(&pkg.PackagePublic) } diff --git a/src/cmd/go/internal/load/icfg.go b/src/cmd/go/internal/load/icfg.go deleted file mode 100644 index d8dd66426b..0000000000 --- a/src/cmd/go/internal/load/icfg.go +++ /dev/null @@ -1,78 +0,0 @@ -// Copyright 2017 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package load - -import ( - "bytes" - "encoding/json" - "errors" - "io/ioutil" -) - -// DebugDeprecatedImportcfg is installed as the undocumented -debug-deprecated-importcfg build flag. -// It is useful for debugging subtle problems in the go command logic but not something -// we want users to depend on. The hope is that the "deprecated" will make that clear. -// We intend to remove this flag in Go 1.11. -var DebugDeprecatedImportcfg debugDeprecatedImportcfgFlag - -type debugDeprecatedImportcfgFlag struct { - enabled bool - Import map[string]string - Pkg map[string]*debugDeprecatedImportcfgPkg -} - -type debugDeprecatedImportcfgPkg struct { - Dir string - Import map[string]string -} - -var ( - debugDeprecatedImportcfgMagic = []byte("# debug-deprecated-importcfg\n") - errImportcfgSyntax = errors.New("malformed syntax") -) - -func (f *debugDeprecatedImportcfgFlag) String() string { return "" } - -func (f *debugDeprecatedImportcfgFlag) Set(x string) error { - if x == "" { - *f = debugDeprecatedImportcfgFlag{} - return nil - } - data, err := ioutil.ReadFile(x) - if err != nil { - return err - } - - if !bytes.HasPrefix(data, debugDeprecatedImportcfgMagic) { - return errImportcfgSyntax - } - data = data[len(debugDeprecatedImportcfgMagic):] - - f.Import = nil - f.Pkg = nil - if err := json.Unmarshal(data, &f); err != nil { - return errImportcfgSyntax - } - f.enabled = true - return nil -} - -func (f *debugDeprecatedImportcfgFlag) lookup(parent *Package, path string) (dir, newPath string) { - newPath = path - if p := f.Import[path]; p != "" { - newPath = p - } - if parent != nil { - if p1 := f.Pkg[parent.ImportPath]; p1 != nil { - if p := p1.Import[path]; p != "" { - newPath = p - } - } - } - if p2 := f.Pkg[newPath]; p2 != nil { - return p2.Dir, newPath - } - return "", "" -} diff --git a/src/cmd/go/internal/load/pkg.go b/src/cmd/go/internal/load/pkg.go index d44c855928..a2f266d24d 100644 --- a/src/cmd/go/internal/load/pkg.go +++ b/src/cmd/go/internal/load/pkg.go @@ -6,6 +6,7 @@ package load import ( + "bytes" "fmt" "go/build" "go/token" @@ -14,6 +15,7 @@ import ( pathpkg "path" "path/filepath" "sort" + "strconv" "strings" "unicode" "unicode/utf8" @@ -168,7 +170,7 @@ func (e *NoGoError) Error() string { return "no Go files in " + e.Package.Dir } -// Vendored returns the vendor-resolved version of imports, +// Resolve returns the resolved version of imports, // which should be p.TestImports or p.XTestImports, NOT p.Imports. // The imports in p.TestImports and p.XTestImports are not recursively // loaded during the initial load of p, so they list the imports found in @@ -178,14 +180,14 @@ func (e *NoGoError) Error() string { // can produce better error messages if it starts with the original paths. // The initial load of p loads all the non-test imports and rewrites // the vendored paths, so nothing should ever call p.vendored(p.Imports). -func (p *Package) Vendored(imports []string) []string { +func (p *Package) Resolve(imports []string) []string { if len(imports) > 0 && len(p.Imports) > 0 && &imports[0] == &p.Imports[0] { - panic("internal error: p.vendored(p.Imports) called") + panic("internal error: p.Resolve(p.Imports) called") } seen := make(map[string]bool) var all []string for _, path := range imports { - path = VendoredImportPath(p, path) + path = ResolveImportPath(p, path) if !seen[path] { seen[path] = true all = append(all, path) @@ -386,16 +388,20 @@ func makeImportValid(r rune) rune { // Mode flags for loadImport and download (in get.go). const ( - // UseVendor means that loadImport should do vendor expansion - // (provided the vendoring experiment is enabled). - // That is, useVendor means that the import path came from - // a source file and has not been vendor-expanded yet. - // Every import path should be loaded initially with useVendor, - // and then the expanded version (with the /vendor/ in it) gets - // recorded as the canonical import path. At that point, future loads - // of that package must not pass useVendor, because + // ResolveImport means that loadImport should do import path expansion. + // That is, ResolveImport means that the import path came from + // a source file and has not been expanded yet to account for + // vendoring or possible module adjustment. + // Every import path should be loaded initially with ResolveImport, + // and then the expanded version (for example with the /vendor/ in it) + // gets recorded as the canonical import path. At that point, future loads + // of that package must not pass ResolveImport, because // disallowVendor will reject direct use of paths containing /vendor/. - UseVendor = 1 << iota + ResolveImport = 1 << iota + + // ResolveModule is for download (part of "go get") and indicates + // that the module adjustment should be done, but not vendor adjustment. + ResolveModule // GetTestDeps is for download (part of "go get") and indicates // that test dependencies should be fetched too. @@ -418,20 +424,17 @@ func LoadImport(path, srcDir string, parent *Package, stk *ImportStack, importPo importPath := path origPath := path isLocal := build.IsLocalImport(path) - var debugDeprecatedImportcfgDir string if isLocal { importPath = dirToImportPath(filepath.Join(srcDir, path)) - } else if DebugDeprecatedImportcfg.enabled { - if d, i := DebugDeprecatedImportcfg.lookup(parent, path); d != "" { - debugDeprecatedImportcfgDir = d - importPath = i - } - } else if mode&UseVendor != 0 { - // We do our own vendor resolution, because we want to + } else if mode&ResolveImport != 0 { + // We do our own path resolution, because we want to // find out the key to use in packageCache without the // overhead of repeated calls to buildContext.Import. // The code is also needed in a few other places anyway. - path = VendoredImportPath(parent, path) + path = ResolveImportPath(parent, path) + importPath = path + } else if mode&ResolveModule != 0 { + path = ModuleImportPath(parent, path) importPath = path } @@ -447,26 +450,17 @@ func LoadImport(path, srcDir string, parent *Package, stk *ImportStack, importPo // Load package. // Import always returns bp != nil, even if an error occurs, // in order to return partial information. - var bp *build.Package - var err error - if debugDeprecatedImportcfgDir != "" { - bp, err = cfg.BuildContext.ImportDir(debugDeprecatedImportcfgDir, 0) - } else if DebugDeprecatedImportcfg.enabled { - bp = new(build.Package) - err = fmt.Errorf("unknown import path %q: not in import cfg", importPath) - } else { - buildMode := build.ImportComment - if mode&UseVendor == 0 || path != origPath { - // Not vendoring, or we already found the vendored path. - buildMode |= build.IgnoreVendor - } - bp, err = cfg.BuildContext.Import(path, srcDir, buildMode) + buildMode := build.ImportComment + if mode&ResolveImport == 0 || path != origPath { + // Not vendoring, or we already found the vendored path. + buildMode |= build.IgnoreVendor } + bp, err := cfg.BuildContext.Import(path, srcDir, buildMode) bp.ImportPath = importPath if cfg.GOBIN != "" { bp.BinDir = cfg.GOBIN } - if debugDeprecatedImportcfgDir == "" && err == nil && !isLocal && bp.ImportComment != "" && bp.ImportComment != path && + if err == nil && !isLocal && bp.ImportComment != "" && bp.ImportComment != path && !strings.Contains(path, "/vendor/") && !strings.HasPrefix(path, "vendor/") { err = fmt.Errorf("code in directory %s expects import %q", bp.Dir, bp.ImportComment) } @@ -475,7 +469,7 @@ func LoadImport(path, srcDir string, parent *Package, stk *ImportStack, importPo p = setErrorPos(p, importPos) } - if debugDeprecatedImportcfgDir == "" && origPath != cleanImport(origPath) { + if origPath != cleanImport(origPath) { p.Error = &PackageError{ ImportStack: stk.Copy(), Err: fmt.Sprintf("non-canonical import path: %q should be %q", origPath, pathpkg.Clean(origPath)), @@ -488,7 +482,7 @@ func LoadImport(path, srcDir string, parent *Package, stk *ImportStack, importPo if perr := disallowInternal(srcDir, p, stk); perr != p { return setErrorPos(perr, importPos) } - if mode&UseVendor != 0 { + if mode&ResolveImport != 0 { if perr := disallowVendor(srcDir, origPath, p, stk); perr != p { return setErrorPos(perr, importPos) } @@ -547,31 +541,31 @@ func isDir(path string) bool { return result } -// VendoredImportPath returns the expansion of path when it appears in parent. -// If parent is x/y/z, then path might expand to x/y/z/vendor/path, x/y/vendor/path, -// x/vendor/path, vendor/path, or else stay path if none of those exist. -// VendoredImportPath returns the expanded path or, if no expansion is found, the original. -func VendoredImportPath(parent *Package, path string) (found string) { - if DebugDeprecatedImportcfg.enabled { - if d, i := DebugDeprecatedImportcfg.lookup(parent, path); d != "" { - return i - } - return path - } - - if parent == nil || parent.Root == "" { - return path - } +// ResolveImportPath returns the true meaning of path when it appears in parent. +// There are two different resolutions applied. +// First, there is Go 1.5 vendoring (golang.org/s/go15vendor). +// If vendor expansion doesn't trigger, then the path is also subject to +// Go 1.11 vgo legacy conversion (golang.org/issue/25069). +func ResolveImportPath(parent *Package, path string) (found string) { + found = VendoredImportPath(parent, path) + if found != path { + return found + } + return ModuleImportPath(parent, path) +} - dir := filepath.Clean(parent.Dir) - root := filepath.Join(parent.Root, "src") - if !str.HasFilePathPrefix(dir, root) || parent.ImportPath != "command-line-arguments" && filepath.Join(root, parent.ImportPath) != dir { +// dirAndRoot returns the source directory and workspace root +// for the package p, guaranteeing that root is a path prefix of dir. +func dirAndRoot(p *Package) (dir, root string) { + dir = filepath.Clean(p.Dir) + root = filepath.Join(p.Root, "src") + if !str.HasFilePathPrefix(dir, root) || p.ImportPath != "command-line-arguments" && filepath.Join(root, p.ImportPath) != dir { // Look for symlinks before reporting error. dir = expandPath(dir) root = expandPath(root) } - if !str.HasFilePathPrefix(dir, root) || len(dir) <= len(root) || dir[len(root)] != filepath.Separator || parent.ImportPath != "command-line-arguments" && !parent.Internal.Local && filepath.Join(root, parent.ImportPath) != dir { + if !str.HasFilePathPrefix(dir, root) || len(dir) <= len(root) || dir[len(root)] != filepath.Separator || p.ImportPath != "command-line-arguments" && !p.Internal.Local && filepath.Join(root, p.ImportPath) != dir { base.Fatalf("unexpected directory layout:\n"+ " import path: %s\n"+ " root: %s\n"+ @@ -579,14 +573,28 @@ func VendoredImportPath(parent *Package, path string) (found string) { " expand root: %s\n"+ " expand dir: %s\n"+ " separator: %s", - parent.ImportPath, - filepath.Join(parent.Root, "src"), - filepath.Clean(parent.Dir), + p.ImportPath, + filepath.Join(p.Root, "src"), + filepath.Clean(p.Dir), root, dir, string(filepath.Separator)) } + return dir, root +} + +// VendoredImportPath returns the vendor-expansion of path when it appears in parent. +// If parent is x/y/z, then path might expand to x/y/z/vendor/path, x/y/vendor/path, +// x/vendor/path, vendor/path, or else stay path if none of those exist. +// VendoredImportPath returns the expanded path or, if no expansion is found, the original. +func VendoredImportPath(parent *Package, path string) (found string) { + if parent == nil || parent.Root == "" { + return path + } + + dir, root := dirAndRoot(parent) + vpath := "vendor/" + path for i := len(dir); i >= len(root); i-- { if i < len(dir) && dir[i] != filepath.Separator { @@ -629,6 +637,164 @@ func VendoredImportPath(parent *Package, path string) (found string) { return path } +var ( + modulePrefix = []byte("\nmodule ") + goModPathCache = make(map[string]string) +) + +// goModPath returns the module path in the go.mod in dir, if any. +func goModPath(dir string) (path string) { + path, ok := goModPathCache[dir] + if ok { + return path + } + defer func() { + goModPathCache[dir] = path + }() + + data, err := ioutil.ReadFile(filepath.Join(dir, "go.mod")) + if err != nil { + return "" + } + var i int + if bytes.HasPrefix(data, modulePrefix[1:]) { + i = 0 + } else { + i = bytes.Index(data, modulePrefix) + if i < 0 { + return "" + } + i++ + } + line := data[i:] + + // Cut line at \n, drop trailing \r if present. + if j := bytes.IndexByte(line, '\n'); j >= 0 { + line = line[:j] + } + if line[len(line)-1] == '\r' { + line = line[:len(line)-1] + } + line = line[len("module "):] + + // If quoted, unquote. + path = strings.TrimSpace(string(line)) + if path != "" && path[0] == '"' { + s, err := strconv.Unquote(path) + if err != nil { + return "" + } + path = s + } + return path +} + +// findVersionElement returns the slice indices of the final version element /vN in path. +// If there is no such element, it returns -1, -1. +func findVersionElement(path string) (i, j int) { + j = len(path) + for i = len(path) - 1; i >= 0; i-- { + if path[i] == '/' { + if isVersionElement(path[i:j]) { + return i, j + } + j = i + } + } + return -1, -1 +} + +// isVersionElement reports whether s is a well-formed path version element: +// v2, v3, v10, etc, but not v0, v05, v1. +func isVersionElement(s string) bool { + if len(s) < 3 || s[0] != '/' || s[1] != 'v' || s[2] == '0' || s[2] == '1' && len(s) == 3 { + return false + } + for i := 2; i < len(s); i++ { + if s[i] < '0' || '9' < s[i] { + return false + } + } + return true +} + +// ModuleImportPath translates import paths found in go modules +// back down to paths that can be resolved in ordinary builds. +// +// Define “new” code as code with a go.mod file in the same directory +// or a parent directory. If an import in new code says x/y/v2/z but +// x/y/v2/z does not exist and x/y/go.mod says “module x/y/v2”, +// then go build will read the import as x/y/z instead. +// See golang.org/issue/25069. +func ModuleImportPath(parent *Package, path string) (found string) { + if parent == nil || parent.Root == "" { + return path + } + + // If there are no vN elements in path, leave it alone. + // (The code below would do the same, but only after + // some other file system accesses that we can avoid + // here by returning early.) + if i, _ := findVersionElement(path); i < 0 { + return path + } + + dir, root := dirAndRoot(parent) + + // Consider dir and parents, up to and including root. + for i := len(dir); i >= len(root); i-- { + if i < len(dir) && dir[i] != filepath.Separator { + continue + } + if goModPath(dir[:i]) != "" { + goto HaveGoMod + } + } + // This code is not in a tree with a go.mod, + // so apply no changes to the path. + return path + +HaveGoMod: + // This import is in a tree with a go.mod. + // Allow it to refer to code in GOPATH/src/x/y/z as x/y/v2/z + // if GOPATH/src/x/y/go.mod says module "x/y/v2", + + // If x/y/v2/z exists, use it unmodified. + if bp, _ := cfg.BuildContext.Import(path, "", build.IgnoreVendor); bp.Dir != "" { + return path + } + + // Otherwise look for a go.mod supplying a version element. + // Some version-like elements may appear in paths but not + // be module versions; we skip over those to look for module + // versions. For example the module m/v2 might have a + // package m/v2/api/v1/foo. + limit := len(path) + for limit > 0 { + i, j := findVersionElement(path[:limit]) + if i < 0 { + return path + } + if bp, _ := cfg.BuildContext.Import(path[:i], "", build.IgnoreVendor); bp.Dir != "" { + if mpath := goModPath(bp.Dir); mpath != "" { + // Found a valid go.mod file, so we're stopping the search. + // If the path is m/v2/p and we found m/go.mod that says + // "module m/v2", then we return "m/p". + if mpath == path[:j] { + return path[:i] + path[j:] + } + // Otherwise just return the original path. + // We didn't find anything worth rewriting, + // and the go.mod indicates that we should + // not consider parent directories. + return path + } + } + limit = i + } + return path +} + // hasGoFiles reports whether dir contains any files with names ending in .go. // For a vendor check we must exclude directories that contain no .go files. // Otherwise it is not possible to vendor just a/b/c and still import the @@ -1087,7 +1253,7 @@ func (p *Package) load(stk *ImportStack, bp *build.Package, err error) { if path == "C" { continue } - p1 := LoadImport(path, p.Dir, p, stk, p.Internal.Build.ImportPos[path], UseVendor) + p1 := LoadImport(path, p.Dir, p, stk, p.Internal.Build.ImportPos[path], ResolveImport) if p.Standard && p.Error == nil && !p1.Standard && p1.Error == nil { p.Error = &PackageError{ ImportStack: stk.Copy(), @@ -1212,6 +1378,7 @@ func (p *Package) load(stk *ImportStack, bp *build.Package, err error) { // GNU binutils flagfile specifiers, sometimes called "response files"). // To be conservative, we reject almost any arg beginning with non-alphanumeric ASCII. // We accept leading . _ and / as likely in file system paths. +// There is a copy of this function in cmd/compile/internal/gc/noder.go. func SafeArg(name string) bool { if name == "" { return false @@ -1597,7 +1764,7 @@ func TestPackagesFor(p *Package, forceTest bool) (ptest, pxtest *Package, err er stk.Push(p.ImportPath + " (test)") rawTestImports := str.StringList(p.TestImports) for i, path := range p.TestImports { - p1 := LoadImport(path, p.Dir, p, &stk, p.Internal.Build.TestImportPos[path], UseVendor) + p1 := LoadImport(path, p.Dir, p, &stk, p.Internal.Build.TestImportPos[path], ResolveImport) if p1.Error != nil { return nil, nil, p1.Error } @@ -1625,7 +1792,7 @@ func TestPackagesFor(p *Package, forceTest bool) (ptest, pxtest *Package, err er pxtestNeedsPtest := false rawXTestImports := str.StringList(p.XTestImports) for i, path := range p.XTestImports { - p1 := LoadImport(path, p.Dir, p, &stk, p.Internal.Build.XTestImportPos[path], UseVendor) + p1 := LoadImport(path, p.Dir, p, &stk, p.Internal.Build.XTestImportPos[path], ResolveImport) if p1.Error != nil { return nil, nil, p1.Error } diff --git a/src/cmd/go/internal/test/test.go b/src/cmd/go/internal/test/test.go index a99c6a5ec2..756804049d 100644 --- a/src/cmd/go/internal/test/test.go +++ b/src/cmd/go/internal/test/test.go @@ -606,10 +606,10 @@ func runTest(cmd *base.Command, args []string) { for _, path := range p.Imports { deps[path] = true } - for _, path := range p.Vendored(p.TestImports) { + for _, path := range p.Resolve(p.TestImports) { deps[path] = true } - for _, path := range p.Vendored(p.XTestImports) { + for _, path := range p.Resolve(p.XTestImports) { deps[path] = true } } @@ -668,6 +668,14 @@ func runTest(cmd *base.Command, args []string) { continue } + // If using the race detector, silently ignore + // attempts to run coverage on the runtime + // packages. It will cause the race detector + // to be invoked before it has been initialized. + if cfg.BuildRace && p.Standard && (p.ImportPath == "runtime" || strings.HasPrefix(p.ImportPath, "runtime/internal")) { + continue + } + if haveMatch { testCoverPkgs = append(testCoverPkgs, p) } diff --git a/src/cmd/go/internal/vet/vet.go b/src/cmd/go/internal/vet/vet.go index 3d095d4508..c792a243bf 100644 --- a/src/cmd/go/internal/vet/vet.go +++ b/src/cmd/go/internal/vet/vet.go @@ -62,11 +62,11 @@ func runVet(cmd *base.Command, args []string) { base.Errorf("%v", err) continue } - if len(ptest.GoFiles) == 0 && pxtest == nil { + if len(ptest.GoFiles) == 0 && len(ptest.CgoFiles) == 0 && pxtest == nil { base.Errorf("go vet %s: no Go files in %s", p.ImportPath, p.Dir) continue } - if len(ptest.GoFiles) > 0 { + if len(ptest.GoFiles) > 0 || len(ptest.CgoFiles) > 0 { root.Deps = append(root.Deps, b.VetAction(work.ModeBuild, work.ModeBuild, ptest)) } if pxtest != nil { diff --git a/src/cmd/go/internal/work/build.go b/src/cmd/go/internal/work/build.go index 57b7b00879..25dfe58d4b 100644 --- a/src/cmd/go/internal/work/build.go +++ b/src/cmd/go/internal/work/build.go @@ -229,7 +229,6 @@ func AddBuildFlags(cmd *base.Command) { // Undocumented, unstable debugging flags. cmd.Flag.StringVar(&cfg.DebugActiongraph, "debug-actiongraph", "", "") - cmd.Flag.Var(&load.DebugDeprecatedImportcfg, "debug-deprecated-importcfg", "") } // fileExtSplit expects a filename and returns the name diff --git a/src/cmd/go/internal/work/buildid.go b/src/cmd/go/internal/work/buildid.go index 39ca20ee4f..bf63b8f472 100644 --- a/src/cmd/go/internal/work/buildid.go +++ b/src/cmd/go/internal/work/buildid.go @@ -397,15 +397,7 @@ func (b *Builder) useCache(a *Action, p *load.Package, actionHash cache.ActionID // If so, it's up to date and we can reuse it instead of rebuilding it. var buildID string if target != "" && !cfg.BuildA { - var err error - buildID, err = buildid.ReadFile(target) - if err != nil && b.ComputeStaleOnly { - if p != nil && !p.Stale { - p.Stale = true - p.StaleReason = "target missing" - } - return true - } + buildID, _ = buildid.ReadFile(target) if strings.HasPrefix(buildID, actionID+buildIDSeparator) { a.buildID = buildID a.built = target @@ -482,7 +474,10 @@ func (b *Builder) useCache(a *Action, p *load.Package, actionHash cache.ActionID } } } - return true + + // Fall through to update a.buildID from the build artifact cache, + // which will affect the computation of buildIDs for targets + // higher up in the dependency graph. } // Check the build artifact cache. @@ -510,6 +505,10 @@ func (b *Builder) useCache(a *Action, p *load.Package, actionHash cache.ActionID a.built = file a.Target = "DO NOT USE - using cache" a.buildID = buildID + if p := a.Package; p != nil { + // Clearer than explaining that something else is stale. + p.StaleReason = "not installed but available in build cache" + } return true } } @@ -520,6 +519,10 @@ func (b *Builder) useCache(a *Action, p *load.Package, actionHash cache.ActionID a.output = []byte{} } + if b.ComputeStaleOnly { + return true + } + return false } diff --git a/src/cmd/go/internal/work/exec.go b/src/cmd/go/internal/work/exec.go index a5ab75f6a8..a50c996041 100644 --- a/src/cmd/go/internal/work/exec.go +++ b/src/cmd/go/internal/work/exec.go @@ -934,16 +934,29 @@ func splitPkgConfigOutput(out []byte) []string { // Calls pkg-config if needed and returns the cflags/ldflags needed to build the package. func (b *Builder) getPkgConfigFlags(p *load.Package) (cflags, ldflags []string, err error) { - if pkgs := p.CgoPkgConfig; len(pkgs) > 0 { + if pcargs := p.CgoPkgConfig; len(pcargs) > 0 { + // pkg-config permits arguments to appear anywhere in + // the command line. Move them all to the front, before --. + var pcflags []string + var pkgs []string + for _, pcarg := range pcargs { + if pcarg == "--" { + // We're going to add our own "--" argument. + } else if strings.HasPrefix(pcarg, "--") { + pcflags = append(pcflags, pcarg) + } else { + pkgs = append(pkgs, pcarg) + } + } for _, pkg := range pkgs { if !load.SafeArg(pkg) { return nil, nil, fmt.Errorf("invalid pkg-config package name: %s", pkg) } } var out []byte - out, err = b.runOut(p.Dir, p.ImportPath, nil, b.PkgconfigCmd(), "--cflags", "--", pkgs) + out, err = b.runOut(p.Dir, p.ImportPath, nil, b.PkgconfigCmd(), "--cflags", pcflags, "--", pkgs) if err != nil { - b.showOutput(nil, p.Dir, b.PkgconfigCmd()+" --cflags "+strings.Join(pkgs, " "), string(out)) + b.showOutput(nil, p.Dir, b.PkgconfigCmd()+" --cflags "+strings.Join(pcflags, " ")+strings.Join(pkgs, " "), string(out)) b.Print(err.Error() + "\n") return nil, nil, errPrintedOutput } @@ -953,15 +966,15 @@ func (b *Builder) getPkgConfigFlags(p *load.Package) (cflags, ldflags []string, return nil, nil, err } } - out, err = b.runOut(p.Dir, p.ImportPath, nil, b.PkgconfigCmd(), "--libs", "--", pkgs) + out, err = b.runOut(p.Dir, p.ImportPath, nil, b.PkgconfigCmd(), "--libs", pcflags, "--", pkgs) if err != nil { - b.showOutput(nil, p.Dir, b.PkgconfigCmd()+" --libs "+strings.Join(pkgs, " "), string(out)) + b.showOutput(nil, p.Dir, b.PkgconfigCmd()+" --libs "+strings.Join(pcflags, " ")+strings.Join(pkgs, " "), string(out)) b.Print(err.Error() + "\n") return nil, nil, errPrintedOutput } if len(out) > 0 { ldflags = strings.Fields(string(out)) - if err := checkLinkerFlags("CFLAGS", "pkg-config --cflags", ldflags); err != nil { + if err := checkLinkerFlags("LDFLAGS", "pkg-config --libs", ldflags); err != nil { return nil, nil, err } } @@ -1080,7 +1093,7 @@ func BuildInstallFunc(b *Builder, a *Action) (err error) { // We want to hide that awful detail as much as possible, so don't // advertise it by touching the mtimes (usually the libraries are up // to date). - if !a.buggyInstall { + if !a.buggyInstall && !b.ComputeStaleOnly { now := time.Now() os.Chtimes(a.Target, now, now) } diff --git a/src/cmd/go/internal/work/security.go b/src/cmd/go/internal/work/security.go index fee5beeb15..1b82af9c97 100644 --- a/src/cmd/go/internal/work/security.go +++ b/src/cmd/go/internal/work/security.go @@ -34,38 +34,95 @@ import ( "fmt" "os" "regexp" + "strings" ) var re = regexp.MustCompile var validCompilerFlags = []*regexp.Regexp{ re(`-D([A-Za-z_].*)`), + re(`-F([^@\-].*)`), re(`-I([^@\-].*)`), re(`-O`), re(`-O([^@\-].*)`), re(`-W`), re(`-W([^@,]+)`), // -Wall but not -Wa,-foo. + re(`-Wa,-mbig-obj`), + re(`-Wp,-D([A-Za-z_].*)`), + re(`-ansi`), + re(`-f(no-)?asynchronous-unwind-tables`), + re(`-f(no-)?blocks`), + re(`-f(no-)builtin-[a-zA-Z0-9_]*`), + re(`-f(no-)?common`), + re(`-f(no-)?constant-cfstrings`), + re(`-fdiagnostics-show-note-include-stack`), + re(`-f(no-)?eliminate-unused-debug-types`), + re(`-f(no-)?exceptions`), + re(`-f(no-)?fast-math`), + re(`-f(no-)?inline-functions`), + re(`-finput-charset=([^@\-].*)`), + re(`-f(no-)?fat-lto-objects`), + re(`-f(no-)?keep-inline-dllexport`), + re(`-f(no-)?lto`), + re(`-fmacro-backtrace-limit=(.+)`), + re(`-fmessage-length=(.+)`), + re(`-f(no-)?modules`), re(`-f(no-)?objc-arc`), + re(`-f(no-)?objc-nonfragile-abi`), + re(`-f(no-)?objc-legacy-dispatch`), re(`-f(no-)?omit-frame-pointer`), + re(`-f(no-)?openmp(-simd)?`), + re(`-f(no-)?permissive`), re(`-f(no-)?(pic|PIC|pie|PIE)`), + re(`-f(no-)?plt`), + re(`-f(no-)?rtti`), re(`-f(no-)?split-stack`), re(`-f(no-)?stack-(.+)`), re(`-f(no-)?strict-aliasing`), + re(`-f(un)signed-char`), + re(`-f(no-)?use-linker-plugin`), // safe if -B is not used; we don't permit -B + re(`-f(no-)?visibility-inlines-hidden`), re(`-fsanitize=(.+)`), + re(`-ftemplate-depth-(.+)`), + re(`-fvisibility=(.+)`), re(`-g([^@\-].*)?`), - re(`-m(arch|cpu|fpu|tune)=([^@\-].*)`), + re(`-m32`), + re(`-m64`), + re(`-m(abi|arch|cpu|fpu|tune)=([^@\-].*)`), + re(`-marm`), + re(`-mfloat-abi=([^@\-].*)`), + re(`-mfpmath=[0-9a-z,+]*`), + re(`-m(no-)?avx[0-9a-z.]*`), + re(`-m(no-)?ms-bitfields`), re(`-m(no-)?stack-(.+)`), re(`-mmacosx-(.+)`), + re(`-mios-simulator-version-min=(.+)`), + re(`-miphoneos-version-min=(.+)`), re(`-mnop-fun-dllimport`), + re(`-m(no-)?sse[0-9.]*`), + re(`-mthumb(-interwork)?`), + re(`-mthreads`), + re(`-mwindows`), + re(`--param=ssp-buffer-size=[0-9]*`), + re(`-pedantic(-errors)?`), + re(`-pipe`), re(`-pthread`), - re(`-std=([^@\-].*)`), + re(`-?-std=([^@\-].*)`), + re(`-?-stdlib=([^@\-].*)`), + re(`--sysroot=([^@\-].*)`), + re(`-w`), re(`-x([^@\-].*)`), } var validCompilerFlagsWithNextArg = []string{ + "-arch", "-D", "-I", "-framework", + "-isysroot", + "-isystem", + "--sysroot", + "-target", "-x", } @@ -73,29 +130,76 @@ var validLinkerFlags = []*regexp.Regexp{ re(`-F([^@\-].*)`), re(`-l([^@\-].*)`), re(`-L([^@\-].*)`), + re(`-O`), + re(`-O([^@\-].*)`), re(`-f(no-)?(pic|PIC|pie|PIE)`), + re(`-f(no-)?openmp(-simd)?`), re(`-fsanitize=([^@\-].*)`), re(`-g([^@\-].*)?`), - re(`-m(arch|cpu|fpu|tune)=([^@\-].*)`), + re(`-headerpad_max_install_names`), + re(`-m(abi|arch|cpu|fpu|tune)=([^@\-].*)`), + re(`-mfloat-abi=([^@\-].*)`), + re(`-mmacosx-(.+)`), + re(`-mios-simulator-version-min=(.+)`), + re(`-miphoneos-version-min=(.+)`), + re(`-mthreads`), + re(`-mwindows`), re(`-(pic|PIC|pie|PIE)`), re(`-pthread`), + re(`-rdynamic`), + re(`-shared`), + re(`-?-static([-a-z0-9+]*)`), + re(`-?-stdlib=([^@\-].*)`), // Note that any wildcards in -Wl need to exclude comma, // since -Wl splits its argument at commas and passes // them all to the linker uninterpreted. Allowing comma // in a wildcard would allow tunnelling arbitrary additional // linker arguments through one of these. - re(`-Wl,-rpath,([^,@\-][^,]+)`), + re(`-Wl,--(no-)?allow-multiple-definition`), + re(`-Wl,--(no-)?allow-shlib-undefined`), + re(`-Wl,--(no-)?as-needed`), + re(`-Wl,-Bdynamic`), + re(`-Wl,-Bstatic`), + re(`-WL,-O([^@,\-][^,]*)?`), + re(`-Wl,-d[ny]`), + re(`-Wl,--disable-new-dtags`), + re(`-Wl,-e[=,][a-zA-Z0-9]*`), + re(`-Wl,--enable-new-dtags`), + re(`-Wl,--end-group`), + re(`-Wl,-framework,[^,@\-][^,]+`), + re(`-Wl,-headerpad_max_install_names`), + re(`-Wl,--no-undefined`), + re(`-Wl,-rpath(-link)?[=,]([^,@\-][^,]+)`), + re(`-Wl,-s`), + re(`-Wl,-search_paths_first`), + re(`-Wl,-sectcreate,([^,@\-][^,]+),([^,@\-][^,]+),([^,@\-][^,]+)`), + re(`-Wl,--start-group`), + re(`-Wl,-?-static`), + re(`-Wl,-?-subsystem,(native|windows|console|posix|xbox)`), + re(`-Wl,-syslibroot[=,]([^,@\-][^,]+)`), + re(`-Wl,-undefined[=,]([^,@\-][^,]+)`), + re(`-Wl,-?-unresolved-symbols=[^,]+`), re(`-Wl,--(no-)?warn-([^,]+)`), + re(`-Wl,-z,(no)?execstack`), + re(`-Wl,-z,relro`), - re(`[a-zA-Z0-9_].*\.(o|obj|dll|dylib|so)`), // direct linker inputs: x.o or libfoo.so (but not -foo.o or @foo.o) + re(`[a-zA-Z0-9_/].*\.(a|o|obj|dll|dylib|so)`), // direct linker inputs: x.o or libfoo.so (but not -foo.o or @foo.o) + re(`\./.*\.(a|o|obj|dll|dylib|so)`), } var validLinkerFlagsWithNextArg = []string{ + "-arch", "-F", "-l", "-L", "-framework", + "-isysroot", + "--sysroot", + "-target", + "-Wl,-framework", + "-Wl,-rpath", + "-Wl,-undefined", } func checkCompilerFlags(name, source string, list []string) error { @@ -147,10 +251,21 @@ Args: i++ continue Args } + + // Permit -Wl,-framework -Wl,name. + if i+1 < len(list) && + strings.HasPrefix(arg, "-Wl,") && + strings.HasPrefix(list[i+1], "-Wl,") && + load.SafeArg(list[i+1][4:]) && + !strings.Contains(list[i+1][4:], ",") { + i++ + continue Args + } + if i+1 < len(list) { - return fmt.Errorf("invalid flag in %s: %s %s", source, arg, list[i+1]) + return fmt.Errorf("invalid flag in %s: %s %s (see https://golang.org/s/invalidflag)", source, arg, list[i+1]) } - return fmt.Errorf("invalid flag in %s: %s without argument", source, arg) + return fmt.Errorf("invalid flag in %s: %s without argument (see https://golang.org/s/invalidflag)", source, arg) } } Bad: diff --git a/src/cmd/go/internal/work/security_test.go b/src/cmd/go/internal/work/security_test.go index 739ab5a6ee..c3a61b8e70 100644 --- a/src/cmd/go/internal/work/security_test.go +++ b/src/cmd/go/internal/work/security_test.go @@ -12,6 +12,7 @@ import ( var goodCompilerFlags = [][]string{ {"-DFOO"}, {"-Dfoo=bar"}, + {"-F/Qt"}, {"-I/"}, {"-I/etc/passwd"}, {"-I."}, @@ -62,6 +63,8 @@ var goodCompilerFlags = [][]string{ var badCompilerFlags = [][]string{ {"-D@X"}, {"-D-X"}, + {"-F@dir"}, + {"-F-dir"}, {"-I@dir"}, {"-I-dir"}, {"-O@1"}, @@ -125,6 +128,7 @@ var goodLinkerFlags = [][]string{ {"-Wl,--no-warn-error"}, {"foo.so"}, {"_世界.dll"}, + {"./x.o"}, {"libcgosotest.dylib"}, {"-F", "framework"}, {"-l", "."}, @@ -132,14 +136,14 @@ var goodLinkerFlags = [][]string{ {"-l", "世界"}, {"-L", "framework"}, {"-framework", "Chocolate"}, + {"-Wl,-framework", "-Wl,Chocolate"}, + {"-Wl,-framework,Chocolate"}, + {"-Wl,-unresolved-symbols=ignore-all"}, } var badLinkerFlags = [][]string{ {"-DFOO"}, {"-Dfoo=bar"}, - {"-O"}, - {"-O2"}, - {"-Osmall"}, {"-W"}, {"-Wall"}, {"-fobjc-arc"}, @@ -152,7 +156,6 @@ var badLinkerFlags = [][]string{ {"-fno-stack-xxx"}, {"-mstack-overflow"}, {"-mno-stack-overflow"}, - {"-mmacosx-version"}, {"-mnop-fun-dllimport"}, {"-std=c99"}, {"-xc"}, @@ -185,9 +188,14 @@ var badLinkerFlags = [][]string{ {"-l", "-foo"}, {"-framework", "-Caffeine"}, {"-framework", "@Home"}, + {"-Wl,-framework,-Caffeine"}, + {"-Wl,-framework", "-Wl,@Home"}, + {"-Wl,-framework", "@Home"}, + {"-Wl,-framework,Chocolate,@Home"}, {"-x", "--c"}, {"-x", "@obj"}, {"-Wl,-rpath,@foo"}, + {"../x.o"}, } func TestCheckLinkerFlags(t *testing.T) { diff --git a/src/cmd/go/testdata/modlegacy/src/new/go.mod b/src/cmd/go/testdata/modlegacy/src/new/go.mod new file mode 100644 index 0000000000..d0dd46d314 --- /dev/null +++ b/src/cmd/go/testdata/modlegacy/src/new/go.mod @@ -0,0 +1 @@ +module "new/v2" diff --git a/src/cmd/go/testdata/modlegacy/src/new/new.go b/src/cmd/go/testdata/modlegacy/src/new/new.go new file mode 100644 index 0000000000..e99c47a6a8 --- /dev/null +++ b/src/cmd/go/testdata/modlegacy/src/new/new.go @@ -0,0 +1,3 @@ +package new + +import _ "new/v2/p2" diff --git a/src/cmd/go/testdata/modlegacy/src/new/p1/p1.go b/src/cmd/go/testdata/modlegacy/src/new/p1/p1.go new file mode 100644 index 0000000000..4539f40919 --- /dev/null +++ b/src/cmd/go/testdata/modlegacy/src/new/p1/p1.go @@ -0,0 +1,7 @@ +package p1 + +import _ "old/p2" +import _ "new/v2" +import _ "new/v2/p2" +import _ "new/sub/v2/x/v1/y" // v2 is module, v1 is directory in module +import _ "new/sub/inner/x" // new/sub/inner/go.mod overrides new/sub/go.mod diff --git a/src/cmd/go/testdata/modlegacy/src/new/p2/p2.go b/src/cmd/go/testdata/modlegacy/src/new/p2/p2.go new file mode 100644 index 0000000000..9b9052f541 --- /dev/null +++ b/src/cmd/go/testdata/modlegacy/src/new/p2/p2.go @@ -0,0 +1 @@ +package p2 diff --git a/src/cmd/go/testdata/modlegacy/src/new/sub/go.mod b/src/cmd/go/testdata/modlegacy/src/new/sub/go.mod new file mode 100644 index 0000000000..484d20c6b2 --- /dev/null +++ b/src/cmd/go/testdata/modlegacy/src/new/sub/go.mod @@ -0,0 +1 @@ +module new/sub/v2 diff --git a/src/cmd/go/testdata/modlegacy/src/new/sub/inner/go.mod b/src/cmd/go/testdata/modlegacy/src/new/sub/inner/go.mod new file mode 100644 index 0000000000..ba3934541f --- /dev/null +++ b/src/cmd/go/testdata/modlegacy/src/new/sub/inner/go.mod @@ -0,0 +1 @@ +module new/sub/inner diff --git a/src/cmd/go/testdata/modlegacy/src/new/sub/inner/x/x.go b/src/cmd/go/testdata/modlegacy/src/new/sub/inner/x/x.go new file mode 100644 index 0000000000..823aafd071 --- /dev/null +++ b/src/cmd/go/testdata/modlegacy/src/new/sub/inner/x/x.go @@ -0,0 +1 @@ +package x diff --git a/src/cmd/go/testdata/modlegacy/src/new/sub/x/v1/y/y.go b/src/cmd/go/testdata/modlegacy/src/new/sub/x/v1/y/y.go new file mode 100644 index 0000000000..789ca715ec --- /dev/null +++ b/src/cmd/go/testdata/modlegacy/src/new/sub/x/v1/y/y.go @@ -0,0 +1 @@ +package y diff --git a/src/cmd/go/testdata/modlegacy/src/old/p1/p1.go b/src/cmd/go/testdata/modlegacy/src/old/p1/p1.go new file mode 100644 index 0000000000..90527483ab --- /dev/null +++ b/src/cmd/go/testdata/modlegacy/src/old/p1/p1.go @@ -0,0 +1,5 @@ +package p1 + +import _ "old/p2" +import _ "new/p1" +import _ "new" diff --git a/src/cmd/go/testdata/modlegacy/src/old/p2/p2.go b/src/cmd/go/testdata/modlegacy/src/old/p2/p2.go new file mode 100644 index 0000000000..9b9052f541 --- /dev/null +++ b/src/cmd/go/testdata/modlegacy/src/old/p2/p2.go @@ -0,0 +1 @@ +package p2 diff --git a/src/cmd/go/vendor_test.go b/src/cmd/go/vendor_test.go index 4f21a510a3..0e7a633240 100644 --- a/src/cmd/go/vendor_test.go +++ b/src/cmd/go/vendor_test.go @@ -10,6 +10,7 @@ import ( "bytes" "fmt" "internal/testenv" + "os" "path/filepath" "regexp" "strings" @@ -328,3 +329,75 @@ func TestVendor12156(t *testing.T) { tg.grepStderrNot("panic", "panicked") tg.grepStderr(`cannot find package "x"`, "wrong error") } + +// Module legacy support does path rewriting very similar to vendoring. + +func TestModLegacy(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + tg.setenv("GOPATH", filepath.Join(tg.pwd(), "testdata/modlegacy")) + tg.run("list", "-f", "{{.Imports}}", "old/p1") + tg.grepStdout("new/p1", "old/p1 should import new/p1") + tg.run("list", "-f", "{{.Imports}}", "new/p1") + tg.grepStdout("new/p2", "new/p1 should import new/p2 (not new/v2/p2)") + tg.grepStdoutNot("new/v2", "new/p1 should NOT import new/v2*") + tg.grepStdout("new/sub/x/v1/y", "new/p1 should import new/sub/x/v1/y (not new/sub/v2/x/v1/y)") + tg.grepStdoutNot("new/sub/v2", "new/p1 should NOT import new/sub/v2*") + tg.grepStdout("new/sub/inner/x", "new/p1 should import new/sub/inner/x (no rewrites)") + tg.run("build", "old/p1", "new/p1") +} + +func TestModLegacyGet(t *testing.T) { + testenv.MustHaveExternalNetwork(t) + + tg := testgo(t) + defer tg.cleanup() + tg.makeTempdir() + tg.setenv("GOPATH", tg.path("d1")) + tg.run("get", "vcs-test.golang.org/git/modlegacy1-old.git/p1") + tg.run("list", "-f", "{{.Deps}}", "vcs-test.golang.org/git/modlegacy1-old.git/p1") + tg.grepStdout("new.git/p2", "old/p1 should depend on new/p2") + tg.grepStdoutNot("new.git/v2/p2", "old/p1 should NOT depend on new/v2/p2") + tg.run("build", "vcs-test.golang.org/git/modlegacy1-old.git/p1", "vcs-test.golang.org/git/modlegacy1-new.git/p1") + + tg.setenv("GOPATH", tg.path("d2")) + + tg.must(os.RemoveAll(tg.path("d2"))) + tg.run("get", "github.com/rsc/vgotest5") + tg.run("get", "github.com/rsc/vgotest4") + tg.run("get", "github.com/myitcv/vgo_example_compat") + + if testing.Short() { + return + } + + tg.must(os.RemoveAll(tg.path("d2"))) + tg.run("get", "github.com/rsc/vgotest4") + tg.run("get", "github.com/rsc/vgotest5") + tg.run("get", "github.com/myitcv/vgo_example_compat") + + tg.must(os.RemoveAll(tg.path("d2"))) + tg.run("get", "github.com/rsc/vgotest4", "github.com/rsc/vgotest5") + tg.run("get", "github.com/myitcv/vgo_example_compat") + + tg.must(os.RemoveAll(tg.path("d2"))) + tg.run("get", "github.com/rsc/vgotest5", "github.com/rsc/vgotest4") + tg.run("get", "github.com/myitcv/vgo_example_compat") + + tg.must(os.RemoveAll(tg.path("d2"))) + tg.run("get", "github.com/myitcv/vgo_example_compat") + tg.run("get", "github.com/rsc/vgotest4", "github.com/rsc/vgotest5") + + pkgs := []string{"github.com/myitcv/vgo_example_compat", "github.com/rsc/vgotest4", "github.com/rsc/vgotest5"} + for i := 0; i < 3; i++ { + for j := 0; j < 3; j++ { + for k := 0; k < 3; k++ { + if i == j || i == k || k == j { + continue + } + tg.must(os.RemoveAll(tg.path("d2"))) + tg.run("get", pkgs[i], pkgs[j], pkgs[k]) + } + } + } +} diff --git a/src/cmd/internal/obj/mips/obj0.go b/src/cmd/internal/obj/mips/obj0.go index ae9d1282a4..9f689929d2 100644 --- a/src/cmd/internal/obj/mips/obj0.go +++ b/src/cmd/internal/obj/mips/obj0.go @@ -531,20 +531,22 @@ func preprocess(ctxt *obj.Link, cursym *obj.LSym, newprog obj.ProgAlloc) { p.Link = q p1 = q.Link - var regOff int16 + var addrOff int64 if c.ctxt.Arch.ByteOrder == binary.BigEndian { - regOff = 1 // load odd register first + addrOff = 4 // swap load/save order } if p.From.Type == obj.TYPE_MEM { reg := REG_F0 + (p.To.Reg-REG_F0)&^1 - p.To.Reg = reg + regOff - q.To.Reg = reg + 1 - regOff - q.From.Offset += 4 + p.To.Reg = reg + q.To.Reg = reg + 1 + p.From.Offset += addrOff + q.From.Offset += 4 - addrOff } else if p.To.Type == obj.TYPE_MEM { reg := REG_F0 + (p.From.Reg-REG_F0)&^1 - p.From.Reg = reg + regOff - q.From.Reg = reg + 1 - regOff - q.To.Offset += 4 + p.From.Reg = reg + q.From.Reg = reg + 1 + p.To.Offset += addrOff + q.To.Offset += 4 - addrOff } } } diff --git a/src/cmd/internal/objabi/funcid.go b/src/cmd/internal/objabi/funcid.go new file mode 100644 index 0000000000..55f1328ba8 --- /dev/null +++ b/src/cmd/internal/objabi/funcid.go @@ -0,0 +1,34 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package objabi + +// A FuncID identifies particular functions that need to be treated +// specially by the runtime. +// Note that in some situations involving plugins, there may be multiple +// copies of a particular special runtime function. +// Note: this list must match the list in runtime/symtab.go. +type FuncID uint32 + +const ( + FuncID_normal FuncID = iota // not a special function + FuncID_goexit + FuncID_jmpdefer + FuncID_mcall + FuncID_morestack + FuncID_mstart + FuncID_rt0_go + FuncID_asmcgocall + FuncID_sigpanic + FuncID_runfinq + FuncID_bgsweep + FuncID_forcegchelper + FuncID_timerproc + FuncID_gcBgMarkWorker + FuncID_systemstack_switch + FuncID_systemstack + FuncID_cgocallback_gofunc + FuncID_gogo + FuncID_externalthreadhandler +) diff --git a/src/cmd/internal/test2json/test2json.go b/src/cmd/internal/test2json/test2json.go index 483fb1de52..f8052136be 100644 --- a/src/cmd/internal/test2json/test2json.go +++ b/src/cmd/internal/test2json/test2json.go @@ -175,6 +175,7 @@ func (c *converter) handleInputLine(line []byte) { // "=== RUN " // "=== PAUSE " // "=== CONT " + actionColon := false origLine := line ok := false indent := 0 @@ -196,6 +197,7 @@ func (c *converter) handleInputLine(line []byte) { } for _, magic := range reports { if bytes.HasPrefix(line, magic) { + actionColon = true ok = true break } @@ -209,7 +211,10 @@ func (c *converter) handleInputLine(line []byte) { } // Parse out action and test name. - i := bytes.IndexByte(line, ':') + 1 + i := 0 + if actionColon { + i = bytes.IndexByte(line, ':') + 1 + } if i == 0 { i = len(updates[0]) } diff --git a/src/cmd/internal/test2json/testdata/issue23920.json b/src/cmd/internal/test2json/testdata/issue23920.json new file mode 100644 index 0000000000..28f7bd56ac --- /dev/null +++ b/src/cmd/internal/test2json/testdata/issue23920.json @@ -0,0 +1,14 @@ +{"Action":"run","Test":"TestWithColons"} +{"Action":"output","Test":"TestWithColons","Output":"=== RUN TestWithColons\n"} +{"Action":"run","Test":"TestWithColons/[::1]"} +{"Action":"output","Test":"TestWithColons/[::1]","Output":"=== RUN TestWithColons/[::1]\n"} +{"Action":"run","Test":"TestWithColons/127.0.0.1:0"} +{"Action":"output","Test":"TestWithColons/127.0.0.1:0","Output":"=== RUN TestWithColons/127.0.0.1:0\n"} +{"Action":"output","Test":"TestWithColons","Output":"--- PASS: TestWithColons (0.00s)\n"} +{"Action":"output","Test":"TestWithColons/[::1]","Output":" --- PASS: TestWithColons/[::1] (0.00s)\n"} +{"Action":"pass","Test":"TestWithColons/[::1]"} +{"Action":"output","Test":"TestWithColons/127.0.0.1:0","Output":" --- PASS: TestWithColons/127.0.0.1:0 (0.00s)\n"} +{"Action":"pass","Test":"TestWithColons/127.0.0.1:0"} +{"Action":"pass","Test":"TestWithColons"} +{"Action":"output","Output":"PASS\n"} +{"Action":"pass"} diff --git a/src/cmd/internal/test2json/testdata/issue23920.test b/src/cmd/internal/test2json/testdata/issue23920.test new file mode 100644 index 0000000000..43bf058034 --- /dev/null +++ b/src/cmd/internal/test2json/testdata/issue23920.test @@ -0,0 +1,7 @@ +=== RUN TestWithColons +=== RUN TestWithColons/[::1] +=== RUN TestWithColons/127.0.0.1:0 +--- PASS: TestWithColons (0.00s) + --- PASS: TestWithColons/[::1] (0.00s) + --- PASS: TestWithColons/127.0.0.1:0 (0.00s) +PASS diff --git a/src/cmd/link/internal/ld/lib.go b/src/cmd/link/internal/ld/lib.go index f866a04936..abaeb15d2e 100644 --- a/src/cmd/link/internal/ld/lib.go +++ b/src/cmd/link/internal/ld/lib.go @@ -1266,7 +1266,7 @@ func (ctxt *Link) hostlink() { // does not work, the resulting programs will not run. See // issue #17847. To avoid this problem pass -no-pie to the // toolchain if it is supported. - if ctxt.BuildMode == BuildModeExe { + if ctxt.BuildMode == BuildModeExe && !ctxt.linkShared { src := filepath.Join(*flagTmpdir, "trivial.c") if err := ioutil.WriteFile(src, []byte("int main() { return 0; }"), 0666); err != nil { Errorf(nil, "WriteFile trivial.c failed: %v", err) diff --git a/src/cmd/link/internal/ld/pcln.go b/src/cmd/link/internal/ld/pcln.go index 9d82677059..8708924d92 100644 --- a/src/cmd/link/internal/ld/pcln.go +++ b/src/cmd/link/internal/ld/pcln.go @@ -312,12 +312,47 @@ func (ctxt *Link) pclntab() { } off = int32(ftab.SetUint32(ctxt.Arch, int64(off), args)) - // frame int32 - // This has been removed (it was never set quite correctly anyway). - // Nothing should use it. - // Leave an obviously incorrect value. - // TODO: Remove entirely. - off = int32(ftab.SetUint32(ctxt.Arch, int64(off), 0x1234567)) + // funcID uint32 + funcID := objabi.FuncID_normal + switch s.Name { + case "runtime.goexit": + funcID = objabi.FuncID_goexit + case "runtime.jmpdefer": + funcID = objabi.FuncID_jmpdefer + case "runtime.mcall": + funcID = objabi.FuncID_mcall + case "runtime.morestack": + funcID = objabi.FuncID_morestack + case "runtime.mstart": + funcID = objabi.FuncID_mstart + case "runtime.rt0_go": + funcID = objabi.FuncID_rt0_go + case "runtime.asmcgocall": + funcID = objabi.FuncID_asmcgocall + case "runtime.sigpanic": + funcID = objabi.FuncID_sigpanic + case "runtime.runfinq": + funcID = objabi.FuncID_runfinq + case "runtime.bgsweep": + funcID = objabi.FuncID_bgsweep + case "runtime.forcegchelper": + funcID = objabi.FuncID_forcegchelper + case "runtime.timerproc": + funcID = objabi.FuncID_timerproc + case "runtime.gcBgMarkWorker": + funcID = objabi.FuncID_gcBgMarkWorker + case "runtime.systemstack_switch": + funcID = objabi.FuncID_systemstack_switch + case "runtime.systemstack": + funcID = objabi.FuncID_systemstack + case "runtime.cgocallback_gofunc": + funcID = objabi.FuncID_cgocallback_gofunc + case "runtime.gogo": + funcID = objabi.FuncID_gogo + case "runtime.externalthreadhandler": + funcID = objabi.FuncID_externalthreadhandler + } + off = int32(ftab.SetUint32(ctxt.Arch, int64(off), uint32(funcID))) if pcln != &pclntabZpcln { renumberfiles(ctxt, pcln.File, &pcln.Pcfile) diff --git a/src/crypto/x509/name_constraints_test.go b/src/crypto/x509/name_constraints_test.go index 10cc348357..e89e70d874 100644 --- a/src/crypto/x509/name_constraints_test.go +++ b/src/crypto/x509/name_constraints_test.go @@ -11,6 +11,7 @@ import ( "crypto/rand" "crypto/x509/pkix" "encoding/asn1" + "encoding/hex" "encoding/pem" "fmt" "io/ioutil" @@ -42,6 +43,7 @@ type nameConstraintsTest struct { roots []constraintsSpec intermediates [][]constraintsSpec leaf leafSpec + requestedEKUs []ExtKeyUsage expectedError string noOpenSSL bool } @@ -1220,8 +1222,9 @@ var nameConstraintsTests = []nameConstraintsTest{ }, }, - // #63: A specified key usage in an intermediate forbids other usages - // in the leaf. + // #63: An intermediate with enumerated EKUs causes a failure if we + // test for an EKU not in that set. (ServerAuth is required by + // default.) nameConstraintsTest{ roots: []constraintsSpec{ constraintsSpec{}, @@ -1237,11 +1240,11 @@ var nameConstraintsTests = []nameConstraintsTest{ sans: []string{"dns:example.com"}, ekus: []string{"serverAuth"}, }, - expectedError: "EKU not permitted", + expectedError: "incompatible key usage", }, - // #64: A specified key usage in an intermediate forbids other usages - // in the leaf, even if we don't recognise them. + // #64: an unknown EKU in the leaf doesn't break anything, even if it's not + // correctly nested. nameConstraintsTest{ roots: []constraintsSpec{ constraintsSpec{}, @@ -1257,7 +1260,7 @@ var nameConstraintsTests = []nameConstraintsTest{ sans: []string{"dns:example.com"}, ekus: []string{"other"}, }, - expectedError: "EKU not permitted", + requestedEKUs: []ExtKeyUsage{ExtKeyUsageAny}, }, // #65: trying to add extra permitted key usages in an intermediate @@ -1282,24 +1285,25 @@ var nameConstraintsTests = []nameConstraintsTest{ }, }, - // #66: EKUs in roots are ignored. + // #66: EKUs in roots are not ignored. nameConstraintsTest{ roots: []constraintsSpec{ constraintsSpec{ - ekus: []string{"serverAuth"}, + ekus: []string{"email"}, }, }, intermediates: [][]constraintsSpec{ []constraintsSpec{ constraintsSpec{ - ekus: []string{"serverAuth", "email"}, + ekus: []string{"serverAuth"}, }, }, }, leaf: leafSpec{ sans: []string{"dns:example.com"}, - ekus: []string{"serverAuth", "email"}, + ekus: []string{"serverAuth"}, }, + expectedError: "incompatible key usage", }, // #67: in order to support COMODO chains, SGC key usages permit @@ -1444,6 +1448,138 @@ var nameConstraintsTests = []nameConstraintsTest{ }, expectedError: "\"https://example.com/test\" is excluded", }, + + // #75: serverAuth in a leaf shouldn't permit clientAuth when requested in + // VerifyOptions. + nameConstraintsTest{ + roots: []constraintsSpec{ + constraintsSpec{}, + }, + intermediates: [][]constraintsSpec{ + []constraintsSpec{ + constraintsSpec{}, + }, + }, + leaf: leafSpec{ + sans: []string{"dns:example.com"}, + ekus: []string{"serverAuth"}, + }, + requestedEKUs: []ExtKeyUsage{ExtKeyUsageClientAuth}, + expectedError: "incompatible key usage", + }, + + // #76: However, MSSGC in a leaf should match a request for serverAuth. + nameConstraintsTest{ + roots: []constraintsSpec{ + constraintsSpec{}, + }, + intermediates: [][]constraintsSpec{ + []constraintsSpec{ + constraintsSpec{}, + }, + }, + leaf: leafSpec{ + sans: []string{"dns:example.com"}, + ekus: []string{"msSGC"}, + }, + requestedEKUs: []ExtKeyUsage{ExtKeyUsageServerAuth}, + }, + + // An invalid DNS SAN should be detected only at validation time so + // that we can process CA certificates in the wild that have invalid SANs. + // See https://github.com/golang/go/issues/23995 + + // #77: an invalid DNS or mail SAN will not be detected if name constaint + // checking is not triggered. + nameConstraintsTest{ + roots: []constraintsSpec{ + constraintsSpec{}, + }, + intermediates: [][]constraintsSpec{ + []constraintsSpec{ + constraintsSpec{}, + }, + }, + leaf: leafSpec{ + sans: []string{"dns:this is invalid", "email:this @ is invalid"}, + }, + }, + + // #78: an invalid DNS SAN will be detected if any name constraint checking + // is triggered. + nameConstraintsTest{ + roots: []constraintsSpec{ + constraintsSpec{ + bad: []string{"uri:"}, + }, + }, + intermediates: [][]constraintsSpec{ + []constraintsSpec{ + constraintsSpec{}, + }, + }, + leaf: leafSpec{ + sans: []string{"dns:this is invalid"}, + }, + expectedError: "cannot parse dnsName", + }, + + // #79: an invalid email SAN will be detected if any name constraint + // checking is triggered. + nameConstraintsTest{ + roots: []constraintsSpec{ + constraintsSpec{ + bad: []string{"uri:"}, + }, + }, + intermediates: [][]constraintsSpec{ + []constraintsSpec{ + constraintsSpec{}, + }, + }, + leaf: leafSpec{ + sans: []string{"email:this @ is invalid"}, + }, + expectedError: "cannot parse rfc822Name", + }, + + // #80: if several EKUs are requested, satisfying any of them is sufficient. + nameConstraintsTest{ + roots: []constraintsSpec{ + constraintsSpec{}, + }, + intermediates: [][]constraintsSpec{ + []constraintsSpec{ + constraintsSpec{}, + }, + }, + leaf: leafSpec{ + sans: []string{"dns:example.com"}, + ekus: []string{"email"}, + }, + requestedEKUs: []ExtKeyUsage{ExtKeyUsageClientAuth, ExtKeyUsageEmailProtection}, + }, + + // #81: EKUs that are not asserted in VerifyOpts are not required to be + // nested. + nameConstraintsTest{ + roots: []constraintsSpec{ + constraintsSpec{}, + }, + intermediates: [][]constraintsSpec{ + []constraintsSpec{ + constraintsSpec{ + ekus: []string{"serverAuth"}, + }, + }, + }, + leaf: leafSpec{ + sans: []string{"dns:example.com"}, + // There's no email EKU in the intermediate. This would be rejected if + // full nesting was required. + ekus: []string{"email", "serverAuth"}, + }, + }, } func makeConstraintsCACert(constraints constraintsSpec, name string, key *ecdsa.PrivateKey, parent *Certificate, parentKey *ecdsa.PrivateKey) (*Certificate, error) { @@ -1512,6 +1648,13 @@ func makeConstraintsLeafCert(leaf leafSpec, key *ecdsa.PrivateKey, parent *Certi } template.IPAddresses = append(template.IPAddresses, ip) + case strings.HasPrefix(name, "invalidip:"): + ipBytes, err := hex.DecodeString(name[10:]) + if err != nil { + return nil, fmt.Errorf("cannot parse invalid IP: %s", err) + } + template.IPAddresses = append(template.IPAddresses, net.IP(ipBytes)) + case strings.HasPrefix(name, "email:"): template.EmailAddresses = append(template.EmailAddresses, name[6:]) @@ -1781,6 +1924,7 @@ func TestConstraintCases(t *testing.T) { Roots: rootPool, Intermediates: intermediatePool, CurrentTime: time.Unix(1500, 0), + KeyUsages: test.requestedEKUs, } _, err = leafCert.Verify(verifyOpts) @@ -1972,12 +2116,13 @@ func TestBadNamesInConstraints(t *testing.T) { } func TestBadNamesInSANs(t *testing.T) { - // Bad names in SANs should not parse. + // Bad names in URI and IP SANs should not parse. Bad DNS and email SANs + // will parse and are tested in name constraint tests at the top of this + // file. badNames := []string{ - "dns:foo.com.", - "email:abc@foo.com.", - "email:foo.com.", "uri:https://example.com./dsf", + "invalidip:0102", + "invalidip:0102030405", } priv, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) diff --git a/src/crypto/x509/root_windows.go b/src/crypto/x509/root_windows.go index 92cc71692d..3621a93aba 100644 --- a/src/crypto/x509/root_windows.go +++ b/src/crypto/x509/root_windows.go @@ -95,6 +95,12 @@ func checkChainTrustStatus(c *Certificate, chainCtx *syscall.CertChainContext) e return nil } +type _CertChainPolicyPara struct { + Size uint32 + Flags uint32 + ExtraPolicyPara unsafe.Pointer +} + // checkChainSSLServerPolicy checks that the certificate chain in chainCtx is valid for // use as a certificate chain for a SSL/TLS server. func checkChainSSLServerPolicy(c *Certificate, chainCtx *syscall.CertChainContext, opts *VerifyOptions) error { @@ -108,13 +114,13 @@ func checkChainSSLServerPolicy(c *Certificate, chainCtx *syscall.CertChainContex } sslPara.Size = uint32(unsafe.Sizeof(*sslPara)) - para := &syscall.CertChainPolicyPara{ - ExtraPolicyPara: uintptr(unsafe.Pointer(sslPara)), + para := &_CertChainPolicyPara{ + ExtraPolicyPara: unsafe.Pointer(sslPara), } para.Size = uint32(unsafe.Sizeof(*para)) status := syscall.CertChainPolicyStatus{} - err = syscall.CertVerifyCertificateChainPolicy(syscall.CERT_CHAIN_POLICY_SSL, chainCtx, para, &status) + err = syscall.CertVerifyCertificateChainPolicy(syscall.CERT_CHAIN_POLICY_SSL, chainCtx, (*syscall.CertChainPolicyPara)(unsafe.Pointer(para)), &status) if err != nil { return err } diff --git a/src/crypto/x509/verify.go b/src/crypto/x509/verify.go index 299ead6512..1e7912555d 100644 --- a/src/crypto/x509/verify.go +++ b/src/crypto/x509/verify.go @@ -6,12 +6,14 @@ package x509 import ( "bytes" + "encoding/asn1" "errors" "fmt" "net" "net/url" "reflect" "runtime" + "strconv" "strings" "time" "unicode/utf8" @@ -54,8 +56,7 @@ const ( // CPU time to verify. TooManyConstraints // CANotAuthorizedForExtKeyUsage results when an intermediate or root - // certificate does not permit an extended key usage that is claimed by - // the leaf certificate. + // certificate does not permit a requested extended key usage. CANotAuthorizedForExtKeyUsage ) @@ -80,7 +81,7 @@ func (e CertificateInvalidError) Error() string { case TooManyIntermediates: return "x509: too many intermediates for path length constraint" case IncompatibleUsage: - return "x509: certificate specifies an incompatible key usage: " + e.Detail + return "x509: certificate specifies an incompatible key usage" case NameMismatch: return "x509: issuer name does not match subject from issuing certificate" case NameConstraintsWithoutSANs: @@ -183,10 +184,13 @@ type VerifyOptions struct { Intermediates *CertPool Roots *CertPool // if nil, the system roots are used CurrentTime time.Time // if zero, the current time is used - // KeyUsage specifies which Extended Key Usage values are acceptable. - // An empty list means ExtKeyUsageServerAuth. Key usage is considered a - // constraint down the chain which mirrors Windows CryptoAPI behavior, - // but not the spec. To accept any key usage, include ExtKeyUsageAny. + // KeyUsage specifies which Extended Key Usage values are acceptable. A leaf + // certificate is accepted if it contains any of the listed values. An empty + // list means ExtKeyUsageServerAuth. To accept any key usage, include + // ExtKeyUsageAny. + // + // Certificate chains are required to nest these extended key usage values. + // (This matches the Windows CryptoAPI behavior, but not the spec.) KeyUsages []ExtKeyUsage // MaxConstraintComparisions is the maximum number of comparisons to // perform when checking a given certificate's name constraints. If @@ -548,41 +552,6 @@ func (c *Certificate) checkNameConstraints(count *int, return nil } -// ekuPermittedBy returns true iff the given extended key usage is permitted by -// the given EKU from a certificate. Normally, this would be a simple -// comparison plus a special case for the “any” EKU. But, in order to support -// existing certificates, some exceptions are made. -func ekuPermittedBy(eku, certEKU ExtKeyUsage) bool { - if certEKU == ExtKeyUsageAny || eku == certEKU { - return true - } - - // Some exceptions are made to support existing certificates. Firstly, - // the ServerAuth and SGC EKUs are treated as a group. - mapServerAuthEKUs := func(eku ExtKeyUsage) ExtKeyUsage { - if eku == ExtKeyUsageNetscapeServerGatedCrypto || eku == ExtKeyUsageMicrosoftServerGatedCrypto { - return ExtKeyUsageServerAuth - } - return eku - } - - eku = mapServerAuthEKUs(eku) - certEKU = mapServerAuthEKUs(certEKU) - - if eku == certEKU || - // ServerAuth in a CA permits ClientAuth in the leaf. - (eku == ExtKeyUsageClientAuth && certEKU == ExtKeyUsageServerAuth) || - // Any CA may issue an OCSP responder certificate. - eku == ExtKeyUsageOCSPSigning || - // Code-signing CAs can use Microsoft's commercial and - // kernel-mode EKUs. - ((eku == ExtKeyUsageMicrosoftCommercialCodeSigning || eku == ExtKeyUsageMicrosoftKernelCodeSigning) && certEKU == ExtKeyUsageCodeSigning) { - return true - } - - return false -} - // isValid performs validity checks on c given that it is a candidate to append // to the chain in currentChain. func (c *Certificate) isValid(certType int, currentChain []*Certificate, opts *VerifyOptions) error { @@ -635,8 +604,7 @@ func (c *Certificate) isValid(certType int, currentChain []*Certificate, opts *V name := string(data) mailbox, ok := parseRFC2821Mailbox(name) if !ok { - // This certificate should not have parsed. - return errors.New("x509: internal error: rfc822Name SAN failed to parse") + return fmt.Errorf("x509: cannot parse rfc822Name %q", mailbox) } if err := c.checkNameConstraints(&comparisonCount, maxConstraintComparisons, "email address", name, mailbox, @@ -648,6 +616,10 @@ func (c *Certificate) isValid(certType int, currentChain []*Certificate, opts *V case nameTypeDNS: name := string(data) + if _, ok := domainToReverseLabels(name); !ok { + return fmt.Errorf("x509: cannot parse dnsName %q", name) + } + if err := c.checkNameConstraints(&comparisonCount, maxConstraintComparisons, "DNS name", name, name, func(parsedName, constraint interface{}) (bool, error) { return matchDomainConstraint(parsedName.(string), constraint.(string)) @@ -694,59 +666,6 @@ func (c *Certificate) isValid(certType int, currentChain []*Certificate, opts *V } } - checkEKUs := certType == intermediateCertificate - - // If no extended key usages are specified, then all are acceptable. - if checkEKUs && (len(c.ExtKeyUsage) == 0 && len(c.UnknownExtKeyUsage) == 0) { - checkEKUs = false - } - - // If the “any” key usage is permitted, then no more checks are needed. - if checkEKUs { - for _, caEKU := range c.ExtKeyUsage { - comparisonCount++ - if caEKU == ExtKeyUsageAny { - checkEKUs = false - break - } - } - } - - if checkEKUs { - NextEKU: - for _, eku := range leaf.ExtKeyUsage { - if comparisonCount > maxConstraintComparisons { - return CertificateInvalidError{c, TooManyConstraints, ""} - } - - for _, caEKU := range c.ExtKeyUsage { - comparisonCount++ - if ekuPermittedBy(eku, caEKU) { - continue NextEKU - } - } - - oid, _ := oidFromExtKeyUsage(eku) - return CertificateInvalidError{c, CANotAuthorizedForExtKeyUsage, fmt.Sprintf("EKU not permitted: %#v", oid)} - } - - NextUnknownEKU: - for _, eku := range leaf.UnknownExtKeyUsage { - if comparisonCount > maxConstraintComparisons { - return CertificateInvalidError{c, TooManyConstraints, ""} - } - - for _, caEKU := range c.UnknownExtKeyUsage { - comparisonCount++ - if caEKU.Equal(eku) { - continue NextUnknownEKU - } - } - - return CertificateInvalidError{c, CANotAuthorizedForExtKeyUsage, fmt.Sprintf("EKU not permitted: %#v", eku)} - } - } - // KeyUsage status flags are ignored. From Engineering Security, Peter // Gutmann: A European government CA marked its signing certificates as // being valid for encryption only, but no-one noticed. Another @@ -785,6 +704,18 @@ func (c *Certificate) isValid(certType int, currentChain []*Certificate, opts *V return nil } +// formatOID formats an ASN.1 OBJECT IDENTIFER in the common, dotted style. +func formatOID(oid asn1.ObjectIdentifier) string { + ret := "" + for i, v := range oid { + if i > 0 { + ret += "." + } + ret += strconv.Itoa(v) + } + return ret +} + // Verify attempts to verify c by building one or more chains from c to a // certificate in opts.Roots, using certificates in opts.Intermediates if // needed. If successful, it returns one or more chains where the first @@ -842,46 +773,38 @@ func (c *Certificate) Verify(opts VerifyOptions) (chains [][]*Certificate, err e } } - requestedKeyUsages := make([]ExtKeyUsage, len(opts.KeyUsages)) - copy(requestedKeyUsages, opts.KeyUsages) - if len(requestedKeyUsages) == 0 { - requestedKeyUsages = append(requestedKeyUsages, ExtKeyUsageServerAuth) + var candidateChains [][]*Certificate + if opts.Roots.contains(c) { + candidateChains = append(candidateChains, []*Certificate{c}) + } else { + if candidateChains, err = c.buildChains(make(map[int][][]*Certificate), []*Certificate{c}, &opts); err != nil { + return nil, err + } } - // If no key usages are specified, then any are acceptable. - checkEKU := len(c.ExtKeyUsage) > 0 + keyUsages := opts.KeyUsages + if len(keyUsages) == 0 { + keyUsages = []ExtKeyUsage{ExtKeyUsageServerAuth} + } - for _, eku := range requestedKeyUsages { - if eku == ExtKeyUsageAny { - checkEKU = false - break + // If any key usage is acceptable then we're done. + for _, usage := range keyUsages { + if usage == ExtKeyUsageAny { + return candidateChains, nil } } - if checkEKU { - NextUsage: - for _, eku := range requestedKeyUsages { - for _, leafEKU := range c.ExtKeyUsage { - if ekuPermittedBy(eku, leafEKU) { - continue NextUsage - } - } - - oid, _ := oidFromExtKeyUsage(eku) - return nil, CertificateInvalidError{c, IncompatibleUsage, fmt.Sprintf("%#v", oid)} + for _, candidate := range candidateChains { + if checkChainForKeyUsage(candidate, keyUsages) { + chains = append(chains, candidate) } } - var candidateChains [][]*Certificate - if opts.Roots.contains(c) { - candidateChains = append(candidateChains, []*Certificate{c}) - } else { - if candidateChains, err = c.buildChains(make(map[int][][]*Certificate), []*Certificate{c}, &opts); err != nil { - return nil, err - } + if len(chains) == 0 { + return nil, CertificateInvalidError{c, IncompatibleUsage, ""} } - return candidateChains, nil + return chains, nil } func appendToFreshChain(chain []*Certificate, cert *Certificate) []*Certificate { @@ -1042,3 +965,65 @@ func (c *Certificate) VerifyHostname(h string) error { return HostnameError{c, h} } + +func checkChainForKeyUsage(chain []*Certificate, keyUsages []ExtKeyUsage) bool { + usages := make([]ExtKeyUsage, len(keyUsages)) + copy(usages, keyUsages) + + if len(chain) == 0 { + return false + } + + usagesRemaining := len(usages) + + // We walk down the list and cross out any usages that aren't supported + // by each certificate. If we cross out all the usages, then the chain + // is unacceptable. + +NextCert: + for i := len(chain) - 1; i >= 0; i-- { + cert := chain[i] + if len(cert.ExtKeyUsage) == 0 && len(cert.UnknownExtKeyUsage) == 0 { + // The certificate doesn't have any extended key usage specified. + continue + } + + for _, usage := range cert.ExtKeyUsage { + if usage == ExtKeyUsageAny { + // The certificate is explicitly good for any usage. + continue NextCert + } + } + + const invalidUsage ExtKeyUsage = -1 + + NextRequestedUsage: + for i, requestedUsage := range usages { + if requestedUsage == invalidUsage { + continue + } + + for _, usage := range cert.ExtKeyUsage { + if requestedUsage == usage { + continue NextRequestedUsage + } else if requestedUsage == ExtKeyUsageServerAuth && + (usage == ExtKeyUsageNetscapeServerGatedCrypto || + usage == ExtKeyUsageMicrosoftServerGatedCrypto) { + // In order to support COMODO + // certificate chains, we have to + // accept Netscape or Microsoft SGC + // usages as equal to ServerAuth. + continue NextRequestedUsage + } + } + + usages[i] = invalidUsage + usagesRemaining-- + if usagesRemaining == 0 { + return false + } + } + } + + return true +} diff --git a/src/crypto/x509/x509.go b/src/crypto/x509/x509.go index 86d9e82aca..8c50a0d474 100644 --- a/src/crypto/x509/x509.go +++ b/src/crypto/x509/x509.go @@ -706,7 +706,9 @@ type Certificate struct { OCSPServer []string IssuingCertificateURL []string - // Subject Alternate Name values + // Subject Alternate Name values. (Note that these values may not be valid + // if invalid values were contained within a parsed certificate. For + // example, an element of DNSNames may not be a valid DNS domain name.) DNSNames []string EmailAddresses []string IPAddresses []net.IP @@ -1126,17 +1128,9 @@ func parseSANExtension(value []byte) (dnsNames, emailAddresses []string, ipAddre err = forEachSAN(value, func(tag int, data []byte) error { switch tag { case nameTypeEmail: - mailbox := string(data) - if _, ok := parseRFC2821Mailbox(mailbox); !ok { - return fmt.Errorf("x509: cannot parse rfc822Name %q", mailbox) - } - emailAddresses = append(emailAddresses, mailbox) + emailAddresses = append(emailAddresses, string(data)) case nameTypeDNS: - domain := string(data) - if _, ok := domainToReverseLabels(domain); !ok { - return fmt.Errorf("x509: cannot parse dnsName %q", string(data)) - } - dnsNames = append(dnsNames, domain) + dnsNames = append(dnsNames, string(data)) case nameTypeURI: uri, err := url.Parse(string(data)) if err != nil { @@ -1153,7 +1147,7 @@ func parseSANExtension(value []byte) (dnsNames, emailAddresses []string, ipAddre case net.IPv4len, net.IPv6len: ipAddresses = append(ipAddresses, data) default: - return errors.New("x509: certificate contained IP address of length " + strconv.Itoa(len(data))) + return errors.New("x509: cannot parse IP address of length " + strconv.Itoa(len(data))) } } diff --git a/src/encoding/json/decode.go b/src/encoding/json/decode.go index 536f25dc7c..730fb920eb 100644 --- a/src/encoding/json/decode.go +++ b/src/encoding/json/decode.go @@ -443,10 +443,25 @@ func (d *decodeState) valueQuoted() interface{} { // if it encounters an Unmarshaler, indirect stops and returns that. // if decodingNull is true, indirect stops at the last pointer so it can be set to nil. func (d *decodeState) indirect(v reflect.Value, decodingNull bool) (Unmarshaler, encoding.TextUnmarshaler, reflect.Value) { + // Issue #24153 indicates that it is generally not a guaranteed property + // that you may round-trip a reflect.Value by calling Value.Addr().Elem() + // and expect the value to still be settable for values derived from + // unexported embedded struct fields. + // + // The logic below effectively does this when it first addresses the value + // (to satisfy possible pointer methods) and continues to dereference + // subsequent pointers as necessary. + // + // After the first round-trip, we set v back to the original value to + // preserve the original RW flags contained in reflect.Value. + v0 := v + haveAddr := false + // If v is a named type and is addressable, // start with its address, so that if the type has pointer methods, // we find them. if v.Kind() != reflect.Ptr && v.Type().Name() != "" && v.CanAddr() { + haveAddr = true v = v.Addr() } for { @@ -455,6 +470,7 @@ func (d *decodeState) indirect(v reflect.Value, decodingNull bool) (Unmarshaler, if v.Kind() == reflect.Interface && !v.IsNil() { e := v.Elem() if e.Kind() == reflect.Ptr && !e.IsNil() && (!decodingNull || e.Elem().Kind() == reflect.Ptr) { + haveAddr = false v = e continue } @@ -480,7 +496,13 @@ func (d *decodeState) indirect(v reflect.Value, decodingNull bool) (Unmarshaler, } } } - v = v.Elem() + + if haveAddr { + v = v0 // restore original value after round-trip Value.Addr().Elem() + haveAddr = false + } else { + v = v.Elem() + } } return nil, nil, v } diff --git a/src/encoding/json/decode_test.go b/src/encoding/json/decode_test.go index 34b7ec6d97..172150c7e9 100644 --- a/src/encoding/json/decode_test.go +++ b/src/encoding/json/decode_test.go @@ -2089,10 +2089,14 @@ func TestInvalidStringOption(t *testing.T) { } } -// Test unmarshal behavior with regards to embedded pointers to unexported structs. -// If unallocated, this returns an error because unmarshal cannot set the field. -// Issue 21357. -func TestUnmarshalEmbeddedPointerUnexported(t *testing.T) { +// Test unmarshal behavior with regards to embedded unexported structs. +// +// (Issue 21357) If the embedded struct is a pointer and is unallocated, +// this returns an error because unmarshal cannot set the field. +// +// (Issue 24152) If the embedded struct is given an explicit name, +// ensure that the normal unmarshal logic does not panic in reflect. +func TestUnmarshalEmbeddedUnexported(t *testing.T) { type ( embed1 struct{ Q int } embed2 struct{ Q int } @@ -2119,6 +2123,18 @@ func TestUnmarshalEmbeddedPointerUnexported(t *testing.T) { *embed3 R int } + S6 struct { + embed1 `json:"embed1"` + } + S7 struct { + embed1 `json:"embed1"` + embed2 + } + S8 struct { + embed1 `json:"embed1"` + embed2 `json:"embed2"` + Q int + } ) tests := []struct { @@ -2154,6 +2170,32 @@ func TestUnmarshalEmbeddedPointerUnexported(t *testing.T) { ptr: new(S5), out: &S5{R: 2}, err: fmt.Errorf("json: cannot set embedded pointer to unexported struct: json.embed3"), + }, { + // Issue 24152, ensure decodeState.indirect does not panic. + in: `{"embed1": {"Q": 1}}`, + ptr: new(S6), + out: &S6{embed1{1}}, + }, { + // Issue 24153, check that we can still set forwarded fields even in + // the presence of a name conflict. + // + // This relies on obscure behavior of reflect where it is possible + // to set a forwarded exported field on an unexported embedded struct + // even though there is a name conflict, even when it would have been + // impossible to do so according to Go visibility rules. + // Go forbids this because it is ambiguous whether S7.Q refers to + // S7.embed1.Q or S7.embed2.Q. Since embed1 and embed2 are unexported, + // it should be impossible for an external package to set either Q. + // + // It is probably okay for a future reflect change to break this. + in: `{"embed1": {"Q": 1}, "Q": 2}`, + ptr: new(S7), + out: &S7{embed1{1}, embed2{2}}, + }, { + // Issue 24153, similar to the S7 case. + in: `{"embed1": {"Q": 1}, "embed2": {"Q": 2}, "Q": 3}`, + ptr: new(S8), + out: &S8{embed1{1}, embed2{2}, 3}, }} for i, tt := range tests { diff --git a/src/go/internal/srcimporter/srcimporter.go b/src/go/internal/srcimporter/srcimporter.go index b0dc8abfc2..9ed7e5e4dc 100644 --- a/src/go/internal/srcimporter/srcimporter.go +++ b/src/go/internal/srcimporter/srcimporter.go @@ -44,9 +44,9 @@ func New(ctxt *build.Context, fset *token.FileSet, packages map[string]*types.Pa // for a package that is in the process of being imported. var importing types.Package -// Import(path) is a shortcut for ImportFrom(path, "", 0). +// Import(path) is a shortcut for ImportFrom(path, ".", 0). func (p *Importer) Import(path string) (*types.Package, error) { - return p.ImportFrom(path, "", 0) + return p.ImportFrom(path, ".", 0) // use "." rather than "" (see issue #24441) } // ImportFrom imports the package with the given import path resolved from the given srcDir, @@ -60,23 +60,10 @@ func (p *Importer) ImportFrom(path, srcDir string, mode types.ImportMode) (*type panic("non-zero import mode") } - // determine package path (do vendor resolution) - var bp *build.Package - var err error - switch { - default: - if abs, err := p.absPath(srcDir); err == nil { // see issue #14282 - srcDir = abs - } - bp, err = p.ctxt.Import(path, srcDir, build.FindOnly) - - case build.IsLocalImport(path): - // "./x" -> "srcDir/x" - bp, err = p.ctxt.ImportDir(filepath.Join(srcDir, path), build.FindOnly) - - case p.isAbsPath(path): - return nil, fmt.Errorf("invalid absolute import path %q", path) + if abs, err := p.absPath(srcDir); err == nil { // see issue #14282 + srcDir = abs } + bp, err := p.ctxt.Import(path, srcDir, 0) if err != nil { return nil, err // err may be *build.NoGoError - return as is } @@ -113,11 +100,6 @@ func (p *Importer) ImportFrom(path, srcDir string, mode types.ImportMode) (*type } }() - // collect package files - bp, err = p.ctxt.ImportDir(bp.Dir, 0) - if err != nil { - return nil, err // err may be *build.NoGoError - return as is - } var filenames []string filenames = append(filenames, bp.GoFiles...) filenames = append(filenames, bp.CgoFiles...) diff --git a/src/go/internal/srcimporter/srcimporter_test.go b/src/go/internal/srcimporter/srcimporter_test.go index 356e71d128..dd4d56ad17 100644 --- a/src/go/internal/srcimporter/srcimporter_test.go +++ b/src/go/internal/srcimporter/srcimporter_test.go @@ -10,6 +10,7 @@ import ( "go/types" "internal/testenv" "io/ioutil" + "path" "path/filepath" "runtime" "strings" @@ -162,3 +163,34 @@ func TestIssue20855(t *testing.T) { t.Error("got no package despite no hard errors") } } + +func testImportPath(t *testing.T, pkgPath string) { + if !testenv.HasSrc() { + t.Skip("no source code available") + } + + pkgName := path.Base(pkgPath) + + pkg, err := importer.Import(pkgPath) + if err != nil { + t.Fatal(err) + } + + if pkg.Name() != pkgName { + t.Errorf("got %q; want %q", pkg.Name(), pkgName) + } + + if pkg.Path() != pkgPath { + t.Errorf("got %q; want %q", pkg.Path(), pkgPath) + } +} + +// TestIssue23092 tests relative imports. +func TestIssue23092(t *testing.T) { + testImportPath(t, "./testdata/issue23092") +} + +// TestIssue24392 tests imports against a path containing 'testdata'. +func TestIssue24392(t *testing.T) { + testImportPath(t, "go/internal/srcimporter/testdata/issue24392") +} diff --git a/src/go/internal/srcimporter/testdata/issue23092/issue23092.go b/src/go/internal/srcimporter/testdata/issue23092/issue23092.go new file mode 100644 index 0000000000..608698bfc5 --- /dev/null +++ b/src/go/internal/srcimporter/testdata/issue23092/issue23092.go @@ -0,0 +1,5 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package issue23092 diff --git a/src/go/internal/srcimporter/testdata/issue24392/issue24392.go b/src/go/internal/srcimporter/testdata/issue24392/issue24392.go new file mode 100644 index 0000000000..8ad52218fc --- /dev/null +++ b/src/go/internal/srcimporter/testdata/issue24392/issue24392.go @@ -0,0 +1,5 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package issue24392 diff --git a/src/internal/singleflight/singleflight.go b/src/internal/singleflight/singleflight.go index 1e9960d575..b2d82e26c2 100644 --- a/src/internal/singleflight/singleflight.go +++ b/src/internal/singleflight/singleflight.go @@ -103,11 +103,21 @@ func (g *Group) doCall(c *call, key string, fn func() (interface{}, error)) { g.mu.Unlock() } -// Forget tells the singleflight to forget about a key. Future calls -// to Do for this key will call the function rather than waiting for -// an earlier call to complete. -func (g *Group) Forget(key string) { +// ForgetUnshared tells the singleflight to forget about a key if it is not +// shared with any other goroutines. Future calls to Do for a forgotten key +// will call the function rather than waiting for an earlier call to complete. +// Returns whether the key was forgotten or unknown--that is, whether no +// other goroutines are waiting for the result. +func (g *Group) ForgetUnshared(key string) bool { g.mu.Lock() - delete(g.m, key) - g.mu.Unlock() + defer g.mu.Unlock() + c, ok := g.m[key] + if !ok { + return true + } + if c.dups == 0 { + delete(g.m, key) + return true + } + return false } diff --git a/src/net/http/pprof/pprof.go b/src/net/http/pprof/pprof.go index 21992d62da..77e0bcdf4f 100644 --- a/src/net/http/pprof/pprof.go +++ b/src/net/http/pprof/pprof.go @@ -80,6 +80,7 @@ func init() { // command line, with arguments separated by NUL bytes. // The package initialization registers it as /debug/pprof/cmdline. func Cmdline(w http.ResponseWriter, r *http.Request) { + w.Header().Set("X-Content-Type-Options", "nosniff") w.Header().Set("Content-Type", "text/plain; charset=utf-8") fmt.Fprintf(w, strings.Join(os.Args, "\x00")) } @@ -100,33 +101,36 @@ func durationExceedsWriteTimeout(r *http.Request, seconds float64) bool { return ok && srv.WriteTimeout != 0 && seconds >= srv.WriteTimeout.Seconds() } +func serveError(w http.ResponseWriter, status int, txt string) { + w.Header().Set("Content-Type", "text/plain; charset=utf-8") + w.Header().Set("X-Go-Pprof", "1") + w.Header().Del("Content-Disposition") + w.WriteHeader(status) + fmt.Fprintln(w, txt) +} + // Profile responds with the pprof-formatted cpu profile. // The package initialization registers it as /debug/pprof/profile. func Profile(w http.ResponseWriter, r *http.Request) { + w.Header().Set("X-Content-Type-Options", "nosniff") sec, _ := strconv.ParseInt(r.FormValue("seconds"), 10, 64) if sec == 0 { sec = 30 } if durationExceedsWriteTimeout(r, float64(sec)) { - w.Header().Set("Content-Type", "text/plain; charset=utf-8") - w.Header().Set("X-Go-Pprof", "1") - w.WriteHeader(http.StatusBadRequest) - fmt.Fprintln(w, "profile duration exceeds server's WriteTimeout") + serveError(w, http.StatusBadRequest, "profile duration exceeds server's WriteTimeout") return } // Set Content Type assuming StartCPUProfile will work, // because if it does it starts writing. w.Header().Set("Content-Type", "application/octet-stream") + w.Header().Set("Content-Disposition", `attachment; filename="profile"`) if err := pprof.StartCPUProfile(w); err != nil { // StartCPUProfile failed, so no writes yet. - // Can change header back to text content - // and send error code. - w.Header().Set("Content-Type", "text/plain; charset=utf-8") - w.Header().Set("X-Go-Pprof", "1") - w.WriteHeader(http.StatusInternalServerError) - fmt.Fprintf(w, "Could not enable CPU profiling: %s\n", err) + serveError(w, http.StatusInternalServerError, + fmt.Sprintf("Could not enable CPU profiling: %s", err)) return } sleep(w, time.Duration(sec)*time.Second) @@ -137,29 +141,25 @@ func Profile(w http.ResponseWriter, r *http.Request) { // Tracing lasts for duration specified in seconds GET parameter, or for 1 second if not specified. // The package initialization registers it as /debug/pprof/trace. func Trace(w http.ResponseWriter, r *http.Request) { + w.Header().Set("X-Content-Type-Options", "nosniff") sec, err := strconv.ParseFloat(r.FormValue("seconds"), 64) if sec <= 0 || err != nil { sec = 1 } if durationExceedsWriteTimeout(r, sec) { - w.Header().Set("Content-Type", "text/plain; charset=utf-8") - w.Header().Set("X-Go-Pprof", "1") - w.WriteHeader(http.StatusBadRequest) - fmt.Fprintln(w, "profile duration exceeds server's WriteTimeout") + serveError(w, http.StatusBadRequest, "profile duration exceeds server's WriteTimeout") return } // Set Content Type assuming trace.Start will work, // because if it does it starts writing. w.Header().Set("Content-Type", "application/octet-stream") + w.Header().Set("Content-Disposition", `attachment; filename="trace"`) if err := trace.Start(w); err != nil { // trace.Start failed, so no writes yet. - // Can change header back to text content and send error code. - w.Header().Set("Content-Type", "text/plain; charset=utf-8") - w.Header().Set("X-Go-Pprof", "1") - w.WriteHeader(http.StatusInternalServerError) - fmt.Fprintf(w, "Could not enable tracing: %s\n", err) + serveError(w, http.StatusInternalServerError, + fmt.Sprintf("Could not enable tracing: %s", err)) return } sleep(w, time.Duration(sec*float64(time.Second))) @@ -170,6 +170,7 @@ func Trace(w http.ResponseWriter, r *http.Request) { // responding with a table mapping program counters to function names. // The package initialization registers it as /debug/pprof/symbol. func Symbol(w http.ResponseWriter, r *http.Request) { + w.Header().Set("X-Content-Type-Options", "nosniff") w.Header().Set("Content-Type", "text/plain; charset=utf-8") // We have to read the whole POST body before @@ -222,18 +223,23 @@ func Handler(name string) http.Handler { type handler string func (name handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "text/plain; charset=utf-8") - debug, _ := strconv.Atoi(r.FormValue("debug")) + w.Header().Set("X-Content-Type-Options", "nosniff") p := pprof.Lookup(string(name)) if p == nil { - w.WriteHeader(404) - fmt.Fprintf(w, "Unknown profile: %s\n", name) + serveError(w, http.StatusNotFound, "Unknown profile") return } gc, _ := strconv.Atoi(r.FormValue("gc")) if name == "heap" && gc > 0 { runtime.GC() } + debug, _ := strconv.Atoi(r.FormValue("debug")) + if debug != 0 { + w.Header().Set("Content-Type", "text/plain; charset=utf-8") + } else { + w.Header().Set("Content-Type", "application/octet-stream") + w.Header().Set("Content-Disposition", fmt.Sprintf(`attachment; filename="%s"`, name)) + } p.WriteTo(w, debug) } diff --git a/src/net/http/pprof/pprof_test.go b/src/net/http/pprof/pprof_test.go new file mode 100644 index 0000000000..47dd35b9b0 --- /dev/null +++ b/src/net/http/pprof/pprof_test.go @@ -0,0 +1,69 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package pprof + +import ( + "bytes" + "io/ioutil" + "net/http" + "net/http/httptest" + "testing" +) + +func TestHandlers(t *testing.T) { + testCases := []struct { + path string + handler http.HandlerFunc + statusCode int + contentType string + contentDisposition string + resp []byte + }{ + {"/debug/pprof/<script>scripty<script>", Index, http.StatusNotFound, "text/plain; charset=utf-8", "", []byte("Unknown profile\n")}, + {"/debug/pprof/heap", Index, http.StatusOK, "application/octet-stream", `attachment; filename="heap"`, nil}, + {"/debug/pprof/heap?debug=1", Index, http.StatusOK, "text/plain; charset=utf-8", "", nil}, + {"/debug/pprof/cmdline", Cmdline, http.StatusOK, "text/plain; charset=utf-8", "", nil}, + {"/debug/pprof/profile?seconds=1", Profile, http.StatusOK, "application/octet-stream", `attachment; filename="profile"`, nil}, + {"/debug/pprof/symbol", Symbol, http.StatusOK, "text/plain; charset=utf-8", "", nil}, + {"/debug/pprof/trace", Trace, http.StatusOK, "application/octet-stream", `attachment; filename="trace"`, nil}, + } + for _, tc := range testCases { + t.Run(tc.path, func(t *testing.T) { + req := httptest.NewRequest("GET", "http://example.com"+tc.path, nil) + w := httptest.NewRecorder() + tc.handler(w, req) + + resp := w.Result() + if got, want := resp.StatusCode, tc.statusCode; got != want { + t.Errorf("status code: got %d; want %d", got, want) + } + + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + t.Errorf("when reading response body, expected non-nil err; got %v", err) + } + if got, want := resp.Header.Get("X-Content-Type-Options"), "nosniff"; got != want { + t.Errorf("X-Content-Type-Options: got %q; want %q", got, want) + } + if got, want := resp.Header.Get("Content-Type"), tc.contentType; got != want { + t.Errorf("Content-Type: got %q; want %q", got, want) + } + if got, want := resp.Header.Get("Content-Disposition"), tc.contentDisposition; got != want { + t.Errorf("Content-Disposition: got %q; want %q", got, want) + } + + if resp.StatusCode == http.StatusOK { + return + } + if got, want := resp.Header.Get("X-Go-Pprof"), "1"; got != want { + t.Errorf("X-Go-Pprof: got %q; want %q", got, want) + } + if !bytes.Equal(body, tc.resp) { + t.Errorf("response: got %q; want %q", body, tc.resp) + } + }) + } + +} diff --git a/src/net/lookup.go b/src/net/lookup.go index 85e472932f..a65b735f92 100644 --- a/src/net/lookup.go +++ b/src/net/lookup.go @@ -194,10 +194,16 @@ func (r *Resolver) LookupIPAddr(ctx context.Context, host string) ([]IPAddr, err resolverFunc = alt } + // We don't want a cancelation of ctx to affect the + // lookupGroup operation. Otherwise if our context gets + // canceled it might cause an error to be returned to a lookup + // using a completely different context. + lookupGroupCtx, lookupGroupCancel := context.WithCancel(context.Background()) + dnsWaitGroup.Add(1) ch, called := lookupGroup.DoChan(host, func() (interface{}, error) { defer dnsWaitGroup.Done() - return testHookLookupIP(ctx, resolverFunc, host) + return testHookLookupIP(lookupGroupCtx, resolverFunc, host) }) if !called { dnsWaitGroup.Done() @@ -205,20 +211,28 @@ func (r *Resolver) LookupIPAddr(ctx context.Context, host string) ([]IPAddr, err select { case <-ctx.Done(): - // If the DNS lookup timed out for some reason, force - // future requests to start the DNS lookup again - // rather than waiting for the current lookup to - // complete. See issue 8602. - ctxErr := ctx.Err() - if ctxErr == context.DeadlineExceeded { - lookupGroup.Forget(host) + // Our context was canceled. If we are the only + // goroutine looking up this key, then drop the key + // from the lookupGroup and cancel the lookup. + // If there are other goroutines looking up this key, + // let the lookup continue uncanceled, and let later + // lookups with the same key share the result. + // See issues 8602, 20703, 22724. + if lookupGroup.ForgetUnshared(host) { + lookupGroupCancel() + } else { + go func() { + <-ch + lookupGroupCancel() + }() } - err := mapErr(ctxErr) + err := mapErr(ctx.Err()) if trace != nil && trace.DNSDone != nil { trace.DNSDone(nil, false, err) } return nil, err case r := <-ch: + lookupGroupCancel() if trace != nil && trace.DNSDone != nil { addrs, _ := r.Val.([]IPAddr) trace.DNSDone(ipAddrsEface(addrs), r.Shared, r.Err) diff --git a/src/net/lookup_test.go b/src/net/lookup_test.go index bfb872551c..24787ccf2b 100644 --- a/src/net/lookup_test.go +++ b/src/net/lookup_test.go @@ -791,3 +791,28 @@ func TestLookupNonLDH(t *testing.T) { t.Fatalf("lookup error = %v, want %v", err, errNoSuchHost) } } + +func TestLookupContextCancel(t *testing.T) { + if testenv.Builder() == "" { + testenv.MustHaveExternalNetwork(t) + } + if runtime.GOOS == "nacl" { + t.Skip("skip on nacl") + } + + defer dnsWaitGroup.Wait() + + ctx, ctxCancel := context.WithCancel(context.Background()) + ctxCancel() + _, err := DefaultResolver.LookupIPAddr(ctx, "google.com") + if err != errCanceled { + testenv.SkipFlakyNet(t) + t.Fatal(err) + } + ctx = context.Background() + _, err = DefaultResolver.LookupIPAddr(ctx, "google.com") + if err != nil { + testenv.SkipFlakyNet(t) + t.Fatal(err) + } +} diff --git a/src/net/tcpsock_unix_test.go b/src/net/tcpsock_unix_test.go index 3af1834455..95c02d2721 100644 --- a/src/net/tcpsock_unix_test.go +++ b/src/net/tcpsock_unix_test.go @@ -87,6 +87,7 @@ func TestTCPSpuriousConnSetupCompletionWithCancel(t *testing.T) { if testenv.Builder() == "" { testenv.MustHaveExternalNetwork(t) } + defer dnsWaitGroup.Wait() t.Parallel() const tries = 10000 var wg sync.WaitGroup diff --git a/src/runtime/error.go b/src/runtime/error.go index 6048272e75..e1291e1543 100644 --- a/src/runtime/error.go +++ b/src/runtime/error.go @@ -73,14 +73,12 @@ func typestring(x interface{}) string { } // printany prints an argument passed to panic. +// If panic is called with a value that has a String or Error method, +// it has already been converted into a string by preprintpanics. func printany(i interface{}) { switch v := i.(type) { case nil: print("nil") - case stringer: - print(v.String()) - case error: - print(v.Error()) case bool: print(v) case int: diff --git a/src/runtime/iface.go b/src/runtime/iface.go index 7c5d3a05b2..bd6bc282f4 100644 --- a/src/runtime/iface.go +++ b/src/runtime/iface.go @@ -113,6 +113,14 @@ func (t *itabTableType) find(inter *interfacetype, typ *_type) *itab { // itabAdd adds the given itab to the itab hash table. // itabLock must be held. func itabAdd(m *itab) { + // Bugs can lead to calling this while mallocing is set, + // typically because this is called while panicing. + // Crash reliably, rather than only when we need to grow + // the hash table. + if getg().m.mallocing != 0 { + throw("malloc deadlock") + } + t := itabTable if t.count >= 3*(t.size/4) { // 75% load factor // Grow hash table. diff --git a/src/runtime/os_windows.go b/src/runtime/os_windows.go index 7aeadd9ef1..2653a94169 100644 --- a/src/runtime/os_windows.go +++ b/src/runtime/os_windows.go @@ -304,8 +304,6 @@ func osinit() { disableWER() - externalthreadhandlerp = funcPC(externalthreadhandler) - initExceptionHandler() stdcall2(_SetConsoleCtrlHandler, funcPC(ctrlhandler), 1) diff --git a/src/runtime/panic.go b/src/runtime/panic.go index c51948bd18..3d0c85f7f9 100644 --- a/src/runtime/panic.go +++ b/src/runtime/panic.go @@ -389,7 +389,6 @@ func Goexit() { // Call all Error and String methods before freezing the world. // Used when crashing with panicking. -// This must match types handled by printany. func preprintpanics(p *_panic) { defer func() { if recover() != nil { @@ -415,8 +414,6 @@ func printpanics(p *_panic) { print("\t") } print("panic: ") - // Because of preprintpanics, p.arg cannot be an error or - // stringer, so this won't call into user code. printany(p.arg) if p.recovered { print(" [recovered]") diff --git a/src/runtime/proc.go b/src/runtime/proc.go index 2e958f7fc5..6e4097517d 100644 --- a/src/runtime/proc.go +++ b/src/runtime/proc.go @@ -393,6 +393,11 @@ func releaseSudog(s *sudog) { // funcPC returns the entry PC of the function f. // It assumes that f is a func value. Otherwise the behavior is undefined. +// CAREFUL: In programs with plugins, funcPC can return different values +// for the same function (because there are actually multiple copies of +// the same function in the address space). To be safe, don't use the +// results of this function in any == expression. It is only safe to +// use the result as an address at which to start executing code. //go:nosplit func funcPC(f interface{}) uintptr { return **(**uintptr)(add(unsafe.Pointer(&f), sys.PtrSize)) @@ -3798,8 +3803,8 @@ func setsSP(pc uintptr) bool { // so assume the worst and stop traceback return true } - switch f.entry { - case gogoPC, systemstackPC, mcallPC, morestackPC: + switch f.funcID { + case funcID_gogo, funcID_systemstack, funcID_mcall, funcID_morestack: return true } return false diff --git a/src/runtime/runtime2.go b/src/runtime/runtime2.go index 556f13d1c1..a783f60ae4 100644 --- a/src/runtime/runtime2.go +++ b/src/runtime/runtime2.go @@ -635,8 +635,8 @@ type _func struct { entry uintptr // start pc nameoff int32 // function name - args int32 // in/out args size - _ int32 // previously legacy frame size; kept for layout compatibility + args int32 // in/out args size + funcID funcID // set for certain special runtime functions pcsp int32 pcfile int32 diff --git a/src/runtime/stack.go b/src/runtime/stack.go index 6149838b6d..1bfd9dd27f 100644 --- a/src/runtime/stack.go +++ b/src/runtime/stack.go @@ -619,7 +619,7 @@ func adjustframe(frame *stkframe, arg unsafe.Pointer) bool { if stackDebug >= 2 { print(" adjusting ", funcname(f), " frame=[", hex(frame.sp), ",", hex(frame.fp), "] pc=", hex(frame.pc), " continpc=", hex(frame.continpc), "\n") } - if f.entry == systemstack_switchPC { + if f.funcID == funcID_systemstack_switch { // A special routine at the bottom of stack of a goroutine that does an systemstack call. // We will allow it to be copied even though we don't // have full GC info for it (because it is written in asm). @@ -1110,7 +1110,8 @@ func shrinkstack(gp *g) { if debug.gcshrinkstackoff > 0 { return } - if gp.startpc == gcBgMarkWorkerPC { + f := findfunc(gp.startpc) + if f.valid() && f.funcID == funcID_gcBgMarkWorker { // We're not allowed to shrink the gcBgMarkWorker // stack (see gcBgMarkWorker for explanation). return diff --git a/src/runtime/symtab.go b/src/runtime/symtab.go index bdf98b9e9d..f50284fb6f 100644 --- a/src/runtime/symtab.go +++ b/src/runtime/symtab.go @@ -133,7 +133,7 @@ again: } se.pcExpander.init(ncallers[0], se.wasPanic) ncallers = ncallers[1:] - se.wasPanic = se.pcExpander.funcInfo.valid() && se.pcExpander.funcInfo.entry == sigpanicPC + se.wasPanic = se.pcExpander.funcInfo.valid() && se.pcExpander.funcInfo.funcID == funcID_sigpanic if se.skip > 0 { for ; se.skip > 0; se.skip-- { se.pcExpander.next() @@ -349,6 +349,35 @@ const ( _ArgsSizeUnknown = -0x80000000 ) +// A FuncID identifies particular functions that need to be treated +// specially by the runtime. +// Note that in some situations involving plugins, there may be multiple +// copies of a particular special runtime function. +// Note: this list must match the list in cmd/internal/objabi/funcid.go. +type funcID uint32 + +const ( + funcID_normal funcID = iota // not a special function + funcID_goexit + funcID_jmpdefer + funcID_mcall + funcID_morestack + funcID_mstart + funcID_rt0_go + funcID_asmcgocall + funcID_sigpanic + funcID_runfinq + funcID_bgsweep + funcID_forcegchelper + funcID_timerproc + funcID_gcBgMarkWorker + funcID_systemstack_switch + funcID_systemstack + funcID_cgocallback_gofunc + funcID_gogo + funcID_externalthreadhandler +) + // moduledata records information about the layout of the executable // image. It is written by the linker. Any changes here must be // matched changes to the code in cmd/internal/ld/symtab.go:symtab. @@ -648,7 +677,6 @@ func findfunc(pc uintptr) funcInfo { idx = uint32(len(datap.ftab) - 1) } if pc < datap.ftab[idx].entry { - // With multiple text sections, the idx might reference a function address that // is higher than the pc being searched, so search backward until the matching address is found. @@ -659,7 +687,6 @@ func findfunc(pc uintptr) funcInfo { throw("findfunc: bad findfunctab entry idx") } } else { - // linear search to find func with pc >= entry. for datap.ftab[idx+1].entry <= pc { idx++ diff --git a/src/runtime/sys_linux_amd64.s b/src/runtime/sys_linux_amd64.s index 5a94bda7c2..a5db5cba23 100644 --- a/src/runtime/sys_linux_amd64.s +++ b/src/runtime/sys_linux_amd64.s @@ -10,9 +10,10 @@ #include "go_tls.h" #include "textflag.h" +#define AT_FDCWD -100 + #define SYS_read 0 #define SYS_write 1 -#define SYS_open 2 #define SYS_close 3 #define SYS_mmap 9 #define SYS_munmap 11 @@ -20,7 +21,6 @@ #define SYS_rt_sigaction 13 #define SYS_rt_sigprocmask 14 #define SYS_rt_sigreturn 15 -#define SYS_access 21 #define SYS_sched_yield 24 #define SYS_mincore 27 #define SYS_madvise 28 @@ -41,9 +41,11 @@ #define SYS_sched_getaffinity 204 #define SYS_epoll_create 213 #define SYS_exit_group 231 -#define SYS_epoll_wait 232 #define SYS_epoll_ctl 233 +#define SYS_openat 257 +#define SYS_faccessat 269 #define SYS_pselect6 270 +#define SYS_epoll_pwait 281 #define SYS_epoll_create1 291 TEXT runtime·exit(SB),NOSPLIT,$0-4 @@ -65,10 +67,12 @@ TEXT runtime·exitThread(SB),NOSPLIT,$0-8 JMP 0(PC) TEXT runtime·open(SB),NOSPLIT,$0-20 - MOVQ name+0(FP), DI - MOVL mode+8(FP), SI - MOVL perm+12(FP), DX - MOVL $SYS_open, AX + // This uses openat instead of open, because Android O blocks open. + MOVL $AT_FDCWD, DI // AT_FDCWD, so this acts like open + MOVQ name+0(FP), SI + MOVL mode+8(FP), DX + MOVL perm+12(FP), R10 + MOVL $SYS_openat, AX SYSCALL CMPQ AX, $0xfffffffffffff001 JLS 2(PC) @@ -599,7 +603,7 @@ TEXT runtime·settls(SB),NOSPLIT,$32 // Same as in sys_darwin_386.s:/ugliness, different constant. // DI currently holds m->tls, which must be fs:0x1d0. // See cgo/gcc_android_amd64.c for the derivation of the constant. - SUBQ $0x1d0, DI // In android, the tls base + SUBQ $0x1d0, DI // In android, the tls base #else ADDQ $8, DI // ELF wants to use -8(FS) #endif @@ -655,11 +659,13 @@ TEXT runtime·epollctl(SB),NOSPLIT,$0 // int32 runtime·epollwait(int32 epfd, EpollEvent *ev, int32 nev, int32 timeout); TEXT runtime·epollwait(SB),NOSPLIT,$0 + // This uses pwait instead of wait, because Android O blocks wait. MOVL epfd+0(FP), DI MOVQ ev+8(FP), SI MOVL nev+16(FP), DX MOVL timeout+20(FP), R10 - MOVL $SYS_epoll_wait, AX + MOVQ $0, R8 + MOVL $SYS_epoll_pwait, AX SYSCALL MOVL AX, ret+24(FP) RET @@ -676,9 +682,12 @@ TEXT runtime·closeonexec(SB),NOSPLIT,$0 // int access(const char *name, int mode) TEXT runtime·access(SB),NOSPLIT,$0 - MOVQ name+0(FP), DI - MOVL mode+8(FP), SI - MOVL $SYS_access, AX + // This uses faccessat instead of access, because Android O blocks access. + MOVL $AT_FDCWD, DI // AT_FDCWD, so this acts like access + MOVQ name+0(FP), SI + MOVL mode+8(FP), DX + MOVL $0, R10 + MOVL $SYS_faccessat, AX SYSCALL MOVL AX, ret+16(FP) RET diff --git a/src/runtime/sys_linux_mips64x.s b/src/runtime/sys_linux_mips64x.s index 7402ae21d4..be0f2ca1e6 100644 --- a/src/runtime/sys_linux_mips64x.s +++ b/src/runtime/sys_linux_mips64x.s @@ -13,10 +13,11 @@ #include "go_tls.h" #include "textflag.h" +#define AT_FDCWD -100 + #define SYS_exit 5058 #define SYS_read 5000 #define SYS_write 5001 -#define SYS_open 5002 #define SYS_close 5003 #define SYS_getpid 5038 #define SYS_kill 5060 @@ -42,7 +43,8 @@ #define SYS_exit_group 5205 #define SYS_epoll_create 5207 #define SYS_epoll_ctl 5208 -#define SYS_epoll_wait 5209 +#define SYS_openat 5247 +#define SYS_epoll_pwait 5272 #define SYS_clock_gettime 5222 #define SYS_epoll_create1 5285 #define SYS_brk 5012 @@ -66,11 +68,13 @@ TEXT runtime·exitThread(SB),NOSPLIT,$-8-8 SYSCALL JMP 0(PC) -TEXT runtime·open(SB),NOSPLIT,$-8-20 - MOVV name+0(FP), R4 - MOVW mode+8(FP), R5 - MOVW perm+12(FP), R6 - MOVV $SYS_open, R2 +TEXT runtime·open(SB),NOSPLIT|NOFRAME,$0-20 + // This uses openat instead of open, because Android O blocks open. + MOVW $AT_FDCWD, R4 // AT_FDCWD, so this acts like open + MOVV name+0(FP), R5 + MOVW mode+8(FP), R6 + MOVW perm+12(FP), R7 + MOVV $SYS_openat, R2 SYSCALL BEQ R7, 2(PC) MOVW $-1, R2 @@ -422,12 +426,14 @@ TEXT runtime·epollctl(SB),NOSPLIT,$-8 RET // int32 runtime·epollwait(int32 epfd, EpollEvent *ev, int32 nev, int32 timeout); -TEXT runtime·epollwait(SB),NOSPLIT,$-8 +TEXT runtime·epollwait(SB),NOSPLIT|NOFRAME,$0 + // This uses pwait instead of wait, because Android O blocks wait. MOVW epfd+0(FP), R4 MOVV ev+8(FP), R5 MOVW nev+16(FP), R6 MOVW timeout+20(FP), R7 - MOVV $SYS_epoll_wait, R2 + MOVV $0, R8 + MOVV $SYS_epoll_pwait, R2 SYSCALL MOVW R2, ret+24(FP) RET diff --git a/src/runtime/traceback.go b/src/runtime/traceback.go index 747176c278..06df9385fd 100644 --- a/src/runtime/traceback.go +++ b/src/runtime/traceback.go @@ -35,56 +35,14 @@ import ( const usesLR = sys.MinFrameSize > 0 -var ( - // initialized in tracebackinit - goexitPC uintptr - jmpdeferPC uintptr - mcallPC uintptr - morestackPC uintptr - mstartPC uintptr - rt0_goPC uintptr - asmcgocallPC uintptr - sigpanicPC uintptr - runfinqPC uintptr - bgsweepPC uintptr - forcegchelperPC uintptr - timerprocPC uintptr - gcBgMarkWorkerPC uintptr - systemstack_switchPC uintptr - systemstackPC uintptr - cgocallback_gofuncPC uintptr - skipPC uintptr - - gogoPC uintptr - - externalthreadhandlerp uintptr // initialized elsewhere -) +var skipPC uintptr func tracebackinit() { // Go variable initialization happens late during runtime startup. // Instead of initializing the variables above in the declarations, // schedinit calls this function so that the variables are // initialized and available earlier in the startup sequence. - goexitPC = funcPC(goexit) - jmpdeferPC = funcPC(jmpdefer) - mcallPC = funcPC(mcall) - morestackPC = funcPC(morestack) - mstartPC = funcPC(mstart) - rt0_goPC = funcPC(rt0_go) - asmcgocallPC = funcPC(asmcgocall) - sigpanicPC = funcPC(sigpanic) - runfinqPC = funcPC(runfinq) - bgsweepPC = funcPC(bgsweep) - forcegchelperPC = funcPC(forcegchelper) - timerprocPC = funcPC(timerproc) - gcBgMarkWorkerPC = funcPC(gcBgMarkWorker) - systemstack_switchPC = funcPC(systemstack_switch) - systemstackPC = funcPC(systemstack) - cgocallback_gofuncPC = funcPC(cgocallback_gofunc) skipPC = funcPC(skipPleaseUseCallersFrames) - - // used by sigprof handler - gogoPC = funcPC(gogo) } // Traceback over the deferred function calls. @@ -137,9 +95,6 @@ func gentraceback(pc0, sp0, lr0 uintptr, gp *g, skip int, pcbuf *uintptr, max in if skip > 0 && callback != nil { throw("gentraceback callback cannot be used with non-zero skip") } - if goexitPC == 0 { - throw("gentraceback before goexitPC initialization") - } g := getg() if g == gp && g == g.m.curg { // The starting sp has been passed in as a uintptr, and the caller may @@ -241,7 +196,7 @@ func gentraceback(pc0, sp0, lr0 uintptr, gp *g, skip int, pcbuf *uintptr, max in // g0, this systemstack is at the top of the stack. // if we're not on g0 or there's a no curg, then this is a regular call. sp := frame.sp - if flags&_TraceJumpStack != 0 && f.entry == systemstackPC && gp == g.m.g0 && gp.m.curg != nil { + if flags&_TraceJumpStack != 0 && f.funcID == funcID_systemstack && gp == g.m.g0 && gp.m.curg != nil { sp = gp.m.curg.sched.sp frame.sp = sp cgoCtxt = gp.m.curg.cgoCtxt @@ -256,7 +211,7 @@ func gentraceback(pc0, sp0, lr0 uintptr, gp *g, skip int, pcbuf *uintptr, max in if topofstack(f, gp.m != nil && gp == gp.m.g0) { frame.lr = 0 flr = funcInfo{} - } else if usesLR && f.entry == jmpdeferPC { + } else if usesLR && f.funcID == funcID_jmpdefer { // jmpdefer modifies SP/LR/PC non-atomically. // If a profiling interrupt arrives during jmpdefer, // the stack unwind may see a mismatched register set @@ -287,7 +242,7 @@ func gentraceback(pc0, sp0, lr0 uintptr, gp *g, skip int, pcbuf *uintptr, max in // But if callback is set, we're doing a garbage collection and must // get everything, so crash loudly. doPrint := printing - if doPrint && gp.m.incgo { + if doPrint && gp.m.incgo && f.funcID == funcID_sigpanic { // We can inject sigpanic // calls directly into C code, // in which case we'll see a C @@ -396,8 +351,8 @@ func gentraceback(pc0, sp0, lr0 uintptr, gp *g, skip int, pcbuf *uintptr, max in // if there's room, pcbuf[1] is a skip PC that encodes the number of skipped frames in pcbuf[0] if n+1 < max { n++ - skipPC := funcPC(skipPleaseUseCallersFrames) + uintptr(logicalSkipped) - (*[1 << 20]uintptr)(unsafe.Pointer(pcbuf))[n] = skipPC + pc := skipPC + uintptr(logicalSkipped) + (*[1 << 20]uintptr)(unsafe.Pointer(pcbuf))[n] = pc } } } @@ -466,7 +421,7 @@ func gentraceback(pc0, sp0, lr0 uintptr, gp *g, skip int, pcbuf *uintptr, max in n++ skipped: - if f.entry == cgocallback_gofuncPC && len(cgoCtxt) > 0 { + if f.funcID == funcID_cgocallback_gofunc && len(cgoCtxt) > 0 { ctxt := cgoCtxt[len(cgoCtxt)-1] cgoCtxt = cgoCtxt[:len(cgoCtxt)-1] @@ -478,7 +433,7 @@ func gentraceback(pc0, sp0, lr0 uintptr, gp *g, skip int, pcbuf *uintptr, max in } } - waspanic = f.entry == sigpanicPC + waspanic = f.funcID == funcID_sigpanic // Do not unwind past the bottom of the stack. if !flr.valid() { @@ -931,30 +886,32 @@ func tracebackHexdump(stk stack, frame *stkframe, bad uintptr) { // Does f mark the top of a goroutine stack? func topofstack(f funcInfo, g0 bool) bool { - pc := f.entry - return pc == goexitPC || - pc == mstartPC || - pc == mcallPC || - pc == morestackPC || - pc == rt0_goPC || - externalthreadhandlerp != 0 && pc == externalthreadhandlerp || + return f.funcID == funcID_goexit || + f.funcID == funcID_mstart || + f.funcID == funcID_mcall || + f.funcID == funcID_morestack || + f.funcID == funcID_rt0_go || + f.funcID == funcID_externalthreadhandler || // asmcgocall is TOS on the system stack because it // switches to the system stack, but in this case we // can come back to the regular stack and still want // to be able to unwind through the call that appeared // on the regular stack. - (g0 && pc == asmcgocallPC) + (g0 && f.funcID == funcID_asmcgocall) } // isSystemGoroutine reports whether the goroutine g must be omitted in // stack dumps and deadlock detector. func isSystemGoroutine(gp *g) bool { - pc := gp.startpc - return pc == runfinqPC && !fingRunning || - pc == bgsweepPC || - pc == forcegchelperPC || - pc == timerprocPC || - pc == gcBgMarkWorkerPC + f := findfunc(gp.startpc) + if !f.valid() { + return false + } + return f.funcID == funcID_runfinq && !fingRunning || + f.funcID == funcID_bgsweep || + f.funcID == funcID_forcegchelper || + f.funcID == funcID_timerproc || + f.funcID == funcID_gcBgMarkWorker } // SetCgoTraceback records three C functions to use to gather diff --git a/src/strings/strings.go b/src/strings/strings.go index 02c032046b..05e8243b2e 100644 --- a/src/strings/strings.go +++ b/src/strings/strings.go @@ -474,7 +474,7 @@ func Map(mapping func(rune) rune, s string) string { b = make([]byte, len(s)+utf8.UTFMax) nbytes = copy(b, s[:i]) if r >= 0 { - if r <= utf8.RuneSelf { + if r < utf8.RuneSelf { b[nbytes] = byte(r) nbytes++ } else { @@ -504,7 +504,7 @@ func Map(mapping func(rune) rune, s string) string { r := mapping(c) // common case - if (0 <= r && r <= utf8.RuneSelf) && nbytes < len(b) { + if (0 <= r && r < utf8.RuneSelf) && nbytes < len(b) { b[nbytes] = byte(r) nbytes++ continue diff --git a/src/strings/strings_test.go b/src/strings/strings_test.go index d8fcb62a87..6c1193873b 100644 --- a/src/strings/strings_test.go +++ b/src/strings/strings_test.go @@ -528,6 +528,7 @@ var upperTests = []StringTest{ {"longStrinGwitHmixofsmaLLandcAps", "LONGSTRINGWITHMIXOFSMALLANDCAPS"}, {"long\u0250string\u0250with\u0250nonascii\u2C6Fchars", "LONG\u2C6FSTRING\u2C6FWITH\u2C6FNONASCII\u2C6FCHARS"}, {"\u0250\u0250\u0250\u0250\u0250", "\u2C6F\u2C6F\u2C6F\u2C6F\u2C6F"}, // grows one byte per char + {"a\u0080\U0010FFFF", "A\u0080\U0010FFFF"}, // test utf8.RuneSelf and utf8.MaxRune } var lowerTests = []StringTest{ @@ -538,6 +539,7 @@ var lowerTests = []StringTest{ {"longStrinGwitHmixofsmaLLandcAps", "longstringwithmixofsmallandcaps"}, {"LONG\u2C6FSTRING\u2C6FWITH\u2C6FNONASCII\u2C6FCHARS", "long\u0250string\u0250with\u0250nonascii\u0250chars"}, {"\u2C6D\u2C6D\u2C6D\u2C6D\u2C6D", "\u0251\u0251\u0251\u0251\u0251"}, // shrinks one byte per char + {"A\u0080\U0010FFFF", "a\u0080\U0010FFFF"}, // test utf8.RuneSelf and utf8.MaxRune } const space = "\t\v\r\f\n\u0085\u00a0\u2000\u3000" @@ -650,6 +652,27 @@ func TestMap(t *testing.T) { if m != expect { t.Errorf("replace invalid sequence: expected %q got %q", expect, m) } + + // 8. Check utf8.RuneSelf and utf8.MaxRune encoding + encode := func(r rune) rune { + switch r { + case utf8.RuneSelf: + return unicode.MaxRune + case unicode.MaxRune: + return utf8.RuneSelf + } + return r + } + s := string(utf8.RuneSelf) + string(utf8.MaxRune) + r := string(utf8.MaxRune) + string(utf8.RuneSelf) // reverse of s + m = Map(encode, s) + if m != r { + t.Errorf("encoding not handled correctly: expected %q got %q", r, m) + } + m = Map(encode, r) + if m != s { + t.Errorf("encoding not handled correctly: expected %q got %q", s, m) + } } func TestToUpper(t *testing.T) { runStringTests(t, ToUpper, "ToUpper", upperTests) } diff --git a/src/syscall/syscall_linux_amd64.go b/src/syscall/syscall_linux_amd64.go index 4b4aa6d414..78c3e8f05a 100644 --- a/src/syscall/syscall_linux_amd64.go +++ b/src/syscall/syscall_linux_amd64.go @@ -39,7 +39,6 @@ const ( //sysnb Setreuid(ruid int, euid int) (err error) //sys Shutdown(fd int, how int) (err error) //sys Splice(rfd int, roff *int64, wfd int, woff *int64, len int, flags int) (n int64, err error) -//sys Stat(path string, stat *Stat_t) (err error) //sys Statfs(path string, buf *Statfs_t) (err error) //sys SyncFileRange(fd int, off int64, n int64, flags int) (err error) //sys Truncate(path string, length int64) (err error) @@ -47,6 +46,7 @@ const ( //sys accept4(s int, rsa *RawSockaddrAny, addrlen *_Socklen, flags int) (fd int, err error) //sys bind(s int, addr unsafe.Pointer, addrlen _Socklen) (err error) //sys connect(s int, addr unsafe.Pointer, addrlen _Socklen) (err error) +//sys fstatat(fd int, path string, stat *Stat_t, flags int) (err error) = SYS_NEWFSTATAT //sysnb getgroups(n int, list *_Gid_t) (nn int, err error) //sysnb setgroups(n int, list *_Gid_t) (err error) //sys getsockopt(s int, level int, name int, val unsafe.Pointer, vallen *_Socklen) (err error) @@ -61,6 +61,10 @@ const ( //sys sendmsg(s int, msg *Msghdr, flags int) (n int, err error) //sys mmap(addr uintptr, length uintptr, prot int, flags int, fd int, offset int64) (xaddr uintptr, err error) +func Stat(path string, stat *Stat_t) (err error) { + return fstatat(_AT_FDCWD, path, stat, 0) +} + //go:noescape func gettimeofday(tv *Timeval) (err Errno) diff --git a/src/syscall/zsyscall_linux_amd64.go b/src/syscall/zsyscall_linux_amd64.go index 6545d1a159..47ea597443 100644 --- a/src/syscall/zsyscall_linux_amd64.go +++ b/src/syscall/zsyscall_linux_amd64.go @@ -1476,21 +1476,6 @@ func Splice(rfd int, roff *int64, wfd int, woff *int64, len int, flags int) (n i // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func Stat(path string, stat *Stat_t) (err error) { - var _p0 *byte - _p0, err = BytePtrFromString(path) - if err != nil { - return - } - _, _, e1 := Syscall(SYS_STAT, uintptr(unsafe.Pointer(_p0)), uintptr(unsafe.Pointer(stat)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Statfs(path string, buf *Statfs_t) (err error) { var _p0 *byte _p0, err = BytePtrFromString(path) @@ -1573,6 +1558,21 @@ func connect(s int, addr unsafe.Pointer, addrlen _Socklen) (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT +func fstatat(fd int, path string, stat *Stat_t, flags int) (err error) { + var _p0 *byte + _p0, err = BytePtrFromString(path) + if err != nil { + return + } + _, _, e1 := Syscall6(SYS_NEWFSTATAT, uintptr(fd), uintptr(unsafe.Pointer(_p0)), uintptr(unsafe.Pointer(stat)), uintptr(flags), 0, 0) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + func getgroups(n int, list *_Gid_t) (nn int, err error) { r0, _, e1 := RawSyscall(SYS_GETGROUPS, uintptr(n), uintptr(unsafe.Pointer(list)), 0) nn = int(r0) diff --git a/test/fixedbugs/issue23812.go b/test/fixedbugs/issue23812.go new file mode 100644 index 0000000000..0a40deb212 --- /dev/null +++ b/test/fixedbugs/issue23812.go @@ -0,0 +1,34 @@ +// run + +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import "fmt" + +func main() { + want := int32(0x3edae8) + got := foo(1) + if want != got { + panic(fmt.Sprintf("want %x, got %x", want, got)) + } +} + +func foo(a int32) int32 { + return shr1(int32(shr2(int64(0x14ff6e2207db5d1f), int(a))), 4) +} + +func shr1(n int32, m int) int32 { return n >> uint(m) } + +func shr2(n int64, m int) int64 { + if m < 0 { + m = -m + } + if m >= 64 { + return n + } + + return n >> uint(m) +} diff --git a/test/fixedbugs/issue24449.go b/test/fixedbugs/issue24449.go new file mode 100644 index 0000000000..b236645103 --- /dev/null +++ b/test/fixedbugs/issue24449.go @@ -0,0 +1,62 @@ +// run + +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "sync/atomic" +) + +var cnt32 int32 + +//go:noinline +func test32(a, b []int) bool { + // Try to generate flag value, issue atomic + // adds and then re-use the flag value to see if + // the atomic add has clobbered them. + atomic.AddInt32(&cnt32, 1) + if len(a) == len(b) { + atomic.AddInt32(&cnt32, 2) + } + atomic.AddInt32(&cnt32, 4) + if len(a) >= len(b) { + atomic.AddInt32(&cnt32, 8) + } + if len(a) <= len(b) { + atomic.AddInt32(&cnt32, 16) + } + return atomic.LoadInt32(&cnt32) == 31 +} + +var cnt64 int64 + +//go:noinline +func test64(a, b []int) bool { + // Try to generate flag value, issue atomic + // adds and then re-use the flag value to see if + // the atomic add has clobbered them. + atomic.AddInt64(&cnt64, 1) + if len(a) == len(b) { + atomic.AddInt64(&cnt64, 2) + } + atomic.AddInt64(&cnt64, 4) + if len(a) >= len(b) { + atomic.AddInt64(&cnt64, 8) + } + if len(a) <= len(b) { + atomic.AddInt64(&cnt64, 16) + } + return atomic.LoadInt64(&cnt64) == 31 +} + +func main() { + if !test32([]int{}, []int{}) { + panic("test32") + } + if !test64([]int{}, []int{}) { + panic("test64") + } +} diff --git a/test/fixedbugs/issue24817.go b/test/fixedbugs/issue24817.go new file mode 100644 index 0000000000..ba2a138ed3 --- /dev/null +++ b/test/fixedbugs/issue24817.go @@ -0,0 +1,64 @@ +// run + +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Check all ways to compare a non-constant string to the empty string. + +package main + +import ( + "fmt" + "os" +) + +var ( + s = "abc" + e = "" + failed bool +) + +func main() { + want(true, "" < s, `"" < s`) + want(false, s < "", `s < ""`) + want(false, "" < e, `"" < e`) + want(false, e < "", `e < ""`) + + want(true, "" <= s, `"" <= s`) + want(false, s <= "", `s <= ""`) + want(true, "" <= e, `"" <= e`) + want(true, e <= "", `e <= ""`) + + want(false, "" > s, `"" > s`) + want(true, s > "", `s > ""`) + want(false, "" > e, `"" > e`) + want(false, e > "", `e > ""`) + + want(false, "" >= s, `"" >= s`) + want(true, s >= "", `s >= ""`) + want(true, "" >= e, `"" >= e`) + want(true, e >= "", `e >= ""`) + + want(false, "" == s, `"" == s`) + want(false, s == "", `s == ""`) + want(true, "" == e, `"" == e`) + want(true, e == "", `e == ""`) + + want(true, "" != s, `"" != s`) + want(true, s != "", `s != ""`) + want(false, "" != e, `"" != e`) + want(false, e != "", `e != ""`) + + if failed { + os.Exit(1) + } +} + +//go:noinline +func want(b bool, have bool, msg string) { + if b != have { + fmt.Println(msg) + failed = true + } +} |