[dev.regabi] cmd/compile,cmd/link: initial support for ABI wrappers

Add compiler support for emitting ABI wrappers by creating real IR as
opposed to introducing ABI aliases. At the moment these are "no-op"
wrappers in the sense that they make a simple call (using the existing
ABI) to their target. The assumption here is that once late call
expansion can handle both ABI0 and the "new" ABIInternal (register
version), it can expand the call to do the right thing.

Note that the runtime contains functions that do not strictly follow
the rules of the current Go ABI0; this has been handled in most cases
by treating these as ABIInternal instead (these changes have been made
in previous patches).

Generation of ABI wrappers (as opposed to ABI aliases) is currently
gated by GOEXPERIMENT=regabi -- wrapper generation is on by default if
GOEXPERIMENT=regabi is set and off otherwise (but can be turned on
using "-gcflags=all=-abiwrap -ldflags=-abiwrap"). Wrapper generation
currently only workd on AMD64; explicitly enabling wrapper for other
architectures (via the command line) is not supported.

Also in this patch are a few other command line options for debugging
(tracing and/or limiting wrapper creation). These will presumably go
away at some point.

Updates #27539, #40724.

Change-Id: I1ee3226fc15a3c32ca2087b8ef8e41dbe6df4a75
Reviewed-on: https://go-review.googlesource.com/c/go/+/270863
Run-TryBot: Than McIntosh <thanm@google.com>
TryBot-Result: Go Bot <gobot@golang.org>
Reviewed-by: Cherry Zhang <cherryyz@google.com>
Trust: Than McIntosh <thanm@google.com>

15 files changed, 320 insertions, 43 deletions

diff --git a/src/cmd/compile/internal/base/debug.go b/src/cmd/compile/internal/base/debug.go
index 45a552a4d9..3acdcea846 100644
--- a/src/cmd/compile/internal/base/debug.go
+++ b/src/cmd/compile/internal/base/debug.go
@@ -51,6 +51,7 @@ type DebugFlags struct {
 	TypeAssert    int    `help:"print information about type assertion inlining"`
 	TypecheckInl  int    `help:"eager typechecking of inline function bodies"`
 	WB            int    `help:"print information about write barriers"`
+	ABIWrap       int    `help:"print information about ABI wrapper generation"`
 
 	any bool // set when any of the values have been set
 }
diff --git a/src/cmd/compile/internal/base/flag.go b/src/cmd/compile/internal/base/flag.go
index aadc70f496..ce87ff730e 100644
--- a/src/cmd/compile/internal/base/flag.go
+++ b/src/cmd/compile/internal/base/flag.go
@@ -81,6 +81,8 @@ type CmdFlags struct {
 	CompilingRuntime bool "flag:\"+\" help:\"compiling runtime\""
 
 	// Longer names
+	ABIWrap            bool         "help:\"enable generation of ABI wrappers\""
+	ABIWrapLimit       int          "help:\"emit at most N ABI wrappers (for debugging)\""
 	AsmHdr             string       "help:\"write assembly header to `file`\""
 	Bench              string       "help:\"append benchmark times to `file`\""
 	BlockProfile       string       "help:\"write block profile to `file`\""
@@ -140,6 +142,7 @@ func ParseFlags() {
 	Flag.LowerP = &Ctxt.Pkgpath
 	Flag.LowerV = &Ctxt.Debugvlog
 
+	Flag.ABIWrap = objabi.Regabi_enabled != 0
 	Flag.Dwarf = objabi.GOARCH != "wasm"
 	Flag.DwarfBASEntries = &Ctxt.UseBASEntries
 	Flag.DwarfLocationLists = &Ctxt.Flag_locationlists
diff --git a/src/cmd/compile/internal/gc/gsubr.go b/src/cmd/compile/internal/gc/gsubr.go
index ddb431d5ab..f3ef14c99b 100644
--- a/src/cmd/compile/internal/gc/gsubr.go
+++ b/src/cmd/compile/internal/gc/gsubr.go
@@ -34,9 +34,12 @@ import (
 	"cmd/compile/internal/base"
 	"cmd/compile/internal/ir"
 	"cmd/compile/internal/ssa"
+	"cmd/compile/internal/types"
 	"cmd/internal/obj"
 	"cmd/internal/objabi"
 	"cmd/internal/src"
+	"fmt"
+	"os"
 )
 
 var sharedProgArray = new([10000]obj.Prog) // *T instead of T to work around issue 19839
@@ -187,32 +190,154 @@ func (pp *Progs) settext(fn *ir.Func) {
 	ptxt.From.Sym = fn.LSym
 }
 
+// makeABIWrapper creates a new function that wraps a cross-ABI call
+// to "f".  The wrapper is marked as an ABIWRAPPER.
+func makeABIWrapper(f *ir.Func, wrapperABI obj.ABI) {
+
+	// Q: is this needed?
+	savepos := base.Pos
+	savedclcontext := dclcontext
+	savedcurfn := Curfn
+
+	base.Pos = autogeneratedPos
+	dclcontext = ir.PEXTERN
+
+	// At the moment we don't support wrapping a method, we'd need machinery
+	// below to handle the receiver. Panic if we see this scenario.
+	ft := f.Nname.Ntype.Type()
+	if ft.NumRecvs() != 0 {
+		panic("makeABIWrapper support for wrapping methods not implemented")
+	}
+
+	// Manufacture a new func type to use for the wrapper.
+	var noReceiver *ir.Field
+	tfn := ir.NewFuncType(base.Pos,
+		noReceiver,
+		structargs(ft.Params(), true),
+		structargs(ft.Results(), false))
+
+	// Reuse f's types.Sym to create a new ODCLFUNC/function.
+	fn := dclfunc(f.Nname.Sym(), tfn)
+	fn.SetDupok(true)
+	fn.SetWrapper(true) // ignore frame for panic+recover matching
+
+	// Select LSYM now.
+	asym := base.Ctxt.LookupABI(f.LSym.Name, wrapperABI)
+	asym.Type = objabi.STEXT
+	if fn.LSym != nil {
+		panic("unexpected")
+	}
+	fn.LSym = asym
+
+	// ABI0-to-ABIInternal wrappers will be mainly loading params from
+	// stack into registers (and/or storing stack locations back to
+	// registers after the wrapped call); in most cases they won't
+	// need to allocate stack space, so it should be OK to mark them
+	// as NOSPLIT in these cases. In addition, my assumption is that
+	// functions written in assembly are NOSPLIT in most (but not all)
+	// cases. In the case of an ABIInternal target that has too many
+	// parameters to fit into registers, the wrapper would need to
+	// allocate stack space, but this seems like an unlikely scenario.
+	// Hence: mark these wrappers NOSPLIT.
+	//
+	// ABIInternal-to-ABI0 wrappers on the other hand will be taking
+	// things in registers and pushing them onto the stack prior to
+	// the ABI0 call, meaning that they will always need to allocate
+	// stack space. If the compiler marks them as NOSPLIT this seems
+	// as though it could lead to situations where the the linker's
+	// nosplit-overflow analysis would trigger a link failure. On the
+	// other hand if they not tagged NOSPLIT then this could cause
+	// problems when building the runtime (since there may be calls to
+	// asm routine in cases where it's not safe to grow the stack). In
+	// most cases the wrapper would be (in effect) inlined, but are
+	// there (perhaps) indirect calls from the runtime that could run
+	// into trouble here.
+	// FIXME: at the moment all.bash does not pass when I leave out
+	// NOSPLIT for these wrappers, so all are currently tagged with NOSPLIT.
+	setupTextLSym(fn, obj.NOSPLIT|obj.ABIWRAPPER)
+
+	// Generate call. Use tail call if no params and no returns,
+	// but a regular call otherwise.
+	//
+	// Note: ideally we would be using a tail call in cases where
+	// there are params but no returns for ABI0->ABIInternal wrappers,
+	// provided that all params fit into registers (e.g. we don't have
+	// to allocate any stack space). Doing this will require some
+	// extra work in typecheck/walk/ssa, might want to add a new node
+	// OTAILCALL or something to this effect.
+	var call ir.Node
+	if tfn.Type().NumResults() == 0 && tfn.Type().NumParams() == 0 && tfn.Type().NumRecvs() == 0 {
+		call = nodSym(ir.ORETJMP, nil, f.Nname.Sym())
+	} else {
+		call = ir.Nod(ir.OCALL, f.Nname, nil)
+		call.PtrList().Set(paramNnames(tfn.Type()))
+		call.SetIsDDD(tfn.Type().IsVariadic())
+		if tfn.Type().NumResults() > 0 {
+			n := ir.Nod(ir.ORETURN, nil, nil)
+			n.PtrList().Set1(call)
+			call = n
+		}
+	}
+	fn.PtrBody().Append(call)
+
+	funcbody()
+	if base.Debug.DclStack != 0 {
+		testdclstack()
+	}
+
+	typecheckFunc(fn)
+	Curfn = fn
+	typecheckslice(fn.Body().Slice(), ctxStmt)
+
+	escapeFuncs([]*ir.Func{fn}, false)
+
+	Target.Decls = append(Target.Decls, fn)
+
+	// Restore previous context.
+	base.Pos = savepos
+	dclcontext = savedclcontext
+	Curfn = savedcurfn
+}
+
 // initLSym defines f's obj.LSym and initializes it based on the
 // properties of f. This includes setting the symbol flags and ABI and
 // creating and initializing related DWARF symbols.
 //
 // initLSym must be called exactly once per function and must be
 // called for both functions with bodies and functions without bodies.
+// For body-less functions, we only create the LSym; for functions
+// with bodies call a helper to setup up / populate the LSym.
 func initLSym(f *ir.Func, hasBody bool) {
+	// FIXME: for new-style ABI wrappers, we set up the lsym at the
+	// point the wrapper is created.
+	if f.LSym != nil && base.Flag.ABIWrap {
+		return
+	}
+	selectLSym(f, hasBody)
+	if hasBody {
+		setupTextLSym(f, 0)
+	}
+}
+
+// selectLSym sets up the LSym for a given function, and
+// makes calls to helpers to create ABI wrappers if needed.
+func selectLSym(f *ir.Func, hasBody bool) {
 	if f.LSym != nil {
 		base.Fatalf("Func.initLSym called twice")
 	}
 
 	if nam := f.Nname; !ir.IsBlank(nam) {
-		f.LSym = nam.Sym().Linksym()
-		if f.Pragma&ir.Systemstack != 0 {
-			f.LSym.Set(obj.AttrCFunc, true)
-		}
 
-		var aliasABI obj.ABI
-		needABIAlias := false
-		defABI, hasDefABI := symabiDefs[f.LSym.Name]
+		var wrapperABI obj.ABI
+		needABIWrapper := false
+		defABI, hasDefABI := symabiDefs[nam.Sym().LinksymName()]
 		if hasDefABI && defABI == obj.ABI0 {
 			// Symbol is defined as ABI0. Create an
 			// Internal -> ABI0 wrapper.
-			f.LSym.SetABI(obj.ABI0)
-			needABIAlias, aliasABI = true, obj.ABIInternal
+			f.LSym = nam.Sym().LinksymABI0()
+			needABIWrapper, wrapperABI = true, obj.ABIInternal
 		} else {
+			f.LSym = nam.Sym().Linksym()
 			// No ABI override. Check that the symbol is
 			// using the expected ABI.
 			want := obj.ABIInternal
@@ -220,6 +345,9 @@ func initLSym(f *ir.Func, hasBody bool) {
 				base.Fatalf("function symbol %s has the wrong ABI %v, expected %v", f.LSym.Name, f.LSym.ABI(), want)
 			}
 		}
+		if f.Pragma&ir.Systemstack != 0 {
+			f.LSym.Set(obj.AttrCFunc, true)
+		}
 
 		isLinknameExported := nam.Sym().Linkname != "" && (hasBody || hasDefABI)
 		if abi, ok := symabiRefs[f.LSym.Name]; (ok && abi == obj.ABI0) || isLinknameExported {
@@ -235,32 +363,39 @@ func initLSym(f *ir.Func, hasBody bool) {
 			// using linkname and we don't want to create
 			// duplicate ABI wrappers.
 			if f.LSym.ABI() != obj.ABI0 {
-				needABIAlias, aliasABI = true, obj.ABI0
+				needABIWrapper, wrapperABI = true, obj.ABI0
 			}
 		}
 
-		if needABIAlias {
-			// These LSyms have the same name as the
-			// native function, so we create them directly
-			// rather than looking them up. The uniqueness
-			// of f.lsym ensures uniqueness of asym.
-			asym := &obj.LSym{
-				Name: f.LSym.Name,
-				Type: objabi.SABIALIAS,
-				R:    []obj.Reloc{{Sym: f.LSym}}, // 0 size, so "informational"
+		if needABIWrapper {
+			if !useABIWrapGen(f) {
+				// Fallback: use alias instead. FIXME.
+
+				// These LSyms have the same name as the
+				// native function, so we create them directly
+				// rather than looking them up. The uniqueness
+				// of f.lsym ensures uniqueness of asym.
+				asym := &obj.LSym{
+					Name: f.LSym.Name,
+					Type: objabi.SABIALIAS,
+					R:    []obj.Reloc{{Sym: f.LSym}}, // 0 size, so "informational"
+				}
+				asym.SetABI(wrapperABI)
+				asym.Set(obj.AttrDuplicateOK, true)
+				base.Ctxt.ABIAliases = append(base.Ctxt.ABIAliases, asym)
+			} else {
+				if base.Debug.ABIWrap != 0 {
+					fmt.Fprintf(os.Stderr, "=-= %v to %v wrapper for %s.%s\n",
+						wrapperABI, 1-wrapperABI, types.LocalPkg.Path, f.LSym.Name)
+				}
+				makeABIWrapper(f, wrapperABI)
 			}
-			asym.SetABI(aliasABI)
-			asym.Set(obj.AttrDuplicateOK, true)
-			base.Ctxt.ABIAliases = append(base.Ctxt.ABIAliases, asym)
 		}
 	}
+}
 
-	if !hasBody {
-		// For body-less functions, we only create the LSym.
-		return
-	}
-
-	var flag int
+// setupTextLsym initializes the LSym for a with-body text symbol.
+func setupTextLSym(f *ir.Func, flag int) {
 	if f.Dupok() {
 		flag |= obj.DUPOK
 	}
diff --git a/src/cmd/compile/internal/gc/main.go b/src/cmd/compile/internal/gc/main.go
index 7f7cd63cdf..de2b3db36a 100644
--- a/src/cmd/compile/internal/gc/main.go
+++ b/src/cmd/compile/internal/gc/main.go
@@ -1144,3 +1144,26 @@ func initializeTypesPackage() {
 
 	initUniverse()
 }
+
+// useNewABIWrapGen returns TRUE if the compiler should generate an
+// ABI wrapper for the function 'f'.
+func useABIWrapGen(f *ir.Func) bool {
+	if !base.Flag.ABIWrap {
+		return false
+	}
+
+	// Support limit option for bisecting.
+	if base.Flag.ABIWrapLimit == 1 {
+		return false
+	}
+	if base.Flag.ABIWrapLimit < 1 {
+		return true
+	}
+	base.Flag.ABIWrapLimit--
+	if base.Debug.ABIWrap != 0 && base.Flag.ABIWrapLimit == 1 {
+		fmt.Fprintf(os.Stderr, "=-= limit reached after new wrapper for %s\n",
+			f.LSym.Name)
+	}
+
+	return true
+}
diff --git a/src/cmd/compile/internal/gc/pgen.go b/src/cmd/compile/internal/gc/pgen.go
index 5b5288c389..dae9d79147 100644
--- a/src/cmd/compile/internal/gc/pgen.go
+++ b/src/cmd/compile/internal/gc/pgen.go
@@ -32,7 +32,6 @@ func emitptrargsmap(fn *ir.Func) {
 		return
 	}
 	lsym := base.Ctxt.Lookup(fn.LSym.Name + ".args_stackmap")
-
 	nptr := int(fn.Type().ArgWidth() / int64(Widthptr))
 	bv := bvalloc(int32(nptr) * 2)
 	nbitmap := 1
@@ -399,7 +398,11 @@ func debuginfo(fnsym *obj.LSym, infosym *obj.LSym, curfn interface{}) ([]dwarf.S
 	fn := curfn.(*ir.Func)
 
 	if fn.Nname != nil {
-		if expect := fn.Sym().Linksym(); fnsym != expect {
+		expect := fn.Sym().Linksym()
+		if fnsym.ABI() == obj.ABI0 {
+			expect = fn.Sym().LinksymABI0()
+		}
+		if fnsym != expect {
 			base.Fatalf("unexpected fnsym: %v != %v", fnsym, expect)
 		}
 	}
diff --git a/src/cmd/compile/internal/gc/racewalk.go b/src/cmd/compile/internal/gc/racewalk.go
index 472deb16e3..61a65368af 100644
--- a/src/cmd/compile/internal/gc/racewalk.go
+++ b/src/cmd/compile/internal/gc/racewalk.go
@@ -61,7 +61,7 @@ func ispkgin(pkgs []string) bool {
 }
 
 func instrument(fn *ir.Func) {
-	if fn.Pragma&ir.Norace != 0 {
+	if fn.Pragma&ir.Norace != 0 || (fn.Sym().Linksym() != nil && fn.Sym().Linksym().ABIWrapper()) {
 		return
 	}
 
diff --git a/src/cmd/compile/internal/gc/ssa.go b/src/cmd/compile/internal/gc/ssa.go
index a5340e7f11..b4cf8b6dc7 100644
--- a/src/cmd/compile/internal/gc/ssa.go
+++ b/src/cmd/compile/internal/gc/ssa.go
@@ -1421,7 +1421,7 @@ func (s *state) stmt(n ir.Node) {
 	case ir.ORETJMP:
 		b := s.exit()
 		b.Kind = ssa.BlockRetJmp // override BlockRet
-		b.Aux = n.Sym().Linksym()
+		b.Aux = callTargetLSym(n.Sym(), s.curfn.LSym)
 
 	case ir.OCONTINUE, ir.OBREAK:
 		var to *ssa.Block
@@ -4826,11 +4826,11 @@ func (s *state) call(n *ir.CallExpr, k callKind, returnResultAddr bool) *ssa.Val
 			}
 		case sym != nil:
 			if testLateExpansion {
-				aux := ssa.StaticAuxCall(sym.Linksym(), ACArgs, ACResults)
+				aux := ssa.StaticAuxCall(callTargetLSym(sym, s.curfn.LSym), ACArgs, ACResults)
 				call = s.newValue0A(ssa.OpStaticLECall, aux.LateExpansionResultType(), aux)
 				call.AddArgs(callArgs...)
 			} else {
-				call = s.newValue1A(ssa.OpStaticCall, types.TypeMem, ssa.StaticAuxCall(sym.Linksym(), ACArgs, ACResults), s.mem())
+				call = s.newValue1A(ssa.OpStaticCall, types.TypeMem, ssa.StaticAuxCall(callTargetLSym(sym, s.curfn.LSym), ACArgs, ACResults), s.mem())
 			}
 		default:
 			s.Fatalf("bad call type %v %v", n.Op(), n)
@@ -7291,3 +7291,46 @@ func clobberBase(n ir.Node) ir.Node {
 	}
 	return n
 }
+
+// callTargetLSym determines the correct LSym for 'callee' when called
+// from function 'caller'. There are a couple of different scenarios
+// to contend with here:
+//
+// 1. if 'caller' is an ABI wrapper, then we always want to use the
+//    LSym from the Func for the callee.
+//
+// 2. if 'caller' is not an ABI wrapper, then we looked at the callee
+//    to see if it corresponds to a "known" ABI0 symbol (e.g. assembly
+//    routine defined in the current package); if so, we want the call to
+//    directly target the ABI0 symbol (effectively bypassing the
+//    ABIInternal->ABI0 wrapper for 'callee').
+//
+// 3. in all other cases, want the regular ABIInternal linksym
+//
+func callTargetLSym(callee *types.Sym, callerLSym *obj.LSym) *obj.LSym {
+	lsym := callee.Linksym()
+	if !base.Flag.ABIWrap {
+		return lsym
+	}
+	if ir.AsNode(callee.Def) == nil {
+		return lsym
+	}
+	ndclfunc := ir.AsNode(callee.Def).Name().Defn
+	if ndclfunc == nil {
+		return lsym
+	}
+	// check for case 1 above
+	if callerLSym.ABIWrapper() {
+		if nlsym := ndclfunc.Func().LSym; nlsym != nil {
+			lsym = nlsym
+		}
+	} else {
+		// check for case 2 above
+		nam := ndclfunc.Func().Nname
+		defABI, hasDefABI := symabiDefs[nam.Sym().LinksymName()]
+		if hasDefABI && defABI == obj.ABI0 {
+			lsym = nam.Sym().LinksymABI0()
+		}
+	}
+	return lsym
+}
diff --git a/src/cmd/compile/internal/types/sym.go b/src/cmd/compile/internal/types/sym.go
index 19f06fcf5b..c512e3a003 100644
--- a/src/cmd/compile/internal/types/sym.go
+++ b/src/cmd/compile/internal/types/sym.go
@@ -93,6 +93,23 @@ func (sym *Sym) Linksym() *obj.LSym {
 	return base.Ctxt.LookupInit(sym.LinksymName(), initPkg)
 }
 
+// LinksymABI0 looks up or creates an ABI0 linker symbol for "sym",
+// in cases where we want to specifically select the ABI0 version of
+// a symbol (typically used only for ABI wrappers).
+func (sym *Sym) LinksymABI0() *obj.LSym {
+	if sym == nil {
+		return nil
+	}
+	initPkg := func(r *obj.LSym) {
+		if sym.Linkname != "" {
+			r.Pkg = "_"
+		} else {
+			r.Pkg = sym.Pkg.Prefix
+		}
+	}
+	return base.Ctxt.LookupABIInit(sym.LinksymName(), obj.ABI0, initPkg)
+}
+
 // Less reports whether symbol a is ordered before symbol b.
 //
 // Symbols are ordered exported before non-exported, then by name, and
diff --git a/src/cmd/internal/obj/link.go b/src/cmd/internal/obj/link.go
index 7b5c990a5d..977c5c3303 100644
--- a/src/cmd/internal/obj/link.go
+++ b/src/cmd/internal/obj/link.go
@@ -635,6 +635,10 @@ const (
 	// ContentAddressable indicates this is a content-addressable symbol.
 	AttrContentAddressable
 
+	// ABI wrapper is set for compiler-generated text symbols that
+	// convert between ABI0 and ABIInternal calling conventions.
+	AttrABIWrapper
+
 	// attrABIBase is the value at which the ABI is encoded in
 	// Attribute. This must be last; all bits after this are
 	// assumed to be an ABI value.
@@ -660,6 +664,7 @@ func (a Attribute) TopFrame() bool           { return a&AttrTopFrame != 0 }
 func (a Attribute) Indexed() bool            { return a&AttrIndexed != 0 }
 func (a Attribute) UsedInIface() bool        { return a&AttrUsedInIface != 0 }
 func (a Attribute) ContentAddressable() bool { return a&AttrContentAddressable != 0 }
+func (a Attribute) ABIWrapper() bool         { return a&AttrABIWrapper != 0 }
 
 func (a *Attribute) Set(flag Attribute, value bool) {
 	if value {
@@ -695,6 +700,7 @@ var textAttrStrings = [...]struct {
 	{bit: AttrTopFrame, s: "TOPFRAME"},
 	{bit: AttrIndexed, s: ""},
 	{bit: AttrContentAddressable, s: ""},
+	{bit: AttrABIWrapper, s: "ABIWRAPPER"},
 }
 
 // TextAttrString formats a for printing in as part of a TEXT prog.
diff --git a/src/cmd/internal/obj/plist.go b/src/cmd/internal/obj/plist.go
index 2b096996f7..679ce7eb8f 100644
--- a/src/cmd/internal/obj/plist.go
+++ b/src/cmd/internal/obj/plist.go
@@ -80,6 +80,11 @@ func Flushplist(ctxt *Link, plist *Plist, newprog ProgAlloc, myimportpath string
 		if !strings.HasPrefix(s.Name, "\"\".") {
 			continue
 		}
+		if s.ABIWrapper() {
+			// Don't create an args_stackmap symbol reference for an ABI
+			// wrapper function
+			continue
+		}
 		found := false
 		for p := s.Func().Text; p != nil; p = p.Link {
 			if p.As == AFUNCDATA && p.From.Type == TYPE_CONST && p.From.Offset == objabi.FUNCDATA_ArgsPointerMaps {
@@ -134,6 +139,7 @@ func (ctxt *Link) InitTextSym(s *LSym, flag int) {
 	s.Set(AttrNoSplit, flag&NOSPLIT != 0)
 	s.Set(AttrReflectMethod, flag&REFLECTMETHOD != 0)
 	s.Set(AttrWrapper, flag&WRAPPER != 0)
+	s.Set(AttrABIWrapper, flag&ABIWRAPPER != 0)
 	s.Set(AttrNeedCtxt, flag&NEEDCTXT != 0)
 	s.Set(AttrNoFrame, flag&NOFRAME != 0)
 	s.Set(AttrTopFrame, flag&TOPFRAME != 0)
diff --git a/src/cmd/internal/obj/textflag.go b/src/cmd/internal/obj/textflag.go
index d2cec734b1..fcc4014aa2 100644
--- a/src/cmd/internal/obj/textflag.go
+++ b/src/cmd/internal/obj/textflag.go
@@ -51,4 +51,7 @@ const (
 	// Function is the top of the call stack. Call stack unwinders should stop
 	// at this function.
 	TOPFRAME = 2048
+
+	// Function is an ABI wrapper.
+	ABIWRAPPER = 4096
 )
diff --git a/src/cmd/internal/obj/x86/obj6.go b/src/cmd/internal/obj/x86/obj6.go
index 184fb4308b..839aeb8fe3 100644
--- a/src/cmd/internal/obj/x86/obj6.go
+++ b/src/cmd/internal/obj/x86/obj6.go
@@ -637,7 +637,7 @@ func preprocess(ctxt *obj.Link, cursym *obj.LSym, newprog obj.ProgAlloc) {
 		}
 	}
 
-	if !p.From.Sym.NoSplit() || p.From.Sym.Wrapper() {
+	if !p.From.Sym.NoSplit() || (p.From.Sym.Wrapper() && !p.From.Sym.ABIWrapper()) {
 		p = obj.Appendp(p, newprog)
 		p = load_g_cx(ctxt, p, newprog) // load g into CX
 	}
@@ -690,7 +690,7 @@ func preprocess(ctxt *obj.Link, cursym *obj.LSym, newprog obj.ProgAlloc) {
 		p.To.Reg = REG_BP
 	}
 
-	if cursym.Func().Text.From.Sym.Wrapper() {
+	if cursym.Func().Text.From.Sym.Wrapper() && !cursym.Func().Text.From.Sym.ABIWrapper() {
 		// if g._panic != nil && g._panic.argp == FP {
 		//   g._panic.argp = bottom-of-frame
 		// }
diff --git a/src/cmd/link/internal/ld/main.go b/src/cmd/link/internal/ld/main.go
index 5c8293810f..1420030eec 100644
--- a/src/cmd/link/internal/ld/main.go
+++ b/src/cmd/link/internal/ld/main.go
@@ -92,11 +92,10 @@ var (
 	FlagRound         = flag.Int("R", -1, "set address rounding `quantum`")
 	FlagTextAddr      = flag.Int64("T", -1, "set text segment `address`")
 	flagEntrySymbol   = flag.String("E", "", "set `entry` symbol name")
-
-	cpuprofile     = flag.String("cpuprofile", "", "write cpu profile to `file`")
-	memprofile     = flag.String("memprofile", "", "write memory profile to `file`")
-	memprofilerate = flag.Int64("memprofilerate", 0, "set runtime.MemProfileRate to `rate`")
-
+	cpuprofile        = flag.String("cpuprofile", "", "write cpu profile to `file`")
+	memprofile        = flag.String("memprofile", "", "write memory profile to `file`")
+	memprofilerate    = flag.Int64("memprofilerate", 0, "set runtime.MemProfileRate to `rate`")
+	flagAbiWrap       = false
 	benchmarkFlag     = flag.String("benchmark", "", "set to 'mem' or 'cpu' to enable phase benchmarking")
 	benchmarkFileFlag = flag.String("benchmarkprofile", "", "emit phase profiles to `base`_phase.{cpu,mem}prof")
 )
@@ -135,6 +134,9 @@ func Main(arch *sys.Arch, theArch Arch) {
 	objabi.Flagfn1("X", "add string value `definition` of the form importpath.name=value", func(s string) { addstrdata1(ctxt, s) })
 	objabi.Flagcount("v", "print link trace", &ctxt.Debugvlog)
 	objabi.Flagfn1("importcfg", "read import configuration from `file`", ctxt.readImportCfg)
+	if objabi.Regabi_enabled != 0 {
+		flag.BoolVar(&flagAbiWrap, "abiwrap", true, "support ABI wrapper functions")
+	}
 
 	objabi.Flagparse(usage)
 
diff --git a/src/cmd/link/internal/ld/symtab.go b/src/cmd/link/internal/ld/symtab.go
index c98e4de03f..3b709baf75 100644
--- a/src/cmd/link/internal/ld/symtab.go
+++ b/src/cmd/link/internal/ld/symtab.go
@@ -102,6 +102,41 @@ func putelfsym(ctxt *Link, x loader.Sym, typ elf.SymType, curbind elf.SymBind) {
 		elfshnum = xosect.Elfsect.(*ElfShdr).shnum
 	}
 
+	sname := ldr.SymExtname(x)
+
+	// For functions with ABI wrappers, we have to make sure that we
+	// don't wind up with two elf symbol table entries with the same
+	// name (since this will generated an error from the external
+	// linker). In the CgoExportStatic case, we want the ABI0 symbol
+	// to have the primary symbol table entry (since it's going to be
+	// called from C), so we rename the ABIInternal symbol. In all
+	// other cases, we rename the ABI0 symbol, since we want
+	// cross-load-module calls to target ABIInternal.
+	//
+	// TODO: generalize this for non-ELF (put the rename code in the
+	// loader, and store the rename result in SymExtname).
+	//
+	// TODO: avoid the ldr.Lookup calls below by instead using an aux
+	// sym or marker relocation to associate the wrapper with the
+	// wrapped function.
+	//
+	if flagAbiWrap {
+		if !ldr.IsExternal(x) && ldr.SymType(x) == sym.STEXT {
+			// First case
+			if ldr.SymVersion(x) == sym.SymVerABIInternal {
+				if s2 := ldr.Lookup(sname, sym.SymVerABI0); s2 != 0 && ldr.AttrCgoExportStatic(s2) && ldr.SymType(s2) == sym.STEXT {
+					sname = sname + ".abiinternal"
+				}
+			}
+			// Second case
+			if ldr.SymVersion(x) == sym.SymVerABI0 && !ldr.AttrCgoExportStatic(x) {
+				if s2 := ldr.Lookup(sname, sym.SymVerABIInternal); s2 != 0 && ldr.SymType(s2) == sym.STEXT {
+					sname = sname + ".abi0"
+				}
+			}
+		}
+	}
+
 	// One pass for each binding: elf.STB_LOCAL, elf.STB_GLOBAL,
 	// maybe one day elf.STB_WEAK.
 	bind := elf.STB_GLOBAL
@@ -140,8 +175,6 @@ func putelfsym(ctxt *Link, x loader.Sym, typ elf.SymType, curbind elf.SymBind) {
 		other |= 3 << 5
 	}
 
-	sname := ldr.SymExtname(x)
-
 	// When dynamically linking, we create Symbols by reading the names from
 	// the symbol tables of the shared libraries and so the names need to
 	// match exactly. Tools like DTrace will have to wait for now.
diff --git a/src/runtime/textflag.h b/src/runtime/textflag.h
index daca36d948..e727208cd0 100644
--- a/src/runtime/textflag.h
+++ b/src/runtime/textflag.h
@@ -35,3 +35,5 @@
 // Function is the top of the call stack. Call stack unwinders should stop
 // at this function.
 #define TOPFRAME 2048
+// Function is an ABI wrapper.
+#define ABIWRAPPER 4096