From 6c34d2f42077bd7757c942c8d1b466366190b45a Mon Sep 17 00:00:00 2001 From: Russ Cox Date: Wed, 23 Dec 2020 00:57:10 -0500 Subject: [dev.regabi] cmd/compile: split out package ssagen [generated] [git-generate] cd src/cmd/compile/internal/gc rf ' # maxOpenDefers is declared in ssa.go but used only by walk. mv maxOpenDefers walk.go # gc.Arch -> ssagen.Arch # It is not as nice but will do for now. mv Arch ArchInfo mv thearch Arch mv Arch ArchInfo arch.go # Pull dwarf out of pgen.go. mv debuginfo declPos createDwarfVars preInliningDcls \ createSimpleVars createSimpleVar \ createComplexVars createComplexVar \ dwarf.go # Pull high-level compilation out of pgen.go, # leaving only the SSA code. mv compilequeue funccompile compile compilenow \ compileFunctions isInlinableButNotInlined \ initLSym \ compile.go mv BoundsCheckFunc GCWriteBarrierReg ssa.go mv largeStack largeStackFrames CheckLargeStacks pgen.go # All that is left in dcl.go is the nowritebarrierrecCheck mv dcl.go nowb.go # Export API and unexport non-API. mv initssaconfig InitConfig mv isIntrinsicCall IsIntrinsicCall mv ssaDumpInline DumpInline mv initSSATables InitTables mv initSSAEnv InitEnv mv compileSSA Compile mv stackOffset StackOffset mv canSSAType TypeOK mv SSAGenState State mv FwdRefAux fwdRefAux mv cgoSymABIs CgoSymABIs mv readSymABIs ReadSymABIs mv initLSym InitLSym mv useABIWrapGen symabiDefs CgoSymABIs ReadSymABIs InitLSym selectLSym makeABIWrapper setupTextLSym abi.go mv arch.go abi.go nowb.go phi.go pgen.go pgen_test.go ssa.go cmd/compile/internal/ssagen ' rm go.go gsubr.go Change-Id: I47fad6cbf1d1e583fd9139003a08401d7cd048a1 Reviewed-on: https://go-review.googlesource.com/c/go/+/279476 Trust: Russ Cox Run-TryBot: Russ Cox Reviewed-by: Matthew Dempsky --- src/cmd/compile/internal/amd64/galign.go | 4 +- src/cmd/compile/internal/amd64/ssa.go | 80 +- src/cmd/compile/internal/arm/galign.go | 6 +- src/cmd/compile/internal/arm/ssa.go | 40 +- src/cmd/compile/internal/arm64/galign.go | 6 +- src/cmd/compile/internal/arm64/ssa.go | 44 +- src/cmd/compile/internal/gc/abiutils_test.go | 15 +- src/cmd/compile/internal/gc/compile.go | 177 + src/cmd/compile/internal/gc/dcl.go | 199 - src/cmd/compile/internal/gc/dwarf.go | 412 ++ src/cmd/compile/internal/gc/go.go | 52 - src/cmd/compile/internal/gc/gsubr.go | 279 - src/cmd/compile/internal/gc/main.go | 164 +- src/cmd/compile/internal/gc/pgen.go | 804 --- src/cmd/compile/internal/gc/pgen_test.go | 208 - src/cmd/compile/internal/gc/phi.go | 556 -- src/cmd/compile/internal/gc/racewalk.go | 3 +- src/cmd/compile/internal/gc/range.go | 3 +- src/cmd/compile/internal/gc/ssa.go | 7455 ------------------------- src/cmd/compile/internal/gc/subr.go | 23 +- src/cmd/compile/internal/gc/walk.go | 23 +- src/cmd/compile/internal/mips/galign.go | 6 +- src/cmd/compile/internal/mips/ssa.go | 34 +- src/cmd/compile/internal/mips64/galign.go | 6 +- src/cmd/compile/internal/mips64/ssa.go | 32 +- src/cmd/compile/internal/ppc64/galign.go | 4 +- src/cmd/compile/internal/ppc64/ssa.go | 34 +- src/cmd/compile/internal/riscv64/galign.go | 4 +- src/cmd/compile/internal/riscv64/ssa.go | 36 +- src/cmd/compile/internal/s390x/galign.go | 4 +- src/cmd/compile/internal/s390x/ssa.go | 56 +- src/cmd/compile/internal/ssagen/abi.go | 367 ++ src/cmd/compile/internal/ssagen/arch.go | 42 + src/cmd/compile/internal/ssagen/nowb.go | 200 + src/cmd/compile/internal/ssagen/pgen.go | 279 + src/cmd/compile/internal/ssagen/pgen_test.go | 209 + src/cmd/compile/internal/ssagen/phi.go | 557 ++ src/cmd/compile/internal/ssagen/ssa.go | 7459 ++++++++++++++++++++++++++ src/cmd/compile/internal/wasm/ssa.go | 36 +- src/cmd/compile/internal/x86/galign.go | 4 +- src/cmd/compile/internal/x86/ssa.go | 64 +- src/cmd/compile/main.go | 3 +- 42 files changed, 10005 insertions(+), 9984 deletions(-) create mode 100644 src/cmd/compile/internal/gc/compile.go delete mode 100644 src/cmd/compile/internal/gc/dcl.go create mode 100644 src/cmd/compile/internal/gc/dwarf.go delete mode 100644 src/cmd/compile/internal/gc/go.go delete mode 100644 src/cmd/compile/internal/gc/gsubr.go delete mode 100644 src/cmd/compile/internal/gc/pgen.go delete mode 100644 src/cmd/compile/internal/gc/pgen_test.go delete mode 100644 src/cmd/compile/internal/gc/phi.go delete mode 100644 src/cmd/compile/internal/gc/ssa.go create mode 100644 src/cmd/compile/internal/ssagen/abi.go create mode 100644 src/cmd/compile/internal/ssagen/arch.go create mode 100644 src/cmd/compile/internal/ssagen/nowb.go create mode 100644 src/cmd/compile/internal/ssagen/pgen.go create mode 100644 src/cmd/compile/internal/ssagen/pgen_test.go create mode 100644 src/cmd/compile/internal/ssagen/phi.go create mode 100644 src/cmd/compile/internal/ssagen/ssa.go diff --git a/src/cmd/compile/internal/amd64/galign.go b/src/cmd/compile/internal/amd64/galign.go index af58440502..ce1c402902 100644 --- a/src/cmd/compile/internal/amd64/galign.go +++ b/src/cmd/compile/internal/amd64/galign.go @@ -5,13 +5,13 @@ package amd64 import ( - "cmd/compile/internal/gc" + "cmd/compile/internal/ssagen" "cmd/internal/obj/x86" ) var leaptr = x86.ALEAQ -func Init(arch *gc.Arch) { +func Init(arch *ssagen.ArchInfo) { arch.LinkArch = &x86.Linkamd64 arch.REGSP = x86.REGSP arch.MAXWIDTH = 1 << 50 diff --git a/src/cmd/compile/internal/amd64/ssa.go b/src/cmd/compile/internal/amd64/ssa.go index 0150bd296a..da355c49d1 100644 --- a/src/cmd/compile/internal/amd64/ssa.go +++ b/src/cmd/compile/internal/amd64/ssa.go @@ -9,17 +9,17 @@ import ( "math" "cmd/compile/internal/base" - "cmd/compile/internal/gc" "cmd/compile/internal/ir" "cmd/compile/internal/logopt" "cmd/compile/internal/ssa" + "cmd/compile/internal/ssagen" "cmd/compile/internal/types" "cmd/internal/obj" "cmd/internal/obj/x86" ) // markMoves marks any MOVXconst ops that need to avoid clobbering flags. -func ssaMarkMoves(s *gc.SSAGenState, b *ssa.Block) { +func ssaMarkMoves(s *ssagen.State, b *ssa.Block) { flive := b.FlagsLiveAtEnd for _, c := range b.ControlValues() { flive = c.Type.IsFlags() || flive @@ -112,7 +112,7 @@ func moveByType(t *types.Type) obj.As { // dest := dest(To) op src(From) // and also returns the created obj.Prog so it // may be further adjusted (offset, scale, etc). -func opregreg(s *gc.SSAGenState, op obj.As, dest, src int16) *obj.Prog { +func opregreg(s *ssagen.State, op obj.As, dest, src int16) *obj.Prog { p := s.Prog(op) p.From.Type = obj.TYPE_REG p.To.Type = obj.TYPE_REG @@ -166,7 +166,7 @@ func duff(size int64) (int64, int64) { return off, adj } -func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { +func ssaGenValue(s *ssagen.State, v *ssa.Value) { switch v.Op { case ssa.OpAMD64VFMADD231SD: p := s.Prog(v.Op.Asm()) @@ -632,12 +632,12 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { p.To.Type = obj.TYPE_REG p.To.Reg = o } - gc.AddAux(&p.From, v) + ssagen.AddAux(&p.From, v) case ssa.OpAMD64LEAQ, ssa.OpAMD64LEAL, ssa.OpAMD64LEAW: p := s.Prog(v.Op.Asm()) p.From.Type = obj.TYPE_MEM p.From.Reg = v.Args[0].Reg() - gc.AddAux(&p.From, v) + ssagen.AddAux(&p.From, v) p.To.Type = obj.TYPE_REG p.To.Reg = v.Reg() case ssa.OpAMD64CMPQ, ssa.OpAMD64CMPL, ssa.OpAMD64CMPW, ssa.OpAMD64CMPB, @@ -673,7 +673,7 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { p := s.Prog(v.Op.Asm()) p.From.Type = obj.TYPE_MEM p.From.Reg = v.Args[0].Reg() - gc.AddAux(&p.From, v) + ssagen.AddAux(&p.From, v) p.To.Type = obj.TYPE_REG p.To.Reg = v.Args[1].Reg() case ssa.OpAMD64CMPQconstload, ssa.OpAMD64CMPLconstload, ssa.OpAMD64CMPWconstload, ssa.OpAMD64CMPBconstload: @@ -681,20 +681,20 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { p := s.Prog(v.Op.Asm()) p.From.Type = obj.TYPE_MEM p.From.Reg = v.Args[0].Reg() - gc.AddAux2(&p.From, v, sc.Off()) + ssagen.AddAux2(&p.From, v, sc.Off()) p.To.Type = obj.TYPE_CONST p.To.Offset = sc.Val() case ssa.OpAMD64CMPQloadidx8, ssa.OpAMD64CMPQloadidx1, ssa.OpAMD64CMPLloadidx4, ssa.OpAMD64CMPLloadidx1, ssa.OpAMD64CMPWloadidx2, ssa.OpAMD64CMPWloadidx1, ssa.OpAMD64CMPBloadidx1: p := s.Prog(v.Op.Asm()) memIdx(&p.From, v) - gc.AddAux(&p.From, v) + ssagen.AddAux(&p.From, v) p.To.Type = obj.TYPE_REG p.To.Reg = v.Args[2].Reg() case ssa.OpAMD64CMPQconstloadidx8, ssa.OpAMD64CMPQconstloadidx1, ssa.OpAMD64CMPLconstloadidx4, ssa.OpAMD64CMPLconstloadidx1, ssa.OpAMD64CMPWconstloadidx2, ssa.OpAMD64CMPWconstloadidx1, ssa.OpAMD64CMPBconstloadidx1: sc := v.AuxValAndOff() p := s.Prog(v.Op.Asm()) memIdx(&p.From, v) - gc.AddAux2(&p.From, v, sc.Off()) + ssagen.AddAux2(&p.From, v, sc.Off()) p.To.Type = obj.TYPE_CONST p.To.Offset = sc.Val() case ssa.OpAMD64MOVLconst, ssa.OpAMD64MOVQconst: @@ -734,14 +734,14 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { p := s.Prog(v.Op.Asm()) p.From.Type = obj.TYPE_MEM p.From.Reg = v.Args[0].Reg() - gc.AddAux(&p.From, v) + ssagen.AddAux(&p.From, v) p.To.Type = obj.TYPE_REG p.To.Reg = v.Reg() case ssa.OpAMD64MOVBloadidx1, ssa.OpAMD64MOVWloadidx1, ssa.OpAMD64MOVLloadidx1, ssa.OpAMD64MOVQloadidx1, ssa.OpAMD64MOVSSloadidx1, ssa.OpAMD64MOVSDloadidx1, ssa.OpAMD64MOVQloadidx8, ssa.OpAMD64MOVSDloadidx8, ssa.OpAMD64MOVLloadidx8, ssa.OpAMD64MOVLloadidx4, ssa.OpAMD64MOVSSloadidx4, ssa.OpAMD64MOVWloadidx2: p := s.Prog(v.Op.Asm()) memIdx(&p.From, v) - gc.AddAux(&p.From, v) + ssagen.AddAux(&p.From, v) p.To.Type = obj.TYPE_REG p.To.Reg = v.Reg() case ssa.OpAMD64MOVQstore, ssa.OpAMD64MOVSSstore, ssa.OpAMD64MOVSDstore, ssa.OpAMD64MOVLstore, ssa.OpAMD64MOVWstore, ssa.OpAMD64MOVBstore, ssa.OpAMD64MOVOstore, @@ -753,7 +753,7 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { p.From.Reg = v.Args[1].Reg() p.To.Type = obj.TYPE_MEM p.To.Reg = v.Args[0].Reg() - gc.AddAux(&p.To, v) + ssagen.AddAux(&p.To, v) case ssa.OpAMD64MOVBstoreidx1, ssa.OpAMD64MOVWstoreidx1, ssa.OpAMD64MOVLstoreidx1, ssa.OpAMD64MOVQstoreidx1, ssa.OpAMD64MOVSSstoreidx1, ssa.OpAMD64MOVSDstoreidx1, ssa.OpAMD64MOVQstoreidx8, ssa.OpAMD64MOVSDstoreidx8, ssa.OpAMD64MOVLstoreidx8, ssa.OpAMD64MOVSSstoreidx4, ssa.OpAMD64MOVLstoreidx4, ssa.OpAMD64MOVWstoreidx2, ssa.OpAMD64ADDLmodifyidx1, ssa.OpAMD64ADDLmodifyidx4, ssa.OpAMD64ADDLmodifyidx8, ssa.OpAMD64ADDQmodifyidx1, ssa.OpAMD64ADDQmodifyidx8, @@ -765,7 +765,7 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { p.From.Type = obj.TYPE_REG p.From.Reg = v.Args[2].Reg() memIdx(&p.To, v) - gc.AddAux(&p.To, v) + ssagen.AddAux(&p.To, v) case ssa.OpAMD64ADDQconstmodify, ssa.OpAMD64ADDLconstmodify: sc := v.AuxValAndOff() off := sc.Off() @@ -788,7 +788,7 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { p := s.Prog(asm) p.To.Type = obj.TYPE_MEM p.To.Reg = v.Args[0].Reg() - gc.AddAux2(&p.To, v, off) + ssagen.AddAux2(&p.To, v, off) break } fallthrough @@ -803,7 +803,7 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { p.From.Offset = val p.To.Type = obj.TYPE_MEM p.To.Reg = v.Args[0].Reg() - gc.AddAux2(&p.To, v, off) + ssagen.AddAux2(&p.To, v, off) case ssa.OpAMD64MOVQstoreconst, ssa.OpAMD64MOVLstoreconst, ssa.OpAMD64MOVWstoreconst, ssa.OpAMD64MOVBstoreconst: p := s.Prog(v.Op.Asm()) @@ -812,7 +812,7 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { p.From.Offset = sc.Val() p.To.Type = obj.TYPE_MEM p.To.Reg = v.Args[0].Reg() - gc.AddAux2(&p.To, v, sc.Off()) + ssagen.AddAux2(&p.To, v, sc.Off()) case ssa.OpAMD64MOVQstoreconstidx1, ssa.OpAMD64MOVQstoreconstidx8, ssa.OpAMD64MOVLstoreconstidx1, ssa.OpAMD64MOVLstoreconstidx4, ssa.OpAMD64MOVWstoreconstidx1, ssa.OpAMD64MOVWstoreconstidx2, ssa.OpAMD64MOVBstoreconstidx1, ssa.OpAMD64ADDLconstmodifyidx1, ssa.OpAMD64ADDLconstmodifyidx4, ssa.OpAMD64ADDLconstmodifyidx8, ssa.OpAMD64ADDQconstmodifyidx1, ssa.OpAMD64ADDQconstmodifyidx8, ssa.OpAMD64ANDLconstmodifyidx1, ssa.OpAMD64ANDLconstmodifyidx4, ssa.OpAMD64ANDLconstmodifyidx8, ssa.OpAMD64ANDQconstmodifyidx1, ssa.OpAMD64ANDQconstmodifyidx8, @@ -837,7 +837,7 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { p.From.Type = obj.TYPE_NONE } memIdx(&p.To, v) - gc.AddAux2(&p.To, v, sc.Off()) + ssagen.AddAux2(&p.To, v, sc.Off()) case ssa.OpAMD64MOVLQSX, ssa.OpAMD64MOVWQSX, ssa.OpAMD64MOVBQSX, ssa.OpAMD64MOVLQZX, ssa.OpAMD64MOVWQZX, ssa.OpAMD64MOVBQZX, ssa.OpAMD64CVTTSS2SL, ssa.OpAMD64CVTTSD2SL, ssa.OpAMD64CVTTSS2SQ, ssa.OpAMD64CVTTSD2SQ, ssa.OpAMD64CVTSS2SD, ssa.OpAMD64CVTSD2SS: @@ -867,7 +867,7 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { p := s.Prog(v.Op.Asm()) p.From.Type = obj.TYPE_MEM p.From.Reg = v.Args[1].Reg() - gc.AddAux(&p.From, v) + ssagen.AddAux(&p.From, v) p.To.Type = obj.TYPE_REG p.To.Reg = v.Reg() if v.Reg() != v.Args[0].Reg() { @@ -893,7 +893,7 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { p.From.Reg = r p.From.Index = i - gc.AddAux(&p.From, v) + ssagen.AddAux(&p.From, v) p.To.Type = obj.TYPE_REG p.To.Reg = v.Reg() if v.Reg() != v.Args[0].Reg() { @@ -951,7 +951,7 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { return } p := s.Prog(loadByType(v.Type)) - gc.AddrAuto(&p.From, v.Args[0]) + ssagen.AddrAuto(&p.From, v.Args[0]) p.To.Type = obj.TYPE_REG p.To.Reg = v.Reg() @@ -963,16 +963,16 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { p := s.Prog(storeByType(v.Type)) p.From.Type = obj.TYPE_REG p.From.Reg = v.Args[0].Reg() - gc.AddrAuto(&p.To, v) + ssagen.AddrAuto(&p.To, v) case ssa.OpAMD64LoweredHasCPUFeature: p := s.Prog(x86.AMOVBQZX) p.From.Type = obj.TYPE_MEM - gc.AddAux(&p.From, v) + ssagen.AddAux(&p.From, v) p.To.Type = obj.TYPE_REG p.To.Reg = v.Reg() case ssa.OpAMD64LoweredGetClosurePtr: // Closure pointer is DX. - gc.CheckLoweredGetClosurePtr(v) + ssagen.CheckLoweredGetClosurePtr(v) case ssa.OpAMD64LoweredGetG: r := v.Reg() // See the comments in cmd/internal/obj/x86/obj6.go @@ -1029,13 +1029,13 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { p.To.Type = obj.TYPE_MEM p.To.Name = obj.NAME_EXTERN // arg0 is in DI. Set sym to match where regalloc put arg1. - p.To.Sym = gc.GCWriteBarrierReg[v.Args[1].Reg()] + p.To.Sym = ssagen.GCWriteBarrierReg[v.Args[1].Reg()] case ssa.OpAMD64LoweredPanicBoundsA, ssa.OpAMD64LoweredPanicBoundsB, ssa.OpAMD64LoweredPanicBoundsC: p := s.Prog(obj.ACALL) p.To.Type = obj.TYPE_MEM p.To.Name = obj.NAME_EXTERN - p.To.Sym = gc.BoundsCheckFunc[v.AuxInt] + p.To.Sym = ssagen.BoundsCheckFunc[v.AuxInt] s.UseArgs(int64(2 * types.PtrSize)) // space used in callee args area by assembly stubs case ssa.OpAMD64NEGQ, ssa.OpAMD64NEGL, @@ -1117,7 +1117,7 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { p := s.Prog(v.Op.Asm()) p.To.Type = obj.TYPE_MEM p.To.Reg = v.Args[0].Reg() - gc.AddAux(&p.To, v) + ssagen.AddAux(&p.To, v) case ssa.OpAMD64SETNEF: p := s.Prog(v.Op.Asm()) @@ -1173,7 +1173,7 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { p := s.Prog(v.Op.Asm()) p.From.Type = obj.TYPE_MEM p.From.Reg = v.Args[0].Reg() - gc.AddAux(&p.From, v) + ssagen.AddAux(&p.From, v) p.To.Type = obj.TYPE_REG p.To.Reg = v.Reg0() case ssa.OpAMD64XCHGB, ssa.OpAMD64XCHGL, ssa.OpAMD64XCHGQ: @@ -1186,7 +1186,7 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { p.From.Reg = r p.To.Type = obj.TYPE_MEM p.To.Reg = v.Args[1].Reg() - gc.AddAux(&p.To, v) + ssagen.AddAux(&p.To, v) case ssa.OpAMD64XADDLlock, ssa.OpAMD64XADDQlock: r := v.Reg0() if r != v.Args[0].Reg() { @@ -1198,7 +1198,7 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { p.From.Reg = r p.To.Type = obj.TYPE_MEM p.To.Reg = v.Args[1].Reg() - gc.AddAux(&p.To, v) + ssagen.AddAux(&p.To, v) case ssa.OpAMD64CMPXCHGLlock, ssa.OpAMD64CMPXCHGQlock: if v.Args[1].Reg() != x86.REG_AX { v.Fatalf("input[1] not in AX %s", v.LongString()) @@ -1209,7 +1209,7 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { p.From.Reg = v.Args[2].Reg() p.To.Type = obj.TYPE_MEM p.To.Reg = v.Args[0].Reg() - gc.AddAux(&p.To, v) + ssagen.AddAux(&p.To, v) p = s.Prog(x86.ASETEQ) p.To.Type = obj.TYPE_REG p.To.Reg = v.Reg0() @@ -1220,20 +1220,20 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { p.From.Reg = v.Args[1].Reg() p.To.Type = obj.TYPE_MEM p.To.Reg = v.Args[0].Reg() - gc.AddAux(&p.To, v) + ssagen.AddAux(&p.To, v) case ssa.OpClobber: p := s.Prog(x86.AMOVL) p.From.Type = obj.TYPE_CONST p.From.Offset = 0xdeaddead p.To.Type = obj.TYPE_MEM p.To.Reg = x86.REG_SP - gc.AddAux(&p.To, v) + ssagen.AddAux(&p.To, v) p = s.Prog(x86.AMOVL) p.From.Type = obj.TYPE_CONST p.From.Offset = 0xdeaddead p.To.Type = obj.TYPE_MEM p.To.Reg = x86.REG_SP - gc.AddAux(&p.To, v) + ssagen.AddAux(&p.To, v) p.To.Offset += 4 default: v.Fatalf("genValue not implemented: %s", v.LongString()) @@ -1259,22 +1259,22 @@ var blockJump = [...]struct { ssa.BlockAMD64NAN: {x86.AJPS, x86.AJPC}, } -var eqfJumps = [2][2]gc.IndexJump{ +var eqfJumps = [2][2]ssagen.IndexJump{ {{Jump: x86.AJNE, Index: 1}, {Jump: x86.AJPS, Index: 1}}, // next == b.Succs[0] {{Jump: x86.AJNE, Index: 1}, {Jump: x86.AJPC, Index: 0}}, // next == b.Succs[1] } -var nefJumps = [2][2]gc.IndexJump{ +var nefJumps = [2][2]ssagen.IndexJump{ {{Jump: x86.AJNE, Index: 0}, {Jump: x86.AJPC, Index: 1}}, // next == b.Succs[0] {{Jump: x86.AJNE, Index: 0}, {Jump: x86.AJPS, Index: 0}}, // next == b.Succs[1] } -func ssaGenBlock(s *gc.SSAGenState, b, next *ssa.Block) { +func ssaGenBlock(s *ssagen.State, b, next *ssa.Block) { switch b.Kind { case ssa.BlockPlain: if b.Succs[0].Block() != next { p := s.Prog(obj.AJMP) p.To.Type = obj.TYPE_BRANCH - s.Branches = append(s.Branches, gc.Branch{P: p, B: b.Succs[0].Block()}) + s.Branches = append(s.Branches, ssagen.Branch{P: p, B: b.Succs[0].Block()}) } case ssa.BlockDefer: // defer returns in rax: @@ -1287,11 +1287,11 @@ func ssaGenBlock(s *gc.SSAGenState, b, next *ssa.Block) { p.To.Reg = x86.REG_AX p = s.Prog(x86.AJNE) p.To.Type = obj.TYPE_BRANCH - s.Branches = append(s.Branches, gc.Branch{P: p, B: b.Succs[1].Block()}) + s.Branches = append(s.Branches, ssagen.Branch{P: p, B: b.Succs[1].Block()}) if b.Succs[0].Block() != next { p := s.Prog(obj.AJMP) p.To.Type = obj.TYPE_BRANCH - s.Branches = append(s.Branches, gc.Branch{P: p, B: b.Succs[0].Block()}) + s.Branches = append(s.Branches, ssagen.Branch{P: p, B: b.Succs[0].Block()}) } case ssa.BlockExit: case ssa.BlockRet: diff --git a/src/cmd/compile/internal/arm/galign.go b/src/cmd/compile/internal/arm/galign.go index 20e2f43a91..81959ae0ab 100644 --- a/src/cmd/compile/internal/arm/galign.go +++ b/src/cmd/compile/internal/arm/galign.go @@ -5,13 +5,13 @@ package arm import ( - "cmd/compile/internal/gc" "cmd/compile/internal/ssa" + "cmd/compile/internal/ssagen" "cmd/internal/obj/arm" "cmd/internal/objabi" ) -func Init(arch *gc.Arch) { +func Init(arch *ssagen.ArchInfo) { arch.LinkArch = &arm.Linkarm arch.REGSP = arm.REGSP arch.MAXWIDTH = (1 << 32) - 1 @@ -20,7 +20,7 @@ func Init(arch *gc.Arch) { arch.Ginsnop = ginsnop arch.Ginsnopdefer = ginsnop - arch.SSAMarkMoves = func(s *gc.SSAGenState, b *ssa.Block) {} + arch.SSAMarkMoves = func(s *ssagen.State, b *ssa.Block) {} arch.SSAGenValue = ssaGenValue arch.SSAGenBlock = ssaGenBlock } diff --git a/src/cmd/compile/internal/arm/ssa.go b/src/cmd/compile/internal/arm/ssa.go index 30eae59331..729d2dab2d 100644 --- a/src/cmd/compile/internal/arm/ssa.go +++ b/src/cmd/compile/internal/arm/ssa.go @@ -10,10 +10,10 @@ import ( "math/bits" "cmd/compile/internal/base" - "cmd/compile/internal/gc" "cmd/compile/internal/ir" "cmd/compile/internal/logopt" "cmd/compile/internal/ssa" + "cmd/compile/internal/ssagen" "cmd/compile/internal/types" "cmd/internal/obj" "cmd/internal/obj/arm" @@ -93,7 +93,7 @@ func makeshift(reg int16, typ int64, s int64) shift { } // genshift generates a Prog for r = r0 op (r1 shifted by n) -func genshift(s *gc.SSAGenState, as obj.As, r0, r1, r int16, typ int64, n int64) *obj.Prog { +func genshift(s *ssagen.State, as obj.As, r0, r1, r int16, typ int64, n int64) *obj.Prog { p := s.Prog(as) p.From.Type = obj.TYPE_SHIFT p.From.Offset = int64(makeshift(r1, typ, n)) @@ -111,7 +111,7 @@ func makeregshift(r1 int16, typ int64, r2 int16) shift { } // genregshift generates a Prog for r = r0 op (r1 shifted by r2) -func genregshift(s *gc.SSAGenState, as obj.As, r0, r1, r2, r int16, typ int64) *obj.Prog { +func genregshift(s *ssagen.State, as obj.As, r0, r1, r2, r int16, typ int64) *obj.Prog { p := s.Prog(as) p.From.Type = obj.TYPE_SHIFT p.From.Offset = int64(makeregshift(r1, typ, r2)) @@ -145,7 +145,7 @@ func getBFC(v uint32) (uint32, uint32) { return 0xffffffff, 0 } -func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { +func ssaGenValue(s *ssagen.State, v *ssa.Value) { switch v.Op { case ssa.OpCopy, ssa.OpARMMOVWreg: if v.Type.IsMemory() { @@ -183,7 +183,7 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { return } p := s.Prog(loadByType(v.Type)) - gc.AddrAuto(&p.From, v.Args[0]) + ssagen.AddrAuto(&p.From, v.Args[0]) p.To.Type = obj.TYPE_REG p.To.Reg = v.Reg() case ssa.OpStoreReg: @@ -194,7 +194,7 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { p := s.Prog(storeByType(v.Type)) p.From.Type = obj.TYPE_REG p.From.Reg = v.Args[0].Reg() - gc.AddrAuto(&p.To, v) + ssagen.AddrAuto(&p.To, v) case ssa.OpARMADD, ssa.OpARMADC, ssa.OpARMSUB, @@ -545,10 +545,10 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { v.Fatalf("aux is of unknown type %T", v.Aux) case *obj.LSym: wantreg = "SB" - gc.AddAux(&p.From, v) + ssagen.AddAux(&p.From, v) case *ir.Name: wantreg = "SP" - gc.AddAux(&p.From, v) + ssagen.AddAux(&p.From, v) case nil: // No sym, just MOVW $off(SP), R wantreg = "SP" @@ -568,7 +568,7 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { p := s.Prog(v.Op.Asm()) p.From.Type = obj.TYPE_MEM p.From.Reg = v.Args[0].Reg() - gc.AddAux(&p.From, v) + ssagen.AddAux(&p.From, v) p.To.Type = obj.TYPE_REG p.To.Reg = v.Reg() case ssa.OpARMMOVBstore, @@ -581,7 +581,7 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { p.From.Reg = v.Args[1].Reg() p.To.Type = obj.TYPE_MEM p.To.Reg = v.Args[0].Reg() - gc.AddAux(&p.To, v) + ssagen.AddAux(&p.To, v) case ssa.OpARMMOVWloadidx, ssa.OpARMMOVBUloadidx, ssa.OpARMMOVBloadidx, ssa.OpARMMOVHUloadidx, ssa.OpARMMOVHloadidx: // this is just shift 0 bits fallthrough @@ -712,13 +712,13 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { p := s.Prog(obj.ACALL) p.To.Type = obj.TYPE_MEM p.To.Name = obj.NAME_EXTERN - p.To.Sym = gc.BoundsCheckFunc[v.AuxInt] + p.To.Sym = ssagen.BoundsCheckFunc[v.AuxInt] s.UseArgs(8) // space used in callee args area by assembly stubs case ssa.OpARMLoweredPanicExtendA, ssa.OpARMLoweredPanicExtendB, ssa.OpARMLoweredPanicExtendC: p := s.Prog(obj.ACALL) p.To.Type = obj.TYPE_MEM p.To.Name = obj.NAME_EXTERN - p.To.Sym = gc.ExtendCheckFunc[v.AuxInt] + p.To.Sym = ssagen.ExtendCheckFunc[v.AuxInt] s.UseArgs(12) // space used in callee args area by assembly stubs case ssa.OpARMDUFFZERO: p := s.Prog(obj.ADUFFZERO) @@ -737,7 +737,7 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { p := s.Prog(arm.AMOVB) p.From.Type = obj.TYPE_MEM p.From.Reg = v.Args[0].Reg() - gc.AddAux(&p.From, v) + ssagen.AddAux(&p.From, v) p.To.Type = obj.TYPE_REG p.To.Reg = arm.REGTMP if logopt.Enabled() { @@ -846,7 +846,7 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { p.To.Reg = v.Reg() case ssa.OpARMLoweredGetClosurePtr: // Closure pointer is R7 (arm.REGCTXT). - gc.CheckLoweredGetClosurePtr(v) + ssagen.CheckLoweredGetClosurePtr(v) case ssa.OpARMLoweredGetCallerSP: // caller's SP is FixedFrameSize below the address of the first arg p := s.Prog(arm.AMOVW) @@ -901,24 +901,24 @@ var blockJump = map[ssa.BlockKind]struct { } // To model a 'LEnoov' ('<=' without overflow checking) branching -var leJumps = [2][2]gc.IndexJump{ +var leJumps = [2][2]ssagen.IndexJump{ {{Jump: arm.ABEQ, Index: 0}, {Jump: arm.ABPL, Index: 1}}, // next == b.Succs[0] {{Jump: arm.ABMI, Index: 0}, {Jump: arm.ABEQ, Index: 0}}, // next == b.Succs[1] } // To model a 'GTnoov' ('>' without overflow checking) branching -var gtJumps = [2][2]gc.IndexJump{ +var gtJumps = [2][2]ssagen.IndexJump{ {{Jump: arm.ABMI, Index: 1}, {Jump: arm.ABEQ, Index: 1}}, // next == b.Succs[0] {{Jump: arm.ABEQ, Index: 1}, {Jump: arm.ABPL, Index: 0}}, // next == b.Succs[1] } -func ssaGenBlock(s *gc.SSAGenState, b, next *ssa.Block) { +func ssaGenBlock(s *ssagen.State, b, next *ssa.Block) { switch b.Kind { case ssa.BlockPlain: if b.Succs[0].Block() != next { p := s.Prog(obj.AJMP) p.To.Type = obj.TYPE_BRANCH - s.Branches = append(s.Branches, gc.Branch{P: p, B: b.Succs[0].Block()}) + s.Branches = append(s.Branches, ssagen.Branch{P: p, B: b.Succs[0].Block()}) } case ssa.BlockDefer: @@ -931,11 +931,11 @@ func ssaGenBlock(s *gc.SSAGenState, b, next *ssa.Block) { p.Reg = arm.REG_R0 p = s.Prog(arm.ABNE) p.To.Type = obj.TYPE_BRANCH - s.Branches = append(s.Branches, gc.Branch{P: p, B: b.Succs[1].Block()}) + s.Branches = append(s.Branches, ssagen.Branch{P: p, B: b.Succs[1].Block()}) if b.Succs[0].Block() != next { p := s.Prog(obj.AJMP) p.To.Type = obj.TYPE_BRANCH - s.Branches = append(s.Branches, gc.Branch{P: p, B: b.Succs[0].Block()}) + s.Branches = append(s.Branches, ssagen.Branch{P: p, B: b.Succs[0].Block()}) } case ssa.BlockExit: diff --git a/src/cmd/compile/internal/arm64/galign.go b/src/cmd/compile/internal/arm64/galign.go index 40d6e17ae2..d3db37e16f 100644 --- a/src/cmd/compile/internal/arm64/galign.go +++ b/src/cmd/compile/internal/arm64/galign.go @@ -5,12 +5,12 @@ package arm64 import ( - "cmd/compile/internal/gc" "cmd/compile/internal/ssa" + "cmd/compile/internal/ssagen" "cmd/internal/obj/arm64" ) -func Init(arch *gc.Arch) { +func Init(arch *ssagen.ArchInfo) { arch.LinkArch = &arm64.Linkarm64 arch.REGSP = arm64.REGSP arch.MAXWIDTH = 1 << 50 @@ -20,7 +20,7 @@ func Init(arch *gc.Arch) { arch.Ginsnop = ginsnop arch.Ginsnopdefer = ginsnop - arch.SSAMarkMoves = func(s *gc.SSAGenState, b *ssa.Block) {} + arch.SSAMarkMoves = func(s *ssagen.State, b *ssa.Block) {} arch.SSAGenValue = ssaGenValue arch.SSAGenBlock = ssaGenBlock } diff --git a/src/cmd/compile/internal/arm64/ssa.go b/src/cmd/compile/internal/arm64/ssa.go index 9bdea3ee2a..8d25fa8592 100644 --- a/src/cmd/compile/internal/arm64/ssa.go +++ b/src/cmd/compile/internal/arm64/ssa.go @@ -8,10 +8,10 @@ import ( "math" "cmd/compile/internal/base" - "cmd/compile/internal/gc" "cmd/compile/internal/ir" "cmd/compile/internal/logopt" "cmd/compile/internal/ssa" + "cmd/compile/internal/ssagen" "cmd/compile/internal/types" "cmd/internal/obj" "cmd/internal/obj/arm64" @@ -83,7 +83,7 @@ func makeshift(reg int16, typ int64, s int64) int64 { } // genshift generates a Prog for r = r0 op (r1 shifted by n) -func genshift(s *gc.SSAGenState, as obj.As, r0, r1, r int16, typ int64, n int64) *obj.Prog { +func genshift(s *ssagen.State, as obj.As, r0, r1, r int16, typ int64, n int64) *obj.Prog { p := s.Prog(as) p.From.Type = obj.TYPE_SHIFT p.From.Offset = makeshift(r1, typ, n) @@ -112,7 +112,7 @@ func genIndexedOperand(v *ssa.Value) obj.Addr { return mop } -func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { +func ssaGenValue(s *ssagen.State, v *ssa.Value) { switch v.Op { case ssa.OpCopy, ssa.OpARM64MOVDreg: if v.Type.IsMemory() { @@ -150,7 +150,7 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { return } p := s.Prog(loadByType(v.Type)) - gc.AddrAuto(&p.From, v.Args[0]) + ssagen.AddrAuto(&p.From, v.Args[0]) p.To.Type = obj.TYPE_REG p.To.Reg = v.Reg() case ssa.OpStoreReg: @@ -161,7 +161,7 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { p := s.Prog(storeByType(v.Type)) p.From.Type = obj.TYPE_REG p.From.Reg = v.Args[0].Reg() - gc.AddrAuto(&p.To, v) + ssagen.AddrAuto(&p.To, v) case ssa.OpARM64ADD, ssa.OpARM64SUB, ssa.OpARM64AND, @@ -395,10 +395,10 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { v.Fatalf("aux is of unknown type %T", v.Aux) case *obj.LSym: wantreg = "SB" - gc.AddAux(&p.From, v) + ssagen.AddAux(&p.From, v) case *ir.Name: wantreg = "SP" - gc.AddAux(&p.From, v) + ssagen.AddAux(&p.From, v) case nil: // No sym, just MOVD $off(SP), R wantreg = "SP" @@ -419,7 +419,7 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { p := s.Prog(v.Op.Asm()) p.From.Type = obj.TYPE_MEM p.From.Reg = v.Args[0].Reg() - gc.AddAux(&p.From, v) + ssagen.AddAux(&p.From, v) p.To.Type = obj.TYPE_REG p.To.Reg = v.Reg() case ssa.OpARM64MOVBloadidx, @@ -446,7 +446,7 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { p := s.Prog(v.Op.Asm()) p.From.Type = obj.TYPE_MEM p.From.Reg = v.Args[0].Reg() - gc.AddAux(&p.From, v) + ssagen.AddAux(&p.From, v) p.To.Type = obj.TYPE_REG p.To.Reg = v.Reg0() case ssa.OpARM64MOVBstore, @@ -463,7 +463,7 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { p.From.Reg = v.Args[1].Reg() p.To.Type = obj.TYPE_MEM p.To.Reg = v.Args[0].Reg() - gc.AddAux(&p.To, v) + ssagen.AddAux(&p.To, v) case ssa.OpARM64MOVBstoreidx, ssa.OpARM64MOVHstoreidx, ssa.OpARM64MOVWstoreidx, @@ -484,7 +484,7 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { p.From.Offset = int64(v.Args[2].Reg()) p.To.Type = obj.TYPE_MEM p.To.Reg = v.Args[0].Reg() - gc.AddAux(&p.To, v) + ssagen.AddAux(&p.To, v) case ssa.OpARM64MOVBstorezero, ssa.OpARM64MOVHstorezero, ssa.OpARM64MOVWstorezero, @@ -494,7 +494,7 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { p.From.Reg = arm64.REGZERO p.To.Type = obj.TYPE_MEM p.To.Reg = v.Args[0].Reg() - gc.AddAux(&p.To, v) + ssagen.AddAux(&p.To, v) case ssa.OpARM64MOVBstorezeroidx, ssa.OpARM64MOVHstorezeroidx, ssa.OpARM64MOVWstorezeroidx, @@ -513,7 +513,7 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { p.From.Offset = int64(arm64.REGZERO) p.To.Type = obj.TYPE_MEM p.To.Reg = v.Args[0].Reg() - gc.AddAux(&p.To, v) + ssagen.AddAux(&p.To, v) case ssa.OpARM64BFI, ssa.OpARM64BFXIL: r := v.Reg() @@ -1027,14 +1027,14 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { p := s.Prog(obj.ACALL) p.To.Type = obj.TYPE_MEM p.To.Name = obj.NAME_EXTERN - p.To.Sym = gc.BoundsCheckFunc[v.AuxInt] + p.To.Sym = ssagen.BoundsCheckFunc[v.AuxInt] s.UseArgs(16) // space used in callee args area by assembly stubs case ssa.OpARM64LoweredNilCheck: // Issue a load which will fault if arg is nil. p := s.Prog(arm64.AMOVB) p.From.Type = obj.TYPE_MEM p.From.Reg = v.Args[0].Reg() - gc.AddAux(&p.From, v) + ssagen.AddAux(&p.From, v) p.To.Type = obj.TYPE_REG p.To.Reg = arm64.REGTMP if logopt.Enabled() { @@ -1065,7 +1065,7 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { p.To.Reg = v.Reg() case ssa.OpARM64LoweredGetClosurePtr: // Closure pointer is R26 (arm64.REGCTXT). - gc.CheckLoweredGetClosurePtr(v) + ssagen.CheckLoweredGetClosurePtr(v) case ssa.OpARM64LoweredGetCallerSP: // caller's SP is FixedFrameSize below the address of the first arg p := s.Prog(arm64.AMOVD) @@ -1134,24 +1134,24 @@ var blockJump = map[ssa.BlockKind]struct { } // To model a 'LEnoov' ('<=' without overflow checking) branching -var leJumps = [2][2]gc.IndexJump{ +var leJumps = [2][2]ssagen.IndexJump{ {{Jump: arm64.ABEQ, Index: 0}, {Jump: arm64.ABPL, Index: 1}}, // next == b.Succs[0] {{Jump: arm64.ABMI, Index: 0}, {Jump: arm64.ABEQ, Index: 0}}, // next == b.Succs[1] } // To model a 'GTnoov' ('>' without overflow checking) branching -var gtJumps = [2][2]gc.IndexJump{ +var gtJumps = [2][2]ssagen.IndexJump{ {{Jump: arm64.ABMI, Index: 1}, {Jump: arm64.ABEQ, Index: 1}}, // next == b.Succs[0] {{Jump: arm64.ABEQ, Index: 1}, {Jump: arm64.ABPL, Index: 0}}, // next == b.Succs[1] } -func ssaGenBlock(s *gc.SSAGenState, b, next *ssa.Block) { +func ssaGenBlock(s *ssagen.State, b, next *ssa.Block) { switch b.Kind { case ssa.BlockPlain: if b.Succs[0].Block() != next { p := s.Prog(obj.AJMP) p.To.Type = obj.TYPE_BRANCH - s.Branches = append(s.Branches, gc.Branch{P: p, B: b.Succs[0].Block()}) + s.Branches = append(s.Branches, ssagen.Branch{P: p, B: b.Succs[0].Block()}) } case ssa.BlockDefer: @@ -1164,11 +1164,11 @@ func ssaGenBlock(s *gc.SSAGenState, b, next *ssa.Block) { p.Reg = arm64.REG_R0 p = s.Prog(arm64.ABNE) p.To.Type = obj.TYPE_BRANCH - s.Branches = append(s.Branches, gc.Branch{P: p, B: b.Succs[1].Block()}) + s.Branches = append(s.Branches, ssagen.Branch{P: p, B: b.Succs[1].Block()}) if b.Succs[0].Block() != next { p := s.Prog(obj.AJMP) p.To.Type = obj.TYPE_BRANCH - s.Branches = append(s.Branches, gc.Branch{P: p, B: b.Succs[0].Block()}) + s.Branches = append(s.Branches, ssagen.Branch{P: p, B: b.Succs[0].Block()}) } case ssa.BlockExit: diff --git a/src/cmd/compile/internal/gc/abiutils_test.go b/src/cmd/compile/internal/gc/abiutils_test.go index 4b2a30d00c..a421a229dc 100644 --- a/src/cmd/compile/internal/gc/abiutils_test.go +++ b/src/cmd/compile/internal/gc/abiutils_test.go @@ -8,6 +8,7 @@ import ( "bufio" "cmd/compile/internal/base" "cmd/compile/internal/reflectdata" + "cmd/compile/internal/ssagen" "cmd/compile/internal/typecheck" "cmd/compile/internal/types" "cmd/internal/obj" @@ -28,16 +29,16 @@ var configAMD64 = ABIConfig{ } func TestMain(m *testing.M) { - thearch.LinkArch = &x86.Linkamd64 - thearch.REGSP = x86.REGSP - thearch.MAXWIDTH = 1 << 50 - types.MaxWidth = thearch.MAXWIDTH - base.Ctxt = obj.Linknew(thearch.LinkArch) + ssagen.Arch.LinkArch = &x86.Linkamd64 + ssagen.Arch.REGSP = x86.REGSP + ssagen.Arch.MAXWIDTH = 1 << 50 + types.MaxWidth = ssagen.Arch.MAXWIDTH + base.Ctxt = obj.Linknew(ssagen.Arch.LinkArch) base.Ctxt.DiagFunc = base.Errorf base.Ctxt.DiagFlush = base.FlushErrors base.Ctxt.Bso = bufio.NewWriter(os.Stdout) - types.PtrSize = thearch.LinkArch.PtrSize - types.RegSize = thearch.LinkArch.RegSize + types.PtrSize = ssagen.Arch.LinkArch.PtrSize + types.RegSize = ssagen.Arch.LinkArch.RegSize types.TypeLinkSym = func(t *types.Type) *obj.LSym { return reflectdata.TypeSym(t).Linksym() } diff --git a/src/cmd/compile/internal/gc/compile.go b/src/cmd/compile/internal/gc/compile.go new file mode 100644 index 0000000000..c2a6a9e327 --- /dev/null +++ b/src/cmd/compile/internal/gc/compile.go @@ -0,0 +1,177 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package gc + +import ( + "internal/race" + "math/rand" + "sort" + "sync" + + "cmd/compile/internal/base" + "cmd/compile/internal/ir" + "cmd/compile/internal/liveness" + "cmd/compile/internal/reflectdata" + "cmd/compile/internal/ssagen" + "cmd/compile/internal/typecheck" + "cmd/compile/internal/types" +) + +// "Portable" code generation. + +var ( + compilequeue []*ir.Func // functions waiting to be compiled +) + +func funccompile(fn *ir.Func) { + if ir.CurFunc != nil { + base.Fatalf("funccompile %v inside %v", fn.Sym(), ir.CurFunc.Sym()) + } + + if fn.Type() == nil { + if base.Errors() == 0 { + base.Fatalf("funccompile missing type") + } + return + } + + // assign parameter offsets + types.CalcSize(fn.Type()) + + if len(fn.Body) == 0 { + // Initialize ABI wrappers if necessary. + ssagen.InitLSym(fn, false) + liveness.WriteFuncMap(fn) + return + } + + typecheck.DeclContext = ir.PAUTO + ir.CurFunc = fn + compile(fn) + ir.CurFunc = nil + typecheck.DeclContext = ir.PEXTERN +} + +func compile(fn *ir.Func) { + // Set up the function's LSym early to avoid data races with the assemblers. + // Do this before walk, as walk needs the LSym to set attributes/relocations + // (e.g. in markTypeUsedInInterface). + ssagen.InitLSym(fn, true) + + errorsBefore := base.Errors() + walk(fn) + if base.Errors() > errorsBefore { + return + } + + // From this point, there should be no uses of Curfn. Enforce that. + ir.CurFunc = nil + + if ir.FuncName(fn) == "_" { + // We don't need to generate code for this function, just report errors in its body. + // At this point we've generated any errors needed. + // (Beyond here we generate only non-spec errors, like "stack frame too large".) + // See issue 29870. + return + } + + // Make sure type syms are declared for all types that might + // be types of stack objects. We need to do this here + // because symbols must be allocated before the parallel + // phase of the compiler. + for _, n := range fn.Dcl { + switch n.Class_ { + case ir.PPARAM, ir.PPARAMOUT, ir.PAUTO: + if liveness.ShouldTrack(n) && n.Addrtaken() { + reflectdata.WriteType(n.Type()) + // Also make sure we allocate a linker symbol + // for the stack object data, for the same reason. + if fn.LSym.Func().StackObjects == nil { + fn.LSym.Func().StackObjects = base.Ctxt.Lookup(fn.LSym.Name + ".stkobj") + } + } + } + } + + if compilenow(fn) { + ssagen.Compile(fn, 0) + } else { + compilequeue = append(compilequeue, fn) + } +} + +// compilenow reports whether to compile immediately. +// If functions are not compiled immediately, +// they are enqueued in compilequeue, +// which is drained by compileFunctions. +func compilenow(fn *ir.Func) bool { + // Issue 38068: if this function is a method AND an inline + // candidate AND was not inlined (yet), put it onto the compile + // queue instead of compiling it immediately. This is in case we + // wind up inlining it into a method wrapper that is generated by + // compiling a function later on in the Target.Decls list. + if ir.IsMethod(fn) && isInlinableButNotInlined(fn) { + return false + } + return base.Flag.LowerC == 1 && base.Debug.CompileLater == 0 +} + +// compileFunctions compiles all functions in compilequeue. +// It fans out nBackendWorkers to do the work +// and waits for them to complete. +func compileFunctions() { + if len(compilequeue) != 0 { + types.CalcSizeDisabled = true // not safe to calculate sizes concurrently + if race.Enabled { + // Randomize compilation order to try to shake out races. + tmp := make([]*ir.Func, len(compilequeue)) + perm := rand.Perm(len(compilequeue)) + for i, v := range perm { + tmp[v] = compilequeue[i] + } + copy(compilequeue, tmp) + } else { + // Compile the longest functions first, + // since they're most likely to be the slowest. + // This helps avoid stragglers. + sort.Slice(compilequeue, func(i, j int) bool { + return len(compilequeue[i].Body) > len(compilequeue[j].Body) + }) + } + var wg sync.WaitGroup + base.Ctxt.InParallel = true + c := make(chan *ir.Func, base.Flag.LowerC) + for i := 0; i < base.Flag.LowerC; i++ { + wg.Add(1) + go func(worker int) { + for fn := range c { + ssagen.Compile(fn, worker) + } + wg.Done() + }(i) + } + for _, fn := range compilequeue { + c <- fn + } + close(c) + compilequeue = nil + wg.Wait() + base.Ctxt.InParallel = false + types.CalcSizeDisabled = false + } +} + +// isInlinableButNotInlined returns true if 'fn' was marked as an +// inline candidate but then never inlined (presumably because we +// found no call sites). +func isInlinableButNotInlined(fn *ir.Func) bool { + if fn.Inl == nil { + return false + } + if fn.Sym() == nil { + return true + } + return !fn.Sym().Linksym().WasInlined() +} diff --git a/src/cmd/compile/internal/gc/dcl.go b/src/cmd/compile/internal/gc/dcl.go deleted file mode 100644 index 7b2bf5b606..0000000000 --- a/src/cmd/compile/internal/gc/dcl.go +++ /dev/null @@ -1,199 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package gc - -import ( - "bytes" - "cmd/compile/internal/base" - "cmd/compile/internal/ir" - "cmd/compile/internal/typecheck" - "cmd/compile/internal/types" - "cmd/internal/obj" - "cmd/internal/src" - "fmt" -) - -func EnableNoWriteBarrierRecCheck() { - nowritebarrierrecCheck = newNowritebarrierrecChecker() -} - -func NoWriteBarrierRecCheck() { - // Write barriers are now known. Check the - // call graph. - nowritebarrierrecCheck.check() - nowritebarrierrecCheck = nil -} - -var nowritebarrierrecCheck *nowritebarrierrecChecker - -type nowritebarrierrecChecker struct { - // extraCalls contains extra function calls that may not be - // visible during later analysis. It maps from the ODCLFUNC of - // the caller to a list of callees. - extraCalls map[*ir.Func][]nowritebarrierrecCall - - // curfn is the current function during AST walks. - curfn *ir.Func -} - -type nowritebarrierrecCall struct { - target *ir.Func // caller or callee - lineno src.XPos // line of call -} - -// newNowritebarrierrecChecker creates a nowritebarrierrecChecker. It -// must be called before transformclosure and walk. -func newNowritebarrierrecChecker() *nowritebarrierrecChecker { - c := &nowritebarrierrecChecker{ - extraCalls: make(map[*ir.Func][]nowritebarrierrecCall), - } - - // Find all systemstack calls and record their targets. In - // general, flow analysis can't see into systemstack, but it's - // important to handle it for this check, so we model it - // directly. This has to happen before transformclosure since - // it's a lot harder to work out the argument after. - for _, n := range typecheck.Target.Decls { - if n.Op() != ir.ODCLFUNC { - continue - } - c.curfn = n.(*ir.Func) - ir.Visit(n, c.findExtraCalls) - } - c.curfn = nil - return c -} - -func (c *nowritebarrierrecChecker) findExtraCalls(nn ir.Node) { - if nn.Op() != ir.OCALLFUNC { - return - } - n := nn.(*ir.CallExpr) - if n.X == nil || n.X.Op() != ir.ONAME { - return - } - fn := n.X.(*ir.Name) - if fn.Class_ != ir.PFUNC || fn.Name().Defn == nil { - return - } - if !types.IsRuntimePkg(fn.Sym().Pkg) || fn.Sym().Name != "systemstack" { - return - } - - var callee *ir.Func - arg := n.Args[0] - switch arg.Op() { - case ir.ONAME: - arg := arg.(*ir.Name) - callee = arg.Name().Defn.(*ir.Func) - case ir.OCLOSURE: - arg := arg.(*ir.ClosureExpr) - callee = arg.Func - default: - base.Fatalf("expected ONAME or OCLOSURE node, got %+v", arg) - } - if callee.Op() != ir.ODCLFUNC { - base.Fatalf("expected ODCLFUNC node, got %+v", callee) - } - c.extraCalls[c.curfn] = append(c.extraCalls[c.curfn], nowritebarrierrecCall{callee, n.Pos()}) -} - -// recordCall records a call from ODCLFUNC node "from", to function -// symbol "to" at position pos. -// -// This should be done as late as possible during compilation to -// capture precise call graphs. The target of the call is an LSym -// because that's all we know after we start SSA. -// -// This can be called concurrently for different from Nodes. -func (c *nowritebarrierrecChecker) recordCall(fn *ir.Func, to *obj.LSym, pos src.XPos) { - // We record this information on the *Func so this is concurrent-safe. - if fn.NWBRCalls == nil { - fn.NWBRCalls = new([]ir.SymAndPos) - } - *fn.NWBRCalls = append(*fn.NWBRCalls, ir.SymAndPos{Sym: to, Pos: pos}) -} - -func (c *nowritebarrierrecChecker) check() { - // We walk the call graph as late as possible so we can - // capture all calls created by lowering, but this means we - // only get to see the obj.LSyms of calls. symToFunc lets us - // get back to the ODCLFUNCs. - symToFunc := make(map[*obj.LSym]*ir.Func) - // funcs records the back-edges of the BFS call graph walk. It - // maps from the ODCLFUNC of each function that must not have - // write barriers to the call that inhibits them. Functions - // that are directly marked go:nowritebarrierrec are in this - // map with a zero-valued nowritebarrierrecCall. This also - // acts as the set of marks for the BFS of the call graph. - funcs := make(map[*ir.Func]nowritebarrierrecCall) - // q is the queue of ODCLFUNC Nodes to visit in BFS order. - var q ir.NameQueue - - for _, n := range typecheck.Target.Decls { - if n.Op() != ir.ODCLFUNC { - continue - } - fn := n.(*ir.Func) - - symToFunc[fn.LSym] = fn - - // Make nowritebarrierrec functions BFS roots. - if fn.Pragma&ir.Nowritebarrierrec != 0 { - funcs[fn] = nowritebarrierrecCall{} - q.PushRight(fn.Nname) - } - // Check go:nowritebarrier functions. - if fn.Pragma&ir.Nowritebarrier != 0 && fn.WBPos.IsKnown() { - base.ErrorfAt(fn.WBPos, "write barrier prohibited") - } - } - - // Perform a BFS of the call graph from all - // go:nowritebarrierrec functions. - enqueue := func(src, target *ir.Func, pos src.XPos) { - if target.Pragma&ir.Yeswritebarrierrec != 0 { - // Don't flow into this function. - return - } - if _, ok := funcs[target]; ok { - // Already found a path to target. - return - } - - // Record the path. - funcs[target] = nowritebarrierrecCall{target: src, lineno: pos} - q.PushRight(target.Nname) - } - for !q.Empty() { - fn := q.PopLeft().Func - - // Check fn. - if fn.WBPos.IsKnown() { - var err bytes.Buffer - call := funcs[fn] - for call.target != nil { - fmt.Fprintf(&err, "\n\t%v: called by %v", base.FmtPos(call.lineno), call.target.Nname) - call = funcs[call.target] - } - base.ErrorfAt(fn.WBPos, "write barrier prohibited by caller; %v%s", fn.Nname, err.String()) - continue - } - - // Enqueue fn's calls. - for _, callee := range c.extraCalls[fn] { - enqueue(fn, callee.target, callee.lineno) - } - if fn.NWBRCalls == nil { - continue - } - for _, callee := range *fn.NWBRCalls { - target := symToFunc[callee.Sym] - if target != nil { - enqueue(fn, target, callee.Pos) - } - } - } -} diff --git a/src/cmd/compile/internal/gc/dwarf.go b/src/cmd/compile/internal/gc/dwarf.go new file mode 100644 index 0000000000..e853c51422 --- /dev/null +++ b/src/cmd/compile/internal/gc/dwarf.go @@ -0,0 +1,412 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package gc + +import ( + "sort" + + "cmd/compile/internal/base" + "cmd/compile/internal/ir" + "cmd/compile/internal/ssa" + "cmd/compile/internal/ssagen" + "cmd/compile/internal/types" + "cmd/internal/dwarf" + "cmd/internal/obj" + "cmd/internal/objabi" + "cmd/internal/src" +) + +func debuginfo(fnsym *obj.LSym, infosym *obj.LSym, curfn interface{}) ([]dwarf.Scope, dwarf.InlCalls) { + fn := curfn.(*ir.Func) + + if fn.Nname != nil { + expect := fn.Sym().Linksym() + if fnsym.ABI() == obj.ABI0 { + expect = fn.Sym().LinksymABI0() + } + if fnsym != expect { + base.Fatalf("unexpected fnsym: %v != %v", fnsym, expect) + } + } + + // Back when there were two different *Funcs for a function, this code + // was not consistent about whether a particular *Node being processed + // was an ODCLFUNC or ONAME node. Partly this is because inlined function + // bodies have no ODCLFUNC node, which was it's own inconsistency. + // In any event, the handling of the two different nodes for DWARF purposes + // was subtly different, likely in unintended ways. CL 272253 merged the + // two nodes' Func fields, so that code sees the same *Func whether it is + // holding the ODCLFUNC or the ONAME. This resulted in changes in the + // DWARF output. To preserve the existing DWARF output and leave an + // intentional change for a future CL, this code does the following when + // fn.Op == ONAME: + // + // 1. Disallow use of createComplexVars in createDwarfVars. + // It was not possible to reach that code for an ONAME before, + // because the DebugInfo was set only on the ODCLFUNC Func. + // Calling into it in the ONAME case causes an index out of bounds panic. + // + // 2. Do not populate apdecls. fn.Func.Dcl was in the ODCLFUNC Func, + // not the ONAME Func. Populating apdecls for the ONAME case results + // in selected being populated after createSimpleVars is called in + // createDwarfVars, and then that causes the loop to skip all the entries + // in dcl, meaning that the RecordAutoType calls don't happen. + // + // These two adjustments keep toolstash -cmp working for now. + // Deciding the right answer is, as they say, future work. + // + // We can tell the difference between the old ODCLFUNC and ONAME + // cases by looking at the infosym.Name. If it's empty, DebugInfo is + // being called from (*obj.Link).populateDWARF, which used to use + // the ODCLFUNC. If it's non-empty (the name will end in $abstract), + // DebugInfo is being called from (*obj.Link).DwarfAbstractFunc, + // which used to use the ONAME form. + isODCLFUNC := infosym.Name == "" + + var apdecls []*ir.Name + // Populate decls for fn. + if isODCLFUNC { + for _, n := range fn.Dcl { + if n.Op() != ir.ONAME { // might be OTYPE or OLITERAL + continue + } + switch n.Class_ { + case ir.PAUTO: + if !n.Used() { + // Text == nil -> generating abstract function + if fnsym.Func().Text != nil { + base.Fatalf("debuginfo unused node (AllocFrame should truncate fn.Func.Dcl)") + } + continue + } + case ir.PPARAM, ir.PPARAMOUT: + default: + continue + } + apdecls = append(apdecls, n) + fnsym.Func().RecordAutoType(ngotype(n).Linksym()) + } + } + + decls, dwarfVars := createDwarfVars(fnsym, isODCLFUNC, fn, apdecls) + + // For each type referenced by the functions auto vars but not + // already referenced by a dwarf var, attach an R_USETYPE relocation to + // the function symbol to insure that the type included in DWARF + // processing during linking. + typesyms := []*obj.LSym{} + for t, _ := range fnsym.Func().Autot { + typesyms = append(typesyms, t) + } + sort.Sort(obj.BySymName(typesyms)) + for _, sym := range typesyms { + r := obj.Addrel(infosym) + r.Sym = sym + r.Type = objabi.R_USETYPE + } + fnsym.Func().Autot = nil + + var varScopes []ir.ScopeID + for _, decl := range decls { + pos := declPos(decl) + varScopes = append(varScopes, findScope(fn.Marks, pos)) + } + + scopes := assembleScopes(fnsym, fn, dwarfVars, varScopes) + var inlcalls dwarf.InlCalls + if base.Flag.GenDwarfInl > 0 { + inlcalls = assembleInlines(fnsym, dwarfVars) + } + return scopes, inlcalls +} + +func declPos(decl *ir.Name) src.XPos { + if decl.Name().Defn != nil && (decl.Name().Captured() || decl.Name().Byval()) { + // It's not clear which position is correct for captured variables here: + // * decl.Pos is the wrong position for captured variables, in the inner + // function, but it is the right position in the outer function. + // * decl.Name.Defn is nil for captured variables that were arguments + // on the outer function, however the decl.Pos for those seems to be + // correct. + // * decl.Name.Defn is the "wrong" thing for variables declared in the + // header of a type switch, it's their position in the header, rather + // than the position of the case statement. In principle this is the + // right thing, but here we prefer the latter because it makes each + // instance of the header variable local to the lexical block of its + // case statement. + // This code is probably wrong for type switch variables that are also + // captured. + return decl.Name().Defn.Pos() + } + return decl.Pos() +} + +// createDwarfVars process fn, returning a list of DWARF variables and the +// Nodes they represent. +func createDwarfVars(fnsym *obj.LSym, complexOK bool, fn *ir.Func, apDecls []*ir.Name) ([]*ir.Name, []*dwarf.Var) { + // Collect a raw list of DWARF vars. + var vars []*dwarf.Var + var decls []*ir.Name + var selected map[*ir.Name]bool + if base.Ctxt.Flag_locationlists && base.Ctxt.Flag_optimize && fn.DebugInfo != nil && complexOK { + decls, vars, selected = createComplexVars(fnsym, fn) + } else { + decls, vars, selected = createSimpleVars(fnsym, apDecls) + } + + dcl := apDecls + if fnsym.WasInlined() { + dcl = preInliningDcls(fnsym) + } + + // If optimization is enabled, the list above will typically be + // missing some of the original pre-optimization variables in the + // function (they may have been promoted to registers, folded into + // constants, dead-coded away, etc). Input arguments not eligible + // for SSA optimization are also missing. Here we add back in entries + // for selected missing vars. Note that the recipe below creates a + // conservative location. The idea here is that we want to + // communicate to the user that "yes, there is a variable named X + // in this function, but no, I don't have enough information to + // reliably report its contents." + // For non-SSA-able arguments, however, the correct information + // is known -- they have a single home on the stack. + for _, n := range dcl { + if _, found := selected[n]; found { + continue + } + c := n.Sym().Name[0] + if c == '.' || n.Type().IsUntyped() { + continue + } + if n.Class_ == ir.PPARAM && !ssagen.TypeOK(n.Type()) { + // SSA-able args get location lists, and may move in and + // out of registers, so those are handled elsewhere. + // Autos and named output params seem to get handled + // with VARDEF, which creates location lists. + // Args not of SSA-able type are treated here; they + // are homed on the stack in a single place for the + // entire call. + vars = append(vars, createSimpleVar(fnsym, n)) + decls = append(decls, n) + continue + } + typename := dwarf.InfoPrefix + types.TypeSymName(n.Type()) + decls = append(decls, n) + abbrev := dwarf.DW_ABRV_AUTO_LOCLIST + isReturnValue := (n.Class_ == ir.PPARAMOUT) + if n.Class_ == ir.PPARAM || n.Class_ == ir.PPARAMOUT { + abbrev = dwarf.DW_ABRV_PARAM_LOCLIST + } else if n.Class_ == ir.PAUTOHEAP { + // If dcl in question has been promoted to heap, do a bit + // of extra work to recover original class (auto or param); + // see issue 30908. This insures that we get the proper + // signature in the abstract function DIE, but leaves a + // misleading location for the param (we want pointer-to-heap + // and not stack). + // TODO(thanm): generate a better location expression + stackcopy := n.Name().Stackcopy + if stackcopy != nil && (stackcopy.Class_ == ir.PPARAM || stackcopy.Class_ == ir.PPARAMOUT) { + abbrev = dwarf.DW_ABRV_PARAM_LOCLIST + isReturnValue = (stackcopy.Class_ == ir.PPARAMOUT) + } + } + inlIndex := 0 + if base.Flag.GenDwarfInl > 1 { + if n.Name().InlFormal() || n.Name().InlLocal() { + inlIndex = posInlIndex(n.Pos()) + 1 + if n.Name().InlFormal() { + abbrev = dwarf.DW_ABRV_PARAM_LOCLIST + } + } + } + declpos := base.Ctxt.InnermostPos(n.Pos()) + vars = append(vars, &dwarf.Var{ + Name: n.Sym().Name, + IsReturnValue: isReturnValue, + Abbrev: abbrev, + StackOffset: int32(n.FrameOffset()), + Type: base.Ctxt.Lookup(typename), + DeclFile: declpos.RelFilename(), + DeclLine: declpos.RelLine(), + DeclCol: declpos.Col(), + InlIndex: int32(inlIndex), + ChildIndex: -1, + }) + // Record go type of to insure that it gets emitted by the linker. + fnsym.Func().RecordAutoType(ngotype(n).Linksym()) + } + + return decls, vars +} + +// Given a function that was inlined at some point during the +// compilation, return a sorted list of nodes corresponding to the +// autos/locals in that function prior to inlining. If this is a +// function that is not local to the package being compiled, then the +// names of the variables may have been "versioned" to avoid conflicts +// with local vars; disregard this versioning when sorting. +func preInliningDcls(fnsym *obj.LSym) []*ir.Name { + fn := base.Ctxt.DwFixups.GetPrecursorFunc(fnsym).(*ir.Func) + var rdcl []*ir.Name + for _, n := range fn.Inl.Dcl { + c := n.Sym().Name[0] + // Avoid reporting "_" parameters, since if there are more than + // one, it can result in a collision later on, as in #23179. + if unversion(n.Sym().Name) == "_" || c == '.' || n.Type().IsUntyped() { + continue + } + rdcl = append(rdcl, n) + } + return rdcl +} + +// createSimpleVars creates a DWARF entry for every variable declared in the +// function, claiming that they are permanently on the stack. +func createSimpleVars(fnsym *obj.LSym, apDecls []*ir.Name) ([]*ir.Name, []*dwarf.Var, map[*ir.Name]bool) { + var vars []*dwarf.Var + var decls []*ir.Name + selected := make(map[*ir.Name]bool) + for _, n := range apDecls { + if ir.IsAutoTmp(n) { + continue + } + + decls = append(decls, n) + vars = append(vars, createSimpleVar(fnsym, n)) + selected[n] = true + } + return decls, vars, selected +} + +func createSimpleVar(fnsym *obj.LSym, n *ir.Name) *dwarf.Var { + var abbrev int + var offs int64 + + switch n.Class_ { + case ir.PAUTO: + offs = n.FrameOffset() + abbrev = dwarf.DW_ABRV_AUTO + if base.Ctxt.FixedFrameSize() == 0 { + offs -= int64(types.PtrSize) + } + if objabi.Framepointer_enabled || objabi.GOARCH == "arm64" { + // There is a word space for FP on ARM64 even if the frame pointer is disabled + offs -= int64(types.PtrSize) + } + + case ir.PPARAM, ir.PPARAMOUT: + abbrev = dwarf.DW_ABRV_PARAM + offs = n.FrameOffset() + base.Ctxt.FixedFrameSize() + default: + base.Fatalf("createSimpleVar unexpected class %v for node %v", n.Class_, n) + } + + typename := dwarf.InfoPrefix + types.TypeSymName(n.Type()) + delete(fnsym.Func().Autot, ngotype(n).Linksym()) + inlIndex := 0 + if base.Flag.GenDwarfInl > 1 { + if n.Name().InlFormal() || n.Name().InlLocal() { + inlIndex = posInlIndex(n.Pos()) + 1 + if n.Name().InlFormal() { + abbrev = dwarf.DW_ABRV_PARAM + } + } + } + declpos := base.Ctxt.InnermostPos(declPos(n)) + return &dwarf.Var{ + Name: n.Sym().Name, + IsReturnValue: n.Class_ == ir.PPARAMOUT, + IsInlFormal: n.Name().InlFormal(), + Abbrev: abbrev, + StackOffset: int32(offs), + Type: base.Ctxt.Lookup(typename), + DeclFile: declpos.RelFilename(), + DeclLine: declpos.RelLine(), + DeclCol: declpos.Col(), + InlIndex: int32(inlIndex), + ChildIndex: -1, + } +} + +// createComplexVars creates recomposed DWARF vars with location lists, +// suitable for describing optimized code. +func createComplexVars(fnsym *obj.LSym, fn *ir.Func) ([]*ir.Name, []*dwarf.Var, map[*ir.Name]bool) { + debugInfo := fn.DebugInfo.(*ssa.FuncDebug) + + // Produce a DWARF variable entry for each user variable. + var decls []*ir.Name + var vars []*dwarf.Var + ssaVars := make(map[*ir.Name]bool) + + for varID, dvar := range debugInfo.Vars { + n := dvar + ssaVars[n] = true + for _, slot := range debugInfo.VarSlots[varID] { + ssaVars[debugInfo.Slots[slot].N] = true + } + + if dvar := createComplexVar(fnsym, fn, ssa.VarID(varID)); dvar != nil { + decls = append(decls, n) + vars = append(vars, dvar) + } + } + + return decls, vars, ssaVars +} + +// createComplexVar builds a single DWARF variable entry and location list. +func createComplexVar(fnsym *obj.LSym, fn *ir.Func, varID ssa.VarID) *dwarf.Var { + debug := fn.DebugInfo.(*ssa.FuncDebug) + n := debug.Vars[varID] + + var abbrev int + switch n.Class_ { + case ir.PAUTO: + abbrev = dwarf.DW_ABRV_AUTO_LOCLIST + case ir.PPARAM, ir.PPARAMOUT: + abbrev = dwarf.DW_ABRV_PARAM_LOCLIST + default: + return nil + } + + gotype := ngotype(n).Linksym() + delete(fnsym.Func().Autot, gotype) + typename := dwarf.InfoPrefix + gotype.Name[len("type."):] + inlIndex := 0 + if base.Flag.GenDwarfInl > 1 { + if n.Name().InlFormal() || n.Name().InlLocal() { + inlIndex = posInlIndex(n.Pos()) + 1 + if n.Name().InlFormal() { + abbrev = dwarf.DW_ABRV_PARAM_LOCLIST + } + } + } + declpos := base.Ctxt.InnermostPos(n.Pos()) + dvar := &dwarf.Var{ + Name: n.Sym().Name, + IsReturnValue: n.Class_ == ir.PPARAMOUT, + IsInlFormal: n.Name().InlFormal(), + Abbrev: abbrev, + Type: base.Ctxt.Lookup(typename), + // The stack offset is used as a sorting key, so for decomposed + // variables just give it the first one. It's not used otherwise. + // This won't work well if the first slot hasn't been assigned a stack + // location, but it's not obvious how to do better. + StackOffset: ssagen.StackOffset(debug.Slots[debug.VarSlots[varID][0]]), + DeclFile: declpos.RelFilename(), + DeclLine: declpos.RelLine(), + DeclCol: declpos.Col(), + InlIndex: int32(inlIndex), + ChildIndex: -1, + } + list := debug.LocationLists[varID] + if len(list) != 0 { + dvar.PutLocationList = func(listSym, startPC dwarf.Sym) { + debug.PutLocationList(list, base.Ctxt, listSym.(*obj.LSym), startPC.(*obj.LSym)) + } + } + return dvar +} diff --git a/src/cmd/compile/internal/gc/go.go b/src/cmd/compile/internal/gc/go.go deleted file mode 100644 index ba838a5ff5..0000000000 --- a/src/cmd/compile/internal/gc/go.go +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package gc - -import ( - "cmd/compile/internal/objw" - "cmd/compile/internal/ssa" - "cmd/internal/obj" -) - -var pragcgobuf [][]string - -// interface to back end - -type Arch struct { - LinkArch *obj.LinkArch - - REGSP int - MAXWIDTH int64 - SoftFloat bool - - PadFrame func(int64) int64 - - // ZeroRange zeroes a range of memory on stack. It is only inserted - // at function entry, and it is ok to clobber registers. - ZeroRange func(*objw.Progs, *obj.Prog, int64, int64, *uint32) *obj.Prog - - Ginsnop func(*objw.Progs) *obj.Prog - Ginsnopdefer func(*objw.Progs) *obj.Prog // special ginsnop for deferreturn - - // SSAMarkMoves marks any MOVXconst ops that need to avoid clobbering flags. - SSAMarkMoves func(*SSAGenState, *ssa.Block) - - // SSAGenValue emits Prog(s) for the Value. - SSAGenValue func(*SSAGenState, *ssa.Value) - - // SSAGenBlock emits end-of-block Progs. SSAGenValue should be called - // for all values in the block before SSAGenBlock. - SSAGenBlock func(s *SSAGenState, b, next *ssa.Block) -} - -var thearch Arch - -var ( - BoundsCheckFunc [ssa.BoundsKindCount]*obj.LSym - ExtendCheckFunc [ssa.BoundsKindCount]*obj.LSym -) - -// GCWriteBarrierReg maps from registers to gcWriteBarrier implementation LSyms. -var GCWriteBarrierReg map[int16]*obj.LSym diff --git a/src/cmd/compile/internal/gc/gsubr.go b/src/cmd/compile/internal/gc/gsubr.go deleted file mode 100644 index 81f7956d2e..0000000000 --- a/src/cmd/compile/internal/gc/gsubr.go +++ /dev/null @@ -1,279 +0,0 @@ -// Derived from Inferno utils/6c/txt.c -// https://bitbucket.org/inferno-os/inferno-os/src/master/utils/6c/txt.c -// -// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. -// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) -// Portions Copyright © 1997-1999 Vita Nuova Limited -// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) -// Portions Copyright © 2004,2006 Bruce Ellis -// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) -// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others -// Portions Copyright © 2009 The Go Authors. All rights reserved. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package gc - -import ( - "cmd/compile/internal/base" - "cmd/compile/internal/escape" - "cmd/compile/internal/ir" - "cmd/compile/internal/typecheck" - "cmd/compile/internal/types" - "cmd/internal/obj" - "cmd/internal/objabi" - "fmt" - "os" -) - -// makeABIWrapper creates a new function that wraps a cross-ABI call -// to "f". The wrapper is marked as an ABIWRAPPER. -func makeABIWrapper(f *ir.Func, wrapperABI obj.ABI) { - - // Q: is this needed? - savepos := base.Pos - savedclcontext := typecheck.DeclContext - savedcurfn := ir.CurFunc - - base.Pos = base.AutogeneratedPos - typecheck.DeclContext = ir.PEXTERN - - // At the moment we don't support wrapping a method, we'd need machinery - // below to handle the receiver. Panic if we see this scenario. - ft := f.Nname.Ntype.Type() - if ft.NumRecvs() != 0 { - panic("makeABIWrapper support for wrapping methods not implemented") - } - - // Manufacture a new func type to use for the wrapper. - var noReceiver *ir.Field - tfn := ir.NewFuncType(base.Pos, - noReceiver, - typecheck.NewFuncParams(ft.Params(), true), - typecheck.NewFuncParams(ft.Results(), false)) - - // Reuse f's types.Sym to create a new ODCLFUNC/function. - fn := typecheck.DeclFunc(f.Nname.Sym(), tfn) - fn.SetDupok(true) - fn.SetWrapper(true) // ignore frame for panic+recover matching - - // Select LSYM now. - asym := base.Ctxt.LookupABI(f.LSym.Name, wrapperABI) - asym.Type = objabi.STEXT - if fn.LSym != nil { - panic("unexpected") - } - fn.LSym = asym - - // ABI0-to-ABIInternal wrappers will be mainly loading params from - // stack into registers (and/or storing stack locations back to - // registers after the wrapped call); in most cases they won't - // need to allocate stack space, so it should be OK to mark them - // as NOSPLIT in these cases. In addition, my assumption is that - // functions written in assembly are NOSPLIT in most (but not all) - // cases. In the case of an ABIInternal target that has too many - // parameters to fit into registers, the wrapper would need to - // allocate stack space, but this seems like an unlikely scenario. - // Hence: mark these wrappers NOSPLIT. - // - // ABIInternal-to-ABI0 wrappers on the other hand will be taking - // things in registers and pushing them onto the stack prior to - // the ABI0 call, meaning that they will always need to allocate - // stack space. If the compiler marks them as NOSPLIT this seems - // as though it could lead to situations where the the linker's - // nosplit-overflow analysis would trigger a link failure. On the - // other hand if they not tagged NOSPLIT then this could cause - // problems when building the runtime (since there may be calls to - // asm routine in cases where it's not safe to grow the stack). In - // most cases the wrapper would be (in effect) inlined, but are - // there (perhaps) indirect calls from the runtime that could run - // into trouble here. - // FIXME: at the moment all.bash does not pass when I leave out - // NOSPLIT for these wrappers, so all are currently tagged with NOSPLIT. - setupTextLSym(fn, obj.NOSPLIT|obj.ABIWRAPPER) - - // Generate call. Use tail call if no params and no returns, - // but a regular call otherwise. - // - // Note: ideally we would be using a tail call in cases where - // there are params but no returns for ABI0->ABIInternal wrappers, - // provided that all params fit into registers (e.g. we don't have - // to allocate any stack space). Doing this will require some - // extra work in typecheck/walk/ssa, might want to add a new node - // OTAILCALL or something to this effect. - var tail ir.Node - if tfn.Type().NumResults() == 0 && tfn.Type().NumParams() == 0 && tfn.Type().NumRecvs() == 0 { - tail = ir.NewBranchStmt(base.Pos, ir.ORETJMP, f.Nname.Sym()) - } else { - call := ir.NewCallExpr(base.Pos, ir.OCALL, f.Nname, nil) - call.Args.Set(ir.ParamNames(tfn.Type())) - call.IsDDD = tfn.Type().IsVariadic() - tail = call - if tfn.Type().NumResults() > 0 { - n := ir.NewReturnStmt(base.Pos, nil) - n.Results = []ir.Node{call} - tail = n - } - } - fn.Body.Append(tail) - - typecheck.FinishFuncBody() - if base.Debug.DclStack != 0 { - types.CheckDclstack() - } - - typecheck.Func(fn) - ir.CurFunc = fn - typecheck.Stmts(fn.Body) - - escape.Batch([]*ir.Func{fn}, false) - - typecheck.Target.Decls = append(typecheck.Target.Decls, fn) - - // Restore previous context. - base.Pos = savepos - typecheck.DeclContext = savedclcontext - ir.CurFunc = savedcurfn -} - -// initLSym defines f's obj.LSym and initializes it based on the -// properties of f. This includes setting the symbol flags and ABI and -// creating and initializing related DWARF symbols. -// -// initLSym must be called exactly once per function and must be -// called for both functions with bodies and functions without bodies. -// For body-less functions, we only create the LSym; for functions -// with bodies call a helper to setup up / populate the LSym. -func initLSym(f *ir.Func, hasBody bool) { - // FIXME: for new-style ABI wrappers, we set up the lsym at the - // point the wrapper is created. - if f.LSym != nil && base.Flag.ABIWrap { - return - } - selectLSym(f, hasBody) - if hasBody { - setupTextLSym(f, 0) - } -} - -// selectLSym sets up the LSym for a given function, and -// makes calls to helpers to create ABI wrappers if needed. -func selectLSym(f *ir.Func, hasBody bool) { - if f.LSym != nil { - base.Fatalf("Func.initLSym called twice") - } - - if nam := f.Nname; !ir.IsBlank(nam) { - - var wrapperABI obj.ABI - needABIWrapper := false - defABI, hasDefABI := symabiDefs[nam.Sym().LinksymName()] - if hasDefABI && defABI == obj.ABI0 { - // Symbol is defined as ABI0. Create an - // Internal -> ABI0 wrapper. - f.LSym = nam.Sym().LinksymABI0() - needABIWrapper, wrapperABI = true, obj.ABIInternal - } else { - f.LSym = nam.Sym().Linksym() - // No ABI override. Check that the symbol is - // using the expected ABI. - want := obj.ABIInternal - if f.LSym.ABI() != want { - base.Fatalf("function symbol %s has the wrong ABI %v, expected %v", f.LSym.Name, f.LSym.ABI(), want) - } - } - if f.Pragma&ir.Systemstack != 0 { - f.LSym.Set(obj.AttrCFunc, true) - } - - isLinknameExported := nam.Sym().Linkname != "" && (hasBody || hasDefABI) - if abi, ok := symabiRefs[f.LSym.Name]; (ok && abi == obj.ABI0) || isLinknameExported { - // Either 1) this symbol is definitely - // referenced as ABI0 from this package; or 2) - // this symbol is defined in this package but - // given a linkname, indicating that it may be - // referenced from another package. Create an - // ABI0 -> Internal wrapper so it can be - // called as ABI0. In case 2, it's important - // that we know it's defined in this package - // since other packages may "pull" symbols - // using linkname and we don't want to create - // duplicate ABI wrappers. - if f.LSym.ABI() != obj.ABI0 { - needABIWrapper, wrapperABI = true, obj.ABI0 - } - } - - if needABIWrapper { - if !useABIWrapGen(f) { - // Fallback: use alias instead. FIXME. - - // These LSyms have the same name as the - // native function, so we create them directly - // rather than looking them up. The uniqueness - // of f.lsym ensures uniqueness of asym. - asym := &obj.LSym{ - Name: f.LSym.Name, - Type: objabi.SABIALIAS, - R: []obj.Reloc{{Sym: f.LSym}}, // 0 size, so "informational" - } - asym.SetABI(wrapperABI) - asym.Set(obj.AttrDuplicateOK, true) - base.Ctxt.ABIAliases = append(base.Ctxt.ABIAliases, asym) - } else { - if base.Debug.ABIWrap != 0 { - fmt.Fprintf(os.Stderr, "=-= %v to %v wrapper for %s.%s\n", - wrapperABI, 1-wrapperABI, types.LocalPkg.Path, f.LSym.Name) - } - makeABIWrapper(f, wrapperABI) - } - } - } -} - -// setupTextLsym initializes the LSym for a with-body text symbol. -func setupTextLSym(f *ir.Func, flag int) { - if f.Dupok() { - flag |= obj.DUPOK - } - if f.Wrapper() { - flag |= obj.WRAPPER - } - if f.Needctxt() { - flag |= obj.NEEDCTXT - } - if f.Pragma&ir.Nosplit != 0 { - flag |= obj.NOSPLIT - } - if f.ReflectMethod() { - flag |= obj.REFLECTMETHOD - } - - // Clumsy but important. - // See test/recover.go for test cases and src/reflect/value.go - // for the actual functions being considered. - if base.Ctxt.Pkgpath == "reflect" { - switch f.Sym().Name { - case "callReflect", "callMethod": - flag |= obj.WRAPPER - } - } - - base.Ctxt.InitTextSym(f.LSym, flag) -} diff --git a/src/cmd/compile/internal/gc/main.go b/src/cmd/compile/internal/gc/main.go index e66b877fd0..154235f744 100644 --- a/src/cmd/compile/internal/gc/main.go +++ b/src/cmd/compile/internal/gc/main.go @@ -17,6 +17,7 @@ import ( "cmd/compile/internal/noder" "cmd/compile/internal/reflectdata" "cmd/compile/internal/ssa" + "cmd/compile/internal/ssagen" "cmd/compile/internal/staticdata" "cmd/compile/internal/typecheck" "cmd/compile/internal/types" @@ -26,12 +27,9 @@ import ( "cmd/internal/src" "flag" "fmt" - "io/ioutil" "log" "os" "runtime" - "sort" - "strings" ) func hidePanic() { @@ -52,14 +50,14 @@ func hidePanic() { // Main parses flags and Go source files specified in the command-line // arguments, type-checks the parsed Go package, compiles functions to machine // code, and finally writes the compiled package definition to disk. -func Main(archInit func(*Arch)) { +func Main(archInit func(*ssagen.ArchInfo)) { base.Timer.Start("fe", "init") defer hidePanic() - archInit(&thearch) + archInit(&ssagen.Arch) - base.Ctxt = obj.Linknew(thearch.LinkArch) + base.Ctxt = obj.Linknew(ssagen.Arch.LinkArch) base.Ctxt.DiagFunc = base.Errorf base.Ctxt.DiagFlush = base.FlushErrors base.Ctxt.Bso = bufio.NewWriter(os.Stdout) @@ -151,7 +149,7 @@ func Main(archInit func(*Arch)) { types.ParseLangFlag() if base.Flag.SymABIs != "" { - readSymABIs(base.Flag.SymABIs, base.Ctxt.Pkgpath) + ssagen.ReadSymABIs(base.Flag.SymABIs, base.Ctxt.Pkgpath) } if base.Compiling(base.NoInstrumentPkgs) { @@ -159,7 +157,7 @@ func Main(archInit func(*Arch)) { base.Flag.MSan = false } - thearch.LinkArch.Init(base.Ctxt) + ssagen.Arch.LinkArch.Init(base.Ctxt) startProfile() if base.Flag.Race { ir.Pkgs.Race = types.NewPkg("runtime/race", "") @@ -174,7 +172,7 @@ func Main(archInit func(*Arch)) { dwarf.EnableLogging(base.Debug.DwarfInl != 0) } if base.Debug.SoftFloat != 0 { - thearch.SoftFloat = true + ssagen.Arch.SoftFloat = true } if base.Flag.JSON != "" { // parse version,destination from json logging optimization. @@ -182,14 +180,14 @@ func Main(archInit func(*Arch)) { } ir.EscFmt = escape.Fmt - ir.IsIntrinsicCall = isIntrinsicCall - inline.SSADumpInline = ssaDumpInline - initSSAEnv() - initSSATables() - - types.PtrSize = thearch.LinkArch.PtrSize - types.RegSize = thearch.LinkArch.RegSize - types.MaxWidth = thearch.MAXWIDTH + ir.IsIntrinsicCall = ssagen.IsIntrinsicCall + inline.SSADumpInline = ssagen.DumpInline + ssagen.InitEnv() + ssagen.InitTables() + + types.PtrSize = ssagen.Arch.LinkArch.PtrSize + types.RegSize = ssagen.Arch.LinkArch.RegSize + types.MaxWidth = ssagen.Arch.MAXWIDTH types.TypeLinkSym = func(t *types.Type) *obj.LSym { return reflectdata.TypeSym(t).Linksym() } @@ -210,7 +208,7 @@ func Main(archInit func(*Arch)) { // Parse input. base.Timer.Start("fe", "parse") lines := noder.ParseFiles(flag.Args()) - cgoSymABIs() + ssagen.CgoSymABIs() base.Timer.Stop() base.Timer.AddEvent(int64(lines), "lines") recordPackageName() @@ -257,7 +255,7 @@ func Main(archInit func(*Arch)) { // We'll do the final check after write barriers are // inserted. if base.Flag.CompilingRuntime { - EnableNoWriteBarrierRecCheck() + ssagen.EnableNoWriteBarrierRecCheck() } // Transform closure bodies to properly reference captured variables. @@ -277,7 +275,7 @@ func Main(archInit func(*Arch)) { // Prepare for SSA compilation. // This must be before peekitabs, because peekitabs // can trigger function compilation. - initssaconfig() + ssagen.InitConfig() // Just before compilation, compile itabs found on // the right side of OCONVIFACE so that methods @@ -302,7 +300,7 @@ func Main(archInit func(*Arch)) { if base.Flag.CompilingRuntime { // Write barriers are now known. Check the call graph. - NoWriteBarrierRecCheck() + ssagen.NoWriteBarrierRecCheck() } // Finalize DWARF inline routine DIEs, then explicitly turn off @@ -323,7 +321,7 @@ func Main(archInit func(*Arch)) { dumpasmhdr() } - CheckLargeStacks() + ssagen.CheckLargeStacks() typecheck.CheckFuncStack() if len(compilequeue) != 0 { @@ -343,34 +341,6 @@ func Main(archInit func(*Arch)) { } } -func CheckLargeStacks() { - // Check whether any of the functions we have compiled have gigantic stack frames. - sort.Slice(largeStackFrames, func(i, j int) bool { - return largeStackFrames[i].pos.Before(largeStackFrames[j].pos) - }) - for _, large := range largeStackFrames { - if large.callee != 0 { - base.ErrorfAt(large.pos, "stack frame too large (>1GB): %d MB locals + %d MB args + %d MB callee", large.locals>>20, large.args>>20, large.callee>>20) - } else { - base.ErrorfAt(large.pos, "stack frame too large (>1GB): %d MB locals + %d MB args", large.locals>>20, large.args>>20) - } - } -} - -func cgoSymABIs() { - // The linker expects an ABI0 wrapper for all cgo-exported - // functions. - for _, prag := range typecheck.Target.CgoPragmas { - switch prag[0] { - case "cgo_export_static", "cgo_export_dynamic": - if symabiRefs == nil { - symabiRefs = make(map[string]obj.ABI) - } - symabiRefs[prag[1]] = obj.ABI0 - } - } -} - func writebench(filename string) error { f, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666) if err != nil { @@ -394,77 +364,6 @@ func writebench(filename string) error { return f.Close() } -// symabiDefs and symabiRefs record the defined and referenced ABIs of -// symbols required by non-Go code. These are keyed by link symbol -// name, where the local package prefix is always `"".` -var symabiDefs, symabiRefs map[string]obj.ABI - -// readSymABIs reads a symabis file that specifies definitions and -// references of text symbols by ABI. -// -// The symabis format is a set of lines, where each line is a sequence -// of whitespace-separated fields. The first field is a verb and is -// either "def" for defining a symbol ABI or "ref" for referencing a -// symbol using an ABI. For both "def" and "ref", the second field is -// the symbol name and the third field is the ABI name, as one of the -// named cmd/internal/obj.ABI constants. -func readSymABIs(file, myimportpath string) { - data, err := ioutil.ReadFile(file) - if err != nil { - log.Fatalf("-symabis: %v", err) - } - - symabiDefs = make(map[string]obj.ABI) - symabiRefs = make(map[string]obj.ABI) - - localPrefix := "" - if myimportpath != "" { - // Symbols in this package may be written either as - // "".X or with the package's import path already in - // the symbol. - localPrefix = objabi.PathToPrefix(myimportpath) + "." - } - - for lineNum, line := range strings.Split(string(data), "\n") { - lineNum++ // 1-based - line = strings.TrimSpace(line) - if line == "" || strings.HasPrefix(line, "#") { - continue - } - - parts := strings.Fields(line) - switch parts[0] { - case "def", "ref": - // Parse line. - if len(parts) != 3 { - log.Fatalf(`%s:%d: invalid symabi: syntax is "%s sym abi"`, file, lineNum, parts[0]) - } - sym, abistr := parts[1], parts[2] - abi, valid := obj.ParseABI(abistr) - if !valid { - log.Fatalf(`%s:%d: invalid symabi: unknown abi "%s"`, file, lineNum, abistr) - } - - // If the symbol is already prefixed with - // myimportpath, rewrite it to start with "" - // so it matches the compiler's internal - // symbol names. - if localPrefix != "" && strings.HasPrefix(sym, localPrefix) { - sym = `"".` + sym[len(localPrefix):] - } - - // Record for later. - if parts[0] == "def" { - symabiDefs[sym] = abi - } else { - symabiRefs[sym] = abi - } - default: - log.Fatalf(`%s:%d: invalid symabi type "%s"`, file, lineNum, parts[0]) - } - } -} - // recordFlags records the specified command-line flags to be placed // in the DWARF info. func recordFlags(flags ...string) { @@ -532,29 +431,6 @@ func recordPackageName() { s.P = []byte(types.LocalPkg.Name) } -// useNewABIWrapGen returns TRUE if the compiler should generate an -// ABI wrapper for the function 'f'. -func useABIWrapGen(f *ir.Func) bool { - if !base.Flag.ABIWrap { - return false - } - - // Support limit option for bisecting. - if base.Flag.ABIWrapLimit == 1 { - return false - } - if base.Flag.ABIWrapLimit < 1 { - return true - } - base.Flag.ABIWrapLimit-- - if base.Debug.ABIWrap != 0 && base.Flag.ABIWrapLimit == 1 { - fmt.Fprintf(os.Stderr, "=-= limit reached after new wrapper for %s\n", - f.LSym.Name) - } - - return true -} - func makePos(b *src.PosBase, line, col uint) src.XPos { return base.Ctxt.PosTable.XPos(src.MakePos(b, line, col)) } diff --git a/src/cmd/compile/internal/gc/pgen.go b/src/cmd/compile/internal/gc/pgen.go deleted file mode 100644 index 4d990e7dba..0000000000 --- a/src/cmd/compile/internal/gc/pgen.go +++ /dev/null @@ -1,804 +0,0 @@ -// Copyright 2011 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package gc - -import ( - "cmd/compile/internal/base" - "cmd/compile/internal/ir" - "cmd/compile/internal/liveness" - "cmd/compile/internal/objw" - "cmd/compile/internal/reflectdata" - "cmd/compile/internal/ssa" - "cmd/compile/internal/typecheck" - "cmd/compile/internal/types" - "cmd/internal/dwarf" - "cmd/internal/obj" - "cmd/internal/objabi" - "cmd/internal/src" - "cmd/internal/sys" - "internal/race" - "math/rand" - "sort" - "sync" - "time" -) - -// "Portable" code generation. - -var ( - compilequeue []*ir.Func // functions waiting to be compiled -) - -// cmpstackvarlt reports whether the stack variable a sorts before b. -// -// Sort the list of stack variables. Autos after anything else, -// within autos, unused after used, within used, things with -// pointers first, zeroed things first, and then decreasing size. -// Because autos are laid out in decreasing addresses -// on the stack, pointers first, zeroed things first and decreasing size -// really means, in memory, things with pointers needing zeroing at -// the top of the stack and increasing in size. -// Non-autos sort on offset. -func cmpstackvarlt(a, b *ir.Name) bool { - if (a.Class_ == ir.PAUTO) != (b.Class_ == ir.PAUTO) { - return b.Class_ == ir.PAUTO - } - - if a.Class_ != ir.PAUTO { - return a.FrameOffset() < b.FrameOffset() - } - - if a.Used() != b.Used() { - return a.Used() - } - - ap := a.Type().HasPointers() - bp := b.Type().HasPointers() - if ap != bp { - return ap - } - - ap = a.Needzero() - bp = b.Needzero() - if ap != bp { - return ap - } - - if a.Type().Width != b.Type().Width { - return a.Type().Width > b.Type().Width - } - - return a.Sym().Name < b.Sym().Name -} - -// byStackvar implements sort.Interface for []*Node using cmpstackvarlt. -type byStackVar []*ir.Name - -func (s byStackVar) Len() int { return len(s) } -func (s byStackVar) Less(i, j int) bool { return cmpstackvarlt(s[i], s[j]) } -func (s byStackVar) Swap(i, j int) { s[i], s[j] = s[j], s[i] } - -func (s *ssafn) AllocFrame(f *ssa.Func) { - s.stksize = 0 - s.stkptrsize = 0 - fn := s.curfn - - // Mark the PAUTO's unused. - for _, ln := range fn.Dcl { - if ln.Class_ == ir.PAUTO { - ln.SetUsed(false) - } - } - - for _, l := range f.RegAlloc { - if ls, ok := l.(ssa.LocalSlot); ok { - ls.N.Name().SetUsed(true) - } - } - - scratchUsed := false - for _, b := range f.Blocks { - for _, v := range b.Values { - if n, ok := v.Aux.(*ir.Name); ok { - switch n.Class_ { - case ir.PPARAM, ir.PPARAMOUT: - // Don't modify nodfp; it is a global. - if n != ir.RegFP { - n.Name().SetUsed(true) - } - case ir.PAUTO: - n.Name().SetUsed(true) - } - } - if !scratchUsed { - scratchUsed = v.Op.UsesScratch() - } - - } - } - - if f.Config.NeedsFpScratch && scratchUsed { - s.scratchFpMem = typecheck.TempAt(src.NoXPos, s.curfn, types.Types[types.TUINT64]) - } - - sort.Sort(byStackVar(fn.Dcl)) - - // Reassign stack offsets of the locals that are used. - lastHasPtr := false - for i, n := range fn.Dcl { - if n.Op() != ir.ONAME || n.Class_ != ir.PAUTO { - continue - } - if !n.Used() { - fn.Dcl = fn.Dcl[:i] - break - } - - types.CalcSize(n.Type()) - w := n.Type().Width - if w >= types.MaxWidth || w < 0 { - base.Fatalf("bad width") - } - if w == 0 && lastHasPtr { - // Pad between a pointer-containing object and a zero-sized object. - // This prevents a pointer to the zero-sized object from being interpreted - // as a pointer to the pointer-containing object (and causing it - // to be scanned when it shouldn't be). See issue 24993. - w = 1 - } - s.stksize += w - s.stksize = types.Rnd(s.stksize, int64(n.Type().Align)) - if n.Type().HasPointers() { - s.stkptrsize = s.stksize - lastHasPtr = true - } else { - lastHasPtr = false - } - if thearch.LinkArch.InFamily(sys.MIPS, sys.MIPS64, sys.ARM, sys.ARM64, sys.PPC64, sys.S390X) { - s.stksize = types.Rnd(s.stksize, int64(types.PtrSize)) - } - n.SetFrameOffset(-s.stksize) - } - - s.stksize = types.Rnd(s.stksize, int64(types.RegSize)) - s.stkptrsize = types.Rnd(s.stkptrsize, int64(types.RegSize)) -} - -func funccompile(fn *ir.Func) { - if ir.CurFunc != nil { - base.Fatalf("funccompile %v inside %v", fn.Sym(), ir.CurFunc.Sym()) - } - - if fn.Type() == nil { - if base.Errors() == 0 { - base.Fatalf("funccompile missing type") - } - return - } - - // assign parameter offsets - types.CalcSize(fn.Type()) - - if len(fn.Body) == 0 { - // Initialize ABI wrappers if necessary. - initLSym(fn, false) - liveness.WriteFuncMap(fn) - return - } - - typecheck.DeclContext = ir.PAUTO - ir.CurFunc = fn - compile(fn) - ir.CurFunc = nil - typecheck.DeclContext = ir.PEXTERN -} - -func compile(fn *ir.Func) { - // Set up the function's LSym early to avoid data races with the assemblers. - // Do this before walk, as walk needs the LSym to set attributes/relocations - // (e.g. in markTypeUsedInInterface). - initLSym(fn, true) - - errorsBefore := base.Errors() - walk(fn) - if base.Errors() > errorsBefore { - return - } - - // From this point, there should be no uses of Curfn. Enforce that. - ir.CurFunc = nil - - if ir.FuncName(fn) == "_" { - // We don't need to generate code for this function, just report errors in its body. - // At this point we've generated any errors needed. - // (Beyond here we generate only non-spec errors, like "stack frame too large".) - // See issue 29870. - return - } - - // Make sure type syms are declared for all types that might - // be types of stack objects. We need to do this here - // because symbols must be allocated before the parallel - // phase of the compiler. - for _, n := range fn.Dcl { - switch n.Class_ { - case ir.PPARAM, ir.PPARAMOUT, ir.PAUTO: - if liveness.ShouldTrack(n) && n.Addrtaken() { - reflectdata.WriteType(n.Type()) - // Also make sure we allocate a linker symbol - // for the stack object data, for the same reason. - if fn.LSym.Func().StackObjects == nil { - fn.LSym.Func().StackObjects = base.Ctxt.Lookup(fn.LSym.Name + ".stkobj") - } - } - } - } - - if compilenow(fn) { - compileSSA(fn, 0) - } else { - compilequeue = append(compilequeue, fn) - } -} - -// compilenow reports whether to compile immediately. -// If functions are not compiled immediately, -// they are enqueued in compilequeue, -// which is drained by compileFunctions. -func compilenow(fn *ir.Func) bool { - // Issue 38068: if this function is a method AND an inline - // candidate AND was not inlined (yet), put it onto the compile - // queue instead of compiling it immediately. This is in case we - // wind up inlining it into a method wrapper that is generated by - // compiling a function later on in the Target.Decls list. - if ir.IsMethod(fn) && isInlinableButNotInlined(fn) { - return false - } - return base.Flag.LowerC == 1 && base.Debug.CompileLater == 0 -} - -// isInlinableButNotInlined returns true if 'fn' was marked as an -// inline candidate but then never inlined (presumably because we -// found no call sites). -func isInlinableButNotInlined(fn *ir.Func) bool { - if fn.Inl == nil { - return false - } - if fn.Sym() == nil { - return true - } - return !fn.Sym().Linksym().WasInlined() -} - -const maxStackSize = 1 << 30 - -// compileSSA builds an SSA backend function, -// uses it to generate a plist, -// and flushes that plist to machine code. -// worker indicates which of the backend workers is doing the processing. -func compileSSA(fn *ir.Func, worker int) { - f := buildssa(fn, worker) - // Note: check arg size to fix issue 25507. - if f.Frontend().(*ssafn).stksize >= maxStackSize || fn.Type().ArgWidth() >= maxStackSize { - largeStackFramesMu.Lock() - largeStackFrames = append(largeStackFrames, largeStack{locals: f.Frontend().(*ssafn).stksize, args: fn.Type().ArgWidth(), pos: fn.Pos()}) - largeStackFramesMu.Unlock() - return - } - pp := objw.NewProgs(fn, worker) - defer pp.Free() - genssa(f, pp) - // Check frame size again. - // The check above included only the space needed for local variables. - // After genssa, the space needed includes local variables and the callee arg region. - // We must do this check prior to calling pp.Flush. - // If there are any oversized stack frames, - // the assembler may emit inscrutable complaints about invalid instructions. - if pp.Text.To.Offset >= maxStackSize { - largeStackFramesMu.Lock() - locals := f.Frontend().(*ssafn).stksize - largeStackFrames = append(largeStackFrames, largeStack{locals: locals, args: fn.Type().ArgWidth(), callee: pp.Text.To.Offset - locals, pos: fn.Pos()}) - largeStackFramesMu.Unlock() - return - } - - pp.Flush() // assemble, fill in boilerplate, etc. - // fieldtrack must be called after pp.Flush. See issue 20014. - fieldtrack(pp.Text.From.Sym, fn.FieldTrack) -} - -func init() { - if race.Enabled { - rand.Seed(time.Now().UnixNano()) - } -} - -// compileFunctions compiles all functions in compilequeue. -// It fans out nBackendWorkers to do the work -// and waits for them to complete. -func compileFunctions() { - if len(compilequeue) != 0 { - types.CalcSizeDisabled = true // not safe to calculate sizes concurrently - if race.Enabled { - // Randomize compilation order to try to shake out races. - tmp := make([]*ir.Func, len(compilequeue)) - perm := rand.Perm(len(compilequeue)) - for i, v := range perm { - tmp[v] = compilequeue[i] - } - copy(compilequeue, tmp) - } else { - // Compile the longest functions first, - // since they're most likely to be the slowest. - // This helps avoid stragglers. - sort.Slice(compilequeue, func(i, j int) bool { - return len(compilequeue[i].Body) > len(compilequeue[j].Body) - }) - } - var wg sync.WaitGroup - base.Ctxt.InParallel = true - c := make(chan *ir.Func, base.Flag.LowerC) - for i := 0; i < base.Flag.LowerC; i++ { - wg.Add(1) - go func(worker int) { - for fn := range c { - compileSSA(fn, worker) - } - wg.Done() - }(i) - } - for _, fn := range compilequeue { - c <- fn - } - close(c) - compilequeue = nil - wg.Wait() - base.Ctxt.InParallel = false - types.CalcSizeDisabled = false - } -} - -func debuginfo(fnsym *obj.LSym, infosym *obj.LSym, curfn interface{}) ([]dwarf.Scope, dwarf.InlCalls) { - fn := curfn.(*ir.Func) - - if fn.Nname != nil { - expect := fn.Sym().Linksym() - if fnsym.ABI() == obj.ABI0 { - expect = fn.Sym().LinksymABI0() - } - if fnsym != expect { - base.Fatalf("unexpected fnsym: %v != %v", fnsym, expect) - } - } - - // Back when there were two different *Funcs for a function, this code - // was not consistent about whether a particular *Node being processed - // was an ODCLFUNC or ONAME node. Partly this is because inlined function - // bodies have no ODCLFUNC node, which was it's own inconsistency. - // In any event, the handling of the two different nodes for DWARF purposes - // was subtly different, likely in unintended ways. CL 272253 merged the - // two nodes' Func fields, so that code sees the same *Func whether it is - // holding the ODCLFUNC or the ONAME. This resulted in changes in the - // DWARF output. To preserve the existing DWARF output and leave an - // intentional change for a future CL, this code does the following when - // fn.Op == ONAME: - // - // 1. Disallow use of createComplexVars in createDwarfVars. - // It was not possible to reach that code for an ONAME before, - // because the DebugInfo was set only on the ODCLFUNC Func. - // Calling into it in the ONAME case causes an index out of bounds panic. - // - // 2. Do not populate apdecls. fn.Func.Dcl was in the ODCLFUNC Func, - // not the ONAME Func. Populating apdecls for the ONAME case results - // in selected being populated after createSimpleVars is called in - // createDwarfVars, and then that causes the loop to skip all the entries - // in dcl, meaning that the RecordAutoType calls don't happen. - // - // These two adjustments keep toolstash -cmp working for now. - // Deciding the right answer is, as they say, future work. - // - // We can tell the difference between the old ODCLFUNC and ONAME - // cases by looking at the infosym.Name. If it's empty, DebugInfo is - // being called from (*obj.Link).populateDWARF, which used to use - // the ODCLFUNC. If it's non-empty (the name will end in $abstract), - // DebugInfo is being called from (*obj.Link).DwarfAbstractFunc, - // which used to use the ONAME form. - isODCLFUNC := infosym.Name == "" - - var apdecls []*ir.Name - // Populate decls for fn. - if isODCLFUNC { - for _, n := range fn.Dcl { - if n.Op() != ir.ONAME { // might be OTYPE or OLITERAL - continue - } - switch n.Class_ { - case ir.PAUTO: - if !n.Used() { - // Text == nil -> generating abstract function - if fnsym.Func().Text != nil { - base.Fatalf("debuginfo unused node (AllocFrame should truncate fn.Func.Dcl)") - } - continue - } - case ir.PPARAM, ir.PPARAMOUT: - default: - continue - } - apdecls = append(apdecls, n) - fnsym.Func().RecordAutoType(ngotype(n).Linksym()) - } - } - - decls, dwarfVars := createDwarfVars(fnsym, isODCLFUNC, fn, apdecls) - - // For each type referenced by the functions auto vars but not - // already referenced by a dwarf var, attach an R_USETYPE relocation to - // the function symbol to insure that the type included in DWARF - // processing during linking. - typesyms := []*obj.LSym{} - for t, _ := range fnsym.Func().Autot { - typesyms = append(typesyms, t) - } - sort.Sort(obj.BySymName(typesyms)) - for _, sym := range typesyms { - r := obj.Addrel(infosym) - r.Sym = sym - r.Type = objabi.R_USETYPE - } - fnsym.Func().Autot = nil - - var varScopes []ir.ScopeID - for _, decl := range decls { - pos := declPos(decl) - varScopes = append(varScopes, findScope(fn.Marks, pos)) - } - - scopes := assembleScopes(fnsym, fn, dwarfVars, varScopes) - var inlcalls dwarf.InlCalls - if base.Flag.GenDwarfInl > 0 { - inlcalls = assembleInlines(fnsym, dwarfVars) - } - return scopes, inlcalls -} - -func declPos(decl *ir.Name) src.XPos { - if decl.Name().Defn != nil && (decl.Name().Captured() || decl.Name().Byval()) { - // It's not clear which position is correct for captured variables here: - // * decl.Pos is the wrong position for captured variables, in the inner - // function, but it is the right position in the outer function. - // * decl.Name.Defn is nil for captured variables that were arguments - // on the outer function, however the decl.Pos for those seems to be - // correct. - // * decl.Name.Defn is the "wrong" thing for variables declared in the - // header of a type switch, it's their position in the header, rather - // than the position of the case statement. In principle this is the - // right thing, but here we prefer the latter because it makes each - // instance of the header variable local to the lexical block of its - // case statement. - // This code is probably wrong for type switch variables that are also - // captured. - return decl.Name().Defn.Pos() - } - return decl.Pos() -} - -// createSimpleVars creates a DWARF entry for every variable declared in the -// function, claiming that they are permanently on the stack. -func createSimpleVars(fnsym *obj.LSym, apDecls []*ir.Name) ([]*ir.Name, []*dwarf.Var, map[*ir.Name]bool) { - var vars []*dwarf.Var - var decls []*ir.Name - selected := make(map[*ir.Name]bool) - for _, n := range apDecls { - if ir.IsAutoTmp(n) { - continue - } - - decls = append(decls, n) - vars = append(vars, createSimpleVar(fnsym, n)) - selected[n] = true - } - return decls, vars, selected -} - -func createSimpleVar(fnsym *obj.LSym, n *ir.Name) *dwarf.Var { - var abbrev int - var offs int64 - - switch n.Class_ { - case ir.PAUTO: - offs = n.FrameOffset() - abbrev = dwarf.DW_ABRV_AUTO - if base.Ctxt.FixedFrameSize() == 0 { - offs -= int64(types.PtrSize) - } - if objabi.Framepointer_enabled || objabi.GOARCH == "arm64" { - // There is a word space for FP on ARM64 even if the frame pointer is disabled - offs -= int64(types.PtrSize) - } - - case ir.PPARAM, ir.PPARAMOUT: - abbrev = dwarf.DW_ABRV_PARAM - offs = n.FrameOffset() + base.Ctxt.FixedFrameSize() - default: - base.Fatalf("createSimpleVar unexpected class %v for node %v", n.Class_, n) - } - - typename := dwarf.InfoPrefix + types.TypeSymName(n.Type()) - delete(fnsym.Func().Autot, ngotype(n).Linksym()) - inlIndex := 0 - if base.Flag.GenDwarfInl > 1 { - if n.Name().InlFormal() || n.Name().InlLocal() { - inlIndex = posInlIndex(n.Pos()) + 1 - if n.Name().InlFormal() { - abbrev = dwarf.DW_ABRV_PARAM - } - } - } - declpos := base.Ctxt.InnermostPos(declPos(n)) - return &dwarf.Var{ - Name: n.Sym().Name, - IsReturnValue: n.Class_ == ir.PPARAMOUT, - IsInlFormal: n.Name().InlFormal(), - Abbrev: abbrev, - StackOffset: int32(offs), - Type: base.Ctxt.Lookup(typename), - DeclFile: declpos.RelFilename(), - DeclLine: declpos.RelLine(), - DeclCol: declpos.Col(), - InlIndex: int32(inlIndex), - ChildIndex: -1, - } -} - -// createComplexVars creates recomposed DWARF vars with location lists, -// suitable for describing optimized code. -func createComplexVars(fnsym *obj.LSym, fn *ir.Func) ([]*ir.Name, []*dwarf.Var, map[*ir.Name]bool) { - debugInfo := fn.DebugInfo.(*ssa.FuncDebug) - - // Produce a DWARF variable entry for each user variable. - var decls []*ir.Name - var vars []*dwarf.Var - ssaVars := make(map[*ir.Name]bool) - - for varID, dvar := range debugInfo.Vars { - n := dvar - ssaVars[n] = true - for _, slot := range debugInfo.VarSlots[varID] { - ssaVars[debugInfo.Slots[slot].N] = true - } - - if dvar := createComplexVar(fnsym, fn, ssa.VarID(varID)); dvar != nil { - decls = append(decls, n) - vars = append(vars, dvar) - } - } - - return decls, vars, ssaVars -} - -// createDwarfVars process fn, returning a list of DWARF variables and the -// Nodes they represent. -func createDwarfVars(fnsym *obj.LSym, complexOK bool, fn *ir.Func, apDecls []*ir.Name) ([]*ir.Name, []*dwarf.Var) { - // Collect a raw list of DWARF vars. - var vars []*dwarf.Var - var decls []*ir.Name - var selected map[*ir.Name]bool - if base.Ctxt.Flag_locationlists && base.Ctxt.Flag_optimize && fn.DebugInfo != nil && complexOK { - decls, vars, selected = createComplexVars(fnsym, fn) - } else { - decls, vars, selected = createSimpleVars(fnsym, apDecls) - } - - dcl := apDecls - if fnsym.WasInlined() { - dcl = preInliningDcls(fnsym) - } - - // If optimization is enabled, the list above will typically be - // missing some of the original pre-optimization variables in the - // function (they may have been promoted to registers, folded into - // constants, dead-coded away, etc). Input arguments not eligible - // for SSA optimization are also missing. Here we add back in entries - // for selected missing vars. Note that the recipe below creates a - // conservative location. The idea here is that we want to - // communicate to the user that "yes, there is a variable named X - // in this function, but no, I don't have enough information to - // reliably report its contents." - // For non-SSA-able arguments, however, the correct information - // is known -- they have a single home on the stack. - for _, n := range dcl { - if _, found := selected[n]; found { - continue - } - c := n.Sym().Name[0] - if c == '.' || n.Type().IsUntyped() { - continue - } - if n.Class_ == ir.PPARAM && !canSSAType(n.Type()) { - // SSA-able args get location lists, and may move in and - // out of registers, so those are handled elsewhere. - // Autos and named output params seem to get handled - // with VARDEF, which creates location lists. - // Args not of SSA-able type are treated here; they - // are homed on the stack in a single place for the - // entire call. - vars = append(vars, createSimpleVar(fnsym, n)) - decls = append(decls, n) - continue - } - typename := dwarf.InfoPrefix + types.TypeSymName(n.Type()) - decls = append(decls, n) - abbrev := dwarf.DW_ABRV_AUTO_LOCLIST - isReturnValue := (n.Class_ == ir.PPARAMOUT) - if n.Class_ == ir.PPARAM || n.Class_ == ir.PPARAMOUT { - abbrev = dwarf.DW_ABRV_PARAM_LOCLIST - } else if n.Class_ == ir.PAUTOHEAP { - // If dcl in question has been promoted to heap, do a bit - // of extra work to recover original class (auto or param); - // see issue 30908. This insures that we get the proper - // signature in the abstract function DIE, but leaves a - // misleading location for the param (we want pointer-to-heap - // and not stack). - // TODO(thanm): generate a better location expression - stackcopy := n.Name().Stackcopy - if stackcopy != nil && (stackcopy.Class_ == ir.PPARAM || stackcopy.Class_ == ir.PPARAMOUT) { - abbrev = dwarf.DW_ABRV_PARAM_LOCLIST - isReturnValue = (stackcopy.Class_ == ir.PPARAMOUT) - } - } - inlIndex := 0 - if base.Flag.GenDwarfInl > 1 { - if n.Name().InlFormal() || n.Name().InlLocal() { - inlIndex = posInlIndex(n.Pos()) + 1 - if n.Name().InlFormal() { - abbrev = dwarf.DW_ABRV_PARAM_LOCLIST - } - } - } - declpos := base.Ctxt.InnermostPos(n.Pos()) - vars = append(vars, &dwarf.Var{ - Name: n.Sym().Name, - IsReturnValue: isReturnValue, - Abbrev: abbrev, - StackOffset: int32(n.FrameOffset()), - Type: base.Ctxt.Lookup(typename), - DeclFile: declpos.RelFilename(), - DeclLine: declpos.RelLine(), - DeclCol: declpos.Col(), - InlIndex: int32(inlIndex), - ChildIndex: -1, - }) - // Record go type of to insure that it gets emitted by the linker. - fnsym.Func().RecordAutoType(ngotype(n).Linksym()) - } - - return decls, vars -} - -// Given a function that was inlined at some point during the -// compilation, return a sorted list of nodes corresponding to the -// autos/locals in that function prior to inlining. If this is a -// function that is not local to the package being compiled, then the -// names of the variables may have been "versioned" to avoid conflicts -// with local vars; disregard this versioning when sorting. -func preInliningDcls(fnsym *obj.LSym) []*ir.Name { - fn := base.Ctxt.DwFixups.GetPrecursorFunc(fnsym).(*ir.Func) - var rdcl []*ir.Name - for _, n := range fn.Inl.Dcl { - c := n.Sym().Name[0] - // Avoid reporting "_" parameters, since if there are more than - // one, it can result in a collision later on, as in #23179. - if unversion(n.Sym().Name) == "_" || c == '.' || n.Type().IsUntyped() { - continue - } - rdcl = append(rdcl, n) - } - return rdcl -} - -// stackOffset returns the stack location of a LocalSlot relative to the -// stack pointer, suitable for use in a DWARF location entry. This has nothing -// to do with its offset in the user variable. -func stackOffset(slot ssa.LocalSlot) int32 { - n := slot.N - var off int64 - switch n.Class_ { - case ir.PAUTO: - off = n.FrameOffset() - if base.Ctxt.FixedFrameSize() == 0 { - off -= int64(types.PtrSize) - } - if objabi.Framepointer_enabled || objabi.GOARCH == "arm64" { - // There is a word space for FP on ARM64 even if the frame pointer is disabled - off -= int64(types.PtrSize) - } - case ir.PPARAM, ir.PPARAMOUT: - off = n.FrameOffset() + base.Ctxt.FixedFrameSize() - } - return int32(off + slot.Off) -} - -// createComplexVar builds a single DWARF variable entry and location list. -func createComplexVar(fnsym *obj.LSym, fn *ir.Func, varID ssa.VarID) *dwarf.Var { - debug := fn.DebugInfo.(*ssa.FuncDebug) - n := debug.Vars[varID] - - var abbrev int - switch n.Class_ { - case ir.PAUTO: - abbrev = dwarf.DW_ABRV_AUTO_LOCLIST - case ir.PPARAM, ir.PPARAMOUT: - abbrev = dwarf.DW_ABRV_PARAM_LOCLIST - default: - return nil - } - - gotype := ngotype(n).Linksym() - delete(fnsym.Func().Autot, gotype) - typename := dwarf.InfoPrefix + gotype.Name[len("type."):] - inlIndex := 0 - if base.Flag.GenDwarfInl > 1 { - if n.Name().InlFormal() || n.Name().InlLocal() { - inlIndex = posInlIndex(n.Pos()) + 1 - if n.Name().InlFormal() { - abbrev = dwarf.DW_ABRV_PARAM_LOCLIST - } - } - } - declpos := base.Ctxt.InnermostPos(n.Pos()) - dvar := &dwarf.Var{ - Name: n.Sym().Name, - IsReturnValue: n.Class_ == ir.PPARAMOUT, - IsInlFormal: n.Name().InlFormal(), - Abbrev: abbrev, - Type: base.Ctxt.Lookup(typename), - // The stack offset is used as a sorting key, so for decomposed - // variables just give it the first one. It's not used otherwise. - // This won't work well if the first slot hasn't been assigned a stack - // location, but it's not obvious how to do better. - StackOffset: stackOffset(debug.Slots[debug.VarSlots[varID][0]]), - DeclFile: declpos.RelFilename(), - DeclLine: declpos.RelLine(), - DeclCol: declpos.Col(), - InlIndex: int32(inlIndex), - ChildIndex: -1, - } - list := debug.LocationLists[varID] - if len(list) != 0 { - dvar.PutLocationList = func(listSym, startPC dwarf.Sym) { - debug.PutLocationList(list, base.Ctxt, listSym.(*obj.LSym), startPC.(*obj.LSym)) - } - } - return dvar -} - -// fieldtrack adds R_USEFIELD relocations to fnsym to record any -// struct fields that it used. -func fieldtrack(fnsym *obj.LSym, tracked map[*types.Sym]struct{}) { - if fnsym == nil { - return - } - if objabi.Fieldtrack_enabled == 0 || len(tracked) == 0 { - return - } - - trackSyms := make([]*types.Sym, 0, len(tracked)) - for sym := range tracked { - trackSyms = append(trackSyms, sym) - } - sort.Sort(symByName(trackSyms)) - for _, sym := range trackSyms { - r := obj.Addrel(fnsym) - r.Sym = sym.Linksym() - r.Type = objabi.R_USEFIELD - } -} - -type symByName []*types.Sym - -func (a symByName) Len() int { return len(a) } -func (a symByName) Less(i, j int) bool { return a[i].Name < a[j].Name } -func (a symByName) Swap(i, j int) { a[i], a[j] = a[j], a[i] } diff --git a/src/cmd/compile/internal/gc/pgen_test.go b/src/cmd/compile/internal/gc/pgen_test.go deleted file mode 100644 index 95c4b24fa1..0000000000 --- a/src/cmd/compile/internal/gc/pgen_test.go +++ /dev/null @@ -1,208 +0,0 @@ -// Copyright 2015 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package gc - -import ( - "cmd/compile/internal/ir" - "cmd/compile/internal/typecheck" - "cmd/compile/internal/types" - "cmd/internal/src" - "reflect" - "sort" - "testing" -) - -func typeWithoutPointers() *types.Type { - return types.NewStruct(types.NoPkg, []*types.Field{ - types.NewField(src.NoXPos, nil, types.New(types.TINT)), - }) -} - -func typeWithPointers() *types.Type { - return types.NewStruct(types.NoPkg, []*types.Field{ - types.NewField(src.NoXPos, nil, types.NewPtr(types.New(types.TINT))), - }) -} - -func markUsed(n *ir.Name) *ir.Name { - n.SetUsed(true) - return n -} - -func markNeedZero(n *ir.Name) *ir.Name { - n.SetNeedzero(true) - return n -} - -// Test all code paths for cmpstackvarlt. -func TestCmpstackvar(t *testing.T) { - nod := func(xoffset int64, t *types.Type, s *types.Sym, cl ir.Class) *ir.Name { - if s == nil { - s = &types.Sym{Name: "."} - } - n := typecheck.NewName(s) - n.SetType(t) - n.SetFrameOffset(xoffset) - n.Class_ = cl - return n - } - testdata := []struct { - a, b *ir.Name - lt bool - }{ - { - nod(0, nil, nil, ir.PAUTO), - nod(0, nil, nil, ir.PFUNC), - false, - }, - { - nod(0, nil, nil, ir.PFUNC), - nod(0, nil, nil, ir.PAUTO), - true, - }, - { - nod(0, nil, nil, ir.PFUNC), - nod(10, nil, nil, ir.PFUNC), - true, - }, - { - nod(20, nil, nil, ir.PFUNC), - nod(10, nil, nil, ir.PFUNC), - false, - }, - { - nod(10, nil, nil, ir.PFUNC), - nod(10, nil, nil, ir.PFUNC), - false, - }, - { - nod(10, nil, nil, ir.PPARAM), - nod(20, nil, nil, ir.PPARAMOUT), - true, - }, - { - nod(10, nil, nil, ir.PPARAMOUT), - nod(20, nil, nil, ir.PPARAM), - true, - }, - { - markUsed(nod(0, nil, nil, ir.PAUTO)), - nod(0, nil, nil, ir.PAUTO), - true, - }, - { - nod(0, nil, nil, ir.PAUTO), - markUsed(nod(0, nil, nil, ir.PAUTO)), - false, - }, - { - nod(0, typeWithoutPointers(), nil, ir.PAUTO), - nod(0, typeWithPointers(), nil, ir.PAUTO), - false, - }, - { - nod(0, typeWithPointers(), nil, ir.PAUTO), - nod(0, typeWithoutPointers(), nil, ir.PAUTO), - true, - }, - { - markNeedZero(nod(0, &types.Type{}, nil, ir.PAUTO)), - nod(0, &types.Type{}, nil, ir.PAUTO), - true, - }, - { - nod(0, &types.Type{}, nil, ir.PAUTO), - markNeedZero(nod(0, &types.Type{}, nil, ir.PAUTO)), - false, - }, - { - nod(0, &types.Type{Width: 1}, nil, ir.PAUTO), - nod(0, &types.Type{Width: 2}, nil, ir.PAUTO), - false, - }, - { - nod(0, &types.Type{Width: 2}, nil, ir.PAUTO), - nod(0, &types.Type{Width: 1}, nil, ir.PAUTO), - true, - }, - { - nod(0, &types.Type{}, &types.Sym{Name: "abc"}, ir.PAUTO), - nod(0, &types.Type{}, &types.Sym{Name: "xyz"}, ir.PAUTO), - true, - }, - { - nod(0, &types.Type{}, &types.Sym{Name: "abc"}, ir.PAUTO), - nod(0, &types.Type{}, &types.Sym{Name: "abc"}, ir.PAUTO), - false, - }, - { - nod(0, &types.Type{}, &types.Sym{Name: "xyz"}, ir.PAUTO), - nod(0, &types.Type{}, &types.Sym{Name: "abc"}, ir.PAUTO), - false, - }, - } - for _, d := range testdata { - got := cmpstackvarlt(d.a, d.b) - if got != d.lt { - t.Errorf("want %v < %v", d.a, d.b) - } - // If we expect a < b to be true, check that b < a is false. - if d.lt && cmpstackvarlt(d.b, d.a) { - t.Errorf("unexpected %v < %v", d.b, d.a) - } - } -} - -func TestStackvarSort(t *testing.T) { - nod := func(xoffset int64, t *types.Type, s *types.Sym, cl ir.Class) *ir.Name { - n := typecheck.NewName(s) - n.SetType(t) - n.SetFrameOffset(xoffset) - n.Class_ = cl - return n - } - inp := []*ir.Name{ - nod(0, &types.Type{}, &types.Sym{}, ir.PFUNC), - nod(0, &types.Type{}, &types.Sym{}, ir.PAUTO), - nod(0, &types.Type{}, &types.Sym{}, ir.PFUNC), - nod(10, &types.Type{}, &types.Sym{}, ir.PFUNC), - nod(20, &types.Type{}, &types.Sym{}, ir.PFUNC), - markUsed(nod(0, &types.Type{}, &types.Sym{}, ir.PAUTO)), - nod(0, typeWithoutPointers(), &types.Sym{}, ir.PAUTO), - nod(0, &types.Type{}, &types.Sym{}, ir.PAUTO), - markNeedZero(nod(0, &types.Type{}, &types.Sym{}, ir.PAUTO)), - nod(0, &types.Type{Width: 1}, &types.Sym{}, ir.PAUTO), - nod(0, &types.Type{Width: 2}, &types.Sym{}, ir.PAUTO), - nod(0, &types.Type{}, &types.Sym{Name: "abc"}, ir.PAUTO), - nod(0, &types.Type{}, &types.Sym{Name: "xyz"}, ir.PAUTO), - } - want := []*ir.Name{ - nod(0, &types.Type{}, &types.Sym{}, ir.PFUNC), - nod(0, &types.Type{}, &types.Sym{}, ir.PFUNC), - nod(10, &types.Type{}, &types.Sym{}, ir.PFUNC), - nod(20, &types.Type{}, &types.Sym{}, ir.PFUNC), - markUsed(nod(0, &types.Type{}, &types.Sym{}, ir.PAUTO)), - markNeedZero(nod(0, &types.Type{}, &types.Sym{}, ir.PAUTO)), - nod(0, &types.Type{Width: 2}, &types.Sym{}, ir.PAUTO), - nod(0, &types.Type{Width: 1}, &types.Sym{}, ir.PAUTO), - nod(0, &types.Type{}, &types.Sym{}, ir.PAUTO), - nod(0, &types.Type{}, &types.Sym{}, ir.PAUTO), - nod(0, &types.Type{}, &types.Sym{Name: "abc"}, ir.PAUTO), - nod(0, &types.Type{}, &types.Sym{Name: "xyz"}, ir.PAUTO), - nod(0, typeWithoutPointers(), &types.Sym{}, ir.PAUTO), - } - sort.Sort(byStackVar(inp)) - if !reflect.DeepEqual(want, inp) { - t.Error("sort failed") - for i := range inp { - g := inp[i] - w := want[i] - eq := reflect.DeepEqual(w, g) - if !eq { - t.Log(i, w, g) - } - } - } -} diff --git a/src/cmd/compile/internal/gc/phi.go b/src/cmd/compile/internal/gc/phi.go deleted file mode 100644 index 75ce18ff84..0000000000 --- a/src/cmd/compile/internal/gc/phi.go +++ /dev/null @@ -1,556 +0,0 @@ -// Copyright 2016 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package gc - -import ( - "cmd/compile/internal/ir" - "cmd/compile/internal/ssa" - "cmd/compile/internal/types" - "cmd/internal/src" - "container/heap" - "fmt" -) - -// This file contains the algorithm to place phi nodes in a function. -// For small functions, we use Braun, Buchwald, Hack, Leißa, Mallon, and Zwinkau. -// https://pp.info.uni-karlsruhe.de/uploads/publikationen/braun13cc.pdf -// For large functions, we use Sreedhar & Gao: A Linear Time Algorithm for Placing Φ-Nodes. -// http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.8.1979&rep=rep1&type=pdf - -const smallBlocks = 500 - -const debugPhi = false - -// FwdRefAux wraps an arbitrary ir.Node as an ssa.Aux for use with OpFwdref. -type FwdRefAux struct { - _ [0]func() // ensure ir.Node isn't compared for equality - N ir.Node -} - -func (FwdRefAux) CanBeAnSSAAux() {} - -// insertPhis finds all the places in the function where a phi is -// necessary and inserts them. -// Uses FwdRef ops to find all uses of variables, and s.defvars to find -// all definitions. -// Phi values are inserted, and all FwdRefs are changed to a Copy -// of the appropriate phi or definition. -// TODO: make this part of cmd/compile/internal/ssa somehow? -func (s *state) insertPhis() { - if len(s.f.Blocks) <= smallBlocks { - sps := simplePhiState{s: s, f: s.f, defvars: s.defvars} - sps.insertPhis() - return - } - ps := phiState{s: s, f: s.f, defvars: s.defvars} - ps.insertPhis() -} - -type phiState struct { - s *state // SSA state - f *ssa.Func // function to work on - defvars []map[ir.Node]*ssa.Value // defined variables at end of each block - - varnum map[ir.Node]int32 // variable numbering - - // properties of the dominator tree - idom []*ssa.Block // dominator parents - tree []domBlock // dominator child+sibling - level []int32 // level in dominator tree (0 = root or unreachable, 1 = children of root, ...) - - // scratch locations - priq blockHeap // priority queue of blocks, higher level (toward leaves) = higher priority - q []*ssa.Block // inner loop queue - queued *sparseSet // has been put in q - hasPhi *sparseSet // has a phi - hasDef *sparseSet // has a write of the variable we're processing - - // miscellaneous - placeholder *ssa.Value // value to use as a "not set yet" placeholder. -} - -func (s *phiState) insertPhis() { - if debugPhi { - fmt.Println(s.f.String()) - } - - // Find all the variables for which we need to match up reads & writes. - // This step prunes any basic-block-only variables from consideration. - // Generate a numbering for these variables. - s.varnum = map[ir.Node]int32{} - var vars []ir.Node - var vartypes []*types.Type - for _, b := range s.f.Blocks { - for _, v := range b.Values { - if v.Op != ssa.OpFwdRef { - continue - } - var_ := v.Aux.(FwdRefAux).N - - // Optimization: look back 1 block for the definition. - if len(b.Preds) == 1 { - c := b.Preds[0].Block() - if w := s.defvars[c.ID][var_]; w != nil { - v.Op = ssa.OpCopy - v.Aux = nil - v.AddArg(w) - continue - } - } - - if _, ok := s.varnum[var_]; ok { - continue - } - s.varnum[var_] = int32(len(vartypes)) - if debugPhi { - fmt.Printf("var%d = %v\n", len(vartypes), var_) - } - vars = append(vars, var_) - vartypes = append(vartypes, v.Type) - } - } - - if len(vartypes) == 0 { - return - } - - // Find all definitions of the variables we need to process. - // defs[n] contains all the blocks in which variable number n is assigned. - defs := make([][]*ssa.Block, len(vartypes)) - for _, b := range s.f.Blocks { - for var_ := range s.defvars[b.ID] { // TODO: encode defvars some other way (explicit ops)? make defvars[n] a slice instead of a map. - if n, ok := s.varnum[var_]; ok { - defs[n] = append(defs[n], b) - } - } - } - - // Make dominator tree. - s.idom = s.f.Idom() - s.tree = make([]domBlock, s.f.NumBlocks()) - for _, b := range s.f.Blocks { - p := s.idom[b.ID] - if p != nil { - s.tree[b.ID].sibling = s.tree[p.ID].firstChild - s.tree[p.ID].firstChild = b - } - } - // Compute levels in dominator tree. - // With parent pointers we can do a depth-first walk without - // any auxiliary storage. - s.level = make([]int32, s.f.NumBlocks()) - b := s.f.Entry -levels: - for { - if p := s.idom[b.ID]; p != nil { - s.level[b.ID] = s.level[p.ID] + 1 - if debugPhi { - fmt.Printf("level %s = %d\n", b, s.level[b.ID]) - } - } - if c := s.tree[b.ID].firstChild; c != nil { - b = c - continue - } - for { - if c := s.tree[b.ID].sibling; c != nil { - b = c - continue levels - } - b = s.idom[b.ID] - if b == nil { - break levels - } - } - } - - // Allocate scratch locations. - s.priq.level = s.level - s.q = make([]*ssa.Block, 0, s.f.NumBlocks()) - s.queued = newSparseSet(s.f.NumBlocks()) - s.hasPhi = newSparseSet(s.f.NumBlocks()) - s.hasDef = newSparseSet(s.f.NumBlocks()) - s.placeholder = s.s.entryNewValue0(ssa.OpUnknown, types.TypeInvalid) - - // Generate phi ops for each variable. - for n := range vartypes { - s.insertVarPhis(n, vars[n], defs[n], vartypes[n]) - } - - // Resolve FwdRefs to the correct write or phi. - s.resolveFwdRefs() - - // Erase variable numbers stored in AuxInt fields of phi ops. They are no longer needed. - for _, b := range s.f.Blocks { - for _, v := range b.Values { - if v.Op == ssa.OpPhi { - v.AuxInt = 0 - } - // Any remaining FwdRefs are dead code. - if v.Op == ssa.OpFwdRef { - v.Op = ssa.OpUnknown - v.Aux = nil - } - } - } -} - -func (s *phiState) insertVarPhis(n int, var_ ir.Node, defs []*ssa.Block, typ *types.Type) { - priq := &s.priq - q := s.q - queued := s.queued - queued.clear() - hasPhi := s.hasPhi - hasPhi.clear() - hasDef := s.hasDef - hasDef.clear() - - // Add defining blocks to priority queue. - for _, b := range defs { - priq.a = append(priq.a, b) - hasDef.add(b.ID) - if debugPhi { - fmt.Printf("def of var%d in %s\n", n, b) - } - } - heap.Init(priq) - - // Visit blocks defining variable n, from deepest to shallowest. - for len(priq.a) > 0 { - currentRoot := heap.Pop(priq).(*ssa.Block) - if debugPhi { - fmt.Printf("currentRoot %s\n", currentRoot) - } - // Walk subtree below definition. - // Skip subtrees we've done in previous iterations. - // Find edges exiting tree dominated by definition (the dominance frontier). - // Insert phis at target blocks. - if queued.contains(currentRoot.ID) { - s.s.Fatalf("root already in queue") - } - q = append(q, currentRoot) - queued.add(currentRoot.ID) - for len(q) > 0 { - b := q[len(q)-1] - q = q[:len(q)-1] - if debugPhi { - fmt.Printf(" processing %s\n", b) - } - - currentRootLevel := s.level[currentRoot.ID] - for _, e := range b.Succs { - c := e.Block() - // TODO: if the variable is dead at c, skip it. - if s.level[c.ID] > currentRootLevel { - // a D-edge, or an edge whose target is in currentRoot's subtree. - continue - } - if hasPhi.contains(c.ID) { - continue - } - // Add a phi to block c for variable n. - hasPhi.add(c.ID) - v := c.NewValue0I(currentRoot.Pos, ssa.OpPhi, typ, int64(n)) // TODO: line number right? - // Note: we store the variable number in the phi's AuxInt field. Used temporarily by phi building. - if var_.Op() == ir.ONAME { - s.s.addNamedValue(var_.(*ir.Name), v) - } - for range c.Preds { - v.AddArg(s.placeholder) // Actual args will be filled in by resolveFwdRefs. - } - if debugPhi { - fmt.Printf("new phi for var%d in %s: %s\n", n, c, v) - } - if !hasDef.contains(c.ID) { - // There's now a new definition of this variable in block c. - // Add it to the priority queue to explore. - heap.Push(priq, c) - hasDef.add(c.ID) - } - } - - // Visit children if they have not been visited yet. - for c := s.tree[b.ID].firstChild; c != nil; c = s.tree[c.ID].sibling { - if !queued.contains(c.ID) { - q = append(q, c) - queued.add(c.ID) - } - } - } - } -} - -// resolveFwdRefs links all FwdRef uses up to their nearest dominating definition. -func (s *phiState) resolveFwdRefs() { - // Do a depth-first walk of the dominator tree, keeping track - // of the most-recently-seen value for each variable. - - // Map from variable ID to SSA value at the current point of the walk. - values := make([]*ssa.Value, len(s.varnum)) - for i := range values { - values[i] = s.placeholder - } - - // Stack of work to do. - type stackEntry struct { - b *ssa.Block // block to explore - - // variable/value pair to reinstate on exit - n int32 // variable ID - v *ssa.Value - - // Note: only one of b or n,v will be set. - } - var stk []stackEntry - - stk = append(stk, stackEntry{b: s.f.Entry}) - for len(stk) > 0 { - work := stk[len(stk)-1] - stk = stk[:len(stk)-1] - - b := work.b - if b == nil { - // On exit from a block, this case will undo any assignments done below. - values[work.n] = work.v - continue - } - - // Process phis as new defs. They come before FwdRefs in this block. - for _, v := range b.Values { - if v.Op != ssa.OpPhi { - continue - } - n := int32(v.AuxInt) - // Remember the old assignment so we can undo it when we exit b. - stk = append(stk, stackEntry{n: n, v: values[n]}) - // Record the new assignment. - values[n] = v - } - - // Replace a FwdRef op with the current incoming value for its variable. - for _, v := range b.Values { - if v.Op != ssa.OpFwdRef { - continue - } - n := s.varnum[v.Aux.(FwdRefAux).N] - v.Op = ssa.OpCopy - v.Aux = nil - v.AddArg(values[n]) - } - - // Establish values for variables defined in b. - for var_, v := range s.defvars[b.ID] { - n, ok := s.varnum[var_] - if !ok { - // some variable not live across a basic block boundary. - continue - } - // Remember the old assignment so we can undo it when we exit b. - stk = append(stk, stackEntry{n: n, v: values[n]}) - // Record the new assignment. - values[n] = v - } - - // Replace phi args in successors with the current incoming value. - for _, e := range b.Succs { - c, i := e.Block(), e.Index() - for j := len(c.Values) - 1; j >= 0; j-- { - v := c.Values[j] - if v.Op != ssa.OpPhi { - break // All phis will be at the end of the block during phi building. - } - // Only set arguments that have been resolved. - // For very wide CFGs, this significantly speeds up phi resolution. - // See golang.org/issue/8225. - if w := values[v.AuxInt]; w.Op != ssa.OpUnknown { - v.SetArg(i, w) - } - } - } - - // Walk children in dominator tree. - for c := s.tree[b.ID].firstChild; c != nil; c = s.tree[c.ID].sibling { - stk = append(stk, stackEntry{b: c}) - } - } -} - -// domBlock contains extra per-block information to record the dominator tree. -type domBlock struct { - firstChild *ssa.Block // first child of block in dominator tree - sibling *ssa.Block // next child of parent in dominator tree -} - -// A block heap is used as a priority queue to implement the PiggyBank -// from Sreedhar and Gao. That paper uses an array which is better -// asymptotically but worse in the common case when the PiggyBank -// holds a sparse set of blocks. -type blockHeap struct { - a []*ssa.Block // block IDs in heap - level []int32 // depth in dominator tree (static, used for determining priority) -} - -func (h *blockHeap) Len() int { return len(h.a) } -func (h *blockHeap) Swap(i, j int) { a := h.a; a[i], a[j] = a[j], a[i] } - -func (h *blockHeap) Push(x interface{}) { - v := x.(*ssa.Block) - h.a = append(h.a, v) -} -func (h *blockHeap) Pop() interface{} { - old := h.a - n := len(old) - x := old[n-1] - h.a = old[:n-1] - return x -} -func (h *blockHeap) Less(i, j int) bool { - return h.level[h.a[i].ID] > h.level[h.a[j].ID] -} - -// TODO: stop walking the iterated domininance frontier when -// the variable is dead. Maybe detect that by checking if the -// node we're on is reverse dominated by all the reads? -// Reverse dominated by the highest common successor of all the reads? - -// copy of ../ssa/sparseset.go -// TODO: move this file to ../ssa, then use sparseSet there. -type sparseSet struct { - dense []ssa.ID - sparse []int32 -} - -// newSparseSet returns a sparseSet that can represent -// integers between 0 and n-1 -func newSparseSet(n int) *sparseSet { - return &sparseSet{dense: nil, sparse: make([]int32, n)} -} - -func (s *sparseSet) contains(x ssa.ID) bool { - i := s.sparse[x] - return i < int32(len(s.dense)) && s.dense[i] == x -} - -func (s *sparseSet) add(x ssa.ID) { - i := s.sparse[x] - if i < int32(len(s.dense)) && s.dense[i] == x { - return - } - s.dense = append(s.dense, x) - s.sparse[x] = int32(len(s.dense)) - 1 -} - -func (s *sparseSet) clear() { - s.dense = s.dense[:0] -} - -// Variant to use for small functions. -type simplePhiState struct { - s *state // SSA state - f *ssa.Func // function to work on - fwdrefs []*ssa.Value // list of FwdRefs to be processed - defvars []map[ir.Node]*ssa.Value // defined variables at end of each block - reachable []bool // which blocks are reachable -} - -func (s *simplePhiState) insertPhis() { - s.reachable = ssa.ReachableBlocks(s.f) - - // Find FwdRef ops. - for _, b := range s.f.Blocks { - for _, v := range b.Values { - if v.Op != ssa.OpFwdRef { - continue - } - s.fwdrefs = append(s.fwdrefs, v) - var_ := v.Aux.(FwdRefAux).N - if _, ok := s.defvars[b.ID][var_]; !ok { - s.defvars[b.ID][var_] = v // treat FwdDefs as definitions. - } - } - } - - var args []*ssa.Value - -loop: - for len(s.fwdrefs) > 0 { - v := s.fwdrefs[len(s.fwdrefs)-1] - s.fwdrefs = s.fwdrefs[:len(s.fwdrefs)-1] - b := v.Block - var_ := v.Aux.(FwdRefAux).N - if b == s.f.Entry { - // No variable should be live at entry. - s.s.Fatalf("Value live at entry. It shouldn't be. func %s, node %v, value %v", s.f.Name, var_, v) - } - if !s.reachable[b.ID] { - // This block is dead. - // It doesn't matter what we use here as long as it is well-formed. - v.Op = ssa.OpUnknown - v.Aux = nil - continue - } - // Find variable value on each predecessor. - args = args[:0] - for _, e := range b.Preds { - args = append(args, s.lookupVarOutgoing(e.Block(), v.Type, var_, v.Pos)) - } - - // Decide if we need a phi or not. We need a phi if there - // are two different args (which are both not v). - var w *ssa.Value - for _, a := range args { - if a == v { - continue // self-reference - } - if a == w { - continue // already have this witness - } - if w != nil { - // two witnesses, need a phi value - v.Op = ssa.OpPhi - v.AddArgs(args...) - v.Aux = nil - continue loop - } - w = a // save witness - } - if w == nil { - s.s.Fatalf("no witness for reachable phi %s", v) - } - // One witness. Make v a copy of w. - v.Op = ssa.OpCopy - v.Aux = nil - v.AddArg(w) - } -} - -// lookupVarOutgoing finds the variable's value at the end of block b. -func (s *simplePhiState) lookupVarOutgoing(b *ssa.Block, t *types.Type, var_ ir.Node, line src.XPos) *ssa.Value { - for { - if v := s.defvars[b.ID][var_]; v != nil { - return v - } - // The variable is not defined by b and we haven't looked it up yet. - // If b has exactly one predecessor, loop to look it up there. - // Otherwise, give up and insert a new FwdRef and resolve it later. - if len(b.Preds) != 1 { - break - } - b = b.Preds[0].Block() - if !s.reachable[b.ID] { - // This is rare; it happens with oddly interleaved infinite loops in dead code. - // See issue 19783. - break - } - } - // Generate a FwdRef for the variable and return that. - v := b.NewValue0A(line, ssa.OpFwdRef, t, FwdRefAux{N: var_}) - s.defvars[b.ID][var_] = v - if var_.Op() == ir.ONAME { - s.s.addNamedValue(var_.(*ir.Name), v) - } - s.fwdrefs = append(s.fwdrefs, v) - return v -} diff --git a/src/cmd/compile/internal/gc/racewalk.go b/src/cmd/compile/internal/gc/racewalk.go index 1ad3b9b422..c52bf1479b 100644 --- a/src/cmd/compile/internal/gc/racewalk.go +++ b/src/cmd/compile/internal/gc/racewalk.go @@ -7,6 +7,7 @@ package gc import ( "cmd/compile/internal/base" "cmd/compile/internal/ir" + "cmd/compile/internal/ssagen" "cmd/compile/internal/types" "cmd/internal/src" "cmd/internal/sys" @@ -25,7 +26,7 @@ func instrument(fn *ir.Func) { lno := base.Pos base.Pos = src.NoXPos - if thearch.LinkArch.Arch.Family != sys.AMD64 { + if ssagen.Arch.LinkArch.Arch.Family != sys.AMD64 { fn.Enter.Prepend(mkcall("racefuncenterfp", nil, nil)) fn.Exit.Append(mkcall("racefuncexit", nil, nil)) } else { diff --git a/src/cmd/compile/internal/gc/range.go b/src/cmd/compile/internal/gc/range.go index 4ba0654aef..2b2178a8bd 100644 --- a/src/cmd/compile/internal/gc/range.go +++ b/src/cmd/compile/internal/gc/range.go @@ -8,6 +8,7 @@ import ( "cmd/compile/internal/base" "cmd/compile/internal/ir" "cmd/compile/internal/reflectdata" + "cmd/compile/internal/ssagen" "cmd/compile/internal/typecheck" "cmd/compile/internal/types" "cmd/internal/sys" @@ -15,7 +16,7 @@ import ( ) func cheapComputableIndex(width int64) bool { - switch thearch.LinkArch.Family { + switch ssagen.Arch.LinkArch.Family { // MIPS does not have R+R addressing // Arm64 may lack ability to generate this code in our assembler, // but the architecture supports it. diff --git a/src/cmd/compile/internal/gc/ssa.go b/src/cmd/compile/internal/gc/ssa.go deleted file mode 100644 index 997bcb6d5e..0000000000 --- a/src/cmd/compile/internal/gc/ssa.go +++ /dev/null @@ -1,7455 +0,0 @@ -// Copyright 2015 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package gc - -import ( - "encoding/binary" - "fmt" - "go/constant" - "html" - "os" - "path/filepath" - "sort" - "strings" - - "bufio" - "bytes" - "cmd/compile/internal/base" - "cmd/compile/internal/ir" - "cmd/compile/internal/liveness" - "cmd/compile/internal/objw" - "cmd/compile/internal/reflectdata" - "cmd/compile/internal/ssa" - "cmd/compile/internal/staticdata" - "cmd/compile/internal/typecheck" - "cmd/compile/internal/types" - "cmd/internal/obj" - "cmd/internal/obj/x86" - "cmd/internal/objabi" - "cmd/internal/src" - "cmd/internal/sys" -) - -var ssaConfig *ssa.Config -var ssaCaches []ssa.Cache - -var ssaDump string // early copy of $GOSSAFUNC; the func name to dump output for -var ssaDir string // optional destination for ssa dump file -var ssaDumpStdout bool // whether to dump to stdout -var ssaDumpCFG string // generate CFGs for these phases -const ssaDumpFile = "ssa.html" - -// The max number of defers in a function using open-coded defers. We enforce this -// limit because the deferBits bitmask is currently a single byte (to minimize code size) -const maxOpenDefers = 8 - -// ssaDumpInlined holds all inlined functions when ssaDump contains a function name. -var ssaDumpInlined []*ir.Func - -func ssaDumpInline(fn *ir.Func) { - if ssaDump != "" && ssaDump == ir.FuncName(fn) { - ssaDumpInlined = append(ssaDumpInlined, fn) - } -} - -func initSSAEnv() { - ssaDump = os.Getenv("GOSSAFUNC") - ssaDir = os.Getenv("GOSSADIR") - if ssaDump != "" { - if strings.HasSuffix(ssaDump, "+") { - ssaDump = ssaDump[:len(ssaDump)-1] - ssaDumpStdout = true - } - spl := strings.Split(ssaDump, ":") - if len(spl) > 1 { - ssaDump = spl[0] - ssaDumpCFG = spl[1] - } - } -} - -func initssaconfig() { - types_ := ssa.NewTypes() - - if thearch.SoftFloat { - softfloatInit() - } - - // Generate a few pointer types that are uncommon in the frontend but common in the backend. - // Caching is disabled in the backend, so generating these here avoids allocations. - _ = types.NewPtr(types.Types[types.TINTER]) // *interface{} - _ = types.NewPtr(types.NewPtr(types.Types[types.TSTRING])) // **string - _ = types.NewPtr(types.NewSlice(types.Types[types.TINTER])) // *[]interface{} - _ = types.NewPtr(types.NewPtr(types.ByteType)) // **byte - _ = types.NewPtr(types.NewSlice(types.ByteType)) // *[]byte - _ = types.NewPtr(types.NewSlice(types.Types[types.TSTRING])) // *[]string - _ = types.NewPtr(types.NewPtr(types.NewPtr(types.Types[types.TUINT8]))) // ***uint8 - _ = types.NewPtr(types.Types[types.TINT16]) // *int16 - _ = types.NewPtr(types.Types[types.TINT64]) // *int64 - _ = types.NewPtr(types.ErrorType) // *error - types.NewPtrCacheEnabled = false - ssaConfig = ssa.NewConfig(base.Ctxt.Arch.Name, *types_, base.Ctxt, base.Flag.N == 0) - ssaConfig.SoftFloat = thearch.SoftFloat - ssaConfig.Race = base.Flag.Race - ssaCaches = make([]ssa.Cache, base.Flag.LowerC) - - // Set up some runtime functions we'll need to call. - ir.Syms.AssertE2I = typecheck.LookupRuntimeFunc("assertE2I") - ir.Syms.AssertE2I2 = typecheck.LookupRuntimeFunc("assertE2I2") - ir.Syms.AssertI2I = typecheck.LookupRuntimeFunc("assertI2I") - ir.Syms.AssertI2I2 = typecheck.LookupRuntimeFunc("assertI2I2") - ir.Syms.Deferproc = typecheck.LookupRuntimeFunc("deferproc") - ir.Syms.DeferprocStack = typecheck.LookupRuntimeFunc("deferprocStack") - ir.Syms.Deferreturn = typecheck.LookupRuntimeFunc("deferreturn") - ir.Syms.Duffcopy = typecheck.LookupRuntimeFunc("duffcopy") - ir.Syms.Duffzero = typecheck.LookupRuntimeFunc("duffzero") - ir.Syms.GCWriteBarrier = typecheck.LookupRuntimeFunc("gcWriteBarrier") - ir.Syms.Goschedguarded = typecheck.LookupRuntimeFunc("goschedguarded") - ir.Syms.Growslice = typecheck.LookupRuntimeFunc("growslice") - ir.Syms.Msanread = typecheck.LookupRuntimeFunc("msanread") - ir.Syms.Msanwrite = typecheck.LookupRuntimeFunc("msanwrite") - ir.Syms.Msanmove = typecheck.LookupRuntimeFunc("msanmove") - ir.Syms.Newobject = typecheck.LookupRuntimeFunc("newobject") - ir.Syms.Newproc = typecheck.LookupRuntimeFunc("newproc") - ir.Syms.Panicdivide = typecheck.LookupRuntimeFunc("panicdivide") - ir.Syms.PanicdottypeE = typecheck.LookupRuntimeFunc("panicdottypeE") - ir.Syms.PanicdottypeI = typecheck.LookupRuntimeFunc("panicdottypeI") - ir.Syms.Panicnildottype = typecheck.LookupRuntimeFunc("panicnildottype") - ir.Syms.Panicoverflow = typecheck.LookupRuntimeFunc("panicoverflow") - ir.Syms.Panicshift = typecheck.LookupRuntimeFunc("panicshift") - ir.Syms.Raceread = typecheck.LookupRuntimeFunc("raceread") - ir.Syms.Racereadrange = typecheck.LookupRuntimeFunc("racereadrange") - ir.Syms.Racewrite = typecheck.LookupRuntimeFunc("racewrite") - ir.Syms.Racewriterange = typecheck.LookupRuntimeFunc("racewriterange") - ir.Syms.X86HasPOPCNT = typecheck.LookupRuntimeVar("x86HasPOPCNT") // bool - ir.Syms.X86HasSSE41 = typecheck.LookupRuntimeVar("x86HasSSE41") // bool - ir.Syms.X86HasFMA = typecheck.LookupRuntimeVar("x86HasFMA") // bool - ir.Syms.ARMHasVFPv4 = typecheck.LookupRuntimeVar("armHasVFPv4") // bool - ir.Syms.ARM64HasATOMICS = typecheck.LookupRuntimeVar("arm64HasATOMICS") // bool - ir.Syms.Typedmemclr = typecheck.LookupRuntimeFunc("typedmemclr") - ir.Syms.Typedmemmove = typecheck.LookupRuntimeFunc("typedmemmove") - ir.Syms.Udiv = typecheck.LookupRuntimeVar("udiv") // asm func with special ABI - ir.Syms.WriteBarrier = typecheck.LookupRuntimeVar("writeBarrier") // struct { bool; ... } - ir.Syms.Zerobase = typecheck.LookupRuntimeVar("zerobase") - - // asm funcs with special ABI - if base.Ctxt.Arch.Name == "amd64" { - GCWriteBarrierReg = map[int16]*obj.LSym{ - x86.REG_AX: typecheck.LookupRuntimeFunc("gcWriteBarrier"), - x86.REG_CX: typecheck.LookupRuntimeFunc("gcWriteBarrierCX"), - x86.REG_DX: typecheck.LookupRuntimeFunc("gcWriteBarrierDX"), - x86.REG_BX: typecheck.LookupRuntimeFunc("gcWriteBarrierBX"), - x86.REG_BP: typecheck.LookupRuntimeFunc("gcWriteBarrierBP"), - x86.REG_SI: typecheck.LookupRuntimeFunc("gcWriteBarrierSI"), - x86.REG_R8: typecheck.LookupRuntimeFunc("gcWriteBarrierR8"), - x86.REG_R9: typecheck.LookupRuntimeFunc("gcWriteBarrierR9"), - } - } - - if thearch.LinkArch.Family == sys.Wasm { - BoundsCheckFunc[ssa.BoundsIndex] = typecheck.LookupRuntimeFunc("goPanicIndex") - BoundsCheckFunc[ssa.BoundsIndexU] = typecheck.LookupRuntimeFunc("goPanicIndexU") - BoundsCheckFunc[ssa.BoundsSliceAlen] = typecheck.LookupRuntimeFunc("goPanicSliceAlen") - BoundsCheckFunc[ssa.BoundsSliceAlenU] = typecheck.LookupRuntimeFunc("goPanicSliceAlenU") - BoundsCheckFunc[ssa.BoundsSliceAcap] = typecheck.LookupRuntimeFunc("goPanicSliceAcap") - BoundsCheckFunc[ssa.BoundsSliceAcapU] = typecheck.LookupRuntimeFunc("goPanicSliceAcapU") - BoundsCheckFunc[ssa.BoundsSliceB] = typecheck.LookupRuntimeFunc("goPanicSliceB") - BoundsCheckFunc[ssa.BoundsSliceBU] = typecheck.LookupRuntimeFunc("goPanicSliceBU") - BoundsCheckFunc[ssa.BoundsSlice3Alen] = typecheck.LookupRuntimeFunc("goPanicSlice3Alen") - BoundsCheckFunc[ssa.BoundsSlice3AlenU] = typecheck.LookupRuntimeFunc("goPanicSlice3AlenU") - BoundsCheckFunc[ssa.BoundsSlice3Acap] = typecheck.LookupRuntimeFunc("goPanicSlice3Acap") - BoundsCheckFunc[ssa.BoundsSlice3AcapU] = typecheck.LookupRuntimeFunc("goPanicSlice3AcapU") - BoundsCheckFunc[ssa.BoundsSlice3B] = typecheck.LookupRuntimeFunc("goPanicSlice3B") - BoundsCheckFunc[ssa.BoundsSlice3BU] = typecheck.LookupRuntimeFunc("goPanicSlice3BU") - BoundsCheckFunc[ssa.BoundsSlice3C] = typecheck.LookupRuntimeFunc("goPanicSlice3C") - BoundsCheckFunc[ssa.BoundsSlice3CU] = typecheck.LookupRuntimeFunc("goPanicSlice3CU") - } else { - BoundsCheckFunc[ssa.BoundsIndex] = typecheck.LookupRuntimeFunc("panicIndex") - BoundsCheckFunc[ssa.BoundsIndexU] = typecheck.LookupRuntimeFunc("panicIndexU") - BoundsCheckFunc[ssa.BoundsSliceAlen] = typecheck.LookupRuntimeFunc("panicSliceAlen") - BoundsCheckFunc[ssa.BoundsSliceAlenU] = typecheck.LookupRuntimeFunc("panicSliceAlenU") - BoundsCheckFunc[ssa.BoundsSliceAcap] = typecheck.LookupRuntimeFunc("panicSliceAcap") - BoundsCheckFunc[ssa.BoundsSliceAcapU] = typecheck.LookupRuntimeFunc("panicSliceAcapU") - BoundsCheckFunc[ssa.BoundsSliceB] = typecheck.LookupRuntimeFunc("panicSliceB") - BoundsCheckFunc[ssa.BoundsSliceBU] = typecheck.LookupRuntimeFunc("panicSliceBU") - BoundsCheckFunc[ssa.BoundsSlice3Alen] = typecheck.LookupRuntimeFunc("panicSlice3Alen") - BoundsCheckFunc[ssa.BoundsSlice3AlenU] = typecheck.LookupRuntimeFunc("panicSlice3AlenU") - BoundsCheckFunc[ssa.BoundsSlice3Acap] = typecheck.LookupRuntimeFunc("panicSlice3Acap") - BoundsCheckFunc[ssa.BoundsSlice3AcapU] = typecheck.LookupRuntimeFunc("panicSlice3AcapU") - BoundsCheckFunc[ssa.BoundsSlice3B] = typecheck.LookupRuntimeFunc("panicSlice3B") - BoundsCheckFunc[ssa.BoundsSlice3BU] = typecheck.LookupRuntimeFunc("panicSlice3BU") - BoundsCheckFunc[ssa.BoundsSlice3C] = typecheck.LookupRuntimeFunc("panicSlice3C") - BoundsCheckFunc[ssa.BoundsSlice3CU] = typecheck.LookupRuntimeFunc("panicSlice3CU") - } - if thearch.LinkArch.PtrSize == 4 { - ExtendCheckFunc[ssa.BoundsIndex] = typecheck.LookupRuntimeVar("panicExtendIndex") - ExtendCheckFunc[ssa.BoundsIndexU] = typecheck.LookupRuntimeVar("panicExtendIndexU") - ExtendCheckFunc[ssa.BoundsSliceAlen] = typecheck.LookupRuntimeVar("panicExtendSliceAlen") - ExtendCheckFunc[ssa.BoundsSliceAlenU] = typecheck.LookupRuntimeVar("panicExtendSliceAlenU") - ExtendCheckFunc[ssa.BoundsSliceAcap] = typecheck.LookupRuntimeVar("panicExtendSliceAcap") - ExtendCheckFunc[ssa.BoundsSliceAcapU] = typecheck.LookupRuntimeVar("panicExtendSliceAcapU") - ExtendCheckFunc[ssa.BoundsSliceB] = typecheck.LookupRuntimeVar("panicExtendSliceB") - ExtendCheckFunc[ssa.BoundsSliceBU] = typecheck.LookupRuntimeVar("panicExtendSliceBU") - ExtendCheckFunc[ssa.BoundsSlice3Alen] = typecheck.LookupRuntimeVar("panicExtendSlice3Alen") - ExtendCheckFunc[ssa.BoundsSlice3AlenU] = typecheck.LookupRuntimeVar("panicExtendSlice3AlenU") - ExtendCheckFunc[ssa.BoundsSlice3Acap] = typecheck.LookupRuntimeVar("panicExtendSlice3Acap") - ExtendCheckFunc[ssa.BoundsSlice3AcapU] = typecheck.LookupRuntimeVar("panicExtendSlice3AcapU") - ExtendCheckFunc[ssa.BoundsSlice3B] = typecheck.LookupRuntimeVar("panicExtendSlice3B") - ExtendCheckFunc[ssa.BoundsSlice3BU] = typecheck.LookupRuntimeVar("panicExtendSlice3BU") - ExtendCheckFunc[ssa.BoundsSlice3C] = typecheck.LookupRuntimeVar("panicExtendSlice3C") - ExtendCheckFunc[ssa.BoundsSlice3CU] = typecheck.LookupRuntimeVar("panicExtendSlice3CU") - } - - // Wasm (all asm funcs with special ABIs) - ir.Syms.WasmMove = typecheck.LookupRuntimeVar("wasmMove") - ir.Syms.WasmZero = typecheck.LookupRuntimeVar("wasmZero") - ir.Syms.WasmDiv = typecheck.LookupRuntimeVar("wasmDiv") - ir.Syms.WasmTruncS = typecheck.LookupRuntimeVar("wasmTruncS") - ir.Syms.WasmTruncU = typecheck.LookupRuntimeVar("wasmTruncU") - ir.Syms.SigPanic = typecheck.LookupRuntimeFunc("sigpanic") -} - -// getParam returns the Field of ith param of node n (which is a -// function/method/interface call), where the receiver of a method call is -// considered as the 0th parameter. This does not include the receiver of an -// interface call. -func getParam(n *ir.CallExpr, i int) *types.Field { - t := n.X.Type() - if n.Op() == ir.OCALLMETH { - if i == 0 { - return t.Recv() - } - return t.Params().Field(i - 1) - } - return t.Params().Field(i) -} - -// dvarint writes a varint v to the funcdata in symbol x and returns the new offset -func dvarint(x *obj.LSym, off int, v int64) int { - if v < 0 || v > 1e9 { - panic(fmt.Sprintf("dvarint: bad offset for funcdata - %v", v)) - } - if v < 1<<7 { - return objw.Uint8(x, off, uint8(v)) - } - off = objw.Uint8(x, off, uint8((v&127)|128)) - if v < 1<<14 { - return objw.Uint8(x, off, uint8(v>>7)) - } - off = objw.Uint8(x, off, uint8(((v>>7)&127)|128)) - if v < 1<<21 { - return objw.Uint8(x, off, uint8(v>>14)) - } - off = objw.Uint8(x, off, uint8(((v>>14)&127)|128)) - if v < 1<<28 { - return objw.Uint8(x, off, uint8(v>>21)) - } - off = objw.Uint8(x, off, uint8(((v>>21)&127)|128)) - return objw.Uint8(x, off, uint8(v>>28)) -} - -// emitOpenDeferInfo emits FUNCDATA information about the defers in a function -// that is using open-coded defers. This funcdata is used to determine the active -// defers in a function and execute those defers during panic processing. -// -// The funcdata is all encoded in varints (since values will almost always be less than -// 128, but stack offsets could potentially be up to 2Gbyte). All "locations" (offsets) -// for stack variables are specified as the number of bytes below varp (pointer to the -// top of the local variables) for their starting address. The format is: -// -// - Max total argument size among all the defers -// - Offset of the deferBits variable -// - Number of defers in the function -// - Information about each defer call, in reverse order of appearance in the function: -// - Total argument size of the call -// - Offset of the closure value to call -// - Number of arguments (including interface receiver or method receiver as first arg) -// - Information about each argument -// - Offset of the stored defer argument in this function's frame -// - Size of the argument -// - Offset of where argument should be placed in the args frame when making call -func (s *state) emitOpenDeferInfo() { - x := base.Ctxt.Lookup(s.curfn.LSym.Name + ".opendefer") - s.curfn.LSym.Func().OpenCodedDeferInfo = x - off := 0 - - // Compute maxargsize (max size of arguments for all defers) - // first, so we can output it first to the funcdata - var maxargsize int64 - for i := len(s.openDefers) - 1; i >= 0; i-- { - r := s.openDefers[i] - argsize := r.n.X.Type().ArgWidth() - if argsize > maxargsize { - maxargsize = argsize - } - } - off = dvarint(x, off, maxargsize) - off = dvarint(x, off, -s.deferBitsTemp.FrameOffset()) - off = dvarint(x, off, int64(len(s.openDefers))) - - // Write in reverse-order, for ease of running in that order at runtime - for i := len(s.openDefers) - 1; i >= 0; i-- { - r := s.openDefers[i] - off = dvarint(x, off, r.n.X.Type().ArgWidth()) - off = dvarint(x, off, -r.closureNode.FrameOffset()) - numArgs := len(r.argNodes) - if r.rcvrNode != nil { - // If there's an interface receiver, treat/place it as the first - // arg. (If there is a method receiver, it's already included as - // first arg in r.argNodes.) - numArgs++ - } - off = dvarint(x, off, int64(numArgs)) - if r.rcvrNode != nil { - off = dvarint(x, off, -r.rcvrNode.FrameOffset()) - off = dvarint(x, off, s.config.PtrSize) - off = dvarint(x, off, 0) - } - for j, arg := range r.argNodes { - f := getParam(r.n, j) - off = dvarint(x, off, -arg.FrameOffset()) - off = dvarint(x, off, f.Type.Size()) - off = dvarint(x, off, f.Offset) - } - } -} - -// buildssa builds an SSA function for fn. -// worker indicates which of the backend workers is doing the processing. -func buildssa(fn *ir.Func, worker int) *ssa.Func { - name := ir.FuncName(fn) - printssa := false - if ssaDump != "" { // match either a simple name e.g. "(*Reader).Reset", or a package.name e.g. "compress/gzip.(*Reader).Reset" - printssa = name == ssaDump || base.Ctxt.Pkgpath+"."+name == ssaDump - } - var astBuf *bytes.Buffer - if printssa { - astBuf = &bytes.Buffer{} - ir.FDumpList(astBuf, "buildssa-enter", fn.Enter) - ir.FDumpList(astBuf, "buildssa-body", fn.Body) - ir.FDumpList(astBuf, "buildssa-exit", fn.Exit) - if ssaDumpStdout { - fmt.Println("generating SSA for", name) - fmt.Print(astBuf.String()) - } - } - - var s state - s.pushLine(fn.Pos()) - defer s.popLine() - - s.hasdefer = fn.HasDefer() - if fn.Pragma&ir.CgoUnsafeArgs != 0 { - s.cgoUnsafeArgs = true - } - - fe := ssafn{ - curfn: fn, - log: printssa && ssaDumpStdout, - } - s.curfn = fn - - s.f = ssa.NewFunc(&fe) - s.config = ssaConfig - s.f.Type = fn.Type() - s.f.Config = ssaConfig - s.f.Cache = &ssaCaches[worker] - s.f.Cache.Reset() - s.f.Name = name - s.f.DebugTest = s.f.DebugHashMatch("GOSSAHASH") - s.f.PrintOrHtmlSSA = printssa - if fn.Pragma&ir.Nosplit != 0 { - s.f.NoSplit = true - } - s.panics = map[funcLine]*ssa.Block{} - s.softFloat = s.config.SoftFloat - - // Allocate starting block - s.f.Entry = s.f.NewBlock(ssa.BlockPlain) - s.f.Entry.Pos = fn.Pos() - - if printssa { - ssaDF := ssaDumpFile - if ssaDir != "" { - ssaDF = filepath.Join(ssaDir, base.Ctxt.Pkgpath+"."+name+".html") - ssaD := filepath.Dir(ssaDF) - os.MkdirAll(ssaD, 0755) - } - s.f.HTMLWriter = ssa.NewHTMLWriter(ssaDF, s.f, ssaDumpCFG) - // TODO: generate and print a mapping from nodes to values and blocks - dumpSourcesColumn(s.f.HTMLWriter, fn) - s.f.HTMLWriter.WriteAST("AST", astBuf) - } - - // Allocate starting values - s.labels = map[string]*ssaLabel{} - s.fwdVars = map[ir.Node]*ssa.Value{} - s.startmem = s.entryNewValue0(ssa.OpInitMem, types.TypeMem) - - s.hasOpenDefers = base.Flag.N == 0 && s.hasdefer && !s.curfn.OpenCodedDeferDisallowed() - switch { - case s.hasOpenDefers && (base.Ctxt.Flag_shared || base.Ctxt.Flag_dynlink) && base.Ctxt.Arch.Name == "386": - // Don't support open-coded defers for 386 ONLY when using shared - // libraries, because there is extra code (added by rewriteToUseGot()) - // preceding the deferreturn/ret code that is generated by gencallret() - // that we don't track correctly. - s.hasOpenDefers = false - } - if s.hasOpenDefers && len(s.curfn.Exit) > 0 { - // Skip doing open defers if there is any extra exit code (likely - // copying heap-allocated return values or race detection), since - // we will not generate that code in the case of the extra - // deferreturn/ret segment. - s.hasOpenDefers = false - } - if s.hasOpenDefers && - s.curfn.NumReturns*s.curfn.NumDefers > 15 { - // Since we are generating defer calls at every exit for - // open-coded defers, skip doing open-coded defers if there are - // too many returns (especially if there are multiple defers). - // Open-coded defers are most important for improving performance - // for smaller functions (which don't have many returns). - s.hasOpenDefers = false - } - - s.sp = s.entryNewValue0(ssa.OpSP, types.Types[types.TUINTPTR]) // TODO: use generic pointer type (unsafe.Pointer?) instead - s.sb = s.entryNewValue0(ssa.OpSB, types.Types[types.TUINTPTR]) - - s.startBlock(s.f.Entry) - s.vars[memVar] = s.startmem - if s.hasOpenDefers { - // Create the deferBits variable and stack slot. deferBits is a - // bitmask showing which of the open-coded defers in this function - // have been activated. - deferBitsTemp := typecheck.TempAt(src.NoXPos, s.curfn, types.Types[types.TUINT8]) - s.deferBitsTemp = deferBitsTemp - // For this value, AuxInt is initialized to zero by default - startDeferBits := s.entryNewValue0(ssa.OpConst8, types.Types[types.TUINT8]) - s.vars[deferBitsVar] = startDeferBits - s.deferBitsAddr = s.addr(deferBitsTemp) - s.store(types.Types[types.TUINT8], s.deferBitsAddr, startDeferBits) - // Make sure that the deferBits stack slot is kept alive (for use - // by panics) and stores to deferBits are not eliminated, even if - // all checking code on deferBits in the function exit can be - // eliminated, because the defer statements were all - // unconditional. - s.vars[memVar] = s.newValue1Apos(ssa.OpVarLive, types.TypeMem, deferBitsTemp, s.mem(), false) - } - - // Generate addresses of local declarations - s.decladdrs = map[*ir.Name]*ssa.Value{} - var args []ssa.Param - var results []ssa.Param - for _, n := range fn.Dcl { - switch n.Class_ { - case ir.PPARAM: - s.decladdrs[n] = s.entryNewValue2A(ssa.OpLocalAddr, types.NewPtr(n.Type()), n, s.sp, s.startmem) - args = append(args, ssa.Param{Type: n.Type(), Offset: int32(n.FrameOffset())}) - case ir.PPARAMOUT: - s.decladdrs[n] = s.entryNewValue2A(ssa.OpLocalAddr, types.NewPtr(n.Type()), n, s.sp, s.startmem) - results = append(results, ssa.Param{Type: n.Type(), Offset: int32(n.FrameOffset())}) - if s.canSSA(n) { - // Save ssa-able PPARAMOUT variables so we can - // store them back to the stack at the end of - // the function. - s.returns = append(s.returns, n) - } - case ir.PAUTO: - // processed at each use, to prevent Addr coming - // before the decl. - case ir.PAUTOHEAP: - // moved to heap - already handled by frontend - case ir.PFUNC: - // local function - already handled by frontend - default: - s.Fatalf("local variable with class %v unimplemented", n.Class_) - } - } - - // Populate SSAable arguments. - for _, n := range fn.Dcl { - if n.Class_ == ir.PPARAM && s.canSSA(n) { - v := s.newValue0A(ssa.OpArg, n.Type(), n) - s.vars[n] = v - s.addNamedValue(n, v) // This helps with debugging information, not needed for compilation itself. - } - } - - // Convert the AST-based IR to the SSA-based IR - s.stmtList(fn.Enter) - s.stmtList(fn.Body) - - // fallthrough to exit - if s.curBlock != nil { - s.pushLine(fn.Endlineno) - s.exit() - s.popLine() - } - - for _, b := range s.f.Blocks { - if b.Pos != src.NoXPos { - s.updateUnsetPredPos(b) - } - } - - s.insertPhis() - - // Main call to ssa package to compile function - ssa.Compile(s.f) - - if s.hasOpenDefers { - s.emitOpenDeferInfo() - } - - return s.f -} - -func dumpSourcesColumn(writer *ssa.HTMLWriter, fn *ir.Func) { - // Read sources of target function fn. - fname := base.Ctxt.PosTable.Pos(fn.Pos()).Filename() - targetFn, err := readFuncLines(fname, fn.Pos().Line(), fn.Endlineno.Line()) - if err != nil { - writer.Logf("cannot read sources for function %v: %v", fn, err) - } - - // Read sources of inlined functions. - var inlFns []*ssa.FuncLines - for _, fi := range ssaDumpInlined { - elno := fi.Endlineno - fname := base.Ctxt.PosTable.Pos(fi.Pos()).Filename() - fnLines, err := readFuncLines(fname, fi.Pos().Line(), elno.Line()) - if err != nil { - writer.Logf("cannot read sources for inlined function %v: %v", fi, err) - continue - } - inlFns = append(inlFns, fnLines) - } - - sort.Sort(ssa.ByTopo(inlFns)) - if targetFn != nil { - inlFns = append([]*ssa.FuncLines{targetFn}, inlFns...) - } - - writer.WriteSources("sources", inlFns) -} - -func readFuncLines(file string, start, end uint) (*ssa.FuncLines, error) { - f, err := os.Open(os.ExpandEnv(file)) - if err != nil { - return nil, err - } - defer f.Close() - var lines []string - ln := uint(1) - scanner := bufio.NewScanner(f) - for scanner.Scan() && ln <= end { - if ln >= start { - lines = append(lines, scanner.Text()) - } - ln++ - } - return &ssa.FuncLines{Filename: file, StartLineno: start, Lines: lines}, nil -} - -// updateUnsetPredPos propagates the earliest-value position information for b -// towards all of b's predecessors that need a position, and recurs on that -// predecessor if its position is updated. B should have a non-empty position. -func (s *state) updateUnsetPredPos(b *ssa.Block) { - if b.Pos == src.NoXPos { - s.Fatalf("Block %s should have a position", b) - } - bestPos := src.NoXPos - for _, e := range b.Preds { - p := e.Block() - if !p.LackingPos() { - continue - } - if bestPos == src.NoXPos { - bestPos = b.Pos - for _, v := range b.Values { - if v.LackingPos() { - continue - } - if v.Pos != src.NoXPos { - // Assume values are still in roughly textual order; - // TODO: could also seek minimum position? - bestPos = v.Pos - break - } - } - } - p.Pos = bestPos - s.updateUnsetPredPos(p) // We do not expect long chains of these, thus recursion is okay. - } -} - -// Information about each open-coded defer. -type openDeferInfo struct { - // The node representing the call of the defer - n *ir.CallExpr - // If defer call is closure call, the address of the argtmp where the - // closure is stored. - closure *ssa.Value - // The node representing the argtmp where the closure is stored - used for - // function, method, or interface call, to store a closure that panic - // processing can use for this defer. - closureNode *ir.Name - // If defer call is interface call, the address of the argtmp where the - // receiver is stored - rcvr *ssa.Value - // The node representing the argtmp where the receiver is stored - rcvrNode *ir.Name - // The addresses of the argtmps where the evaluated arguments of the defer - // function call are stored. - argVals []*ssa.Value - // The nodes representing the argtmps where the args of the defer are stored - argNodes []*ir.Name -} - -type state struct { - // configuration (arch) information - config *ssa.Config - - // function we're building - f *ssa.Func - - // Node for function - curfn *ir.Func - - // labels in f - labels map[string]*ssaLabel - - // unlabeled break and continue statement tracking - breakTo *ssa.Block // current target for plain break statement - continueTo *ssa.Block // current target for plain continue statement - - // current location where we're interpreting the AST - curBlock *ssa.Block - - // variable assignments in the current block (map from variable symbol to ssa value) - // *Node is the unique identifier (an ONAME Node) for the variable. - // TODO: keep a single varnum map, then make all of these maps slices instead? - vars map[ir.Node]*ssa.Value - - // fwdVars are variables that are used before they are defined in the current block. - // This map exists just to coalesce multiple references into a single FwdRef op. - // *Node is the unique identifier (an ONAME Node) for the variable. - fwdVars map[ir.Node]*ssa.Value - - // all defined variables at the end of each block. Indexed by block ID. - defvars []map[ir.Node]*ssa.Value - - // addresses of PPARAM and PPARAMOUT variables. - decladdrs map[*ir.Name]*ssa.Value - - // starting values. Memory, stack pointer, and globals pointer - startmem *ssa.Value - sp *ssa.Value - sb *ssa.Value - // value representing address of where deferBits autotmp is stored - deferBitsAddr *ssa.Value - deferBitsTemp *ir.Name - - // line number stack. The current line number is top of stack - line []src.XPos - // the last line number processed; it may have been popped - lastPos src.XPos - - // list of panic calls by function name and line number. - // Used to deduplicate panic calls. - panics map[funcLine]*ssa.Block - - // list of PPARAMOUT (return) variables. - returns []*ir.Name - - cgoUnsafeArgs bool - hasdefer bool // whether the function contains a defer statement - softFloat bool - hasOpenDefers bool // whether we are doing open-coded defers - - // If doing open-coded defers, list of info about the defer calls in - // scanning order. Hence, at exit we should run these defers in reverse - // order of this list - openDefers []*openDeferInfo - // For open-coded defers, this is the beginning and end blocks of the last - // defer exit code that we have generated so far. We use these to share - // code between exits if the shareDeferExits option (disabled by default) - // is on. - lastDeferExit *ssa.Block // Entry block of last defer exit code we generated - lastDeferFinalBlock *ssa.Block // Final block of last defer exit code we generated - lastDeferCount int // Number of defers encountered at that point - - prevCall *ssa.Value // the previous call; use this to tie results to the call op. -} - -type funcLine struct { - f *obj.LSym - base *src.PosBase - line uint -} - -type ssaLabel struct { - target *ssa.Block // block identified by this label - breakTarget *ssa.Block // block to break to in control flow node identified by this label - continueTarget *ssa.Block // block to continue to in control flow node identified by this label -} - -// label returns the label associated with sym, creating it if necessary. -func (s *state) label(sym *types.Sym) *ssaLabel { - lab := s.labels[sym.Name] - if lab == nil { - lab = new(ssaLabel) - s.labels[sym.Name] = lab - } - return lab -} - -func (s *state) Logf(msg string, args ...interface{}) { s.f.Logf(msg, args...) } -func (s *state) Log() bool { return s.f.Log() } -func (s *state) Fatalf(msg string, args ...interface{}) { - s.f.Frontend().Fatalf(s.peekPos(), msg, args...) -} -func (s *state) Warnl(pos src.XPos, msg string, args ...interface{}) { s.f.Warnl(pos, msg, args...) } -func (s *state) Debug_checknil() bool { return s.f.Frontend().Debug_checknil() } - -func ssaMarker(name string) *ir.Name { - return typecheck.NewName(&types.Sym{Name: name}) -} - -var ( - // marker node for the memory variable - memVar = ssaMarker("mem") - - // marker nodes for temporary variables - ptrVar = ssaMarker("ptr") - lenVar = ssaMarker("len") - newlenVar = ssaMarker("newlen") - capVar = ssaMarker("cap") - typVar = ssaMarker("typ") - okVar = ssaMarker("ok") - deferBitsVar = ssaMarker("deferBits") -) - -// startBlock sets the current block we're generating code in to b. -func (s *state) startBlock(b *ssa.Block) { - if s.curBlock != nil { - s.Fatalf("starting block %v when block %v has not ended", b, s.curBlock) - } - s.curBlock = b - s.vars = map[ir.Node]*ssa.Value{} - for n := range s.fwdVars { - delete(s.fwdVars, n) - } -} - -// endBlock marks the end of generating code for the current block. -// Returns the (former) current block. Returns nil if there is no current -// block, i.e. if no code flows to the current execution point. -func (s *state) endBlock() *ssa.Block { - b := s.curBlock - if b == nil { - return nil - } - for len(s.defvars) <= int(b.ID) { - s.defvars = append(s.defvars, nil) - } - s.defvars[b.ID] = s.vars - s.curBlock = nil - s.vars = nil - if b.LackingPos() { - // Empty plain blocks get the line of their successor (handled after all blocks created), - // except for increment blocks in For statements (handled in ssa conversion of OFOR), - // and for blocks ending in GOTO/BREAK/CONTINUE. - b.Pos = src.NoXPos - } else { - b.Pos = s.lastPos - } - return b -} - -// pushLine pushes a line number on the line number stack. -func (s *state) pushLine(line src.XPos) { - if !line.IsKnown() { - // the frontend may emit node with line number missing, - // use the parent line number in this case. - line = s.peekPos() - if base.Flag.K != 0 { - base.Warn("buildssa: unknown position (line 0)") - } - } else { - s.lastPos = line - } - - s.line = append(s.line, line) -} - -// popLine pops the top of the line number stack. -func (s *state) popLine() { - s.line = s.line[:len(s.line)-1] -} - -// peekPos peeks the top of the line number stack. -func (s *state) peekPos() src.XPos { - return s.line[len(s.line)-1] -} - -// newValue0 adds a new value with no arguments to the current block. -func (s *state) newValue0(op ssa.Op, t *types.Type) *ssa.Value { - return s.curBlock.NewValue0(s.peekPos(), op, t) -} - -// newValue0A adds a new value with no arguments and an aux value to the current block. -func (s *state) newValue0A(op ssa.Op, t *types.Type, aux ssa.Aux) *ssa.Value { - return s.curBlock.NewValue0A(s.peekPos(), op, t, aux) -} - -// newValue0I adds a new value with no arguments and an auxint value to the current block. -func (s *state) newValue0I(op ssa.Op, t *types.Type, auxint int64) *ssa.Value { - return s.curBlock.NewValue0I(s.peekPos(), op, t, auxint) -} - -// newValue1 adds a new value with one argument to the current block. -func (s *state) newValue1(op ssa.Op, t *types.Type, arg *ssa.Value) *ssa.Value { - return s.curBlock.NewValue1(s.peekPos(), op, t, arg) -} - -// newValue1A adds a new value with one argument and an aux value to the current block. -func (s *state) newValue1A(op ssa.Op, t *types.Type, aux ssa.Aux, arg *ssa.Value) *ssa.Value { - return s.curBlock.NewValue1A(s.peekPos(), op, t, aux, arg) -} - -// newValue1Apos adds a new value with one argument and an aux value to the current block. -// isStmt determines whether the created values may be a statement or not -// (i.e., false means never, yes means maybe). -func (s *state) newValue1Apos(op ssa.Op, t *types.Type, aux ssa.Aux, arg *ssa.Value, isStmt bool) *ssa.Value { - if isStmt { - return s.curBlock.NewValue1A(s.peekPos(), op, t, aux, arg) - } - return s.curBlock.NewValue1A(s.peekPos().WithNotStmt(), op, t, aux, arg) -} - -// newValue1I adds a new value with one argument and an auxint value to the current block. -func (s *state) newValue1I(op ssa.Op, t *types.Type, aux int64, arg *ssa.Value) *ssa.Value { - return s.curBlock.NewValue1I(s.peekPos(), op, t, aux, arg) -} - -// newValue2 adds a new value with two arguments to the current block. -func (s *state) newValue2(op ssa.Op, t *types.Type, arg0, arg1 *ssa.Value) *ssa.Value { - return s.curBlock.NewValue2(s.peekPos(), op, t, arg0, arg1) -} - -// newValue2A adds a new value with two arguments and an aux value to the current block. -func (s *state) newValue2A(op ssa.Op, t *types.Type, aux ssa.Aux, arg0, arg1 *ssa.Value) *ssa.Value { - return s.curBlock.NewValue2A(s.peekPos(), op, t, aux, arg0, arg1) -} - -// newValue2Apos adds a new value with two arguments and an aux value to the current block. -// isStmt determines whether the created values may be a statement or not -// (i.e., false means never, yes means maybe). -func (s *state) newValue2Apos(op ssa.Op, t *types.Type, aux ssa.Aux, arg0, arg1 *ssa.Value, isStmt bool) *ssa.Value { - if isStmt { - return s.curBlock.NewValue2A(s.peekPos(), op, t, aux, arg0, arg1) - } - return s.curBlock.NewValue2A(s.peekPos().WithNotStmt(), op, t, aux, arg0, arg1) -} - -// newValue2I adds a new value with two arguments and an auxint value to the current block. -func (s *state) newValue2I(op ssa.Op, t *types.Type, aux int64, arg0, arg1 *ssa.Value) *ssa.Value { - return s.curBlock.NewValue2I(s.peekPos(), op, t, aux, arg0, arg1) -} - -// newValue3 adds a new value with three arguments to the current block. -func (s *state) newValue3(op ssa.Op, t *types.Type, arg0, arg1, arg2 *ssa.Value) *ssa.Value { - return s.curBlock.NewValue3(s.peekPos(), op, t, arg0, arg1, arg2) -} - -// newValue3I adds a new value with three arguments and an auxint value to the current block. -func (s *state) newValue3I(op ssa.Op, t *types.Type, aux int64, arg0, arg1, arg2 *ssa.Value) *ssa.Value { - return s.curBlock.NewValue3I(s.peekPos(), op, t, aux, arg0, arg1, arg2) -} - -// newValue3A adds a new value with three arguments and an aux value to the current block. -func (s *state) newValue3A(op ssa.Op, t *types.Type, aux ssa.Aux, arg0, arg1, arg2 *ssa.Value) *ssa.Value { - return s.curBlock.NewValue3A(s.peekPos(), op, t, aux, arg0, arg1, arg2) -} - -// newValue3Apos adds a new value with three arguments and an aux value to the current block. -// isStmt determines whether the created values may be a statement or not -// (i.e., false means never, yes means maybe). -func (s *state) newValue3Apos(op ssa.Op, t *types.Type, aux ssa.Aux, arg0, arg1, arg2 *ssa.Value, isStmt bool) *ssa.Value { - if isStmt { - return s.curBlock.NewValue3A(s.peekPos(), op, t, aux, arg0, arg1, arg2) - } - return s.curBlock.NewValue3A(s.peekPos().WithNotStmt(), op, t, aux, arg0, arg1, arg2) -} - -// newValue4 adds a new value with four arguments to the current block. -func (s *state) newValue4(op ssa.Op, t *types.Type, arg0, arg1, arg2, arg3 *ssa.Value) *ssa.Value { - return s.curBlock.NewValue4(s.peekPos(), op, t, arg0, arg1, arg2, arg3) -} - -// newValue4 adds a new value with four arguments and an auxint value to the current block. -func (s *state) newValue4I(op ssa.Op, t *types.Type, aux int64, arg0, arg1, arg2, arg3 *ssa.Value) *ssa.Value { - return s.curBlock.NewValue4I(s.peekPos(), op, t, aux, arg0, arg1, arg2, arg3) -} - -// entryNewValue0 adds a new value with no arguments to the entry block. -func (s *state) entryNewValue0(op ssa.Op, t *types.Type) *ssa.Value { - return s.f.Entry.NewValue0(src.NoXPos, op, t) -} - -// entryNewValue0A adds a new value with no arguments and an aux value to the entry block. -func (s *state) entryNewValue0A(op ssa.Op, t *types.Type, aux ssa.Aux) *ssa.Value { - return s.f.Entry.NewValue0A(src.NoXPos, op, t, aux) -} - -// entryNewValue1 adds a new value with one argument to the entry block. -func (s *state) entryNewValue1(op ssa.Op, t *types.Type, arg *ssa.Value) *ssa.Value { - return s.f.Entry.NewValue1(src.NoXPos, op, t, arg) -} - -// entryNewValue1 adds a new value with one argument and an auxint value to the entry block. -func (s *state) entryNewValue1I(op ssa.Op, t *types.Type, auxint int64, arg *ssa.Value) *ssa.Value { - return s.f.Entry.NewValue1I(src.NoXPos, op, t, auxint, arg) -} - -// entryNewValue1A adds a new value with one argument and an aux value to the entry block. -func (s *state) entryNewValue1A(op ssa.Op, t *types.Type, aux ssa.Aux, arg *ssa.Value) *ssa.Value { - return s.f.Entry.NewValue1A(src.NoXPos, op, t, aux, arg) -} - -// entryNewValue2 adds a new value with two arguments to the entry block. -func (s *state) entryNewValue2(op ssa.Op, t *types.Type, arg0, arg1 *ssa.Value) *ssa.Value { - return s.f.Entry.NewValue2(src.NoXPos, op, t, arg0, arg1) -} - -// entryNewValue2A adds a new value with two arguments and an aux value to the entry block. -func (s *state) entryNewValue2A(op ssa.Op, t *types.Type, aux ssa.Aux, arg0, arg1 *ssa.Value) *ssa.Value { - return s.f.Entry.NewValue2A(src.NoXPos, op, t, aux, arg0, arg1) -} - -// const* routines add a new const value to the entry block. -func (s *state) constSlice(t *types.Type) *ssa.Value { - return s.f.ConstSlice(t) -} -func (s *state) constInterface(t *types.Type) *ssa.Value { - return s.f.ConstInterface(t) -} -func (s *state) constNil(t *types.Type) *ssa.Value { return s.f.ConstNil(t) } -func (s *state) constEmptyString(t *types.Type) *ssa.Value { - return s.f.ConstEmptyString(t) -} -func (s *state) constBool(c bool) *ssa.Value { - return s.f.ConstBool(types.Types[types.TBOOL], c) -} -func (s *state) constInt8(t *types.Type, c int8) *ssa.Value { - return s.f.ConstInt8(t, c) -} -func (s *state) constInt16(t *types.Type, c int16) *ssa.Value { - return s.f.ConstInt16(t, c) -} -func (s *state) constInt32(t *types.Type, c int32) *ssa.Value { - return s.f.ConstInt32(t, c) -} -func (s *state) constInt64(t *types.Type, c int64) *ssa.Value { - return s.f.ConstInt64(t, c) -} -func (s *state) constFloat32(t *types.Type, c float64) *ssa.Value { - return s.f.ConstFloat32(t, c) -} -func (s *state) constFloat64(t *types.Type, c float64) *ssa.Value { - return s.f.ConstFloat64(t, c) -} -func (s *state) constInt(t *types.Type, c int64) *ssa.Value { - if s.config.PtrSize == 8 { - return s.constInt64(t, c) - } - if int64(int32(c)) != c { - s.Fatalf("integer constant too big %d", c) - } - return s.constInt32(t, int32(c)) -} -func (s *state) constOffPtrSP(t *types.Type, c int64) *ssa.Value { - return s.f.ConstOffPtrSP(t, c, s.sp) -} - -// newValueOrSfCall* are wrappers around newValue*, which may create a call to a -// soft-float runtime function instead (when emitting soft-float code). -func (s *state) newValueOrSfCall1(op ssa.Op, t *types.Type, arg *ssa.Value) *ssa.Value { - if s.softFloat { - if c, ok := s.sfcall(op, arg); ok { - return c - } - } - return s.newValue1(op, t, arg) -} -func (s *state) newValueOrSfCall2(op ssa.Op, t *types.Type, arg0, arg1 *ssa.Value) *ssa.Value { - if s.softFloat { - if c, ok := s.sfcall(op, arg0, arg1); ok { - return c - } - } - return s.newValue2(op, t, arg0, arg1) -} - -type instrumentKind uint8 - -const ( - instrumentRead = iota - instrumentWrite - instrumentMove -) - -func (s *state) instrument(t *types.Type, addr *ssa.Value, kind instrumentKind) { - s.instrument2(t, addr, nil, kind) -} - -// instrumentFields instruments a read/write operation on addr. -// If it is instrumenting for MSAN and t is a struct type, it instruments -// operation for each field, instead of for the whole struct. -func (s *state) instrumentFields(t *types.Type, addr *ssa.Value, kind instrumentKind) { - if !base.Flag.MSan || !t.IsStruct() { - s.instrument(t, addr, kind) - return - } - for _, f := range t.Fields().Slice() { - if f.Sym.IsBlank() { - continue - } - offptr := s.newValue1I(ssa.OpOffPtr, types.NewPtr(f.Type), f.Offset, addr) - s.instrumentFields(f.Type, offptr, kind) - } -} - -func (s *state) instrumentMove(t *types.Type, dst, src *ssa.Value) { - if base.Flag.MSan { - s.instrument2(t, dst, src, instrumentMove) - } else { - s.instrument(t, src, instrumentRead) - s.instrument(t, dst, instrumentWrite) - } -} - -func (s *state) instrument2(t *types.Type, addr, addr2 *ssa.Value, kind instrumentKind) { - if !s.curfn.InstrumentBody() { - return - } - - w := t.Size() - if w == 0 { - return // can't race on zero-sized things - } - - if ssa.IsSanitizerSafeAddr(addr) { - return - } - - var fn *obj.LSym - needWidth := false - - if addr2 != nil && kind != instrumentMove { - panic("instrument2: non-nil addr2 for non-move instrumentation") - } - - if base.Flag.MSan { - switch kind { - case instrumentRead: - fn = ir.Syms.Msanread - case instrumentWrite: - fn = ir.Syms.Msanwrite - case instrumentMove: - fn = ir.Syms.Msanmove - default: - panic("unreachable") - } - needWidth = true - } else if base.Flag.Race && t.NumComponents(types.CountBlankFields) > 1 { - // for composite objects we have to write every address - // because a write might happen to any subobject. - // composites with only one element don't have subobjects, though. - switch kind { - case instrumentRead: - fn = ir.Syms.Racereadrange - case instrumentWrite: - fn = ir.Syms.Racewriterange - default: - panic("unreachable") - } - needWidth = true - } else if base.Flag.Race { - // for non-composite objects we can write just the start - // address, as any write must write the first byte. - switch kind { - case instrumentRead: - fn = ir.Syms.Raceread - case instrumentWrite: - fn = ir.Syms.Racewrite - default: - panic("unreachable") - } - } else { - panic("unreachable") - } - - args := []*ssa.Value{addr} - if addr2 != nil { - args = append(args, addr2) - } - if needWidth { - args = append(args, s.constInt(types.Types[types.TUINTPTR], w)) - } - s.rtcall(fn, true, nil, args...) -} - -func (s *state) load(t *types.Type, src *ssa.Value) *ssa.Value { - s.instrumentFields(t, src, instrumentRead) - return s.rawLoad(t, src) -} - -func (s *state) rawLoad(t *types.Type, src *ssa.Value) *ssa.Value { - return s.newValue2(ssa.OpLoad, t, src, s.mem()) -} - -func (s *state) store(t *types.Type, dst, val *ssa.Value) { - s.vars[memVar] = s.newValue3A(ssa.OpStore, types.TypeMem, t, dst, val, s.mem()) -} - -func (s *state) zero(t *types.Type, dst *ssa.Value) { - s.instrument(t, dst, instrumentWrite) - store := s.newValue2I(ssa.OpZero, types.TypeMem, t.Size(), dst, s.mem()) - store.Aux = t - s.vars[memVar] = store -} - -func (s *state) move(t *types.Type, dst, src *ssa.Value) { - s.instrumentMove(t, dst, src) - store := s.newValue3I(ssa.OpMove, types.TypeMem, t.Size(), dst, src, s.mem()) - store.Aux = t - s.vars[memVar] = store -} - -// stmtList converts the statement list n to SSA and adds it to s. -func (s *state) stmtList(l ir.Nodes) { - for _, n := range l { - s.stmt(n) - } -} - -// stmt converts the statement n to SSA and adds it to s. -func (s *state) stmt(n ir.Node) { - if !(n.Op() == ir.OVARKILL || n.Op() == ir.OVARLIVE || n.Op() == ir.OVARDEF) { - // OVARKILL, OVARLIVE, and OVARDEF are invisible to the programmer, so we don't use their line numbers to avoid confusion in debugging. - s.pushLine(n.Pos()) - defer s.popLine() - } - - // If s.curBlock is nil, and n isn't a label (which might have an associated goto somewhere), - // then this code is dead. Stop here. - if s.curBlock == nil && n.Op() != ir.OLABEL { - return - } - - s.stmtList(n.Init()) - switch n.Op() { - - case ir.OBLOCK: - n := n.(*ir.BlockStmt) - s.stmtList(n.List) - - // No-ops - case ir.ODCLCONST, ir.ODCLTYPE, ir.OFALL: - - // Expression statements - case ir.OCALLFUNC: - n := n.(*ir.CallExpr) - if ir.IsIntrinsicCall(n) { - s.intrinsicCall(n) - return - } - fallthrough - - case ir.OCALLMETH, ir.OCALLINTER: - n := n.(*ir.CallExpr) - s.callResult(n, callNormal) - if n.Op() == ir.OCALLFUNC && n.X.Op() == ir.ONAME && n.X.(*ir.Name).Class_ == ir.PFUNC { - if fn := n.X.Sym().Name; base.Flag.CompilingRuntime && fn == "throw" || - n.X.Sym().Pkg == ir.Pkgs.Runtime && (fn == "throwinit" || fn == "gopanic" || fn == "panicwrap" || fn == "block" || fn == "panicmakeslicelen" || fn == "panicmakeslicecap") { - m := s.mem() - b := s.endBlock() - b.Kind = ssa.BlockExit - b.SetControl(m) - // TODO: never rewrite OPANIC to OCALLFUNC in the - // first place. Need to wait until all backends - // go through SSA. - } - } - case ir.ODEFER: - n := n.(*ir.GoDeferStmt) - if base.Debug.Defer > 0 { - var defertype string - if s.hasOpenDefers { - defertype = "open-coded" - } else if n.Esc() == ir.EscNever { - defertype = "stack-allocated" - } else { - defertype = "heap-allocated" - } - base.WarnfAt(n.Pos(), "%s defer", defertype) - } - if s.hasOpenDefers { - s.openDeferRecord(n.Call.(*ir.CallExpr)) - } else { - d := callDefer - if n.Esc() == ir.EscNever { - d = callDeferStack - } - s.callResult(n.Call.(*ir.CallExpr), d) - } - case ir.OGO: - n := n.(*ir.GoDeferStmt) - s.callResult(n.Call.(*ir.CallExpr), callGo) - - case ir.OAS2DOTTYPE: - n := n.(*ir.AssignListStmt) - res, resok := s.dottype(n.Rhs[0].(*ir.TypeAssertExpr), true) - deref := false - if !canSSAType(n.Rhs[0].Type()) { - if res.Op != ssa.OpLoad { - s.Fatalf("dottype of non-load") - } - mem := s.mem() - if mem.Op == ssa.OpVarKill { - mem = mem.Args[0] - } - if res.Args[1] != mem { - s.Fatalf("memory no longer live from 2-result dottype load") - } - deref = true - res = res.Args[0] - } - s.assign(n.Lhs[0], res, deref, 0) - s.assign(n.Lhs[1], resok, false, 0) - return - - case ir.OAS2FUNC: - // We come here only when it is an intrinsic call returning two values. - n := n.(*ir.AssignListStmt) - call := n.Rhs[0].(*ir.CallExpr) - if !ir.IsIntrinsicCall(call) { - s.Fatalf("non-intrinsic AS2FUNC not expanded %v", call) - } - v := s.intrinsicCall(call) - v1 := s.newValue1(ssa.OpSelect0, n.Lhs[0].Type(), v) - v2 := s.newValue1(ssa.OpSelect1, n.Lhs[1].Type(), v) - s.assign(n.Lhs[0], v1, false, 0) - s.assign(n.Lhs[1], v2, false, 0) - return - - case ir.ODCL: - n := n.(*ir.Decl) - if n.X.(*ir.Name).Class_ == ir.PAUTOHEAP { - s.Fatalf("DCL %v", n) - } - - case ir.OLABEL: - n := n.(*ir.LabelStmt) - sym := n.Label - lab := s.label(sym) - - // The label might already have a target block via a goto. - if lab.target == nil { - lab.target = s.f.NewBlock(ssa.BlockPlain) - } - - // Go to that label. - // (We pretend "label:" is preceded by "goto label", unless the predecessor is unreachable.) - if s.curBlock != nil { - b := s.endBlock() - b.AddEdgeTo(lab.target) - } - s.startBlock(lab.target) - - case ir.OGOTO: - n := n.(*ir.BranchStmt) - sym := n.Label - - lab := s.label(sym) - if lab.target == nil { - lab.target = s.f.NewBlock(ssa.BlockPlain) - } - - b := s.endBlock() - b.Pos = s.lastPos.WithIsStmt() // Do this even if b is an empty block. - b.AddEdgeTo(lab.target) - - case ir.OAS: - n := n.(*ir.AssignStmt) - if n.X == n.Y && n.X.Op() == ir.ONAME { - // An x=x assignment. No point in doing anything - // here. In addition, skipping this assignment - // prevents generating: - // VARDEF x - // COPY x -> x - // which is bad because x is incorrectly considered - // dead before the vardef. See issue #14904. - return - } - - // Evaluate RHS. - rhs := n.Y - if rhs != nil { - switch rhs.Op() { - case ir.OSTRUCTLIT, ir.OARRAYLIT, ir.OSLICELIT: - // All literals with nonzero fields have already been - // rewritten during walk. Any that remain are just T{} - // or equivalents. Use the zero value. - if !ir.IsZero(rhs) { - s.Fatalf("literal with nonzero value in SSA: %v", rhs) - } - rhs = nil - case ir.OAPPEND: - rhs := rhs.(*ir.CallExpr) - // Check whether we're writing the result of an append back to the same slice. - // If so, we handle it specially to avoid write barriers on the fast - // (non-growth) path. - if !ir.SameSafeExpr(n.X, rhs.Args[0]) || base.Flag.N != 0 { - break - } - // If the slice can be SSA'd, it'll be on the stack, - // so there will be no write barriers, - // so there's no need to attempt to prevent them. - if s.canSSA(n.X) { - if base.Debug.Append > 0 { // replicating old diagnostic message - base.WarnfAt(n.Pos(), "append: len-only update (in local slice)") - } - break - } - if base.Debug.Append > 0 { - base.WarnfAt(n.Pos(), "append: len-only update") - } - s.append(rhs, true) - return - } - } - - if ir.IsBlank(n.X) { - // _ = rhs - // Just evaluate rhs for side-effects. - if rhs != nil { - s.expr(rhs) - } - return - } - - var t *types.Type - if n.Y != nil { - t = n.Y.Type() - } else { - t = n.X.Type() - } - - var r *ssa.Value - deref := !canSSAType(t) - if deref { - if rhs == nil { - r = nil // Signal assign to use OpZero. - } else { - r = s.addr(rhs) - } - } else { - if rhs == nil { - r = s.zeroVal(t) - } else { - r = s.expr(rhs) - } - } - - var skip skipMask - if rhs != nil && (rhs.Op() == ir.OSLICE || rhs.Op() == ir.OSLICE3 || rhs.Op() == ir.OSLICESTR) && ir.SameSafeExpr(rhs.(*ir.SliceExpr).X, n.X) { - // We're assigning a slicing operation back to its source. - // Don't write back fields we aren't changing. See issue #14855. - rhs := rhs.(*ir.SliceExpr) - i, j, k := rhs.SliceBounds() - if i != nil && (i.Op() == ir.OLITERAL && i.Val().Kind() == constant.Int && ir.Int64Val(i) == 0) { - // [0:...] is the same as [:...] - i = nil - } - // TODO: detect defaults for len/cap also. - // Currently doesn't really work because (*p)[:len(*p)] appears here as: - // tmp = len(*p) - // (*p)[:tmp] - //if j != nil && (j.Op == OLEN && samesafeexpr(j.Left, n.Left)) { - // j = nil - //} - //if k != nil && (k.Op == OCAP && samesafeexpr(k.Left, n.Left)) { - // k = nil - //} - if i == nil { - skip |= skipPtr - if j == nil { - skip |= skipLen - } - if k == nil { - skip |= skipCap - } - } - } - - s.assign(n.X, r, deref, skip) - - case ir.OIF: - n := n.(*ir.IfStmt) - if ir.IsConst(n.Cond, constant.Bool) { - s.stmtList(n.Cond.Init()) - if ir.BoolVal(n.Cond) { - s.stmtList(n.Body) - } else { - s.stmtList(n.Else) - } - break - } - - bEnd := s.f.NewBlock(ssa.BlockPlain) - var likely int8 - if n.Likely { - likely = 1 - } - var bThen *ssa.Block - if len(n.Body) != 0 { - bThen = s.f.NewBlock(ssa.BlockPlain) - } else { - bThen = bEnd - } - var bElse *ssa.Block - if len(n.Else) != 0 { - bElse = s.f.NewBlock(ssa.BlockPlain) - } else { - bElse = bEnd - } - s.condBranch(n.Cond, bThen, bElse, likely) - - if len(n.Body) != 0 { - s.startBlock(bThen) - s.stmtList(n.Body) - if b := s.endBlock(); b != nil { - b.AddEdgeTo(bEnd) - } - } - if len(n.Else) != 0 { - s.startBlock(bElse) - s.stmtList(n.Else) - if b := s.endBlock(); b != nil { - b.AddEdgeTo(bEnd) - } - } - s.startBlock(bEnd) - - case ir.ORETURN: - n := n.(*ir.ReturnStmt) - s.stmtList(n.Results) - b := s.exit() - b.Pos = s.lastPos.WithIsStmt() - - case ir.ORETJMP: - n := n.(*ir.BranchStmt) - b := s.exit() - b.Kind = ssa.BlockRetJmp // override BlockRet - b.Aux = callTargetLSym(n.Label, s.curfn.LSym) - - case ir.OCONTINUE, ir.OBREAK: - n := n.(*ir.BranchStmt) - var to *ssa.Block - if n.Label == nil { - // plain break/continue - switch n.Op() { - case ir.OCONTINUE: - to = s.continueTo - case ir.OBREAK: - to = s.breakTo - } - } else { - // labeled break/continue; look up the target - sym := n.Label - lab := s.label(sym) - switch n.Op() { - case ir.OCONTINUE: - to = lab.continueTarget - case ir.OBREAK: - to = lab.breakTarget - } - } - - b := s.endBlock() - b.Pos = s.lastPos.WithIsStmt() // Do this even if b is an empty block. - b.AddEdgeTo(to) - - case ir.OFOR, ir.OFORUNTIL: - // OFOR: for Ninit; Left; Right { Nbody } - // cond (Left); body (Nbody); incr (Right) - // - // OFORUNTIL: for Ninit; Left; Right; List { Nbody } - // => body: { Nbody }; incr: Right; if Left { lateincr: List; goto body }; end: - n := n.(*ir.ForStmt) - bCond := s.f.NewBlock(ssa.BlockPlain) - bBody := s.f.NewBlock(ssa.BlockPlain) - bIncr := s.f.NewBlock(ssa.BlockPlain) - bEnd := s.f.NewBlock(ssa.BlockPlain) - - // ensure empty for loops have correct position; issue #30167 - bBody.Pos = n.Pos() - - // first, jump to condition test (OFOR) or body (OFORUNTIL) - b := s.endBlock() - if n.Op() == ir.OFOR { - b.AddEdgeTo(bCond) - // generate code to test condition - s.startBlock(bCond) - if n.Cond != nil { - s.condBranch(n.Cond, bBody, bEnd, 1) - } else { - b := s.endBlock() - b.Kind = ssa.BlockPlain - b.AddEdgeTo(bBody) - } - - } else { - b.AddEdgeTo(bBody) - } - - // set up for continue/break in body - prevContinue := s.continueTo - prevBreak := s.breakTo - s.continueTo = bIncr - s.breakTo = bEnd - var lab *ssaLabel - if sym := n.Label; sym != nil { - // labeled for loop - lab = s.label(sym) - lab.continueTarget = bIncr - lab.breakTarget = bEnd - } - - // generate body - s.startBlock(bBody) - s.stmtList(n.Body) - - // tear down continue/break - s.continueTo = prevContinue - s.breakTo = prevBreak - if lab != nil { - lab.continueTarget = nil - lab.breakTarget = nil - } - - // done with body, goto incr - if b := s.endBlock(); b != nil { - b.AddEdgeTo(bIncr) - } - - // generate incr (and, for OFORUNTIL, condition) - s.startBlock(bIncr) - if n.Post != nil { - s.stmt(n.Post) - } - if n.Op() == ir.OFOR { - if b := s.endBlock(); b != nil { - b.AddEdgeTo(bCond) - // It can happen that bIncr ends in a block containing only VARKILL, - // and that muddles the debugging experience. - if n.Op() != ir.OFORUNTIL && b.Pos == src.NoXPos { - b.Pos = bCond.Pos - } - } - } else { - // bCond is unused in OFORUNTIL, so repurpose it. - bLateIncr := bCond - // test condition - s.condBranch(n.Cond, bLateIncr, bEnd, 1) - // generate late increment - s.startBlock(bLateIncr) - s.stmtList(n.Late) - s.endBlock().AddEdgeTo(bBody) - } - - s.startBlock(bEnd) - - case ir.OSWITCH, ir.OSELECT: - // These have been mostly rewritten by the front end into their Nbody fields. - // Our main task is to correctly hook up any break statements. - bEnd := s.f.NewBlock(ssa.BlockPlain) - - prevBreak := s.breakTo - s.breakTo = bEnd - var sym *types.Sym - var body ir.Nodes - if n.Op() == ir.OSWITCH { - n := n.(*ir.SwitchStmt) - sym = n.Label - body = n.Compiled - } else { - n := n.(*ir.SelectStmt) - sym = n.Label - body = n.Compiled - } - - var lab *ssaLabel - if sym != nil { - // labeled - lab = s.label(sym) - lab.breakTarget = bEnd - } - - // generate body code - s.stmtList(body) - - s.breakTo = prevBreak - if lab != nil { - lab.breakTarget = nil - } - - // walk adds explicit OBREAK nodes to the end of all reachable code paths. - // If we still have a current block here, then mark it unreachable. - if s.curBlock != nil { - m := s.mem() - b := s.endBlock() - b.Kind = ssa.BlockExit - b.SetControl(m) - } - s.startBlock(bEnd) - - case ir.OVARDEF: - n := n.(*ir.UnaryExpr) - if !s.canSSA(n.X) { - s.vars[memVar] = s.newValue1Apos(ssa.OpVarDef, types.TypeMem, n.X.(*ir.Name), s.mem(), false) - } - case ir.OVARKILL: - // Insert a varkill op to record that a variable is no longer live. - // We only care about liveness info at call sites, so putting the - // varkill in the store chain is enough to keep it correctly ordered - // with respect to call ops. - n := n.(*ir.UnaryExpr) - if !s.canSSA(n.X) { - s.vars[memVar] = s.newValue1Apos(ssa.OpVarKill, types.TypeMem, n.X.(*ir.Name), s.mem(), false) - } - - case ir.OVARLIVE: - // Insert a varlive op to record that a variable is still live. - n := n.(*ir.UnaryExpr) - v := n.X.(*ir.Name) - if !v.Addrtaken() { - s.Fatalf("VARLIVE variable %v must have Addrtaken set", v) - } - switch v.Class_ { - case ir.PAUTO, ir.PPARAM, ir.PPARAMOUT: - default: - s.Fatalf("VARLIVE variable %v must be Auto or Arg", v) - } - s.vars[memVar] = s.newValue1A(ssa.OpVarLive, types.TypeMem, v, s.mem()) - - case ir.OCHECKNIL: - n := n.(*ir.UnaryExpr) - p := s.expr(n.X) - s.nilCheck(p) - - case ir.OINLMARK: - n := n.(*ir.InlineMarkStmt) - s.newValue1I(ssa.OpInlMark, types.TypeVoid, n.Index, s.mem()) - - default: - s.Fatalf("unhandled stmt %v", n.Op()) - } -} - -// If true, share as many open-coded defer exits as possible (with the downside of -// worse line-number information) -const shareDeferExits = false - -// exit processes any code that needs to be generated just before returning. -// It returns a BlockRet block that ends the control flow. Its control value -// will be set to the final memory state. -func (s *state) exit() *ssa.Block { - if s.hasdefer { - if s.hasOpenDefers { - if shareDeferExits && s.lastDeferExit != nil && len(s.openDefers) == s.lastDeferCount { - if s.curBlock.Kind != ssa.BlockPlain { - panic("Block for an exit should be BlockPlain") - } - s.curBlock.AddEdgeTo(s.lastDeferExit) - s.endBlock() - return s.lastDeferFinalBlock - } - s.openDeferExit() - } else { - s.rtcall(ir.Syms.Deferreturn, true, nil) - } - } - - // Run exit code. Typically, this code copies heap-allocated PPARAMOUT - // variables back to the stack. - s.stmtList(s.curfn.Exit) - - // Store SSAable PPARAMOUT variables back to stack locations. - for _, n := range s.returns { - addr := s.decladdrs[n] - val := s.variable(n, n.Type()) - s.vars[memVar] = s.newValue1A(ssa.OpVarDef, types.TypeMem, n, s.mem()) - s.store(n.Type(), addr, val) - // TODO: if val is ever spilled, we'd like to use the - // PPARAMOUT slot for spilling it. That won't happen - // currently. - } - - // Do actual return. - m := s.mem() - b := s.endBlock() - b.Kind = ssa.BlockRet - b.SetControl(m) - if s.hasdefer && s.hasOpenDefers { - s.lastDeferFinalBlock = b - } - return b -} - -type opAndType struct { - op ir.Op - etype types.Kind -} - -var opToSSA = map[opAndType]ssa.Op{ - opAndType{ir.OADD, types.TINT8}: ssa.OpAdd8, - opAndType{ir.OADD, types.TUINT8}: ssa.OpAdd8, - opAndType{ir.OADD, types.TINT16}: ssa.OpAdd16, - opAndType{ir.OADD, types.TUINT16}: ssa.OpAdd16, - opAndType{ir.OADD, types.TINT32}: ssa.OpAdd32, - opAndType{ir.OADD, types.TUINT32}: ssa.OpAdd32, - opAndType{ir.OADD, types.TINT64}: ssa.OpAdd64, - opAndType{ir.OADD, types.TUINT64}: ssa.OpAdd64, - opAndType{ir.OADD, types.TFLOAT32}: ssa.OpAdd32F, - opAndType{ir.OADD, types.TFLOAT64}: ssa.OpAdd64F, - - opAndType{ir.OSUB, types.TINT8}: ssa.OpSub8, - opAndType{ir.OSUB, types.TUINT8}: ssa.OpSub8, - opAndType{ir.OSUB, types.TINT16}: ssa.OpSub16, - opAndType{ir.OSUB, types.TUINT16}: ssa.OpSub16, - opAndType{ir.OSUB, types.TINT32}: ssa.OpSub32, - opAndType{ir.OSUB, types.TUINT32}: ssa.OpSub32, - opAndType{ir.OSUB, types.TINT64}: ssa.OpSub64, - opAndType{ir.OSUB, types.TUINT64}: ssa.OpSub64, - opAndType{ir.OSUB, types.TFLOAT32}: ssa.OpSub32F, - opAndType{ir.OSUB, types.TFLOAT64}: ssa.OpSub64F, - - opAndType{ir.ONOT, types.TBOOL}: ssa.OpNot, - - opAndType{ir.ONEG, types.TINT8}: ssa.OpNeg8, - opAndType{ir.ONEG, types.TUINT8}: ssa.OpNeg8, - opAndType{ir.ONEG, types.TINT16}: ssa.OpNeg16, - opAndType{ir.ONEG, types.TUINT16}: ssa.OpNeg16, - opAndType{ir.ONEG, types.TINT32}: ssa.OpNeg32, - opAndType{ir.ONEG, types.TUINT32}: ssa.OpNeg32, - opAndType{ir.ONEG, types.TINT64}: ssa.OpNeg64, - opAndType{ir.ONEG, types.TUINT64}: ssa.OpNeg64, - opAndType{ir.ONEG, types.TFLOAT32}: ssa.OpNeg32F, - opAndType{ir.ONEG, types.TFLOAT64}: ssa.OpNeg64F, - - opAndType{ir.OBITNOT, types.TINT8}: ssa.OpCom8, - opAndType{ir.OBITNOT, types.TUINT8}: ssa.OpCom8, - opAndType{ir.OBITNOT, types.TINT16}: ssa.OpCom16, - opAndType{ir.OBITNOT, types.TUINT16}: ssa.OpCom16, - opAndType{ir.OBITNOT, types.TINT32}: ssa.OpCom32, - opAndType{ir.OBITNOT, types.TUINT32}: ssa.OpCom32, - opAndType{ir.OBITNOT, types.TINT64}: ssa.OpCom64, - opAndType{ir.OBITNOT, types.TUINT64}: ssa.OpCom64, - - opAndType{ir.OIMAG, types.TCOMPLEX64}: ssa.OpComplexImag, - opAndType{ir.OIMAG, types.TCOMPLEX128}: ssa.OpComplexImag, - opAndType{ir.OREAL, types.TCOMPLEX64}: ssa.OpComplexReal, - opAndType{ir.OREAL, types.TCOMPLEX128}: ssa.OpComplexReal, - - opAndType{ir.OMUL, types.TINT8}: ssa.OpMul8, - opAndType{ir.OMUL, types.TUINT8}: ssa.OpMul8, - opAndType{ir.OMUL, types.TINT16}: ssa.OpMul16, - opAndType{ir.OMUL, types.TUINT16}: ssa.OpMul16, - opAndType{ir.OMUL, types.TINT32}: ssa.OpMul32, - opAndType{ir.OMUL, types.TUINT32}: ssa.OpMul32, - opAndType{ir.OMUL, types.TINT64}: ssa.OpMul64, - opAndType{ir.OMUL, types.TUINT64}: ssa.OpMul64, - opAndType{ir.OMUL, types.TFLOAT32}: ssa.OpMul32F, - opAndType{ir.OMUL, types.TFLOAT64}: ssa.OpMul64F, - - opAndType{ir.ODIV, types.TFLOAT32}: ssa.OpDiv32F, - opAndType{ir.ODIV, types.TFLOAT64}: ssa.OpDiv64F, - - opAndType{ir.ODIV, types.TINT8}: ssa.OpDiv8, - opAndType{ir.ODIV, types.TUINT8}: ssa.OpDiv8u, - opAndType{ir.ODIV, types.TINT16}: ssa.OpDiv16, - opAndType{ir.ODIV, types.TUINT16}: ssa.OpDiv16u, - opAndType{ir.ODIV, types.TINT32}: ssa.OpDiv32, - opAndType{ir.ODIV, types.TUINT32}: ssa.OpDiv32u, - opAndType{ir.ODIV, types.TINT64}: ssa.OpDiv64, - opAndType{ir.ODIV, types.TUINT64}: ssa.OpDiv64u, - - opAndType{ir.OMOD, types.TINT8}: ssa.OpMod8, - opAndType{ir.OMOD, types.TUINT8}: ssa.OpMod8u, - opAndType{ir.OMOD, types.TINT16}: ssa.OpMod16, - opAndType{ir.OMOD, types.TUINT16}: ssa.OpMod16u, - opAndType{ir.OMOD, types.TINT32}: ssa.OpMod32, - opAndType{ir.OMOD, types.TUINT32}: ssa.OpMod32u, - opAndType{ir.OMOD, types.TINT64}: ssa.OpMod64, - opAndType{ir.OMOD, types.TUINT64}: ssa.OpMod64u, - - opAndType{ir.OAND, types.TINT8}: ssa.OpAnd8, - opAndType{ir.OAND, types.TUINT8}: ssa.OpAnd8, - opAndType{ir.OAND, types.TINT16}: ssa.OpAnd16, - opAndType{ir.OAND, types.TUINT16}: ssa.OpAnd16, - opAndType{ir.OAND, types.TINT32}: ssa.OpAnd32, - opAndType{ir.OAND, types.TUINT32}: ssa.OpAnd32, - opAndType{ir.OAND, types.TINT64}: ssa.OpAnd64, - opAndType{ir.OAND, types.TUINT64}: ssa.OpAnd64, - - opAndType{ir.OOR, types.TINT8}: ssa.OpOr8, - opAndType{ir.OOR, types.TUINT8}: ssa.OpOr8, - opAndType{ir.OOR, types.TINT16}: ssa.OpOr16, - opAndType{ir.OOR, types.TUINT16}: ssa.OpOr16, - opAndType{ir.OOR, types.TINT32}: ssa.OpOr32, - opAndType{ir.OOR, types.TUINT32}: ssa.OpOr32, - opAndType{ir.OOR, types.TINT64}: ssa.OpOr64, - opAndType{ir.OOR, types.TUINT64}: ssa.OpOr64, - - opAndType{ir.OXOR, types.TINT8}: ssa.OpXor8, - opAndType{ir.OXOR, types.TUINT8}: ssa.OpXor8, - opAndType{ir.OXOR, types.TINT16}: ssa.OpXor16, - opAndType{ir.OXOR, types.TUINT16}: ssa.OpXor16, - opAndType{ir.OXOR, types.TINT32}: ssa.OpXor32, - opAndType{ir.OXOR, types.TUINT32}: ssa.OpXor32, - opAndType{ir.OXOR, types.TINT64}: ssa.OpXor64, - opAndType{ir.OXOR, types.TUINT64}: ssa.OpXor64, - - opAndType{ir.OEQ, types.TBOOL}: ssa.OpEqB, - opAndType{ir.OEQ, types.TINT8}: ssa.OpEq8, - opAndType{ir.OEQ, types.TUINT8}: ssa.OpEq8, - opAndType{ir.OEQ, types.TINT16}: ssa.OpEq16, - opAndType{ir.OEQ, types.TUINT16}: ssa.OpEq16, - opAndType{ir.OEQ, types.TINT32}: ssa.OpEq32, - opAndType{ir.OEQ, types.TUINT32}: ssa.OpEq32, - opAndType{ir.OEQ, types.TINT64}: ssa.OpEq64, - opAndType{ir.OEQ, types.TUINT64}: ssa.OpEq64, - opAndType{ir.OEQ, types.TINTER}: ssa.OpEqInter, - opAndType{ir.OEQ, types.TSLICE}: ssa.OpEqSlice, - opAndType{ir.OEQ, types.TFUNC}: ssa.OpEqPtr, - opAndType{ir.OEQ, types.TMAP}: ssa.OpEqPtr, - opAndType{ir.OEQ, types.TCHAN}: ssa.OpEqPtr, - opAndType{ir.OEQ, types.TPTR}: ssa.OpEqPtr, - opAndType{ir.OEQ, types.TUINTPTR}: ssa.OpEqPtr, - opAndType{ir.OEQ, types.TUNSAFEPTR}: ssa.OpEqPtr, - opAndType{ir.OEQ, types.TFLOAT64}: ssa.OpEq64F, - opAndType{ir.OEQ, types.TFLOAT32}: ssa.OpEq32F, - - opAndType{ir.ONE, types.TBOOL}: ssa.OpNeqB, - opAndType{ir.ONE, types.TINT8}: ssa.OpNeq8, - opAndType{ir.ONE, types.TUINT8}: ssa.OpNeq8, - opAndType{ir.ONE, types.TINT16}: ssa.OpNeq16, - opAndType{ir.ONE, types.TUINT16}: ssa.OpNeq16, - opAndType{ir.ONE, types.TINT32}: ssa.OpNeq32, - opAndType{ir.ONE, types.TUINT32}: ssa.OpNeq32, - opAndType{ir.ONE, types.TINT64}: ssa.OpNeq64, - opAndType{ir.ONE, types.TUINT64}: ssa.OpNeq64, - opAndType{ir.ONE, types.TINTER}: ssa.OpNeqInter, - opAndType{ir.ONE, types.TSLICE}: ssa.OpNeqSlice, - opAndType{ir.ONE, types.TFUNC}: ssa.OpNeqPtr, - opAndType{ir.ONE, types.TMAP}: ssa.OpNeqPtr, - opAndType{ir.ONE, types.TCHAN}: ssa.OpNeqPtr, - opAndType{ir.ONE, types.TPTR}: ssa.OpNeqPtr, - opAndType{ir.ONE, types.TUINTPTR}: ssa.OpNeqPtr, - opAndType{ir.ONE, types.TUNSAFEPTR}: ssa.OpNeqPtr, - opAndType{ir.ONE, types.TFLOAT64}: ssa.OpNeq64F, - opAndType{ir.ONE, types.TFLOAT32}: ssa.OpNeq32F, - - opAndType{ir.OLT, types.TINT8}: ssa.OpLess8, - opAndType{ir.OLT, types.TUINT8}: ssa.OpLess8U, - opAndType{ir.OLT, types.TINT16}: ssa.OpLess16, - opAndType{ir.OLT, types.TUINT16}: ssa.OpLess16U, - opAndType{ir.OLT, types.TINT32}: ssa.OpLess32, - opAndType{ir.OLT, types.TUINT32}: ssa.OpLess32U, - opAndType{ir.OLT, types.TINT64}: ssa.OpLess64, - opAndType{ir.OLT, types.TUINT64}: ssa.OpLess64U, - opAndType{ir.OLT, types.TFLOAT64}: ssa.OpLess64F, - opAndType{ir.OLT, types.TFLOAT32}: ssa.OpLess32F, - - opAndType{ir.OLE, types.TINT8}: ssa.OpLeq8, - opAndType{ir.OLE, types.TUINT8}: ssa.OpLeq8U, - opAndType{ir.OLE, types.TINT16}: ssa.OpLeq16, - opAndType{ir.OLE, types.TUINT16}: ssa.OpLeq16U, - opAndType{ir.OLE, types.TINT32}: ssa.OpLeq32, - opAndType{ir.OLE, types.TUINT32}: ssa.OpLeq32U, - opAndType{ir.OLE, types.TINT64}: ssa.OpLeq64, - opAndType{ir.OLE, types.TUINT64}: ssa.OpLeq64U, - opAndType{ir.OLE, types.TFLOAT64}: ssa.OpLeq64F, - opAndType{ir.OLE, types.TFLOAT32}: ssa.OpLeq32F, -} - -func (s *state) concreteEtype(t *types.Type) types.Kind { - e := t.Kind() - switch e { - default: - return e - case types.TINT: - if s.config.PtrSize == 8 { - return types.TINT64 - } - return types.TINT32 - case types.TUINT: - if s.config.PtrSize == 8 { - return types.TUINT64 - } - return types.TUINT32 - case types.TUINTPTR: - if s.config.PtrSize == 8 { - return types.TUINT64 - } - return types.TUINT32 - } -} - -func (s *state) ssaOp(op ir.Op, t *types.Type) ssa.Op { - etype := s.concreteEtype(t) - x, ok := opToSSA[opAndType{op, etype}] - if !ok { - s.Fatalf("unhandled binary op %v %s", op, etype) - } - return x -} - -type opAndTwoTypes struct { - op ir.Op - etype1 types.Kind - etype2 types.Kind -} - -type twoTypes struct { - etype1 types.Kind - etype2 types.Kind -} - -type twoOpsAndType struct { - op1 ssa.Op - op2 ssa.Op - intermediateType types.Kind -} - -var fpConvOpToSSA = map[twoTypes]twoOpsAndType{ - - twoTypes{types.TINT8, types.TFLOAT32}: twoOpsAndType{ssa.OpSignExt8to32, ssa.OpCvt32to32F, types.TINT32}, - twoTypes{types.TINT16, types.TFLOAT32}: twoOpsAndType{ssa.OpSignExt16to32, ssa.OpCvt32to32F, types.TINT32}, - twoTypes{types.TINT32, types.TFLOAT32}: twoOpsAndType{ssa.OpCopy, ssa.OpCvt32to32F, types.TINT32}, - twoTypes{types.TINT64, types.TFLOAT32}: twoOpsAndType{ssa.OpCopy, ssa.OpCvt64to32F, types.TINT64}, - - twoTypes{types.TINT8, types.TFLOAT64}: twoOpsAndType{ssa.OpSignExt8to32, ssa.OpCvt32to64F, types.TINT32}, - twoTypes{types.TINT16, types.TFLOAT64}: twoOpsAndType{ssa.OpSignExt16to32, ssa.OpCvt32to64F, types.TINT32}, - twoTypes{types.TINT32, types.TFLOAT64}: twoOpsAndType{ssa.OpCopy, ssa.OpCvt32to64F, types.TINT32}, - twoTypes{types.TINT64, types.TFLOAT64}: twoOpsAndType{ssa.OpCopy, ssa.OpCvt64to64F, types.TINT64}, - - twoTypes{types.TFLOAT32, types.TINT8}: twoOpsAndType{ssa.OpCvt32Fto32, ssa.OpTrunc32to8, types.TINT32}, - twoTypes{types.TFLOAT32, types.TINT16}: twoOpsAndType{ssa.OpCvt32Fto32, ssa.OpTrunc32to16, types.TINT32}, - twoTypes{types.TFLOAT32, types.TINT32}: twoOpsAndType{ssa.OpCvt32Fto32, ssa.OpCopy, types.TINT32}, - twoTypes{types.TFLOAT32, types.TINT64}: twoOpsAndType{ssa.OpCvt32Fto64, ssa.OpCopy, types.TINT64}, - - twoTypes{types.TFLOAT64, types.TINT8}: twoOpsAndType{ssa.OpCvt64Fto32, ssa.OpTrunc32to8, types.TINT32}, - twoTypes{types.TFLOAT64, types.TINT16}: twoOpsAndType{ssa.OpCvt64Fto32, ssa.OpTrunc32to16, types.TINT32}, - twoTypes{types.TFLOAT64, types.TINT32}: twoOpsAndType{ssa.OpCvt64Fto32, ssa.OpCopy, types.TINT32}, - twoTypes{types.TFLOAT64, types.TINT64}: twoOpsAndType{ssa.OpCvt64Fto64, ssa.OpCopy, types.TINT64}, - // unsigned - twoTypes{types.TUINT8, types.TFLOAT32}: twoOpsAndType{ssa.OpZeroExt8to32, ssa.OpCvt32to32F, types.TINT32}, - twoTypes{types.TUINT16, types.TFLOAT32}: twoOpsAndType{ssa.OpZeroExt16to32, ssa.OpCvt32to32F, types.TINT32}, - twoTypes{types.TUINT32, types.TFLOAT32}: twoOpsAndType{ssa.OpZeroExt32to64, ssa.OpCvt64to32F, types.TINT64}, // go wide to dodge unsigned - twoTypes{types.TUINT64, types.TFLOAT32}: twoOpsAndType{ssa.OpCopy, ssa.OpInvalid, types.TUINT64}, // Cvt64Uto32F, branchy code expansion instead - - twoTypes{types.TUINT8, types.TFLOAT64}: twoOpsAndType{ssa.OpZeroExt8to32, ssa.OpCvt32to64F, types.TINT32}, - twoTypes{types.TUINT16, types.TFLOAT64}: twoOpsAndType{ssa.OpZeroExt16to32, ssa.OpCvt32to64F, types.TINT32}, - twoTypes{types.TUINT32, types.TFLOAT64}: twoOpsAndType{ssa.OpZeroExt32to64, ssa.OpCvt64to64F, types.TINT64}, // go wide to dodge unsigned - twoTypes{types.TUINT64, types.TFLOAT64}: twoOpsAndType{ssa.OpCopy, ssa.OpInvalid, types.TUINT64}, // Cvt64Uto64F, branchy code expansion instead - - twoTypes{types.TFLOAT32, types.TUINT8}: twoOpsAndType{ssa.OpCvt32Fto32, ssa.OpTrunc32to8, types.TINT32}, - twoTypes{types.TFLOAT32, types.TUINT16}: twoOpsAndType{ssa.OpCvt32Fto32, ssa.OpTrunc32to16, types.TINT32}, - twoTypes{types.TFLOAT32, types.TUINT32}: twoOpsAndType{ssa.OpCvt32Fto64, ssa.OpTrunc64to32, types.TINT64}, // go wide to dodge unsigned - twoTypes{types.TFLOAT32, types.TUINT64}: twoOpsAndType{ssa.OpInvalid, ssa.OpCopy, types.TUINT64}, // Cvt32Fto64U, branchy code expansion instead - - twoTypes{types.TFLOAT64, types.TUINT8}: twoOpsAndType{ssa.OpCvt64Fto32, ssa.OpTrunc32to8, types.TINT32}, - twoTypes{types.TFLOAT64, types.TUINT16}: twoOpsAndType{ssa.OpCvt64Fto32, ssa.OpTrunc32to16, types.TINT32}, - twoTypes{types.TFLOAT64, types.TUINT32}: twoOpsAndType{ssa.OpCvt64Fto64, ssa.OpTrunc64to32, types.TINT64}, // go wide to dodge unsigned - twoTypes{types.TFLOAT64, types.TUINT64}: twoOpsAndType{ssa.OpInvalid, ssa.OpCopy, types.TUINT64}, // Cvt64Fto64U, branchy code expansion instead - - // float - twoTypes{types.TFLOAT64, types.TFLOAT32}: twoOpsAndType{ssa.OpCvt64Fto32F, ssa.OpCopy, types.TFLOAT32}, - twoTypes{types.TFLOAT64, types.TFLOAT64}: twoOpsAndType{ssa.OpRound64F, ssa.OpCopy, types.TFLOAT64}, - twoTypes{types.TFLOAT32, types.TFLOAT32}: twoOpsAndType{ssa.OpRound32F, ssa.OpCopy, types.TFLOAT32}, - twoTypes{types.TFLOAT32, types.TFLOAT64}: twoOpsAndType{ssa.OpCvt32Fto64F, ssa.OpCopy, types.TFLOAT64}, -} - -// this map is used only for 32-bit arch, and only includes the difference -// on 32-bit arch, don't use int64<->float conversion for uint32 -var fpConvOpToSSA32 = map[twoTypes]twoOpsAndType{ - twoTypes{types.TUINT32, types.TFLOAT32}: twoOpsAndType{ssa.OpCopy, ssa.OpCvt32Uto32F, types.TUINT32}, - twoTypes{types.TUINT32, types.TFLOAT64}: twoOpsAndType{ssa.OpCopy, ssa.OpCvt32Uto64F, types.TUINT32}, - twoTypes{types.TFLOAT32, types.TUINT32}: twoOpsAndType{ssa.OpCvt32Fto32U, ssa.OpCopy, types.TUINT32}, - twoTypes{types.TFLOAT64, types.TUINT32}: twoOpsAndType{ssa.OpCvt64Fto32U, ssa.OpCopy, types.TUINT32}, -} - -// uint64<->float conversions, only on machines that have instructions for that -var uint64fpConvOpToSSA = map[twoTypes]twoOpsAndType{ - twoTypes{types.TUINT64, types.TFLOAT32}: twoOpsAndType{ssa.OpCopy, ssa.OpCvt64Uto32F, types.TUINT64}, - twoTypes{types.TUINT64, types.TFLOAT64}: twoOpsAndType{ssa.OpCopy, ssa.OpCvt64Uto64F, types.TUINT64}, - twoTypes{types.TFLOAT32, types.TUINT64}: twoOpsAndType{ssa.OpCvt32Fto64U, ssa.OpCopy, types.TUINT64}, - twoTypes{types.TFLOAT64, types.TUINT64}: twoOpsAndType{ssa.OpCvt64Fto64U, ssa.OpCopy, types.TUINT64}, -} - -var shiftOpToSSA = map[opAndTwoTypes]ssa.Op{ - opAndTwoTypes{ir.OLSH, types.TINT8, types.TUINT8}: ssa.OpLsh8x8, - opAndTwoTypes{ir.OLSH, types.TUINT8, types.TUINT8}: ssa.OpLsh8x8, - opAndTwoTypes{ir.OLSH, types.TINT8, types.TUINT16}: ssa.OpLsh8x16, - opAndTwoTypes{ir.OLSH, types.TUINT8, types.TUINT16}: ssa.OpLsh8x16, - opAndTwoTypes{ir.OLSH, types.TINT8, types.TUINT32}: ssa.OpLsh8x32, - opAndTwoTypes{ir.OLSH, types.TUINT8, types.TUINT32}: ssa.OpLsh8x32, - opAndTwoTypes{ir.OLSH, types.TINT8, types.TUINT64}: ssa.OpLsh8x64, - opAndTwoTypes{ir.OLSH, types.TUINT8, types.TUINT64}: ssa.OpLsh8x64, - - opAndTwoTypes{ir.OLSH, types.TINT16, types.TUINT8}: ssa.OpLsh16x8, - opAndTwoTypes{ir.OLSH, types.TUINT16, types.TUINT8}: ssa.OpLsh16x8, - opAndTwoTypes{ir.OLSH, types.TINT16, types.TUINT16}: ssa.OpLsh16x16, - opAndTwoTypes{ir.OLSH, types.TUINT16, types.TUINT16}: ssa.OpLsh16x16, - opAndTwoTypes{ir.OLSH, types.TINT16, types.TUINT32}: ssa.OpLsh16x32, - opAndTwoTypes{ir.OLSH, types.TUINT16, types.TUINT32}: ssa.OpLsh16x32, - opAndTwoTypes{ir.OLSH, types.TINT16, types.TUINT64}: ssa.OpLsh16x64, - opAndTwoTypes{ir.OLSH, types.TUINT16, types.TUINT64}: ssa.OpLsh16x64, - - opAndTwoTypes{ir.OLSH, types.TINT32, types.TUINT8}: ssa.OpLsh32x8, - opAndTwoTypes{ir.OLSH, types.TUINT32, types.TUINT8}: ssa.OpLsh32x8, - opAndTwoTypes{ir.OLSH, types.TINT32, types.TUINT16}: ssa.OpLsh32x16, - opAndTwoTypes{ir.OLSH, types.TUINT32, types.TUINT16}: ssa.OpLsh32x16, - opAndTwoTypes{ir.OLSH, types.TINT32, types.TUINT32}: ssa.OpLsh32x32, - opAndTwoTypes{ir.OLSH, types.TUINT32, types.TUINT32}: ssa.OpLsh32x32, - opAndTwoTypes{ir.OLSH, types.TINT32, types.TUINT64}: ssa.OpLsh32x64, - opAndTwoTypes{ir.OLSH, types.TUINT32, types.TUINT64}: ssa.OpLsh32x64, - - opAndTwoTypes{ir.OLSH, types.TINT64, types.TUINT8}: ssa.OpLsh64x8, - opAndTwoTypes{ir.OLSH, types.TUINT64, types.TUINT8}: ssa.OpLsh64x8, - opAndTwoTypes{ir.OLSH, types.TINT64, types.TUINT16}: ssa.OpLsh64x16, - opAndTwoTypes{ir.OLSH, types.TUINT64, types.TUINT16}: ssa.OpLsh64x16, - opAndTwoTypes{ir.OLSH, types.TINT64, types.TUINT32}: ssa.OpLsh64x32, - opAndTwoTypes{ir.OLSH, types.TUINT64, types.TUINT32}: ssa.OpLsh64x32, - opAndTwoTypes{ir.OLSH, types.TINT64, types.TUINT64}: ssa.OpLsh64x64, - opAndTwoTypes{ir.OLSH, types.TUINT64, types.TUINT64}: ssa.OpLsh64x64, - - opAndTwoTypes{ir.ORSH, types.TINT8, types.TUINT8}: ssa.OpRsh8x8, - opAndTwoTypes{ir.ORSH, types.TUINT8, types.TUINT8}: ssa.OpRsh8Ux8, - opAndTwoTypes{ir.ORSH, types.TINT8, types.TUINT16}: ssa.OpRsh8x16, - opAndTwoTypes{ir.ORSH, types.TUINT8, types.TUINT16}: ssa.OpRsh8Ux16, - opAndTwoTypes{ir.ORSH, types.TINT8, types.TUINT32}: ssa.OpRsh8x32, - opAndTwoTypes{ir.ORSH, types.TUINT8, types.TUINT32}: ssa.OpRsh8Ux32, - opAndTwoTypes{ir.ORSH, types.TINT8, types.TUINT64}: ssa.OpRsh8x64, - opAndTwoTypes{ir.ORSH, types.TUINT8, types.TUINT64}: ssa.OpRsh8Ux64, - - opAndTwoTypes{ir.ORSH, types.TINT16, types.TUINT8}: ssa.OpRsh16x8, - opAndTwoTypes{ir.ORSH, types.TUINT16, types.TUINT8}: ssa.OpRsh16Ux8, - opAndTwoTypes{ir.ORSH, types.TINT16, types.TUINT16}: ssa.OpRsh16x16, - opAndTwoTypes{ir.ORSH, types.TUINT16, types.TUINT16}: ssa.OpRsh16Ux16, - opAndTwoTypes{ir.ORSH, types.TINT16, types.TUINT32}: ssa.OpRsh16x32, - opAndTwoTypes{ir.ORSH, types.TUINT16, types.TUINT32}: ssa.OpRsh16Ux32, - opAndTwoTypes{ir.ORSH, types.TINT16, types.TUINT64}: ssa.OpRsh16x64, - opAndTwoTypes{ir.ORSH, types.TUINT16, types.TUINT64}: ssa.OpRsh16Ux64, - - opAndTwoTypes{ir.ORSH, types.TINT32, types.TUINT8}: ssa.OpRsh32x8, - opAndTwoTypes{ir.ORSH, types.TUINT32, types.TUINT8}: ssa.OpRsh32Ux8, - opAndTwoTypes{ir.ORSH, types.TINT32, types.TUINT16}: ssa.OpRsh32x16, - opAndTwoTypes{ir.ORSH, types.TUINT32, types.TUINT16}: ssa.OpRsh32Ux16, - opAndTwoTypes{ir.ORSH, types.TINT32, types.TUINT32}: ssa.OpRsh32x32, - opAndTwoTypes{ir.ORSH, types.TUINT32, types.TUINT32}: ssa.OpRsh32Ux32, - opAndTwoTypes{ir.ORSH, types.TINT32, types.TUINT64}: ssa.OpRsh32x64, - opAndTwoTypes{ir.ORSH, types.TUINT32, types.TUINT64}: ssa.OpRsh32Ux64, - - opAndTwoTypes{ir.ORSH, types.TINT64, types.TUINT8}: ssa.OpRsh64x8, - opAndTwoTypes{ir.ORSH, types.TUINT64, types.TUINT8}: ssa.OpRsh64Ux8, - opAndTwoTypes{ir.ORSH, types.TINT64, types.TUINT16}: ssa.OpRsh64x16, - opAndTwoTypes{ir.ORSH, types.TUINT64, types.TUINT16}: ssa.OpRsh64Ux16, - opAndTwoTypes{ir.ORSH, types.TINT64, types.TUINT32}: ssa.OpRsh64x32, - opAndTwoTypes{ir.ORSH, types.TUINT64, types.TUINT32}: ssa.OpRsh64Ux32, - opAndTwoTypes{ir.ORSH, types.TINT64, types.TUINT64}: ssa.OpRsh64x64, - opAndTwoTypes{ir.ORSH, types.TUINT64, types.TUINT64}: ssa.OpRsh64Ux64, -} - -func (s *state) ssaShiftOp(op ir.Op, t *types.Type, u *types.Type) ssa.Op { - etype1 := s.concreteEtype(t) - etype2 := s.concreteEtype(u) - x, ok := shiftOpToSSA[opAndTwoTypes{op, etype1, etype2}] - if !ok { - s.Fatalf("unhandled shift op %v etype=%s/%s", op, etype1, etype2) - } - return x -} - -// expr converts the expression n to ssa, adds it to s and returns the ssa result. -func (s *state) expr(n ir.Node) *ssa.Value { - if ir.HasUniquePos(n) { - // ONAMEs and named OLITERALs have the line number - // of the decl, not the use. See issue 14742. - s.pushLine(n.Pos()) - defer s.popLine() - } - - s.stmtList(n.Init()) - switch n.Op() { - case ir.OBYTES2STRTMP: - n := n.(*ir.ConvExpr) - slice := s.expr(n.X) - ptr := s.newValue1(ssa.OpSlicePtr, s.f.Config.Types.BytePtr, slice) - len := s.newValue1(ssa.OpSliceLen, types.Types[types.TINT], slice) - return s.newValue2(ssa.OpStringMake, n.Type(), ptr, len) - case ir.OSTR2BYTESTMP: - n := n.(*ir.ConvExpr) - str := s.expr(n.X) - ptr := s.newValue1(ssa.OpStringPtr, s.f.Config.Types.BytePtr, str) - len := s.newValue1(ssa.OpStringLen, types.Types[types.TINT], str) - return s.newValue3(ssa.OpSliceMake, n.Type(), ptr, len, len) - case ir.OCFUNC: - n := n.(*ir.UnaryExpr) - aux := n.X.Sym().Linksym() - return s.entryNewValue1A(ssa.OpAddr, n.Type(), aux, s.sb) - case ir.OMETHEXPR: - n := n.(*ir.MethodExpr) - sym := staticdata.FuncSym(n.FuncName().Sym()).Linksym() - return s.entryNewValue1A(ssa.OpAddr, types.NewPtr(n.Type()), sym, s.sb) - case ir.ONAME: - n := n.(*ir.Name) - if n.Class_ == ir.PFUNC { - // "value" of a function is the address of the function's closure - sym := staticdata.FuncSym(n.Sym()).Linksym() - return s.entryNewValue1A(ssa.OpAddr, types.NewPtr(n.Type()), sym, s.sb) - } - if s.canSSA(n) { - return s.variable(n, n.Type()) - } - addr := s.addr(n) - return s.load(n.Type(), addr) - case ir.ONAMEOFFSET: - n := n.(*ir.NameOffsetExpr) - if s.canSSAName(n.Name_) && canSSAType(n.Type()) { - return s.variable(n, n.Type()) - } - addr := s.addr(n) - return s.load(n.Type(), addr) - case ir.OCLOSUREREAD: - addr := s.addr(n) - return s.load(n.Type(), addr) - case ir.ONIL: - n := n.(*ir.NilExpr) - t := n.Type() - switch { - case t.IsSlice(): - return s.constSlice(t) - case t.IsInterface(): - return s.constInterface(t) - default: - return s.constNil(t) - } - case ir.OLITERAL: - switch u := n.Val(); u.Kind() { - case constant.Int: - i := ir.IntVal(n.Type(), u) - switch n.Type().Size() { - case 1: - return s.constInt8(n.Type(), int8(i)) - case 2: - return s.constInt16(n.Type(), int16(i)) - case 4: - return s.constInt32(n.Type(), int32(i)) - case 8: - return s.constInt64(n.Type(), i) - default: - s.Fatalf("bad integer size %d", n.Type().Size()) - return nil - } - case constant.String: - i := constant.StringVal(u) - if i == "" { - return s.constEmptyString(n.Type()) - } - return s.entryNewValue0A(ssa.OpConstString, n.Type(), ssa.StringToAux(i)) - case constant.Bool: - return s.constBool(constant.BoolVal(u)) - case constant.Float: - f, _ := constant.Float64Val(u) - switch n.Type().Size() { - case 4: - return s.constFloat32(n.Type(), f) - case 8: - return s.constFloat64(n.Type(), f) - default: - s.Fatalf("bad float size %d", n.Type().Size()) - return nil - } - case constant.Complex: - re, _ := constant.Float64Val(constant.Real(u)) - im, _ := constant.Float64Val(constant.Imag(u)) - switch n.Type().Size() { - case 8: - pt := types.Types[types.TFLOAT32] - return s.newValue2(ssa.OpComplexMake, n.Type(), - s.constFloat32(pt, re), - s.constFloat32(pt, im)) - case 16: - pt := types.Types[types.TFLOAT64] - return s.newValue2(ssa.OpComplexMake, n.Type(), - s.constFloat64(pt, re), - s.constFloat64(pt, im)) - default: - s.Fatalf("bad complex size %d", n.Type().Size()) - return nil - } - default: - s.Fatalf("unhandled OLITERAL %v", u.Kind()) - return nil - } - case ir.OCONVNOP: - n := n.(*ir.ConvExpr) - to := n.Type() - from := n.X.Type() - - // Assume everything will work out, so set up our return value. - // Anything interesting that happens from here is a fatal. - x := s.expr(n.X) - if to == from { - return x - } - - // Special case for not confusing GC and liveness. - // We don't want pointers accidentally classified - // as not-pointers or vice-versa because of copy - // elision. - if to.IsPtrShaped() != from.IsPtrShaped() { - return s.newValue2(ssa.OpConvert, to, x, s.mem()) - } - - v := s.newValue1(ssa.OpCopy, to, x) // ensure that v has the right type - - // CONVNOP closure - if to.Kind() == types.TFUNC && from.IsPtrShaped() { - return v - } - - // named <--> unnamed type or typed <--> untyped const - if from.Kind() == to.Kind() { - return v - } - - // unsafe.Pointer <--> *T - if to.IsUnsafePtr() && from.IsPtrShaped() || from.IsUnsafePtr() && to.IsPtrShaped() { - return v - } - - // map <--> *hmap - if to.Kind() == types.TMAP && from.IsPtr() && - to.MapType().Hmap == from.Elem() { - return v - } - - types.CalcSize(from) - types.CalcSize(to) - if from.Width != to.Width { - s.Fatalf("CONVNOP width mismatch %v (%d) -> %v (%d)\n", from, from.Width, to, to.Width) - return nil - } - if etypesign(from.Kind()) != etypesign(to.Kind()) { - s.Fatalf("CONVNOP sign mismatch %v (%s) -> %v (%s)\n", from, from.Kind(), to, to.Kind()) - return nil - } - - if base.Flag.Cfg.Instrumenting { - // These appear to be fine, but they fail the - // integer constraint below, so okay them here. - // Sample non-integer conversion: map[string]string -> *uint8 - return v - } - - if etypesign(from.Kind()) == 0 { - s.Fatalf("CONVNOP unrecognized non-integer %v -> %v\n", from, to) - return nil - } - - // integer, same width, same sign - return v - - case ir.OCONV: - n := n.(*ir.ConvExpr) - x := s.expr(n.X) - ft := n.X.Type() // from type - tt := n.Type() // to type - if ft.IsBoolean() && tt.IsKind(types.TUINT8) { - // Bool -> uint8 is generated internally when indexing into runtime.staticbyte. - return s.newValue1(ssa.OpCopy, n.Type(), x) - } - if ft.IsInteger() && tt.IsInteger() { - var op ssa.Op - if tt.Size() == ft.Size() { - op = ssa.OpCopy - } else if tt.Size() < ft.Size() { - // truncation - switch 10*ft.Size() + tt.Size() { - case 21: - op = ssa.OpTrunc16to8 - case 41: - op = ssa.OpTrunc32to8 - case 42: - op = ssa.OpTrunc32to16 - case 81: - op = ssa.OpTrunc64to8 - case 82: - op = ssa.OpTrunc64to16 - case 84: - op = ssa.OpTrunc64to32 - default: - s.Fatalf("weird integer truncation %v -> %v", ft, tt) - } - } else if ft.IsSigned() { - // sign extension - switch 10*ft.Size() + tt.Size() { - case 12: - op = ssa.OpSignExt8to16 - case 14: - op = ssa.OpSignExt8to32 - case 18: - op = ssa.OpSignExt8to64 - case 24: - op = ssa.OpSignExt16to32 - case 28: - op = ssa.OpSignExt16to64 - case 48: - op = ssa.OpSignExt32to64 - default: - s.Fatalf("bad integer sign extension %v -> %v", ft, tt) - } - } else { - // zero extension - switch 10*ft.Size() + tt.Size() { - case 12: - op = ssa.OpZeroExt8to16 - case 14: - op = ssa.OpZeroExt8to32 - case 18: - op = ssa.OpZeroExt8to64 - case 24: - op = ssa.OpZeroExt16to32 - case 28: - op = ssa.OpZeroExt16to64 - case 48: - op = ssa.OpZeroExt32to64 - default: - s.Fatalf("weird integer sign extension %v -> %v", ft, tt) - } - } - return s.newValue1(op, n.Type(), x) - } - - if ft.IsFloat() || tt.IsFloat() { - conv, ok := fpConvOpToSSA[twoTypes{s.concreteEtype(ft), s.concreteEtype(tt)}] - if s.config.RegSize == 4 && thearch.LinkArch.Family != sys.MIPS && !s.softFloat { - if conv1, ok1 := fpConvOpToSSA32[twoTypes{s.concreteEtype(ft), s.concreteEtype(tt)}]; ok1 { - conv = conv1 - } - } - if thearch.LinkArch.Family == sys.ARM64 || thearch.LinkArch.Family == sys.Wasm || thearch.LinkArch.Family == sys.S390X || s.softFloat { - if conv1, ok1 := uint64fpConvOpToSSA[twoTypes{s.concreteEtype(ft), s.concreteEtype(tt)}]; ok1 { - conv = conv1 - } - } - - if thearch.LinkArch.Family == sys.MIPS && !s.softFloat { - if ft.Size() == 4 && ft.IsInteger() && !ft.IsSigned() { - // tt is float32 or float64, and ft is also unsigned - if tt.Size() == 4 { - return s.uint32Tofloat32(n, x, ft, tt) - } - if tt.Size() == 8 { - return s.uint32Tofloat64(n, x, ft, tt) - } - } else if tt.Size() == 4 && tt.IsInteger() && !tt.IsSigned() { - // ft is float32 or float64, and tt is unsigned integer - if ft.Size() == 4 { - return s.float32ToUint32(n, x, ft, tt) - } - if ft.Size() == 8 { - return s.float64ToUint32(n, x, ft, tt) - } - } - } - - if !ok { - s.Fatalf("weird float conversion %v -> %v", ft, tt) - } - op1, op2, it := conv.op1, conv.op2, conv.intermediateType - - if op1 != ssa.OpInvalid && op2 != ssa.OpInvalid { - // normal case, not tripping over unsigned 64 - if op1 == ssa.OpCopy { - if op2 == ssa.OpCopy { - return x - } - return s.newValueOrSfCall1(op2, n.Type(), x) - } - if op2 == ssa.OpCopy { - return s.newValueOrSfCall1(op1, n.Type(), x) - } - return s.newValueOrSfCall1(op2, n.Type(), s.newValueOrSfCall1(op1, types.Types[it], x)) - } - // Tricky 64-bit unsigned cases. - if ft.IsInteger() { - // tt is float32 or float64, and ft is also unsigned - if tt.Size() == 4 { - return s.uint64Tofloat32(n, x, ft, tt) - } - if tt.Size() == 8 { - return s.uint64Tofloat64(n, x, ft, tt) - } - s.Fatalf("weird unsigned integer to float conversion %v -> %v", ft, tt) - } - // ft is float32 or float64, and tt is unsigned integer - if ft.Size() == 4 { - return s.float32ToUint64(n, x, ft, tt) - } - if ft.Size() == 8 { - return s.float64ToUint64(n, x, ft, tt) - } - s.Fatalf("weird float to unsigned integer conversion %v -> %v", ft, tt) - return nil - } - - if ft.IsComplex() && tt.IsComplex() { - var op ssa.Op - if ft.Size() == tt.Size() { - switch ft.Size() { - case 8: - op = ssa.OpRound32F - case 16: - op = ssa.OpRound64F - default: - s.Fatalf("weird complex conversion %v -> %v", ft, tt) - } - } else if ft.Size() == 8 && tt.Size() == 16 { - op = ssa.OpCvt32Fto64F - } else if ft.Size() == 16 && tt.Size() == 8 { - op = ssa.OpCvt64Fto32F - } else { - s.Fatalf("weird complex conversion %v -> %v", ft, tt) - } - ftp := types.FloatForComplex(ft) - ttp := types.FloatForComplex(tt) - return s.newValue2(ssa.OpComplexMake, tt, - s.newValueOrSfCall1(op, ttp, s.newValue1(ssa.OpComplexReal, ftp, x)), - s.newValueOrSfCall1(op, ttp, s.newValue1(ssa.OpComplexImag, ftp, x))) - } - - s.Fatalf("unhandled OCONV %s -> %s", n.X.Type().Kind(), n.Type().Kind()) - return nil - - case ir.ODOTTYPE: - n := n.(*ir.TypeAssertExpr) - res, _ := s.dottype(n, false) - return res - - // binary ops - case ir.OLT, ir.OEQ, ir.ONE, ir.OLE, ir.OGE, ir.OGT: - n := n.(*ir.BinaryExpr) - a := s.expr(n.X) - b := s.expr(n.Y) - if n.X.Type().IsComplex() { - pt := types.FloatForComplex(n.X.Type()) - op := s.ssaOp(ir.OEQ, pt) - r := s.newValueOrSfCall2(op, types.Types[types.TBOOL], s.newValue1(ssa.OpComplexReal, pt, a), s.newValue1(ssa.OpComplexReal, pt, b)) - i := s.newValueOrSfCall2(op, types.Types[types.TBOOL], s.newValue1(ssa.OpComplexImag, pt, a), s.newValue1(ssa.OpComplexImag, pt, b)) - c := s.newValue2(ssa.OpAndB, types.Types[types.TBOOL], r, i) - switch n.Op() { - case ir.OEQ: - return c - case ir.ONE: - return s.newValue1(ssa.OpNot, types.Types[types.TBOOL], c) - default: - s.Fatalf("ordered complex compare %v", n.Op()) - } - } - - // Convert OGE and OGT into OLE and OLT. - op := n.Op() - switch op { - case ir.OGE: - op, a, b = ir.OLE, b, a - case ir.OGT: - op, a, b = ir.OLT, b, a - } - if n.X.Type().IsFloat() { - // float comparison - return s.newValueOrSfCall2(s.ssaOp(op, n.X.Type()), types.Types[types.TBOOL], a, b) - } - // integer comparison - return s.newValue2(s.ssaOp(op, n.X.Type()), types.Types[types.TBOOL], a, b) - case ir.OMUL: - n := n.(*ir.BinaryExpr) - a := s.expr(n.X) - b := s.expr(n.Y) - if n.Type().IsComplex() { - mulop := ssa.OpMul64F - addop := ssa.OpAdd64F - subop := ssa.OpSub64F - pt := types.FloatForComplex(n.Type()) // Could be Float32 or Float64 - wt := types.Types[types.TFLOAT64] // Compute in Float64 to minimize cancellation error - - areal := s.newValue1(ssa.OpComplexReal, pt, a) - breal := s.newValue1(ssa.OpComplexReal, pt, b) - aimag := s.newValue1(ssa.OpComplexImag, pt, a) - bimag := s.newValue1(ssa.OpComplexImag, pt, b) - - if pt != wt { // Widen for calculation - areal = s.newValueOrSfCall1(ssa.OpCvt32Fto64F, wt, areal) - breal = s.newValueOrSfCall1(ssa.OpCvt32Fto64F, wt, breal) - aimag = s.newValueOrSfCall1(ssa.OpCvt32Fto64F, wt, aimag) - bimag = s.newValueOrSfCall1(ssa.OpCvt32Fto64F, wt, bimag) - } - - xreal := s.newValueOrSfCall2(subop, wt, s.newValueOrSfCall2(mulop, wt, areal, breal), s.newValueOrSfCall2(mulop, wt, aimag, bimag)) - ximag := s.newValueOrSfCall2(addop, wt, s.newValueOrSfCall2(mulop, wt, areal, bimag), s.newValueOrSfCall2(mulop, wt, aimag, breal)) - - if pt != wt { // Narrow to store back - xreal = s.newValueOrSfCall1(ssa.OpCvt64Fto32F, pt, xreal) - ximag = s.newValueOrSfCall1(ssa.OpCvt64Fto32F, pt, ximag) - } - - return s.newValue2(ssa.OpComplexMake, n.Type(), xreal, ximag) - } - - if n.Type().IsFloat() { - return s.newValueOrSfCall2(s.ssaOp(n.Op(), n.Type()), a.Type, a, b) - } - - return s.newValue2(s.ssaOp(n.Op(), n.Type()), a.Type, a, b) - - case ir.ODIV: - n := n.(*ir.BinaryExpr) - a := s.expr(n.X) - b := s.expr(n.Y) - if n.Type().IsComplex() { - // TODO this is not executed because the front-end substitutes a runtime call. - // That probably ought to change; with modest optimization the widen/narrow - // conversions could all be elided in larger expression trees. - mulop := ssa.OpMul64F - addop := ssa.OpAdd64F - subop := ssa.OpSub64F - divop := ssa.OpDiv64F - pt := types.FloatForComplex(n.Type()) // Could be Float32 or Float64 - wt := types.Types[types.TFLOAT64] // Compute in Float64 to minimize cancellation error - - areal := s.newValue1(ssa.OpComplexReal, pt, a) - breal := s.newValue1(ssa.OpComplexReal, pt, b) - aimag := s.newValue1(ssa.OpComplexImag, pt, a) - bimag := s.newValue1(ssa.OpComplexImag, pt, b) - - if pt != wt { // Widen for calculation - areal = s.newValueOrSfCall1(ssa.OpCvt32Fto64F, wt, areal) - breal = s.newValueOrSfCall1(ssa.OpCvt32Fto64F, wt, breal) - aimag = s.newValueOrSfCall1(ssa.OpCvt32Fto64F, wt, aimag) - bimag = s.newValueOrSfCall1(ssa.OpCvt32Fto64F, wt, bimag) - } - - denom := s.newValueOrSfCall2(addop, wt, s.newValueOrSfCall2(mulop, wt, breal, breal), s.newValueOrSfCall2(mulop, wt, bimag, bimag)) - xreal := s.newValueOrSfCall2(addop, wt, s.newValueOrSfCall2(mulop, wt, areal, breal), s.newValueOrSfCall2(mulop, wt, aimag, bimag)) - ximag := s.newValueOrSfCall2(subop, wt, s.newValueOrSfCall2(mulop, wt, aimag, breal), s.newValueOrSfCall2(mulop, wt, areal, bimag)) - - // TODO not sure if this is best done in wide precision or narrow - // Double-rounding might be an issue. - // Note that the pre-SSA implementation does the entire calculation - // in wide format, so wide is compatible. - xreal = s.newValueOrSfCall2(divop, wt, xreal, denom) - ximag = s.newValueOrSfCall2(divop, wt, ximag, denom) - - if pt != wt { // Narrow to store back - xreal = s.newValueOrSfCall1(ssa.OpCvt64Fto32F, pt, xreal) - ximag = s.newValueOrSfCall1(ssa.OpCvt64Fto32F, pt, ximag) - } - return s.newValue2(ssa.OpComplexMake, n.Type(), xreal, ximag) - } - if n.Type().IsFloat() { - return s.newValueOrSfCall2(s.ssaOp(n.Op(), n.Type()), a.Type, a, b) - } - return s.intDivide(n, a, b) - case ir.OMOD: - n := n.(*ir.BinaryExpr) - a := s.expr(n.X) - b := s.expr(n.Y) - return s.intDivide(n, a, b) - case ir.OADD, ir.OSUB: - n := n.(*ir.BinaryExpr) - a := s.expr(n.X) - b := s.expr(n.Y) - if n.Type().IsComplex() { - pt := types.FloatForComplex(n.Type()) - op := s.ssaOp(n.Op(), pt) - return s.newValue2(ssa.OpComplexMake, n.Type(), - s.newValueOrSfCall2(op, pt, s.newValue1(ssa.OpComplexReal, pt, a), s.newValue1(ssa.OpComplexReal, pt, b)), - s.newValueOrSfCall2(op, pt, s.newValue1(ssa.OpComplexImag, pt, a), s.newValue1(ssa.OpComplexImag, pt, b))) - } - if n.Type().IsFloat() { - return s.newValueOrSfCall2(s.ssaOp(n.Op(), n.Type()), a.Type, a, b) - } - return s.newValue2(s.ssaOp(n.Op(), n.Type()), a.Type, a, b) - case ir.OAND, ir.OOR, ir.OXOR: - n := n.(*ir.BinaryExpr) - a := s.expr(n.X) - b := s.expr(n.Y) - return s.newValue2(s.ssaOp(n.Op(), n.Type()), a.Type, a, b) - case ir.OANDNOT: - n := n.(*ir.BinaryExpr) - a := s.expr(n.X) - b := s.expr(n.Y) - b = s.newValue1(s.ssaOp(ir.OBITNOT, b.Type), b.Type, b) - return s.newValue2(s.ssaOp(ir.OAND, n.Type()), a.Type, a, b) - case ir.OLSH, ir.ORSH: - n := n.(*ir.BinaryExpr) - a := s.expr(n.X) - b := s.expr(n.Y) - bt := b.Type - if bt.IsSigned() { - cmp := s.newValue2(s.ssaOp(ir.OLE, bt), types.Types[types.TBOOL], s.zeroVal(bt), b) - s.check(cmp, ir.Syms.Panicshift) - bt = bt.ToUnsigned() - } - return s.newValue2(s.ssaShiftOp(n.Op(), n.Type(), bt), a.Type, a, b) - case ir.OANDAND, ir.OOROR: - // To implement OANDAND (and OOROR), we introduce a - // new temporary variable to hold the result. The - // variable is associated with the OANDAND node in the - // s.vars table (normally variables are only - // associated with ONAME nodes). We convert - // A && B - // to - // var = A - // if var { - // var = B - // } - // Using var in the subsequent block introduces the - // necessary phi variable. - n := n.(*ir.LogicalExpr) - el := s.expr(n.X) - s.vars[n] = el - - b := s.endBlock() - b.Kind = ssa.BlockIf - b.SetControl(el) - // In theory, we should set b.Likely here based on context. - // However, gc only gives us likeliness hints - // in a single place, for plain OIF statements, - // and passing around context is finnicky, so don't bother for now. - - bRight := s.f.NewBlock(ssa.BlockPlain) - bResult := s.f.NewBlock(ssa.BlockPlain) - if n.Op() == ir.OANDAND { - b.AddEdgeTo(bRight) - b.AddEdgeTo(bResult) - } else if n.Op() == ir.OOROR { - b.AddEdgeTo(bResult) - b.AddEdgeTo(bRight) - } - - s.startBlock(bRight) - er := s.expr(n.Y) - s.vars[n] = er - - b = s.endBlock() - b.AddEdgeTo(bResult) - - s.startBlock(bResult) - return s.variable(n, types.Types[types.TBOOL]) - case ir.OCOMPLEX: - n := n.(*ir.BinaryExpr) - r := s.expr(n.X) - i := s.expr(n.Y) - return s.newValue2(ssa.OpComplexMake, n.Type(), r, i) - - // unary ops - case ir.ONEG: - n := n.(*ir.UnaryExpr) - a := s.expr(n.X) - if n.Type().IsComplex() { - tp := types.FloatForComplex(n.Type()) - negop := s.ssaOp(n.Op(), tp) - return s.newValue2(ssa.OpComplexMake, n.Type(), - s.newValue1(negop, tp, s.newValue1(ssa.OpComplexReal, tp, a)), - s.newValue1(negop, tp, s.newValue1(ssa.OpComplexImag, tp, a))) - } - return s.newValue1(s.ssaOp(n.Op(), n.Type()), a.Type, a) - case ir.ONOT, ir.OBITNOT: - n := n.(*ir.UnaryExpr) - a := s.expr(n.X) - return s.newValue1(s.ssaOp(n.Op(), n.Type()), a.Type, a) - case ir.OIMAG, ir.OREAL: - n := n.(*ir.UnaryExpr) - a := s.expr(n.X) - return s.newValue1(s.ssaOp(n.Op(), n.X.Type()), n.Type(), a) - case ir.OPLUS: - n := n.(*ir.UnaryExpr) - return s.expr(n.X) - - case ir.OADDR: - n := n.(*ir.AddrExpr) - return s.addr(n.X) - - case ir.ORESULT: - n := n.(*ir.ResultExpr) - if s.prevCall == nil || s.prevCall.Op != ssa.OpStaticLECall && s.prevCall.Op != ssa.OpInterLECall && s.prevCall.Op != ssa.OpClosureLECall { - // Do the old thing - addr := s.constOffPtrSP(types.NewPtr(n.Type()), n.Offset) - return s.rawLoad(n.Type(), addr) - } - which := s.prevCall.Aux.(*ssa.AuxCall).ResultForOffset(n.Offset) - if which == -1 { - // Do the old thing // TODO: Panic instead. - addr := s.constOffPtrSP(types.NewPtr(n.Type()), n.Offset) - return s.rawLoad(n.Type(), addr) - } - if canSSAType(n.Type()) { - return s.newValue1I(ssa.OpSelectN, n.Type(), which, s.prevCall) - } else { - addr := s.newValue1I(ssa.OpSelectNAddr, types.NewPtr(n.Type()), which, s.prevCall) - return s.rawLoad(n.Type(), addr) - } - - case ir.ODEREF: - n := n.(*ir.StarExpr) - p := s.exprPtr(n.X, n.Bounded(), n.Pos()) - return s.load(n.Type(), p) - - case ir.ODOT: - n := n.(*ir.SelectorExpr) - if n.X.Op() == ir.OSTRUCTLIT { - // All literals with nonzero fields have already been - // rewritten during walk. Any that remain are just T{} - // or equivalents. Use the zero value. - if !ir.IsZero(n.X) { - s.Fatalf("literal with nonzero value in SSA: %v", n.X) - } - return s.zeroVal(n.Type()) - } - // If n is addressable and can't be represented in - // SSA, then load just the selected field. This - // prevents false memory dependencies in race/msan - // instrumentation. - if ir.IsAssignable(n) && !s.canSSA(n) { - p := s.addr(n) - return s.load(n.Type(), p) - } - v := s.expr(n.X) - return s.newValue1I(ssa.OpStructSelect, n.Type(), int64(fieldIdx(n)), v) - - case ir.ODOTPTR: - n := n.(*ir.SelectorExpr) - p := s.exprPtr(n.X, n.Bounded(), n.Pos()) - p = s.newValue1I(ssa.OpOffPtr, types.NewPtr(n.Type()), n.Offset, p) - return s.load(n.Type(), p) - - case ir.OINDEX: - n := n.(*ir.IndexExpr) - switch { - case n.X.Type().IsString(): - if n.Bounded() && ir.IsConst(n.X, constant.String) && ir.IsConst(n.Index, constant.Int) { - // Replace "abc"[1] with 'b'. - // Delayed until now because "abc"[1] is not an ideal constant. - // See test/fixedbugs/issue11370.go. - return s.newValue0I(ssa.OpConst8, types.Types[types.TUINT8], int64(int8(ir.StringVal(n.X)[ir.Int64Val(n.Index)]))) - } - a := s.expr(n.X) - i := s.expr(n.Index) - len := s.newValue1(ssa.OpStringLen, types.Types[types.TINT], a) - i = s.boundsCheck(i, len, ssa.BoundsIndex, n.Bounded()) - ptrtyp := s.f.Config.Types.BytePtr - ptr := s.newValue1(ssa.OpStringPtr, ptrtyp, a) - if ir.IsConst(n.Index, constant.Int) { - ptr = s.newValue1I(ssa.OpOffPtr, ptrtyp, ir.Int64Val(n.Index), ptr) - } else { - ptr = s.newValue2(ssa.OpAddPtr, ptrtyp, ptr, i) - } - return s.load(types.Types[types.TUINT8], ptr) - case n.X.Type().IsSlice(): - p := s.addr(n) - return s.load(n.X.Type().Elem(), p) - case n.X.Type().IsArray(): - if canSSAType(n.X.Type()) { - // SSA can handle arrays of length at most 1. - bound := n.X.Type().NumElem() - a := s.expr(n.X) - i := s.expr(n.Index) - if bound == 0 { - // Bounds check will never succeed. Might as well - // use constants for the bounds check. - z := s.constInt(types.Types[types.TINT], 0) - s.boundsCheck(z, z, ssa.BoundsIndex, false) - // The return value won't be live, return junk. - return s.newValue0(ssa.OpUnknown, n.Type()) - } - len := s.constInt(types.Types[types.TINT], bound) - s.boundsCheck(i, len, ssa.BoundsIndex, n.Bounded()) // checks i == 0 - return s.newValue1I(ssa.OpArraySelect, n.Type(), 0, a) - } - p := s.addr(n) - return s.load(n.X.Type().Elem(), p) - default: - s.Fatalf("bad type for index %v", n.X.Type()) - return nil - } - - case ir.OLEN, ir.OCAP: - n := n.(*ir.UnaryExpr) - switch { - case n.X.Type().IsSlice(): - op := ssa.OpSliceLen - if n.Op() == ir.OCAP { - op = ssa.OpSliceCap - } - return s.newValue1(op, types.Types[types.TINT], s.expr(n.X)) - case n.X.Type().IsString(): // string; not reachable for OCAP - return s.newValue1(ssa.OpStringLen, types.Types[types.TINT], s.expr(n.X)) - case n.X.Type().IsMap(), n.X.Type().IsChan(): - return s.referenceTypeBuiltin(n, s.expr(n.X)) - default: // array - return s.constInt(types.Types[types.TINT], n.X.Type().NumElem()) - } - - case ir.OSPTR: - n := n.(*ir.UnaryExpr) - a := s.expr(n.X) - if n.X.Type().IsSlice() { - return s.newValue1(ssa.OpSlicePtr, n.Type(), a) - } else { - return s.newValue1(ssa.OpStringPtr, n.Type(), a) - } - - case ir.OITAB: - n := n.(*ir.UnaryExpr) - a := s.expr(n.X) - return s.newValue1(ssa.OpITab, n.Type(), a) - - case ir.OIDATA: - n := n.(*ir.UnaryExpr) - a := s.expr(n.X) - return s.newValue1(ssa.OpIData, n.Type(), a) - - case ir.OEFACE: - n := n.(*ir.BinaryExpr) - tab := s.expr(n.X) - data := s.expr(n.Y) - return s.newValue2(ssa.OpIMake, n.Type(), tab, data) - - case ir.OSLICEHEADER: - n := n.(*ir.SliceHeaderExpr) - p := s.expr(n.Ptr) - l := s.expr(n.LenCap[0]) - c := s.expr(n.LenCap[1]) - return s.newValue3(ssa.OpSliceMake, n.Type(), p, l, c) - - case ir.OSLICE, ir.OSLICEARR, ir.OSLICE3, ir.OSLICE3ARR: - n := n.(*ir.SliceExpr) - v := s.expr(n.X) - var i, j, k *ssa.Value - low, high, max := n.SliceBounds() - if low != nil { - i = s.expr(low) - } - if high != nil { - j = s.expr(high) - } - if max != nil { - k = s.expr(max) - } - p, l, c := s.slice(v, i, j, k, n.Bounded()) - return s.newValue3(ssa.OpSliceMake, n.Type(), p, l, c) - - case ir.OSLICESTR: - n := n.(*ir.SliceExpr) - v := s.expr(n.X) - var i, j *ssa.Value - low, high, _ := n.SliceBounds() - if low != nil { - i = s.expr(low) - } - if high != nil { - j = s.expr(high) - } - p, l, _ := s.slice(v, i, j, nil, n.Bounded()) - return s.newValue2(ssa.OpStringMake, n.Type(), p, l) - - case ir.OCALLFUNC: - n := n.(*ir.CallExpr) - if ir.IsIntrinsicCall(n) { - return s.intrinsicCall(n) - } - fallthrough - - case ir.OCALLINTER, ir.OCALLMETH: - n := n.(*ir.CallExpr) - return s.callResult(n, callNormal) - - case ir.OGETG: - n := n.(*ir.CallExpr) - return s.newValue1(ssa.OpGetG, n.Type(), s.mem()) - - case ir.OAPPEND: - return s.append(n.(*ir.CallExpr), false) - - case ir.OSTRUCTLIT, ir.OARRAYLIT: - // All literals with nonzero fields have already been - // rewritten during walk. Any that remain are just T{} - // or equivalents. Use the zero value. - n := n.(*ir.CompLitExpr) - if !ir.IsZero(n) { - s.Fatalf("literal with nonzero value in SSA: %v", n) - } - return s.zeroVal(n.Type()) - - case ir.ONEWOBJ: - n := n.(*ir.UnaryExpr) - if n.Type().Elem().Size() == 0 { - return s.newValue1A(ssa.OpAddr, n.Type(), ir.Syms.Zerobase, s.sb) - } - typ := s.expr(n.X) - vv := s.rtcall(ir.Syms.Newobject, true, []*types.Type{n.Type()}, typ) - return vv[0] - - default: - s.Fatalf("unhandled expr %v", n.Op()) - return nil - } -} - -// append converts an OAPPEND node to SSA. -// If inplace is false, it converts the OAPPEND expression n to an ssa.Value, -// adds it to s, and returns the Value. -// If inplace is true, it writes the result of the OAPPEND expression n -// back to the slice being appended to, and returns nil. -// inplace MUST be set to false if the slice can be SSA'd. -func (s *state) append(n *ir.CallExpr, inplace bool) *ssa.Value { - // If inplace is false, process as expression "append(s, e1, e2, e3)": - // - // ptr, len, cap := s - // newlen := len + 3 - // if newlen > cap { - // ptr, len, cap = growslice(s, newlen) - // newlen = len + 3 // recalculate to avoid a spill - // } - // // with write barriers, if needed: - // *(ptr+len) = e1 - // *(ptr+len+1) = e2 - // *(ptr+len+2) = e3 - // return makeslice(ptr, newlen, cap) - // - // - // If inplace is true, process as statement "s = append(s, e1, e2, e3)": - // - // a := &s - // ptr, len, cap := s - // newlen := len + 3 - // if uint(newlen) > uint(cap) { - // newptr, len, newcap = growslice(ptr, len, cap, newlen) - // vardef(a) // if necessary, advise liveness we are writing a new a - // *a.cap = newcap // write before ptr to avoid a spill - // *a.ptr = newptr // with write barrier - // } - // newlen = len + 3 // recalculate to avoid a spill - // *a.len = newlen - // // with write barriers, if needed: - // *(ptr+len) = e1 - // *(ptr+len+1) = e2 - // *(ptr+len+2) = e3 - - et := n.Type().Elem() - pt := types.NewPtr(et) - - // Evaluate slice - sn := n.Args[0] // the slice node is the first in the list - - var slice, addr *ssa.Value - if inplace { - addr = s.addr(sn) - slice = s.load(n.Type(), addr) - } else { - slice = s.expr(sn) - } - - // Allocate new blocks - grow := s.f.NewBlock(ssa.BlockPlain) - assign := s.f.NewBlock(ssa.BlockPlain) - - // Decide if we need to grow - nargs := int64(len(n.Args) - 1) - p := s.newValue1(ssa.OpSlicePtr, pt, slice) - l := s.newValue1(ssa.OpSliceLen, types.Types[types.TINT], slice) - c := s.newValue1(ssa.OpSliceCap, types.Types[types.TINT], slice) - nl := s.newValue2(s.ssaOp(ir.OADD, types.Types[types.TINT]), types.Types[types.TINT], l, s.constInt(types.Types[types.TINT], nargs)) - - cmp := s.newValue2(s.ssaOp(ir.OLT, types.Types[types.TUINT]), types.Types[types.TBOOL], c, nl) - s.vars[ptrVar] = p - - if !inplace { - s.vars[newlenVar] = nl - s.vars[capVar] = c - } else { - s.vars[lenVar] = l - } - - b := s.endBlock() - b.Kind = ssa.BlockIf - b.Likely = ssa.BranchUnlikely - b.SetControl(cmp) - b.AddEdgeTo(grow) - b.AddEdgeTo(assign) - - // Call growslice - s.startBlock(grow) - taddr := s.expr(n.X) - r := s.rtcall(ir.Syms.Growslice, true, []*types.Type{pt, types.Types[types.TINT], types.Types[types.TINT]}, taddr, p, l, c, nl) - - if inplace { - if sn.Op() == ir.ONAME { - sn := sn.(*ir.Name) - if sn.Class_ != ir.PEXTERN { - // Tell liveness we're about to build a new slice - s.vars[memVar] = s.newValue1A(ssa.OpVarDef, types.TypeMem, sn, s.mem()) - } - } - capaddr := s.newValue1I(ssa.OpOffPtr, s.f.Config.Types.IntPtr, types.SliceCapOffset, addr) - s.store(types.Types[types.TINT], capaddr, r[2]) - s.store(pt, addr, r[0]) - // load the value we just stored to avoid having to spill it - s.vars[ptrVar] = s.load(pt, addr) - s.vars[lenVar] = r[1] // avoid a spill in the fast path - } else { - s.vars[ptrVar] = r[0] - s.vars[newlenVar] = s.newValue2(s.ssaOp(ir.OADD, types.Types[types.TINT]), types.Types[types.TINT], r[1], s.constInt(types.Types[types.TINT], nargs)) - s.vars[capVar] = r[2] - } - - b = s.endBlock() - b.AddEdgeTo(assign) - - // assign new elements to slots - s.startBlock(assign) - - if inplace { - l = s.variable(lenVar, types.Types[types.TINT]) // generates phi for len - nl = s.newValue2(s.ssaOp(ir.OADD, types.Types[types.TINT]), types.Types[types.TINT], l, s.constInt(types.Types[types.TINT], nargs)) - lenaddr := s.newValue1I(ssa.OpOffPtr, s.f.Config.Types.IntPtr, types.SliceLenOffset, addr) - s.store(types.Types[types.TINT], lenaddr, nl) - } - - // Evaluate args - type argRec struct { - // if store is true, we're appending the value v. If false, we're appending the - // value at *v. - v *ssa.Value - store bool - } - args := make([]argRec, 0, nargs) - for _, n := range n.Args[1:] { - if canSSAType(n.Type()) { - args = append(args, argRec{v: s.expr(n), store: true}) - } else { - v := s.addr(n) - args = append(args, argRec{v: v}) - } - } - - p = s.variable(ptrVar, pt) // generates phi for ptr - if !inplace { - nl = s.variable(newlenVar, types.Types[types.TINT]) // generates phi for nl - c = s.variable(capVar, types.Types[types.TINT]) // generates phi for cap - } - p2 := s.newValue2(ssa.OpPtrIndex, pt, p, l) - for i, arg := range args { - addr := s.newValue2(ssa.OpPtrIndex, pt, p2, s.constInt(types.Types[types.TINT], int64(i))) - if arg.store { - s.storeType(et, addr, arg.v, 0, true) - } else { - s.move(et, addr, arg.v) - } - } - - delete(s.vars, ptrVar) - if inplace { - delete(s.vars, lenVar) - return nil - } - delete(s.vars, newlenVar) - delete(s.vars, capVar) - // make result - return s.newValue3(ssa.OpSliceMake, n.Type(), p, nl, c) -} - -// condBranch evaluates the boolean expression cond and branches to yes -// if cond is true and no if cond is false. -// This function is intended to handle && and || better than just calling -// s.expr(cond) and branching on the result. -func (s *state) condBranch(cond ir.Node, yes, no *ssa.Block, likely int8) { - switch cond.Op() { - case ir.OANDAND: - cond := cond.(*ir.LogicalExpr) - mid := s.f.NewBlock(ssa.BlockPlain) - s.stmtList(cond.Init()) - s.condBranch(cond.X, mid, no, max8(likely, 0)) - s.startBlock(mid) - s.condBranch(cond.Y, yes, no, likely) - return - // Note: if likely==1, then both recursive calls pass 1. - // If likely==-1, then we don't have enough information to decide - // whether the first branch is likely or not. So we pass 0 for - // the likeliness of the first branch. - // TODO: have the frontend give us branch prediction hints for - // OANDAND and OOROR nodes (if it ever has such info). - case ir.OOROR: - cond := cond.(*ir.LogicalExpr) - mid := s.f.NewBlock(ssa.BlockPlain) - s.stmtList(cond.Init()) - s.condBranch(cond.X, yes, mid, min8(likely, 0)) - s.startBlock(mid) - s.condBranch(cond.Y, yes, no, likely) - return - // Note: if likely==-1, then both recursive calls pass -1. - // If likely==1, then we don't have enough info to decide - // the likelihood of the first branch. - case ir.ONOT: - cond := cond.(*ir.UnaryExpr) - s.stmtList(cond.Init()) - s.condBranch(cond.X, no, yes, -likely) - return - case ir.OCONVNOP: - cond := cond.(*ir.ConvExpr) - s.stmtList(cond.Init()) - s.condBranch(cond.X, yes, no, likely) - return - } - c := s.expr(cond) - b := s.endBlock() - b.Kind = ssa.BlockIf - b.SetControl(c) - b.Likely = ssa.BranchPrediction(likely) // gc and ssa both use -1/0/+1 for likeliness - b.AddEdgeTo(yes) - b.AddEdgeTo(no) -} - -type skipMask uint8 - -const ( - skipPtr skipMask = 1 << iota - skipLen - skipCap -) - -// assign does left = right. -// Right has already been evaluated to ssa, left has not. -// If deref is true, then we do left = *right instead (and right has already been nil-checked). -// If deref is true and right == nil, just do left = 0. -// skip indicates assignments (at the top level) that can be avoided. -func (s *state) assign(left ir.Node, right *ssa.Value, deref bool, skip skipMask) { - if left.Op() == ir.ONAME && ir.IsBlank(left) { - return - } - t := left.Type() - types.CalcSize(t) - if s.canSSA(left) { - if deref { - s.Fatalf("can SSA LHS %v but not RHS %s", left, right) - } - if left.Op() == ir.ODOT { - // We're assigning to a field of an ssa-able value. - // We need to build a new structure with the new value for the - // field we're assigning and the old values for the other fields. - // For instance: - // type T struct {a, b, c int} - // var T x - // x.b = 5 - // For the x.b = 5 assignment we want to generate x = T{x.a, 5, x.c} - - // Grab information about the structure type. - left := left.(*ir.SelectorExpr) - t := left.X.Type() - nf := t.NumFields() - idx := fieldIdx(left) - - // Grab old value of structure. - old := s.expr(left.X) - - // Make new structure. - new := s.newValue0(ssa.StructMakeOp(t.NumFields()), t) - - // Add fields as args. - for i := 0; i < nf; i++ { - if i == idx { - new.AddArg(right) - } else { - new.AddArg(s.newValue1I(ssa.OpStructSelect, t.FieldType(i), int64(i), old)) - } - } - - // Recursively assign the new value we've made to the base of the dot op. - s.assign(left.X, new, false, 0) - // TODO: do we need to update named values here? - return - } - if left.Op() == ir.OINDEX && left.(*ir.IndexExpr).X.Type().IsArray() { - left := left.(*ir.IndexExpr) - s.pushLine(left.Pos()) - defer s.popLine() - // We're assigning to an element of an ssa-able array. - // a[i] = v - t := left.X.Type() - n := t.NumElem() - - i := s.expr(left.Index) // index - if n == 0 { - // The bounds check must fail. Might as well - // ignore the actual index and just use zeros. - z := s.constInt(types.Types[types.TINT], 0) - s.boundsCheck(z, z, ssa.BoundsIndex, false) - return - } - if n != 1 { - s.Fatalf("assigning to non-1-length array") - } - // Rewrite to a = [1]{v} - len := s.constInt(types.Types[types.TINT], 1) - s.boundsCheck(i, len, ssa.BoundsIndex, false) // checks i == 0 - v := s.newValue1(ssa.OpArrayMake1, t, right) - s.assign(left.X, v, false, 0) - return - } - left := left.(*ir.Name) - // Update variable assignment. - s.vars[left] = right - s.addNamedValue(left, right) - return - } - - // If this assignment clobbers an entire local variable, then emit - // OpVarDef so liveness analysis knows the variable is redefined. - if base := clobberBase(left); base.Op() == ir.ONAME && base.(*ir.Name).Class_ != ir.PEXTERN && skip == 0 { - s.vars[memVar] = s.newValue1Apos(ssa.OpVarDef, types.TypeMem, base.(*ir.Name), s.mem(), !ir.IsAutoTmp(base)) - } - - // Left is not ssa-able. Compute its address. - addr := s.addr(left) - if ir.IsReflectHeaderDataField(left) { - // Package unsafe's documentation says storing pointers into - // reflect.SliceHeader and reflect.StringHeader's Data fields - // is valid, even though they have type uintptr (#19168). - // Mark it pointer type to signal the writebarrier pass to - // insert a write barrier. - t = types.Types[types.TUNSAFEPTR] - } - if deref { - // Treat as a mem->mem move. - if right == nil { - s.zero(t, addr) - } else { - s.move(t, addr, right) - } - return - } - // Treat as a store. - s.storeType(t, addr, right, skip, !ir.IsAutoTmp(left)) -} - -// zeroVal returns the zero value for type t. -func (s *state) zeroVal(t *types.Type) *ssa.Value { - switch { - case t.IsInteger(): - switch t.Size() { - case 1: - return s.constInt8(t, 0) - case 2: - return s.constInt16(t, 0) - case 4: - return s.constInt32(t, 0) - case 8: - return s.constInt64(t, 0) - default: - s.Fatalf("bad sized integer type %v", t) - } - case t.IsFloat(): - switch t.Size() { - case 4: - return s.constFloat32(t, 0) - case 8: - return s.constFloat64(t, 0) - default: - s.Fatalf("bad sized float type %v", t) - } - case t.IsComplex(): - switch t.Size() { - case 8: - z := s.constFloat32(types.Types[types.TFLOAT32], 0) - return s.entryNewValue2(ssa.OpComplexMake, t, z, z) - case 16: - z := s.constFloat64(types.Types[types.TFLOAT64], 0) - return s.entryNewValue2(ssa.OpComplexMake, t, z, z) - default: - s.Fatalf("bad sized complex type %v", t) - } - - case t.IsString(): - return s.constEmptyString(t) - case t.IsPtrShaped(): - return s.constNil(t) - case t.IsBoolean(): - return s.constBool(false) - case t.IsInterface(): - return s.constInterface(t) - case t.IsSlice(): - return s.constSlice(t) - case t.IsStruct(): - n := t.NumFields() - v := s.entryNewValue0(ssa.StructMakeOp(t.NumFields()), t) - for i := 0; i < n; i++ { - v.AddArg(s.zeroVal(t.FieldType(i))) - } - return v - case t.IsArray(): - switch t.NumElem() { - case 0: - return s.entryNewValue0(ssa.OpArrayMake0, t) - case 1: - return s.entryNewValue1(ssa.OpArrayMake1, t, s.zeroVal(t.Elem())) - } - } - s.Fatalf("zero for type %v not implemented", t) - return nil -} - -type callKind int8 - -const ( - callNormal callKind = iota - callDefer - callDeferStack - callGo -) - -type sfRtCallDef struct { - rtfn *obj.LSym - rtype types.Kind -} - -var softFloatOps map[ssa.Op]sfRtCallDef - -func softfloatInit() { - // Some of these operations get transformed by sfcall. - softFloatOps = map[ssa.Op]sfRtCallDef{ - ssa.OpAdd32F: sfRtCallDef{typecheck.LookupRuntimeFunc("fadd32"), types.TFLOAT32}, - ssa.OpAdd64F: sfRtCallDef{typecheck.LookupRuntimeFunc("fadd64"), types.TFLOAT64}, - ssa.OpSub32F: sfRtCallDef{typecheck.LookupRuntimeFunc("fadd32"), types.TFLOAT32}, - ssa.OpSub64F: sfRtCallDef{typecheck.LookupRuntimeFunc("fadd64"), types.TFLOAT64}, - ssa.OpMul32F: sfRtCallDef{typecheck.LookupRuntimeFunc("fmul32"), types.TFLOAT32}, - ssa.OpMul64F: sfRtCallDef{typecheck.LookupRuntimeFunc("fmul64"), types.TFLOAT64}, - ssa.OpDiv32F: sfRtCallDef{typecheck.LookupRuntimeFunc("fdiv32"), types.TFLOAT32}, - ssa.OpDiv64F: sfRtCallDef{typecheck.LookupRuntimeFunc("fdiv64"), types.TFLOAT64}, - - ssa.OpEq64F: sfRtCallDef{typecheck.LookupRuntimeFunc("feq64"), types.TBOOL}, - ssa.OpEq32F: sfRtCallDef{typecheck.LookupRuntimeFunc("feq32"), types.TBOOL}, - ssa.OpNeq64F: sfRtCallDef{typecheck.LookupRuntimeFunc("feq64"), types.TBOOL}, - ssa.OpNeq32F: sfRtCallDef{typecheck.LookupRuntimeFunc("feq32"), types.TBOOL}, - ssa.OpLess64F: sfRtCallDef{typecheck.LookupRuntimeFunc("fgt64"), types.TBOOL}, - ssa.OpLess32F: sfRtCallDef{typecheck.LookupRuntimeFunc("fgt32"), types.TBOOL}, - ssa.OpLeq64F: sfRtCallDef{typecheck.LookupRuntimeFunc("fge64"), types.TBOOL}, - ssa.OpLeq32F: sfRtCallDef{typecheck.LookupRuntimeFunc("fge32"), types.TBOOL}, - - ssa.OpCvt32to32F: sfRtCallDef{typecheck.LookupRuntimeFunc("fint32to32"), types.TFLOAT32}, - ssa.OpCvt32Fto32: sfRtCallDef{typecheck.LookupRuntimeFunc("f32toint32"), types.TINT32}, - ssa.OpCvt64to32F: sfRtCallDef{typecheck.LookupRuntimeFunc("fint64to32"), types.TFLOAT32}, - ssa.OpCvt32Fto64: sfRtCallDef{typecheck.LookupRuntimeFunc("f32toint64"), types.TINT64}, - ssa.OpCvt64Uto32F: sfRtCallDef{typecheck.LookupRuntimeFunc("fuint64to32"), types.TFLOAT32}, - ssa.OpCvt32Fto64U: sfRtCallDef{typecheck.LookupRuntimeFunc("f32touint64"), types.TUINT64}, - ssa.OpCvt32to64F: sfRtCallDef{typecheck.LookupRuntimeFunc("fint32to64"), types.TFLOAT64}, - ssa.OpCvt64Fto32: sfRtCallDef{typecheck.LookupRuntimeFunc("f64toint32"), types.TINT32}, - ssa.OpCvt64to64F: sfRtCallDef{typecheck.LookupRuntimeFunc("fint64to64"), types.TFLOAT64}, - ssa.OpCvt64Fto64: sfRtCallDef{typecheck.LookupRuntimeFunc("f64toint64"), types.TINT64}, - ssa.OpCvt64Uto64F: sfRtCallDef{typecheck.LookupRuntimeFunc("fuint64to64"), types.TFLOAT64}, - ssa.OpCvt64Fto64U: sfRtCallDef{typecheck.LookupRuntimeFunc("f64touint64"), types.TUINT64}, - ssa.OpCvt32Fto64F: sfRtCallDef{typecheck.LookupRuntimeFunc("f32to64"), types.TFLOAT64}, - ssa.OpCvt64Fto32F: sfRtCallDef{typecheck.LookupRuntimeFunc("f64to32"), types.TFLOAT32}, - } -} - -// TODO: do not emit sfcall if operation can be optimized to constant in later -// opt phase -func (s *state) sfcall(op ssa.Op, args ...*ssa.Value) (*ssa.Value, bool) { - if callDef, ok := softFloatOps[op]; ok { - switch op { - case ssa.OpLess32F, - ssa.OpLess64F, - ssa.OpLeq32F, - ssa.OpLeq64F: - args[0], args[1] = args[1], args[0] - case ssa.OpSub32F, - ssa.OpSub64F: - args[1] = s.newValue1(s.ssaOp(ir.ONEG, types.Types[callDef.rtype]), args[1].Type, args[1]) - } - - result := s.rtcall(callDef.rtfn, true, []*types.Type{types.Types[callDef.rtype]}, args...)[0] - if op == ssa.OpNeq32F || op == ssa.OpNeq64F { - result = s.newValue1(ssa.OpNot, result.Type, result) - } - return result, true - } - return nil, false -} - -var intrinsics map[intrinsicKey]intrinsicBuilder - -// An intrinsicBuilder converts a call node n into an ssa value that -// implements that call as an intrinsic. args is a list of arguments to the func. -type intrinsicBuilder func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value - -type intrinsicKey struct { - arch *sys.Arch - pkg string - fn string -} - -func initSSATables() { - intrinsics = map[intrinsicKey]intrinsicBuilder{} - - var all []*sys.Arch - var p4 []*sys.Arch - var p8 []*sys.Arch - var lwatomics []*sys.Arch - for _, a := range &sys.Archs { - all = append(all, a) - if a.PtrSize == 4 { - p4 = append(p4, a) - } else { - p8 = append(p8, a) - } - if a.Family != sys.PPC64 { - lwatomics = append(lwatomics, a) - } - } - - // add adds the intrinsic b for pkg.fn for the given list of architectures. - add := func(pkg, fn string, b intrinsicBuilder, archs ...*sys.Arch) { - for _, a := range archs { - intrinsics[intrinsicKey{a, pkg, fn}] = b - } - } - // addF does the same as add but operates on architecture families. - addF := func(pkg, fn string, b intrinsicBuilder, archFamilies ...sys.ArchFamily) { - m := 0 - for _, f := range archFamilies { - if f >= 32 { - panic("too many architecture families") - } - m |= 1 << uint(f) - } - for _, a := range all { - if m>>uint(a.Family)&1 != 0 { - intrinsics[intrinsicKey{a, pkg, fn}] = b - } - } - } - // alias defines pkg.fn = pkg2.fn2 for all architectures in archs for which pkg2.fn2 exists. - alias := func(pkg, fn, pkg2, fn2 string, archs ...*sys.Arch) { - aliased := false - for _, a := range archs { - if b, ok := intrinsics[intrinsicKey{a, pkg2, fn2}]; ok { - intrinsics[intrinsicKey{a, pkg, fn}] = b - aliased = true - } - } - if !aliased { - panic(fmt.Sprintf("attempted to alias undefined intrinsic: %s.%s", pkg, fn)) - } - } - - /******** runtime ********/ - if !base.Flag.Cfg.Instrumenting { - add("runtime", "slicebytetostringtmp", - func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { - // Compiler frontend optimizations emit OBYTES2STRTMP nodes - // for the backend instead of slicebytetostringtmp calls - // when not instrumenting. - return s.newValue2(ssa.OpStringMake, n.Type(), args[0], args[1]) - }, - all...) - } - addF("runtime/internal/math", "MulUintptr", - func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { - if s.config.PtrSize == 4 { - return s.newValue2(ssa.OpMul32uover, types.NewTuple(types.Types[types.TUINT], types.Types[types.TUINT]), args[0], args[1]) - } - return s.newValue2(ssa.OpMul64uover, types.NewTuple(types.Types[types.TUINT], types.Types[types.TUINT]), args[0], args[1]) - }, - sys.AMD64, sys.I386, sys.MIPS64) - add("runtime", "KeepAlive", - func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { - data := s.newValue1(ssa.OpIData, s.f.Config.Types.BytePtr, args[0]) - s.vars[memVar] = s.newValue2(ssa.OpKeepAlive, types.TypeMem, data, s.mem()) - return nil - }, - all...) - add("runtime", "getclosureptr", - func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { - return s.newValue0(ssa.OpGetClosurePtr, s.f.Config.Types.Uintptr) - }, - all...) - - add("runtime", "getcallerpc", - func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { - return s.newValue0(ssa.OpGetCallerPC, s.f.Config.Types.Uintptr) - }, - all...) - - add("runtime", "getcallersp", - func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { - return s.newValue0(ssa.OpGetCallerSP, s.f.Config.Types.Uintptr) - }, - all...) - - /******** runtime/internal/sys ********/ - addF("runtime/internal/sys", "Ctz32", - func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { - return s.newValue1(ssa.OpCtz32, types.Types[types.TINT], args[0]) - }, - sys.AMD64, sys.ARM64, sys.ARM, sys.S390X, sys.MIPS, sys.PPC64) - addF("runtime/internal/sys", "Ctz64", - func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { - return s.newValue1(ssa.OpCtz64, types.Types[types.TINT], args[0]) - }, - sys.AMD64, sys.ARM64, sys.ARM, sys.S390X, sys.MIPS, sys.PPC64) - addF("runtime/internal/sys", "Bswap32", - func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { - return s.newValue1(ssa.OpBswap32, types.Types[types.TUINT32], args[0]) - }, - sys.AMD64, sys.ARM64, sys.ARM, sys.S390X) - addF("runtime/internal/sys", "Bswap64", - func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { - return s.newValue1(ssa.OpBswap64, types.Types[types.TUINT64], args[0]) - }, - sys.AMD64, sys.ARM64, sys.ARM, sys.S390X) - - /******** runtime/internal/atomic ********/ - addF("runtime/internal/atomic", "Load", - func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { - v := s.newValue2(ssa.OpAtomicLoad32, types.NewTuple(types.Types[types.TUINT32], types.TypeMem), args[0], s.mem()) - s.vars[memVar] = s.newValue1(ssa.OpSelect1, types.TypeMem, v) - return s.newValue1(ssa.OpSelect0, types.Types[types.TUINT32], v) - }, - sys.AMD64, sys.ARM64, sys.MIPS, sys.MIPS64, sys.PPC64, sys.RISCV64, sys.S390X) - addF("runtime/internal/atomic", "Load8", - func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { - v := s.newValue2(ssa.OpAtomicLoad8, types.NewTuple(types.Types[types.TUINT8], types.TypeMem), args[0], s.mem()) - s.vars[memVar] = s.newValue1(ssa.OpSelect1, types.TypeMem, v) - return s.newValue1(ssa.OpSelect0, types.Types[types.TUINT8], v) - }, - sys.AMD64, sys.ARM64, sys.MIPS, sys.MIPS64, sys.PPC64, sys.RISCV64, sys.S390X) - addF("runtime/internal/atomic", "Load64", - func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { - v := s.newValue2(ssa.OpAtomicLoad64, types.NewTuple(types.Types[types.TUINT64], types.TypeMem), args[0], s.mem()) - s.vars[memVar] = s.newValue1(ssa.OpSelect1, types.TypeMem, v) - return s.newValue1(ssa.OpSelect0, types.Types[types.TUINT64], v) - }, - sys.AMD64, sys.ARM64, sys.MIPS64, sys.PPC64, sys.RISCV64, sys.S390X) - addF("runtime/internal/atomic", "LoadAcq", - func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { - v := s.newValue2(ssa.OpAtomicLoadAcq32, types.NewTuple(types.Types[types.TUINT32], types.TypeMem), args[0], s.mem()) - s.vars[memVar] = s.newValue1(ssa.OpSelect1, types.TypeMem, v) - return s.newValue1(ssa.OpSelect0, types.Types[types.TUINT32], v) - }, - sys.PPC64, sys.S390X) - addF("runtime/internal/atomic", "LoadAcq64", - func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { - v := s.newValue2(ssa.OpAtomicLoadAcq64, types.NewTuple(types.Types[types.TUINT64], types.TypeMem), args[0], s.mem()) - s.vars[memVar] = s.newValue1(ssa.OpSelect1, types.TypeMem, v) - return s.newValue1(ssa.OpSelect0, types.Types[types.TUINT64], v) - }, - sys.PPC64) - addF("runtime/internal/atomic", "Loadp", - func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { - v := s.newValue2(ssa.OpAtomicLoadPtr, types.NewTuple(s.f.Config.Types.BytePtr, types.TypeMem), args[0], s.mem()) - s.vars[memVar] = s.newValue1(ssa.OpSelect1, types.TypeMem, v) - return s.newValue1(ssa.OpSelect0, s.f.Config.Types.BytePtr, v) - }, - sys.AMD64, sys.ARM64, sys.MIPS, sys.MIPS64, sys.PPC64, sys.RISCV64, sys.S390X) - - addF("runtime/internal/atomic", "Store", - func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { - s.vars[memVar] = s.newValue3(ssa.OpAtomicStore32, types.TypeMem, args[0], args[1], s.mem()) - return nil - }, - sys.AMD64, sys.ARM64, sys.MIPS, sys.MIPS64, sys.PPC64, sys.RISCV64, sys.S390X) - addF("runtime/internal/atomic", "Store8", - func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { - s.vars[memVar] = s.newValue3(ssa.OpAtomicStore8, types.TypeMem, args[0], args[1], s.mem()) - return nil - }, - sys.AMD64, sys.ARM64, sys.MIPS, sys.MIPS64, sys.PPC64, sys.RISCV64, sys.S390X) - addF("runtime/internal/atomic", "Store64", - func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { - s.vars[memVar] = s.newValue3(ssa.OpAtomicStore64, types.TypeMem, args[0], args[1], s.mem()) - return nil - }, - sys.AMD64, sys.ARM64, sys.MIPS64, sys.PPC64, sys.RISCV64, sys.S390X) - addF("runtime/internal/atomic", "StorepNoWB", - func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { - s.vars[memVar] = s.newValue3(ssa.OpAtomicStorePtrNoWB, types.TypeMem, args[0], args[1], s.mem()) - return nil - }, - sys.AMD64, sys.ARM64, sys.MIPS, sys.MIPS64, sys.RISCV64, sys.S390X) - addF("runtime/internal/atomic", "StoreRel", - func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { - s.vars[memVar] = s.newValue3(ssa.OpAtomicStoreRel32, types.TypeMem, args[0], args[1], s.mem()) - return nil - }, - sys.PPC64, sys.S390X) - addF("runtime/internal/atomic", "StoreRel64", - func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { - s.vars[memVar] = s.newValue3(ssa.OpAtomicStoreRel64, types.TypeMem, args[0], args[1], s.mem()) - return nil - }, - sys.PPC64) - - addF("runtime/internal/atomic", "Xchg", - func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { - v := s.newValue3(ssa.OpAtomicExchange32, types.NewTuple(types.Types[types.TUINT32], types.TypeMem), args[0], args[1], s.mem()) - s.vars[memVar] = s.newValue1(ssa.OpSelect1, types.TypeMem, v) - return s.newValue1(ssa.OpSelect0, types.Types[types.TUINT32], v) - }, - sys.AMD64, sys.MIPS, sys.MIPS64, sys.PPC64, sys.RISCV64, sys.S390X) - addF("runtime/internal/atomic", "Xchg64", - func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { - v := s.newValue3(ssa.OpAtomicExchange64, types.NewTuple(types.Types[types.TUINT64], types.TypeMem), args[0], args[1], s.mem()) - s.vars[memVar] = s.newValue1(ssa.OpSelect1, types.TypeMem, v) - return s.newValue1(ssa.OpSelect0, types.Types[types.TUINT64], v) - }, - sys.AMD64, sys.MIPS64, sys.PPC64, sys.RISCV64, sys.S390X) - - type atomicOpEmitter func(s *state, n *ir.CallExpr, args []*ssa.Value, op ssa.Op, typ types.Kind) - - makeAtomicGuardedIntrinsicARM64 := func(op0, op1 ssa.Op, typ, rtyp types.Kind, emit atomicOpEmitter) intrinsicBuilder { - - return func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { - // Target Atomic feature is identified by dynamic detection - addr := s.entryNewValue1A(ssa.OpAddr, types.Types[types.TBOOL].PtrTo(), ir.Syms.ARM64HasATOMICS, s.sb) - v := s.load(types.Types[types.TBOOL], addr) - b := s.endBlock() - b.Kind = ssa.BlockIf - b.SetControl(v) - bTrue := s.f.NewBlock(ssa.BlockPlain) - bFalse := s.f.NewBlock(ssa.BlockPlain) - bEnd := s.f.NewBlock(ssa.BlockPlain) - b.AddEdgeTo(bTrue) - b.AddEdgeTo(bFalse) - b.Likely = ssa.BranchLikely - - // We have atomic instructions - use it directly. - s.startBlock(bTrue) - emit(s, n, args, op1, typ) - s.endBlock().AddEdgeTo(bEnd) - - // Use original instruction sequence. - s.startBlock(bFalse) - emit(s, n, args, op0, typ) - s.endBlock().AddEdgeTo(bEnd) - - // Merge results. - s.startBlock(bEnd) - if rtyp == types.TNIL { - return nil - } else { - return s.variable(n, types.Types[rtyp]) - } - } - } - - atomicXchgXaddEmitterARM64 := func(s *state, n *ir.CallExpr, args []*ssa.Value, op ssa.Op, typ types.Kind) { - v := s.newValue3(op, types.NewTuple(types.Types[typ], types.TypeMem), args[0], args[1], s.mem()) - s.vars[memVar] = s.newValue1(ssa.OpSelect1, types.TypeMem, v) - s.vars[n] = s.newValue1(ssa.OpSelect0, types.Types[typ], v) - } - addF("runtime/internal/atomic", "Xchg", - makeAtomicGuardedIntrinsicARM64(ssa.OpAtomicExchange32, ssa.OpAtomicExchange32Variant, types.TUINT32, types.TUINT32, atomicXchgXaddEmitterARM64), - sys.ARM64) - addF("runtime/internal/atomic", "Xchg64", - makeAtomicGuardedIntrinsicARM64(ssa.OpAtomicExchange64, ssa.OpAtomicExchange64Variant, types.TUINT64, types.TUINT64, atomicXchgXaddEmitterARM64), - sys.ARM64) - - addF("runtime/internal/atomic", "Xadd", - func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { - v := s.newValue3(ssa.OpAtomicAdd32, types.NewTuple(types.Types[types.TUINT32], types.TypeMem), args[0], args[1], s.mem()) - s.vars[memVar] = s.newValue1(ssa.OpSelect1, types.TypeMem, v) - return s.newValue1(ssa.OpSelect0, types.Types[types.TUINT32], v) - }, - sys.AMD64, sys.MIPS, sys.MIPS64, sys.PPC64, sys.RISCV64, sys.S390X) - addF("runtime/internal/atomic", "Xadd64", - func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { - v := s.newValue3(ssa.OpAtomicAdd64, types.NewTuple(types.Types[types.TUINT64], types.TypeMem), args[0], args[1], s.mem()) - s.vars[memVar] = s.newValue1(ssa.OpSelect1, types.TypeMem, v) - return s.newValue1(ssa.OpSelect0, types.Types[types.TUINT64], v) - }, - sys.AMD64, sys.MIPS64, sys.PPC64, sys.RISCV64, sys.S390X) - - addF("runtime/internal/atomic", "Xadd", - makeAtomicGuardedIntrinsicARM64(ssa.OpAtomicAdd32, ssa.OpAtomicAdd32Variant, types.TUINT32, types.TUINT32, atomicXchgXaddEmitterARM64), - sys.ARM64) - addF("runtime/internal/atomic", "Xadd64", - makeAtomicGuardedIntrinsicARM64(ssa.OpAtomicAdd64, ssa.OpAtomicAdd64Variant, types.TUINT64, types.TUINT64, atomicXchgXaddEmitterARM64), - sys.ARM64) - - addF("runtime/internal/atomic", "Cas", - func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { - v := s.newValue4(ssa.OpAtomicCompareAndSwap32, types.NewTuple(types.Types[types.TBOOL], types.TypeMem), args[0], args[1], args[2], s.mem()) - s.vars[memVar] = s.newValue1(ssa.OpSelect1, types.TypeMem, v) - return s.newValue1(ssa.OpSelect0, types.Types[types.TBOOL], v) - }, - sys.AMD64, sys.MIPS, sys.MIPS64, sys.PPC64, sys.RISCV64, sys.S390X) - addF("runtime/internal/atomic", "Cas64", - func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { - v := s.newValue4(ssa.OpAtomicCompareAndSwap64, types.NewTuple(types.Types[types.TBOOL], types.TypeMem), args[0], args[1], args[2], s.mem()) - s.vars[memVar] = s.newValue1(ssa.OpSelect1, types.TypeMem, v) - return s.newValue1(ssa.OpSelect0, types.Types[types.TBOOL], v) - }, - sys.AMD64, sys.MIPS64, sys.PPC64, sys.RISCV64, sys.S390X) - addF("runtime/internal/atomic", "CasRel", - func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { - v := s.newValue4(ssa.OpAtomicCompareAndSwap32, types.NewTuple(types.Types[types.TBOOL], types.TypeMem), args[0], args[1], args[2], s.mem()) - s.vars[memVar] = s.newValue1(ssa.OpSelect1, types.TypeMem, v) - return s.newValue1(ssa.OpSelect0, types.Types[types.TBOOL], v) - }, - sys.PPC64) - - atomicCasEmitterARM64 := func(s *state, n *ir.CallExpr, args []*ssa.Value, op ssa.Op, typ types.Kind) { - v := s.newValue4(op, types.NewTuple(types.Types[types.TBOOL], types.TypeMem), args[0], args[1], args[2], s.mem()) - s.vars[memVar] = s.newValue1(ssa.OpSelect1, types.TypeMem, v) - s.vars[n] = s.newValue1(ssa.OpSelect0, types.Types[typ], v) - } - - addF("runtime/internal/atomic", "Cas", - makeAtomicGuardedIntrinsicARM64(ssa.OpAtomicCompareAndSwap32, ssa.OpAtomicCompareAndSwap32Variant, types.TUINT32, types.TBOOL, atomicCasEmitterARM64), - sys.ARM64) - addF("runtime/internal/atomic", "Cas64", - makeAtomicGuardedIntrinsicARM64(ssa.OpAtomicCompareAndSwap64, ssa.OpAtomicCompareAndSwap64Variant, types.TUINT64, types.TBOOL, atomicCasEmitterARM64), - sys.ARM64) - - addF("runtime/internal/atomic", "And8", - func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { - s.vars[memVar] = s.newValue3(ssa.OpAtomicAnd8, types.TypeMem, args[0], args[1], s.mem()) - return nil - }, - sys.AMD64, sys.MIPS, sys.PPC64, sys.S390X) - addF("runtime/internal/atomic", "And", - func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { - s.vars[memVar] = s.newValue3(ssa.OpAtomicAnd32, types.TypeMem, args[0], args[1], s.mem()) - return nil - }, - sys.AMD64, sys.MIPS, sys.PPC64, sys.S390X) - addF("runtime/internal/atomic", "Or8", - func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { - s.vars[memVar] = s.newValue3(ssa.OpAtomicOr8, types.TypeMem, args[0], args[1], s.mem()) - return nil - }, - sys.AMD64, sys.ARM64, sys.MIPS, sys.PPC64, sys.S390X) - addF("runtime/internal/atomic", "Or", - func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { - s.vars[memVar] = s.newValue3(ssa.OpAtomicOr32, types.TypeMem, args[0], args[1], s.mem()) - return nil - }, - sys.AMD64, sys.MIPS, sys.PPC64, sys.S390X) - - atomicAndOrEmitterARM64 := func(s *state, n *ir.CallExpr, args []*ssa.Value, op ssa.Op, typ types.Kind) { - s.vars[memVar] = s.newValue3(op, types.TypeMem, args[0], args[1], s.mem()) - } - - addF("runtime/internal/atomic", "And8", - makeAtomicGuardedIntrinsicARM64(ssa.OpAtomicAnd8, ssa.OpAtomicAnd8Variant, types.TNIL, types.TNIL, atomicAndOrEmitterARM64), - sys.ARM64) - addF("runtime/internal/atomic", "And", - makeAtomicGuardedIntrinsicARM64(ssa.OpAtomicAnd32, ssa.OpAtomicAnd32Variant, types.TNIL, types.TNIL, atomicAndOrEmitterARM64), - sys.ARM64) - addF("runtime/internal/atomic", "Or8", - makeAtomicGuardedIntrinsicARM64(ssa.OpAtomicOr8, ssa.OpAtomicOr8Variant, types.TNIL, types.TNIL, atomicAndOrEmitterARM64), - sys.ARM64) - addF("runtime/internal/atomic", "Or", - makeAtomicGuardedIntrinsicARM64(ssa.OpAtomicOr32, ssa.OpAtomicOr32Variant, types.TNIL, types.TNIL, atomicAndOrEmitterARM64), - sys.ARM64) - - alias("runtime/internal/atomic", "Loadint64", "runtime/internal/atomic", "Load64", all...) - alias("runtime/internal/atomic", "Xaddint64", "runtime/internal/atomic", "Xadd64", all...) - alias("runtime/internal/atomic", "Loaduint", "runtime/internal/atomic", "Load", p4...) - alias("runtime/internal/atomic", "Loaduint", "runtime/internal/atomic", "Load64", p8...) - alias("runtime/internal/atomic", "Loaduintptr", "runtime/internal/atomic", "Load", p4...) - alias("runtime/internal/atomic", "Loaduintptr", "runtime/internal/atomic", "Load64", p8...) - alias("runtime/internal/atomic", "LoadAcq", "runtime/internal/atomic", "Load", lwatomics...) - alias("runtime/internal/atomic", "LoadAcq64", "runtime/internal/atomic", "Load64", lwatomics...) - alias("runtime/internal/atomic", "LoadAcquintptr", "runtime/internal/atomic", "LoadAcq", p4...) - alias("sync", "runtime_LoadAcquintptr", "runtime/internal/atomic", "LoadAcq", p4...) // linknamed - alias("runtime/internal/atomic", "LoadAcquintptr", "runtime/internal/atomic", "LoadAcq64", p8...) - alias("sync", "runtime_LoadAcquintptr", "runtime/internal/atomic", "LoadAcq64", p8...) // linknamed - alias("runtime/internal/atomic", "Storeuintptr", "runtime/internal/atomic", "Store", p4...) - alias("runtime/internal/atomic", "Storeuintptr", "runtime/internal/atomic", "Store64", p8...) - alias("runtime/internal/atomic", "StoreRel", "runtime/internal/atomic", "Store", lwatomics...) - alias("runtime/internal/atomic", "StoreRel64", "runtime/internal/atomic", "Store64", lwatomics...) - alias("runtime/internal/atomic", "StoreReluintptr", "runtime/internal/atomic", "StoreRel", p4...) - alias("sync", "runtime_StoreReluintptr", "runtime/internal/atomic", "StoreRel", p4...) // linknamed - alias("runtime/internal/atomic", "StoreReluintptr", "runtime/internal/atomic", "StoreRel64", p8...) - alias("sync", "runtime_StoreReluintptr", "runtime/internal/atomic", "StoreRel64", p8...) // linknamed - alias("runtime/internal/atomic", "Xchguintptr", "runtime/internal/atomic", "Xchg", p4...) - alias("runtime/internal/atomic", "Xchguintptr", "runtime/internal/atomic", "Xchg64", p8...) - alias("runtime/internal/atomic", "Xadduintptr", "runtime/internal/atomic", "Xadd", p4...) - alias("runtime/internal/atomic", "Xadduintptr", "runtime/internal/atomic", "Xadd64", p8...) - alias("runtime/internal/atomic", "Casuintptr", "runtime/internal/atomic", "Cas", p4...) - alias("runtime/internal/atomic", "Casuintptr", "runtime/internal/atomic", "Cas64", p8...) - alias("runtime/internal/atomic", "Casp1", "runtime/internal/atomic", "Cas", p4...) - alias("runtime/internal/atomic", "Casp1", "runtime/internal/atomic", "Cas64", p8...) - alias("runtime/internal/atomic", "CasRel", "runtime/internal/atomic", "Cas", lwatomics...) - - /******** math ********/ - addF("math", "Sqrt", - func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { - return s.newValue1(ssa.OpSqrt, types.Types[types.TFLOAT64], args[0]) - }, - sys.I386, sys.AMD64, sys.ARM, sys.ARM64, sys.MIPS, sys.MIPS64, sys.PPC64, sys.RISCV64, sys.S390X, sys.Wasm) - addF("math", "Trunc", - func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { - return s.newValue1(ssa.OpTrunc, types.Types[types.TFLOAT64], args[0]) - }, - sys.ARM64, sys.PPC64, sys.S390X, sys.Wasm) - addF("math", "Ceil", - func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { - return s.newValue1(ssa.OpCeil, types.Types[types.TFLOAT64], args[0]) - }, - sys.ARM64, sys.PPC64, sys.S390X, sys.Wasm) - addF("math", "Floor", - func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { - return s.newValue1(ssa.OpFloor, types.Types[types.TFLOAT64], args[0]) - }, - sys.ARM64, sys.PPC64, sys.S390X, sys.Wasm) - addF("math", "Round", - func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { - return s.newValue1(ssa.OpRound, types.Types[types.TFLOAT64], args[0]) - }, - sys.ARM64, sys.PPC64, sys.S390X) - addF("math", "RoundToEven", - func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { - return s.newValue1(ssa.OpRoundToEven, types.Types[types.TFLOAT64], args[0]) - }, - sys.ARM64, sys.S390X, sys.Wasm) - addF("math", "Abs", - func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { - return s.newValue1(ssa.OpAbs, types.Types[types.TFLOAT64], args[0]) - }, - sys.ARM64, sys.ARM, sys.PPC64, sys.Wasm) - addF("math", "Copysign", - func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { - return s.newValue2(ssa.OpCopysign, types.Types[types.TFLOAT64], args[0], args[1]) - }, - sys.PPC64, sys.Wasm) - addF("math", "FMA", - func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { - return s.newValue3(ssa.OpFMA, types.Types[types.TFLOAT64], args[0], args[1], args[2]) - }, - sys.ARM64, sys.PPC64, sys.S390X) - addF("math", "FMA", - func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { - if !s.config.UseFMA { - s.vars[n] = s.callResult(n, callNormal) // types.Types[TFLOAT64] - return s.variable(n, types.Types[types.TFLOAT64]) - } - v := s.entryNewValue0A(ssa.OpHasCPUFeature, types.Types[types.TBOOL], ir.Syms.X86HasFMA) - b := s.endBlock() - b.Kind = ssa.BlockIf - b.SetControl(v) - bTrue := s.f.NewBlock(ssa.BlockPlain) - bFalse := s.f.NewBlock(ssa.BlockPlain) - bEnd := s.f.NewBlock(ssa.BlockPlain) - b.AddEdgeTo(bTrue) - b.AddEdgeTo(bFalse) - b.Likely = ssa.BranchLikely // >= haswell cpus are common - - // We have the intrinsic - use it directly. - s.startBlock(bTrue) - s.vars[n] = s.newValue3(ssa.OpFMA, types.Types[types.TFLOAT64], args[0], args[1], args[2]) - s.endBlock().AddEdgeTo(bEnd) - - // Call the pure Go version. - s.startBlock(bFalse) - s.vars[n] = s.callResult(n, callNormal) // types.Types[TFLOAT64] - s.endBlock().AddEdgeTo(bEnd) - - // Merge results. - s.startBlock(bEnd) - return s.variable(n, types.Types[types.TFLOAT64]) - }, - sys.AMD64) - addF("math", "FMA", - func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { - if !s.config.UseFMA { - s.vars[n] = s.callResult(n, callNormal) // types.Types[TFLOAT64] - return s.variable(n, types.Types[types.TFLOAT64]) - } - addr := s.entryNewValue1A(ssa.OpAddr, types.Types[types.TBOOL].PtrTo(), ir.Syms.ARMHasVFPv4, s.sb) - v := s.load(types.Types[types.TBOOL], addr) - b := s.endBlock() - b.Kind = ssa.BlockIf - b.SetControl(v) - bTrue := s.f.NewBlock(ssa.BlockPlain) - bFalse := s.f.NewBlock(ssa.BlockPlain) - bEnd := s.f.NewBlock(ssa.BlockPlain) - b.AddEdgeTo(bTrue) - b.AddEdgeTo(bFalse) - b.Likely = ssa.BranchLikely - - // We have the intrinsic - use it directly. - s.startBlock(bTrue) - s.vars[n] = s.newValue3(ssa.OpFMA, types.Types[types.TFLOAT64], args[0], args[1], args[2]) - s.endBlock().AddEdgeTo(bEnd) - - // Call the pure Go version. - s.startBlock(bFalse) - s.vars[n] = s.callResult(n, callNormal) // types.Types[TFLOAT64] - s.endBlock().AddEdgeTo(bEnd) - - // Merge results. - s.startBlock(bEnd) - return s.variable(n, types.Types[types.TFLOAT64]) - }, - sys.ARM) - - makeRoundAMD64 := func(op ssa.Op) func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { - return func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { - v := s.entryNewValue0A(ssa.OpHasCPUFeature, types.Types[types.TBOOL], ir.Syms.X86HasSSE41) - b := s.endBlock() - b.Kind = ssa.BlockIf - b.SetControl(v) - bTrue := s.f.NewBlock(ssa.BlockPlain) - bFalse := s.f.NewBlock(ssa.BlockPlain) - bEnd := s.f.NewBlock(ssa.BlockPlain) - b.AddEdgeTo(bTrue) - b.AddEdgeTo(bFalse) - b.Likely = ssa.BranchLikely // most machines have sse4.1 nowadays - - // We have the intrinsic - use it directly. - s.startBlock(bTrue) - s.vars[n] = s.newValue1(op, types.Types[types.TFLOAT64], args[0]) - s.endBlock().AddEdgeTo(bEnd) - - // Call the pure Go version. - s.startBlock(bFalse) - s.vars[n] = s.callResult(n, callNormal) // types.Types[TFLOAT64] - s.endBlock().AddEdgeTo(bEnd) - - // Merge results. - s.startBlock(bEnd) - return s.variable(n, types.Types[types.TFLOAT64]) - } - } - addF("math", "RoundToEven", - makeRoundAMD64(ssa.OpRoundToEven), - sys.AMD64) - addF("math", "Floor", - makeRoundAMD64(ssa.OpFloor), - sys.AMD64) - addF("math", "Ceil", - makeRoundAMD64(ssa.OpCeil), - sys.AMD64) - addF("math", "Trunc", - makeRoundAMD64(ssa.OpTrunc), - sys.AMD64) - - /******** math/bits ********/ - addF("math/bits", "TrailingZeros64", - func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { - return s.newValue1(ssa.OpCtz64, types.Types[types.TINT], args[0]) - }, - sys.AMD64, sys.ARM64, sys.ARM, sys.S390X, sys.MIPS, sys.PPC64, sys.Wasm) - addF("math/bits", "TrailingZeros32", - func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { - return s.newValue1(ssa.OpCtz32, types.Types[types.TINT], args[0]) - }, - sys.AMD64, sys.ARM64, sys.ARM, sys.S390X, sys.MIPS, sys.PPC64, sys.Wasm) - addF("math/bits", "TrailingZeros16", - func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { - x := s.newValue1(ssa.OpZeroExt16to32, types.Types[types.TUINT32], args[0]) - c := s.constInt32(types.Types[types.TUINT32], 1<<16) - y := s.newValue2(ssa.OpOr32, types.Types[types.TUINT32], x, c) - return s.newValue1(ssa.OpCtz32, types.Types[types.TINT], y) - }, - sys.MIPS) - addF("math/bits", "TrailingZeros16", - func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { - return s.newValue1(ssa.OpCtz16, types.Types[types.TINT], args[0]) - }, - sys.AMD64, sys.I386, sys.ARM, sys.ARM64, sys.Wasm) - addF("math/bits", "TrailingZeros16", - func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { - x := s.newValue1(ssa.OpZeroExt16to64, types.Types[types.TUINT64], args[0]) - c := s.constInt64(types.Types[types.TUINT64], 1<<16) - y := s.newValue2(ssa.OpOr64, types.Types[types.TUINT64], x, c) - return s.newValue1(ssa.OpCtz64, types.Types[types.TINT], y) - }, - sys.S390X, sys.PPC64) - addF("math/bits", "TrailingZeros8", - func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { - x := s.newValue1(ssa.OpZeroExt8to32, types.Types[types.TUINT32], args[0]) - c := s.constInt32(types.Types[types.TUINT32], 1<<8) - y := s.newValue2(ssa.OpOr32, types.Types[types.TUINT32], x, c) - return s.newValue1(ssa.OpCtz32, types.Types[types.TINT], y) - }, - sys.MIPS) - addF("math/bits", "TrailingZeros8", - func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { - return s.newValue1(ssa.OpCtz8, types.Types[types.TINT], args[0]) - }, - sys.AMD64, sys.ARM, sys.ARM64, sys.Wasm) - addF("math/bits", "TrailingZeros8", - func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { - x := s.newValue1(ssa.OpZeroExt8to64, types.Types[types.TUINT64], args[0]) - c := s.constInt64(types.Types[types.TUINT64], 1<<8) - y := s.newValue2(ssa.OpOr64, types.Types[types.TUINT64], x, c) - return s.newValue1(ssa.OpCtz64, types.Types[types.TINT], y) - }, - sys.S390X) - alias("math/bits", "ReverseBytes64", "runtime/internal/sys", "Bswap64", all...) - alias("math/bits", "ReverseBytes32", "runtime/internal/sys", "Bswap32", all...) - // ReverseBytes inlines correctly, no need to intrinsify it. - // ReverseBytes16 lowers to a rotate, no need for anything special here. - addF("math/bits", "Len64", - func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { - return s.newValue1(ssa.OpBitLen64, types.Types[types.TINT], args[0]) - }, - sys.AMD64, sys.ARM64, sys.ARM, sys.S390X, sys.MIPS, sys.PPC64, sys.Wasm) - addF("math/bits", "Len32", - func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { - return s.newValue1(ssa.OpBitLen32, types.Types[types.TINT], args[0]) - }, - sys.AMD64, sys.ARM64) - addF("math/bits", "Len32", - func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { - if s.config.PtrSize == 4 { - return s.newValue1(ssa.OpBitLen32, types.Types[types.TINT], args[0]) - } - x := s.newValue1(ssa.OpZeroExt32to64, types.Types[types.TUINT64], args[0]) - return s.newValue1(ssa.OpBitLen64, types.Types[types.TINT], x) - }, - sys.ARM, sys.S390X, sys.MIPS, sys.PPC64, sys.Wasm) - addF("math/bits", "Len16", - func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { - if s.config.PtrSize == 4 { - x := s.newValue1(ssa.OpZeroExt16to32, types.Types[types.TUINT32], args[0]) - return s.newValue1(ssa.OpBitLen32, types.Types[types.TINT], x) - } - x := s.newValue1(ssa.OpZeroExt16to64, types.Types[types.TUINT64], args[0]) - return s.newValue1(ssa.OpBitLen64, types.Types[types.TINT], x) - }, - sys.ARM64, sys.ARM, sys.S390X, sys.MIPS, sys.PPC64, sys.Wasm) - addF("math/bits", "Len16", - func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { - return s.newValue1(ssa.OpBitLen16, types.Types[types.TINT], args[0]) - }, - sys.AMD64) - addF("math/bits", "Len8", - func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { - if s.config.PtrSize == 4 { - x := s.newValue1(ssa.OpZeroExt8to32, types.Types[types.TUINT32], args[0]) - return s.newValue1(ssa.OpBitLen32, types.Types[types.TINT], x) - } - x := s.newValue1(ssa.OpZeroExt8to64, types.Types[types.TUINT64], args[0]) - return s.newValue1(ssa.OpBitLen64, types.Types[types.TINT], x) - }, - sys.ARM64, sys.ARM, sys.S390X, sys.MIPS, sys.PPC64, sys.Wasm) - addF("math/bits", "Len8", - func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { - return s.newValue1(ssa.OpBitLen8, types.Types[types.TINT], args[0]) - }, - sys.AMD64) - addF("math/bits", "Len", - func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { - if s.config.PtrSize == 4 { - return s.newValue1(ssa.OpBitLen32, types.Types[types.TINT], args[0]) - } - return s.newValue1(ssa.OpBitLen64, types.Types[types.TINT], args[0]) - }, - sys.AMD64, sys.ARM64, sys.ARM, sys.S390X, sys.MIPS, sys.PPC64, sys.Wasm) - // LeadingZeros is handled because it trivially calls Len. - addF("math/bits", "Reverse64", - func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { - return s.newValue1(ssa.OpBitRev64, types.Types[types.TINT], args[0]) - }, - sys.ARM64) - addF("math/bits", "Reverse32", - func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { - return s.newValue1(ssa.OpBitRev32, types.Types[types.TINT], args[0]) - }, - sys.ARM64) - addF("math/bits", "Reverse16", - func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { - return s.newValue1(ssa.OpBitRev16, types.Types[types.TINT], args[0]) - }, - sys.ARM64) - addF("math/bits", "Reverse8", - func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { - return s.newValue1(ssa.OpBitRev8, types.Types[types.TINT], args[0]) - }, - sys.ARM64) - addF("math/bits", "Reverse", - func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { - if s.config.PtrSize == 4 { - return s.newValue1(ssa.OpBitRev32, types.Types[types.TINT], args[0]) - } - return s.newValue1(ssa.OpBitRev64, types.Types[types.TINT], args[0]) - }, - sys.ARM64) - addF("math/bits", "RotateLeft8", - func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { - return s.newValue2(ssa.OpRotateLeft8, types.Types[types.TUINT8], args[0], args[1]) - }, - sys.AMD64) - addF("math/bits", "RotateLeft16", - func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { - return s.newValue2(ssa.OpRotateLeft16, types.Types[types.TUINT16], args[0], args[1]) - }, - sys.AMD64) - addF("math/bits", "RotateLeft32", - func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { - return s.newValue2(ssa.OpRotateLeft32, types.Types[types.TUINT32], args[0], args[1]) - }, - sys.AMD64, sys.ARM, sys.ARM64, sys.S390X, sys.PPC64, sys.Wasm) - addF("math/bits", "RotateLeft64", - func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { - return s.newValue2(ssa.OpRotateLeft64, types.Types[types.TUINT64], args[0], args[1]) - }, - sys.AMD64, sys.ARM64, sys.S390X, sys.PPC64, sys.Wasm) - alias("math/bits", "RotateLeft", "math/bits", "RotateLeft64", p8...) - - makeOnesCountAMD64 := func(op64 ssa.Op, op32 ssa.Op) func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { - return func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { - v := s.entryNewValue0A(ssa.OpHasCPUFeature, types.Types[types.TBOOL], ir.Syms.X86HasPOPCNT) - b := s.endBlock() - b.Kind = ssa.BlockIf - b.SetControl(v) - bTrue := s.f.NewBlock(ssa.BlockPlain) - bFalse := s.f.NewBlock(ssa.BlockPlain) - bEnd := s.f.NewBlock(ssa.BlockPlain) - b.AddEdgeTo(bTrue) - b.AddEdgeTo(bFalse) - b.Likely = ssa.BranchLikely // most machines have popcnt nowadays - - // We have the intrinsic - use it directly. - s.startBlock(bTrue) - op := op64 - if s.config.PtrSize == 4 { - op = op32 - } - s.vars[n] = s.newValue1(op, types.Types[types.TINT], args[0]) - s.endBlock().AddEdgeTo(bEnd) - - // Call the pure Go version. - s.startBlock(bFalse) - s.vars[n] = s.callResult(n, callNormal) // types.Types[TINT] - s.endBlock().AddEdgeTo(bEnd) - - // Merge results. - s.startBlock(bEnd) - return s.variable(n, types.Types[types.TINT]) - } - } - addF("math/bits", "OnesCount64", - makeOnesCountAMD64(ssa.OpPopCount64, ssa.OpPopCount64), - sys.AMD64) - addF("math/bits", "OnesCount64", - func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { - return s.newValue1(ssa.OpPopCount64, types.Types[types.TINT], args[0]) - }, - sys.PPC64, sys.ARM64, sys.S390X, sys.Wasm) - addF("math/bits", "OnesCount32", - makeOnesCountAMD64(ssa.OpPopCount32, ssa.OpPopCount32), - sys.AMD64) - addF("math/bits", "OnesCount32", - func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { - return s.newValue1(ssa.OpPopCount32, types.Types[types.TINT], args[0]) - }, - sys.PPC64, sys.ARM64, sys.S390X, sys.Wasm) - addF("math/bits", "OnesCount16", - makeOnesCountAMD64(ssa.OpPopCount16, ssa.OpPopCount16), - sys.AMD64) - addF("math/bits", "OnesCount16", - func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { - return s.newValue1(ssa.OpPopCount16, types.Types[types.TINT], args[0]) - }, - sys.ARM64, sys.S390X, sys.PPC64, sys.Wasm) - addF("math/bits", "OnesCount8", - func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { - return s.newValue1(ssa.OpPopCount8, types.Types[types.TINT], args[0]) - }, - sys.S390X, sys.PPC64, sys.Wasm) - addF("math/bits", "OnesCount", - makeOnesCountAMD64(ssa.OpPopCount64, ssa.OpPopCount32), - sys.AMD64) - addF("math/bits", "Mul64", - func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { - return s.newValue2(ssa.OpMul64uhilo, types.NewTuple(types.Types[types.TUINT64], types.Types[types.TUINT64]), args[0], args[1]) - }, - sys.AMD64, sys.ARM64, sys.PPC64, sys.S390X, sys.MIPS64) - alias("math/bits", "Mul", "math/bits", "Mul64", sys.ArchAMD64, sys.ArchARM64, sys.ArchPPC64, sys.ArchS390X, sys.ArchMIPS64, sys.ArchMIPS64LE) - addF("math/bits", "Add64", - func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { - return s.newValue3(ssa.OpAdd64carry, types.NewTuple(types.Types[types.TUINT64], types.Types[types.TUINT64]), args[0], args[1], args[2]) - }, - sys.AMD64, sys.ARM64, sys.PPC64, sys.S390X) - alias("math/bits", "Add", "math/bits", "Add64", sys.ArchAMD64, sys.ArchARM64, sys.ArchPPC64, sys.ArchS390X) - addF("math/bits", "Sub64", - func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { - return s.newValue3(ssa.OpSub64borrow, types.NewTuple(types.Types[types.TUINT64], types.Types[types.TUINT64]), args[0], args[1], args[2]) - }, - sys.AMD64, sys.ARM64, sys.S390X) - alias("math/bits", "Sub", "math/bits", "Sub64", sys.ArchAMD64, sys.ArchARM64, sys.ArchS390X) - addF("math/bits", "Div64", - func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { - // check for divide-by-zero/overflow and panic with appropriate message - cmpZero := s.newValue2(s.ssaOp(ir.ONE, types.Types[types.TUINT64]), types.Types[types.TBOOL], args[2], s.zeroVal(types.Types[types.TUINT64])) - s.check(cmpZero, ir.Syms.Panicdivide) - cmpOverflow := s.newValue2(s.ssaOp(ir.OLT, types.Types[types.TUINT64]), types.Types[types.TBOOL], args[0], args[2]) - s.check(cmpOverflow, ir.Syms.Panicoverflow) - return s.newValue3(ssa.OpDiv128u, types.NewTuple(types.Types[types.TUINT64], types.Types[types.TUINT64]), args[0], args[1], args[2]) - }, - sys.AMD64) - alias("math/bits", "Div", "math/bits", "Div64", sys.ArchAMD64) - - alias("runtime/internal/sys", "Ctz8", "math/bits", "TrailingZeros8", all...) - alias("runtime/internal/sys", "TrailingZeros8", "math/bits", "TrailingZeros8", all...) - alias("runtime/internal/sys", "TrailingZeros64", "math/bits", "TrailingZeros64", all...) - alias("runtime/internal/sys", "Len8", "math/bits", "Len8", all...) - alias("runtime/internal/sys", "Len64", "math/bits", "Len64", all...) - alias("runtime/internal/sys", "OnesCount64", "math/bits", "OnesCount64", all...) - - /******** sync/atomic ********/ - - // Note: these are disabled by flag_race in findIntrinsic below. - alias("sync/atomic", "LoadInt32", "runtime/internal/atomic", "Load", all...) - alias("sync/atomic", "LoadInt64", "runtime/internal/atomic", "Load64", all...) - alias("sync/atomic", "LoadPointer", "runtime/internal/atomic", "Loadp", all...) - alias("sync/atomic", "LoadUint32", "runtime/internal/atomic", "Load", all...) - alias("sync/atomic", "LoadUint64", "runtime/internal/atomic", "Load64", all...) - alias("sync/atomic", "LoadUintptr", "runtime/internal/atomic", "Load", p4...) - alias("sync/atomic", "LoadUintptr", "runtime/internal/atomic", "Load64", p8...) - - alias("sync/atomic", "StoreInt32", "runtime/internal/atomic", "Store", all...) - alias("sync/atomic", "StoreInt64", "runtime/internal/atomic", "Store64", all...) - // Note: not StorePointer, that needs a write barrier. Same below for {CompareAnd}Swap. - alias("sync/atomic", "StoreUint32", "runtime/internal/atomic", "Store", all...) - alias("sync/atomic", "StoreUint64", "runtime/internal/atomic", "Store64", all...) - alias("sync/atomic", "StoreUintptr", "runtime/internal/atomic", "Store", p4...) - alias("sync/atomic", "StoreUintptr", "runtime/internal/atomic", "Store64", p8...) - - alias("sync/atomic", "SwapInt32", "runtime/internal/atomic", "Xchg", all...) - alias("sync/atomic", "SwapInt64", "runtime/internal/atomic", "Xchg64", all...) - alias("sync/atomic", "SwapUint32", "runtime/internal/atomic", "Xchg", all...) - alias("sync/atomic", "SwapUint64", "runtime/internal/atomic", "Xchg64", all...) - alias("sync/atomic", "SwapUintptr", "runtime/internal/atomic", "Xchg", p4...) - alias("sync/atomic", "SwapUintptr", "runtime/internal/atomic", "Xchg64", p8...) - - alias("sync/atomic", "CompareAndSwapInt32", "runtime/internal/atomic", "Cas", all...) - alias("sync/atomic", "CompareAndSwapInt64", "runtime/internal/atomic", "Cas64", all...) - alias("sync/atomic", "CompareAndSwapUint32", "runtime/internal/atomic", "Cas", all...) - alias("sync/atomic", "CompareAndSwapUint64", "runtime/internal/atomic", "Cas64", all...) - alias("sync/atomic", "CompareAndSwapUintptr", "runtime/internal/atomic", "Cas", p4...) - alias("sync/atomic", "CompareAndSwapUintptr", "runtime/internal/atomic", "Cas64", p8...) - - alias("sync/atomic", "AddInt32", "runtime/internal/atomic", "Xadd", all...) - alias("sync/atomic", "AddInt64", "runtime/internal/atomic", "Xadd64", all...) - alias("sync/atomic", "AddUint32", "runtime/internal/atomic", "Xadd", all...) - alias("sync/atomic", "AddUint64", "runtime/internal/atomic", "Xadd64", all...) - alias("sync/atomic", "AddUintptr", "runtime/internal/atomic", "Xadd", p4...) - alias("sync/atomic", "AddUintptr", "runtime/internal/atomic", "Xadd64", p8...) - - /******** math/big ********/ - add("math/big", "mulWW", - func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { - return s.newValue2(ssa.OpMul64uhilo, types.NewTuple(types.Types[types.TUINT64], types.Types[types.TUINT64]), args[0], args[1]) - }, - sys.ArchAMD64, sys.ArchARM64, sys.ArchPPC64LE, sys.ArchPPC64, sys.ArchS390X) -} - -// findIntrinsic returns a function which builds the SSA equivalent of the -// function identified by the symbol sym. If sym is not an intrinsic call, returns nil. -func findIntrinsic(sym *types.Sym) intrinsicBuilder { - if sym == nil || sym.Pkg == nil { - return nil - } - pkg := sym.Pkg.Path - if sym.Pkg == types.LocalPkg { - pkg = base.Ctxt.Pkgpath - } - if base.Flag.Race && pkg == "sync/atomic" { - // The race detector needs to be able to intercept these calls. - // We can't intrinsify them. - return nil - } - // Skip intrinsifying math functions (which may contain hard-float - // instructions) when soft-float - if thearch.SoftFloat && pkg == "math" { - return nil - } - - fn := sym.Name - if ssa.IntrinsicsDisable { - if pkg == "runtime" && (fn == "getcallerpc" || fn == "getcallersp" || fn == "getclosureptr") { - // These runtime functions don't have definitions, must be intrinsics. - } else { - return nil - } - } - return intrinsics[intrinsicKey{thearch.LinkArch.Arch, pkg, fn}] -} - -func isIntrinsicCall(n *ir.CallExpr) bool { - if n == nil { - return false - } - name, ok := n.X.(*ir.Name) - if !ok { - return false - } - return findIntrinsic(name.Sym()) != nil -} - -// intrinsicCall converts a call to a recognized intrinsic function into the intrinsic SSA operation. -func (s *state) intrinsicCall(n *ir.CallExpr) *ssa.Value { - v := findIntrinsic(n.X.Sym())(s, n, s.intrinsicArgs(n)) - if ssa.IntrinsicsDebug > 0 { - x := v - if x == nil { - x = s.mem() - } - if x.Op == ssa.OpSelect0 || x.Op == ssa.OpSelect1 { - x = x.Args[0] - } - base.WarnfAt(n.Pos(), "intrinsic substitution for %v with %s", n.X.Sym().Name, x.LongString()) - } - return v -} - -// intrinsicArgs extracts args from n, evaluates them to SSA values, and returns them. -func (s *state) intrinsicArgs(n *ir.CallExpr) []*ssa.Value { - // Construct map of temps; see comments in s.call about the structure of n. - temps := map[ir.Node]*ssa.Value{} - for _, a := range n.Args { - if a.Op() != ir.OAS { - s.Fatalf("non-assignment as a temp function argument %v", a.Op()) - } - a := a.(*ir.AssignStmt) - l, r := a.X, a.Y - if l.Op() != ir.ONAME { - s.Fatalf("non-ONAME temp function argument %v", a.Op()) - } - // Evaluate and store to "temporary". - // Walk ensures these temporaries are dead outside of n. - temps[l] = s.expr(r) - } - args := make([]*ssa.Value, len(n.Rargs)) - for i, n := range n.Rargs { - // Store a value to an argument slot. - if x, ok := temps[n]; ok { - // This is a previously computed temporary. - args[i] = x - continue - } - // This is an explicit value; evaluate it. - args[i] = s.expr(n) - } - return args -} - -// openDeferRecord adds code to evaluate and store the args for an open-code defer -// call, and records info about the defer, so we can generate proper code on the -// exit paths. n is the sub-node of the defer node that is the actual function -// call. We will also record funcdata information on where the args are stored -// (as well as the deferBits variable), and this will enable us to run the proper -// defer calls during panics. -func (s *state) openDeferRecord(n *ir.CallExpr) { - // Do any needed expression evaluation for the args (including the - // receiver, if any). This may be evaluating something like 'autotmp_3 = - // once.mutex'. Such a statement will create a mapping in s.vars[] from - // the autotmp name to the evaluated SSA arg value, but won't do any - // stores to the stack. - s.stmtList(n.Args) - - var args []*ssa.Value - var argNodes []*ir.Name - - opendefer := &openDeferInfo{ - n: n, - } - fn := n.X - if n.Op() == ir.OCALLFUNC { - // We must always store the function value in a stack slot for the - // runtime panic code to use. But in the defer exit code, we will - // call the function directly if it is a static function. - closureVal := s.expr(fn) - closure := s.openDeferSave(nil, fn.Type(), closureVal) - opendefer.closureNode = closure.Aux.(*ir.Name) - if !(fn.Op() == ir.ONAME && fn.(*ir.Name).Class_ == ir.PFUNC) { - opendefer.closure = closure - } - } else if n.Op() == ir.OCALLMETH { - if fn.Op() != ir.ODOTMETH { - base.Fatalf("OCALLMETH: n.Left not an ODOTMETH: %v", fn) - } - fn := fn.(*ir.SelectorExpr) - closureVal := s.getMethodClosure(fn) - // We must always store the function value in a stack slot for the - // runtime panic code to use. But in the defer exit code, we will - // call the method directly. - closure := s.openDeferSave(nil, fn.Type(), closureVal) - opendefer.closureNode = closure.Aux.(*ir.Name) - } else { - if fn.Op() != ir.ODOTINTER { - base.Fatalf("OCALLINTER: n.Left not an ODOTINTER: %v", fn.Op()) - } - fn := fn.(*ir.SelectorExpr) - closure, rcvr := s.getClosureAndRcvr(fn) - opendefer.closure = s.openDeferSave(nil, closure.Type, closure) - // Important to get the receiver type correct, so it is recognized - // as a pointer for GC purposes. - opendefer.rcvr = s.openDeferSave(nil, fn.Type().Recv().Type, rcvr) - opendefer.closureNode = opendefer.closure.Aux.(*ir.Name) - opendefer.rcvrNode = opendefer.rcvr.Aux.(*ir.Name) - } - for _, argn := range n.Rargs { - var v *ssa.Value - if canSSAType(argn.Type()) { - v = s.openDeferSave(nil, argn.Type(), s.expr(argn)) - } else { - v = s.openDeferSave(argn, argn.Type(), nil) - } - args = append(args, v) - argNodes = append(argNodes, v.Aux.(*ir.Name)) - } - opendefer.argVals = args - opendefer.argNodes = argNodes - index := len(s.openDefers) - s.openDefers = append(s.openDefers, opendefer) - - // Update deferBits only after evaluation and storage to stack of - // args/receiver/interface is successful. - bitvalue := s.constInt8(types.Types[types.TUINT8], 1<= 0; i-- { - r := s.openDefers[i] - bCond := s.f.NewBlock(ssa.BlockPlain) - bEnd := s.f.NewBlock(ssa.BlockPlain) - - deferBits := s.variable(deferBitsVar, types.Types[types.TUINT8]) - // Generate code to check if the bit associated with the current - // defer is set. - bitval := s.constInt8(types.Types[types.TUINT8], 1< int64(4*types.PtrSize) { - // 4*Widthptr is an arbitrary constant. We want it - // to be at least 3*Widthptr so slices can be registerized. - // Too big and we'll introduce too much register pressure. - return false - } - switch t.Kind() { - case types.TARRAY: - // We can't do larger arrays because dynamic indexing is - // not supported on SSA variables. - // TODO: allow if all indexes are constant. - if t.NumElem() <= 1 { - return canSSAType(t.Elem()) - } - return false - case types.TSTRUCT: - if t.NumFields() > ssa.MaxStruct { - return false - } - for _, t1 := range t.Fields().Slice() { - if !canSSAType(t1.Type) { - return false - } - } - return true - default: - return true - } -} - -// exprPtr evaluates n to a pointer and nil-checks it. -func (s *state) exprPtr(n ir.Node, bounded bool, lineno src.XPos) *ssa.Value { - p := s.expr(n) - if bounded || n.NonNil() { - if s.f.Frontend().Debug_checknil() && lineno.Line() > 1 { - s.f.Warnl(lineno, "removed nil check") - } - return p - } - s.nilCheck(p) - return p -} - -// nilCheck generates nil pointer checking code. -// Used only for automatically inserted nil checks, -// not for user code like 'x != nil'. -func (s *state) nilCheck(ptr *ssa.Value) { - if base.Debug.DisableNil != 0 || s.curfn.NilCheckDisabled() { - return - } - s.newValue2(ssa.OpNilCheck, types.TypeVoid, ptr, s.mem()) -} - -// boundsCheck generates bounds checking code. Checks if 0 <= idx <[=] len, branches to exit if not. -// Starts a new block on return. -// On input, len must be converted to full int width and be nonnegative. -// Returns idx converted to full int width. -// If bounded is true then caller guarantees the index is not out of bounds -// (but boundsCheck will still extend the index to full int width). -func (s *state) boundsCheck(idx, len *ssa.Value, kind ssa.BoundsKind, bounded bool) *ssa.Value { - idx = s.extendIndex(idx, len, kind, bounded) - - if bounded || base.Flag.B != 0 { - // If bounded or bounds checking is flag-disabled, then no check necessary, - // just return the extended index. - // - // Here, bounded == true if the compiler generated the index itself, - // such as in the expansion of a slice initializer. These indexes are - // compiler-generated, not Go program variables, so they cannot be - // attacker-controlled, so we can omit Spectre masking as well. - // - // Note that we do not want to omit Spectre masking in code like: - // - // if 0 <= i && i < len(x) { - // use(x[i]) - // } - // - // Lucky for us, bounded==false for that code. - // In that case (handled below), we emit a bound check (and Spectre mask) - // and then the prove pass will remove the bounds check. - // In theory the prove pass could potentially remove certain - // Spectre masks, but it's very delicate and probably better - // to be conservative and leave them all in. - return idx - } - - bNext := s.f.NewBlock(ssa.BlockPlain) - bPanic := s.f.NewBlock(ssa.BlockExit) - - if !idx.Type.IsSigned() { - switch kind { - case ssa.BoundsIndex: - kind = ssa.BoundsIndexU - case ssa.BoundsSliceAlen: - kind = ssa.BoundsSliceAlenU - case ssa.BoundsSliceAcap: - kind = ssa.BoundsSliceAcapU - case ssa.BoundsSliceB: - kind = ssa.BoundsSliceBU - case ssa.BoundsSlice3Alen: - kind = ssa.BoundsSlice3AlenU - case ssa.BoundsSlice3Acap: - kind = ssa.BoundsSlice3AcapU - case ssa.BoundsSlice3B: - kind = ssa.BoundsSlice3BU - case ssa.BoundsSlice3C: - kind = ssa.BoundsSlice3CU - } - } - - var cmp *ssa.Value - if kind == ssa.BoundsIndex || kind == ssa.BoundsIndexU { - cmp = s.newValue2(ssa.OpIsInBounds, types.Types[types.TBOOL], idx, len) - } else { - cmp = s.newValue2(ssa.OpIsSliceInBounds, types.Types[types.TBOOL], idx, len) - } - b := s.endBlock() - b.Kind = ssa.BlockIf - b.SetControl(cmp) - b.Likely = ssa.BranchLikely - b.AddEdgeTo(bNext) - b.AddEdgeTo(bPanic) - - s.startBlock(bPanic) - if thearch.LinkArch.Family == sys.Wasm { - // TODO(khr): figure out how to do "register" based calling convention for bounds checks. - // Should be similar to gcWriteBarrier, but I can't make it work. - s.rtcall(BoundsCheckFunc[kind], false, nil, idx, len) - } else { - mem := s.newValue3I(ssa.OpPanicBounds, types.TypeMem, int64(kind), idx, len, s.mem()) - s.endBlock().SetControl(mem) - } - s.startBlock(bNext) - - // In Spectre index mode, apply an appropriate mask to avoid speculative out-of-bounds accesses. - if base.Flag.Cfg.SpectreIndex { - op := ssa.OpSpectreIndex - if kind != ssa.BoundsIndex && kind != ssa.BoundsIndexU { - op = ssa.OpSpectreSliceIndex - } - idx = s.newValue2(op, types.Types[types.TINT], idx, len) - } - - return idx -} - -// If cmp (a bool) is false, panic using the given function. -func (s *state) check(cmp *ssa.Value, fn *obj.LSym) { - b := s.endBlock() - b.Kind = ssa.BlockIf - b.SetControl(cmp) - b.Likely = ssa.BranchLikely - bNext := s.f.NewBlock(ssa.BlockPlain) - line := s.peekPos() - pos := base.Ctxt.PosTable.Pos(line) - fl := funcLine{f: fn, base: pos.Base(), line: pos.Line()} - bPanic := s.panics[fl] - if bPanic == nil { - bPanic = s.f.NewBlock(ssa.BlockPlain) - s.panics[fl] = bPanic - s.startBlock(bPanic) - // The panic call takes/returns memory to ensure that the right - // memory state is observed if the panic happens. - s.rtcall(fn, false, nil) - } - b.AddEdgeTo(bNext) - b.AddEdgeTo(bPanic) - s.startBlock(bNext) -} - -func (s *state) intDivide(n ir.Node, a, b *ssa.Value) *ssa.Value { - needcheck := true - switch b.Op { - case ssa.OpConst8, ssa.OpConst16, ssa.OpConst32, ssa.OpConst64: - if b.AuxInt != 0 { - needcheck = false - } - } - if needcheck { - // do a size-appropriate check for zero - cmp := s.newValue2(s.ssaOp(ir.ONE, n.Type()), types.Types[types.TBOOL], b, s.zeroVal(n.Type())) - s.check(cmp, ir.Syms.Panicdivide) - } - return s.newValue2(s.ssaOp(n.Op(), n.Type()), a.Type, a, b) -} - -// rtcall issues a call to the given runtime function fn with the listed args. -// Returns a slice of results of the given result types. -// The call is added to the end of the current block. -// If returns is false, the block is marked as an exit block. -func (s *state) rtcall(fn *obj.LSym, returns bool, results []*types.Type, args ...*ssa.Value) []*ssa.Value { - s.prevCall = nil - // Write args to the stack - off := base.Ctxt.FixedFrameSize() - testLateExpansion := ssa.LateCallExpansionEnabledWithin(s.f) - var ACArgs []ssa.Param - var ACResults []ssa.Param - var callArgs []*ssa.Value - - for _, arg := range args { - t := arg.Type - off = types.Rnd(off, t.Alignment()) - size := t.Size() - ACArgs = append(ACArgs, ssa.Param{Type: t, Offset: int32(off)}) - if testLateExpansion { - callArgs = append(callArgs, arg) - } else { - ptr := s.constOffPtrSP(t.PtrTo(), off) - s.store(t, ptr, arg) - } - off += size - } - off = types.Rnd(off, int64(types.RegSize)) - - // Accumulate results types and offsets - offR := off - for _, t := range results { - offR = types.Rnd(offR, t.Alignment()) - ACResults = append(ACResults, ssa.Param{Type: t, Offset: int32(offR)}) - offR += t.Size() - } - - // Issue call - var call *ssa.Value - aux := ssa.StaticAuxCall(fn, ACArgs, ACResults) - if testLateExpansion { - callArgs = append(callArgs, s.mem()) - call = s.newValue0A(ssa.OpStaticLECall, aux.LateExpansionResultType(), aux) - call.AddArgs(callArgs...) - s.vars[memVar] = s.newValue1I(ssa.OpSelectN, types.TypeMem, int64(len(ACResults)), call) - } else { - call = s.newValue1A(ssa.OpStaticCall, types.TypeMem, aux, s.mem()) - s.vars[memVar] = call - } - - if !returns { - // Finish block - b := s.endBlock() - b.Kind = ssa.BlockExit - b.SetControl(call) - call.AuxInt = off - base.Ctxt.FixedFrameSize() - if len(results) > 0 { - s.Fatalf("panic call can't have results") - } - return nil - } - - // Load results - res := make([]*ssa.Value, len(results)) - if testLateExpansion { - for i, t := range results { - off = types.Rnd(off, t.Alignment()) - if canSSAType(t) { - res[i] = s.newValue1I(ssa.OpSelectN, t, int64(i), call) - } else { - addr := s.newValue1I(ssa.OpSelectNAddr, types.NewPtr(t), int64(i), call) - res[i] = s.rawLoad(t, addr) - } - off += t.Size() - } - } else { - for i, t := range results { - off = types.Rnd(off, t.Alignment()) - ptr := s.constOffPtrSP(types.NewPtr(t), off) - res[i] = s.load(t, ptr) - off += t.Size() - } - } - off = types.Rnd(off, int64(types.PtrSize)) - - // Remember how much callee stack space we needed. - call.AuxInt = off - - return res -} - -// do *left = right for type t. -func (s *state) storeType(t *types.Type, left, right *ssa.Value, skip skipMask, leftIsStmt bool) { - s.instrument(t, left, instrumentWrite) - - if skip == 0 && (!t.HasPointers() || ssa.IsStackAddr(left)) { - // Known to not have write barrier. Store the whole type. - s.vars[memVar] = s.newValue3Apos(ssa.OpStore, types.TypeMem, t, left, right, s.mem(), leftIsStmt) - return - } - - // store scalar fields first, so write barrier stores for - // pointer fields can be grouped together, and scalar values - // don't need to be live across the write barrier call. - // TODO: if the writebarrier pass knows how to reorder stores, - // we can do a single store here as long as skip==0. - s.storeTypeScalars(t, left, right, skip) - if skip&skipPtr == 0 && t.HasPointers() { - s.storeTypePtrs(t, left, right) - } -} - -// do *left = right for all scalar (non-pointer) parts of t. -func (s *state) storeTypeScalars(t *types.Type, left, right *ssa.Value, skip skipMask) { - switch { - case t.IsBoolean() || t.IsInteger() || t.IsFloat() || t.IsComplex(): - s.store(t, left, right) - case t.IsPtrShaped(): - if t.IsPtr() && t.Elem().NotInHeap() { - s.store(t, left, right) // see issue 42032 - } - // otherwise, no scalar fields. - case t.IsString(): - if skip&skipLen != 0 { - return - } - len := s.newValue1(ssa.OpStringLen, types.Types[types.TINT], right) - lenAddr := s.newValue1I(ssa.OpOffPtr, s.f.Config.Types.IntPtr, s.config.PtrSize, left) - s.store(types.Types[types.TINT], lenAddr, len) - case t.IsSlice(): - if skip&skipLen == 0 { - len := s.newValue1(ssa.OpSliceLen, types.Types[types.TINT], right) - lenAddr := s.newValue1I(ssa.OpOffPtr, s.f.Config.Types.IntPtr, s.config.PtrSize, left) - s.store(types.Types[types.TINT], lenAddr, len) - } - if skip&skipCap == 0 { - cap := s.newValue1(ssa.OpSliceCap, types.Types[types.TINT], right) - capAddr := s.newValue1I(ssa.OpOffPtr, s.f.Config.Types.IntPtr, 2*s.config.PtrSize, left) - s.store(types.Types[types.TINT], capAddr, cap) - } - case t.IsInterface(): - // itab field doesn't need a write barrier (even though it is a pointer). - itab := s.newValue1(ssa.OpITab, s.f.Config.Types.BytePtr, right) - s.store(types.Types[types.TUINTPTR], left, itab) - case t.IsStruct(): - n := t.NumFields() - for i := 0; i < n; i++ { - ft := t.FieldType(i) - addr := s.newValue1I(ssa.OpOffPtr, ft.PtrTo(), t.FieldOff(i), left) - val := s.newValue1I(ssa.OpStructSelect, ft, int64(i), right) - s.storeTypeScalars(ft, addr, val, 0) - } - case t.IsArray() && t.NumElem() == 0: - // nothing - case t.IsArray() && t.NumElem() == 1: - s.storeTypeScalars(t.Elem(), left, s.newValue1I(ssa.OpArraySelect, t.Elem(), 0, right), 0) - default: - s.Fatalf("bad write barrier type %v", t) - } -} - -// do *left = right for all pointer parts of t. -func (s *state) storeTypePtrs(t *types.Type, left, right *ssa.Value) { - switch { - case t.IsPtrShaped(): - if t.IsPtr() && t.Elem().NotInHeap() { - break // see issue 42032 - } - s.store(t, left, right) - case t.IsString(): - ptr := s.newValue1(ssa.OpStringPtr, s.f.Config.Types.BytePtr, right) - s.store(s.f.Config.Types.BytePtr, left, ptr) - case t.IsSlice(): - elType := types.NewPtr(t.Elem()) - ptr := s.newValue1(ssa.OpSlicePtr, elType, right) - s.store(elType, left, ptr) - case t.IsInterface(): - // itab field is treated as a scalar. - idata := s.newValue1(ssa.OpIData, s.f.Config.Types.BytePtr, right) - idataAddr := s.newValue1I(ssa.OpOffPtr, s.f.Config.Types.BytePtrPtr, s.config.PtrSize, left) - s.store(s.f.Config.Types.BytePtr, idataAddr, idata) - case t.IsStruct(): - n := t.NumFields() - for i := 0; i < n; i++ { - ft := t.FieldType(i) - if !ft.HasPointers() { - continue - } - addr := s.newValue1I(ssa.OpOffPtr, ft.PtrTo(), t.FieldOff(i), left) - val := s.newValue1I(ssa.OpStructSelect, ft, int64(i), right) - s.storeTypePtrs(ft, addr, val) - } - case t.IsArray() && t.NumElem() == 0: - // nothing - case t.IsArray() && t.NumElem() == 1: - s.storeTypePtrs(t.Elem(), left, s.newValue1I(ssa.OpArraySelect, t.Elem(), 0, right)) - default: - s.Fatalf("bad write barrier type %v", t) - } -} - -// putArg evaluates n for the purpose of passing it as an argument to a function and returns the corresponding Param for the call. -// If forLateExpandedCall is true, it returns the argument value to pass to the call operation. -// If forLateExpandedCall is false, then the value is stored at the specified stack offset, and the returned value is nil. -func (s *state) putArg(n ir.Node, t *types.Type, off int64, forLateExpandedCall bool) (ssa.Param, *ssa.Value) { - var a *ssa.Value - if forLateExpandedCall { - if !canSSAType(t) { - a = s.newValue2(ssa.OpDereference, t, s.addr(n), s.mem()) - } else { - a = s.expr(n) - } - } else { - s.storeArgWithBase(n, t, s.sp, off) - } - return ssa.Param{Type: t, Offset: int32(off)}, a -} - -func (s *state) storeArgWithBase(n ir.Node, t *types.Type, base *ssa.Value, off int64) { - pt := types.NewPtr(t) - var addr *ssa.Value - if base == s.sp { - // Use special routine that avoids allocation on duplicate offsets. - addr = s.constOffPtrSP(pt, off) - } else { - addr = s.newValue1I(ssa.OpOffPtr, pt, off, base) - } - - if !canSSAType(t) { - a := s.addr(n) - s.move(t, addr, a) - return - } - - a := s.expr(n) - s.storeType(t, addr, a, 0, false) -} - -// slice computes the slice v[i:j:k] and returns ptr, len, and cap of result. -// i,j,k may be nil, in which case they are set to their default value. -// v may be a slice, string or pointer to an array. -func (s *state) slice(v, i, j, k *ssa.Value, bounded bool) (p, l, c *ssa.Value) { - t := v.Type - var ptr, len, cap *ssa.Value - switch { - case t.IsSlice(): - ptr = s.newValue1(ssa.OpSlicePtr, types.NewPtr(t.Elem()), v) - len = s.newValue1(ssa.OpSliceLen, types.Types[types.TINT], v) - cap = s.newValue1(ssa.OpSliceCap, types.Types[types.TINT], v) - case t.IsString(): - ptr = s.newValue1(ssa.OpStringPtr, types.NewPtr(types.Types[types.TUINT8]), v) - len = s.newValue1(ssa.OpStringLen, types.Types[types.TINT], v) - cap = len - case t.IsPtr(): - if !t.Elem().IsArray() { - s.Fatalf("bad ptr to array in slice %v\n", t) - } - s.nilCheck(v) - ptr = s.newValue1(ssa.OpCopy, types.NewPtr(t.Elem().Elem()), v) - len = s.constInt(types.Types[types.TINT], t.Elem().NumElem()) - cap = len - default: - s.Fatalf("bad type in slice %v\n", t) - } - - // Set default values - if i == nil { - i = s.constInt(types.Types[types.TINT], 0) - } - if j == nil { - j = len - } - three := true - if k == nil { - three = false - k = cap - } - - // Panic if slice indices are not in bounds. - // Make sure we check these in reverse order so that we're always - // comparing against a value known to be nonnegative. See issue 28797. - if three { - if k != cap { - kind := ssa.BoundsSlice3Alen - if t.IsSlice() { - kind = ssa.BoundsSlice3Acap - } - k = s.boundsCheck(k, cap, kind, bounded) - } - if j != k { - j = s.boundsCheck(j, k, ssa.BoundsSlice3B, bounded) - } - i = s.boundsCheck(i, j, ssa.BoundsSlice3C, bounded) - } else { - if j != k { - kind := ssa.BoundsSliceAlen - if t.IsSlice() { - kind = ssa.BoundsSliceAcap - } - j = s.boundsCheck(j, k, kind, bounded) - } - i = s.boundsCheck(i, j, ssa.BoundsSliceB, bounded) - } - - // Word-sized integer operations. - subOp := s.ssaOp(ir.OSUB, types.Types[types.TINT]) - mulOp := s.ssaOp(ir.OMUL, types.Types[types.TINT]) - andOp := s.ssaOp(ir.OAND, types.Types[types.TINT]) - - // Calculate the length (rlen) and capacity (rcap) of the new slice. - // For strings the capacity of the result is unimportant. However, - // we use rcap to test if we've generated a zero-length slice. - // Use length of strings for that. - rlen := s.newValue2(subOp, types.Types[types.TINT], j, i) - rcap := rlen - if j != k && !t.IsString() { - rcap = s.newValue2(subOp, types.Types[types.TINT], k, i) - } - - if (i.Op == ssa.OpConst64 || i.Op == ssa.OpConst32) && i.AuxInt == 0 { - // No pointer arithmetic necessary. - return ptr, rlen, rcap - } - - // Calculate the base pointer (rptr) for the new slice. - // - // Generate the following code assuming that indexes are in bounds. - // The masking is to make sure that we don't generate a slice - // that points to the next object in memory. We cannot just set - // the pointer to nil because then we would create a nil slice or - // string. - // - // rcap = k - i - // rlen = j - i - // rptr = ptr + (mask(rcap) & (i * stride)) - // - // Where mask(x) is 0 if x==0 and -1 if x>0 and stride is the width - // of the element type. - stride := s.constInt(types.Types[types.TINT], ptr.Type.Elem().Width) - - // The delta is the number of bytes to offset ptr by. - delta := s.newValue2(mulOp, types.Types[types.TINT], i, stride) - - // If we're slicing to the point where the capacity is zero, - // zero out the delta. - mask := s.newValue1(ssa.OpSlicemask, types.Types[types.TINT], rcap) - delta = s.newValue2(andOp, types.Types[types.TINT], delta, mask) - - // Compute rptr = ptr + delta. - rptr := s.newValue2(ssa.OpAddPtr, ptr.Type, ptr, delta) - - return rptr, rlen, rcap -} - -type u642fcvtTab struct { - leq, cvt2F, and, rsh, or, add ssa.Op - one func(*state, *types.Type, int64) *ssa.Value -} - -var u64_f64 = u642fcvtTab{ - leq: ssa.OpLeq64, - cvt2F: ssa.OpCvt64to64F, - and: ssa.OpAnd64, - rsh: ssa.OpRsh64Ux64, - or: ssa.OpOr64, - add: ssa.OpAdd64F, - one: (*state).constInt64, -} - -var u64_f32 = u642fcvtTab{ - leq: ssa.OpLeq64, - cvt2F: ssa.OpCvt64to32F, - and: ssa.OpAnd64, - rsh: ssa.OpRsh64Ux64, - or: ssa.OpOr64, - add: ssa.OpAdd32F, - one: (*state).constInt64, -} - -func (s *state) uint64Tofloat64(n ir.Node, x *ssa.Value, ft, tt *types.Type) *ssa.Value { - return s.uint64Tofloat(&u64_f64, n, x, ft, tt) -} - -func (s *state) uint64Tofloat32(n ir.Node, x *ssa.Value, ft, tt *types.Type) *ssa.Value { - return s.uint64Tofloat(&u64_f32, n, x, ft, tt) -} - -func (s *state) uint64Tofloat(cvttab *u642fcvtTab, n ir.Node, x *ssa.Value, ft, tt *types.Type) *ssa.Value { - // if x >= 0 { - // result = (floatY) x - // } else { - // y = uintX(x) ; y = x & 1 - // z = uintX(x) ; z = z >> 1 - // z = z >> 1 - // z = z | y - // result = floatY(z) - // result = result + result - // } - // - // Code borrowed from old code generator. - // What's going on: large 64-bit "unsigned" looks like - // negative number to hardware's integer-to-float - // conversion. However, because the mantissa is only - // 63 bits, we don't need the LSB, so instead we do an - // unsigned right shift (divide by two), convert, and - // double. However, before we do that, we need to be - // sure that we do not lose a "1" if that made the - // difference in the resulting rounding. Therefore, we - // preserve it, and OR (not ADD) it back in. The case - // that matters is when the eleven discarded bits are - // equal to 10000000001; that rounds up, and the 1 cannot - // be lost else it would round down if the LSB of the - // candidate mantissa is 0. - cmp := s.newValue2(cvttab.leq, types.Types[types.TBOOL], s.zeroVal(ft), x) - b := s.endBlock() - b.Kind = ssa.BlockIf - b.SetControl(cmp) - b.Likely = ssa.BranchLikely - - bThen := s.f.NewBlock(ssa.BlockPlain) - bElse := s.f.NewBlock(ssa.BlockPlain) - bAfter := s.f.NewBlock(ssa.BlockPlain) - - b.AddEdgeTo(bThen) - s.startBlock(bThen) - a0 := s.newValue1(cvttab.cvt2F, tt, x) - s.vars[n] = a0 - s.endBlock() - bThen.AddEdgeTo(bAfter) - - b.AddEdgeTo(bElse) - s.startBlock(bElse) - one := cvttab.one(s, ft, 1) - y := s.newValue2(cvttab.and, ft, x, one) - z := s.newValue2(cvttab.rsh, ft, x, one) - z = s.newValue2(cvttab.or, ft, z, y) - a := s.newValue1(cvttab.cvt2F, tt, z) - a1 := s.newValue2(cvttab.add, tt, a, a) - s.vars[n] = a1 - s.endBlock() - bElse.AddEdgeTo(bAfter) - - s.startBlock(bAfter) - return s.variable(n, n.Type()) -} - -type u322fcvtTab struct { - cvtI2F, cvtF2F ssa.Op -} - -var u32_f64 = u322fcvtTab{ - cvtI2F: ssa.OpCvt32to64F, - cvtF2F: ssa.OpCopy, -} - -var u32_f32 = u322fcvtTab{ - cvtI2F: ssa.OpCvt32to32F, - cvtF2F: ssa.OpCvt64Fto32F, -} - -func (s *state) uint32Tofloat64(n ir.Node, x *ssa.Value, ft, tt *types.Type) *ssa.Value { - return s.uint32Tofloat(&u32_f64, n, x, ft, tt) -} - -func (s *state) uint32Tofloat32(n ir.Node, x *ssa.Value, ft, tt *types.Type) *ssa.Value { - return s.uint32Tofloat(&u32_f32, n, x, ft, tt) -} - -func (s *state) uint32Tofloat(cvttab *u322fcvtTab, n ir.Node, x *ssa.Value, ft, tt *types.Type) *ssa.Value { - // if x >= 0 { - // result = floatY(x) - // } else { - // result = floatY(float64(x) + (1<<32)) - // } - cmp := s.newValue2(ssa.OpLeq32, types.Types[types.TBOOL], s.zeroVal(ft), x) - b := s.endBlock() - b.Kind = ssa.BlockIf - b.SetControl(cmp) - b.Likely = ssa.BranchLikely - - bThen := s.f.NewBlock(ssa.BlockPlain) - bElse := s.f.NewBlock(ssa.BlockPlain) - bAfter := s.f.NewBlock(ssa.BlockPlain) - - b.AddEdgeTo(bThen) - s.startBlock(bThen) - a0 := s.newValue1(cvttab.cvtI2F, tt, x) - s.vars[n] = a0 - s.endBlock() - bThen.AddEdgeTo(bAfter) - - b.AddEdgeTo(bElse) - s.startBlock(bElse) - a1 := s.newValue1(ssa.OpCvt32to64F, types.Types[types.TFLOAT64], x) - twoToThe32 := s.constFloat64(types.Types[types.TFLOAT64], float64(1<<32)) - a2 := s.newValue2(ssa.OpAdd64F, types.Types[types.TFLOAT64], a1, twoToThe32) - a3 := s.newValue1(cvttab.cvtF2F, tt, a2) - - s.vars[n] = a3 - s.endBlock() - bElse.AddEdgeTo(bAfter) - - s.startBlock(bAfter) - return s.variable(n, n.Type()) -} - -// referenceTypeBuiltin generates code for the len/cap builtins for maps and channels. -func (s *state) referenceTypeBuiltin(n *ir.UnaryExpr, x *ssa.Value) *ssa.Value { - if !n.X.Type().IsMap() && !n.X.Type().IsChan() { - s.Fatalf("node must be a map or a channel") - } - // if n == nil { - // return 0 - // } else { - // // len - // return *((*int)n) - // // cap - // return *(((*int)n)+1) - // } - lenType := n.Type() - nilValue := s.constNil(types.Types[types.TUINTPTR]) - cmp := s.newValue2(ssa.OpEqPtr, types.Types[types.TBOOL], x, nilValue) - b := s.endBlock() - b.Kind = ssa.BlockIf - b.SetControl(cmp) - b.Likely = ssa.BranchUnlikely - - bThen := s.f.NewBlock(ssa.BlockPlain) - bElse := s.f.NewBlock(ssa.BlockPlain) - bAfter := s.f.NewBlock(ssa.BlockPlain) - - // length/capacity of a nil map/chan is zero - b.AddEdgeTo(bThen) - s.startBlock(bThen) - s.vars[n] = s.zeroVal(lenType) - s.endBlock() - bThen.AddEdgeTo(bAfter) - - b.AddEdgeTo(bElse) - s.startBlock(bElse) - switch n.Op() { - case ir.OLEN: - // length is stored in the first word for map/chan - s.vars[n] = s.load(lenType, x) - case ir.OCAP: - // capacity is stored in the second word for chan - sw := s.newValue1I(ssa.OpOffPtr, lenType.PtrTo(), lenType.Width, x) - s.vars[n] = s.load(lenType, sw) - default: - s.Fatalf("op must be OLEN or OCAP") - } - s.endBlock() - bElse.AddEdgeTo(bAfter) - - s.startBlock(bAfter) - return s.variable(n, lenType) -} - -type f2uCvtTab struct { - ltf, cvt2U, subf, or ssa.Op - floatValue func(*state, *types.Type, float64) *ssa.Value - intValue func(*state, *types.Type, int64) *ssa.Value - cutoff uint64 -} - -var f32_u64 = f2uCvtTab{ - ltf: ssa.OpLess32F, - cvt2U: ssa.OpCvt32Fto64, - subf: ssa.OpSub32F, - or: ssa.OpOr64, - floatValue: (*state).constFloat32, - intValue: (*state).constInt64, - cutoff: 1 << 63, -} - -var f64_u64 = f2uCvtTab{ - ltf: ssa.OpLess64F, - cvt2U: ssa.OpCvt64Fto64, - subf: ssa.OpSub64F, - or: ssa.OpOr64, - floatValue: (*state).constFloat64, - intValue: (*state).constInt64, - cutoff: 1 << 63, -} - -var f32_u32 = f2uCvtTab{ - ltf: ssa.OpLess32F, - cvt2U: ssa.OpCvt32Fto32, - subf: ssa.OpSub32F, - or: ssa.OpOr32, - floatValue: (*state).constFloat32, - intValue: func(s *state, t *types.Type, v int64) *ssa.Value { return s.constInt32(t, int32(v)) }, - cutoff: 1 << 31, -} - -var f64_u32 = f2uCvtTab{ - ltf: ssa.OpLess64F, - cvt2U: ssa.OpCvt64Fto32, - subf: ssa.OpSub64F, - or: ssa.OpOr32, - floatValue: (*state).constFloat64, - intValue: func(s *state, t *types.Type, v int64) *ssa.Value { return s.constInt32(t, int32(v)) }, - cutoff: 1 << 31, -} - -func (s *state) float32ToUint64(n ir.Node, x *ssa.Value, ft, tt *types.Type) *ssa.Value { - return s.floatToUint(&f32_u64, n, x, ft, tt) -} -func (s *state) float64ToUint64(n ir.Node, x *ssa.Value, ft, tt *types.Type) *ssa.Value { - return s.floatToUint(&f64_u64, n, x, ft, tt) -} - -func (s *state) float32ToUint32(n ir.Node, x *ssa.Value, ft, tt *types.Type) *ssa.Value { - return s.floatToUint(&f32_u32, n, x, ft, tt) -} - -func (s *state) float64ToUint32(n ir.Node, x *ssa.Value, ft, tt *types.Type) *ssa.Value { - return s.floatToUint(&f64_u32, n, x, ft, tt) -} - -func (s *state) floatToUint(cvttab *f2uCvtTab, n ir.Node, x *ssa.Value, ft, tt *types.Type) *ssa.Value { - // cutoff:=1<<(intY_Size-1) - // if x < floatX(cutoff) { - // result = uintY(x) - // } else { - // y = x - floatX(cutoff) - // z = uintY(y) - // result = z | -(cutoff) - // } - cutoff := cvttab.floatValue(s, ft, float64(cvttab.cutoff)) - cmp := s.newValue2(cvttab.ltf, types.Types[types.TBOOL], x, cutoff) - b := s.endBlock() - b.Kind = ssa.BlockIf - b.SetControl(cmp) - b.Likely = ssa.BranchLikely - - bThen := s.f.NewBlock(ssa.BlockPlain) - bElse := s.f.NewBlock(ssa.BlockPlain) - bAfter := s.f.NewBlock(ssa.BlockPlain) - - b.AddEdgeTo(bThen) - s.startBlock(bThen) - a0 := s.newValue1(cvttab.cvt2U, tt, x) - s.vars[n] = a0 - s.endBlock() - bThen.AddEdgeTo(bAfter) - - b.AddEdgeTo(bElse) - s.startBlock(bElse) - y := s.newValue2(cvttab.subf, ft, x, cutoff) - y = s.newValue1(cvttab.cvt2U, tt, y) - z := cvttab.intValue(s, tt, int64(-cvttab.cutoff)) - a1 := s.newValue2(cvttab.or, tt, y, z) - s.vars[n] = a1 - s.endBlock() - bElse.AddEdgeTo(bAfter) - - s.startBlock(bAfter) - return s.variable(n, n.Type()) -} - -// dottype generates SSA for a type assertion node. -// commaok indicates whether to panic or return a bool. -// If commaok is false, resok will be nil. -func (s *state) dottype(n *ir.TypeAssertExpr, commaok bool) (res, resok *ssa.Value) { - iface := s.expr(n.X) // input interface - target := s.expr(n.Ntype) // target type - byteptr := s.f.Config.Types.BytePtr - - if n.Type().IsInterface() { - if n.Type().IsEmptyInterface() { - // Converting to an empty interface. - // Input could be an empty or nonempty interface. - if base.Debug.TypeAssert > 0 { - base.WarnfAt(n.Pos(), "type assertion inlined") - } - - // Get itab/type field from input. - itab := s.newValue1(ssa.OpITab, byteptr, iface) - // Conversion succeeds iff that field is not nil. - cond := s.newValue2(ssa.OpNeqPtr, types.Types[types.TBOOL], itab, s.constNil(byteptr)) - - if n.X.Type().IsEmptyInterface() && commaok { - // Converting empty interface to empty interface with ,ok is just a nil check. - return iface, cond - } - - // Branch on nilness. - b := s.endBlock() - b.Kind = ssa.BlockIf - b.SetControl(cond) - b.Likely = ssa.BranchLikely - bOk := s.f.NewBlock(ssa.BlockPlain) - bFail := s.f.NewBlock(ssa.BlockPlain) - b.AddEdgeTo(bOk) - b.AddEdgeTo(bFail) - - if !commaok { - // On failure, panic by calling panicnildottype. - s.startBlock(bFail) - s.rtcall(ir.Syms.Panicnildottype, false, nil, target) - - // On success, return (perhaps modified) input interface. - s.startBlock(bOk) - if n.X.Type().IsEmptyInterface() { - res = iface // Use input interface unchanged. - return - } - // Load type out of itab, build interface with existing idata. - off := s.newValue1I(ssa.OpOffPtr, byteptr, int64(types.PtrSize), itab) - typ := s.load(byteptr, off) - idata := s.newValue1(ssa.OpIData, byteptr, iface) - res = s.newValue2(ssa.OpIMake, n.Type(), typ, idata) - return - } - - s.startBlock(bOk) - // nonempty -> empty - // Need to load type from itab - off := s.newValue1I(ssa.OpOffPtr, byteptr, int64(types.PtrSize), itab) - s.vars[typVar] = s.load(byteptr, off) - s.endBlock() - - // itab is nil, might as well use that as the nil result. - s.startBlock(bFail) - s.vars[typVar] = itab - s.endBlock() - - // Merge point. - bEnd := s.f.NewBlock(ssa.BlockPlain) - bOk.AddEdgeTo(bEnd) - bFail.AddEdgeTo(bEnd) - s.startBlock(bEnd) - idata := s.newValue1(ssa.OpIData, byteptr, iface) - res = s.newValue2(ssa.OpIMake, n.Type(), s.variable(typVar, byteptr), idata) - resok = cond - delete(s.vars, typVar) - return - } - // converting to a nonempty interface needs a runtime call. - if base.Debug.TypeAssert > 0 { - base.WarnfAt(n.Pos(), "type assertion not inlined") - } - if n.X.Type().IsEmptyInterface() { - if commaok { - call := s.rtcall(ir.Syms.AssertE2I2, true, []*types.Type{n.Type(), types.Types[types.TBOOL]}, target, iface) - return call[0], call[1] - } - return s.rtcall(ir.Syms.AssertE2I, true, []*types.Type{n.Type()}, target, iface)[0], nil - } - if commaok { - call := s.rtcall(ir.Syms.AssertI2I2, true, []*types.Type{n.Type(), types.Types[types.TBOOL]}, target, iface) - return call[0], call[1] - } - return s.rtcall(ir.Syms.AssertI2I, true, []*types.Type{n.Type()}, target, iface)[0], nil - } - - if base.Debug.TypeAssert > 0 { - base.WarnfAt(n.Pos(), "type assertion inlined") - } - - // Converting to a concrete type. - direct := types.IsDirectIface(n.Type()) - itab := s.newValue1(ssa.OpITab, byteptr, iface) // type word of interface - if base.Debug.TypeAssert > 0 { - base.WarnfAt(n.Pos(), "type assertion inlined") - } - var targetITab *ssa.Value - if n.X.Type().IsEmptyInterface() { - // Looking for pointer to target type. - targetITab = target - } else { - // Looking for pointer to itab for target type and source interface. - targetITab = s.expr(n.Itab[0]) - } - - var tmp ir.Node // temporary for use with large types - var addr *ssa.Value // address of tmp - if commaok && !canSSAType(n.Type()) { - // unSSAable type, use temporary. - // TODO: get rid of some of these temporaries. - tmp = typecheck.TempAt(n.Pos(), s.curfn, n.Type()) - s.vars[memVar] = s.newValue1A(ssa.OpVarDef, types.TypeMem, tmp.(*ir.Name), s.mem()) - addr = s.addr(tmp) - } - - cond := s.newValue2(ssa.OpEqPtr, types.Types[types.TBOOL], itab, targetITab) - b := s.endBlock() - b.Kind = ssa.BlockIf - b.SetControl(cond) - b.Likely = ssa.BranchLikely - - bOk := s.f.NewBlock(ssa.BlockPlain) - bFail := s.f.NewBlock(ssa.BlockPlain) - b.AddEdgeTo(bOk) - b.AddEdgeTo(bFail) - - if !commaok { - // on failure, panic by calling panicdottype - s.startBlock(bFail) - taddr := s.expr(n.Ntype.(*ir.AddrExpr).Alloc) - if n.X.Type().IsEmptyInterface() { - s.rtcall(ir.Syms.PanicdottypeE, false, nil, itab, target, taddr) - } else { - s.rtcall(ir.Syms.PanicdottypeI, false, nil, itab, target, taddr) - } - - // on success, return data from interface - s.startBlock(bOk) - if direct { - return s.newValue1(ssa.OpIData, n.Type(), iface), nil - } - p := s.newValue1(ssa.OpIData, types.NewPtr(n.Type()), iface) - return s.load(n.Type(), p), nil - } - - // commaok is the more complicated case because we have - // a control flow merge point. - bEnd := s.f.NewBlock(ssa.BlockPlain) - // Note that we need a new valVar each time (unlike okVar where we can - // reuse the variable) because it might have a different type every time. - valVar := ssaMarker("val") - - // type assertion succeeded - s.startBlock(bOk) - if tmp == nil { - if direct { - s.vars[valVar] = s.newValue1(ssa.OpIData, n.Type(), iface) - } else { - p := s.newValue1(ssa.OpIData, types.NewPtr(n.Type()), iface) - s.vars[valVar] = s.load(n.Type(), p) - } - } else { - p := s.newValue1(ssa.OpIData, types.NewPtr(n.Type()), iface) - s.move(n.Type(), addr, p) - } - s.vars[okVar] = s.constBool(true) - s.endBlock() - bOk.AddEdgeTo(bEnd) - - // type assertion failed - s.startBlock(bFail) - if tmp == nil { - s.vars[valVar] = s.zeroVal(n.Type()) - } else { - s.zero(n.Type(), addr) - } - s.vars[okVar] = s.constBool(false) - s.endBlock() - bFail.AddEdgeTo(bEnd) - - // merge point - s.startBlock(bEnd) - if tmp == nil { - res = s.variable(valVar, n.Type()) - delete(s.vars, valVar) - } else { - res = s.load(n.Type(), addr) - s.vars[memVar] = s.newValue1A(ssa.OpVarKill, types.TypeMem, tmp.(*ir.Name), s.mem()) - } - resok = s.variable(okVar, types.Types[types.TBOOL]) - delete(s.vars, okVar) - return res, resok -} - -// variable returns the value of a variable at the current location. -func (s *state) variable(n ir.Node, t *types.Type) *ssa.Value { - v := s.vars[n] - if v != nil { - return v - } - v = s.fwdVars[n] - if v != nil { - return v - } - - if s.curBlock == s.f.Entry { - // No variable should be live at entry. - s.Fatalf("Value live at entry. It shouldn't be. func %s, node %v, value %v", s.f.Name, n, v) - } - // Make a FwdRef, which records a value that's live on block input. - // We'll find the matching definition as part of insertPhis. - v = s.newValue0A(ssa.OpFwdRef, t, FwdRefAux{N: n}) - s.fwdVars[n] = v - if n.Op() == ir.ONAME { - s.addNamedValue(n.(*ir.Name), v) - } - return v -} - -func (s *state) mem() *ssa.Value { - return s.variable(memVar, types.TypeMem) -} - -func (s *state) addNamedValue(n *ir.Name, v *ssa.Value) { - if n.Class_ == ir.Pxxx { - // Don't track our marker nodes (memVar etc.). - return - } - if ir.IsAutoTmp(n) { - // Don't track temporary variables. - return - } - if n.Class_ == ir.PPARAMOUT { - // Don't track named output values. This prevents return values - // from being assigned too early. See #14591 and #14762. TODO: allow this. - return - } - loc := ssa.LocalSlot{N: n.Name(), Type: n.Type(), Off: 0} - values, ok := s.f.NamedValues[loc] - if !ok { - s.f.Names = append(s.f.Names, loc) - } - s.f.NamedValues[loc] = append(values, v) -} - -// Generate a disconnected call to a runtime routine and a return. -func gencallret(pp *objw.Progs, sym *obj.LSym) *obj.Prog { - p := pp.Prog(obj.ACALL) - p.To.Type = obj.TYPE_MEM - p.To.Name = obj.NAME_EXTERN - p.To.Sym = sym - p = pp.Prog(obj.ARET) - return p -} - -// Branch is an unresolved branch. -type Branch struct { - P *obj.Prog // branch instruction - B *ssa.Block // target -} - -// SSAGenState contains state needed during Prog generation. -type SSAGenState struct { - pp *objw.Progs - - // Branches remembers all the branch instructions we've seen - // and where they would like to go. - Branches []Branch - - // bstart remembers where each block starts (indexed by block ID) - bstart []*obj.Prog - - // Some architectures require a 64-bit temporary for FP-related register shuffling. Examples include PPC and Sparc V8. - ScratchFpMem *ir.Name - - maxarg int64 // largest frame size for arguments to calls made by the function - - // Map from GC safe points to liveness index, generated by - // liveness analysis. - livenessMap liveness.Map - - // lineRunStart records the beginning of the current run of instructions - // within a single block sharing the same line number - // Used to move statement marks to the beginning of such runs. - lineRunStart *obj.Prog - - // wasm: The number of values on the WebAssembly stack. This is only used as a safeguard. - OnWasmStackSkipped int -} - -// Prog appends a new Prog. -func (s *SSAGenState) Prog(as obj.As) *obj.Prog { - p := s.pp.Prog(as) - if ssa.LosesStmtMark(as) { - return p - } - // Float a statement start to the beginning of any same-line run. - // lineRunStart is reset at block boundaries, which appears to work well. - if s.lineRunStart == nil || s.lineRunStart.Pos.Line() != p.Pos.Line() { - s.lineRunStart = p - } else if p.Pos.IsStmt() == src.PosIsStmt { - s.lineRunStart.Pos = s.lineRunStart.Pos.WithIsStmt() - p.Pos = p.Pos.WithNotStmt() - } - return p -} - -// Pc returns the current Prog. -func (s *SSAGenState) Pc() *obj.Prog { - return s.pp.Next -} - -// SetPos sets the current source position. -func (s *SSAGenState) SetPos(pos src.XPos) { - s.pp.Pos = pos -} - -// Br emits a single branch instruction and returns the instruction. -// Not all architectures need the returned instruction, but otherwise -// the boilerplate is common to all. -func (s *SSAGenState) Br(op obj.As, target *ssa.Block) *obj.Prog { - p := s.Prog(op) - p.To.Type = obj.TYPE_BRANCH - s.Branches = append(s.Branches, Branch{P: p, B: target}) - return p -} - -// DebugFriendlySetPosFrom adjusts Pos.IsStmt subject to heuristics -// that reduce "jumpy" line number churn when debugging. -// Spill/fill/copy instructions from the register allocator, -// phi functions, and instructions with a no-pos position -// are examples of instructions that can cause churn. -func (s *SSAGenState) DebugFriendlySetPosFrom(v *ssa.Value) { - switch v.Op { - case ssa.OpPhi, ssa.OpCopy, ssa.OpLoadReg, ssa.OpStoreReg: - // These are not statements - s.SetPos(v.Pos.WithNotStmt()) - default: - p := v.Pos - if p != src.NoXPos { - // If the position is defined, update the position. - // Also convert default IsStmt to NotStmt; only - // explicit statement boundaries should appear - // in the generated code. - if p.IsStmt() != src.PosIsStmt { - p = p.WithNotStmt() - // Calls use the pos attached to v, but copy the statement mark from SSAGenState - } - s.SetPos(p) - } else { - s.SetPos(s.pp.Pos.WithNotStmt()) - } - } -} - -// byXoffset implements sort.Interface for []*ir.Name using Xoffset as the ordering. -type byXoffset []*ir.Name - -func (s byXoffset) Len() int { return len(s) } -func (s byXoffset) Less(i, j int) bool { return s[i].FrameOffset() < s[j].FrameOffset() } -func (s byXoffset) Swap(i, j int) { s[i], s[j] = s[j], s[i] } - -func emitStackObjects(e *ssafn, pp *objw.Progs) { - var vars []*ir.Name - for _, n := range e.curfn.Dcl { - if liveness.ShouldTrack(n) && n.Addrtaken() { - vars = append(vars, n) - } - } - if len(vars) == 0 { - return - } - - // Sort variables from lowest to highest address. - sort.Sort(byXoffset(vars)) - - // Populate the stack object data. - // Format must match runtime/stack.go:stackObjectRecord. - x := e.curfn.LSym.Func().StackObjects - off := 0 - off = objw.Uintptr(x, off, uint64(len(vars))) - for _, v := range vars { - // Note: arguments and return values have non-negative Xoffset, - // in which case the offset is relative to argp. - // Locals have a negative Xoffset, in which case the offset is relative to varp. - off = objw.Uintptr(x, off, uint64(v.FrameOffset())) - if !types.TypeSym(v.Type()).Siggen() { - e.Fatalf(v.Pos(), "stack object's type symbol not generated for type %s", v.Type()) - } - off = objw.SymPtr(x, off, reflectdata.WriteType(v.Type()), 0) - } - - // Emit a funcdata pointing at the stack object data. - p := pp.Prog(obj.AFUNCDATA) - p.From.SetConst(objabi.FUNCDATA_StackObjects) - p.To.Type = obj.TYPE_MEM - p.To.Name = obj.NAME_EXTERN - p.To.Sym = x - - if base.Flag.Live != 0 { - for _, v := range vars { - base.WarnfAt(v.Pos(), "stack object %v %s", v, v.Type().String()) - } - } -} - -// genssa appends entries to pp for each instruction in f. -func genssa(f *ssa.Func, pp *objw.Progs) { - var s SSAGenState - - e := f.Frontend().(*ssafn) - - s.livenessMap = liveness.Compute(e.curfn, f, e.stkptrsize, pp) - emitStackObjects(e, pp) - - openDeferInfo := e.curfn.LSym.Func().OpenCodedDeferInfo - if openDeferInfo != nil { - // This function uses open-coded defers -- write out the funcdata - // info that we computed at the end of genssa. - p := pp.Prog(obj.AFUNCDATA) - p.From.SetConst(objabi.FUNCDATA_OpenCodedDeferInfo) - p.To.Type = obj.TYPE_MEM - p.To.Name = obj.NAME_EXTERN - p.To.Sym = openDeferInfo - } - - // Remember where each block starts. - s.bstart = make([]*obj.Prog, f.NumBlocks()) - s.pp = pp - var progToValue map[*obj.Prog]*ssa.Value - var progToBlock map[*obj.Prog]*ssa.Block - var valueToProgAfter []*obj.Prog // The first Prog following computation of a value v; v is visible at this point. - if f.PrintOrHtmlSSA { - progToValue = make(map[*obj.Prog]*ssa.Value, f.NumValues()) - progToBlock = make(map[*obj.Prog]*ssa.Block, f.NumBlocks()) - f.Logf("genssa %s\n", f.Name) - progToBlock[s.pp.Next] = f.Blocks[0] - } - - s.ScratchFpMem = e.scratchFpMem - - if base.Ctxt.Flag_locationlists { - if cap(f.Cache.ValueToProgAfter) < f.NumValues() { - f.Cache.ValueToProgAfter = make([]*obj.Prog, f.NumValues()) - } - valueToProgAfter = f.Cache.ValueToProgAfter[:f.NumValues()] - for i := range valueToProgAfter { - valueToProgAfter[i] = nil - } - } - - // If the very first instruction is not tagged as a statement, - // debuggers may attribute it to previous function in program. - firstPos := src.NoXPos - for _, v := range f.Entry.Values { - if v.Pos.IsStmt() == src.PosIsStmt { - firstPos = v.Pos - v.Pos = firstPos.WithDefaultStmt() - break - } - } - - // inlMarks has an entry for each Prog that implements an inline mark. - // It maps from that Prog to the global inlining id of the inlined body - // which should unwind to this Prog's location. - var inlMarks map[*obj.Prog]int32 - var inlMarkList []*obj.Prog - - // inlMarksByPos maps from a (column 1) source position to the set of - // Progs that are in the set above and have that source position. - var inlMarksByPos map[src.XPos][]*obj.Prog - - // Emit basic blocks - for i, b := range f.Blocks { - s.bstart[b.ID] = s.pp.Next - s.lineRunStart = nil - - // Attach a "default" liveness info. Normally this will be - // overwritten in the Values loop below for each Value. But - // for an empty block this will be used for its control - // instruction. We won't use the actual liveness map on a - // control instruction. Just mark it something that is - // preemptible, unless this function is "all unsafe". - s.pp.NextLive = objw.LivenessIndex{StackMapIndex: -1, IsUnsafePoint: liveness.IsUnsafe(f)} - - // Emit values in block - thearch.SSAMarkMoves(&s, b) - for _, v := range b.Values { - x := s.pp.Next - s.DebugFriendlySetPosFrom(v) - - switch v.Op { - case ssa.OpInitMem: - // memory arg needs no code - case ssa.OpArg: - // input args need no code - case ssa.OpSP, ssa.OpSB: - // nothing to do - case ssa.OpSelect0, ssa.OpSelect1: - // nothing to do - case ssa.OpGetG: - // nothing to do when there's a g register, - // and checkLower complains if there's not - case ssa.OpVarDef, ssa.OpVarLive, ssa.OpKeepAlive, ssa.OpVarKill: - // nothing to do; already used by liveness - case ssa.OpPhi: - CheckLoweredPhi(v) - case ssa.OpConvert: - // nothing to do; no-op conversion for liveness - if v.Args[0].Reg() != v.Reg() { - v.Fatalf("OpConvert should be a no-op: %s; %s", v.Args[0].LongString(), v.LongString()) - } - case ssa.OpInlMark: - p := thearch.Ginsnop(s.pp) - if inlMarks == nil { - inlMarks = map[*obj.Prog]int32{} - inlMarksByPos = map[src.XPos][]*obj.Prog{} - } - inlMarks[p] = v.AuxInt32() - inlMarkList = append(inlMarkList, p) - pos := v.Pos.AtColumn1() - inlMarksByPos[pos] = append(inlMarksByPos[pos], p) - - default: - // Attach this safe point to the next - // instruction. - s.pp.NextLive = s.livenessMap.Get(v) - - // Special case for first line in function; move it to the start. - if firstPos != src.NoXPos { - s.SetPos(firstPos) - firstPos = src.NoXPos - } - // let the backend handle it - thearch.SSAGenValue(&s, v) - } - - if base.Ctxt.Flag_locationlists { - valueToProgAfter[v.ID] = s.pp.Next - } - - if f.PrintOrHtmlSSA { - for ; x != s.pp.Next; x = x.Link { - progToValue[x] = v - } - } - } - // If this is an empty infinite loop, stick a hardware NOP in there so that debuggers are less confused. - if s.bstart[b.ID] == s.pp.Next && len(b.Succs) == 1 && b.Succs[0].Block() == b { - p := thearch.Ginsnop(s.pp) - p.Pos = p.Pos.WithIsStmt() - if b.Pos == src.NoXPos { - b.Pos = p.Pos // It needs a file, otherwise a no-file non-zero line causes confusion. See #35652. - if b.Pos == src.NoXPos { - b.Pos = pp.Text.Pos // Sometimes p.Pos is empty. See #35695. - } - } - b.Pos = b.Pos.WithBogusLine() // Debuggers are not good about infinite loops, force a change in line number - } - // Emit control flow instructions for block - var next *ssa.Block - if i < len(f.Blocks)-1 && base.Flag.N == 0 { - // If -N, leave next==nil so every block with successors - // ends in a JMP (except call blocks - plive doesn't like - // select{send,recv} followed by a JMP call). Helps keep - // line numbers for otherwise empty blocks. - next = f.Blocks[i+1] - } - x := s.pp.Next - s.SetPos(b.Pos) - thearch.SSAGenBlock(&s, b, next) - if f.PrintOrHtmlSSA { - for ; x != s.pp.Next; x = x.Link { - progToBlock[x] = b - } - } - } - if f.Blocks[len(f.Blocks)-1].Kind == ssa.BlockExit { - // We need the return address of a panic call to - // still be inside the function in question. So if - // it ends in a call which doesn't return, add a - // nop (which will never execute) after the call. - thearch.Ginsnop(pp) - } - if openDeferInfo != nil { - // When doing open-coded defers, generate a disconnected call to - // deferreturn and a return. This will be used to during panic - // recovery to unwind the stack and return back to the runtime. - s.pp.NextLive = s.livenessMap.DeferReturn - gencallret(pp, ir.Syms.Deferreturn) - } - - if inlMarks != nil { - // We have some inline marks. Try to find other instructions we're - // going to emit anyway, and use those instructions instead of the - // inline marks. - for p := pp.Text; p != nil; p = p.Link { - if p.As == obj.ANOP || p.As == obj.AFUNCDATA || p.As == obj.APCDATA || p.As == obj.ATEXT || p.As == obj.APCALIGN || thearch.LinkArch.Family == sys.Wasm { - // Don't use 0-sized instructions as inline marks, because we need - // to identify inline mark instructions by pc offset. - // (Some of these instructions are sometimes zero-sized, sometimes not. - // We must not use anything that even might be zero-sized.) - // TODO: are there others? - continue - } - if _, ok := inlMarks[p]; ok { - // Don't use inline marks themselves. We don't know - // whether they will be zero-sized or not yet. - continue - } - pos := p.Pos.AtColumn1() - s := inlMarksByPos[pos] - if len(s) == 0 { - continue - } - for _, m := range s { - // We found an instruction with the same source position as - // some of the inline marks. - // Use this instruction instead. - p.Pos = p.Pos.WithIsStmt() // promote position to a statement - pp.CurFunc.LSym.Func().AddInlMark(p, inlMarks[m]) - // Make the inline mark a real nop, so it doesn't generate any code. - m.As = obj.ANOP - m.Pos = src.NoXPos - m.From = obj.Addr{} - m.To = obj.Addr{} - } - delete(inlMarksByPos, pos) - } - // Any unmatched inline marks now need to be added to the inlining tree (and will generate a nop instruction). - for _, p := range inlMarkList { - if p.As != obj.ANOP { - pp.CurFunc.LSym.Func().AddInlMark(p, inlMarks[p]) - } - } - } - - if base.Ctxt.Flag_locationlists { - debugInfo := ssa.BuildFuncDebug(base.Ctxt, f, base.Debug.LocationLists > 1, stackOffset) - e.curfn.DebugInfo = debugInfo - bstart := s.bstart - // Note that at this moment, Prog.Pc is a sequence number; it's - // not a real PC until after assembly, so this mapping has to - // be done later. - debugInfo.GetPC = func(b, v ssa.ID) int64 { - switch v { - case ssa.BlockStart.ID: - if b == f.Entry.ID { - return 0 // Start at the very beginning, at the assembler-generated prologue. - // this should only happen for function args (ssa.OpArg) - } - return bstart[b].Pc - case ssa.BlockEnd.ID: - return e.curfn.LSym.Size - default: - return valueToProgAfter[v].Pc - } - } - } - - // Resolve branches, and relax DefaultStmt into NotStmt - for _, br := range s.Branches { - br.P.To.SetTarget(s.bstart[br.B.ID]) - if br.P.Pos.IsStmt() != src.PosIsStmt { - br.P.Pos = br.P.Pos.WithNotStmt() - } else if v0 := br.B.FirstPossibleStmtValue(); v0 != nil && v0.Pos.Line() == br.P.Pos.Line() && v0.Pos.IsStmt() == src.PosIsStmt { - br.P.Pos = br.P.Pos.WithNotStmt() - } - - } - - if e.log { // spew to stdout - filename := "" - for p := pp.Text; p != nil; p = p.Link { - if p.Pos.IsKnown() && p.InnermostFilename() != filename { - filename = p.InnermostFilename() - f.Logf("# %s\n", filename) - } - - var s string - if v, ok := progToValue[p]; ok { - s = v.String() - } else if b, ok := progToBlock[p]; ok { - s = b.String() - } else { - s = " " // most value and branch strings are 2-3 characters long - } - f.Logf(" %-6s\t%.5d (%s)\t%s\n", s, p.Pc, p.InnermostLineNumber(), p.InstructionString()) - } - } - if f.HTMLWriter != nil { // spew to ssa.html - var buf bytes.Buffer - buf.WriteString("") - buf.WriteString("
") - filename := "" - for p := pp.Text; p != nil; p = p.Link { - // Don't spam every line with the file name, which is often huge. - // Only print changes, and "unknown" is not a change. - if p.Pos.IsKnown() && p.InnermostFilename() != filename { - filename = p.InnermostFilename() - buf.WriteString("
") - buf.WriteString(html.EscapeString("# " + filename)) - buf.WriteString("
") - } - - buf.WriteString("
") - if v, ok := progToValue[p]; ok { - buf.WriteString(v.HTML()) - } else if b, ok := progToBlock[p]; ok { - buf.WriteString("" + b.HTML() + "") - } - buf.WriteString("
") - buf.WriteString("
") - buf.WriteString(fmt.Sprintf("%.5d (%s) %s", p.Pc, p.InnermostLineNumber(), p.InnermostLineNumberHTML(), html.EscapeString(p.InstructionString()))) - buf.WriteString("
") - } - buf.WriteString("
") - buf.WriteString("") - f.HTMLWriter.WriteColumn("genssa", "genssa", "ssa-prog", buf.String()) - } - - defframe(&s, e) - - f.HTMLWriter.Close() - f.HTMLWriter = nil -} - -func defframe(s *SSAGenState, e *ssafn) { - pp := s.pp - - frame := types.Rnd(s.maxarg+e.stksize, int64(types.RegSize)) - if thearch.PadFrame != nil { - frame = thearch.PadFrame(frame) - } - - // Fill in argument and frame size. - pp.Text.To.Type = obj.TYPE_TEXTSIZE - pp.Text.To.Val = int32(types.Rnd(e.curfn.Type().ArgWidth(), int64(types.RegSize))) - pp.Text.To.Offset = frame - - // Insert code to zero ambiguously live variables so that the - // garbage collector only sees initialized values when it - // looks for pointers. - p := pp.Text - var lo, hi int64 - - // Opaque state for backend to use. Current backends use it to - // keep track of which helper registers have been zeroed. - var state uint32 - - // Iterate through declarations. They are sorted in decreasing Xoffset order. - for _, n := range e.curfn.Dcl { - if !n.Needzero() { - continue - } - if n.Class_ != ir.PAUTO { - e.Fatalf(n.Pos(), "needzero class %d", n.Class_) - } - if n.Type().Size()%int64(types.PtrSize) != 0 || n.FrameOffset()%int64(types.PtrSize) != 0 || n.Type().Size() == 0 { - e.Fatalf(n.Pos(), "var %L has size %d offset %d", n, n.Type().Size(), n.Offset_) - } - - if lo != hi && n.FrameOffset()+n.Type().Size() >= lo-int64(2*types.RegSize) { - // Merge with range we already have. - lo = n.FrameOffset() - continue - } - - // Zero old range - p = thearch.ZeroRange(pp, p, frame+lo, hi-lo, &state) - - // Set new range. - lo = n.FrameOffset() - hi = lo + n.Type().Size() - } - - // Zero final range. - thearch.ZeroRange(pp, p, frame+lo, hi-lo, &state) -} - -// For generating consecutive jump instructions to model a specific branching -type IndexJump struct { - Jump obj.As - Index int -} - -func (s *SSAGenState) oneJump(b *ssa.Block, jump *IndexJump) { - p := s.Br(jump.Jump, b.Succs[jump.Index].Block()) - p.Pos = b.Pos -} - -// CombJump generates combinational instructions (2 at present) for a block jump, -// thereby the behaviour of non-standard condition codes could be simulated -func (s *SSAGenState) CombJump(b, next *ssa.Block, jumps *[2][2]IndexJump) { - switch next { - case b.Succs[0].Block(): - s.oneJump(b, &jumps[0][0]) - s.oneJump(b, &jumps[0][1]) - case b.Succs[1].Block(): - s.oneJump(b, &jumps[1][0]) - s.oneJump(b, &jumps[1][1]) - default: - var q *obj.Prog - if b.Likely != ssa.BranchUnlikely { - s.oneJump(b, &jumps[1][0]) - s.oneJump(b, &jumps[1][1]) - q = s.Br(obj.AJMP, b.Succs[1].Block()) - } else { - s.oneJump(b, &jumps[0][0]) - s.oneJump(b, &jumps[0][1]) - q = s.Br(obj.AJMP, b.Succs[0].Block()) - } - q.Pos = b.Pos - } -} - -// AddAux adds the offset in the aux fields (AuxInt and Aux) of v to a. -func AddAux(a *obj.Addr, v *ssa.Value) { - AddAux2(a, v, v.AuxInt) -} -func AddAux2(a *obj.Addr, v *ssa.Value, offset int64) { - if a.Type != obj.TYPE_MEM && a.Type != obj.TYPE_ADDR { - v.Fatalf("bad AddAux addr %v", a) - } - // add integer offset - a.Offset += offset - - // If no additional symbol offset, we're done. - if v.Aux == nil { - return - } - // Add symbol's offset from its base register. - switch n := v.Aux.(type) { - case *ssa.AuxCall: - a.Name = obj.NAME_EXTERN - a.Sym = n.Fn - case *obj.LSym: - a.Name = obj.NAME_EXTERN - a.Sym = n - case *ir.Name: - if n.Class_ == ir.PPARAM || n.Class_ == ir.PPARAMOUT { - a.Name = obj.NAME_PARAM - a.Sym = ir.Orig(n).Sym().Linksym() - a.Offset += n.FrameOffset() - break - } - a.Name = obj.NAME_AUTO - a.Sym = n.Sym().Linksym() - a.Offset += n.FrameOffset() - default: - v.Fatalf("aux in %s not implemented %#v", v, v.Aux) - } -} - -// extendIndex extends v to a full int width. -// panic with the given kind if v does not fit in an int (only on 32-bit archs). -func (s *state) extendIndex(idx, len *ssa.Value, kind ssa.BoundsKind, bounded bool) *ssa.Value { - size := idx.Type.Size() - if size == s.config.PtrSize { - return idx - } - if size > s.config.PtrSize { - // truncate 64-bit indexes on 32-bit pointer archs. Test the - // high word and branch to out-of-bounds failure if it is not 0. - var lo *ssa.Value - if idx.Type.IsSigned() { - lo = s.newValue1(ssa.OpInt64Lo, types.Types[types.TINT], idx) - } else { - lo = s.newValue1(ssa.OpInt64Lo, types.Types[types.TUINT], idx) - } - if bounded || base.Flag.B != 0 { - return lo - } - bNext := s.f.NewBlock(ssa.BlockPlain) - bPanic := s.f.NewBlock(ssa.BlockExit) - hi := s.newValue1(ssa.OpInt64Hi, types.Types[types.TUINT32], idx) - cmp := s.newValue2(ssa.OpEq32, types.Types[types.TBOOL], hi, s.constInt32(types.Types[types.TUINT32], 0)) - if !idx.Type.IsSigned() { - switch kind { - case ssa.BoundsIndex: - kind = ssa.BoundsIndexU - case ssa.BoundsSliceAlen: - kind = ssa.BoundsSliceAlenU - case ssa.BoundsSliceAcap: - kind = ssa.BoundsSliceAcapU - case ssa.BoundsSliceB: - kind = ssa.BoundsSliceBU - case ssa.BoundsSlice3Alen: - kind = ssa.BoundsSlice3AlenU - case ssa.BoundsSlice3Acap: - kind = ssa.BoundsSlice3AcapU - case ssa.BoundsSlice3B: - kind = ssa.BoundsSlice3BU - case ssa.BoundsSlice3C: - kind = ssa.BoundsSlice3CU - } - } - b := s.endBlock() - b.Kind = ssa.BlockIf - b.SetControl(cmp) - b.Likely = ssa.BranchLikely - b.AddEdgeTo(bNext) - b.AddEdgeTo(bPanic) - - s.startBlock(bPanic) - mem := s.newValue4I(ssa.OpPanicExtend, types.TypeMem, int64(kind), hi, lo, len, s.mem()) - s.endBlock().SetControl(mem) - s.startBlock(bNext) - - return lo - } - - // Extend value to the required size - var op ssa.Op - if idx.Type.IsSigned() { - switch 10*size + s.config.PtrSize { - case 14: - op = ssa.OpSignExt8to32 - case 18: - op = ssa.OpSignExt8to64 - case 24: - op = ssa.OpSignExt16to32 - case 28: - op = ssa.OpSignExt16to64 - case 48: - op = ssa.OpSignExt32to64 - default: - s.Fatalf("bad signed index extension %s", idx.Type) - } - } else { - switch 10*size + s.config.PtrSize { - case 14: - op = ssa.OpZeroExt8to32 - case 18: - op = ssa.OpZeroExt8to64 - case 24: - op = ssa.OpZeroExt16to32 - case 28: - op = ssa.OpZeroExt16to64 - case 48: - op = ssa.OpZeroExt32to64 - default: - s.Fatalf("bad unsigned index extension %s", idx.Type) - } - } - return s.newValue1(op, types.Types[types.TINT], idx) -} - -// CheckLoweredPhi checks that regalloc and stackalloc correctly handled phi values. -// Called during ssaGenValue. -func CheckLoweredPhi(v *ssa.Value) { - if v.Op != ssa.OpPhi { - v.Fatalf("CheckLoweredPhi called with non-phi value: %v", v.LongString()) - } - if v.Type.IsMemory() { - return - } - f := v.Block.Func - loc := f.RegAlloc[v.ID] - for _, a := range v.Args { - if aloc := f.RegAlloc[a.ID]; aloc != loc { // TODO: .Equal() instead? - v.Fatalf("phi arg at different location than phi: %v @ %s, but arg %v @ %s\n%s\n", v, loc, a, aloc, v.Block.Func) - } - } -} - -// CheckLoweredGetClosurePtr checks that v is the first instruction in the function's entry block. -// The output of LoweredGetClosurePtr is generally hardwired to the correct register. -// That register contains the closure pointer on closure entry. -func CheckLoweredGetClosurePtr(v *ssa.Value) { - entry := v.Block.Func.Entry - if entry != v.Block || entry.Values[0] != v { - base.Fatalf("in %s, badly placed LoweredGetClosurePtr: %v %v", v.Block.Func.Name, v.Block, v) - } -} - -func AddrAuto(a *obj.Addr, v *ssa.Value) { - n, off := ssa.AutoVar(v) - a.Type = obj.TYPE_MEM - a.Sym = n.Sym().Linksym() - a.Reg = int16(thearch.REGSP) - a.Offset = n.FrameOffset() + off - if n.Class_ == ir.PPARAM || n.Class_ == ir.PPARAMOUT { - a.Name = obj.NAME_PARAM - } else { - a.Name = obj.NAME_AUTO - } -} - -func (s *SSAGenState) AddrScratch(a *obj.Addr) { - if s.ScratchFpMem == nil { - panic("no scratch memory available; forgot to declare usesScratch for Op?") - } - a.Type = obj.TYPE_MEM - a.Name = obj.NAME_AUTO - a.Sym = s.ScratchFpMem.Sym().Linksym() - a.Reg = int16(thearch.REGSP) - a.Offset = s.ScratchFpMem.Offset_ -} - -// Call returns a new CALL instruction for the SSA value v. -// It uses PrepareCall to prepare the call. -func (s *SSAGenState) Call(v *ssa.Value) *obj.Prog { - pPosIsStmt := s.pp.Pos.IsStmt() // The statement-ness fo the call comes from ssaGenState - s.PrepareCall(v) - - p := s.Prog(obj.ACALL) - if pPosIsStmt == src.PosIsStmt { - p.Pos = v.Pos.WithIsStmt() - } else { - p.Pos = v.Pos.WithNotStmt() - } - if sym, ok := v.Aux.(*ssa.AuxCall); ok && sym.Fn != nil { - p.To.Type = obj.TYPE_MEM - p.To.Name = obj.NAME_EXTERN - p.To.Sym = sym.Fn - } else { - // TODO(mdempsky): Can these differences be eliminated? - switch thearch.LinkArch.Family { - case sys.AMD64, sys.I386, sys.PPC64, sys.RISCV64, sys.S390X, sys.Wasm: - p.To.Type = obj.TYPE_REG - case sys.ARM, sys.ARM64, sys.MIPS, sys.MIPS64: - p.To.Type = obj.TYPE_MEM - default: - base.Fatalf("unknown indirect call family") - } - p.To.Reg = v.Args[0].Reg() - } - return p -} - -// PrepareCall prepares to emit a CALL instruction for v and does call-related bookkeeping. -// It must be called immediately before emitting the actual CALL instruction, -// since it emits PCDATA for the stack map at the call (calls are safe points). -func (s *SSAGenState) PrepareCall(v *ssa.Value) { - idx := s.livenessMap.Get(v) - if !idx.StackMapValid() { - // See Liveness.hasStackMap. - if sym, ok := v.Aux.(*ssa.AuxCall); !ok || !(sym.Fn == ir.Syms.Typedmemclr || sym.Fn == ir.Syms.Typedmemmove) { - base.Fatalf("missing stack map index for %v", v.LongString()) - } - } - - call, ok := v.Aux.(*ssa.AuxCall) - - if ok && call.Fn == ir.Syms.Deferreturn { - // Deferred calls will appear to be returning to - // the CALL deferreturn(SB) that we are about to emit. - // However, the stack trace code will show the line - // of the instruction byte before the return PC. - // To avoid that being an unrelated instruction, - // insert an actual hardware NOP that will have the right line number. - // This is different from obj.ANOP, which is a virtual no-op - // that doesn't make it into the instruction stream. - thearch.Ginsnopdefer(s.pp) - } - - if ok { - // Record call graph information for nowritebarrierrec - // analysis. - if nowritebarrierrecCheck != nil { - nowritebarrierrecCheck.recordCall(s.pp.CurFunc, call.Fn, v.Pos) - } - } - - if s.maxarg < v.AuxInt { - s.maxarg = v.AuxInt - } -} - -// UseArgs records the fact that an instruction needs a certain amount of -// callee args space for its use. -func (s *SSAGenState) UseArgs(n int64) { - if s.maxarg < n { - s.maxarg = n - } -} - -// fieldIdx finds the index of the field referred to by the ODOT node n. -func fieldIdx(n *ir.SelectorExpr) int { - t := n.X.Type() - f := n.Sel - if !t.IsStruct() { - panic("ODOT's LHS is not a struct") - } - - var i int - for _, t1 := range t.Fields().Slice() { - if t1.Sym != f { - i++ - continue - } - if t1.Offset != n.Offset { - panic("field offset doesn't match") - } - return i - } - panic(fmt.Sprintf("can't find field in expr %v\n", n)) - - // TODO: keep the result of this function somewhere in the ODOT Node - // so we don't have to recompute it each time we need it. -} - -// ssafn holds frontend information about a function that the backend is processing. -// It also exports a bunch of compiler services for the ssa backend. -type ssafn struct { - curfn *ir.Func - strings map[string]*obj.LSym // map from constant string to data symbols - scratchFpMem *ir.Name // temp for floating point register / memory moves on some architectures - stksize int64 // stack size for current frame - stkptrsize int64 // prefix of stack containing pointers - log bool // print ssa debug to the stdout -} - -// StringData returns a symbol which -// is the data component of a global string constant containing s. -func (e *ssafn) StringData(s string) *obj.LSym { - if aux, ok := e.strings[s]; ok { - return aux - } - if e.strings == nil { - e.strings = make(map[string]*obj.LSym) - } - data := staticdata.StringSym(e.curfn.Pos(), s) - e.strings[s] = data - return data -} - -func (e *ssafn) Auto(pos src.XPos, t *types.Type) *ir.Name { - return typecheck.TempAt(pos, e.curfn, t) // Note: adds new auto to e.curfn.Func.Dcl list -} - -func (e *ssafn) SplitString(name ssa.LocalSlot) (ssa.LocalSlot, ssa.LocalSlot) { - ptrType := types.NewPtr(types.Types[types.TUINT8]) - lenType := types.Types[types.TINT] - // Split this string up into two separate variables. - p := e.SplitSlot(&name, ".ptr", 0, ptrType) - l := e.SplitSlot(&name, ".len", ptrType.Size(), lenType) - return p, l -} - -func (e *ssafn) SplitInterface(name ssa.LocalSlot) (ssa.LocalSlot, ssa.LocalSlot) { - n := name.N - u := types.Types[types.TUINTPTR] - t := types.NewPtr(types.Types[types.TUINT8]) - // Split this interface up into two separate variables. - f := ".itab" - if n.Type().IsEmptyInterface() { - f = ".type" - } - c := e.SplitSlot(&name, f, 0, u) // see comment in plive.go:onebitwalktype1. - d := e.SplitSlot(&name, ".data", u.Size(), t) - return c, d -} - -func (e *ssafn) SplitSlice(name ssa.LocalSlot) (ssa.LocalSlot, ssa.LocalSlot, ssa.LocalSlot) { - ptrType := types.NewPtr(name.Type.Elem()) - lenType := types.Types[types.TINT] - p := e.SplitSlot(&name, ".ptr", 0, ptrType) - l := e.SplitSlot(&name, ".len", ptrType.Size(), lenType) - c := e.SplitSlot(&name, ".cap", ptrType.Size()+lenType.Size(), lenType) - return p, l, c -} - -func (e *ssafn) SplitComplex(name ssa.LocalSlot) (ssa.LocalSlot, ssa.LocalSlot) { - s := name.Type.Size() / 2 - var t *types.Type - if s == 8 { - t = types.Types[types.TFLOAT64] - } else { - t = types.Types[types.TFLOAT32] - } - r := e.SplitSlot(&name, ".real", 0, t) - i := e.SplitSlot(&name, ".imag", t.Size(), t) - return r, i -} - -func (e *ssafn) SplitInt64(name ssa.LocalSlot) (ssa.LocalSlot, ssa.LocalSlot) { - var t *types.Type - if name.Type.IsSigned() { - t = types.Types[types.TINT32] - } else { - t = types.Types[types.TUINT32] - } - if thearch.LinkArch.ByteOrder == binary.BigEndian { - return e.SplitSlot(&name, ".hi", 0, t), e.SplitSlot(&name, ".lo", t.Size(), types.Types[types.TUINT32]) - } - return e.SplitSlot(&name, ".hi", t.Size(), t), e.SplitSlot(&name, ".lo", 0, types.Types[types.TUINT32]) -} - -func (e *ssafn) SplitStruct(name ssa.LocalSlot, i int) ssa.LocalSlot { - st := name.Type - // Note: the _ field may appear several times. But - // have no fear, identically-named but distinct Autos are - // ok, albeit maybe confusing for a debugger. - return e.SplitSlot(&name, "."+st.FieldName(i), st.FieldOff(i), st.FieldType(i)) -} - -func (e *ssafn) SplitArray(name ssa.LocalSlot) ssa.LocalSlot { - n := name.N - at := name.Type - if at.NumElem() != 1 { - e.Fatalf(n.Pos(), "bad array size") - } - et := at.Elem() - return e.SplitSlot(&name, "[0]", 0, et) -} - -func (e *ssafn) DerefItab(it *obj.LSym, offset int64) *obj.LSym { - return reflectdata.ITabSym(it, offset) -} - -// SplitSlot returns a slot representing the data of parent starting at offset. -func (e *ssafn) SplitSlot(parent *ssa.LocalSlot, suffix string, offset int64, t *types.Type) ssa.LocalSlot { - node := parent.N - - if node.Class_ != ir.PAUTO || node.Name().Addrtaken() { - // addressed things and non-autos retain their parents (i.e., cannot truly be split) - return ssa.LocalSlot{N: node, Type: t, Off: parent.Off + offset} - } - - s := &types.Sym{Name: node.Sym().Name + suffix, Pkg: types.LocalPkg} - n := ir.NewNameAt(parent.N.Pos(), s) - s.Def = n - ir.AsNode(s.Def).Name().SetUsed(true) - n.SetType(t) - n.Class_ = ir.PAUTO - n.SetEsc(ir.EscNever) - n.Curfn = e.curfn - e.curfn.Dcl = append(e.curfn.Dcl, n) - types.CalcSize(t) - return ssa.LocalSlot{N: n, Type: t, Off: 0, SplitOf: parent, SplitOffset: offset} -} - -func (e *ssafn) CanSSA(t *types.Type) bool { - return canSSAType(t) -} - -func (e *ssafn) Line(pos src.XPos) string { - return base.FmtPos(pos) -} - -// Log logs a message from the compiler. -func (e *ssafn) Logf(msg string, args ...interface{}) { - if e.log { - fmt.Printf(msg, args...) - } -} - -func (e *ssafn) Log() bool { - return e.log -} - -// Fatal reports a compiler error and exits. -func (e *ssafn) Fatalf(pos src.XPos, msg string, args ...interface{}) { - base.Pos = pos - nargs := append([]interface{}{ir.FuncName(e.curfn)}, args...) - base.Fatalf("'%s': "+msg, nargs...) -} - -// Warnl reports a "warning", which is usually flag-triggered -// logging output for the benefit of tests. -func (e *ssafn) Warnl(pos src.XPos, fmt_ string, args ...interface{}) { - base.WarnfAt(pos, fmt_, args...) -} - -func (e *ssafn) Debug_checknil() bool { - return base.Debug.Nil != 0 -} - -func (e *ssafn) UseWriteBarrier() bool { - return base.Flag.WB -} - -func (e *ssafn) Syslook(name string) *obj.LSym { - switch name { - case "goschedguarded": - return ir.Syms.Goschedguarded - case "writeBarrier": - return ir.Syms.WriteBarrier - case "gcWriteBarrier": - return ir.Syms.GCWriteBarrier - case "typedmemmove": - return ir.Syms.Typedmemmove - case "typedmemclr": - return ir.Syms.Typedmemclr - } - e.Fatalf(src.NoXPos, "unknown Syslook func %v", name) - return nil -} - -func (e *ssafn) SetWBPos(pos src.XPos) { - e.curfn.SetWBPos(pos) -} - -func (e *ssafn) MyImportPath() string { - return base.Ctxt.Pkgpath -} - -func clobberBase(n ir.Node) ir.Node { - if n.Op() == ir.ODOT { - n := n.(*ir.SelectorExpr) - if n.X.Type().NumFields() == 1 { - return clobberBase(n.X) - } - } - if n.Op() == ir.OINDEX { - n := n.(*ir.IndexExpr) - if n.X.Type().IsArray() && n.X.Type().NumElem() == 1 { - return clobberBase(n.X) - } - } - return n -} - -// callTargetLSym determines the correct LSym for 'callee' when called -// from function 'caller'. There are a couple of different scenarios -// to contend with here: -// -// 1. if 'caller' is an ABI wrapper, then we always want to use the -// LSym from the Func for the callee. -// -// 2. if 'caller' is not an ABI wrapper, then we looked at the callee -// to see if it corresponds to a "known" ABI0 symbol (e.g. assembly -// routine defined in the current package); if so, we want the call to -// directly target the ABI0 symbol (effectively bypassing the -// ABIInternal->ABI0 wrapper for 'callee'). -// -// 3. in all other cases, want the regular ABIInternal linksym -// -func callTargetLSym(callee *types.Sym, callerLSym *obj.LSym) *obj.LSym { - lsym := callee.Linksym() - if !base.Flag.ABIWrap { - return lsym - } - if ir.AsNode(callee.Def) == nil { - return lsym - } - defn := ir.AsNode(callee.Def).Name().Defn - if defn == nil { - return lsym - } - ndclfunc := defn.(*ir.Func) - - // check for case 1 above - if callerLSym.ABIWrapper() { - if nlsym := ndclfunc.LSym; nlsym != nil { - lsym = nlsym - } - } else { - // check for case 2 above - nam := ndclfunc.Nname - defABI, hasDefABI := symabiDefs[nam.Sym().LinksymName()] - if hasDefABI && defABI == obj.ABI0 { - lsym = nam.Sym().LinksymABI0() - } - } - return lsym -} - -func min8(a, b int8) int8 { - if a < b { - return a - } - return b -} - -func max8(a, b int8) int8 { - if a > b { - return a - } - return b -} - -// deferstruct makes a runtime._defer structure, with additional space for -// stksize bytes of args. -func deferstruct(stksize int64) *types.Type { - makefield := func(name string, typ *types.Type) *types.Field { - // Unlike the global makefield function, this one needs to set Pkg - // because these types might be compared (in SSA CSE sorting). - // TODO: unify this makefield and the global one above. - sym := &types.Sym{Name: name, Pkg: types.LocalPkg} - return types.NewField(src.NoXPos, sym, typ) - } - argtype := types.NewArray(types.Types[types.TUINT8], stksize) - argtype.Width = stksize - argtype.Align = 1 - // These fields must match the ones in runtime/runtime2.go:_defer and - // cmd/compile/internal/gc/ssa.go:(*state).call. - fields := []*types.Field{ - makefield("siz", types.Types[types.TUINT32]), - makefield("started", types.Types[types.TBOOL]), - makefield("heap", types.Types[types.TBOOL]), - makefield("openDefer", types.Types[types.TBOOL]), - makefield("sp", types.Types[types.TUINTPTR]), - makefield("pc", types.Types[types.TUINTPTR]), - // Note: the types here don't really matter. Defer structures - // are always scanned explicitly during stack copying and GC, - // so we make them uintptr type even though they are real pointers. - makefield("fn", types.Types[types.TUINTPTR]), - makefield("_panic", types.Types[types.TUINTPTR]), - makefield("link", types.Types[types.TUINTPTR]), - makefield("framepc", types.Types[types.TUINTPTR]), - makefield("varp", types.Types[types.TUINTPTR]), - makefield("fd", types.Types[types.TUINTPTR]), - makefield("args", argtype), - } - - // build struct holding the above fields - s := types.NewStruct(types.NoPkg, fields) - s.SetNoalg(true) - types.CalcStructSize(s) - return s -} diff --git a/src/cmd/compile/internal/gc/subr.go b/src/cmd/compile/internal/gc/subr.go index 89baaf7eee..02a4c0a688 100644 --- a/src/cmd/compile/internal/gc/subr.go +++ b/src/cmd/compile/internal/gc/subr.go @@ -8,24 +8,11 @@ import ( "cmd/compile/internal/base" "cmd/compile/internal/ir" "cmd/compile/internal/reflectdata" + "cmd/compile/internal/ssagen" "cmd/compile/internal/typecheck" "cmd/compile/internal/types" "cmd/internal/src" "fmt" - "sync" -) - -// largeStack is info about a function whose stack frame is too large (rare). -type largeStack struct { - locals int64 - args int64 - callee int64 - pos src.XPos -} - -var ( - largeStackFramesMu sync.Mutex // protects largeStackFrames - largeStackFrames []largeStack ) // backingArrayPtrLen extracts the pointer and length from a slice or string. @@ -91,25 +78,25 @@ func calcHasCall(n ir.Node) bool { // so we ensure they are evaluated first. case ir.OADD, ir.OSUB, ir.OMUL: n := n.(*ir.BinaryExpr) - if thearch.SoftFloat && (types.IsFloat[n.Type().Kind()] || types.IsComplex[n.Type().Kind()]) { + if ssagen.Arch.SoftFloat && (types.IsFloat[n.Type().Kind()] || types.IsComplex[n.Type().Kind()]) { return true } return n.X.HasCall() || n.Y.HasCall() case ir.ONEG: n := n.(*ir.UnaryExpr) - if thearch.SoftFloat && (types.IsFloat[n.Type().Kind()] || types.IsComplex[n.Type().Kind()]) { + if ssagen.Arch.SoftFloat && (types.IsFloat[n.Type().Kind()] || types.IsComplex[n.Type().Kind()]) { return true } return n.X.HasCall() case ir.OLT, ir.OEQ, ir.ONE, ir.OLE, ir.OGE, ir.OGT: n := n.(*ir.BinaryExpr) - if thearch.SoftFloat && (types.IsFloat[n.X.Type().Kind()] || types.IsComplex[n.X.Type().Kind()]) { + if ssagen.Arch.SoftFloat && (types.IsFloat[n.X.Type().Kind()] || types.IsComplex[n.X.Type().Kind()]) { return true } return n.X.HasCall() || n.Y.HasCall() case ir.OCONV: n := n.(*ir.ConvExpr) - if thearch.SoftFloat && ((types.IsFloat[n.Type().Kind()] || types.IsComplex[n.Type().Kind()]) || (types.IsFloat[n.X.Type().Kind()] || types.IsComplex[n.X.Type().Kind()])) { + if ssagen.Arch.SoftFloat && ((types.IsFloat[n.Type().Kind()] || types.IsComplex[n.Type().Kind()]) || (types.IsFloat[n.X.Type().Kind()] || types.IsComplex[n.X.Type().Kind()])) { return true } return n.X.HasCall() diff --git a/src/cmd/compile/internal/gc/walk.go b/src/cmd/compile/internal/gc/walk.go index 9b49b06c34..f86dbba2c9 100644 --- a/src/cmd/compile/internal/gc/walk.go +++ b/src/cmd/compile/internal/gc/walk.go @@ -9,6 +9,7 @@ import ( "cmd/compile/internal/escape" "cmd/compile/internal/ir" "cmd/compile/internal/reflectdata" + "cmd/compile/internal/ssagen" "cmd/compile/internal/staticdata" "cmd/compile/internal/typecheck" "cmd/compile/internal/types" @@ -977,7 +978,7 @@ func walkexpr1(n ir.Node, init *ir.Nodes) ir.Node { n.X = cheapexpr(n.X, init) // byteindex widens n.Left so that the multiplication doesn't overflow. index := ir.NewBinaryExpr(base.Pos, ir.OLSH, byteindex(n.X), ir.NewInt(3)) - if thearch.LinkArch.ByteOrder == binary.BigEndian { + if ssagen.Arch.LinkArch.ByteOrder == binary.BigEndian { index = ir.NewBinaryExpr(base.Pos, ir.OADD, index, ir.NewInt(7)) } xe := ir.NewIndexExpr(base.Pos, ir.Names.Staticuint64s, index) @@ -1675,7 +1676,7 @@ func walkexpr1(n ir.Node, init *ir.Nodes) ir.Node { return mkcall("stringtoslicerune", n.Type(), init, a, typecheck.Conv(n.X, types.Types[types.TSTRING])) case ir.OARRAYLIT, ir.OSLICELIT, ir.OMAPLIT, ir.OSTRUCTLIT, ir.OPTRLIT: - if isStaticCompositeLiteral(n) && !canSSAType(n.Type()) { + if isStaticCompositeLiteral(n) && !ssagen.TypeOK(n.Type()) { n := n.(*ir.CompLitExpr) // not OPTRLIT // n can be directly represented in the read-only data section. // Make direct reference to the static data. See issue 12841. @@ -1739,11 +1740,11 @@ func markUsedIfaceMethod(n *ir.CallExpr) { // // If no such function is necessary, it returns (Txxx, Txxx). func rtconvfn(src, dst *types.Type) (param, result types.Kind) { - if thearch.SoftFloat { + if ssagen.Arch.SoftFloat { return types.Txxx, types.Txxx } - switch thearch.LinkArch.Family { + switch ssagen.Arch.LinkArch.Family { case sys.ARM, sys.MIPS: if src.IsFloat() { switch dst.Kind() { @@ -3229,7 +3230,7 @@ func walkcompare(n *ir.BinaryExpr, init *ir.Nodes) ir.Node { unalignedLoad := canMergeLoads() if unalignedLoad { // Keep this low enough to generate less code than a function call. - maxcmpsize = 2 * int64(thearch.LinkArch.RegSize) + maxcmpsize = 2 * int64(ssagen.Arch.LinkArch.RegSize) } switch t.Kind() { @@ -3469,8 +3470,8 @@ func walkcompareString(n *ir.BinaryExpr, init *ir.Nodes) ir.Node { combine64bit := false if canCombineLoads { // Keep this low enough to generate less code than a function call. - maxRewriteLen = 2 * thearch.LinkArch.RegSize - combine64bit = thearch.LinkArch.RegSize >= 8 + maxRewriteLen = 2 * ssagen.Arch.LinkArch.RegSize + combine64bit = ssagen.Arch.LinkArch.RegSize >= 8 } var and ir.Op @@ -3909,12 +3910,12 @@ func wrapCall(n *ir.CallExpr, init *ir.Nodes) ir.Node { // larger, possibly unaligned, load. Note that currently the // optimizations must be able to handle little endian byte order. func canMergeLoads() bool { - switch thearch.LinkArch.Family { + switch ssagen.Arch.LinkArch.Family { case sys.ARM64, sys.AMD64, sys.I386, sys.S390X: return true case sys.PPC64: // Load combining only supported on ppc64le. - return thearch.LinkArch.ByteOrder == binary.LittleEndian + return ssagen.Arch.LinkArch.ByteOrder == binary.LittleEndian } return false } @@ -4032,3 +4033,7 @@ func appendWalkStmt(init *ir.Nodes, stmt ir.Node) { } init.Append(n) } + +// The max number of defers in a function using open-coded defers. We enforce this +// limit because the deferBits bitmask is currently a single byte (to minimize code size) +const maxOpenDefers = 8 diff --git a/src/cmd/compile/internal/mips/galign.go b/src/cmd/compile/internal/mips/galign.go index be40c16dde..599163550b 100644 --- a/src/cmd/compile/internal/mips/galign.go +++ b/src/cmd/compile/internal/mips/galign.go @@ -5,13 +5,13 @@ package mips import ( - "cmd/compile/internal/gc" "cmd/compile/internal/ssa" + "cmd/compile/internal/ssagen" "cmd/internal/obj/mips" "cmd/internal/objabi" ) -func Init(arch *gc.Arch) { +func Init(arch *ssagen.ArchInfo) { arch.LinkArch = &mips.Linkmips if objabi.GOARCH == "mipsle" { arch.LinkArch = &mips.Linkmipsle @@ -22,7 +22,7 @@ func Init(arch *gc.Arch) { arch.ZeroRange = zerorange arch.Ginsnop = ginsnop arch.Ginsnopdefer = ginsnop - arch.SSAMarkMoves = func(s *gc.SSAGenState, b *ssa.Block) {} + arch.SSAMarkMoves = func(s *ssagen.State, b *ssa.Block) {} arch.SSAGenValue = ssaGenValue arch.SSAGenBlock = ssaGenBlock } diff --git a/src/cmd/compile/internal/mips/ssa.go b/src/cmd/compile/internal/mips/ssa.go index e46d87e17d..f1cdbd3241 100644 --- a/src/cmd/compile/internal/mips/ssa.go +++ b/src/cmd/compile/internal/mips/ssa.go @@ -8,10 +8,10 @@ import ( "math" "cmd/compile/internal/base" - "cmd/compile/internal/gc" "cmd/compile/internal/ir" "cmd/compile/internal/logopt" "cmd/compile/internal/ssa" + "cmd/compile/internal/ssagen" "cmd/compile/internal/types" "cmd/internal/obj" "cmd/internal/obj/mips" @@ -77,7 +77,7 @@ func storeByType(t *types.Type, r int16) obj.As { panic("bad store type") } -func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { +func ssaGenValue(s *ssagen.State, v *ssa.Value) { switch v.Op { case ssa.OpCopy, ssa.OpMIPSMOVWreg: t := v.Type @@ -123,7 +123,7 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { } r := v.Reg() p := s.Prog(loadByType(v.Type, r)) - gc.AddrAuto(&p.From, v.Args[0]) + ssagen.AddrAuto(&p.From, v.Args[0]) p.To.Type = obj.TYPE_REG p.To.Reg = r if isHILO(r) { @@ -153,7 +153,7 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { p := s.Prog(storeByType(v.Type, r)) p.From.Type = obj.TYPE_REG p.From.Reg = r - gc.AddrAuto(&p.To, v) + ssagen.AddrAuto(&p.To, v) case ssa.OpMIPSADD, ssa.OpMIPSSUB, ssa.OpMIPSAND, @@ -288,10 +288,10 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { v.Fatalf("aux is of unknown type %T", v.Aux) case *obj.LSym: wantreg = "SB" - gc.AddAux(&p.From, v) + ssagen.AddAux(&p.From, v) case *ir.Name: wantreg = "SP" - gc.AddAux(&p.From, v) + ssagen.AddAux(&p.From, v) case nil: // No sym, just MOVW $off(SP), R wantreg = "SP" @@ -312,7 +312,7 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { p := s.Prog(v.Op.Asm()) p.From.Type = obj.TYPE_MEM p.From.Reg = v.Args[0].Reg() - gc.AddAux(&p.From, v) + ssagen.AddAux(&p.From, v) p.To.Type = obj.TYPE_REG p.To.Reg = v.Reg() case ssa.OpMIPSMOVBstore, @@ -325,7 +325,7 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { p.From.Reg = v.Args[1].Reg() p.To.Type = obj.TYPE_MEM p.To.Reg = v.Args[0].Reg() - gc.AddAux(&p.To, v) + ssagen.AddAux(&p.To, v) case ssa.OpMIPSMOVBstorezero, ssa.OpMIPSMOVHstorezero, ssa.OpMIPSMOVWstorezero: @@ -334,7 +334,7 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { p.From.Reg = mips.REGZERO p.To.Type = obj.TYPE_MEM p.To.Reg = v.Args[0].Reg() - gc.AddAux(&p.To, v) + ssagen.AddAux(&p.To, v) case ssa.OpMIPSMOVBreg, ssa.OpMIPSMOVBUreg, ssa.OpMIPSMOVHreg, @@ -492,13 +492,13 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { p := s.Prog(obj.ACALL) p.To.Type = obj.TYPE_MEM p.To.Name = obj.NAME_EXTERN - p.To.Sym = gc.BoundsCheckFunc[v.AuxInt] + p.To.Sym = ssagen.BoundsCheckFunc[v.AuxInt] s.UseArgs(8) // space used in callee args area by assembly stubs case ssa.OpMIPSLoweredPanicExtendA, ssa.OpMIPSLoweredPanicExtendB, ssa.OpMIPSLoweredPanicExtendC: p := s.Prog(obj.ACALL) p.To.Type = obj.TYPE_MEM p.To.Name = obj.NAME_EXTERN - p.To.Sym = gc.ExtendCheckFunc[v.AuxInt] + p.To.Sym = ssagen.ExtendCheckFunc[v.AuxInt] s.UseArgs(12) // space used in callee args area by assembly stubs case ssa.OpMIPSLoweredAtomicLoad8, ssa.OpMIPSLoweredAtomicLoad32: @@ -762,7 +762,7 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { p := s.Prog(mips.AMOVB) p.From.Type = obj.TYPE_MEM p.From.Reg = v.Args[0].Reg() - gc.AddAux(&p.From, v) + ssagen.AddAux(&p.From, v) p.To.Type = obj.TYPE_REG p.To.Reg = mips.REGTMP if logopt.Enabled() { @@ -793,7 +793,7 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { case ssa.OpMIPSLoweredGetClosurePtr: // Closure pointer is R22 (mips.REGCTXT). - gc.CheckLoweredGetClosurePtr(v) + ssagen.CheckLoweredGetClosurePtr(v) case ssa.OpMIPSLoweredGetCallerSP: // caller's SP is FixedFrameSize below the address of the first arg p := s.Prog(mips.AMOVW) @@ -826,13 +826,13 @@ var blockJump = map[ssa.BlockKind]struct { ssa.BlockMIPSFPF: {mips.ABFPF, mips.ABFPT}, } -func ssaGenBlock(s *gc.SSAGenState, b, next *ssa.Block) { +func ssaGenBlock(s *ssagen.State, b, next *ssa.Block) { switch b.Kind { case ssa.BlockPlain: if b.Succs[0].Block() != next { p := s.Prog(obj.AJMP) p.To.Type = obj.TYPE_BRANCH - s.Branches = append(s.Branches, gc.Branch{P: p, B: b.Succs[0].Block()}) + s.Branches = append(s.Branches, ssagen.Branch{P: p, B: b.Succs[0].Block()}) } case ssa.BlockDefer: // defer returns in R1: @@ -843,11 +843,11 @@ func ssaGenBlock(s *gc.SSAGenState, b, next *ssa.Block) { p.From.Reg = mips.REGZERO p.Reg = mips.REG_R1 p.To.Type = obj.TYPE_BRANCH - s.Branches = append(s.Branches, gc.Branch{P: p, B: b.Succs[1].Block()}) + s.Branches = append(s.Branches, ssagen.Branch{P: p, B: b.Succs[1].Block()}) if b.Succs[0].Block() != next { p := s.Prog(obj.AJMP) p.To.Type = obj.TYPE_BRANCH - s.Branches = append(s.Branches, gc.Branch{P: p, B: b.Succs[0].Block()}) + s.Branches = append(s.Branches, ssagen.Branch{P: p, B: b.Succs[0].Block()}) } case ssa.BlockExit: case ssa.BlockRet: diff --git a/src/cmd/compile/internal/mips64/galign.go b/src/cmd/compile/internal/mips64/galign.go index 90c381a50b..fc0a34228c 100644 --- a/src/cmd/compile/internal/mips64/galign.go +++ b/src/cmd/compile/internal/mips64/galign.go @@ -5,13 +5,13 @@ package mips64 import ( - "cmd/compile/internal/gc" "cmd/compile/internal/ssa" + "cmd/compile/internal/ssagen" "cmd/internal/obj/mips" "cmd/internal/objabi" ) -func Init(arch *gc.Arch) { +func Init(arch *ssagen.ArchInfo) { arch.LinkArch = &mips.Linkmips64 if objabi.GOARCH == "mips64le" { arch.LinkArch = &mips.Linkmips64le @@ -23,7 +23,7 @@ func Init(arch *gc.Arch) { arch.Ginsnop = ginsnop arch.Ginsnopdefer = ginsnop - arch.SSAMarkMoves = func(s *gc.SSAGenState, b *ssa.Block) {} + arch.SSAMarkMoves = func(s *ssagen.State, b *ssa.Block) {} arch.SSAGenValue = ssaGenValue arch.SSAGenBlock = ssaGenBlock } diff --git a/src/cmd/compile/internal/mips64/ssa.go b/src/cmd/compile/internal/mips64/ssa.go index 096e7048ce..14cf7af143 100644 --- a/src/cmd/compile/internal/mips64/ssa.go +++ b/src/cmd/compile/internal/mips64/ssa.go @@ -8,10 +8,10 @@ import ( "math" "cmd/compile/internal/base" - "cmd/compile/internal/gc" "cmd/compile/internal/ir" "cmd/compile/internal/logopt" "cmd/compile/internal/ssa" + "cmd/compile/internal/ssagen" "cmd/compile/internal/types" "cmd/internal/obj" "cmd/internal/obj/mips" @@ -85,7 +85,7 @@ func storeByType(t *types.Type, r int16) obj.As { panic("bad store type") } -func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { +func ssaGenValue(s *ssagen.State, v *ssa.Value) { switch v.Op { case ssa.OpCopy, ssa.OpMIPS64MOVVreg: if v.Type.IsMemory() { @@ -126,7 +126,7 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { } r := v.Reg() p := s.Prog(loadByType(v.Type, r)) - gc.AddrAuto(&p.From, v.Args[0]) + ssagen.AddrAuto(&p.From, v.Args[0]) p.To.Type = obj.TYPE_REG p.To.Reg = r if isHILO(r) { @@ -156,7 +156,7 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { p := s.Prog(storeByType(v.Type, r)) p.From.Type = obj.TYPE_REG p.From.Reg = r - gc.AddrAuto(&p.To, v) + ssagen.AddrAuto(&p.To, v) case ssa.OpMIPS64ADDV, ssa.OpMIPS64SUBV, ssa.OpMIPS64AND, @@ -262,10 +262,10 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { v.Fatalf("aux is of unknown type %T", v.Aux) case *obj.LSym: wantreg = "SB" - gc.AddAux(&p.From, v) + ssagen.AddAux(&p.From, v) case *ir.Name: wantreg = "SP" - gc.AddAux(&p.From, v) + ssagen.AddAux(&p.From, v) case nil: // No sym, just MOVV $off(SP), R wantreg = "SP" @@ -288,7 +288,7 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { p := s.Prog(v.Op.Asm()) p.From.Type = obj.TYPE_MEM p.From.Reg = v.Args[0].Reg() - gc.AddAux(&p.From, v) + ssagen.AddAux(&p.From, v) p.To.Type = obj.TYPE_REG p.To.Reg = v.Reg() case ssa.OpMIPS64MOVBstore, @@ -302,7 +302,7 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { p.From.Reg = v.Args[1].Reg() p.To.Type = obj.TYPE_MEM p.To.Reg = v.Args[0].Reg() - gc.AddAux(&p.To, v) + ssagen.AddAux(&p.To, v) case ssa.OpMIPS64MOVBstorezero, ssa.OpMIPS64MOVHstorezero, ssa.OpMIPS64MOVWstorezero, @@ -312,7 +312,7 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { p.From.Reg = mips.REGZERO p.To.Type = obj.TYPE_MEM p.To.Reg = v.Args[0].Reg() - gc.AddAux(&p.To, v) + ssagen.AddAux(&p.To, v) case ssa.OpMIPS64MOVBreg, ssa.OpMIPS64MOVBUreg, ssa.OpMIPS64MOVHreg, @@ -502,7 +502,7 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { p := s.Prog(obj.ACALL) p.To.Type = obj.TYPE_MEM p.To.Name = obj.NAME_EXTERN - p.To.Sym = gc.BoundsCheckFunc[v.AuxInt] + p.To.Sym = ssagen.BoundsCheckFunc[v.AuxInt] s.UseArgs(16) // space used in callee args area by assembly stubs case ssa.OpMIPS64LoweredAtomicLoad8, ssa.OpMIPS64LoweredAtomicLoad32, ssa.OpMIPS64LoweredAtomicLoad64: as := mips.AMOVV @@ -720,7 +720,7 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { p := s.Prog(mips.AMOVB) p.From.Type = obj.TYPE_MEM p.From.Reg = v.Args[0].Reg() - gc.AddAux(&p.From, v) + ssagen.AddAux(&p.From, v) p.To.Type = obj.TYPE_REG p.To.Reg = mips.REGTMP if logopt.Enabled() { @@ -754,7 +754,7 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { p2.To.SetTarget(p4) case ssa.OpMIPS64LoweredGetClosurePtr: // Closure pointer is R22 (mips.REGCTXT). - gc.CheckLoweredGetClosurePtr(v) + ssagen.CheckLoweredGetClosurePtr(v) case ssa.OpMIPS64LoweredGetCallerSP: // caller's SP is FixedFrameSize below the address of the first arg p := s.Prog(mips.AMOVV) @@ -787,13 +787,13 @@ var blockJump = map[ssa.BlockKind]struct { ssa.BlockMIPS64FPF: {mips.ABFPF, mips.ABFPT}, } -func ssaGenBlock(s *gc.SSAGenState, b, next *ssa.Block) { +func ssaGenBlock(s *ssagen.State, b, next *ssa.Block) { switch b.Kind { case ssa.BlockPlain: if b.Succs[0].Block() != next { p := s.Prog(obj.AJMP) p.To.Type = obj.TYPE_BRANCH - s.Branches = append(s.Branches, gc.Branch{P: p, B: b.Succs[0].Block()}) + s.Branches = append(s.Branches, ssagen.Branch{P: p, B: b.Succs[0].Block()}) } case ssa.BlockDefer: // defer returns in R1: @@ -804,11 +804,11 @@ func ssaGenBlock(s *gc.SSAGenState, b, next *ssa.Block) { p.From.Reg = mips.REGZERO p.Reg = mips.REG_R1 p.To.Type = obj.TYPE_BRANCH - s.Branches = append(s.Branches, gc.Branch{P: p, B: b.Succs[1].Block()}) + s.Branches = append(s.Branches, ssagen.Branch{P: p, B: b.Succs[1].Block()}) if b.Succs[0].Block() != next { p := s.Prog(obj.AJMP) p.To.Type = obj.TYPE_BRANCH - s.Branches = append(s.Branches, gc.Branch{P: p, B: b.Succs[0].Block()}) + s.Branches = append(s.Branches, ssagen.Branch{P: p, B: b.Succs[0].Block()}) } case ssa.BlockExit: case ssa.BlockRet: diff --git a/src/cmd/compile/internal/ppc64/galign.go b/src/cmd/compile/internal/ppc64/galign.go index c8ef567dc3..c72d1aa834 100644 --- a/src/cmd/compile/internal/ppc64/galign.go +++ b/src/cmd/compile/internal/ppc64/galign.go @@ -5,12 +5,12 @@ package ppc64 import ( - "cmd/compile/internal/gc" + "cmd/compile/internal/ssagen" "cmd/internal/obj/ppc64" "cmd/internal/objabi" ) -func Init(arch *gc.Arch) { +func Init(arch *ssagen.ArchInfo) { arch.LinkArch = &ppc64.Linkppc64 if objabi.GOARCH == "ppc64le" { arch.LinkArch = &ppc64.Linkppc64le diff --git a/src/cmd/compile/internal/ppc64/ssa.go b/src/cmd/compile/internal/ppc64/ssa.go index edcaad03ec..c85e110ed3 100644 --- a/src/cmd/compile/internal/ppc64/ssa.go +++ b/src/cmd/compile/internal/ppc64/ssa.go @@ -6,10 +6,10 @@ package ppc64 import ( "cmd/compile/internal/base" - "cmd/compile/internal/gc" "cmd/compile/internal/ir" "cmd/compile/internal/logopt" "cmd/compile/internal/ssa" + "cmd/compile/internal/ssagen" "cmd/compile/internal/types" "cmd/internal/obj" "cmd/internal/obj/ppc64" @@ -19,7 +19,7 @@ import ( ) // markMoves marks any MOVXconst ops that need to avoid clobbering flags. -func ssaMarkMoves(s *gc.SSAGenState, b *ssa.Block) { +func ssaMarkMoves(s *ssagen.State, b *ssa.Block) { // flive := b.FlagsLiveAtEnd // if b.Control != nil && b.Control.Type.IsFlags() { // flive = true @@ -101,7 +101,7 @@ func storeByType(t *types.Type) obj.As { panic("bad store type") } -func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { +func ssaGenValue(s *ssagen.State, v *ssa.Value) { switch v.Op { case ssa.OpCopy: t := v.Type @@ -469,7 +469,7 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { case ssa.OpPPC64LoweredGetClosurePtr: // Closure pointer is R11 (already) - gc.CheckLoweredGetClosurePtr(v) + ssagen.CheckLoweredGetClosurePtr(v) case ssa.OpPPC64LoweredGetCallerSP: // caller's SP is FixedFrameSize below the address of the first arg @@ -491,7 +491,7 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { case ssa.OpLoadReg: loadOp := loadByType(v.Type) p := s.Prog(loadOp) - gc.AddrAuto(&p.From, v.Args[0]) + ssagen.AddrAuto(&p.From, v.Args[0]) p.To.Type = obj.TYPE_REG p.To.Reg = v.Reg() @@ -500,7 +500,7 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { p := s.Prog(storeOp) p.From.Type = obj.TYPE_REG p.From.Reg = v.Args[0].Reg() - gc.AddrAuto(&p.To, v) + ssagen.AddrAuto(&p.To, v) case ssa.OpPPC64DIVD: // For now, @@ -758,7 +758,7 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { p.From.Reg = v.Args[0].Reg() p.To.Type = obj.TYPE_REG p.To.Reg = v.Reg() - gc.AddAux(&p.From, v) + ssagen.AddAux(&p.From, v) } @@ -819,7 +819,7 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { p := s.Prog(ppc64.AMOVD) p.From.Type = obj.TYPE_ADDR p.From.Reg = v.Args[0].Reg() - gc.AddAux(&p.From, v) + ssagen.AddAux(&p.From, v) p.To.Type = obj.TYPE_REG p.To.Reg = v.Reg() // Load go.string using 0 offset @@ -837,7 +837,7 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { p := s.Prog(v.Op.Asm()) p.From.Type = obj.TYPE_MEM p.From.Reg = v.Args[0].Reg() - gc.AddAux(&p.From, v) + ssagen.AddAux(&p.From, v) p.To.Type = obj.TYPE_REG p.To.Reg = v.Reg() @@ -871,7 +871,7 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { p.From.Reg = ppc64.REGZERO p.To.Type = obj.TYPE_MEM p.To.Reg = v.Args[0].Reg() - gc.AddAux(&p.To, v) + ssagen.AddAux(&p.To, v) case ssa.OpPPC64MOVDstore, ssa.OpPPC64MOVWstore, ssa.OpPPC64MOVHstore, ssa.OpPPC64MOVBstore, ssa.OpPPC64FMOVDstore, ssa.OpPPC64FMOVSstore: p := s.Prog(v.Op.Asm()) @@ -879,7 +879,7 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { p.From.Reg = v.Args[1].Reg() p.To.Type = obj.TYPE_MEM p.To.Reg = v.Args[0].Reg() - gc.AddAux(&p.To, v) + ssagen.AddAux(&p.To, v) case ssa.OpPPC64MOVDstoreidx, ssa.OpPPC64MOVWstoreidx, ssa.OpPPC64MOVHstoreidx, ssa.OpPPC64MOVBstoreidx, ssa.OpPPC64FMOVDstoreidx, ssa.OpPPC64FMOVSstoreidx, ssa.OpPPC64MOVDBRstoreidx, ssa.OpPPC64MOVWBRstoreidx, @@ -1809,7 +1809,7 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { p := s.Prog(obj.ACALL) p.To.Type = obj.TYPE_MEM p.To.Name = obj.NAME_EXTERN - p.To.Sym = gc.BoundsCheckFunc[v.AuxInt] + p.To.Sym = ssagen.BoundsCheckFunc[v.AuxInt] s.UseArgs(16) // space used in callee args area by assembly stubs case ssa.OpPPC64LoweredNilCheck: @@ -1847,7 +1847,7 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { p := s.Prog(ppc64.AMOVBZ) p.From.Type = obj.TYPE_MEM p.From.Reg = v.Args[0].Reg() - gc.AddAux(&p.From, v) + ssagen.AddAux(&p.From, v) p.To.Type = obj.TYPE_REG p.To.Reg = ppc64.REGTMP } @@ -1893,7 +1893,7 @@ var blockJump = [...]struct { ssa.BlockPPC64FGT: {ppc64.ABGT, ppc64.ABLE, false, false}, } -func ssaGenBlock(s *gc.SSAGenState, b, next *ssa.Block) { +func ssaGenBlock(s *ssagen.State, b, next *ssa.Block) { switch b.Kind { case ssa.BlockDefer: // defer returns in R3: @@ -1907,18 +1907,18 @@ func ssaGenBlock(s *gc.SSAGenState, b, next *ssa.Block) { p = s.Prog(ppc64.ABNE) p.To.Type = obj.TYPE_BRANCH - s.Branches = append(s.Branches, gc.Branch{P: p, B: b.Succs[1].Block()}) + s.Branches = append(s.Branches, ssagen.Branch{P: p, B: b.Succs[1].Block()}) if b.Succs[0].Block() != next { p := s.Prog(obj.AJMP) p.To.Type = obj.TYPE_BRANCH - s.Branches = append(s.Branches, gc.Branch{P: p, B: b.Succs[0].Block()}) + s.Branches = append(s.Branches, ssagen.Branch{P: p, B: b.Succs[0].Block()}) } case ssa.BlockPlain: if b.Succs[0].Block() != next { p := s.Prog(obj.AJMP) p.To.Type = obj.TYPE_BRANCH - s.Branches = append(s.Branches, gc.Branch{P: p, B: b.Succs[0].Block()}) + s.Branches = append(s.Branches, ssagen.Branch{P: p, B: b.Succs[0].Block()}) } case ssa.BlockExit: case ssa.BlockRet: diff --git a/src/cmd/compile/internal/riscv64/galign.go b/src/cmd/compile/internal/riscv64/galign.go index 4db0fac52e..338248a7cf 100644 --- a/src/cmd/compile/internal/riscv64/galign.go +++ b/src/cmd/compile/internal/riscv64/galign.go @@ -5,11 +5,11 @@ package riscv64 import ( - "cmd/compile/internal/gc" + "cmd/compile/internal/ssagen" "cmd/internal/obj/riscv" ) -func Init(arch *gc.Arch) { +func Init(arch *ssagen.ArchInfo) { arch.LinkArch = &riscv.LinkRISCV64 arch.REGSP = riscv.REG_SP diff --git a/src/cmd/compile/internal/riscv64/ssa.go b/src/cmd/compile/internal/riscv64/ssa.go index d08cebdcf5..70c29a4b7b 100644 --- a/src/cmd/compile/internal/riscv64/ssa.go +++ b/src/cmd/compile/internal/riscv64/ssa.go @@ -6,9 +6,9 @@ package riscv64 import ( "cmd/compile/internal/base" - "cmd/compile/internal/gc" "cmd/compile/internal/ir" "cmd/compile/internal/ssa" + "cmd/compile/internal/ssagen" "cmd/compile/internal/types" "cmd/internal/obj" "cmd/internal/obj/riscv" @@ -180,9 +180,9 @@ func largestMove(alignment int64) (obj.As, int64) { // markMoves marks any MOVXconst ops that need to avoid clobbering flags. // RISC-V has no flags, so this is a no-op. -func ssaMarkMoves(s *gc.SSAGenState, b *ssa.Block) {} +func ssaMarkMoves(s *ssagen.State, b *ssa.Block) {} -func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { +func ssaGenValue(s *ssagen.State, v *ssa.Value) { s.SetPos(v.Pos) switch v.Op { @@ -191,7 +191,7 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { case ssa.OpArg: // input args need no code case ssa.OpPhi: - gc.CheckLoweredPhi(v) + ssagen.CheckLoweredPhi(v) case ssa.OpCopy, ssa.OpRISCV64MOVconvert, ssa.OpRISCV64MOVDreg: if v.Type.IsMemory() { return @@ -221,7 +221,7 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { return } p := s.Prog(loadByType(v.Type)) - gc.AddrAuto(&p.From, v.Args[0]) + ssagen.AddrAuto(&p.From, v.Args[0]) p.To.Type = obj.TYPE_REG p.To.Reg = v.Reg() case ssa.OpStoreReg: @@ -232,7 +232,7 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { p := s.Prog(storeByType(v.Type)) p.From.Type = obj.TYPE_REG p.From.Reg = v.Args[0].Reg() - gc.AddrAuto(&p.To, v) + ssagen.AddrAuto(&p.To, v) case ssa.OpSP, ssa.OpSB, ssa.OpGetG: // nothing to do case ssa.OpRISCV64MOVBreg, ssa.OpRISCV64MOVHreg, ssa.OpRISCV64MOVWreg, @@ -323,10 +323,10 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { v.Fatalf("aux is of unknown type %T", v.Aux) case *obj.LSym: wantreg = "SB" - gc.AddAux(&p.From, v) + ssagen.AddAux(&p.From, v) case *ir.Name: wantreg = "SP" - gc.AddAux(&p.From, v) + ssagen.AddAux(&p.From, v) case nil: // No sym, just MOVW $off(SP), R wantreg = "SP" @@ -342,7 +342,7 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { p := s.Prog(v.Op.Asm()) p.From.Type = obj.TYPE_MEM p.From.Reg = v.Args[0].Reg() - gc.AddAux(&p.From, v) + ssagen.AddAux(&p.From, v) p.To.Type = obj.TYPE_REG p.To.Reg = v.Reg() case ssa.OpRISCV64MOVBstore, ssa.OpRISCV64MOVHstore, ssa.OpRISCV64MOVWstore, ssa.OpRISCV64MOVDstore, @@ -352,14 +352,14 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { p.From.Reg = v.Args[1].Reg() p.To.Type = obj.TYPE_MEM p.To.Reg = v.Args[0].Reg() - gc.AddAux(&p.To, v) + ssagen.AddAux(&p.To, v) case ssa.OpRISCV64MOVBstorezero, ssa.OpRISCV64MOVHstorezero, ssa.OpRISCV64MOVWstorezero, ssa.OpRISCV64MOVDstorezero: p := s.Prog(v.Op.Asm()) p.From.Type = obj.TYPE_REG p.From.Reg = riscv.REG_ZERO p.To.Type = obj.TYPE_MEM p.To.Reg = v.Args[0].Reg() - gc.AddAux(&p.To, v) + ssagen.AddAux(&p.To, v) case ssa.OpRISCV64SEQZ, ssa.OpRISCV64SNEZ: p := s.Prog(v.Op.Asm()) p.From.Type = obj.TYPE_REG @@ -377,7 +377,7 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { p := s.Prog(obj.ACALL) p.To.Type = obj.TYPE_MEM p.To.Name = obj.NAME_EXTERN - p.To.Sym = gc.BoundsCheckFunc[v.AuxInt] + p.To.Sym = ssagen.BoundsCheckFunc[v.AuxInt] s.UseArgs(16) // space used in callee args area by assembly stubs case ssa.OpRISCV64LoweredAtomicLoad8: @@ -585,7 +585,7 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { p := s.Prog(riscv.AMOVB) p.From.Type = obj.TYPE_MEM p.From.Reg = v.Args[0].Reg() - gc.AddAux(&p.From, v) + ssagen.AddAux(&p.From, v) p.To.Type = obj.TYPE_REG p.To.Reg = riscv.REG_ZERO if base.Debug.Nil != 0 && v.Pos.Line() > 1 { // v.Pos == 1 in generated wrappers @@ -594,7 +594,7 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { case ssa.OpRISCV64LoweredGetClosurePtr: // Closure pointer is S4 (riscv.REG_CTXT). - gc.CheckLoweredGetClosurePtr(v) + ssagen.CheckLoweredGetClosurePtr(v) case ssa.OpRISCV64LoweredGetCallerSP: // caller's SP is FixedFrameSize below the address of the first arg @@ -644,7 +644,7 @@ var blockBranch = [...]obj.As{ ssa.BlockRISCV64BNEZ: riscv.ABNEZ, } -func ssaGenBlock(s *gc.SSAGenState, b, next *ssa.Block) { +func ssaGenBlock(s *ssagen.State, b, next *ssa.Block) { s.SetPos(b.Pos) switch b.Kind { @@ -657,17 +657,17 @@ func ssaGenBlock(s *gc.SSAGenState, b, next *ssa.Block) { p.From.Type = obj.TYPE_REG p.From.Reg = riscv.REG_ZERO p.Reg = riscv.REG_A0 - s.Branches = append(s.Branches, gc.Branch{P: p, B: b.Succs[1].Block()}) + s.Branches = append(s.Branches, ssagen.Branch{P: p, B: b.Succs[1].Block()}) if b.Succs[0].Block() != next { p := s.Prog(obj.AJMP) p.To.Type = obj.TYPE_BRANCH - s.Branches = append(s.Branches, gc.Branch{P: p, B: b.Succs[0].Block()}) + s.Branches = append(s.Branches, ssagen.Branch{P: p, B: b.Succs[0].Block()}) } case ssa.BlockPlain: if b.Succs[0].Block() != next { p := s.Prog(obj.AJMP) p.To.Type = obj.TYPE_BRANCH - s.Branches = append(s.Branches, gc.Branch{P: p, B: b.Succs[0].Block()}) + s.Branches = append(s.Branches, ssagen.Branch{P: p, B: b.Succs[0].Block()}) } case ssa.BlockExit: case ssa.BlockRet: diff --git a/src/cmd/compile/internal/s390x/galign.go b/src/cmd/compile/internal/s390x/galign.go index cb68fd36c1..b004a2db0a 100644 --- a/src/cmd/compile/internal/s390x/galign.go +++ b/src/cmd/compile/internal/s390x/galign.go @@ -5,11 +5,11 @@ package s390x import ( - "cmd/compile/internal/gc" + "cmd/compile/internal/ssagen" "cmd/internal/obj/s390x" ) -func Init(arch *gc.Arch) { +func Init(arch *ssagen.ArchInfo) { arch.LinkArch = &s390x.Links390x arch.REGSP = s390x.REGSP arch.MAXWIDTH = 1 << 50 diff --git a/src/cmd/compile/internal/s390x/ssa.go b/src/cmd/compile/internal/s390x/ssa.go index dc01401348..d4c7a286e2 100644 --- a/src/cmd/compile/internal/s390x/ssa.go +++ b/src/cmd/compile/internal/s390x/ssa.go @@ -8,16 +8,16 @@ import ( "math" "cmd/compile/internal/base" - "cmd/compile/internal/gc" "cmd/compile/internal/logopt" "cmd/compile/internal/ssa" + "cmd/compile/internal/ssagen" "cmd/compile/internal/types" "cmd/internal/obj" "cmd/internal/obj/s390x" ) // markMoves marks any MOVXconst ops that need to avoid clobbering flags. -func ssaMarkMoves(s *gc.SSAGenState, b *ssa.Block) { +func ssaMarkMoves(s *ssagen.State, b *ssa.Block) { flive := b.FlagsLiveAtEnd for _, c := range b.ControlValues() { flive = c.Type.IsFlags() || flive @@ -135,7 +135,7 @@ func moveByType(t *types.Type) obj.As { // dest := dest(To) op src(From) // and also returns the created obj.Prog so it // may be further adjusted (offset, scale, etc). -func opregreg(s *gc.SSAGenState, op obj.As, dest, src int16) *obj.Prog { +func opregreg(s *ssagen.State, op obj.As, dest, src int16) *obj.Prog { p := s.Prog(op) p.From.Type = obj.TYPE_REG p.To.Type = obj.TYPE_REG @@ -148,7 +148,7 @@ func opregreg(s *gc.SSAGenState, op obj.As, dest, src int16) *obj.Prog { // dest := src(From) op off // and also returns the created obj.Prog so it // may be further adjusted (offset, scale, etc). -func opregregimm(s *gc.SSAGenState, op obj.As, dest, src int16, off int64) *obj.Prog { +func opregregimm(s *ssagen.State, op obj.As, dest, src int16, off int64) *obj.Prog { p := s.Prog(op) p.From.Type = obj.TYPE_CONST p.From.Offset = off @@ -158,7 +158,7 @@ func opregregimm(s *gc.SSAGenState, op obj.As, dest, src int16, off int64) *obj. return p } -func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { +func ssaGenValue(s *ssagen.State, v *ssa.Value) { switch v.Op { case ssa.OpS390XSLD, ssa.OpS390XSLW, ssa.OpS390XSRD, ssa.OpS390XSRW, @@ -395,14 +395,14 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { p.From.Type = obj.TYPE_ADDR p.From.Reg = r p.From.Index = i - gc.AddAux(&p.From, v) + ssagen.AddAux(&p.From, v) p.To.Type = obj.TYPE_REG p.To.Reg = v.Reg() case ssa.OpS390XMOVDaddr: p := s.Prog(s390x.AMOVD) p.From.Type = obj.TYPE_ADDR p.From.Reg = v.Args[0].Reg() - gc.AddAux(&p.From, v) + ssagen.AddAux(&p.From, v) p.To.Type = obj.TYPE_REG p.To.Reg = v.Reg() case ssa.OpS390XCMP, ssa.OpS390XCMPW, ssa.OpS390XCMPU, ssa.OpS390XCMPWU: @@ -448,7 +448,7 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { p := s.Prog(v.Op.Asm()) p.From.Type = obj.TYPE_MEM p.From.Reg = v.Args[1].Reg() - gc.AddAux(&p.From, v) + ssagen.AddAux(&p.From, v) p.To.Type = obj.TYPE_REG p.To.Reg = r case ssa.OpS390XMOVDload, @@ -459,7 +459,7 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { p := s.Prog(v.Op.Asm()) p.From.Type = obj.TYPE_MEM p.From.Reg = v.Args[0].Reg() - gc.AddAux(&p.From, v) + ssagen.AddAux(&p.From, v) p.To.Type = obj.TYPE_REG p.To.Reg = v.Reg() case ssa.OpS390XMOVBZloadidx, ssa.OpS390XMOVHZloadidx, ssa.OpS390XMOVWZloadidx, @@ -476,7 +476,7 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { p.From.Reg = r p.From.Scale = 1 p.From.Index = i - gc.AddAux(&p.From, v) + ssagen.AddAux(&p.From, v) p.To.Type = obj.TYPE_REG p.To.Reg = v.Reg() case ssa.OpS390XMOVBstore, ssa.OpS390XMOVHstore, ssa.OpS390XMOVWstore, ssa.OpS390XMOVDstore, @@ -487,7 +487,7 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { p.From.Reg = v.Args[1].Reg() p.To.Type = obj.TYPE_MEM p.To.Reg = v.Args[0].Reg() - gc.AddAux(&p.To, v) + ssagen.AddAux(&p.To, v) case ssa.OpS390XMOVBstoreidx, ssa.OpS390XMOVHstoreidx, ssa.OpS390XMOVWstoreidx, ssa.OpS390XMOVDstoreidx, ssa.OpS390XMOVHBRstoreidx, ssa.OpS390XMOVWBRstoreidx, ssa.OpS390XMOVDBRstoreidx, ssa.OpS390XFMOVSstoreidx, ssa.OpS390XFMOVDstoreidx: @@ -503,7 +503,7 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { p.To.Reg = r p.To.Scale = 1 p.To.Index = i - gc.AddAux(&p.To, v) + ssagen.AddAux(&p.To, v) case ssa.OpS390XMOVDstoreconst, ssa.OpS390XMOVWstoreconst, ssa.OpS390XMOVHstoreconst, ssa.OpS390XMOVBstoreconst: p := s.Prog(v.Op.Asm()) p.From.Type = obj.TYPE_CONST @@ -511,7 +511,7 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { p.From.Offset = sc.Val() p.To.Type = obj.TYPE_MEM p.To.Reg = v.Args[0].Reg() - gc.AddAux2(&p.To, v, sc.Off()) + ssagen.AddAux2(&p.To, v, sc.Off()) case ssa.OpS390XMOVBreg, ssa.OpS390XMOVHreg, ssa.OpS390XMOVWreg, ssa.OpS390XMOVBZreg, ssa.OpS390XMOVHZreg, ssa.OpS390XMOVWZreg, ssa.OpS390XLDGR, ssa.OpS390XLGDR, @@ -530,7 +530,7 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { p.From.Offset = sc.Val() p.To.Type = obj.TYPE_MEM p.To.Reg = v.Args[0].Reg() - gc.AddAux2(&p.To, v, sc.Off()) + ssagen.AddAux2(&p.To, v, sc.Off()) case ssa.OpCopy: if v.Type.IsMemory() { return @@ -546,7 +546,7 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { return } p := s.Prog(loadByType(v.Type)) - gc.AddrAuto(&p.From, v.Args[0]) + ssagen.AddrAuto(&p.From, v.Args[0]) p.To.Type = obj.TYPE_REG p.To.Reg = v.Reg() case ssa.OpStoreReg: @@ -557,10 +557,10 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { p := s.Prog(storeByType(v.Type)) p.From.Type = obj.TYPE_REG p.From.Reg = v.Args[0].Reg() - gc.AddrAuto(&p.To, v) + ssagen.AddrAuto(&p.To, v) case ssa.OpS390XLoweredGetClosurePtr: // Closure pointer is R12 (already) - gc.CheckLoweredGetClosurePtr(v) + ssagen.CheckLoweredGetClosurePtr(v) case ssa.OpS390XLoweredRound32F, ssa.OpS390XLoweredRound64F: // input is already rounded case ssa.OpS390XLoweredGetG: @@ -593,7 +593,7 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { p := s.Prog(obj.ACALL) p.To.Type = obj.TYPE_MEM p.To.Name = obj.NAME_EXTERN - p.To.Sym = gc.BoundsCheckFunc[v.AuxInt] + p.To.Sym = ssagen.BoundsCheckFunc[v.AuxInt] s.UseArgs(16) // space used in callee args area by assembly stubs case ssa.OpS390XFLOGR, ssa.OpS390XPOPCNT, ssa.OpS390XNEG, ssa.OpS390XNEGW, @@ -637,7 +637,7 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { p := s.Prog(s390x.AMOVBZ) p.From.Type = obj.TYPE_MEM p.From.Reg = v.Args[0].Reg() - gc.AddAux(&p.From, v) + ssagen.AddAux(&p.From, v) p.To.Type = obj.TYPE_REG p.To.Reg = s390x.REGTMP if logopt.Enabled() { @@ -672,7 +672,7 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { p.Reg = v.Args[len(v.Args)-2].Reg() p.To.Type = obj.TYPE_MEM p.To.Reg = v.Args[0].Reg() - gc.AddAux(&p.To, v) + ssagen.AddAux(&p.To, v) case ssa.OpS390XLoweredMove: // Inputs must be valid pointers to memory, // so adjust arg0 and arg1 as part of the expansion. @@ -764,7 +764,7 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { p := s.Prog(v.Op.Asm()) p.From.Type = obj.TYPE_MEM p.From.Reg = v.Args[0].Reg() - gc.AddAux(&p.From, v) + ssagen.AddAux(&p.From, v) p.To.Type = obj.TYPE_REG p.To.Reg = v.Reg0() case ssa.OpS390XMOVBatomicstore, ssa.OpS390XMOVWatomicstore, ssa.OpS390XMOVDatomicstore: @@ -773,7 +773,7 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { p.From.Reg = v.Args[1].Reg() p.To.Type = obj.TYPE_MEM p.To.Reg = v.Args[0].Reg() - gc.AddAux(&p.To, v) + ssagen.AddAux(&p.To, v) case ssa.OpS390XLAN, ssa.OpS390XLAO: // LA(N|O) Ry, TMP, 0(Rx) op := s.Prog(v.Op.Asm()) @@ -808,7 +808,7 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { p.From.Reg = v.Args[1].Reg() p.To.Type = obj.TYPE_MEM p.To.Reg = v.Args[0].Reg() - gc.AddAux(&p.To, v) + ssagen.AddAux(&p.To, v) case ssa.OpS390XLoweredAtomicCas32, ssa.OpS390XLoweredAtomicCas64: // Convert the flags output of CS{,G} into a bool. // CS{,G} arg1, arg2, arg0 @@ -824,7 +824,7 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { cs.Reg = v.Args[2].Reg() // new cs.To.Type = obj.TYPE_MEM cs.To.Reg = v.Args[0].Reg() - gc.AddAux(&cs.To, v) + ssagen.AddAux(&cs.To, v) // MOVD $0, ret movd := s.Prog(s390x.AMOVD) @@ -859,7 +859,7 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { load.From.Reg = v.Args[0].Reg() load.To.Type = obj.TYPE_REG load.To.Reg = v.Reg0() - gc.AddAux(&load.From, v) + ssagen.AddAux(&load.From, v) // CS{,G} ret, arg1, arg0 cs := s.Prog(v.Op.Asm()) @@ -868,7 +868,7 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { cs.Reg = v.Args[1].Reg() // new cs.To.Type = obj.TYPE_MEM cs.To.Reg = v.Args[0].Reg() - gc.AddAux(&cs.To, v) + ssagen.AddAux(&cs.To, v) // BNE cs bne := s.Prog(s390x.ABNE) @@ -908,14 +908,14 @@ func blockAsm(b *ssa.Block) obj.As { panic("unreachable") } -func ssaGenBlock(s *gc.SSAGenState, b, next *ssa.Block) { +func ssaGenBlock(s *ssagen.State, b, next *ssa.Block) { // Handle generic blocks first. switch b.Kind { case ssa.BlockPlain: if b.Succs[0].Block() != next { p := s.Prog(s390x.ABR) p.To.Type = obj.TYPE_BRANCH - s.Branches = append(s.Branches, gc.Branch{P: p, B: b.Succs[0].Block()}) + s.Branches = append(s.Branches, ssagen.Branch{P: p, B: b.Succs[0].Block()}) } return case ssa.BlockDefer: diff --git a/src/cmd/compile/internal/ssagen/abi.go b/src/cmd/compile/internal/ssagen/abi.go new file mode 100644 index 0000000000..af08fcb7c3 --- /dev/null +++ b/src/cmd/compile/internal/ssagen/abi.go @@ -0,0 +1,367 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:generate go run mkbuiltin.go + +package ssagen + +import ( + "fmt" + "io/ioutil" + "log" + "os" + "strings" + + "cmd/compile/internal/base" + "cmd/compile/internal/escape" + "cmd/compile/internal/ir" + "cmd/compile/internal/typecheck" + "cmd/compile/internal/types" + "cmd/internal/obj" + "cmd/internal/objabi" +) + +// useNewABIWrapGen returns TRUE if the compiler should generate an +// ABI wrapper for the function 'f'. +func useABIWrapGen(f *ir.Func) bool { + if !base.Flag.ABIWrap { + return false + } + + // Support limit option for bisecting. + if base.Flag.ABIWrapLimit == 1 { + return false + } + if base.Flag.ABIWrapLimit < 1 { + return true + } + base.Flag.ABIWrapLimit-- + if base.Debug.ABIWrap != 0 && base.Flag.ABIWrapLimit == 1 { + fmt.Fprintf(os.Stderr, "=-= limit reached after new wrapper for %s\n", + f.LSym.Name) + } + + return true +} + +// symabiDefs and symabiRefs record the defined and referenced ABIs of +// symbols required by non-Go code. These are keyed by link symbol +// name, where the local package prefix is always `"".` +var symabiDefs, symabiRefs map[string]obj.ABI + +func CgoSymABIs() { + // The linker expects an ABI0 wrapper for all cgo-exported + // functions. + for _, prag := range typecheck.Target.CgoPragmas { + switch prag[0] { + case "cgo_export_static", "cgo_export_dynamic": + if symabiRefs == nil { + symabiRefs = make(map[string]obj.ABI) + } + symabiRefs[prag[1]] = obj.ABI0 + } + } +} + +// ReadSymABIs reads a symabis file that specifies definitions and +// references of text symbols by ABI. +// +// The symabis format is a set of lines, where each line is a sequence +// of whitespace-separated fields. The first field is a verb and is +// either "def" for defining a symbol ABI or "ref" for referencing a +// symbol using an ABI. For both "def" and "ref", the second field is +// the symbol name and the third field is the ABI name, as one of the +// named cmd/internal/obj.ABI constants. +func ReadSymABIs(file, myimportpath string) { + data, err := ioutil.ReadFile(file) + if err != nil { + log.Fatalf("-symabis: %v", err) + } + + symabiDefs = make(map[string]obj.ABI) + symabiRefs = make(map[string]obj.ABI) + + localPrefix := "" + if myimportpath != "" { + // Symbols in this package may be written either as + // "".X or with the package's import path already in + // the symbol. + localPrefix = objabi.PathToPrefix(myimportpath) + "." + } + + for lineNum, line := range strings.Split(string(data), "\n") { + lineNum++ // 1-based + line = strings.TrimSpace(line) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + + parts := strings.Fields(line) + switch parts[0] { + case "def", "ref": + // Parse line. + if len(parts) != 3 { + log.Fatalf(`%s:%d: invalid symabi: syntax is "%s sym abi"`, file, lineNum, parts[0]) + } + sym, abistr := parts[1], parts[2] + abi, valid := obj.ParseABI(abistr) + if !valid { + log.Fatalf(`%s:%d: invalid symabi: unknown abi "%s"`, file, lineNum, abistr) + } + + // If the symbol is already prefixed with + // myimportpath, rewrite it to start with "" + // so it matches the compiler's internal + // symbol names. + if localPrefix != "" && strings.HasPrefix(sym, localPrefix) { + sym = `"".` + sym[len(localPrefix):] + } + + // Record for later. + if parts[0] == "def" { + symabiDefs[sym] = abi + } else { + symabiRefs[sym] = abi + } + default: + log.Fatalf(`%s:%d: invalid symabi type "%s"`, file, lineNum, parts[0]) + } + } +} + +// InitLSym defines f's obj.LSym and initializes it based on the +// properties of f. This includes setting the symbol flags and ABI and +// creating and initializing related DWARF symbols. +// +// InitLSym must be called exactly once per function and must be +// called for both functions with bodies and functions without bodies. +// For body-less functions, we only create the LSym; for functions +// with bodies call a helper to setup up / populate the LSym. +func InitLSym(f *ir.Func, hasBody bool) { + // FIXME: for new-style ABI wrappers, we set up the lsym at the + // point the wrapper is created. + if f.LSym != nil && base.Flag.ABIWrap { + return + } + selectLSym(f, hasBody) + if hasBody { + setupTextLSym(f, 0) + } +} + +// selectLSym sets up the LSym for a given function, and +// makes calls to helpers to create ABI wrappers if needed. +func selectLSym(f *ir.Func, hasBody bool) { + if f.LSym != nil { + base.Fatalf("Func.initLSym called twice") + } + + if nam := f.Nname; !ir.IsBlank(nam) { + + var wrapperABI obj.ABI + needABIWrapper := false + defABI, hasDefABI := symabiDefs[nam.Sym().LinksymName()] + if hasDefABI && defABI == obj.ABI0 { + // Symbol is defined as ABI0. Create an + // Internal -> ABI0 wrapper. + f.LSym = nam.Sym().LinksymABI0() + needABIWrapper, wrapperABI = true, obj.ABIInternal + } else { + f.LSym = nam.Sym().Linksym() + // No ABI override. Check that the symbol is + // using the expected ABI. + want := obj.ABIInternal + if f.LSym.ABI() != want { + base.Fatalf("function symbol %s has the wrong ABI %v, expected %v", f.LSym.Name, f.LSym.ABI(), want) + } + } + if f.Pragma&ir.Systemstack != 0 { + f.LSym.Set(obj.AttrCFunc, true) + } + + isLinknameExported := nam.Sym().Linkname != "" && (hasBody || hasDefABI) + if abi, ok := symabiRefs[f.LSym.Name]; (ok && abi == obj.ABI0) || isLinknameExported { + // Either 1) this symbol is definitely + // referenced as ABI0 from this package; or 2) + // this symbol is defined in this package but + // given a linkname, indicating that it may be + // referenced from another package. Create an + // ABI0 -> Internal wrapper so it can be + // called as ABI0. In case 2, it's important + // that we know it's defined in this package + // since other packages may "pull" symbols + // using linkname and we don't want to create + // duplicate ABI wrappers. + if f.LSym.ABI() != obj.ABI0 { + needABIWrapper, wrapperABI = true, obj.ABI0 + } + } + + if needABIWrapper { + if !useABIWrapGen(f) { + // Fallback: use alias instead. FIXME. + + // These LSyms have the same name as the + // native function, so we create them directly + // rather than looking them up. The uniqueness + // of f.lsym ensures uniqueness of asym. + asym := &obj.LSym{ + Name: f.LSym.Name, + Type: objabi.SABIALIAS, + R: []obj.Reloc{{Sym: f.LSym}}, // 0 size, so "informational" + } + asym.SetABI(wrapperABI) + asym.Set(obj.AttrDuplicateOK, true) + base.Ctxt.ABIAliases = append(base.Ctxt.ABIAliases, asym) + } else { + if base.Debug.ABIWrap != 0 { + fmt.Fprintf(os.Stderr, "=-= %v to %v wrapper for %s.%s\n", + wrapperABI, 1-wrapperABI, types.LocalPkg.Path, f.LSym.Name) + } + makeABIWrapper(f, wrapperABI) + } + } + } +} + +// makeABIWrapper creates a new function that wraps a cross-ABI call +// to "f". The wrapper is marked as an ABIWRAPPER. +func makeABIWrapper(f *ir.Func, wrapperABI obj.ABI) { + + // Q: is this needed? + savepos := base.Pos + savedclcontext := typecheck.DeclContext + savedcurfn := ir.CurFunc + + base.Pos = base.AutogeneratedPos + typecheck.DeclContext = ir.PEXTERN + + // At the moment we don't support wrapping a method, we'd need machinery + // below to handle the receiver. Panic if we see this scenario. + ft := f.Nname.Ntype.Type() + if ft.NumRecvs() != 0 { + panic("makeABIWrapper support for wrapping methods not implemented") + } + + // Manufacture a new func type to use for the wrapper. + var noReceiver *ir.Field + tfn := ir.NewFuncType(base.Pos, + noReceiver, + typecheck.NewFuncParams(ft.Params(), true), + typecheck.NewFuncParams(ft.Results(), false)) + + // Reuse f's types.Sym to create a new ODCLFUNC/function. + fn := typecheck.DeclFunc(f.Nname.Sym(), tfn) + fn.SetDupok(true) + fn.SetWrapper(true) // ignore frame for panic+recover matching + + // Select LSYM now. + asym := base.Ctxt.LookupABI(f.LSym.Name, wrapperABI) + asym.Type = objabi.STEXT + if fn.LSym != nil { + panic("unexpected") + } + fn.LSym = asym + + // ABI0-to-ABIInternal wrappers will be mainly loading params from + // stack into registers (and/or storing stack locations back to + // registers after the wrapped call); in most cases they won't + // need to allocate stack space, so it should be OK to mark them + // as NOSPLIT in these cases. In addition, my assumption is that + // functions written in assembly are NOSPLIT in most (but not all) + // cases. In the case of an ABIInternal target that has too many + // parameters to fit into registers, the wrapper would need to + // allocate stack space, but this seems like an unlikely scenario. + // Hence: mark these wrappers NOSPLIT. + // + // ABIInternal-to-ABI0 wrappers on the other hand will be taking + // things in registers and pushing them onto the stack prior to + // the ABI0 call, meaning that they will always need to allocate + // stack space. If the compiler marks them as NOSPLIT this seems + // as though it could lead to situations where the the linker's + // nosplit-overflow analysis would trigger a link failure. On the + // other hand if they not tagged NOSPLIT then this could cause + // problems when building the runtime (since there may be calls to + // asm routine in cases where it's not safe to grow the stack). In + // most cases the wrapper would be (in effect) inlined, but are + // there (perhaps) indirect calls from the runtime that could run + // into trouble here. + // FIXME: at the moment all.bash does not pass when I leave out + // NOSPLIT for these wrappers, so all are currently tagged with NOSPLIT. + setupTextLSym(fn, obj.NOSPLIT|obj.ABIWRAPPER) + + // Generate call. Use tail call if no params and no returns, + // but a regular call otherwise. + // + // Note: ideally we would be using a tail call in cases where + // there are params but no returns for ABI0->ABIInternal wrappers, + // provided that all params fit into registers (e.g. we don't have + // to allocate any stack space). Doing this will require some + // extra work in typecheck/walk/ssa, might want to add a new node + // OTAILCALL or something to this effect. + var tail ir.Node + if tfn.Type().NumResults() == 0 && tfn.Type().NumParams() == 0 && tfn.Type().NumRecvs() == 0 { + tail = ir.NewBranchStmt(base.Pos, ir.ORETJMP, f.Nname.Sym()) + } else { + call := ir.NewCallExpr(base.Pos, ir.OCALL, f.Nname, nil) + call.Args.Set(ir.ParamNames(tfn.Type())) + call.IsDDD = tfn.Type().IsVariadic() + tail = call + if tfn.Type().NumResults() > 0 { + n := ir.NewReturnStmt(base.Pos, nil) + n.Results = []ir.Node{call} + tail = n + } + } + fn.Body.Append(tail) + + typecheck.FinishFuncBody() + if base.Debug.DclStack != 0 { + types.CheckDclstack() + } + + typecheck.Func(fn) + ir.CurFunc = fn + typecheck.Stmts(fn.Body) + + escape.Batch([]*ir.Func{fn}, false) + + typecheck.Target.Decls = append(typecheck.Target.Decls, fn) + + // Restore previous context. + base.Pos = savepos + typecheck.DeclContext = savedclcontext + ir.CurFunc = savedcurfn +} + +// setupTextLsym initializes the LSym for a with-body text symbol. +func setupTextLSym(f *ir.Func, flag int) { + if f.Dupok() { + flag |= obj.DUPOK + } + if f.Wrapper() { + flag |= obj.WRAPPER + } + if f.Needctxt() { + flag |= obj.NEEDCTXT + } + if f.Pragma&ir.Nosplit != 0 { + flag |= obj.NOSPLIT + } + if f.ReflectMethod() { + flag |= obj.REFLECTMETHOD + } + + // Clumsy but important. + // See test/recover.go for test cases and src/reflect/value.go + // for the actual functions being considered. + if base.Ctxt.Pkgpath == "reflect" { + switch f.Sym().Name { + case "callReflect", "callMethod": + flag |= obj.WRAPPER + } + } + + base.Ctxt.InitTextSym(f.LSym, flag) +} diff --git a/src/cmd/compile/internal/ssagen/arch.go b/src/cmd/compile/internal/ssagen/arch.go new file mode 100644 index 0000000000..cc50ab36b5 --- /dev/null +++ b/src/cmd/compile/internal/ssagen/arch.go @@ -0,0 +1,42 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ssagen + +import ( + "cmd/compile/internal/objw" + "cmd/compile/internal/ssa" + "cmd/internal/obj" +) + +var Arch ArchInfo + +// interface to back end + +type ArchInfo struct { + LinkArch *obj.LinkArch + + REGSP int + MAXWIDTH int64 + SoftFloat bool + + PadFrame func(int64) int64 + + // ZeroRange zeroes a range of memory on stack. It is only inserted + // at function entry, and it is ok to clobber registers. + ZeroRange func(*objw.Progs, *obj.Prog, int64, int64, *uint32) *obj.Prog + + Ginsnop func(*objw.Progs) *obj.Prog + Ginsnopdefer func(*objw.Progs) *obj.Prog // special ginsnop for deferreturn + + // SSAMarkMoves marks any MOVXconst ops that need to avoid clobbering flags. + SSAMarkMoves func(*State, *ssa.Block) + + // SSAGenValue emits Prog(s) for the Value. + SSAGenValue func(*State, *ssa.Value) + + // SSAGenBlock emits end-of-block Progs. SSAGenValue should be called + // for all values in the block before SSAGenBlock. + SSAGenBlock func(s *State, b, next *ssa.Block) +} diff --git a/src/cmd/compile/internal/ssagen/nowb.go b/src/cmd/compile/internal/ssagen/nowb.go new file mode 100644 index 0000000000..7b2e68c8e7 --- /dev/null +++ b/src/cmd/compile/internal/ssagen/nowb.go @@ -0,0 +1,200 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ssagen + +import ( + "bytes" + "fmt" + + "cmd/compile/internal/base" + "cmd/compile/internal/ir" + "cmd/compile/internal/typecheck" + "cmd/compile/internal/types" + "cmd/internal/obj" + "cmd/internal/src" +) + +func EnableNoWriteBarrierRecCheck() { + nowritebarrierrecCheck = newNowritebarrierrecChecker() +} + +func NoWriteBarrierRecCheck() { + // Write barriers are now known. Check the + // call graph. + nowritebarrierrecCheck.check() + nowritebarrierrecCheck = nil +} + +var nowritebarrierrecCheck *nowritebarrierrecChecker + +type nowritebarrierrecChecker struct { + // extraCalls contains extra function calls that may not be + // visible during later analysis. It maps from the ODCLFUNC of + // the caller to a list of callees. + extraCalls map[*ir.Func][]nowritebarrierrecCall + + // curfn is the current function during AST walks. + curfn *ir.Func +} + +type nowritebarrierrecCall struct { + target *ir.Func // caller or callee + lineno src.XPos // line of call +} + +// newNowritebarrierrecChecker creates a nowritebarrierrecChecker. It +// must be called before transformclosure and walk. +func newNowritebarrierrecChecker() *nowritebarrierrecChecker { + c := &nowritebarrierrecChecker{ + extraCalls: make(map[*ir.Func][]nowritebarrierrecCall), + } + + // Find all systemstack calls and record their targets. In + // general, flow analysis can't see into systemstack, but it's + // important to handle it for this check, so we model it + // directly. This has to happen before transformclosure since + // it's a lot harder to work out the argument after. + for _, n := range typecheck.Target.Decls { + if n.Op() != ir.ODCLFUNC { + continue + } + c.curfn = n.(*ir.Func) + ir.Visit(n, c.findExtraCalls) + } + c.curfn = nil + return c +} + +func (c *nowritebarrierrecChecker) findExtraCalls(nn ir.Node) { + if nn.Op() != ir.OCALLFUNC { + return + } + n := nn.(*ir.CallExpr) + if n.X == nil || n.X.Op() != ir.ONAME { + return + } + fn := n.X.(*ir.Name) + if fn.Class_ != ir.PFUNC || fn.Name().Defn == nil { + return + } + if !types.IsRuntimePkg(fn.Sym().Pkg) || fn.Sym().Name != "systemstack" { + return + } + + var callee *ir.Func + arg := n.Args[0] + switch arg.Op() { + case ir.ONAME: + arg := arg.(*ir.Name) + callee = arg.Name().Defn.(*ir.Func) + case ir.OCLOSURE: + arg := arg.(*ir.ClosureExpr) + callee = arg.Func + default: + base.Fatalf("expected ONAME or OCLOSURE node, got %+v", arg) + } + if callee.Op() != ir.ODCLFUNC { + base.Fatalf("expected ODCLFUNC node, got %+v", callee) + } + c.extraCalls[c.curfn] = append(c.extraCalls[c.curfn], nowritebarrierrecCall{callee, n.Pos()}) +} + +// recordCall records a call from ODCLFUNC node "from", to function +// symbol "to" at position pos. +// +// This should be done as late as possible during compilation to +// capture precise call graphs. The target of the call is an LSym +// because that's all we know after we start SSA. +// +// This can be called concurrently for different from Nodes. +func (c *nowritebarrierrecChecker) recordCall(fn *ir.Func, to *obj.LSym, pos src.XPos) { + // We record this information on the *Func so this is concurrent-safe. + if fn.NWBRCalls == nil { + fn.NWBRCalls = new([]ir.SymAndPos) + } + *fn.NWBRCalls = append(*fn.NWBRCalls, ir.SymAndPos{Sym: to, Pos: pos}) +} + +func (c *nowritebarrierrecChecker) check() { + // We walk the call graph as late as possible so we can + // capture all calls created by lowering, but this means we + // only get to see the obj.LSyms of calls. symToFunc lets us + // get back to the ODCLFUNCs. + symToFunc := make(map[*obj.LSym]*ir.Func) + // funcs records the back-edges of the BFS call graph walk. It + // maps from the ODCLFUNC of each function that must not have + // write barriers to the call that inhibits them. Functions + // that are directly marked go:nowritebarrierrec are in this + // map with a zero-valued nowritebarrierrecCall. This also + // acts as the set of marks for the BFS of the call graph. + funcs := make(map[*ir.Func]nowritebarrierrecCall) + // q is the queue of ODCLFUNC Nodes to visit in BFS order. + var q ir.NameQueue + + for _, n := range typecheck.Target.Decls { + if n.Op() != ir.ODCLFUNC { + continue + } + fn := n.(*ir.Func) + + symToFunc[fn.LSym] = fn + + // Make nowritebarrierrec functions BFS roots. + if fn.Pragma&ir.Nowritebarrierrec != 0 { + funcs[fn] = nowritebarrierrecCall{} + q.PushRight(fn.Nname) + } + // Check go:nowritebarrier functions. + if fn.Pragma&ir.Nowritebarrier != 0 && fn.WBPos.IsKnown() { + base.ErrorfAt(fn.WBPos, "write barrier prohibited") + } + } + + // Perform a BFS of the call graph from all + // go:nowritebarrierrec functions. + enqueue := func(src, target *ir.Func, pos src.XPos) { + if target.Pragma&ir.Yeswritebarrierrec != 0 { + // Don't flow into this function. + return + } + if _, ok := funcs[target]; ok { + // Already found a path to target. + return + } + + // Record the path. + funcs[target] = nowritebarrierrecCall{target: src, lineno: pos} + q.PushRight(target.Nname) + } + for !q.Empty() { + fn := q.PopLeft().Func + + // Check fn. + if fn.WBPos.IsKnown() { + var err bytes.Buffer + call := funcs[fn] + for call.target != nil { + fmt.Fprintf(&err, "\n\t%v: called by %v", base.FmtPos(call.lineno), call.target.Nname) + call = funcs[call.target] + } + base.ErrorfAt(fn.WBPos, "write barrier prohibited by caller; %v%s", fn.Nname, err.String()) + continue + } + + // Enqueue fn's calls. + for _, callee := range c.extraCalls[fn] { + enqueue(fn, callee.target, callee.lineno) + } + if fn.NWBRCalls == nil { + continue + } + for _, callee := range *fn.NWBRCalls { + target := symToFunc[callee.Sym] + if target != nil { + enqueue(fn, target, callee.Pos) + } + } + } +} diff --git a/src/cmd/compile/internal/ssagen/pgen.go b/src/cmd/compile/internal/ssagen/pgen.go new file mode 100644 index 0000000000..bc6be20d86 --- /dev/null +++ b/src/cmd/compile/internal/ssagen/pgen.go @@ -0,0 +1,279 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ssagen + +import ( + "internal/race" + "math/rand" + "sort" + "sync" + "time" + + "cmd/compile/internal/base" + "cmd/compile/internal/ir" + "cmd/compile/internal/objw" + "cmd/compile/internal/ssa" + "cmd/compile/internal/typecheck" + "cmd/compile/internal/types" + "cmd/internal/obj" + "cmd/internal/objabi" + "cmd/internal/src" + "cmd/internal/sys" +) + +// cmpstackvarlt reports whether the stack variable a sorts before b. +// +// Sort the list of stack variables. Autos after anything else, +// within autos, unused after used, within used, things with +// pointers first, zeroed things first, and then decreasing size. +// Because autos are laid out in decreasing addresses +// on the stack, pointers first, zeroed things first and decreasing size +// really means, in memory, things with pointers needing zeroing at +// the top of the stack and increasing in size. +// Non-autos sort on offset. +func cmpstackvarlt(a, b *ir.Name) bool { + if (a.Class_ == ir.PAUTO) != (b.Class_ == ir.PAUTO) { + return b.Class_ == ir.PAUTO + } + + if a.Class_ != ir.PAUTO { + return a.FrameOffset() < b.FrameOffset() + } + + if a.Used() != b.Used() { + return a.Used() + } + + ap := a.Type().HasPointers() + bp := b.Type().HasPointers() + if ap != bp { + return ap + } + + ap = a.Needzero() + bp = b.Needzero() + if ap != bp { + return ap + } + + if a.Type().Width != b.Type().Width { + return a.Type().Width > b.Type().Width + } + + return a.Sym().Name < b.Sym().Name +} + +// byStackvar implements sort.Interface for []*Node using cmpstackvarlt. +type byStackVar []*ir.Name + +func (s byStackVar) Len() int { return len(s) } +func (s byStackVar) Less(i, j int) bool { return cmpstackvarlt(s[i], s[j]) } +func (s byStackVar) Swap(i, j int) { s[i], s[j] = s[j], s[i] } + +func (s *ssafn) AllocFrame(f *ssa.Func) { + s.stksize = 0 + s.stkptrsize = 0 + fn := s.curfn + + // Mark the PAUTO's unused. + for _, ln := range fn.Dcl { + if ln.Class_ == ir.PAUTO { + ln.SetUsed(false) + } + } + + for _, l := range f.RegAlloc { + if ls, ok := l.(ssa.LocalSlot); ok { + ls.N.Name().SetUsed(true) + } + } + + scratchUsed := false + for _, b := range f.Blocks { + for _, v := range b.Values { + if n, ok := v.Aux.(*ir.Name); ok { + switch n.Class_ { + case ir.PPARAM, ir.PPARAMOUT: + // Don't modify nodfp; it is a global. + if n != ir.RegFP { + n.Name().SetUsed(true) + } + case ir.PAUTO: + n.Name().SetUsed(true) + } + } + if !scratchUsed { + scratchUsed = v.Op.UsesScratch() + } + + } + } + + if f.Config.NeedsFpScratch && scratchUsed { + s.scratchFpMem = typecheck.TempAt(src.NoXPos, s.curfn, types.Types[types.TUINT64]) + } + + sort.Sort(byStackVar(fn.Dcl)) + + // Reassign stack offsets of the locals that are used. + lastHasPtr := false + for i, n := range fn.Dcl { + if n.Op() != ir.ONAME || n.Class_ != ir.PAUTO { + continue + } + if !n.Used() { + fn.Dcl = fn.Dcl[:i] + break + } + + types.CalcSize(n.Type()) + w := n.Type().Width + if w >= types.MaxWidth || w < 0 { + base.Fatalf("bad width") + } + if w == 0 && lastHasPtr { + // Pad between a pointer-containing object and a zero-sized object. + // This prevents a pointer to the zero-sized object from being interpreted + // as a pointer to the pointer-containing object (and causing it + // to be scanned when it shouldn't be). See issue 24993. + w = 1 + } + s.stksize += w + s.stksize = types.Rnd(s.stksize, int64(n.Type().Align)) + if n.Type().HasPointers() { + s.stkptrsize = s.stksize + lastHasPtr = true + } else { + lastHasPtr = false + } + if Arch.LinkArch.InFamily(sys.MIPS, sys.MIPS64, sys.ARM, sys.ARM64, sys.PPC64, sys.S390X) { + s.stksize = types.Rnd(s.stksize, int64(types.PtrSize)) + } + n.SetFrameOffset(-s.stksize) + } + + s.stksize = types.Rnd(s.stksize, int64(types.RegSize)) + s.stkptrsize = types.Rnd(s.stkptrsize, int64(types.RegSize)) +} + +const maxStackSize = 1 << 30 + +// Compile builds an SSA backend function, +// uses it to generate a plist, +// and flushes that plist to machine code. +// worker indicates which of the backend workers is doing the processing. +func Compile(fn *ir.Func, worker int) { + f := buildssa(fn, worker) + // Note: check arg size to fix issue 25507. + if f.Frontend().(*ssafn).stksize >= maxStackSize || fn.Type().ArgWidth() >= maxStackSize { + largeStackFramesMu.Lock() + largeStackFrames = append(largeStackFrames, largeStack{locals: f.Frontend().(*ssafn).stksize, args: fn.Type().ArgWidth(), pos: fn.Pos()}) + largeStackFramesMu.Unlock() + return + } + pp := objw.NewProgs(fn, worker) + defer pp.Free() + genssa(f, pp) + // Check frame size again. + // The check above included only the space needed for local variables. + // After genssa, the space needed includes local variables and the callee arg region. + // We must do this check prior to calling pp.Flush. + // If there are any oversized stack frames, + // the assembler may emit inscrutable complaints about invalid instructions. + if pp.Text.To.Offset >= maxStackSize { + largeStackFramesMu.Lock() + locals := f.Frontend().(*ssafn).stksize + largeStackFrames = append(largeStackFrames, largeStack{locals: locals, args: fn.Type().ArgWidth(), callee: pp.Text.To.Offset - locals, pos: fn.Pos()}) + largeStackFramesMu.Unlock() + return + } + + pp.Flush() // assemble, fill in boilerplate, etc. + // fieldtrack must be called after pp.Flush. See issue 20014. + fieldtrack(pp.Text.From.Sym, fn.FieldTrack) +} + +func init() { + if race.Enabled { + rand.Seed(time.Now().UnixNano()) + } +} + +// StackOffset returns the stack location of a LocalSlot relative to the +// stack pointer, suitable for use in a DWARF location entry. This has nothing +// to do with its offset in the user variable. +func StackOffset(slot ssa.LocalSlot) int32 { + n := slot.N + var off int64 + switch n.Class_ { + case ir.PAUTO: + off = n.FrameOffset() + if base.Ctxt.FixedFrameSize() == 0 { + off -= int64(types.PtrSize) + } + if objabi.Framepointer_enabled || objabi.GOARCH == "arm64" { + // There is a word space for FP on ARM64 even if the frame pointer is disabled + off -= int64(types.PtrSize) + } + case ir.PPARAM, ir.PPARAMOUT: + off = n.FrameOffset() + base.Ctxt.FixedFrameSize() + } + return int32(off + slot.Off) +} + +// fieldtrack adds R_USEFIELD relocations to fnsym to record any +// struct fields that it used. +func fieldtrack(fnsym *obj.LSym, tracked map[*types.Sym]struct{}) { + if fnsym == nil { + return + } + if objabi.Fieldtrack_enabled == 0 || len(tracked) == 0 { + return + } + + trackSyms := make([]*types.Sym, 0, len(tracked)) + for sym := range tracked { + trackSyms = append(trackSyms, sym) + } + sort.Sort(symByName(trackSyms)) + for _, sym := range trackSyms { + r := obj.Addrel(fnsym) + r.Sym = sym.Linksym() + r.Type = objabi.R_USEFIELD + } +} + +type symByName []*types.Sym + +func (a symByName) Len() int { return len(a) } +func (a symByName) Less(i, j int) bool { return a[i].Name < a[j].Name } +func (a symByName) Swap(i, j int) { a[i], a[j] = a[j], a[i] } + +// largeStack is info about a function whose stack frame is too large (rare). +type largeStack struct { + locals int64 + args int64 + callee int64 + pos src.XPos +} + +var ( + largeStackFramesMu sync.Mutex // protects largeStackFrames + largeStackFrames []largeStack +) + +func CheckLargeStacks() { + // Check whether any of the functions we have compiled have gigantic stack frames. + sort.Slice(largeStackFrames, func(i, j int) bool { + return largeStackFrames[i].pos.Before(largeStackFrames[j].pos) + }) + for _, large := range largeStackFrames { + if large.callee != 0 { + base.ErrorfAt(large.pos, "stack frame too large (>1GB): %d MB locals + %d MB args + %d MB callee", large.locals>>20, large.args>>20, large.callee>>20) + } else { + base.ErrorfAt(large.pos, "stack frame too large (>1GB): %d MB locals + %d MB args", large.locals>>20, large.args>>20) + } + } +} diff --git a/src/cmd/compile/internal/ssagen/pgen_test.go b/src/cmd/compile/internal/ssagen/pgen_test.go new file mode 100644 index 0000000000..82d8447e9f --- /dev/null +++ b/src/cmd/compile/internal/ssagen/pgen_test.go @@ -0,0 +1,209 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ssagen + +import ( + "reflect" + "sort" + "testing" + + "cmd/compile/internal/ir" + "cmd/compile/internal/typecheck" + "cmd/compile/internal/types" + "cmd/internal/src" +) + +func typeWithoutPointers() *types.Type { + return types.NewStruct(types.NoPkg, []*types.Field{ + types.NewField(src.NoXPos, nil, types.New(types.TINT)), + }) +} + +func typeWithPointers() *types.Type { + return types.NewStruct(types.NoPkg, []*types.Field{ + types.NewField(src.NoXPos, nil, types.NewPtr(types.New(types.TINT))), + }) +} + +func markUsed(n *ir.Name) *ir.Name { + n.SetUsed(true) + return n +} + +func markNeedZero(n *ir.Name) *ir.Name { + n.SetNeedzero(true) + return n +} + +// Test all code paths for cmpstackvarlt. +func TestCmpstackvar(t *testing.T) { + nod := func(xoffset int64, t *types.Type, s *types.Sym, cl ir.Class) *ir.Name { + if s == nil { + s = &types.Sym{Name: "."} + } + n := typecheck.NewName(s) + n.SetType(t) + n.SetFrameOffset(xoffset) + n.Class_ = cl + return n + } + testdata := []struct { + a, b *ir.Name + lt bool + }{ + { + nod(0, nil, nil, ir.PAUTO), + nod(0, nil, nil, ir.PFUNC), + false, + }, + { + nod(0, nil, nil, ir.PFUNC), + nod(0, nil, nil, ir.PAUTO), + true, + }, + { + nod(0, nil, nil, ir.PFUNC), + nod(10, nil, nil, ir.PFUNC), + true, + }, + { + nod(20, nil, nil, ir.PFUNC), + nod(10, nil, nil, ir.PFUNC), + false, + }, + { + nod(10, nil, nil, ir.PFUNC), + nod(10, nil, nil, ir.PFUNC), + false, + }, + { + nod(10, nil, nil, ir.PPARAM), + nod(20, nil, nil, ir.PPARAMOUT), + true, + }, + { + nod(10, nil, nil, ir.PPARAMOUT), + nod(20, nil, nil, ir.PPARAM), + true, + }, + { + markUsed(nod(0, nil, nil, ir.PAUTO)), + nod(0, nil, nil, ir.PAUTO), + true, + }, + { + nod(0, nil, nil, ir.PAUTO), + markUsed(nod(0, nil, nil, ir.PAUTO)), + false, + }, + { + nod(0, typeWithoutPointers(), nil, ir.PAUTO), + nod(0, typeWithPointers(), nil, ir.PAUTO), + false, + }, + { + nod(0, typeWithPointers(), nil, ir.PAUTO), + nod(0, typeWithoutPointers(), nil, ir.PAUTO), + true, + }, + { + markNeedZero(nod(0, &types.Type{}, nil, ir.PAUTO)), + nod(0, &types.Type{}, nil, ir.PAUTO), + true, + }, + { + nod(0, &types.Type{}, nil, ir.PAUTO), + markNeedZero(nod(0, &types.Type{}, nil, ir.PAUTO)), + false, + }, + { + nod(0, &types.Type{Width: 1}, nil, ir.PAUTO), + nod(0, &types.Type{Width: 2}, nil, ir.PAUTO), + false, + }, + { + nod(0, &types.Type{Width: 2}, nil, ir.PAUTO), + nod(0, &types.Type{Width: 1}, nil, ir.PAUTO), + true, + }, + { + nod(0, &types.Type{}, &types.Sym{Name: "abc"}, ir.PAUTO), + nod(0, &types.Type{}, &types.Sym{Name: "xyz"}, ir.PAUTO), + true, + }, + { + nod(0, &types.Type{}, &types.Sym{Name: "abc"}, ir.PAUTO), + nod(0, &types.Type{}, &types.Sym{Name: "abc"}, ir.PAUTO), + false, + }, + { + nod(0, &types.Type{}, &types.Sym{Name: "xyz"}, ir.PAUTO), + nod(0, &types.Type{}, &types.Sym{Name: "abc"}, ir.PAUTO), + false, + }, + } + for _, d := range testdata { + got := cmpstackvarlt(d.a, d.b) + if got != d.lt { + t.Errorf("want %v < %v", d.a, d.b) + } + // If we expect a < b to be true, check that b < a is false. + if d.lt && cmpstackvarlt(d.b, d.a) { + t.Errorf("unexpected %v < %v", d.b, d.a) + } + } +} + +func TestStackvarSort(t *testing.T) { + nod := func(xoffset int64, t *types.Type, s *types.Sym, cl ir.Class) *ir.Name { + n := typecheck.NewName(s) + n.SetType(t) + n.SetFrameOffset(xoffset) + n.Class_ = cl + return n + } + inp := []*ir.Name{ + nod(0, &types.Type{}, &types.Sym{}, ir.PFUNC), + nod(0, &types.Type{}, &types.Sym{}, ir.PAUTO), + nod(0, &types.Type{}, &types.Sym{}, ir.PFUNC), + nod(10, &types.Type{}, &types.Sym{}, ir.PFUNC), + nod(20, &types.Type{}, &types.Sym{}, ir.PFUNC), + markUsed(nod(0, &types.Type{}, &types.Sym{}, ir.PAUTO)), + nod(0, typeWithoutPointers(), &types.Sym{}, ir.PAUTO), + nod(0, &types.Type{}, &types.Sym{}, ir.PAUTO), + markNeedZero(nod(0, &types.Type{}, &types.Sym{}, ir.PAUTO)), + nod(0, &types.Type{Width: 1}, &types.Sym{}, ir.PAUTO), + nod(0, &types.Type{Width: 2}, &types.Sym{}, ir.PAUTO), + nod(0, &types.Type{}, &types.Sym{Name: "abc"}, ir.PAUTO), + nod(0, &types.Type{}, &types.Sym{Name: "xyz"}, ir.PAUTO), + } + want := []*ir.Name{ + nod(0, &types.Type{}, &types.Sym{}, ir.PFUNC), + nod(0, &types.Type{}, &types.Sym{}, ir.PFUNC), + nod(10, &types.Type{}, &types.Sym{}, ir.PFUNC), + nod(20, &types.Type{}, &types.Sym{}, ir.PFUNC), + markUsed(nod(0, &types.Type{}, &types.Sym{}, ir.PAUTO)), + markNeedZero(nod(0, &types.Type{}, &types.Sym{}, ir.PAUTO)), + nod(0, &types.Type{Width: 2}, &types.Sym{}, ir.PAUTO), + nod(0, &types.Type{Width: 1}, &types.Sym{}, ir.PAUTO), + nod(0, &types.Type{}, &types.Sym{}, ir.PAUTO), + nod(0, &types.Type{}, &types.Sym{}, ir.PAUTO), + nod(0, &types.Type{}, &types.Sym{Name: "abc"}, ir.PAUTO), + nod(0, &types.Type{}, &types.Sym{Name: "xyz"}, ir.PAUTO), + nod(0, typeWithoutPointers(), &types.Sym{}, ir.PAUTO), + } + sort.Sort(byStackVar(inp)) + if !reflect.DeepEqual(want, inp) { + t.Error("sort failed") + for i := range inp { + g := inp[i] + w := want[i] + eq := reflect.DeepEqual(w, g) + if !eq { + t.Log(i, w, g) + } + } + } +} diff --git a/src/cmd/compile/internal/ssagen/phi.go b/src/cmd/compile/internal/ssagen/phi.go new file mode 100644 index 0000000000..01ad211282 --- /dev/null +++ b/src/cmd/compile/internal/ssagen/phi.go @@ -0,0 +1,557 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ssagen + +import ( + "container/heap" + "fmt" + + "cmd/compile/internal/ir" + "cmd/compile/internal/ssa" + "cmd/compile/internal/types" + "cmd/internal/src" +) + +// This file contains the algorithm to place phi nodes in a function. +// For small functions, we use Braun, Buchwald, Hack, Leißa, Mallon, and Zwinkau. +// https://pp.info.uni-karlsruhe.de/uploads/publikationen/braun13cc.pdf +// For large functions, we use Sreedhar & Gao: A Linear Time Algorithm for Placing Φ-Nodes. +// http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.8.1979&rep=rep1&type=pdf + +const smallBlocks = 500 + +const debugPhi = false + +// fwdRefAux wraps an arbitrary ir.Node as an ssa.Aux for use with OpFwdref. +type fwdRefAux struct { + _ [0]func() // ensure ir.Node isn't compared for equality + N ir.Node +} + +func (fwdRefAux) CanBeAnSSAAux() {} + +// insertPhis finds all the places in the function where a phi is +// necessary and inserts them. +// Uses FwdRef ops to find all uses of variables, and s.defvars to find +// all definitions. +// Phi values are inserted, and all FwdRefs are changed to a Copy +// of the appropriate phi or definition. +// TODO: make this part of cmd/compile/internal/ssa somehow? +func (s *state) insertPhis() { + if len(s.f.Blocks) <= smallBlocks { + sps := simplePhiState{s: s, f: s.f, defvars: s.defvars} + sps.insertPhis() + return + } + ps := phiState{s: s, f: s.f, defvars: s.defvars} + ps.insertPhis() +} + +type phiState struct { + s *state // SSA state + f *ssa.Func // function to work on + defvars []map[ir.Node]*ssa.Value // defined variables at end of each block + + varnum map[ir.Node]int32 // variable numbering + + // properties of the dominator tree + idom []*ssa.Block // dominator parents + tree []domBlock // dominator child+sibling + level []int32 // level in dominator tree (0 = root or unreachable, 1 = children of root, ...) + + // scratch locations + priq blockHeap // priority queue of blocks, higher level (toward leaves) = higher priority + q []*ssa.Block // inner loop queue + queued *sparseSet // has been put in q + hasPhi *sparseSet // has a phi + hasDef *sparseSet // has a write of the variable we're processing + + // miscellaneous + placeholder *ssa.Value // value to use as a "not set yet" placeholder. +} + +func (s *phiState) insertPhis() { + if debugPhi { + fmt.Println(s.f.String()) + } + + // Find all the variables for which we need to match up reads & writes. + // This step prunes any basic-block-only variables from consideration. + // Generate a numbering for these variables. + s.varnum = map[ir.Node]int32{} + var vars []ir.Node + var vartypes []*types.Type + for _, b := range s.f.Blocks { + for _, v := range b.Values { + if v.Op != ssa.OpFwdRef { + continue + } + var_ := v.Aux.(fwdRefAux).N + + // Optimization: look back 1 block for the definition. + if len(b.Preds) == 1 { + c := b.Preds[0].Block() + if w := s.defvars[c.ID][var_]; w != nil { + v.Op = ssa.OpCopy + v.Aux = nil + v.AddArg(w) + continue + } + } + + if _, ok := s.varnum[var_]; ok { + continue + } + s.varnum[var_] = int32(len(vartypes)) + if debugPhi { + fmt.Printf("var%d = %v\n", len(vartypes), var_) + } + vars = append(vars, var_) + vartypes = append(vartypes, v.Type) + } + } + + if len(vartypes) == 0 { + return + } + + // Find all definitions of the variables we need to process. + // defs[n] contains all the blocks in which variable number n is assigned. + defs := make([][]*ssa.Block, len(vartypes)) + for _, b := range s.f.Blocks { + for var_ := range s.defvars[b.ID] { // TODO: encode defvars some other way (explicit ops)? make defvars[n] a slice instead of a map. + if n, ok := s.varnum[var_]; ok { + defs[n] = append(defs[n], b) + } + } + } + + // Make dominator tree. + s.idom = s.f.Idom() + s.tree = make([]domBlock, s.f.NumBlocks()) + for _, b := range s.f.Blocks { + p := s.idom[b.ID] + if p != nil { + s.tree[b.ID].sibling = s.tree[p.ID].firstChild + s.tree[p.ID].firstChild = b + } + } + // Compute levels in dominator tree. + // With parent pointers we can do a depth-first walk without + // any auxiliary storage. + s.level = make([]int32, s.f.NumBlocks()) + b := s.f.Entry +levels: + for { + if p := s.idom[b.ID]; p != nil { + s.level[b.ID] = s.level[p.ID] + 1 + if debugPhi { + fmt.Printf("level %s = %d\n", b, s.level[b.ID]) + } + } + if c := s.tree[b.ID].firstChild; c != nil { + b = c + continue + } + for { + if c := s.tree[b.ID].sibling; c != nil { + b = c + continue levels + } + b = s.idom[b.ID] + if b == nil { + break levels + } + } + } + + // Allocate scratch locations. + s.priq.level = s.level + s.q = make([]*ssa.Block, 0, s.f.NumBlocks()) + s.queued = newSparseSet(s.f.NumBlocks()) + s.hasPhi = newSparseSet(s.f.NumBlocks()) + s.hasDef = newSparseSet(s.f.NumBlocks()) + s.placeholder = s.s.entryNewValue0(ssa.OpUnknown, types.TypeInvalid) + + // Generate phi ops for each variable. + for n := range vartypes { + s.insertVarPhis(n, vars[n], defs[n], vartypes[n]) + } + + // Resolve FwdRefs to the correct write or phi. + s.resolveFwdRefs() + + // Erase variable numbers stored in AuxInt fields of phi ops. They are no longer needed. + for _, b := range s.f.Blocks { + for _, v := range b.Values { + if v.Op == ssa.OpPhi { + v.AuxInt = 0 + } + // Any remaining FwdRefs are dead code. + if v.Op == ssa.OpFwdRef { + v.Op = ssa.OpUnknown + v.Aux = nil + } + } + } +} + +func (s *phiState) insertVarPhis(n int, var_ ir.Node, defs []*ssa.Block, typ *types.Type) { + priq := &s.priq + q := s.q + queued := s.queued + queued.clear() + hasPhi := s.hasPhi + hasPhi.clear() + hasDef := s.hasDef + hasDef.clear() + + // Add defining blocks to priority queue. + for _, b := range defs { + priq.a = append(priq.a, b) + hasDef.add(b.ID) + if debugPhi { + fmt.Printf("def of var%d in %s\n", n, b) + } + } + heap.Init(priq) + + // Visit blocks defining variable n, from deepest to shallowest. + for len(priq.a) > 0 { + currentRoot := heap.Pop(priq).(*ssa.Block) + if debugPhi { + fmt.Printf("currentRoot %s\n", currentRoot) + } + // Walk subtree below definition. + // Skip subtrees we've done in previous iterations. + // Find edges exiting tree dominated by definition (the dominance frontier). + // Insert phis at target blocks. + if queued.contains(currentRoot.ID) { + s.s.Fatalf("root already in queue") + } + q = append(q, currentRoot) + queued.add(currentRoot.ID) + for len(q) > 0 { + b := q[len(q)-1] + q = q[:len(q)-1] + if debugPhi { + fmt.Printf(" processing %s\n", b) + } + + currentRootLevel := s.level[currentRoot.ID] + for _, e := range b.Succs { + c := e.Block() + // TODO: if the variable is dead at c, skip it. + if s.level[c.ID] > currentRootLevel { + // a D-edge, or an edge whose target is in currentRoot's subtree. + continue + } + if hasPhi.contains(c.ID) { + continue + } + // Add a phi to block c for variable n. + hasPhi.add(c.ID) + v := c.NewValue0I(currentRoot.Pos, ssa.OpPhi, typ, int64(n)) // TODO: line number right? + // Note: we store the variable number in the phi's AuxInt field. Used temporarily by phi building. + if var_.Op() == ir.ONAME { + s.s.addNamedValue(var_.(*ir.Name), v) + } + for range c.Preds { + v.AddArg(s.placeholder) // Actual args will be filled in by resolveFwdRefs. + } + if debugPhi { + fmt.Printf("new phi for var%d in %s: %s\n", n, c, v) + } + if !hasDef.contains(c.ID) { + // There's now a new definition of this variable in block c. + // Add it to the priority queue to explore. + heap.Push(priq, c) + hasDef.add(c.ID) + } + } + + // Visit children if they have not been visited yet. + for c := s.tree[b.ID].firstChild; c != nil; c = s.tree[c.ID].sibling { + if !queued.contains(c.ID) { + q = append(q, c) + queued.add(c.ID) + } + } + } + } +} + +// resolveFwdRefs links all FwdRef uses up to their nearest dominating definition. +func (s *phiState) resolveFwdRefs() { + // Do a depth-first walk of the dominator tree, keeping track + // of the most-recently-seen value for each variable. + + // Map from variable ID to SSA value at the current point of the walk. + values := make([]*ssa.Value, len(s.varnum)) + for i := range values { + values[i] = s.placeholder + } + + // Stack of work to do. + type stackEntry struct { + b *ssa.Block // block to explore + + // variable/value pair to reinstate on exit + n int32 // variable ID + v *ssa.Value + + // Note: only one of b or n,v will be set. + } + var stk []stackEntry + + stk = append(stk, stackEntry{b: s.f.Entry}) + for len(stk) > 0 { + work := stk[len(stk)-1] + stk = stk[:len(stk)-1] + + b := work.b + if b == nil { + // On exit from a block, this case will undo any assignments done below. + values[work.n] = work.v + continue + } + + // Process phis as new defs. They come before FwdRefs in this block. + for _, v := range b.Values { + if v.Op != ssa.OpPhi { + continue + } + n := int32(v.AuxInt) + // Remember the old assignment so we can undo it when we exit b. + stk = append(stk, stackEntry{n: n, v: values[n]}) + // Record the new assignment. + values[n] = v + } + + // Replace a FwdRef op with the current incoming value for its variable. + for _, v := range b.Values { + if v.Op != ssa.OpFwdRef { + continue + } + n := s.varnum[v.Aux.(fwdRefAux).N] + v.Op = ssa.OpCopy + v.Aux = nil + v.AddArg(values[n]) + } + + // Establish values for variables defined in b. + for var_, v := range s.defvars[b.ID] { + n, ok := s.varnum[var_] + if !ok { + // some variable not live across a basic block boundary. + continue + } + // Remember the old assignment so we can undo it when we exit b. + stk = append(stk, stackEntry{n: n, v: values[n]}) + // Record the new assignment. + values[n] = v + } + + // Replace phi args in successors with the current incoming value. + for _, e := range b.Succs { + c, i := e.Block(), e.Index() + for j := len(c.Values) - 1; j >= 0; j-- { + v := c.Values[j] + if v.Op != ssa.OpPhi { + break // All phis will be at the end of the block during phi building. + } + // Only set arguments that have been resolved. + // For very wide CFGs, this significantly speeds up phi resolution. + // See golang.org/issue/8225. + if w := values[v.AuxInt]; w.Op != ssa.OpUnknown { + v.SetArg(i, w) + } + } + } + + // Walk children in dominator tree. + for c := s.tree[b.ID].firstChild; c != nil; c = s.tree[c.ID].sibling { + stk = append(stk, stackEntry{b: c}) + } + } +} + +// domBlock contains extra per-block information to record the dominator tree. +type domBlock struct { + firstChild *ssa.Block // first child of block in dominator tree + sibling *ssa.Block // next child of parent in dominator tree +} + +// A block heap is used as a priority queue to implement the PiggyBank +// from Sreedhar and Gao. That paper uses an array which is better +// asymptotically but worse in the common case when the PiggyBank +// holds a sparse set of blocks. +type blockHeap struct { + a []*ssa.Block // block IDs in heap + level []int32 // depth in dominator tree (static, used for determining priority) +} + +func (h *blockHeap) Len() int { return len(h.a) } +func (h *blockHeap) Swap(i, j int) { a := h.a; a[i], a[j] = a[j], a[i] } + +func (h *blockHeap) Push(x interface{}) { + v := x.(*ssa.Block) + h.a = append(h.a, v) +} +func (h *blockHeap) Pop() interface{} { + old := h.a + n := len(old) + x := old[n-1] + h.a = old[:n-1] + return x +} +func (h *blockHeap) Less(i, j int) bool { + return h.level[h.a[i].ID] > h.level[h.a[j].ID] +} + +// TODO: stop walking the iterated domininance frontier when +// the variable is dead. Maybe detect that by checking if the +// node we're on is reverse dominated by all the reads? +// Reverse dominated by the highest common successor of all the reads? + +// copy of ../ssa/sparseset.go +// TODO: move this file to ../ssa, then use sparseSet there. +type sparseSet struct { + dense []ssa.ID + sparse []int32 +} + +// newSparseSet returns a sparseSet that can represent +// integers between 0 and n-1 +func newSparseSet(n int) *sparseSet { + return &sparseSet{dense: nil, sparse: make([]int32, n)} +} + +func (s *sparseSet) contains(x ssa.ID) bool { + i := s.sparse[x] + return i < int32(len(s.dense)) && s.dense[i] == x +} + +func (s *sparseSet) add(x ssa.ID) { + i := s.sparse[x] + if i < int32(len(s.dense)) && s.dense[i] == x { + return + } + s.dense = append(s.dense, x) + s.sparse[x] = int32(len(s.dense)) - 1 +} + +func (s *sparseSet) clear() { + s.dense = s.dense[:0] +} + +// Variant to use for small functions. +type simplePhiState struct { + s *state // SSA state + f *ssa.Func // function to work on + fwdrefs []*ssa.Value // list of FwdRefs to be processed + defvars []map[ir.Node]*ssa.Value // defined variables at end of each block + reachable []bool // which blocks are reachable +} + +func (s *simplePhiState) insertPhis() { + s.reachable = ssa.ReachableBlocks(s.f) + + // Find FwdRef ops. + for _, b := range s.f.Blocks { + for _, v := range b.Values { + if v.Op != ssa.OpFwdRef { + continue + } + s.fwdrefs = append(s.fwdrefs, v) + var_ := v.Aux.(fwdRefAux).N + if _, ok := s.defvars[b.ID][var_]; !ok { + s.defvars[b.ID][var_] = v // treat FwdDefs as definitions. + } + } + } + + var args []*ssa.Value + +loop: + for len(s.fwdrefs) > 0 { + v := s.fwdrefs[len(s.fwdrefs)-1] + s.fwdrefs = s.fwdrefs[:len(s.fwdrefs)-1] + b := v.Block + var_ := v.Aux.(fwdRefAux).N + if b == s.f.Entry { + // No variable should be live at entry. + s.s.Fatalf("Value live at entry. It shouldn't be. func %s, node %v, value %v", s.f.Name, var_, v) + } + if !s.reachable[b.ID] { + // This block is dead. + // It doesn't matter what we use here as long as it is well-formed. + v.Op = ssa.OpUnknown + v.Aux = nil + continue + } + // Find variable value on each predecessor. + args = args[:0] + for _, e := range b.Preds { + args = append(args, s.lookupVarOutgoing(e.Block(), v.Type, var_, v.Pos)) + } + + // Decide if we need a phi or not. We need a phi if there + // are two different args (which are both not v). + var w *ssa.Value + for _, a := range args { + if a == v { + continue // self-reference + } + if a == w { + continue // already have this witness + } + if w != nil { + // two witnesses, need a phi value + v.Op = ssa.OpPhi + v.AddArgs(args...) + v.Aux = nil + continue loop + } + w = a // save witness + } + if w == nil { + s.s.Fatalf("no witness for reachable phi %s", v) + } + // One witness. Make v a copy of w. + v.Op = ssa.OpCopy + v.Aux = nil + v.AddArg(w) + } +} + +// lookupVarOutgoing finds the variable's value at the end of block b. +func (s *simplePhiState) lookupVarOutgoing(b *ssa.Block, t *types.Type, var_ ir.Node, line src.XPos) *ssa.Value { + for { + if v := s.defvars[b.ID][var_]; v != nil { + return v + } + // The variable is not defined by b and we haven't looked it up yet. + // If b has exactly one predecessor, loop to look it up there. + // Otherwise, give up and insert a new FwdRef and resolve it later. + if len(b.Preds) != 1 { + break + } + b = b.Preds[0].Block() + if !s.reachable[b.ID] { + // This is rare; it happens with oddly interleaved infinite loops in dead code. + // See issue 19783. + break + } + } + // Generate a FwdRef for the variable and return that. + v := b.NewValue0A(line, ssa.OpFwdRef, t, fwdRefAux{N: var_}) + s.defvars[b.ID][var_] = v + if var_.Op() == ir.ONAME { + s.s.addNamedValue(var_.(*ir.Name), v) + } + s.fwdrefs = append(s.fwdrefs, v) + return v +} diff --git a/src/cmd/compile/internal/ssagen/ssa.go b/src/cmd/compile/internal/ssagen/ssa.go new file mode 100644 index 0000000000..a77e57a5b6 --- /dev/null +++ b/src/cmd/compile/internal/ssagen/ssa.go @@ -0,0 +1,7459 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ssagen + +import ( + "bufio" + "bytes" + "encoding/binary" + "fmt" + "go/constant" + "html" + "os" + "path/filepath" + "sort" + "strings" + + "cmd/compile/internal/base" + "cmd/compile/internal/ir" + "cmd/compile/internal/liveness" + "cmd/compile/internal/objw" + "cmd/compile/internal/reflectdata" + "cmd/compile/internal/ssa" + "cmd/compile/internal/staticdata" + "cmd/compile/internal/typecheck" + "cmd/compile/internal/types" + "cmd/internal/obj" + "cmd/internal/obj/x86" + "cmd/internal/objabi" + "cmd/internal/src" + "cmd/internal/sys" +) + +var ssaConfig *ssa.Config +var ssaCaches []ssa.Cache + +var ssaDump string // early copy of $GOSSAFUNC; the func name to dump output for +var ssaDir string // optional destination for ssa dump file +var ssaDumpStdout bool // whether to dump to stdout +var ssaDumpCFG string // generate CFGs for these phases +const ssaDumpFile = "ssa.html" + +// ssaDumpInlined holds all inlined functions when ssaDump contains a function name. +var ssaDumpInlined []*ir.Func + +func DumpInline(fn *ir.Func) { + if ssaDump != "" && ssaDump == ir.FuncName(fn) { + ssaDumpInlined = append(ssaDumpInlined, fn) + } +} + +func InitEnv() { + ssaDump = os.Getenv("GOSSAFUNC") + ssaDir = os.Getenv("GOSSADIR") + if ssaDump != "" { + if strings.HasSuffix(ssaDump, "+") { + ssaDump = ssaDump[:len(ssaDump)-1] + ssaDumpStdout = true + } + spl := strings.Split(ssaDump, ":") + if len(spl) > 1 { + ssaDump = spl[0] + ssaDumpCFG = spl[1] + } + } +} + +func InitConfig() { + types_ := ssa.NewTypes() + + if Arch.SoftFloat { + softfloatInit() + } + + // Generate a few pointer types that are uncommon in the frontend but common in the backend. + // Caching is disabled in the backend, so generating these here avoids allocations. + _ = types.NewPtr(types.Types[types.TINTER]) // *interface{} + _ = types.NewPtr(types.NewPtr(types.Types[types.TSTRING])) // **string + _ = types.NewPtr(types.NewSlice(types.Types[types.TINTER])) // *[]interface{} + _ = types.NewPtr(types.NewPtr(types.ByteType)) // **byte + _ = types.NewPtr(types.NewSlice(types.ByteType)) // *[]byte + _ = types.NewPtr(types.NewSlice(types.Types[types.TSTRING])) // *[]string + _ = types.NewPtr(types.NewPtr(types.NewPtr(types.Types[types.TUINT8]))) // ***uint8 + _ = types.NewPtr(types.Types[types.TINT16]) // *int16 + _ = types.NewPtr(types.Types[types.TINT64]) // *int64 + _ = types.NewPtr(types.ErrorType) // *error + types.NewPtrCacheEnabled = false + ssaConfig = ssa.NewConfig(base.Ctxt.Arch.Name, *types_, base.Ctxt, base.Flag.N == 0) + ssaConfig.SoftFloat = Arch.SoftFloat + ssaConfig.Race = base.Flag.Race + ssaCaches = make([]ssa.Cache, base.Flag.LowerC) + + // Set up some runtime functions we'll need to call. + ir.Syms.AssertE2I = typecheck.LookupRuntimeFunc("assertE2I") + ir.Syms.AssertE2I2 = typecheck.LookupRuntimeFunc("assertE2I2") + ir.Syms.AssertI2I = typecheck.LookupRuntimeFunc("assertI2I") + ir.Syms.AssertI2I2 = typecheck.LookupRuntimeFunc("assertI2I2") + ir.Syms.Deferproc = typecheck.LookupRuntimeFunc("deferproc") + ir.Syms.DeferprocStack = typecheck.LookupRuntimeFunc("deferprocStack") + ir.Syms.Deferreturn = typecheck.LookupRuntimeFunc("deferreturn") + ir.Syms.Duffcopy = typecheck.LookupRuntimeFunc("duffcopy") + ir.Syms.Duffzero = typecheck.LookupRuntimeFunc("duffzero") + ir.Syms.GCWriteBarrier = typecheck.LookupRuntimeFunc("gcWriteBarrier") + ir.Syms.Goschedguarded = typecheck.LookupRuntimeFunc("goschedguarded") + ir.Syms.Growslice = typecheck.LookupRuntimeFunc("growslice") + ir.Syms.Msanread = typecheck.LookupRuntimeFunc("msanread") + ir.Syms.Msanwrite = typecheck.LookupRuntimeFunc("msanwrite") + ir.Syms.Msanmove = typecheck.LookupRuntimeFunc("msanmove") + ir.Syms.Newobject = typecheck.LookupRuntimeFunc("newobject") + ir.Syms.Newproc = typecheck.LookupRuntimeFunc("newproc") + ir.Syms.Panicdivide = typecheck.LookupRuntimeFunc("panicdivide") + ir.Syms.PanicdottypeE = typecheck.LookupRuntimeFunc("panicdottypeE") + ir.Syms.PanicdottypeI = typecheck.LookupRuntimeFunc("panicdottypeI") + ir.Syms.Panicnildottype = typecheck.LookupRuntimeFunc("panicnildottype") + ir.Syms.Panicoverflow = typecheck.LookupRuntimeFunc("panicoverflow") + ir.Syms.Panicshift = typecheck.LookupRuntimeFunc("panicshift") + ir.Syms.Raceread = typecheck.LookupRuntimeFunc("raceread") + ir.Syms.Racereadrange = typecheck.LookupRuntimeFunc("racereadrange") + ir.Syms.Racewrite = typecheck.LookupRuntimeFunc("racewrite") + ir.Syms.Racewriterange = typecheck.LookupRuntimeFunc("racewriterange") + ir.Syms.X86HasPOPCNT = typecheck.LookupRuntimeVar("x86HasPOPCNT") // bool + ir.Syms.X86HasSSE41 = typecheck.LookupRuntimeVar("x86HasSSE41") // bool + ir.Syms.X86HasFMA = typecheck.LookupRuntimeVar("x86HasFMA") // bool + ir.Syms.ARMHasVFPv4 = typecheck.LookupRuntimeVar("armHasVFPv4") // bool + ir.Syms.ARM64HasATOMICS = typecheck.LookupRuntimeVar("arm64HasATOMICS") // bool + ir.Syms.Typedmemclr = typecheck.LookupRuntimeFunc("typedmemclr") + ir.Syms.Typedmemmove = typecheck.LookupRuntimeFunc("typedmemmove") + ir.Syms.Udiv = typecheck.LookupRuntimeVar("udiv") // asm func with special ABI + ir.Syms.WriteBarrier = typecheck.LookupRuntimeVar("writeBarrier") // struct { bool; ... } + ir.Syms.Zerobase = typecheck.LookupRuntimeVar("zerobase") + + // asm funcs with special ABI + if base.Ctxt.Arch.Name == "amd64" { + GCWriteBarrierReg = map[int16]*obj.LSym{ + x86.REG_AX: typecheck.LookupRuntimeFunc("gcWriteBarrier"), + x86.REG_CX: typecheck.LookupRuntimeFunc("gcWriteBarrierCX"), + x86.REG_DX: typecheck.LookupRuntimeFunc("gcWriteBarrierDX"), + x86.REG_BX: typecheck.LookupRuntimeFunc("gcWriteBarrierBX"), + x86.REG_BP: typecheck.LookupRuntimeFunc("gcWriteBarrierBP"), + x86.REG_SI: typecheck.LookupRuntimeFunc("gcWriteBarrierSI"), + x86.REG_R8: typecheck.LookupRuntimeFunc("gcWriteBarrierR8"), + x86.REG_R9: typecheck.LookupRuntimeFunc("gcWriteBarrierR9"), + } + } + + if Arch.LinkArch.Family == sys.Wasm { + BoundsCheckFunc[ssa.BoundsIndex] = typecheck.LookupRuntimeFunc("goPanicIndex") + BoundsCheckFunc[ssa.BoundsIndexU] = typecheck.LookupRuntimeFunc("goPanicIndexU") + BoundsCheckFunc[ssa.BoundsSliceAlen] = typecheck.LookupRuntimeFunc("goPanicSliceAlen") + BoundsCheckFunc[ssa.BoundsSliceAlenU] = typecheck.LookupRuntimeFunc("goPanicSliceAlenU") + BoundsCheckFunc[ssa.BoundsSliceAcap] = typecheck.LookupRuntimeFunc("goPanicSliceAcap") + BoundsCheckFunc[ssa.BoundsSliceAcapU] = typecheck.LookupRuntimeFunc("goPanicSliceAcapU") + BoundsCheckFunc[ssa.BoundsSliceB] = typecheck.LookupRuntimeFunc("goPanicSliceB") + BoundsCheckFunc[ssa.BoundsSliceBU] = typecheck.LookupRuntimeFunc("goPanicSliceBU") + BoundsCheckFunc[ssa.BoundsSlice3Alen] = typecheck.LookupRuntimeFunc("goPanicSlice3Alen") + BoundsCheckFunc[ssa.BoundsSlice3AlenU] = typecheck.LookupRuntimeFunc("goPanicSlice3AlenU") + BoundsCheckFunc[ssa.BoundsSlice3Acap] = typecheck.LookupRuntimeFunc("goPanicSlice3Acap") + BoundsCheckFunc[ssa.BoundsSlice3AcapU] = typecheck.LookupRuntimeFunc("goPanicSlice3AcapU") + BoundsCheckFunc[ssa.BoundsSlice3B] = typecheck.LookupRuntimeFunc("goPanicSlice3B") + BoundsCheckFunc[ssa.BoundsSlice3BU] = typecheck.LookupRuntimeFunc("goPanicSlice3BU") + BoundsCheckFunc[ssa.BoundsSlice3C] = typecheck.LookupRuntimeFunc("goPanicSlice3C") + BoundsCheckFunc[ssa.BoundsSlice3CU] = typecheck.LookupRuntimeFunc("goPanicSlice3CU") + } else { + BoundsCheckFunc[ssa.BoundsIndex] = typecheck.LookupRuntimeFunc("panicIndex") + BoundsCheckFunc[ssa.BoundsIndexU] = typecheck.LookupRuntimeFunc("panicIndexU") + BoundsCheckFunc[ssa.BoundsSliceAlen] = typecheck.LookupRuntimeFunc("panicSliceAlen") + BoundsCheckFunc[ssa.BoundsSliceAlenU] = typecheck.LookupRuntimeFunc("panicSliceAlenU") + BoundsCheckFunc[ssa.BoundsSliceAcap] = typecheck.LookupRuntimeFunc("panicSliceAcap") + BoundsCheckFunc[ssa.BoundsSliceAcapU] = typecheck.LookupRuntimeFunc("panicSliceAcapU") + BoundsCheckFunc[ssa.BoundsSliceB] = typecheck.LookupRuntimeFunc("panicSliceB") + BoundsCheckFunc[ssa.BoundsSliceBU] = typecheck.LookupRuntimeFunc("panicSliceBU") + BoundsCheckFunc[ssa.BoundsSlice3Alen] = typecheck.LookupRuntimeFunc("panicSlice3Alen") + BoundsCheckFunc[ssa.BoundsSlice3AlenU] = typecheck.LookupRuntimeFunc("panicSlice3AlenU") + BoundsCheckFunc[ssa.BoundsSlice3Acap] = typecheck.LookupRuntimeFunc("panicSlice3Acap") + BoundsCheckFunc[ssa.BoundsSlice3AcapU] = typecheck.LookupRuntimeFunc("panicSlice3AcapU") + BoundsCheckFunc[ssa.BoundsSlice3B] = typecheck.LookupRuntimeFunc("panicSlice3B") + BoundsCheckFunc[ssa.BoundsSlice3BU] = typecheck.LookupRuntimeFunc("panicSlice3BU") + BoundsCheckFunc[ssa.BoundsSlice3C] = typecheck.LookupRuntimeFunc("panicSlice3C") + BoundsCheckFunc[ssa.BoundsSlice3CU] = typecheck.LookupRuntimeFunc("panicSlice3CU") + } + if Arch.LinkArch.PtrSize == 4 { + ExtendCheckFunc[ssa.BoundsIndex] = typecheck.LookupRuntimeVar("panicExtendIndex") + ExtendCheckFunc[ssa.BoundsIndexU] = typecheck.LookupRuntimeVar("panicExtendIndexU") + ExtendCheckFunc[ssa.BoundsSliceAlen] = typecheck.LookupRuntimeVar("panicExtendSliceAlen") + ExtendCheckFunc[ssa.BoundsSliceAlenU] = typecheck.LookupRuntimeVar("panicExtendSliceAlenU") + ExtendCheckFunc[ssa.BoundsSliceAcap] = typecheck.LookupRuntimeVar("panicExtendSliceAcap") + ExtendCheckFunc[ssa.BoundsSliceAcapU] = typecheck.LookupRuntimeVar("panicExtendSliceAcapU") + ExtendCheckFunc[ssa.BoundsSliceB] = typecheck.LookupRuntimeVar("panicExtendSliceB") + ExtendCheckFunc[ssa.BoundsSliceBU] = typecheck.LookupRuntimeVar("panicExtendSliceBU") + ExtendCheckFunc[ssa.BoundsSlice3Alen] = typecheck.LookupRuntimeVar("panicExtendSlice3Alen") + ExtendCheckFunc[ssa.BoundsSlice3AlenU] = typecheck.LookupRuntimeVar("panicExtendSlice3AlenU") + ExtendCheckFunc[ssa.BoundsSlice3Acap] = typecheck.LookupRuntimeVar("panicExtendSlice3Acap") + ExtendCheckFunc[ssa.BoundsSlice3AcapU] = typecheck.LookupRuntimeVar("panicExtendSlice3AcapU") + ExtendCheckFunc[ssa.BoundsSlice3B] = typecheck.LookupRuntimeVar("panicExtendSlice3B") + ExtendCheckFunc[ssa.BoundsSlice3BU] = typecheck.LookupRuntimeVar("panicExtendSlice3BU") + ExtendCheckFunc[ssa.BoundsSlice3C] = typecheck.LookupRuntimeVar("panicExtendSlice3C") + ExtendCheckFunc[ssa.BoundsSlice3CU] = typecheck.LookupRuntimeVar("panicExtendSlice3CU") + } + + // Wasm (all asm funcs with special ABIs) + ir.Syms.WasmMove = typecheck.LookupRuntimeVar("wasmMove") + ir.Syms.WasmZero = typecheck.LookupRuntimeVar("wasmZero") + ir.Syms.WasmDiv = typecheck.LookupRuntimeVar("wasmDiv") + ir.Syms.WasmTruncS = typecheck.LookupRuntimeVar("wasmTruncS") + ir.Syms.WasmTruncU = typecheck.LookupRuntimeVar("wasmTruncU") + ir.Syms.SigPanic = typecheck.LookupRuntimeFunc("sigpanic") +} + +// getParam returns the Field of ith param of node n (which is a +// function/method/interface call), where the receiver of a method call is +// considered as the 0th parameter. This does not include the receiver of an +// interface call. +func getParam(n *ir.CallExpr, i int) *types.Field { + t := n.X.Type() + if n.Op() == ir.OCALLMETH { + if i == 0 { + return t.Recv() + } + return t.Params().Field(i - 1) + } + return t.Params().Field(i) +} + +// dvarint writes a varint v to the funcdata in symbol x and returns the new offset +func dvarint(x *obj.LSym, off int, v int64) int { + if v < 0 || v > 1e9 { + panic(fmt.Sprintf("dvarint: bad offset for funcdata - %v", v)) + } + if v < 1<<7 { + return objw.Uint8(x, off, uint8(v)) + } + off = objw.Uint8(x, off, uint8((v&127)|128)) + if v < 1<<14 { + return objw.Uint8(x, off, uint8(v>>7)) + } + off = objw.Uint8(x, off, uint8(((v>>7)&127)|128)) + if v < 1<<21 { + return objw.Uint8(x, off, uint8(v>>14)) + } + off = objw.Uint8(x, off, uint8(((v>>14)&127)|128)) + if v < 1<<28 { + return objw.Uint8(x, off, uint8(v>>21)) + } + off = objw.Uint8(x, off, uint8(((v>>21)&127)|128)) + return objw.Uint8(x, off, uint8(v>>28)) +} + +// emitOpenDeferInfo emits FUNCDATA information about the defers in a function +// that is using open-coded defers. This funcdata is used to determine the active +// defers in a function and execute those defers during panic processing. +// +// The funcdata is all encoded in varints (since values will almost always be less than +// 128, but stack offsets could potentially be up to 2Gbyte). All "locations" (offsets) +// for stack variables are specified as the number of bytes below varp (pointer to the +// top of the local variables) for their starting address. The format is: +// +// - Max total argument size among all the defers +// - Offset of the deferBits variable +// - Number of defers in the function +// - Information about each defer call, in reverse order of appearance in the function: +// - Total argument size of the call +// - Offset of the closure value to call +// - Number of arguments (including interface receiver or method receiver as first arg) +// - Information about each argument +// - Offset of the stored defer argument in this function's frame +// - Size of the argument +// - Offset of where argument should be placed in the args frame when making call +func (s *state) emitOpenDeferInfo() { + x := base.Ctxt.Lookup(s.curfn.LSym.Name + ".opendefer") + s.curfn.LSym.Func().OpenCodedDeferInfo = x + off := 0 + + // Compute maxargsize (max size of arguments for all defers) + // first, so we can output it first to the funcdata + var maxargsize int64 + for i := len(s.openDefers) - 1; i >= 0; i-- { + r := s.openDefers[i] + argsize := r.n.X.Type().ArgWidth() + if argsize > maxargsize { + maxargsize = argsize + } + } + off = dvarint(x, off, maxargsize) + off = dvarint(x, off, -s.deferBitsTemp.FrameOffset()) + off = dvarint(x, off, int64(len(s.openDefers))) + + // Write in reverse-order, for ease of running in that order at runtime + for i := len(s.openDefers) - 1; i >= 0; i-- { + r := s.openDefers[i] + off = dvarint(x, off, r.n.X.Type().ArgWidth()) + off = dvarint(x, off, -r.closureNode.FrameOffset()) + numArgs := len(r.argNodes) + if r.rcvrNode != nil { + // If there's an interface receiver, treat/place it as the first + // arg. (If there is a method receiver, it's already included as + // first arg in r.argNodes.) + numArgs++ + } + off = dvarint(x, off, int64(numArgs)) + if r.rcvrNode != nil { + off = dvarint(x, off, -r.rcvrNode.FrameOffset()) + off = dvarint(x, off, s.config.PtrSize) + off = dvarint(x, off, 0) + } + for j, arg := range r.argNodes { + f := getParam(r.n, j) + off = dvarint(x, off, -arg.FrameOffset()) + off = dvarint(x, off, f.Type.Size()) + off = dvarint(x, off, f.Offset) + } + } +} + +// buildssa builds an SSA function for fn. +// worker indicates which of the backend workers is doing the processing. +func buildssa(fn *ir.Func, worker int) *ssa.Func { + name := ir.FuncName(fn) + printssa := false + if ssaDump != "" { // match either a simple name e.g. "(*Reader).Reset", or a package.name e.g. "compress/gzip.(*Reader).Reset" + printssa = name == ssaDump || base.Ctxt.Pkgpath+"."+name == ssaDump + } + var astBuf *bytes.Buffer + if printssa { + astBuf = &bytes.Buffer{} + ir.FDumpList(astBuf, "buildssa-enter", fn.Enter) + ir.FDumpList(astBuf, "buildssa-body", fn.Body) + ir.FDumpList(astBuf, "buildssa-exit", fn.Exit) + if ssaDumpStdout { + fmt.Println("generating SSA for", name) + fmt.Print(astBuf.String()) + } + } + + var s state + s.pushLine(fn.Pos()) + defer s.popLine() + + s.hasdefer = fn.HasDefer() + if fn.Pragma&ir.CgoUnsafeArgs != 0 { + s.cgoUnsafeArgs = true + } + + fe := ssafn{ + curfn: fn, + log: printssa && ssaDumpStdout, + } + s.curfn = fn + + s.f = ssa.NewFunc(&fe) + s.config = ssaConfig + s.f.Type = fn.Type() + s.f.Config = ssaConfig + s.f.Cache = &ssaCaches[worker] + s.f.Cache.Reset() + s.f.Name = name + s.f.DebugTest = s.f.DebugHashMatch("GOSSAHASH") + s.f.PrintOrHtmlSSA = printssa + if fn.Pragma&ir.Nosplit != 0 { + s.f.NoSplit = true + } + s.panics = map[funcLine]*ssa.Block{} + s.softFloat = s.config.SoftFloat + + // Allocate starting block + s.f.Entry = s.f.NewBlock(ssa.BlockPlain) + s.f.Entry.Pos = fn.Pos() + + if printssa { + ssaDF := ssaDumpFile + if ssaDir != "" { + ssaDF = filepath.Join(ssaDir, base.Ctxt.Pkgpath+"."+name+".html") + ssaD := filepath.Dir(ssaDF) + os.MkdirAll(ssaD, 0755) + } + s.f.HTMLWriter = ssa.NewHTMLWriter(ssaDF, s.f, ssaDumpCFG) + // TODO: generate and print a mapping from nodes to values and blocks + dumpSourcesColumn(s.f.HTMLWriter, fn) + s.f.HTMLWriter.WriteAST("AST", astBuf) + } + + // Allocate starting values + s.labels = map[string]*ssaLabel{} + s.fwdVars = map[ir.Node]*ssa.Value{} + s.startmem = s.entryNewValue0(ssa.OpInitMem, types.TypeMem) + + s.hasOpenDefers = base.Flag.N == 0 && s.hasdefer && !s.curfn.OpenCodedDeferDisallowed() + switch { + case s.hasOpenDefers && (base.Ctxt.Flag_shared || base.Ctxt.Flag_dynlink) && base.Ctxt.Arch.Name == "386": + // Don't support open-coded defers for 386 ONLY when using shared + // libraries, because there is extra code (added by rewriteToUseGot()) + // preceding the deferreturn/ret code that is generated by gencallret() + // that we don't track correctly. + s.hasOpenDefers = false + } + if s.hasOpenDefers && len(s.curfn.Exit) > 0 { + // Skip doing open defers if there is any extra exit code (likely + // copying heap-allocated return values or race detection), since + // we will not generate that code in the case of the extra + // deferreturn/ret segment. + s.hasOpenDefers = false + } + if s.hasOpenDefers && + s.curfn.NumReturns*s.curfn.NumDefers > 15 { + // Since we are generating defer calls at every exit for + // open-coded defers, skip doing open-coded defers if there are + // too many returns (especially if there are multiple defers). + // Open-coded defers are most important for improving performance + // for smaller functions (which don't have many returns). + s.hasOpenDefers = false + } + + s.sp = s.entryNewValue0(ssa.OpSP, types.Types[types.TUINTPTR]) // TODO: use generic pointer type (unsafe.Pointer?) instead + s.sb = s.entryNewValue0(ssa.OpSB, types.Types[types.TUINTPTR]) + + s.startBlock(s.f.Entry) + s.vars[memVar] = s.startmem + if s.hasOpenDefers { + // Create the deferBits variable and stack slot. deferBits is a + // bitmask showing which of the open-coded defers in this function + // have been activated. + deferBitsTemp := typecheck.TempAt(src.NoXPos, s.curfn, types.Types[types.TUINT8]) + s.deferBitsTemp = deferBitsTemp + // For this value, AuxInt is initialized to zero by default + startDeferBits := s.entryNewValue0(ssa.OpConst8, types.Types[types.TUINT8]) + s.vars[deferBitsVar] = startDeferBits + s.deferBitsAddr = s.addr(deferBitsTemp) + s.store(types.Types[types.TUINT8], s.deferBitsAddr, startDeferBits) + // Make sure that the deferBits stack slot is kept alive (for use + // by panics) and stores to deferBits are not eliminated, even if + // all checking code on deferBits in the function exit can be + // eliminated, because the defer statements were all + // unconditional. + s.vars[memVar] = s.newValue1Apos(ssa.OpVarLive, types.TypeMem, deferBitsTemp, s.mem(), false) + } + + // Generate addresses of local declarations + s.decladdrs = map[*ir.Name]*ssa.Value{} + var args []ssa.Param + var results []ssa.Param + for _, n := range fn.Dcl { + switch n.Class_ { + case ir.PPARAM: + s.decladdrs[n] = s.entryNewValue2A(ssa.OpLocalAddr, types.NewPtr(n.Type()), n, s.sp, s.startmem) + args = append(args, ssa.Param{Type: n.Type(), Offset: int32(n.FrameOffset())}) + case ir.PPARAMOUT: + s.decladdrs[n] = s.entryNewValue2A(ssa.OpLocalAddr, types.NewPtr(n.Type()), n, s.sp, s.startmem) + results = append(results, ssa.Param{Type: n.Type(), Offset: int32(n.FrameOffset())}) + if s.canSSA(n) { + // Save ssa-able PPARAMOUT variables so we can + // store them back to the stack at the end of + // the function. + s.returns = append(s.returns, n) + } + case ir.PAUTO: + // processed at each use, to prevent Addr coming + // before the decl. + case ir.PAUTOHEAP: + // moved to heap - already handled by frontend + case ir.PFUNC: + // local function - already handled by frontend + default: + s.Fatalf("local variable with class %v unimplemented", n.Class_) + } + } + + // Populate SSAable arguments. + for _, n := range fn.Dcl { + if n.Class_ == ir.PPARAM && s.canSSA(n) { + v := s.newValue0A(ssa.OpArg, n.Type(), n) + s.vars[n] = v + s.addNamedValue(n, v) // This helps with debugging information, not needed for compilation itself. + } + } + + // Convert the AST-based IR to the SSA-based IR + s.stmtList(fn.Enter) + s.stmtList(fn.Body) + + // fallthrough to exit + if s.curBlock != nil { + s.pushLine(fn.Endlineno) + s.exit() + s.popLine() + } + + for _, b := range s.f.Blocks { + if b.Pos != src.NoXPos { + s.updateUnsetPredPos(b) + } + } + + s.insertPhis() + + // Main call to ssa package to compile function + ssa.Compile(s.f) + + if s.hasOpenDefers { + s.emitOpenDeferInfo() + } + + return s.f +} + +func dumpSourcesColumn(writer *ssa.HTMLWriter, fn *ir.Func) { + // Read sources of target function fn. + fname := base.Ctxt.PosTable.Pos(fn.Pos()).Filename() + targetFn, err := readFuncLines(fname, fn.Pos().Line(), fn.Endlineno.Line()) + if err != nil { + writer.Logf("cannot read sources for function %v: %v", fn, err) + } + + // Read sources of inlined functions. + var inlFns []*ssa.FuncLines + for _, fi := range ssaDumpInlined { + elno := fi.Endlineno + fname := base.Ctxt.PosTable.Pos(fi.Pos()).Filename() + fnLines, err := readFuncLines(fname, fi.Pos().Line(), elno.Line()) + if err != nil { + writer.Logf("cannot read sources for inlined function %v: %v", fi, err) + continue + } + inlFns = append(inlFns, fnLines) + } + + sort.Sort(ssa.ByTopo(inlFns)) + if targetFn != nil { + inlFns = append([]*ssa.FuncLines{targetFn}, inlFns...) + } + + writer.WriteSources("sources", inlFns) +} + +func readFuncLines(file string, start, end uint) (*ssa.FuncLines, error) { + f, err := os.Open(os.ExpandEnv(file)) + if err != nil { + return nil, err + } + defer f.Close() + var lines []string + ln := uint(1) + scanner := bufio.NewScanner(f) + for scanner.Scan() && ln <= end { + if ln >= start { + lines = append(lines, scanner.Text()) + } + ln++ + } + return &ssa.FuncLines{Filename: file, StartLineno: start, Lines: lines}, nil +} + +// updateUnsetPredPos propagates the earliest-value position information for b +// towards all of b's predecessors that need a position, and recurs on that +// predecessor if its position is updated. B should have a non-empty position. +func (s *state) updateUnsetPredPos(b *ssa.Block) { + if b.Pos == src.NoXPos { + s.Fatalf("Block %s should have a position", b) + } + bestPos := src.NoXPos + for _, e := range b.Preds { + p := e.Block() + if !p.LackingPos() { + continue + } + if bestPos == src.NoXPos { + bestPos = b.Pos + for _, v := range b.Values { + if v.LackingPos() { + continue + } + if v.Pos != src.NoXPos { + // Assume values are still in roughly textual order; + // TODO: could also seek minimum position? + bestPos = v.Pos + break + } + } + } + p.Pos = bestPos + s.updateUnsetPredPos(p) // We do not expect long chains of these, thus recursion is okay. + } +} + +// Information about each open-coded defer. +type openDeferInfo struct { + // The node representing the call of the defer + n *ir.CallExpr + // If defer call is closure call, the address of the argtmp where the + // closure is stored. + closure *ssa.Value + // The node representing the argtmp where the closure is stored - used for + // function, method, or interface call, to store a closure that panic + // processing can use for this defer. + closureNode *ir.Name + // If defer call is interface call, the address of the argtmp where the + // receiver is stored + rcvr *ssa.Value + // The node representing the argtmp where the receiver is stored + rcvrNode *ir.Name + // The addresses of the argtmps where the evaluated arguments of the defer + // function call are stored. + argVals []*ssa.Value + // The nodes representing the argtmps where the args of the defer are stored + argNodes []*ir.Name +} + +type state struct { + // configuration (arch) information + config *ssa.Config + + // function we're building + f *ssa.Func + + // Node for function + curfn *ir.Func + + // labels in f + labels map[string]*ssaLabel + + // unlabeled break and continue statement tracking + breakTo *ssa.Block // current target for plain break statement + continueTo *ssa.Block // current target for plain continue statement + + // current location where we're interpreting the AST + curBlock *ssa.Block + + // variable assignments in the current block (map from variable symbol to ssa value) + // *Node is the unique identifier (an ONAME Node) for the variable. + // TODO: keep a single varnum map, then make all of these maps slices instead? + vars map[ir.Node]*ssa.Value + + // fwdVars are variables that are used before they are defined in the current block. + // This map exists just to coalesce multiple references into a single FwdRef op. + // *Node is the unique identifier (an ONAME Node) for the variable. + fwdVars map[ir.Node]*ssa.Value + + // all defined variables at the end of each block. Indexed by block ID. + defvars []map[ir.Node]*ssa.Value + + // addresses of PPARAM and PPARAMOUT variables. + decladdrs map[*ir.Name]*ssa.Value + + // starting values. Memory, stack pointer, and globals pointer + startmem *ssa.Value + sp *ssa.Value + sb *ssa.Value + // value representing address of where deferBits autotmp is stored + deferBitsAddr *ssa.Value + deferBitsTemp *ir.Name + + // line number stack. The current line number is top of stack + line []src.XPos + // the last line number processed; it may have been popped + lastPos src.XPos + + // list of panic calls by function name and line number. + // Used to deduplicate panic calls. + panics map[funcLine]*ssa.Block + + // list of PPARAMOUT (return) variables. + returns []*ir.Name + + cgoUnsafeArgs bool + hasdefer bool // whether the function contains a defer statement + softFloat bool + hasOpenDefers bool // whether we are doing open-coded defers + + // If doing open-coded defers, list of info about the defer calls in + // scanning order. Hence, at exit we should run these defers in reverse + // order of this list + openDefers []*openDeferInfo + // For open-coded defers, this is the beginning and end blocks of the last + // defer exit code that we have generated so far. We use these to share + // code between exits if the shareDeferExits option (disabled by default) + // is on. + lastDeferExit *ssa.Block // Entry block of last defer exit code we generated + lastDeferFinalBlock *ssa.Block // Final block of last defer exit code we generated + lastDeferCount int // Number of defers encountered at that point + + prevCall *ssa.Value // the previous call; use this to tie results to the call op. +} + +type funcLine struct { + f *obj.LSym + base *src.PosBase + line uint +} + +type ssaLabel struct { + target *ssa.Block // block identified by this label + breakTarget *ssa.Block // block to break to in control flow node identified by this label + continueTarget *ssa.Block // block to continue to in control flow node identified by this label +} + +// label returns the label associated with sym, creating it if necessary. +func (s *state) label(sym *types.Sym) *ssaLabel { + lab := s.labels[sym.Name] + if lab == nil { + lab = new(ssaLabel) + s.labels[sym.Name] = lab + } + return lab +} + +func (s *state) Logf(msg string, args ...interface{}) { s.f.Logf(msg, args...) } +func (s *state) Log() bool { return s.f.Log() } +func (s *state) Fatalf(msg string, args ...interface{}) { + s.f.Frontend().Fatalf(s.peekPos(), msg, args...) +} +func (s *state) Warnl(pos src.XPos, msg string, args ...interface{}) { s.f.Warnl(pos, msg, args...) } +func (s *state) Debug_checknil() bool { return s.f.Frontend().Debug_checknil() } + +func ssaMarker(name string) *ir.Name { + return typecheck.NewName(&types.Sym{Name: name}) +} + +var ( + // marker node for the memory variable + memVar = ssaMarker("mem") + + // marker nodes for temporary variables + ptrVar = ssaMarker("ptr") + lenVar = ssaMarker("len") + newlenVar = ssaMarker("newlen") + capVar = ssaMarker("cap") + typVar = ssaMarker("typ") + okVar = ssaMarker("ok") + deferBitsVar = ssaMarker("deferBits") +) + +// startBlock sets the current block we're generating code in to b. +func (s *state) startBlock(b *ssa.Block) { + if s.curBlock != nil { + s.Fatalf("starting block %v when block %v has not ended", b, s.curBlock) + } + s.curBlock = b + s.vars = map[ir.Node]*ssa.Value{} + for n := range s.fwdVars { + delete(s.fwdVars, n) + } +} + +// endBlock marks the end of generating code for the current block. +// Returns the (former) current block. Returns nil if there is no current +// block, i.e. if no code flows to the current execution point. +func (s *state) endBlock() *ssa.Block { + b := s.curBlock + if b == nil { + return nil + } + for len(s.defvars) <= int(b.ID) { + s.defvars = append(s.defvars, nil) + } + s.defvars[b.ID] = s.vars + s.curBlock = nil + s.vars = nil + if b.LackingPos() { + // Empty plain blocks get the line of their successor (handled after all blocks created), + // except for increment blocks in For statements (handled in ssa conversion of OFOR), + // and for blocks ending in GOTO/BREAK/CONTINUE. + b.Pos = src.NoXPos + } else { + b.Pos = s.lastPos + } + return b +} + +// pushLine pushes a line number on the line number stack. +func (s *state) pushLine(line src.XPos) { + if !line.IsKnown() { + // the frontend may emit node with line number missing, + // use the parent line number in this case. + line = s.peekPos() + if base.Flag.K != 0 { + base.Warn("buildssa: unknown position (line 0)") + } + } else { + s.lastPos = line + } + + s.line = append(s.line, line) +} + +// popLine pops the top of the line number stack. +func (s *state) popLine() { + s.line = s.line[:len(s.line)-1] +} + +// peekPos peeks the top of the line number stack. +func (s *state) peekPos() src.XPos { + return s.line[len(s.line)-1] +} + +// newValue0 adds a new value with no arguments to the current block. +func (s *state) newValue0(op ssa.Op, t *types.Type) *ssa.Value { + return s.curBlock.NewValue0(s.peekPos(), op, t) +} + +// newValue0A adds a new value with no arguments and an aux value to the current block. +func (s *state) newValue0A(op ssa.Op, t *types.Type, aux ssa.Aux) *ssa.Value { + return s.curBlock.NewValue0A(s.peekPos(), op, t, aux) +} + +// newValue0I adds a new value with no arguments and an auxint value to the current block. +func (s *state) newValue0I(op ssa.Op, t *types.Type, auxint int64) *ssa.Value { + return s.curBlock.NewValue0I(s.peekPos(), op, t, auxint) +} + +// newValue1 adds a new value with one argument to the current block. +func (s *state) newValue1(op ssa.Op, t *types.Type, arg *ssa.Value) *ssa.Value { + return s.curBlock.NewValue1(s.peekPos(), op, t, arg) +} + +// newValue1A adds a new value with one argument and an aux value to the current block. +func (s *state) newValue1A(op ssa.Op, t *types.Type, aux ssa.Aux, arg *ssa.Value) *ssa.Value { + return s.curBlock.NewValue1A(s.peekPos(), op, t, aux, arg) +} + +// newValue1Apos adds a new value with one argument and an aux value to the current block. +// isStmt determines whether the created values may be a statement or not +// (i.e., false means never, yes means maybe). +func (s *state) newValue1Apos(op ssa.Op, t *types.Type, aux ssa.Aux, arg *ssa.Value, isStmt bool) *ssa.Value { + if isStmt { + return s.curBlock.NewValue1A(s.peekPos(), op, t, aux, arg) + } + return s.curBlock.NewValue1A(s.peekPos().WithNotStmt(), op, t, aux, arg) +} + +// newValue1I adds a new value with one argument and an auxint value to the current block. +func (s *state) newValue1I(op ssa.Op, t *types.Type, aux int64, arg *ssa.Value) *ssa.Value { + return s.curBlock.NewValue1I(s.peekPos(), op, t, aux, arg) +} + +// newValue2 adds a new value with two arguments to the current block. +func (s *state) newValue2(op ssa.Op, t *types.Type, arg0, arg1 *ssa.Value) *ssa.Value { + return s.curBlock.NewValue2(s.peekPos(), op, t, arg0, arg1) +} + +// newValue2A adds a new value with two arguments and an aux value to the current block. +func (s *state) newValue2A(op ssa.Op, t *types.Type, aux ssa.Aux, arg0, arg1 *ssa.Value) *ssa.Value { + return s.curBlock.NewValue2A(s.peekPos(), op, t, aux, arg0, arg1) +} + +// newValue2Apos adds a new value with two arguments and an aux value to the current block. +// isStmt determines whether the created values may be a statement or not +// (i.e., false means never, yes means maybe). +func (s *state) newValue2Apos(op ssa.Op, t *types.Type, aux ssa.Aux, arg0, arg1 *ssa.Value, isStmt bool) *ssa.Value { + if isStmt { + return s.curBlock.NewValue2A(s.peekPos(), op, t, aux, arg0, arg1) + } + return s.curBlock.NewValue2A(s.peekPos().WithNotStmt(), op, t, aux, arg0, arg1) +} + +// newValue2I adds a new value with two arguments and an auxint value to the current block. +func (s *state) newValue2I(op ssa.Op, t *types.Type, aux int64, arg0, arg1 *ssa.Value) *ssa.Value { + return s.curBlock.NewValue2I(s.peekPos(), op, t, aux, arg0, arg1) +} + +// newValue3 adds a new value with three arguments to the current block. +func (s *state) newValue3(op ssa.Op, t *types.Type, arg0, arg1, arg2 *ssa.Value) *ssa.Value { + return s.curBlock.NewValue3(s.peekPos(), op, t, arg0, arg1, arg2) +} + +// newValue3I adds a new value with three arguments and an auxint value to the current block. +func (s *state) newValue3I(op ssa.Op, t *types.Type, aux int64, arg0, arg1, arg2 *ssa.Value) *ssa.Value { + return s.curBlock.NewValue3I(s.peekPos(), op, t, aux, arg0, arg1, arg2) +} + +// newValue3A adds a new value with three arguments and an aux value to the current block. +func (s *state) newValue3A(op ssa.Op, t *types.Type, aux ssa.Aux, arg0, arg1, arg2 *ssa.Value) *ssa.Value { + return s.curBlock.NewValue3A(s.peekPos(), op, t, aux, arg0, arg1, arg2) +} + +// newValue3Apos adds a new value with three arguments and an aux value to the current block. +// isStmt determines whether the created values may be a statement or not +// (i.e., false means never, yes means maybe). +func (s *state) newValue3Apos(op ssa.Op, t *types.Type, aux ssa.Aux, arg0, arg1, arg2 *ssa.Value, isStmt bool) *ssa.Value { + if isStmt { + return s.curBlock.NewValue3A(s.peekPos(), op, t, aux, arg0, arg1, arg2) + } + return s.curBlock.NewValue3A(s.peekPos().WithNotStmt(), op, t, aux, arg0, arg1, arg2) +} + +// newValue4 adds a new value with four arguments to the current block. +func (s *state) newValue4(op ssa.Op, t *types.Type, arg0, arg1, arg2, arg3 *ssa.Value) *ssa.Value { + return s.curBlock.NewValue4(s.peekPos(), op, t, arg0, arg1, arg2, arg3) +} + +// newValue4 adds a new value with four arguments and an auxint value to the current block. +func (s *state) newValue4I(op ssa.Op, t *types.Type, aux int64, arg0, arg1, arg2, arg3 *ssa.Value) *ssa.Value { + return s.curBlock.NewValue4I(s.peekPos(), op, t, aux, arg0, arg1, arg2, arg3) +} + +// entryNewValue0 adds a new value with no arguments to the entry block. +func (s *state) entryNewValue0(op ssa.Op, t *types.Type) *ssa.Value { + return s.f.Entry.NewValue0(src.NoXPos, op, t) +} + +// entryNewValue0A adds a new value with no arguments and an aux value to the entry block. +func (s *state) entryNewValue0A(op ssa.Op, t *types.Type, aux ssa.Aux) *ssa.Value { + return s.f.Entry.NewValue0A(src.NoXPos, op, t, aux) +} + +// entryNewValue1 adds a new value with one argument to the entry block. +func (s *state) entryNewValue1(op ssa.Op, t *types.Type, arg *ssa.Value) *ssa.Value { + return s.f.Entry.NewValue1(src.NoXPos, op, t, arg) +} + +// entryNewValue1 adds a new value with one argument and an auxint value to the entry block. +func (s *state) entryNewValue1I(op ssa.Op, t *types.Type, auxint int64, arg *ssa.Value) *ssa.Value { + return s.f.Entry.NewValue1I(src.NoXPos, op, t, auxint, arg) +} + +// entryNewValue1A adds a new value with one argument and an aux value to the entry block. +func (s *state) entryNewValue1A(op ssa.Op, t *types.Type, aux ssa.Aux, arg *ssa.Value) *ssa.Value { + return s.f.Entry.NewValue1A(src.NoXPos, op, t, aux, arg) +} + +// entryNewValue2 adds a new value with two arguments to the entry block. +func (s *state) entryNewValue2(op ssa.Op, t *types.Type, arg0, arg1 *ssa.Value) *ssa.Value { + return s.f.Entry.NewValue2(src.NoXPos, op, t, arg0, arg1) +} + +// entryNewValue2A adds a new value with two arguments and an aux value to the entry block. +func (s *state) entryNewValue2A(op ssa.Op, t *types.Type, aux ssa.Aux, arg0, arg1 *ssa.Value) *ssa.Value { + return s.f.Entry.NewValue2A(src.NoXPos, op, t, aux, arg0, arg1) +} + +// const* routines add a new const value to the entry block. +func (s *state) constSlice(t *types.Type) *ssa.Value { + return s.f.ConstSlice(t) +} +func (s *state) constInterface(t *types.Type) *ssa.Value { + return s.f.ConstInterface(t) +} +func (s *state) constNil(t *types.Type) *ssa.Value { return s.f.ConstNil(t) } +func (s *state) constEmptyString(t *types.Type) *ssa.Value { + return s.f.ConstEmptyString(t) +} +func (s *state) constBool(c bool) *ssa.Value { + return s.f.ConstBool(types.Types[types.TBOOL], c) +} +func (s *state) constInt8(t *types.Type, c int8) *ssa.Value { + return s.f.ConstInt8(t, c) +} +func (s *state) constInt16(t *types.Type, c int16) *ssa.Value { + return s.f.ConstInt16(t, c) +} +func (s *state) constInt32(t *types.Type, c int32) *ssa.Value { + return s.f.ConstInt32(t, c) +} +func (s *state) constInt64(t *types.Type, c int64) *ssa.Value { + return s.f.ConstInt64(t, c) +} +func (s *state) constFloat32(t *types.Type, c float64) *ssa.Value { + return s.f.ConstFloat32(t, c) +} +func (s *state) constFloat64(t *types.Type, c float64) *ssa.Value { + return s.f.ConstFloat64(t, c) +} +func (s *state) constInt(t *types.Type, c int64) *ssa.Value { + if s.config.PtrSize == 8 { + return s.constInt64(t, c) + } + if int64(int32(c)) != c { + s.Fatalf("integer constant too big %d", c) + } + return s.constInt32(t, int32(c)) +} +func (s *state) constOffPtrSP(t *types.Type, c int64) *ssa.Value { + return s.f.ConstOffPtrSP(t, c, s.sp) +} + +// newValueOrSfCall* are wrappers around newValue*, which may create a call to a +// soft-float runtime function instead (when emitting soft-float code). +func (s *state) newValueOrSfCall1(op ssa.Op, t *types.Type, arg *ssa.Value) *ssa.Value { + if s.softFloat { + if c, ok := s.sfcall(op, arg); ok { + return c + } + } + return s.newValue1(op, t, arg) +} +func (s *state) newValueOrSfCall2(op ssa.Op, t *types.Type, arg0, arg1 *ssa.Value) *ssa.Value { + if s.softFloat { + if c, ok := s.sfcall(op, arg0, arg1); ok { + return c + } + } + return s.newValue2(op, t, arg0, arg1) +} + +type instrumentKind uint8 + +const ( + instrumentRead = iota + instrumentWrite + instrumentMove +) + +func (s *state) instrument(t *types.Type, addr *ssa.Value, kind instrumentKind) { + s.instrument2(t, addr, nil, kind) +} + +// instrumentFields instruments a read/write operation on addr. +// If it is instrumenting for MSAN and t is a struct type, it instruments +// operation for each field, instead of for the whole struct. +func (s *state) instrumentFields(t *types.Type, addr *ssa.Value, kind instrumentKind) { + if !base.Flag.MSan || !t.IsStruct() { + s.instrument(t, addr, kind) + return + } + for _, f := range t.Fields().Slice() { + if f.Sym.IsBlank() { + continue + } + offptr := s.newValue1I(ssa.OpOffPtr, types.NewPtr(f.Type), f.Offset, addr) + s.instrumentFields(f.Type, offptr, kind) + } +} + +func (s *state) instrumentMove(t *types.Type, dst, src *ssa.Value) { + if base.Flag.MSan { + s.instrument2(t, dst, src, instrumentMove) + } else { + s.instrument(t, src, instrumentRead) + s.instrument(t, dst, instrumentWrite) + } +} + +func (s *state) instrument2(t *types.Type, addr, addr2 *ssa.Value, kind instrumentKind) { + if !s.curfn.InstrumentBody() { + return + } + + w := t.Size() + if w == 0 { + return // can't race on zero-sized things + } + + if ssa.IsSanitizerSafeAddr(addr) { + return + } + + var fn *obj.LSym + needWidth := false + + if addr2 != nil && kind != instrumentMove { + panic("instrument2: non-nil addr2 for non-move instrumentation") + } + + if base.Flag.MSan { + switch kind { + case instrumentRead: + fn = ir.Syms.Msanread + case instrumentWrite: + fn = ir.Syms.Msanwrite + case instrumentMove: + fn = ir.Syms.Msanmove + default: + panic("unreachable") + } + needWidth = true + } else if base.Flag.Race && t.NumComponents(types.CountBlankFields) > 1 { + // for composite objects we have to write every address + // because a write might happen to any subobject. + // composites with only one element don't have subobjects, though. + switch kind { + case instrumentRead: + fn = ir.Syms.Racereadrange + case instrumentWrite: + fn = ir.Syms.Racewriterange + default: + panic("unreachable") + } + needWidth = true + } else if base.Flag.Race { + // for non-composite objects we can write just the start + // address, as any write must write the first byte. + switch kind { + case instrumentRead: + fn = ir.Syms.Raceread + case instrumentWrite: + fn = ir.Syms.Racewrite + default: + panic("unreachable") + } + } else { + panic("unreachable") + } + + args := []*ssa.Value{addr} + if addr2 != nil { + args = append(args, addr2) + } + if needWidth { + args = append(args, s.constInt(types.Types[types.TUINTPTR], w)) + } + s.rtcall(fn, true, nil, args...) +} + +func (s *state) load(t *types.Type, src *ssa.Value) *ssa.Value { + s.instrumentFields(t, src, instrumentRead) + return s.rawLoad(t, src) +} + +func (s *state) rawLoad(t *types.Type, src *ssa.Value) *ssa.Value { + return s.newValue2(ssa.OpLoad, t, src, s.mem()) +} + +func (s *state) store(t *types.Type, dst, val *ssa.Value) { + s.vars[memVar] = s.newValue3A(ssa.OpStore, types.TypeMem, t, dst, val, s.mem()) +} + +func (s *state) zero(t *types.Type, dst *ssa.Value) { + s.instrument(t, dst, instrumentWrite) + store := s.newValue2I(ssa.OpZero, types.TypeMem, t.Size(), dst, s.mem()) + store.Aux = t + s.vars[memVar] = store +} + +func (s *state) move(t *types.Type, dst, src *ssa.Value) { + s.instrumentMove(t, dst, src) + store := s.newValue3I(ssa.OpMove, types.TypeMem, t.Size(), dst, src, s.mem()) + store.Aux = t + s.vars[memVar] = store +} + +// stmtList converts the statement list n to SSA and adds it to s. +func (s *state) stmtList(l ir.Nodes) { + for _, n := range l { + s.stmt(n) + } +} + +// stmt converts the statement n to SSA and adds it to s. +func (s *state) stmt(n ir.Node) { + if !(n.Op() == ir.OVARKILL || n.Op() == ir.OVARLIVE || n.Op() == ir.OVARDEF) { + // OVARKILL, OVARLIVE, and OVARDEF are invisible to the programmer, so we don't use their line numbers to avoid confusion in debugging. + s.pushLine(n.Pos()) + defer s.popLine() + } + + // If s.curBlock is nil, and n isn't a label (which might have an associated goto somewhere), + // then this code is dead. Stop here. + if s.curBlock == nil && n.Op() != ir.OLABEL { + return + } + + s.stmtList(n.Init()) + switch n.Op() { + + case ir.OBLOCK: + n := n.(*ir.BlockStmt) + s.stmtList(n.List) + + // No-ops + case ir.ODCLCONST, ir.ODCLTYPE, ir.OFALL: + + // Expression statements + case ir.OCALLFUNC: + n := n.(*ir.CallExpr) + if ir.IsIntrinsicCall(n) { + s.intrinsicCall(n) + return + } + fallthrough + + case ir.OCALLMETH, ir.OCALLINTER: + n := n.(*ir.CallExpr) + s.callResult(n, callNormal) + if n.Op() == ir.OCALLFUNC && n.X.Op() == ir.ONAME && n.X.(*ir.Name).Class_ == ir.PFUNC { + if fn := n.X.Sym().Name; base.Flag.CompilingRuntime && fn == "throw" || + n.X.Sym().Pkg == ir.Pkgs.Runtime && (fn == "throwinit" || fn == "gopanic" || fn == "panicwrap" || fn == "block" || fn == "panicmakeslicelen" || fn == "panicmakeslicecap") { + m := s.mem() + b := s.endBlock() + b.Kind = ssa.BlockExit + b.SetControl(m) + // TODO: never rewrite OPANIC to OCALLFUNC in the + // first place. Need to wait until all backends + // go through SSA. + } + } + case ir.ODEFER: + n := n.(*ir.GoDeferStmt) + if base.Debug.Defer > 0 { + var defertype string + if s.hasOpenDefers { + defertype = "open-coded" + } else if n.Esc() == ir.EscNever { + defertype = "stack-allocated" + } else { + defertype = "heap-allocated" + } + base.WarnfAt(n.Pos(), "%s defer", defertype) + } + if s.hasOpenDefers { + s.openDeferRecord(n.Call.(*ir.CallExpr)) + } else { + d := callDefer + if n.Esc() == ir.EscNever { + d = callDeferStack + } + s.callResult(n.Call.(*ir.CallExpr), d) + } + case ir.OGO: + n := n.(*ir.GoDeferStmt) + s.callResult(n.Call.(*ir.CallExpr), callGo) + + case ir.OAS2DOTTYPE: + n := n.(*ir.AssignListStmt) + res, resok := s.dottype(n.Rhs[0].(*ir.TypeAssertExpr), true) + deref := false + if !TypeOK(n.Rhs[0].Type()) { + if res.Op != ssa.OpLoad { + s.Fatalf("dottype of non-load") + } + mem := s.mem() + if mem.Op == ssa.OpVarKill { + mem = mem.Args[0] + } + if res.Args[1] != mem { + s.Fatalf("memory no longer live from 2-result dottype load") + } + deref = true + res = res.Args[0] + } + s.assign(n.Lhs[0], res, deref, 0) + s.assign(n.Lhs[1], resok, false, 0) + return + + case ir.OAS2FUNC: + // We come here only when it is an intrinsic call returning two values. + n := n.(*ir.AssignListStmt) + call := n.Rhs[0].(*ir.CallExpr) + if !ir.IsIntrinsicCall(call) { + s.Fatalf("non-intrinsic AS2FUNC not expanded %v", call) + } + v := s.intrinsicCall(call) + v1 := s.newValue1(ssa.OpSelect0, n.Lhs[0].Type(), v) + v2 := s.newValue1(ssa.OpSelect1, n.Lhs[1].Type(), v) + s.assign(n.Lhs[0], v1, false, 0) + s.assign(n.Lhs[1], v2, false, 0) + return + + case ir.ODCL: + n := n.(*ir.Decl) + if n.X.(*ir.Name).Class_ == ir.PAUTOHEAP { + s.Fatalf("DCL %v", n) + } + + case ir.OLABEL: + n := n.(*ir.LabelStmt) + sym := n.Label + lab := s.label(sym) + + // The label might already have a target block via a goto. + if lab.target == nil { + lab.target = s.f.NewBlock(ssa.BlockPlain) + } + + // Go to that label. + // (We pretend "label:" is preceded by "goto label", unless the predecessor is unreachable.) + if s.curBlock != nil { + b := s.endBlock() + b.AddEdgeTo(lab.target) + } + s.startBlock(lab.target) + + case ir.OGOTO: + n := n.(*ir.BranchStmt) + sym := n.Label + + lab := s.label(sym) + if lab.target == nil { + lab.target = s.f.NewBlock(ssa.BlockPlain) + } + + b := s.endBlock() + b.Pos = s.lastPos.WithIsStmt() // Do this even if b is an empty block. + b.AddEdgeTo(lab.target) + + case ir.OAS: + n := n.(*ir.AssignStmt) + if n.X == n.Y && n.X.Op() == ir.ONAME { + // An x=x assignment. No point in doing anything + // here. In addition, skipping this assignment + // prevents generating: + // VARDEF x + // COPY x -> x + // which is bad because x is incorrectly considered + // dead before the vardef. See issue #14904. + return + } + + // Evaluate RHS. + rhs := n.Y + if rhs != nil { + switch rhs.Op() { + case ir.OSTRUCTLIT, ir.OARRAYLIT, ir.OSLICELIT: + // All literals with nonzero fields have already been + // rewritten during walk. Any that remain are just T{} + // or equivalents. Use the zero value. + if !ir.IsZero(rhs) { + s.Fatalf("literal with nonzero value in SSA: %v", rhs) + } + rhs = nil + case ir.OAPPEND: + rhs := rhs.(*ir.CallExpr) + // Check whether we're writing the result of an append back to the same slice. + // If so, we handle it specially to avoid write barriers on the fast + // (non-growth) path. + if !ir.SameSafeExpr(n.X, rhs.Args[0]) || base.Flag.N != 0 { + break + } + // If the slice can be SSA'd, it'll be on the stack, + // so there will be no write barriers, + // so there's no need to attempt to prevent them. + if s.canSSA(n.X) { + if base.Debug.Append > 0 { // replicating old diagnostic message + base.WarnfAt(n.Pos(), "append: len-only update (in local slice)") + } + break + } + if base.Debug.Append > 0 { + base.WarnfAt(n.Pos(), "append: len-only update") + } + s.append(rhs, true) + return + } + } + + if ir.IsBlank(n.X) { + // _ = rhs + // Just evaluate rhs for side-effects. + if rhs != nil { + s.expr(rhs) + } + return + } + + var t *types.Type + if n.Y != nil { + t = n.Y.Type() + } else { + t = n.X.Type() + } + + var r *ssa.Value + deref := !TypeOK(t) + if deref { + if rhs == nil { + r = nil // Signal assign to use OpZero. + } else { + r = s.addr(rhs) + } + } else { + if rhs == nil { + r = s.zeroVal(t) + } else { + r = s.expr(rhs) + } + } + + var skip skipMask + if rhs != nil && (rhs.Op() == ir.OSLICE || rhs.Op() == ir.OSLICE3 || rhs.Op() == ir.OSLICESTR) && ir.SameSafeExpr(rhs.(*ir.SliceExpr).X, n.X) { + // We're assigning a slicing operation back to its source. + // Don't write back fields we aren't changing. See issue #14855. + rhs := rhs.(*ir.SliceExpr) + i, j, k := rhs.SliceBounds() + if i != nil && (i.Op() == ir.OLITERAL && i.Val().Kind() == constant.Int && ir.Int64Val(i) == 0) { + // [0:...] is the same as [:...] + i = nil + } + // TODO: detect defaults for len/cap also. + // Currently doesn't really work because (*p)[:len(*p)] appears here as: + // tmp = len(*p) + // (*p)[:tmp] + //if j != nil && (j.Op == OLEN && samesafeexpr(j.Left, n.Left)) { + // j = nil + //} + //if k != nil && (k.Op == OCAP && samesafeexpr(k.Left, n.Left)) { + // k = nil + //} + if i == nil { + skip |= skipPtr + if j == nil { + skip |= skipLen + } + if k == nil { + skip |= skipCap + } + } + } + + s.assign(n.X, r, deref, skip) + + case ir.OIF: + n := n.(*ir.IfStmt) + if ir.IsConst(n.Cond, constant.Bool) { + s.stmtList(n.Cond.Init()) + if ir.BoolVal(n.Cond) { + s.stmtList(n.Body) + } else { + s.stmtList(n.Else) + } + break + } + + bEnd := s.f.NewBlock(ssa.BlockPlain) + var likely int8 + if n.Likely { + likely = 1 + } + var bThen *ssa.Block + if len(n.Body) != 0 { + bThen = s.f.NewBlock(ssa.BlockPlain) + } else { + bThen = bEnd + } + var bElse *ssa.Block + if len(n.Else) != 0 { + bElse = s.f.NewBlock(ssa.BlockPlain) + } else { + bElse = bEnd + } + s.condBranch(n.Cond, bThen, bElse, likely) + + if len(n.Body) != 0 { + s.startBlock(bThen) + s.stmtList(n.Body) + if b := s.endBlock(); b != nil { + b.AddEdgeTo(bEnd) + } + } + if len(n.Else) != 0 { + s.startBlock(bElse) + s.stmtList(n.Else) + if b := s.endBlock(); b != nil { + b.AddEdgeTo(bEnd) + } + } + s.startBlock(bEnd) + + case ir.ORETURN: + n := n.(*ir.ReturnStmt) + s.stmtList(n.Results) + b := s.exit() + b.Pos = s.lastPos.WithIsStmt() + + case ir.ORETJMP: + n := n.(*ir.BranchStmt) + b := s.exit() + b.Kind = ssa.BlockRetJmp // override BlockRet + b.Aux = callTargetLSym(n.Label, s.curfn.LSym) + + case ir.OCONTINUE, ir.OBREAK: + n := n.(*ir.BranchStmt) + var to *ssa.Block + if n.Label == nil { + // plain break/continue + switch n.Op() { + case ir.OCONTINUE: + to = s.continueTo + case ir.OBREAK: + to = s.breakTo + } + } else { + // labeled break/continue; look up the target + sym := n.Label + lab := s.label(sym) + switch n.Op() { + case ir.OCONTINUE: + to = lab.continueTarget + case ir.OBREAK: + to = lab.breakTarget + } + } + + b := s.endBlock() + b.Pos = s.lastPos.WithIsStmt() // Do this even if b is an empty block. + b.AddEdgeTo(to) + + case ir.OFOR, ir.OFORUNTIL: + // OFOR: for Ninit; Left; Right { Nbody } + // cond (Left); body (Nbody); incr (Right) + // + // OFORUNTIL: for Ninit; Left; Right; List { Nbody } + // => body: { Nbody }; incr: Right; if Left { lateincr: List; goto body }; end: + n := n.(*ir.ForStmt) + bCond := s.f.NewBlock(ssa.BlockPlain) + bBody := s.f.NewBlock(ssa.BlockPlain) + bIncr := s.f.NewBlock(ssa.BlockPlain) + bEnd := s.f.NewBlock(ssa.BlockPlain) + + // ensure empty for loops have correct position; issue #30167 + bBody.Pos = n.Pos() + + // first, jump to condition test (OFOR) or body (OFORUNTIL) + b := s.endBlock() + if n.Op() == ir.OFOR { + b.AddEdgeTo(bCond) + // generate code to test condition + s.startBlock(bCond) + if n.Cond != nil { + s.condBranch(n.Cond, bBody, bEnd, 1) + } else { + b := s.endBlock() + b.Kind = ssa.BlockPlain + b.AddEdgeTo(bBody) + } + + } else { + b.AddEdgeTo(bBody) + } + + // set up for continue/break in body + prevContinue := s.continueTo + prevBreak := s.breakTo + s.continueTo = bIncr + s.breakTo = bEnd + var lab *ssaLabel + if sym := n.Label; sym != nil { + // labeled for loop + lab = s.label(sym) + lab.continueTarget = bIncr + lab.breakTarget = bEnd + } + + // generate body + s.startBlock(bBody) + s.stmtList(n.Body) + + // tear down continue/break + s.continueTo = prevContinue + s.breakTo = prevBreak + if lab != nil { + lab.continueTarget = nil + lab.breakTarget = nil + } + + // done with body, goto incr + if b := s.endBlock(); b != nil { + b.AddEdgeTo(bIncr) + } + + // generate incr (and, for OFORUNTIL, condition) + s.startBlock(bIncr) + if n.Post != nil { + s.stmt(n.Post) + } + if n.Op() == ir.OFOR { + if b := s.endBlock(); b != nil { + b.AddEdgeTo(bCond) + // It can happen that bIncr ends in a block containing only VARKILL, + // and that muddles the debugging experience. + if n.Op() != ir.OFORUNTIL && b.Pos == src.NoXPos { + b.Pos = bCond.Pos + } + } + } else { + // bCond is unused in OFORUNTIL, so repurpose it. + bLateIncr := bCond + // test condition + s.condBranch(n.Cond, bLateIncr, bEnd, 1) + // generate late increment + s.startBlock(bLateIncr) + s.stmtList(n.Late) + s.endBlock().AddEdgeTo(bBody) + } + + s.startBlock(bEnd) + + case ir.OSWITCH, ir.OSELECT: + // These have been mostly rewritten by the front end into their Nbody fields. + // Our main task is to correctly hook up any break statements. + bEnd := s.f.NewBlock(ssa.BlockPlain) + + prevBreak := s.breakTo + s.breakTo = bEnd + var sym *types.Sym + var body ir.Nodes + if n.Op() == ir.OSWITCH { + n := n.(*ir.SwitchStmt) + sym = n.Label + body = n.Compiled + } else { + n := n.(*ir.SelectStmt) + sym = n.Label + body = n.Compiled + } + + var lab *ssaLabel + if sym != nil { + // labeled + lab = s.label(sym) + lab.breakTarget = bEnd + } + + // generate body code + s.stmtList(body) + + s.breakTo = prevBreak + if lab != nil { + lab.breakTarget = nil + } + + // walk adds explicit OBREAK nodes to the end of all reachable code paths. + // If we still have a current block here, then mark it unreachable. + if s.curBlock != nil { + m := s.mem() + b := s.endBlock() + b.Kind = ssa.BlockExit + b.SetControl(m) + } + s.startBlock(bEnd) + + case ir.OVARDEF: + n := n.(*ir.UnaryExpr) + if !s.canSSA(n.X) { + s.vars[memVar] = s.newValue1Apos(ssa.OpVarDef, types.TypeMem, n.X.(*ir.Name), s.mem(), false) + } + case ir.OVARKILL: + // Insert a varkill op to record that a variable is no longer live. + // We only care about liveness info at call sites, so putting the + // varkill in the store chain is enough to keep it correctly ordered + // with respect to call ops. + n := n.(*ir.UnaryExpr) + if !s.canSSA(n.X) { + s.vars[memVar] = s.newValue1Apos(ssa.OpVarKill, types.TypeMem, n.X.(*ir.Name), s.mem(), false) + } + + case ir.OVARLIVE: + // Insert a varlive op to record that a variable is still live. + n := n.(*ir.UnaryExpr) + v := n.X.(*ir.Name) + if !v.Addrtaken() { + s.Fatalf("VARLIVE variable %v must have Addrtaken set", v) + } + switch v.Class_ { + case ir.PAUTO, ir.PPARAM, ir.PPARAMOUT: + default: + s.Fatalf("VARLIVE variable %v must be Auto or Arg", v) + } + s.vars[memVar] = s.newValue1A(ssa.OpVarLive, types.TypeMem, v, s.mem()) + + case ir.OCHECKNIL: + n := n.(*ir.UnaryExpr) + p := s.expr(n.X) + s.nilCheck(p) + + case ir.OINLMARK: + n := n.(*ir.InlineMarkStmt) + s.newValue1I(ssa.OpInlMark, types.TypeVoid, n.Index, s.mem()) + + default: + s.Fatalf("unhandled stmt %v", n.Op()) + } +} + +// If true, share as many open-coded defer exits as possible (with the downside of +// worse line-number information) +const shareDeferExits = false + +// exit processes any code that needs to be generated just before returning. +// It returns a BlockRet block that ends the control flow. Its control value +// will be set to the final memory state. +func (s *state) exit() *ssa.Block { + if s.hasdefer { + if s.hasOpenDefers { + if shareDeferExits && s.lastDeferExit != nil && len(s.openDefers) == s.lastDeferCount { + if s.curBlock.Kind != ssa.BlockPlain { + panic("Block for an exit should be BlockPlain") + } + s.curBlock.AddEdgeTo(s.lastDeferExit) + s.endBlock() + return s.lastDeferFinalBlock + } + s.openDeferExit() + } else { + s.rtcall(ir.Syms.Deferreturn, true, nil) + } + } + + // Run exit code. Typically, this code copies heap-allocated PPARAMOUT + // variables back to the stack. + s.stmtList(s.curfn.Exit) + + // Store SSAable PPARAMOUT variables back to stack locations. + for _, n := range s.returns { + addr := s.decladdrs[n] + val := s.variable(n, n.Type()) + s.vars[memVar] = s.newValue1A(ssa.OpVarDef, types.TypeMem, n, s.mem()) + s.store(n.Type(), addr, val) + // TODO: if val is ever spilled, we'd like to use the + // PPARAMOUT slot for spilling it. That won't happen + // currently. + } + + // Do actual return. + m := s.mem() + b := s.endBlock() + b.Kind = ssa.BlockRet + b.SetControl(m) + if s.hasdefer && s.hasOpenDefers { + s.lastDeferFinalBlock = b + } + return b +} + +type opAndType struct { + op ir.Op + etype types.Kind +} + +var opToSSA = map[opAndType]ssa.Op{ + opAndType{ir.OADD, types.TINT8}: ssa.OpAdd8, + opAndType{ir.OADD, types.TUINT8}: ssa.OpAdd8, + opAndType{ir.OADD, types.TINT16}: ssa.OpAdd16, + opAndType{ir.OADD, types.TUINT16}: ssa.OpAdd16, + opAndType{ir.OADD, types.TINT32}: ssa.OpAdd32, + opAndType{ir.OADD, types.TUINT32}: ssa.OpAdd32, + opAndType{ir.OADD, types.TINT64}: ssa.OpAdd64, + opAndType{ir.OADD, types.TUINT64}: ssa.OpAdd64, + opAndType{ir.OADD, types.TFLOAT32}: ssa.OpAdd32F, + opAndType{ir.OADD, types.TFLOAT64}: ssa.OpAdd64F, + + opAndType{ir.OSUB, types.TINT8}: ssa.OpSub8, + opAndType{ir.OSUB, types.TUINT8}: ssa.OpSub8, + opAndType{ir.OSUB, types.TINT16}: ssa.OpSub16, + opAndType{ir.OSUB, types.TUINT16}: ssa.OpSub16, + opAndType{ir.OSUB, types.TINT32}: ssa.OpSub32, + opAndType{ir.OSUB, types.TUINT32}: ssa.OpSub32, + opAndType{ir.OSUB, types.TINT64}: ssa.OpSub64, + opAndType{ir.OSUB, types.TUINT64}: ssa.OpSub64, + opAndType{ir.OSUB, types.TFLOAT32}: ssa.OpSub32F, + opAndType{ir.OSUB, types.TFLOAT64}: ssa.OpSub64F, + + opAndType{ir.ONOT, types.TBOOL}: ssa.OpNot, + + opAndType{ir.ONEG, types.TINT8}: ssa.OpNeg8, + opAndType{ir.ONEG, types.TUINT8}: ssa.OpNeg8, + opAndType{ir.ONEG, types.TINT16}: ssa.OpNeg16, + opAndType{ir.ONEG, types.TUINT16}: ssa.OpNeg16, + opAndType{ir.ONEG, types.TINT32}: ssa.OpNeg32, + opAndType{ir.ONEG, types.TUINT32}: ssa.OpNeg32, + opAndType{ir.ONEG, types.TINT64}: ssa.OpNeg64, + opAndType{ir.ONEG, types.TUINT64}: ssa.OpNeg64, + opAndType{ir.ONEG, types.TFLOAT32}: ssa.OpNeg32F, + opAndType{ir.ONEG, types.TFLOAT64}: ssa.OpNeg64F, + + opAndType{ir.OBITNOT, types.TINT8}: ssa.OpCom8, + opAndType{ir.OBITNOT, types.TUINT8}: ssa.OpCom8, + opAndType{ir.OBITNOT, types.TINT16}: ssa.OpCom16, + opAndType{ir.OBITNOT, types.TUINT16}: ssa.OpCom16, + opAndType{ir.OBITNOT, types.TINT32}: ssa.OpCom32, + opAndType{ir.OBITNOT, types.TUINT32}: ssa.OpCom32, + opAndType{ir.OBITNOT, types.TINT64}: ssa.OpCom64, + opAndType{ir.OBITNOT, types.TUINT64}: ssa.OpCom64, + + opAndType{ir.OIMAG, types.TCOMPLEX64}: ssa.OpComplexImag, + opAndType{ir.OIMAG, types.TCOMPLEX128}: ssa.OpComplexImag, + opAndType{ir.OREAL, types.TCOMPLEX64}: ssa.OpComplexReal, + opAndType{ir.OREAL, types.TCOMPLEX128}: ssa.OpComplexReal, + + opAndType{ir.OMUL, types.TINT8}: ssa.OpMul8, + opAndType{ir.OMUL, types.TUINT8}: ssa.OpMul8, + opAndType{ir.OMUL, types.TINT16}: ssa.OpMul16, + opAndType{ir.OMUL, types.TUINT16}: ssa.OpMul16, + opAndType{ir.OMUL, types.TINT32}: ssa.OpMul32, + opAndType{ir.OMUL, types.TUINT32}: ssa.OpMul32, + opAndType{ir.OMUL, types.TINT64}: ssa.OpMul64, + opAndType{ir.OMUL, types.TUINT64}: ssa.OpMul64, + opAndType{ir.OMUL, types.TFLOAT32}: ssa.OpMul32F, + opAndType{ir.OMUL, types.TFLOAT64}: ssa.OpMul64F, + + opAndType{ir.ODIV, types.TFLOAT32}: ssa.OpDiv32F, + opAndType{ir.ODIV, types.TFLOAT64}: ssa.OpDiv64F, + + opAndType{ir.ODIV, types.TINT8}: ssa.OpDiv8, + opAndType{ir.ODIV, types.TUINT8}: ssa.OpDiv8u, + opAndType{ir.ODIV, types.TINT16}: ssa.OpDiv16, + opAndType{ir.ODIV, types.TUINT16}: ssa.OpDiv16u, + opAndType{ir.ODIV, types.TINT32}: ssa.OpDiv32, + opAndType{ir.ODIV, types.TUINT32}: ssa.OpDiv32u, + opAndType{ir.ODIV, types.TINT64}: ssa.OpDiv64, + opAndType{ir.ODIV, types.TUINT64}: ssa.OpDiv64u, + + opAndType{ir.OMOD, types.TINT8}: ssa.OpMod8, + opAndType{ir.OMOD, types.TUINT8}: ssa.OpMod8u, + opAndType{ir.OMOD, types.TINT16}: ssa.OpMod16, + opAndType{ir.OMOD, types.TUINT16}: ssa.OpMod16u, + opAndType{ir.OMOD, types.TINT32}: ssa.OpMod32, + opAndType{ir.OMOD, types.TUINT32}: ssa.OpMod32u, + opAndType{ir.OMOD, types.TINT64}: ssa.OpMod64, + opAndType{ir.OMOD, types.TUINT64}: ssa.OpMod64u, + + opAndType{ir.OAND, types.TINT8}: ssa.OpAnd8, + opAndType{ir.OAND, types.TUINT8}: ssa.OpAnd8, + opAndType{ir.OAND, types.TINT16}: ssa.OpAnd16, + opAndType{ir.OAND, types.TUINT16}: ssa.OpAnd16, + opAndType{ir.OAND, types.TINT32}: ssa.OpAnd32, + opAndType{ir.OAND, types.TUINT32}: ssa.OpAnd32, + opAndType{ir.OAND, types.TINT64}: ssa.OpAnd64, + opAndType{ir.OAND, types.TUINT64}: ssa.OpAnd64, + + opAndType{ir.OOR, types.TINT8}: ssa.OpOr8, + opAndType{ir.OOR, types.TUINT8}: ssa.OpOr8, + opAndType{ir.OOR, types.TINT16}: ssa.OpOr16, + opAndType{ir.OOR, types.TUINT16}: ssa.OpOr16, + opAndType{ir.OOR, types.TINT32}: ssa.OpOr32, + opAndType{ir.OOR, types.TUINT32}: ssa.OpOr32, + opAndType{ir.OOR, types.TINT64}: ssa.OpOr64, + opAndType{ir.OOR, types.TUINT64}: ssa.OpOr64, + + opAndType{ir.OXOR, types.TINT8}: ssa.OpXor8, + opAndType{ir.OXOR, types.TUINT8}: ssa.OpXor8, + opAndType{ir.OXOR, types.TINT16}: ssa.OpXor16, + opAndType{ir.OXOR, types.TUINT16}: ssa.OpXor16, + opAndType{ir.OXOR, types.TINT32}: ssa.OpXor32, + opAndType{ir.OXOR, types.TUINT32}: ssa.OpXor32, + opAndType{ir.OXOR, types.TINT64}: ssa.OpXor64, + opAndType{ir.OXOR, types.TUINT64}: ssa.OpXor64, + + opAndType{ir.OEQ, types.TBOOL}: ssa.OpEqB, + opAndType{ir.OEQ, types.TINT8}: ssa.OpEq8, + opAndType{ir.OEQ, types.TUINT8}: ssa.OpEq8, + opAndType{ir.OEQ, types.TINT16}: ssa.OpEq16, + opAndType{ir.OEQ, types.TUINT16}: ssa.OpEq16, + opAndType{ir.OEQ, types.TINT32}: ssa.OpEq32, + opAndType{ir.OEQ, types.TUINT32}: ssa.OpEq32, + opAndType{ir.OEQ, types.TINT64}: ssa.OpEq64, + opAndType{ir.OEQ, types.TUINT64}: ssa.OpEq64, + opAndType{ir.OEQ, types.TINTER}: ssa.OpEqInter, + opAndType{ir.OEQ, types.TSLICE}: ssa.OpEqSlice, + opAndType{ir.OEQ, types.TFUNC}: ssa.OpEqPtr, + opAndType{ir.OEQ, types.TMAP}: ssa.OpEqPtr, + opAndType{ir.OEQ, types.TCHAN}: ssa.OpEqPtr, + opAndType{ir.OEQ, types.TPTR}: ssa.OpEqPtr, + opAndType{ir.OEQ, types.TUINTPTR}: ssa.OpEqPtr, + opAndType{ir.OEQ, types.TUNSAFEPTR}: ssa.OpEqPtr, + opAndType{ir.OEQ, types.TFLOAT64}: ssa.OpEq64F, + opAndType{ir.OEQ, types.TFLOAT32}: ssa.OpEq32F, + + opAndType{ir.ONE, types.TBOOL}: ssa.OpNeqB, + opAndType{ir.ONE, types.TINT8}: ssa.OpNeq8, + opAndType{ir.ONE, types.TUINT8}: ssa.OpNeq8, + opAndType{ir.ONE, types.TINT16}: ssa.OpNeq16, + opAndType{ir.ONE, types.TUINT16}: ssa.OpNeq16, + opAndType{ir.ONE, types.TINT32}: ssa.OpNeq32, + opAndType{ir.ONE, types.TUINT32}: ssa.OpNeq32, + opAndType{ir.ONE, types.TINT64}: ssa.OpNeq64, + opAndType{ir.ONE, types.TUINT64}: ssa.OpNeq64, + opAndType{ir.ONE, types.TINTER}: ssa.OpNeqInter, + opAndType{ir.ONE, types.TSLICE}: ssa.OpNeqSlice, + opAndType{ir.ONE, types.TFUNC}: ssa.OpNeqPtr, + opAndType{ir.ONE, types.TMAP}: ssa.OpNeqPtr, + opAndType{ir.ONE, types.TCHAN}: ssa.OpNeqPtr, + opAndType{ir.ONE, types.TPTR}: ssa.OpNeqPtr, + opAndType{ir.ONE, types.TUINTPTR}: ssa.OpNeqPtr, + opAndType{ir.ONE, types.TUNSAFEPTR}: ssa.OpNeqPtr, + opAndType{ir.ONE, types.TFLOAT64}: ssa.OpNeq64F, + opAndType{ir.ONE, types.TFLOAT32}: ssa.OpNeq32F, + + opAndType{ir.OLT, types.TINT8}: ssa.OpLess8, + opAndType{ir.OLT, types.TUINT8}: ssa.OpLess8U, + opAndType{ir.OLT, types.TINT16}: ssa.OpLess16, + opAndType{ir.OLT, types.TUINT16}: ssa.OpLess16U, + opAndType{ir.OLT, types.TINT32}: ssa.OpLess32, + opAndType{ir.OLT, types.TUINT32}: ssa.OpLess32U, + opAndType{ir.OLT, types.TINT64}: ssa.OpLess64, + opAndType{ir.OLT, types.TUINT64}: ssa.OpLess64U, + opAndType{ir.OLT, types.TFLOAT64}: ssa.OpLess64F, + opAndType{ir.OLT, types.TFLOAT32}: ssa.OpLess32F, + + opAndType{ir.OLE, types.TINT8}: ssa.OpLeq8, + opAndType{ir.OLE, types.TUINT8}: ssa.OpLeq8U, + opAndType{ir.OLE, types.TINT16}: ssa.OpLeq16, + opAndType{ir.OLE, types.TUINT16}: ssa.OpLeq16U, + opAndType{ir.OLE, types.TINT32}: ssa.OpLeq32, + opAndType{ir.OLE, types.TUINT32}: ssa.OpLeq32U, + opAndType{ir.OLE, types.TINT64}: ssa.OpLeq64, + opAndType{ir.OLE, types.TUINT64}: ssa.OpLeq64U, + opAndType{ir.OLE, types.TFLOAT64}: ssa.OpLeq64F, + opAndType{ir.OLE, types.TFLOAT32}: ssa.OpLeq32F, +} + +func (s *state) concreteEtype(t *types.Type) types.Kind { + e := t.Kind() + switch e { + default: + return e + case types.TINT: + if s.config.PtrSize == 8 { + return types.TINT64 + } + return types.TINT32 + case types.TUINT: + if s.config.PtrSize == 8 { + return types.TUINT64 + } + return types.TUINT32 + case types.TUINTPTR: + if s.config.PtrSize == 8 { + return types.TUINT64 + } + return types.TUINT32 + } +} + +func (s *state) ssaOp(op ir.Op, t *types.Type) ssa.Op { + etype := s.concreteEtype(t) + x, ok := opToSSA[opAndType{op, etype}] + if !ok { + s.Fatalf("unhandled binary op %v %s", op, etype) + } + return x +} + +type opAndTwoTypes struct { + op ir.Op + etype1 types.Kind + etype2 types.Kind +} + +type twoTypes struct { + etype1 types.Kind + etype2 types.Kind +} + +type twoOpsAndType struct { + op1 ssa.Op + op2 ssa.Op + intermediateType types.Kind +} + +var fpConvOpToSSA = map[twoTypes]twoOpsAndType{ + + twoTypes{types.TINT8, types.TFLOAT32}: twoOpsAndType{ssa.OpSignExt8to32, ssa.OpCvt32to32F, types.TINT32}, + twoTypes{types.TINT16, types.TFLOAT32}: twoOpsAndType{ssa.OpSignExt16to32, ssa.OpCvt32to32F, types.TINT32}, + twoTypes{types.TINT32, types.TFLOAT32}: twoOpsAndType{ssa.OpCopy, ssa.OpCvt32to32F, types.TINT32}, + twoTypes{types.TINT64, types.TFLOAT32}: twoOpsAndType{ssa.OpCopy, ssa.OpCvt64to32F, types.TINT64}, + + twoTypes{types.TINT8, types.TFLOAT64}: twoOpsAndType{ssa.OpSignExt8to32, ssa.OpCvt32to64F, types.TINT32}, + twoTypes{types.TINT16, types.TFLOAT64}: twoOpsAndType{ssa.OpSignExt16to32, ssa.OpCvt32to64F, types.TINT32}, + twoTypes{types.TINT32, types.TFLOAT64}: twoOpsAndType{ssa.OpCopy, ssa.OpCvt32to64F, types.TINT32}, + twoTypes{types.TINT64, types.TFLOAT64}: twoOpsAndType{ssa.OpCopy, ssa.OpCvt64to64F, types.TINT64}, + + twoTypes{types.TFLOAT32, types.TINT8}: twoOpsAndType{ssa.OpCvt32Fto32, ssa.OpTrunc32to8, types.TINT32}, + twoTypes{types.TFLOAT32, types.TINT16}: twoOpsAndType{ssa.OpCvt32Fto32, ssa.OpTrunc32to16, types.TINT32}, + twoTypes{types.TFLOAT32, types.TINT32}: twoOpsAndType{ssa.OpCvt32Fto32, ssa.OpCopy, types.TINT32}, + twoTypes{types.TFLOAT32, types.TINT64}: twoOpsAndType{ssa.OpCvt32Fto64, ssa.OpCopy, types.TINT64}, + + twoTypes{types.TFLOAT64, types.TINT8}: twoOpsAndType{ssa.OpCvt64Fto32, ssa.OpTrunc32to8, types.TINT32}, + twoTypes{types.TFLOAT64, types.TINT16}: twoOpsAndType{ssa.OpCvt64Fto32, ssa.OpTrunc32to16, types.TINT32}, + twoTypes{types.TFLOAT64, types.TINT32}: twoOpsAndType{ssa.OpCvt64Fto32, ssa.OpCopy, types.TINT32}, + twoTypes{types.TFLOAT64, types.TINT64}: twoOpsAndType{ssa.OpCvt64Fto64, ssa.OpCopy, types.TINT64}, + // unsigned + twoTypes{types.TUINT8, types.TFLOAT32}: twoOpsAndType{ssa.OpZeroExt8to32, ssa.OpCvt32to32F, types.TINT32}, + twoTypes{types.TUINT16, types.TFLOAT32}: twoOpsAndType{ssa.OpZeroExt16to32, ssa.OpCvt32to32F, types.TINT32}, + twoTypes{types.TUINT32, types.TFLOAT32}: twoOpsAndType{ssa.OpZeroExt32to64, ssa.OpCvt64to32F, types.TINT64}, // go wide to dodge unsigned + twoTypes{types.TUINT64, types.TFLOAT32}: twoOpsAndType{ssa.OpCopy, ssa.OpInvalid, types.TUINT64}, // Cvt64Uto32F, branchy code expansion instead + + twoTypes{types.TUINT8, types.TFLOAT64}: twoOpsAndType{ssa.OpZeroExt8to32, ssa.OpCvt32to64F, types.TINT32}, + twoTypes{types.TUINT16, types.TFLOAT64}: twoOpsAndType{ssa.OpZeroExt16to32, ssa.OpCvt32to64F, types.TINT32}, + twoTypes{types.TUINT32, types.TFLOAT64}: twoOpsAndType{ssa.OpZeroExt32to64, ssa.OpCvt64to64F, types.TINT64}, // go wide to dodge unsigned + twoTypes{types.TUINT64, types.TFLOAT64}: twoOpsAndType{ssa.OpCopy, ssa.OpInvalid, types.TUINT64}, // Cvt64Uto64F, branchy code expansion instead + + twoTypes{types.TFLOAT32, types.TUINT8}: twoOpsAndType{ssa.OpCvt32Fto32, ssa.OpTrunc32to8, types.TINT32}, + twoTypes{types.TFLOAT32, types.TUINT16}: twoOpsAndType{ssa.OpCvt32Fto32, ssa.OpTrunc32to16, types.TINT32}, + twoTypes{types.TFLOAT32, types.TUINT32}: twoOpsAndType{ssa.OpCvt32Fto64, ssa.OpTrunc64to32, types.TINT64}, // go wide to dodge unsigned + twoTypes{types.TFLOAT32, types.TUINT64}: twoOpsAndType{ssa.OpInvalid, ssa.OpCopy, types.TUINT64}, // Cvt32Fto64U, branchy code expansion instead + + twoTypes{types.TFLOAT64, types.TUINT8}: twoOpsAndType{ssa.OpCvt64Fto32, ssa.OpTrunc32to8, types.TINT32}, + twoTypes{types.TFLOAT64, types.TUINT16}: twoOpsAndType{ssa.OpCvt64Fto32, ssa.OpTrunc32to16, types.TINT32}, + twoTypes{types.TFLOAT64, types.TUINT32}: twoOpsAndType{ssa.OpCvt64Fto64, ssa.OpTrunc64to32, types.TINT64}, // go wide to dodge unsigned + twoTypes{types.TFLOAT64, types.TUINT64}: twoOpsAndType{ssa.OpInvalid, ssa.OpCopy, types.TUINT64}, // Cvt64Fto64U, branchy code expansion instead + + // float + twoTypes{types.TFLOAT64, types.TFLOAT32}: twoOpsAndType{ssa.OpCvt64Fto32F, ssa.OpCopy, types.TFLOAT32}, + twoTypes{types.TFLOAT64, types.TFLOAT64}: twoOpsAndType{ssa.OpRound64F, ssa.OpCopy, types.TFLOAT64}, + twoTypes{types.TFLOAT32, types.TFLOAT32}: twoOpsAndType{ssa.OpRound32F, ssa.OpCopy, types.TFLOAT32}, + twoTypes{types.TFLOAT32, types.TFLOAT64}: twoOpsAndType{ssa.OpCvt32Fto64F, ssa.OpCopy, types.TFLOAT64}, +} + +// this map is used only for 32-bit arch, and only includes the difference +// on 32-bit arch, don't use int64<->float conversion for uint32 +var fpConvOpToSSA32 = map[twoTypes]twoOpsAndType{ + twoTypes{types.TUINT32, types.TFLOAT32}: twoOpsAndType{ssa.OpCopy, ssa.OpCvt32Uto32F, types.TUINT32}, + twoTypes{types.TUINT32, types.TFLOAT64}: twoOpsAndType{ssa.OpCopy, ssa.OpCvt32Uto64F, types.TUINT32}, + twoTypes{types.TFLOAT32, types.TUINT32}: twoOpsAndType{ssa.OpCvt32Fto32U, ssa.OpCopy, types.TUINT32}, + twoTypes{types.TFLOAT64, types.TUINT32}: twoOpsAndType{ssa.OpCvt64Fto32U, ssa.OpCopy, types.TUINT32}, +} + +// uint64<->float conversions, only on machines that have instructions for that +var uint64fpConvOpToSSA = map[twoTypes]twoOpsAndType{ + twoTypes{types.TUINT64, types.TFLOAT32}: twoOpsAndType{ssa.OpCopy, ssa.OpCvt64Uto32F, types.TUINT64}, + twoTypes{types.TUINT64, types.TFLOAT64}: twoOpsAndType{ssa.OpCopy, ssa.OpCvt64Uto64F, types.TUINT64}, + twoTypes{types.TFLOAT32, types.TUINT64}: twoOpsAndType{ssa.OpCvt32Fto64U, ssa.OpCopy, types.TUINT64}, + twoTypes{types.TFLOAT64, types.TUINT64}: twoOpsAndType{ssa.OpCvt64Fto64U, ssa.OpCopy, types.TUINT64}, +} + +var shiftOpToSSA = map[opAndTwoTypes]ssa.Op{ + opAndTwoTypes{ir.OLSH, types.TINT8, types.TUINT8}: ssa.OpLsh8x8, + opAndTwoTypes{ir.OLSH, types.TUINT8, types.TUINT8}: ssa.OpLsh8x8, + opAndTwoTypes{ir.OLSH, types.TINT8, types.TUINT16}: ssa.OpLsh8x16, + opAndTwoTypes{ir.OLSH, types.TUINT8, types.TUINT16}: ssa.OpLsh8x16, + opAndTwoTypes{ir.OLSH, types.TINT8, types.TUINT32}: ssa.OpLsh8x32, + opAndTwoTypes{ir.OLSH, types.TUINT8, types.TUINT32}: ssa.OpLsh8x32, + opAndTwoTypes{ir.OLSH, types.TINT8, types.TUINT64}: ssa.OpLsh8x64, + opAndTwoTypes{ir.OLSH, types.TUINT8, types.TUINT64}: ssa.OpLsh8x64, + + opAndTwoTypes{ir.OLSH, types.TINT16, types.TUINT8}: ssa.OpLsh16x8, + opAndTwoTypes{ir.OLSH, types.TUINT16, types.TUINT8}: ssa.OpLsh16x8, + opAndTwoTypes{ir.OLSH, types.TINT16, types.TUINT16}: ssa.OpLsh16x16, + opAndTwoTypes{ir.OLSH, types.TUINT16, types.TUINT16}: ssa.OpLsh16x16, + opAndTwoTypes{ir.OLSH, types.TINT16, types.TUINT32}: ssa.OpLsh16x32, + opAndTwoTypes{ir.OLSH, types.TUINT16, types.TUINT32}: ssa.OpLsh16x32, + opAndTwoTypes{ir.OLSH, types.TINT16, types.TUINT64}: ssa.OpLsh16x64, + opAndTwoTypes{ir.OLSH, types.TUINT16, types.TUINT64}: ssa.OpLsh16x64, + + opAndTwoTypes{ir.OLSH, types.TINT32, types.TUINT8}: ssa.OpLsh32x8, + opAndTwoTypes{ir.OLSH, types.TUINT32, types.TUINT8}: ssa.OpLsh32x8, + opAndTwoTypes{ir.OLSH, types.TINT32, types.TUINT16}: ssa.OpLsh32x16, + opAndTwoTypes{ir.OLSH, types.TUINT32, types.TUINT16}: ssa.OpLsh32x16, + opAndTwoTypes{ir.OLSH, types.TINT32, types.TUINT32}: ssa.OpLsh32x32, + opAndTwoTypes{ir.OLSH, types.TUINT32, types.TUINT32}: ssa.OpLsh32x32, + opAndTwoTypes{ir.OLSH, types.TINT32, types.TUINT64}: ssa.OpLsh32x64, + opAndTwoTypes{ir.OLSH, types.TUINT32, types.TUINT64}: ssa.OpLsh32x64, + + opAndTwoTypes{ir.OLSH, types.TINT64, types.TUINT8}: ssa.OpLsh64x8, + opAndTwoTypes{ir.OLSH, types.TUINT64, types.TUINT8}: ssa.OpLsh64x8, + opAndTwoTypes{ir.OLSH, types.TINT64, types.TUINT16}: ssa.OpLsh64x16, + opAndTwoTypes{ir.OLSH, types.TUINT64, types.TUINT16}: ssa.OpLsh64x16, + opAndTwoTypes{ir.OLSH, types.TINT64, types.TUINT32}: ssa.OpLsh64x32, + opAndTwoTypes{ir.OLSH, types.TUINT64, types.TUINT32}: ssa.OpLsh64x32, + opAndTwoTypes{ir.OLSH, types.TINT64, types.TUINT64}: ssa.OpLsh64x64, + opAndTwoTypes{ir.OLSH, types.TUINT64, types.TUINT64}: ssa.OpLsh64x64, + + opAndTwoTypes{ir.ORSH, types.TINT8, types.TUINT8}: ssa.OpRsh8x8, + opAndTwoTypes{ir.ORSH, types.TUINT8, types.TUINT8}: ssa.OpRsh8Ux8, + opAndTwoTypes{ir.ORSH, types.TINT8, types.TUINT16}: ssa.OpRsh8x16, + opAndTwoTypes{ir.ORSH, types.TUINT8, types.TUINT16}: ssa.OpRsh8Ux16, + opAndTwoTypes{ir.ORSH, types.TINT8, types.TUINT32}: ssa.OpRsh8x32, + opAndTwoTypes{ir.ORSH, types.TUINT8, types.TUINT32}: ssa.OpRsh8Ux32, + opAndTwoTypes{ir.ORSH, types.TINT8, types.TUINT64}: ssa.OpRsh8x64, + opAndTwoTypes{ir.ORSH, types.TUINT8, types.TUINT64}: ssa.OpRsh8Ux64, + + opAndTwoTypes{ir.ORSH, types.TINT16, types.TUINT8}: ssa.OpRsh16x8, + opAndTwoTypes{ir.ORSH, types.TUINT16, types.TUINT8}: ssa.OpRsh16Ux8, + opAndTwoTypes{ir.ORSH, types.TINT16, types.TUINT16}: ssa.OpRsh16x16, + opAndTwoTypes{ir.ORSH, types.TUINT16, types.TUINT16}: ssa.OpRsh16Ux16, + opAndTwoTypes{ir.ORSH, types.TINT16, types.TUINT32}: ssa.OpRsh16x32, + opAndTwoTypes{ir.ORSH, types.TUINT16, types.TUINT32}: ssa.OpRsh16Ux32, + opAndTwoTypes{ir.ORSH, types.TINT16, types.TUINT64}: ssa.OpRsh16x64, + opAndTwoTypes{ir.ORSH, types.TUINT16, types.TUINT64}: ssa.OpRsh16Ux64, + + opAndTwoTypes{ir.ORSH, types.TINT32, types.TUINT8}: ssa.OpRsh32x8, + opAndTwoTypes{ir.ORSH, types.TUINT32, types.TUINT8}: ssa.OpRsh32Ux8, + opAndTwoTypes{ir.ORSH, types.TINT32, types.TUINT16}: ssa.OpRsh32x16, + opAndTwoTypes{ir.ORSH, types.TUINT32, types.TUINT16}: ssa.OpRsh32Ux16, + opAndTwoTypes{ir.ORSH, types.TINT32, types.TUINT32}: ssa.OpRsh32x32, + opAndTwoTypes{ir.ORSH, types.TUINT32, types.TUINT32}: ssa.OpRsh32Ux32, + opAndTwoTypes{ir.ORSH, types.TINT32, types.TUINT64}: ssa.OpRsh32x64, + opAndTwoTypes{ir.ORSH, types.TUINT32, types.TUINT64}: ssa.OpRsh32Ux64, + + opAndTwoTypes{ir.ORSH, types.TINT64, types.TUINT8}: ssa.OpRsh64x8, + opAndTwoTypes{ir.ORSH, types.TUINT64, types.TUINT8}: ssa.OpRsh64Ux8, + opAndTwoTypes{ir.ORSH, types.TINT64, types.TUINT16}: ssa.OpRsh64x16, + opAndTwoTypes{ir.ORSH, types.TUINT64, types.TUINT16}: ssa.OpRsh64Ux16, + opAndTwoTypes{ir.ORSH, types.TINT64, types.TUINT32}: ssa.OpRsh64x32, + opAndTwoTypes{ir.ORSH, types.TUINT64, types.TUINT32}: ssa.OpRsh64Ux32, + opAndTwoTypes{ir.ORSH, types.TINT64, types.TUINT64}: ssa.OpRsh64x64, + opAndTwoTypes{ir.ORSH, types.TUINT64, types.TUINT64}: ssa.OpRsh64Ux64, +} + +func (s *state) ssaShiftOp(op ir.Op, t *types.Type, u *types.Type) ssa.Op { + etype1 := s.concreteEtype(t) + etype2 := s.concreteEtype(u) + x, ok := shiftOpToSSA[opAndTwoTypes{op, etype1, etype2}] + if !ok { + s.Fatalf("unhandled shift op %v etype=%s/%s", op, etype1, etype2) + } + return x +} + +// expr converts the expression n to ssa, adds it to s and returns the ssa result. +func (s *state) expr(n ir.Node) *ssa.Value { + if ir.HasUniquePos(n) { + // ONAMEs and named OLITERALs have the line number + // of the decl, not the use. See issue 14742. + s.pushLine(n.Pos()) + defer s.popLine() + } + + s.stmtList(n.Init()) + switch n.Op() { + case ir.OBYTES2STRTMP: + n := n.(*ir.ConvExpr) + slice := s.expr(n.X) + ptr := s.newValue1(ssa.OpSlicePtr, s.f.Config.Types.BytePtr, slice) + len := s.newValue1(ssa.OpSliceLen, types.Types[types.TINT], slice) + return s.newValue2(ssa.OpStringMake, n.Type(), ptr, len) + case ir.OSTR2BYTESTMP: + n := n.(*ir.ConvExpr) + str := s.expr(n.X) + ptr := s.newValue1(ssa.OpStringPtr, s.f.Config.Types.BytePtr, str) + len := s.newValue1(ssa.OpStringLen, types.Types[types.TINT], str) + return s.newValue3(ssa.OpSliceMake, n.Type(), ptr, len, len) + case ir.OCFUNC: + n := n.(*ir.UnaryExpr) + aux := n.X.Sym().Linksym() + return s.entryNewValue1A(ssa.OpAddr, n.Type(), aux, s.sb) + case ir.OMETHEXPR: + n := n.(*ir.MethodExpr) + sym := staticdata.FuncSym(n.FuncName().Sym()).Linksym() + return s.entryNewValue1A(ssa.OpAddr, types.NewPtr(n.Type()), sym, s.sb) + case ir.ONAME: + n := n.(*ir.Name) + if n.Class_ == ir.PFUNC { + // "value" of a function is the address of the function's closure + sym := staticdata.FuncSym(n.Sym()).Linksym() + return s.entryNewValue1A(ssa.OpAddr, types.NewPtr(n.Type()), sym, s.sb) + } + if s.canSSA(n) { + return s.variable(n, n.Type()) + } + addr := s.addr(n) + return s.load(n.Type(), addr) + case ir.ONAMEOFFSET: + n := n.(*ir.NameOffsetExpr) + if s.canSSAName(n.Name_) && TypeOK(n.Type()) { + return s.variable(n, n.Type()) + } + addr := s.addr(n) + return s.load(n.Type(), addr) + case ir.OCLOSUREREAD: + addr := s.addr(n) + return s.load(n.Type(), addr) + case ir.ONIL: + n := n.(*ir.NilExpr) + t := n.Type() + switch { + case t.IsSlice(): + return s.constSlice(t) + case t.IsInterface(): + return s.constInterface(t) + default: + return s.constNil(t) + } + case ir.OLITERAL: + switch u := n.Val(); u.Kind() { + case constant.Int: + i := ir.IntVal(n.Type(), u) + switch n.Type().Size() { + case 1: + return s.constInt8(n.Type(), int8(i)) + case 2: + return s.constInt16(n.Type(), int16(i)) + case 4: + return s.constInt32(n.Type(), int32(i)) + case 8: + return s.constInt64(n.Type(), i) + default: + s.Fatalf("bad integer size %d", n.Type().Size()) + return nil + } + case constant.String: + i := constant.StringVal(u) + if i == "" { + return s.constEmptyString(n.Type()) + } + return s.entryNewValue0A(ssa.OpConstString, n.Type(), ssa.StringToAux(i)) + case constant.Bool: + return s.constBool(constant.BoolVal(u)) + case constant.Float: + f, _ := constant.Float64Val(u) + switch n.Type().Size() { + case 4: + return s.constFloat32(n.Type(), f) + case 8: + return s.constFloat64(n.Type(), f) + default: + s.Fatalf("bad float size %d", n.Type().Size()) + return nil + } + case constant.Complex: + re, _ := constant.Float64Val(constant.Real(u)) + im, _ := constant.Float64Val(constant.Imag(u)) + switch n.Type().Size() { + case 8: + pt := types.Types[types.TFLOAT32] + return s.newValue2(ssa.OpComplexMake, n.Type(), + s.constFloat32(pt, re), + s.constFloat32(pt, im)) + case 16: + pt := types.Types[types.TFLOAT64] + return s.newValue2(ssa.OpComplexMake, n.Type(), + s.constFloat64(pt, re), + s.constFloat64(pt, im)) + default: + s.Fatalf("bad complex size %d", n.Type().Size()) + return nil + } + default: + s.Fatalf("unhandled OLITERAL %v", u.Kind()) + return nil + } + case ir.OCONVNOP: + n := n.(*ir.ConvExpr) + to := n.Type() + from := n.X.Type() + + // Assume everything will work out, so set up our return value. + // Anything interesting that happens from here is a fatal. + x := s.expr(n.X) + if to == from { + return x + } + + // Special case for not confusing GC and liveness. + // We don't want pointers accidentally classified + // as not-pointers or vice-versa because of copy + // elision. + if to.IsPtrShaped() != from.IsPtrShaped() { + return s.newValue2(ssa.OpConvert, to, x, s.mem()) + } + + v := s.newValue1(ssa.OpCopy, to, x) // ensure that v has the right type + + // CONVNOP closure + if to.Kind() == types.TFUNC && from.IsPtrShaped() { + return v + } + + // named <--> unnamed type or typed <--> untyped const + if from.Kind() == to.Kind() { + return v + } + + // unsafe.Pointer <--> *T + if to.IsUnsafePtr() && from.IsPtrShaped() || from.IsUnsafePtr() && to.IsPtrShaped() { + return v + } + + // map <--> *hmap + if to.Kind() == types.TMAP && from.IsPtr() && + to.MapType().Hmap == from.Elem() { + return v + } + + types.CalcSize(from) + types.CalcSize(to) + if from.Width != to.Width { + s.Fatalf("CONVNOP width mismatch %v (%d) -> %v (%d)\n", from, from.Width, to, to.Width) + return nil + } + if etypesign(from.Kind()) != etypesign(to.Kind()) { + s.Fatalf("CONVNOP sign mismatch %v (%s) -> %v (%s)\n", from, from.Kind(), to, to.Kind()) + return nil + } + + if base.Flag.Cfg.Instrumenting { + // These appear to be fine, but they fail the + // integer constraint below, so okay them here. + // Sample non-integer conversion: map[string]string -> *uint8 + return v + } + + if etypesign(from.Kind()) == 0 { + s.Fatalf("CONVNOP unrecognized non-integer %v -> %v\n", from, to) + return nil + } + + // integer, same width, same sign + return v + + case ir.OCONV: + n := n.(*ir.ConvExpr) + x := s.expr(n.X) + ft := n.X.Type() // from type + tt := n.Type() // to type + if ft.IsBoolean() && tt.IsKind(types.TUINT8) { + // Bool -> uint8 is generated internally when indexing into runtime.staticbyte. + return s.newValue1(ssa.OpCopy, n.Type(), x) + } + if ft.IsInteger() && tt.IsInteger() { + var op ssa.Op + if tt.Size() == ft.Size() { + op = ssa.OpCopy + } else if tt.Size() < ft.Size() { + // truncation + switch 10*ft.Size() + tt.Size() { + case 21: + op = ssa.OpTrunc16to8 + case 41: + op = ssa.OpTrunc32to8 + case 42: + op = ssa.OpTrunc32to16 + case 81: + op = ssa.OpTrunc64to8 + case 82: + op = ssa.OpTrunc64to16 + case 84: + op = ssa.OpTrunc64to32 + default: + s.Fatalf("weird integer truncation %v -> %v", ft, tt) + } + } else if ft.IsSigned() { + // sign extension + switch 10*ft.Size() + tt.Size() { + case 12: + op = ssa.OpSignExt8to16 + case 14: + op = ssa.OpSignExt8to32 + case 18: + op = ssa.OpSignExt8to64 + case 24: + op = ssa.OpSignExt16to32 + case 28: + op = ssa.OpSignExt16to64 + case 48: + op = ssa.OpSignExt32to64 + default: + s.Fatalf("bad integer sign extension %v -> %v", ft, tt) + } + } else { + // zero extension + switch 10*ft.Size() + tt.Size() { + case 12: + op = ssa.OpZeroExt8to16 + case 14: + op = ssa.OpZeroExt8to32 + case 18: + op = ssa.OpZeroExt8to64 + case 24: + op = ssa.OpZeroExt16to32 + case 28: + op = ssa.OpZeroExt16to64 + case 48: + op = ssa.OpZeroExt32to64 + default: + s.Fatalf("weird integer sign extension %v -> %v", ft, tt) + } + } + return s.newValue1(op, n.Type(), x) + } + + if ft.IsFloat() || tt.IsFloat() { + conv, ok := fpConvOpToSSA[twoTypes{s.concreteEtype(ft), s.concreteEtype(tt)}] + if s.config.RegSize == 4 && Arch.LinkArch.Family != sys.MIPS && !s.softFloat { + if conv1, ok1 := fpConvOpToSSA32[twoTypes{s.concreteEtype(ft), s.concreteEtype(tt)}]; ok1 { + conv = conv1 + } + } + if Arch.LinkArch.Family == sys.ARM64 || Arch.LinkArch.Family == sys.Wasm || Arch.LinkArch.Family == sys.S390X || s.softFloat { + if conv1, ok1 := uint64fpConvOpToSSA[twoTypes{s.concreteEtype(ft), s.concreteEtype(tt)}]; ok1 { + conv = conv1 + } + } + + if Arch.LinkArch.Family == sys.MIPS && !s.softFloat { + if ft.Size() == 4 && ft.IsInteger() && !ft.IsSigned() { + // tt is float32 or float64, and ft is also unsigned + if tt.Size() == 4 { + return s.uint32Tofloat32(n, x, ft, tt) + } + if tt.Size() == 8 { + return s.uint32Tofloat64(n, x, ft, tt) + } + } else if tt.Size() == 4 && tt.IsInteger() && !tt.IsSigned() { + // ft is float32 or float64, and tt is unsigned integer + if ft.Size() == 4 { + return s.float32ToUint32(n, x, ft, tt) + } + if ft.Size() == 8 { + return s.float64ToUint32(n, x, ft, tt) + } + } + } + + if !ok { + s.Fatalf("weird float conversion %v -> %v", ft, tt) + } + op1, op2, it := conv.op1, conv.op2, conv.intermediateType + + if op1 != ssa.OpInvalid && op2 != ssa.OpInvalid { + // normal case, not tripping over unsigned 64 + if op1 == ssa.OpCopy { + if op2 == ssa.OpCopy { + return x + } + return s.newValueOrSfCall1(op2, n.Type(), x) + } + if op2 == ssa.OpCopy { + return s.newValueOrSfCall1(op1, n.Type(), x) + } + return s.newValueOrSfCall1(op2, n.Type(), s.newValueOrSfCall1(op1, types.Types[it], x)) + } + // Tricky 64-bit unsigned cases. + if ft.IsInteger() { + // tt is float32 or float64, and ft is also unsigned + if tt.Size() == 4 { + return s.uint64Tofloat32(n, x, ft, tt) + } + if tt.Size() == 8 { + return s.uint64Tofloat64(n, x, ft, tt) + } + s.Fatalf("weird unsigned integer to float conversion %v -> %v", ft, tt) + } + // ft is float32 or float64, and tt is unsigned integer + if ft.Size() == 4 { + return s.float32ToUint64(n, x, ft, tt) + } + if ft.Size() == 8 { + return s.float64ToUint64(n, x, ft, tt) + } + s.Fatalf("weird float to unsigned integer conversion %v -> %v", ft, tt) + return nil + } + + if ft.IsComplex() && tt.IsComplex() { + var op ssa.Op + if ft.Size() == tt.Size() { + switch ft.Size() { + case 8: + op = ssa.OpRound32F + case 16: + op = ssa.OpRound64F + default: + s.Fatalf("weird complex conversion %v -> %v", ft, tt) + } + } else if ft.Size() == 8 && tt.Size() == 16 { + op = ssa.OpCvt32Fto64F + } else if ft.Size() == 16 && tt.Size() == 8 { + op = ssa.OpCvt64Fto32F + } else { + s.Fatalf("weird complex conversion %v -> %v", ft, tt) + } + ftp := types.FloatForComplex(ft) + ttp := types.FloatForComplex(tt) + return s.newValue2(ssa.OpComplexMake, tt, + s.newValueOrSfCall1(op, ttp, s.newValue1(ssa.OpComplexReal, ftp, x)), + s.newValueOrSfCall1(op, ttp, s.newValue1(ssa.OpComplexImag, ftp, x))) + } + + s.Fatalf("unhandled OCONV %s -> %s", n.X.Type().Kind(), n.Type().Kind()) + return nil + + case ir.ODOTTYPE: + n := n.(*ir.TypeAssertExpr) + res, _ := s.dottype(n, false) + return res + + // binary ops + case ir.OLT, ir.OEQ, ir.ONE, ir.OLE, ir.OGE, ir.OGT: + n := n.(*ir.BinaryExpr) + a := s.expr(n.X) + b := s.expr(n.Y) + if n.X.Type().IsComplex() { + pt := types.FloatForComplex(n.X.Type()) + op := s.ssaOp(ir.OEQ, pt) + r := s.newValueOrSfCall2(op, types.Types[types.TBOOL], s.newValue1(ssa.OpComplexReal, pt, a), s.newValue1(ssa.OpComplexReal, pt, b)) + i := s.newValueOrSfCall2(op, types.Types[types.TBOOL], s.newValue1(ssa.OpComplexImag, pt, a), s.newValue1(ssa.OpComplexImag, pt, b)) + c := s.newValue2(ssa.OpAndB, types.Types[types.TBOOL], r, i) + switch n.Op() { + case ir.OEQ: + return c + case ir.ONE: + return s.newValue1(ssa.OpNot, types.Types[types.TBOOL], c) + default: + s.Fatalf("ordered complex compare %v", n.Op()) + } + } + + // Convert OGE and OGT into OLE and OLT. + op := n.Op() + switch op { + case ir.OGE: + op, a, b = ir.OLE, b, a + case ir.OGT: + op, a, b = ir.OLT, b, a + } + if n.X.Type().IsFloat() { + // float comparison + return s.newValueOrSfCall2(s.ssaOp(op, n.X.Type()), types.Types[types.TBOOL], a, b) + } + // integer comparison + return s.newValue2(s.ssaOp(op, n.X.Type()), types.Types[types.TBOOL], a, b) + case ir.OMUL: + n := n.(*ir.BinaryExpr) + a := s.expr(n.X) + b := s.expr(n.Y) + if n.Type().IsComplex() { + mulop := ssa.OpMul64F + addop := ssa.OpAdd64F + subop := ssa.OpSub64F + pt := types.FloatForComplex(n.Type()) // Could be Float32 or Float64 + wt := types.Types[types.TFLOAT64] // Compute in Float64 to minimize cancellation error + + areal := s.newValue1(ssa.OpComplexReal, pt, a) + breal := s.newValue1(ssa.OpComplexReal, pt, b) + aimag := s.newValue1(ssa.OpComplexImag, pt, a) + bimag := s.newValue1(ssa.OpComplexImag, pt, b) + + if pt != wt { // Widen for calculation + areal = s.newValueOrSfCall1(ssa.OpCvt32Fto64F, wt, areal) + breal = s.newValueOrSfCall1(ssa.OpCvt32Fto64F, wt, breal) + aimag = s.newValueOrSfCall1(ssa.OpCvt32Fto64F, wt, aimag) + bimag = s.newValueOrSfCall1(ssa.OpCvt32Fto64F, wt, bimag) + } + + xreal := s.newValueOrSfCall2(subop, wt, s.newValueOrSfCall2(mulop, wt, areal, breal), s.newValueOrSfCall2(mulop, wt, aimag, bimag)) + ximag := s.newValueOrSfCall2(addop, wt, s.newValueOrSfCall2(mulop, wt, areal, bimag), s.newValueOrSfCall2(mulop, wt, aimag, breal)) + + if pt != wt { // Narrow to store back + xreal = s.newValueOrSfCall1(ssa.OpCvt64Fto32F, pt, xreal) + ximag = s.newValueOrSfCall1(ssa.OpCvt64Fto32F, pt, ximag) + } + + return s.newValue2(ssa.OpComplexMake, n.Type(), xreal, ximag) + } + + if n.Type().IsFloat() { + return s.newValueOrSfCall2(s.ssaOp(n.Op(), n.Type()), a.Type, a, b) + } + + return s.newValue2(s.ssaOp(n.Op(), n.Type()), a.Type, a, b) + + case ir.ODIV: + n := n.(*ir.BinaryExpr) + a := s.expr(n.X) + b := s.expr(n.Y) + if n.Type().IsComplex() { + // TODO this is not executed because the front-end substitutes a runtime call. + // That probably ought to change; with modest optimization the widen/narrow + // conversions could all be elided in larger expression trees. + mulop := ssa.OpMul64F + addop := ssa.OpAdd64F + subop := ssa.OpSub64F + divop := ssa.OpDiv64F + pt := types.FloatForComplex(n.Type()) // Could be Float32 or Float64 + wt := types.Types[types.TFLOAT64] // Compute in Float64 to minimize cancellation error + + areal := s.newValue1(ssa.OpComplexReal, pt, a) + breal := s.newValue1(ssa.OpComplexReal, pt, b) + aimag := s.newValue1(ssa.OpComplexImag, pt, a) + bimag := s.newValue1(ssa.OpComplexImag, pt, b) + + if pt != wt { // Widen for calculation + areal = s.newValueOrSfCall1(ssa.OpCvt32Fto64F, wt, areal) + breal = s.newValueOrSfCall1(ssa.OpCvt32Fto64F, wt, breal) + aimag = s.newValueOrSfCall1(ssa.OpCvt32Fto64F, wt, aimag) + bimag = s.newValueOrSfCall1(ssa.OpCvt32Fto64F, wt, bimag) + } + + denom := s.newValueOrSfCall2(addop, wt, s.newValueOrSfCall2(mulop, wt, breal, breal), s.newValueOrSfCall2(mulop, wt, bimag, bimag)) + xreal := s.newValueOrSfCall2(addop, wt, s.newValueOrSfCall2(mulop, wt, areal, breal), s.newValueOrSfCall2(mulop, wt, aimag, bimag)) + ximag := s.newValueOrSfCall2(subop, wt, s.newValueOrSfCall2(mulop, wt, aimag, breal), s.newValueOrSfCall2(mulop, wt, areal, bimag)) + + // TODO not sure if this is best done in wide precision or narrow + // Double-rounding might be an issue. + // Note that the pre-SSA implementation does the entire calculation + // in wide format, so wide is compatible. + xreal = s.newValueOrSfCall2(divop, wt, xreal, denom) + ximag = s.newValueOrSfCall2(divop, wt, ximag, denom) + + if pt != wt { // Narrow to store back + xreal = s.newValueOrSfCall1(ssa.OpCvt64Fto32F, pt, xreal) + ximag = s.newValueOrSfCall1(ssa.OpCvt64Fto32F, pt, ximag) + } + return s.newValue2(ssa.OpComplexMake, n.Type(), xreal, ximag) + } + if n.Type().IsFloat() { + return s.newValueOrSfCall2(s.ssaOp(n.Op(), n.Type()), a.Type, a, b) + } + return s.intDivide(n, a, b) + case ir.OMOD: + n := n.(*ir.BinaryExpr) + a := s.expr(n.X) + b := s.expr(n.Y) + return s.intDivide(n, a, b) + case ir.OADD, ir.OSUB: + n := n.(*ir.BinaryExpr) + a := s.expr(n.X) + b := s.expr(n.Y) + if n.Type().IsComplex() { + pt := types.FloatForComplex(n.Type()) + op := s.ssaOp(n.Op(), pt) + return s.newValue2(ssa.OpComplexMake, n.Type(), + s.newValueOrSfCall2(op, pt, s.newValue1(ssa.OpComplexReal, pt, a), s.newValue1(ssa.OpComplexReal, pt, b)), + s.newValueOrSfCall2(op, pt, s.newValue1(ssa.OpComplexImag, pt, a), s.newValue1(ssa.OpComplexImag, pt, b))) + } + if n.Type().IsFloat() { + return s.newValueOrSfCall2(s.ssaOp(n.Op(), n.Type()), a.Type, a, b) + } + return s.newValue2(s.ssaOp(n.Op(), n.Type()), a.Type, a, b) + case ir.OAND, ir.OOR, ir.OXOR: + n := n.(*ir.BinaryExpr) + a := s.expr(n.X) + b := s.expr(n.Y) + return s.newValue2(s.ssaOp(n.Op(), n.Type()), a.Type, a, b) + case ir.OANDNOT: + n := n.(*ir.BinaryExpr) + a := s.expr(n.X) + b := s.expr(n.Y) + b = s.newValue1(s.ssaOp(ir.OBITNOT, b.Type), b.Type, b) + return s.newValue2(s.ssaOp(ir.OAND, n.Type()), a.Type, a, b) + case ir.OLSH, ir.ORSH: + n := n.(*ir.BinaryExpr) + a := s.expr(n.X) + b := s.expr(n.Y) + bt := b.Type + if bt.IsSigned() { + cmp := s.newValue2(s.ssaOp(ir.OLE, bt), types.Types[types.TBOOL], s.zeroVal(bt), b) + s.check(cmp, ir.Syms.Panicshift) + bt = bt.ToUnsigned() + } + return s.newValue2(s.ssaShiftOp(n.Op(), n.Type(), bt), a.Type, a, b) + case ir.OANDAND, ir.OOROR: + // To implement OANDAND (and OOROR), we introduce a + // new temporary variable to hold the result. The + // variable is associated with the OANDAND node in the + // s.vars table (normally variables are only + // associated with ONAME nodes). We convert + // A && B + // to + // var = A + // if var { + // var = B + // } + // Using var in the subsequent block introduces the + // necessary phi variable. + n := n.(*ir.LogicalExpr) + el := s.expr(n.X) + s.vars[n] = el + + b := s.endBlock() + b.Kind = ssa.BlockIf + b.SetControl(el) + // In theory, we should set b.Likely here based on context. + // However, gc only gives us likeliness hints + // in a single place, for plain OIF statements, + // and passing around context is finnicky, so don't bother for now. + + bRight := s.f.NewBlock(ssa.BlockPlain) + bResult := s.f.NewBlock(ssa.BlockPlain) + if n.Op() == ir.OANDAND { + b.AddEdgeTo(bRight) + b.AddEdgeTo(bResult) + } else if n.Op() == ir.OOROR { + b.AddEdgeTo(bResult) + b.AddEdgeTo(bRight) + } + + s.startBlock(bRight) + er := s.expr(n.Y) + s.vars[n] = er + + b = s.endBlock() + b.AddEdgeTo(bResult) + + s.startBlock(bResult) + return s.variable(n, types.Types[types.TBOOL]) + case ir.OCOMPLEX: + n := n.(*ir.BinaryExpr) + r := s.expr(n.X) + i := s.expr(n.Y) + return s.newValue2(ssa.OpComplexMake, n.Type(), r, i) + + // unary ops + case ir.ONEG: + n := n.(*ir.UnaryExpr) + a := s.expr(n.X) + if n.Type().IsComplex() { + tp := types.FloatForComplex(n.Type()) + negop := s.ssaOp(n.Op(), tp) + return s.newValue2(ssa.OpComplexMake, n.Type(), + s.newValue1(negop, tp, s.newValue1(ssa.OpComplexReal, tp, a)), + s.newValue1(negop, tp, s.newValue1(ssa.OpComplexImag, tp, a))) + } + return s.newValue1(s.ssaOp(n.Op(), n.Type()), a.Type, a) + case ir.ONOT, ir.OBITNOT: + n := n.(*ir.UnaryExpr) + a := s.expr(n.X) + return s.newValue1(s.ssaOp(n.Op(), n.Type()), a.Type, a) + case ir.OIMAG, ir.OREAL: + n := n.(*ir.UnaryExpr) + a := s.expr(n.X) + return s.newValue1(s.ssaOp(n.Op(), n.X.Type()), n.Type(), a) + case ir.OPLUS: + n := n.(*ir.UnaryExpr) + return s.expr(n.X) + + case ir.OADDR: + n := n.(*ir.AddrExpr) + return s.addr(n.X) + + case ir.ORESULT: + n := n.(*ir.ResultExpr) + if s.prevCall == nil || s.prevCall.Op != ssa.OpStaticLECall && s.prevCall.Op != ssa.OpInterLECall && s.prevCall.Op != ssa.OpClosureLECall { + // Do the old thing + addr := s.constOffPtrSP(types.NewPtr(n.Type()), n.Offset) + return s.rawLoad(n.Type(), addr) + } + which := s.prevCall.Aux.(*ssa.AuxCall).ResultForOffset(n.Offset) + if which == -1 { + // Do the old thing // TODO: Panic instead. + addr := s.constOffPtrSP(types.NewPtr(n.Type()), n.Offset) + return s.rawLoad(n.Type(), addr) + } + if TypeOK(n.Type()) { + return s.newValue1I(ssa.OpSelectN, n.Type(), which, s.prevCall) + } else { + addr := s.newValue1I(ssa.OpSelectNAddr, types.NewPtr(n.Type()), which, s.prevCall) + return s.rawLoad(n.Type(), addr) + } + + case ir.ODEREF: + n := n.(*ir.StarExpr) + p := s.exprPtr(n.X, n.Bounded(), n.Pos()) + return s.load(n.Type(), p) + + case ir.ODOT: + n := n.(*ir.SelectorExpr) + if n.X.Op() == ir.OSTRUCTLIT { + // All literals with nonzero fields have already been + // rewritten during walk. Any that remain are just T{} + // or equivalents. Use the zero value. + if !ir.IsZero(n.X) { + s.Fatalf("literal with nonzero value in SSA: %v", n.X) + } + return s.zeroVal(n.Type()) + } + // If n is addressable and can't be represented in + // SSA, then load just the selected field. This + // prevents false memory dependencies in race/msan + // instrumentation. + if ir.IsAssignable(n) && !s.canSSA(n) { + p := s.addr(n) + return s.load(n.Type(), p) + } + v := s.expr(n.X) + return s.newValue1I(ssa.OpStructSelect, n.Type(), int64(fieldIdx(n)), v) + + case ir.ODOTPTR: + n := n.(*ir.SelectorExpr) + p := s.exprPtr(n.X, n.Bounded(), n.Pos()) + p = s.newValue1I(ssa.OpOffPtr, types.NewPtr(n.Type()), n.Offset, p) + return s.load(n.Type(), p) + + case ir.OINDEX: + n := n.(*ir.IndexExpr) + switch { + case n.X.Type().IsString(): + if n.Bounded() && ir.IsConst(n.X, constant.String) && ir.IsConst(n.Index, constant.Int) { + // Replace "abc"[1] with 'b'. + // Delayed until now because "abc"[1] is not an ideal constant. + // See test/fixedbugs/issue11370.go. + return s.newValue0I(ssa.OpConst8, types.Types[types.TUINT8], int64(int8(ir.StringVal(n.X)[ir.Int64Val(n.Index)]))) + } + a := s.expr(n.X) + i := s.expr(n.Index) + len := s.newValue1(ssa.OpStringLen, types.Types[types.TINT], a) + i = s.boundsCheck(i, len, ssa.BoundsIndex, n.Bounded()) + ptrtyp := s.f.Config.Types.BytePtr + ptr := s.newValue1(ssa.OpStringPtr, ptrtyp, a) + if ir.IsConst(n.Index, constant.Int) { + ptr = s.newValue1I(ssa.OpOffPtr, ptrtyp, ir.Int64Val(n.Index), ptr) + } else { + ptr = s.newValue2(ssa.OpAddPtr, ptrtyp, ptr, i) + } + return s.load(types.Types[types.TUINT8], ptr) + case n.X.Type().IsSlice(): + p := s.addr(n) + return s.load(n.X.Type().Elem(), p) + case n.X.Type().IsArray(): + if TypeOK(n.X.Type()) { + // SSA can handle arrays of length at most 1. + bound := n.X.Type().NumElem() + a := s.expr(n.X) + i := s.expr(n.Index) + if bound == 0 { + // Bounds check will never succeed. Might as well + // use constants for the bounds check. + z := s.constInt(types.Types[types.TINT], 0) + s.boundsCheck(z, z, ssa.BoundsIndex, false) + // The return value won't be live, return junk. + return s.newValue0(ssa.OpUnknown, n.Type()) + } + len := s.constInt(types.Types[types.TINT], bound) + s.boundsCheck(i, len, ssa.BoundsIndex, n.Bounded()) // checks i == 0 + return s.newValue1I(ssa.OpArraySelect, n.Type(), 0, a) + } + p := s.addr(n) + return s.load(n.X.Type().Elem(), p) + default: + s.Fatalf("bad type for index %v", n.X.Type()) + return nil + } + + case ir.OLEN, ir.OCAP: + n := n.(*ir.UnaryExpr) + switch { + case n.X.Type().IsSlice(): + op := ssa.OpSliceLen + if n.Op() == ir.OCAP { + op = ssa.OpSliceCap + } + return s.newValue1(op, types.Types[types.TINT], s.expr(n.X)) + case n.X.Type().IsString(): // string; not reachable for OCAP + return s.newValue1(ssa.OpStringLen, types.Types[types.TINT], s.expr(n.X)) + case n.X.Type().IsMap(), n.X.Type().IsChan(): + return s.referenceTypeBuiltin(n, s.expr(n.X)) + default: // array + return s.constInt(types.Types[types.TINT], n.X.Type().NumElem()) + } + + case ir.OSPTR: + n := n.(*ir.UnaryExpr) + a := s.expr(n.X) + if n.X.Type().IsSlice() { + return s.newValue1(ssa.OpSlicePtr, n.Type(), a) + } else { + return s.newValue1(ssa.OpStringPtr, n.Type(), a) + } + + case ir.OITAB: + n := n.(*ir.UnaryExpr) + a := s.expr(n.X) + return s.newValue1(ssa.OpITab, n.Type(), a) + + case ir.OIDATA: + n := n.(*ir.UnaryExpr) + a := s.expr(n.X) + return s.newValue1(ssa.OpIData, n.Type(), a) + + case ir.OEFACE: + n := n.(*ir.BinaryExpr) + tab := s.expr(n.X) + data := s.expr(n.Y) + return s.newValue2(ssa.OpIMake, n.Type(), tab, data) + + case ir.OSLICEHEADER: + n := n.(*ir.SliceHeaderExpr) + p := s.expr(n.Ptr) + l := s.expr(n.LenCap[0]) + c := s.expr(n.LenCap[1]) + return s.newValue3(ssa.OpSliceMake, n.Type(), p, l, c) + + case ir.OSLICE, ir.OSLICEARR, ir.OSLICE3, ir.OSLICE3ARR: + n := n.(*ir.SliceExpr) + v := s.expr(n.X) + var i, j, k *ssa.Value + low, high, max := n.SliceBounds() + if low != nil { + i = s.expr(low) + } + if high != nil { + j = s.expr(high) + } + if max != nil { + k = s.expr(max) + } + p, l, c := s.slice(v, i, j, k, n.Bounded()) + return s.newValue3(ssa.OpSliceMake, n.Type(), p, l, c) + + case ir.OSLICESTR: + n := n.(*ir.SliceExpr) + v := s.expr(n.X) + var i, j *ssa.Value + low, high, _ := n.SliceBounds() + if low != nil { + i = s.expr(low) + } + if high != nil { + j = s.expr(high) + } + p, l, _ := s.slice(v, i, j, nil, n.Bounded()) + return s.newValue2(ssa.OpStringMake, n.Type(), p, l) + + case ir.OCALLFUNC: + n := n.(*ir.CallExpr) + if ir.IsIntrinsicCall(n) { + return s.intrinsicCall(n) + } + fallthrough + + case ir.OCALLINTER, ir.OCALLMETH: + n := n.(*ir.CallExpr) + return s.callResult(n, callNormal) + + case ir.OGETG: + n := n.(*ir.CallExpr) + return s.newValue1(ssa.OpGetG, n.Type(), s.mem()) + + case ir.OAPPEND: + return s.append(n.(*ir.CallExpr), false) + + case ir.OSTRUCTLIT, ir.OARRAYLIT: + // All literals with nonzero fields have already been + // rewritten during walk. Any that remain are just T{} + // or equivalents. Use the zero value. + n := n.(*ir.CompLitExpr) + if !ir.IsZero(n) { + s.Fatalf("literal with nonzero value in SSA: %v", n) + } + return s.zeroVal(n.Type()) + + case ir.ONEWOBJ: + n := n.(*ir.UnaryExpr) + if n.Type().Elem().Size() == 0 { + return s.newValue1A(ssa.OpAddr, n.Type(), ir.Syms.Zerobase, s.sb) + } + typ := s.expr(n.X) + vv := s.rtcall(ir.Syms.Newobject, true, []*types.Type{n.Type()}, typ) + return vv[0] + + default: + s.Fatalf("unhandled expr %v", n.Op()) + return nil + } +} + +// append converts an OAPPEND node to SSA. +// If inplace is false, it converts the OAPPEND expression n to an ssa.Value, +// adds it to s, and returns the Value. +// If inplace is true, it writes the result of the OAPPEND expression n +// back to the slice being appended to, and returns nil. +// inplace MUST be set to false if the slice can be SSA'd. +func (s *state) append(n *ir.CallExpr, inplace bool) *ssa.Value { + // If inplace is false, process as expression "append(s, e1, e2, e3)": + // + // ptr, len, cap := s + // newlen := len + 3 + // if newlen > cap { + // ptr, len, cap = growslice(s, newlen) + // newlen = len + 3 // recalculate to avoid a spill + // } + // // with write barriers, if needed: + // *(ptr+len) = e1 + // *(ptr+len+1) = e2 + // *(ptr+len+2) = e3 + // return makeslice(ptr, newlen, cap) + // + // + // If inplace is true, process as statement "s = append(s, e1, e2, e3)": + // + // a := &s + // ptr, len, cap := s + // newlen := len + 3 + // if uint(newlen) > uint(cap) { + // newptr, len, newcap = growslice(ptr, len, cap, newlen) + // vardef(a) // if necessary, advise liveness we are writing a new a + // *a.cap = newcap // write before ptr to avoid a spill + // *a.ptr = newptr // with write barrier + // } + // newlen = len + 3 // recalculate to avoid a spill + // *a.len = newlen + // // with write barriers, if needed: + // *(ptr+len) = e1 + // *(ptr+len+1) = e2 + // *(ptr+len+2) = e3 + + et := n.Type().Elem() + pt := types.NewPtr(et) + + // Evaluate slice + sn := n.Args[0] // the slice node is the first in the list + + var slice, addr *ssa.Value + if inplace { + addr = s.addr(sn) + slice = s.load(n.Type(), addr) + } else { + slice = s.expr(sn) + } + + // Allocate new blocks + grow := s.f.NewBlock(ssa.BlockPlain) + assign := s.f.NewBlock(ssa.BlockPlain) + + // Decide if we need to grow + nargs := int64(len(n.Args) - 1) + p := s.newValue1(ssa.OpSlicePtr, pt, slice) + l := s.newValue1(ssa.OpSliceLen, types.Types[types.TINT], slice) + c := s.newValue1(ssa.OpSliceCap, types.Types[types.TINT], slice) + nl := s.newValue2(s.ssaOp(ir.OADD, types.Types[types.TINT]), types.Types[types.TINT], l, s.constInt(types.Types[types.TINT], nargs)) + + cmp := s.newValue2(s.ssaOp(ir.OLT, types.Types[types.TUINT]), types.Types[types.TBOOL], c, nl) + s.vars[ptrVar] = p + + if !inplace { + s.vars[newlenVar] = nl + s.vars[capVar] = c + } else { + s.vars[lenVar] = l + } + + b := s.endBlock() + b.Kind = ssa.BlockIf + b.Likely = ssa.BranchUnlikely + b.SetControl(cmp) + b.AddEdgeTo(grow) + b.AddEdgeTo(assign) + + // Call growslice + s.startBlock(grow) + taddr := s.expr(n.X) + r := s.rtcall(ir.Syms.Growslice, true, []*types.Type{pt, types.Types[types.TINT], types.Types[types.TINT]}, taddr, p, l, c, nl) + + if inplace { + if sn.Op() == ir.ONAME { + sn := sn.(*ir.Name) + if sn.Class_ != ir.PEXTERN { + // Tell liveness we're about to build a new slice + s.vars[memVar] = s.newValue1A(ssa.OpVarDef, types.TypeMem, sn, s.mem()) + } + } + capaddr := s.newValue1I(ssa.OpOffPtr, s.f.Config.Types.IntPtr, types.SliceCapOffset, addr) + s.store(types.Types[types.TINT], capaddr, r[2]) + s.store(pt, addr, r[0]) + // load the value we just stored to avoid having to spill it + s.vars[ptrVar] = s.load(pt, addr) + s.vars[lenVar] = r[1] // avoid a spill in the fast path + } else { + s.vars[ptrVar] = r[0] + s.vars[newlenVar] = s.newValue2(s.ssaOp(ir.OADD, types.Types[types.TINT]), types.Types[types.TINT], r[1], s.constInt(types.Types[types.TINT], nargs)) + s.vars[capVar] = r[2] + } + + b = s.endBlock() + b.AddEdgeTo(assign) + + // assign new elements to slots + s.startBlock(assign) + + if inplace { + l = s.variable(lenVar, types.Types[types.TINT]) // generates phi for len + nl = s.newValue2(s.ssaOp(ir.OADD, types.Types[types.TINT]), types.Types[types.TINT], l, s.constInt(types.Types[types.TINT], nargs)) + lenaddr := s.newValue1I(ssa.OpOffPtr, s.f.Config.Types.IntPtr, types.SliceLenOffset, addr) + s.store(types.Types[types.TINT], lenaddr, nl) + } + + // Evaluate args + type argRec struct { + // if store is true, we're appending the value v. If false, we're appending the + // value at *v. + v *ssa.Value + store bool + } + args := make([]argRec, 0, nargs) + for _, n := range n.Args[1:] { + if TypeOK(n.Type()) { + args = append(args, argRec{v: s.expr(n), store: true}) + } else { + v := s.addr(n) + args = append(args, argRec{v: v}) + } + } + + p = s.variable(ptrVar, pt) // generates phi for ptr + if !inplace { + nl = s.variable(newlenVar, types.Types[types.TINT]) // generates phi for nl + c = s.variable(capVar, types.Types[types.TINT]) // generates phi for cap + } + p2 := s.newValue2(ssa.OpPtrIndex, pt, p, l) + for i, arg := range args { + addr := s.newValue2(ssa.OpPtrIndex, pt, p2, s.constInt(types.Types[types.TINT], int64(i))) + if arg.store { + s.storeType(et, addr, arg.v, 0, true) + } else { + s.move(et, addr, arg.v) + } + } + + delete(s.vars, ptrVar) + if inplace { + delete(s.vars, lenVar) + return nil + } + delete(s.vars, newlenVar) + delete(s.vars, capVar) + // make result + return s.newValue3(ssa.OpSliceMake, n.Type(), p, nl, c) +} + +// condBranch evaluates the boolean expression cond and branches to yes +// if cond is true and no if cond is false. +// This function is intended to handle && and || better than just calling +// s.expr(cond) and branching on the result. +func (s *state) condBranch(cond ir.Node, yes, no *ssa.Block, likely int8) { + switch cond.Op() { + case ir.OANDAND: + cond := cond.(*ir.LogicalExpr) + mid := s.f.NewBlock(ssa.BlockPlain) + s.stmtList(cond.Init()) + s.condBranch(cond.X, mid, no, max8(likely, 0)) + s.startBlock(mid) + s.condBranch(cond.Y, yes, no, likely) + return + // Note: if likely==1, then both recursive calls pass 1. + // If likely==-1, then we don't have enough information to decide + // whether the first branch is likely or not. So we pass 0 for + // the likeliness of the first branch. + // TODO: have the frontend give us branch prediction hints for + // OANDAND and OOROR nodes (if it ever has such info). + case ir.OOROR: + cond := cond.(*ir.LogicalExpr) + mid := s.f.NewBlock(ssa.BlockPlain) + s.stmtList(cond.Init()) + s.condBranch(cond.X, yes, mid, min8(likely, 0)) + s.startBlock(mid) + s.condBranch(cond.Y, yes, no, likely) + return + // Note: if likely==-1, then both recursive calls pass -1. + // If likely==1, then we don't have enough info to decide + // the likelihood of the first branch. + case ir.ONOT: + cond := cond.(*ir.UnaryExpr) + s.stmtList(cond.Init()) + s.condBranch(cond.X, no, yes, -likely) + return + case ir.OCONVNOP: + cond := cond.(*ir.ConvExpr) + s.stmtList(cond.Init()) + s.condBranch(cond.X, yes, no, likely) + return + } + c := s.expr(cond) + b := s.endBlock() + b.Kind = ssa.BlockIf + b.SetControl(c) + b.Likely = ssa.BranchPrediction(likely) // gc and ssa both use -1/0/+1 for likeliness + b.AddEdgeTo(yes) + b.AddEdgeTo(no) +} + +type skipMask uint8 + +const ( + skipPtr skipMask = 1 << iota + skipLen + skipCap +) + +// assign does left = right. +// Right has already been evaluated to ssa, left has not. +// If deref is true, then we do left = *right instead (and right has already been nil-checked). +// If deref is true and right == nil, just do left = 0. +// skip indicates assignments (at the top level) that can be avoided. +func (s *state) assign(left ir.Node, right *ssa.Value, deref bool, skip skipMask) { + if left.Op() == ir.ONAME && ir.IsBlank(left) { + return + } + t := left.Type() + types.CalcSize(t) + if s.canSSA(left) { + if deref { + s.Fatalf("can SSA LHS %v but not RHS %s", left, right) + } + if left.Op() == ir.ODOT { + // We're assigning to a field of an ssa-able value. + // We need to build a new structure with the new value for the + // field we're assigning and the old values for the other fields. + // For instance: + // type T struct {a, b, c int} + // var T x + // x.b = 5 + // For the x.b = 5 assignment we want to generate x = T{x.a, 5, x.c} + + // Grab information about the structure type. + left := left.(*ir.SelectorExpr) + t := left.X.Type() + nf := t.NumFields() + idx := fieldIdx(left) + + // Grab old value of structure. + old := s.expr(left.X) + + // Make new structure. + new := s.newValue0(ssa.StructMakeOp(t.NumFields()), t) + + // Add fields as args. + for i := 0; i < nf; i++ { + if i == idx { + new.AddArg(right) + } else { + new.AddArg(s.newValue1I(ssa.OpStructSelect, t.FieldType(i), int64(i), old)) + } + } + + // Recursively assign the new value we've made to the base of the dot op. + s.assign(left.X, new, false, 0) + // TODO: do we need to update named values here? + return + } + if left.Op() == ir.OINDEX && left.(*ir.IndexExpr).X.Type().IsArray() { + left := left.(*ir.IndexExpr) + s.pushLine(left.Pos()) + defer s.popLine() + // We're assigning to an element of an ssa-able array. + // a[i] = v + t := left.X.Type() + n := t.NumElem() + + i := s.expr(left.Index) // index + if n == 0 { + // The bounds check must fail. Might as well + // ignore the actual index and just use zeros. + z := s.constInt(types.Types[types.TINT], 0) + s.boundsCheck(z, z, ssa.BoundsIndex, false) + return + } + if n != 1 { + s.Fatalf("assigning to non-1-length array") + } + // Rewrite to a = [1]{v} + len := s.constInt(types.Types[types.TINT], 1) + s.boundsCheck(i, len, ssa.BoundsIndex, false) // checks i == 0 + v := s.newValue1(ssa.OpArrayMake1, t, right) + s.assign(left.X, v, false, 0) + return + } + left := left.(*ir.Name) + // Update variable assignment. + s.vars[left] = right + s.addNamedValue(left, right) + return + } + + // If this assignment clobbers an entire local variable, then emit + // OpVarDef so liveness analysis knows the variable is redefined. + if base := clobberBase(left); base.Op() == ir.ONAME && base.(*ir.Name).Class_ != ir.PEXTERN && skip == 0 { + s.vars[memVar] = s.newValue1Apos(ssa.OpVarDef, types.TypeMem, base.(*ir.Name), s.mem(), !ir.IsAutoTmp(base)) + } + + // Left is not ssa-able. Compute its address. + addr := s.addr(left) + if ir.IsReflectHeaderDataField(left) { + // Package unsafe's documentation says storing pointers into + // reflect.SliceHeader and reflect.StringHeader's Data fields + // is valid, even though they have type uintptr (#19168). + // Mark it pointer type to signal the writebarrier pass to + // insert a write barrier. + t = types.Types[types.TUNSAFEPTR] + } + if deref { + // Treat as a mem->mem move. + if right == nil { + s.zero(t, addr) + } else { + s.move(t, addr, right) + } + return + } + // Treat as a store. + s.storeType(t, addr, right, skip, !ir.IsAutoTmp(left)) +} + +// zeroVal returns the zero value for type t. +func (s *state) zeroVal(t *types.Type) *ssa.Value { + switch { + case t.IsInteger(): + switch t.Size() { + case 1: + return s.constInt8(t, 0) + case 2: + return s.constInt16(t, 0) + case 4: + return s.constInt32(t, 0) + case 8: + return s.constInt64(t, 0) + default: + s.Fatalf("bad sized integer type %v", t) + } + case t.IsFloat(): + switch t.Size() { + case 4: + return s.constFloat32(t, 0) + case 8: + return s.constFloat64(t, 0) + default: + s.Fatalf("bad sized float type %v", t) + } + case t.IsComplex(): + switch t.Size() { + case 8: + z := s.constFloat32(types.Types[types.TFLOAT32], 0) + return s.entryNewValue2(ssa.OpComplexMake, t, z, z) + case 16: + z := s.constFloat64(types.Types[types.TFLOAT64], 0) + return s.entryNewValue2(ssa.OpComplexMake, t, z, z) + default: + s.Fatalf("bad sized complex type %v", t) + } + + case t.IsString(): + return s.constEmptyString(t) + case t.IsPtrShaped(): + return s.constNil(t) + case t.IsBoolean(): + return s.constBool(false) + case t.IsInterface(): + return s.constInterface(t) + case t.IsSlice(): + return s.constSlice(t) + case t.IsStruct(): + n := t.NumFields() + v := s.entryNewValue0(ssa.StructMakeOp(t.NumFields()), t) + for i := 0; i < n; i++ { + v.AddArg(s.zeroVal(t.FieldType(i))) + } + return v + case t.IsArray(): + switch t.NumElem() { + case 0: + return s.entryNewValue0(ssa.OpArrayMake0, t) + case 1: + return s.entryNewValue1(ssa.OpArrayMake1, t, s.zeroVal(t.Elem())) + } + } + s.Fatalf("zero for type %v not implemented", t) + return nil +} + +type callKind int8 + +const ( + callNormal callKind = iota + callDefer + callDeferStack + callGo +) + +type sfRtCallDef struct { + rtfn *obj.LSym + rtype types.Kind +} + +var softFloatOps map[ssa.Op]sfRtCallDef + +func softfloatInit() { + // Some of these operations get transformed by sfcall. + softFloatOps = map[ssa.Op]sfRtCallDef{ + ssa.OpAdd32F: sfRtCallDef{typecheck.LookupRuntimeFunc("fadd32"), types.TFLOAT32}, + ssa.OpAdd64F: sfRtCallDef{typecheck.LookupRuntimeFunc("fadd64"), types.TFLOAT64}, + ssa.OpSub32F: sfRtCallDef{typecheck.LookupRuntimeFunc("fadd32"), types.TFLOAT32}, + ssa.OpSub64F: sfRtCallDef{typecheck.LookupRuntimeFunc("fadd64"), types.TFLOAT64}, + ssa.OpMul32F: sfRtCallDef{typecheck.LookupRuntimeFunc("fmul32"), types.TFLOAT32}, + ssa.OpMul64F: sfRtCallDef{typecheck.LookupRuntimeFunc("fmul64"), types.TFLOAT64}, + ssa.OpDiv32F: sfRtCallDef{typecheck.LookupRuntimeFunc("fdiv32"), types.TFLOAT32}, + ssa.OpDiv64F: sfRtCallDef{typecheck.LookupRuntimeFunc("fdiv64"), types.TFLOAT64}, + + ssa.OpEq64F: sfRtCallDef{typecheck.LookupRuntimeFunc("feq64"), types.TBOOL}, + ssa.OpEq32F: sfRtCallDef{typecheck.LookupRuntimeFunc("feq32"), types.TBOOL}, + ssa.OpNeq64F: sfRtCallDef{typecheck.LookupRuntimeFunc("feq64"), types.TBOOL}, + ssa.OpNeq32F: sfRtCallDef{typecheck.LookupRuntimeFunc("feq32"), types.TBOOL}, + ssa.OpLess64F: sfRtCallDef{typecheck.LookupRuntimeFunc("fgt64"), types.TBOOL}, + ssa.OpLess32F: sfRtCallDef{typecheck.LookupRuntimeFunc("fgt32"), types.TBOOL}, + ssa.OpLeq64F: sfRtCallDef{typecheck.LookupRuntimeFunc("fge64"), types.TBOOL}, + ssa.OpLeq32F: sfRtCallDef{typecheck.LookupRuntimeFunc("fge32"), types.TBOOL}, + + ssa.OpCvt32to32F: sfRtCallDef{typecheck.LookupRuntimeFunc("fint32to32"), types.TFLOAT32}, + ssa.OpCvt32Fto32: sfRtCallDef{typecheck.LookupRuntimeFunc("f32toint32"), types.TINT32}, + ssa.OpCvt64to32F: sfRtCallDef{typecheck.LookupRuntimeFunc("fint64to32"), types.TFLOAT32}, + ssa.OpCvt32Fto64: sfRtCallDef{typecheck.LookupRuntimeFunc("f32toint64"), types.TINT64}, + ssa.OpCvt64Uto32F: sfRtCallDef{typecheck.LookupRuntimeFunc("fuint64to32"), types.TFLOAT32}, + ssa.OpCvt32Fto64U: sfRtCallDef{typecheck.LookupRuntimeFunc("f32touint64"), types.TUINT64}, + ssa.OpCvt32to64F: sfRtCallDef{typecheck.LookupRuntimeFunc("fint32to64"), types.TFLOAT64}, + ssa.OpCvt64Fto32: sfRtCallDef{typecheck.LookupRuntimeFunc("f64toint32"), types.TINT32}, + ssa.OpCvt64to64F: sfRtCallDef{typecheck.LookupRuntimeFunc("fint64to64"), types.TFLOAT64}, + ssa.OpCvt64Fto64: sfRtCallDef{typecheck.LookupRuntimeFunc("f64toint64"), types.TINT64}, + ssa.OpCvt64Uto64F: sfRtCallDef{typecheck.LookupRuntimeFunc("fuint64to64"), types.TFLOAT64}, + ssa.OpCvt64Fto64U: sfRtCallDef{typecheck.LookupRuntimeFunc("f64touint64"), types.TUINT64}, + ssa.OpCvt32Fto64F: sfRtCallDef{typecheck.LookupRuntimeFunc("f32to64"), types.TFLOAT64}, + ssa.OpCvt64Fto32F: sfRtCallDef{typecheck.LookupRuntimeFunc("f64to32"), types.TFLOAT32}, + } +} + +// TODO: do not emit sfcall if operation can be optimized to constant in later +// opt phase +func (s *state) sfcall(op ssa.Op, args ...*ssa.Value) (*ssa.Value, bool) { + if callDef, ok := softFloatOps[op]; ok { + switch op { + case ssa.OpLess32F, + ssa.OpLess64F, + ssa.OpLeq32F, + ssa.OpLeq64F: + args[0], args[1] = args[1], args[0] + case ssa.OpSub32F, + ssa.OpSub64F: + args[1] = s.newValue1(s.ssaOp(ir.ONEG, types.Types[callDef.rtype]), args[1].Type, args[1]) + } + + result := s.rtcall(callDef.rtfn, true, []*types.Type{types.Types[callDef.rtype]}, args...)[0] + if op == ssa.OpNeq32F || op == ssa.OpNeq64F { + result = s.newValue1(ssa.OpNot, result.Type, result) + } + return result, true + } + return nil, false +} + +var intrinsics map[intrinsicKey]intrinsicBuilder + +// An intrinsicBuilder converts a call node n into an ssa value that +// implements that call as an intrinsic. args is a list of arguments to the func. +type intrinsicBuilder func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value + +type intrinsicKey struct { + arch *sys.Arch + pkg string + fn string +} + +func InitTables() { + intrinsics = map[intrinsicKey]intrinsicBuilder{} + + var all []*sys.Arch + var p4 []*sys.Arch + var p8 []*sys.Arch + var lwatomics []*sys.Arch + for _, a := range &sys.Archs { + all = append(all, a) + if a.PtrSize == 4 { + p4 = append(p4, a) + } else { + p8 = append(p8, a) + } + if a.Family != sys.PPC64 { + lwatomics = append(lwatomics, a) + } + } + + // add adds the intrinsic b for pkg.fn for the given list of architectures. + add := func(pkg, fn string, b intrinsicBuilder, archs ...*sys.Arch) { + for _, a := range archs { + intrinsics[intrinsicKey{a, pkg, fn}] = b + } + } + // addF does the same as add but operates on architecture families. + addF := func(pkg, fn string, b intrinsicBuilder, archFamilies ...sys.ArchFamily) { + m := 0 + for _, f := range archFamilies { + if f >= 32 { + panic("too many architecture families") + } + m |= 1 << uint(f) + } + for _, a := range all { + if m>>uint(a.Family)&1 != 0 { + intrinsics[intrinsicKey{a, pkg, fn}] = b + } + } + } + // alias defines pkg.fn = pkg2.fn2 for all architectures in archs for which pkg2.fn2 exists. + alias := func(pkg, fn, pkg2, fn2 string, archs ...*sys.Arch) { + aliased := false + for _, a := range archs { + if b, ok := intrinsics[intrinsicKey{a, pkg2, fn2}]; ok { + intrinsics[intrinsicKey{a, pkg, fn}] = b + aliased = true + } + } + if !aliased { + panic(fmt.Sprintf("attempted to alias undefined intrinsic: %s.%s", pkg, fn)) + } + } + + /******** runtime ********/ + if !base.Flag.Cfg.Instrumenting { + add("runtime", "slicebytetostringtmp", + func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { + // Compiler frontend optimizations emit OBYTES2STRTMP nodes + // for the backend instead of slicebytetostringtmp calls + // when not instrumenting. + return s.newValue2(ssa.OpStringMake, n.Type(), args[0], args[1]) + }, + all...) + } + addF("runtime/internal/math", "MulUintptr", + func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { + if s.config.PtrSize == 4 { + return s.newValue2(ssa.OpMul32uover, types.NewTuple(types.Types[types.TUINT], types.Types[types.TUINT]), args[0], args[1]) + } + return s.newValue2(ssa.OpMul64uover, types.NewTuple(types.Types[types.TUINT], types.Types[types.TUINT]), args[0], args[1]) + }, + sys.AMD64, sys.I386, sys.MIPS64) + add("runtime", "KeepAlive", + func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { + data := s.newValue1(ssa.OpIData, s.f.Config.Types.BytePtr, args[0]) + s.vars[memVar] = s.newValue2(ssa.OpKeepAlive, types.TypeMem, data, s.mem()) + return nil + }, + all...) + add("runtime", "getclosureptr", + func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { + return s.newValue0(ssa.OpGetClosurePtr, s.f.Config.Types.Uintptr) + }, + all...) + + add("runtime", "getcallerpc", + func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { + return s.newValue0(ssa.OpGetCallerPC, s.f.Config.Types.Uintptr) + }, + all...) + + add("runtime", "getcallersp", + func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { + return s.newValue0(ssa.OpGetCallerSP, s.f.Config.Types.Uintptr) + }, + all...) + + /******** runtime/internal/sys ********/ + addF("runtime/internal/sys", "Ctz32", + func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { + return s.newValue1(ssa.OpCtz32, types.Types[types.TINT], args[0]) + }, + sys.AMD64, sys.ARM64, sys.ARM, sys.S390X, sys.MIPS, sys.PPC64) + addF("runtime/internal/sys", "Ctz64", + func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { + return s.newValue1(ssa.OpCtz64, types.Types[types.TINT], args[0]) + }, + sys.AMD64, sys.ARM64, sys.ARM, sys.S390X, sys.MIPS, sys.PPC64) + addF("runtime/internal/sys", "Bswap32", + func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { + return s.newValue1(ssa.OpBswap32, types.Types[types.TUINT32], args[0]) + }, + sys.AMD64, sys.ARM64, sys.ARM, sys.S390X) + addF("runtime/internal/sys", "Bswap64", + func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { + return s.newValue1(ssa.OpBswap64, types.Types[types.TUINT64], args[0]) + }, + sys.AMD64, sys.ARM64, sys.ARM, sys.S390X) + + /******** runtime/internal/atomic ********/ + addF("runtime/internal/atomic", "Load", + func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { + v := s.newValue2(ssa.OpAtomicLoad32, types.NewTuple(types.Types[types.TUINT32], types.TypeMem), args[0], s.mem()) + s.vars[memVar] = s.newValue1(ssa.OpSelect1, types.TypeMem, v) + return s.newValue1(ssa.OpSelect0, types.Types[types.TUINT32], v) + }, + sys.AMD64, sys.ARM64, sys.MIPS, sys.MIPS64, sys.PPC64, sys.RISCV64, sys.S390X) + addF("runtime/internal/atomic", "Load8", + func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { + v := s.newValue2(ssa.OpAtomicLoad8, types.NewTuple(types.Types[types.TUINT8], types.TypeMem), args[0], s.mem()) + s.vars[memVar] = s.newValue1(ssa.OpSelect1, types.TypeMem, v) + return s.newValue1(ssa.OpSelect0, types.Types[types.TUINT8], v) + }, + sys.AMD64, sys.ARM64, sys.MIPS, sys.MIPS64, sys.PPC64, sys.RISCV64, sys.S390X) + addF("runtime/internal/atomic", "Load64", + func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { + v := s.newValue2(ssa.OpAtomicLoad64, types.NewTuple(types.Types[types.TUINT64], types.TypeMem), args[0], s.mem()) + s.vars[memVar] = s.newValue1(ssa.OpSelect1, types.TypeMem, v) + return s.newValue1(ssa.OpSelect0, types.Types[types.TUINT64], v) + }, + sys.AMD64, sys.ARM64, sys.MIPS64, sys.PPC64, sys.RISCV64, sys.S390X) + addF("runtime/internal/atomic", "LoadAcq", + func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { + v := s.newValue2(ssa.OpAtomicLoadAcq32, types.NewTuple(types.Types[types.TUINT32], types.TypeMem), args[0], s.mem()) + s.vars[memVar] = s.newValue1(ssa.OpSelect1, types.TypeMem, v) + return s.newValue1(ssa.OpSelect0, types.Types[types.TUINT32], v) + }, + sys.PPC64, sys.S390X) + addF("runtime/internal/atomic", "LoadAcq64", + func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { + v := s.newValue2(ssa.OpAtomicLoadAcq64, types.NewTuple(types.Types[types.TUINT64], types.TypeMem), args[0], s.mem()) + s.vars[memVar] = s.newValue1(ssa.OpSelect1, types.TypeMem, v) + return s.newValue1(ssa.OpSelect0, types.Types[types.TUINT64], v) + }, + sys.PPC64) + addF("runtime/internal/atomic", "Loadp", + func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { + v := s.newValue2(ssa.OpAtomicLoadPtr, types.NewTuple(s.f.Config.Types.BytePtr, types.TypeMem), args[0], s.mem()) + s.vars[memVar] = s.newValue1(ssa.OpSelect1, types.TypeMem, v) + return s.newValue1(ssa.OpSelect0, s.f.Config.Types.BytePtr, v) + }, + sys.AMD64, sys.ARM64, sys.MIPS, sys.MIPS64, sys.PPC64, sys.RISCV64, sys.S390X) + + addF("runtime/internal/atomic", "Store", + func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { + s.vars[memVar] = s.newValue3(ssa.OpAtomicStore32, types.TypeMem, args[0], args[1], s.mem()) + return nil + }, + sys.AMD64, sys.ARM64, sys.MIPS, sys.MIPS64, sys.PPC64, sys.RISCV64, sys.S390X) + addF("runtime/internal/atomic", "Store8", + func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { + s.vars[memVar] = s.newValue3(ssa.OpAtomicStore8, types.TypeMem, args[0], args[1], s.mem()) + return nil + }, + sys.AMD64, sys.ARM64, sys.MIPS, sys.MIPS64, sys.PPC64, sys.RISCV64, sys.S390X) + addF("runtime/internal/atomic", "Store64", + func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { + s.vars[memVar] = s.newValue3(ssa.OpAtomicStore64, types.TypeMem, args[0], args[1], s.mem()) + return nil + }, + sys.AMD64, sys.ARM64, sys.MIPS64, sys.PPC64, sys.RISCV64, sys.S390X) + addF("runtime/internal/atomic", "StorepNoWB", + func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { + s.vars[memVar] = s.newValue3(ssa.OpAtomicStorePtrNoWB, types.TypeMem, args[0], args[1], s.mem()) + return nil + }, + sys.AMD64, sys.ARM64, sys.MIPS, sys.MIPS64, sys.RISCV64, sys.S390X) + addF("runtime/internal/atomic", "StoreRel", + func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { + s.vars[memVar] = s.newValue3(ssa.OpAtomicStoreRel32, types.TypeMem, args[0], args[1], s.mem()) + return nil + }, + sys.PPC64, sys.S390X) + addF("runtime/internal/atomic", "StoreRel64", + func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { + s.vars[memVar] = s.newValue3(ssa.OpAtomicStoreRel64, types.TypeMem, args[0], args[1], s.mem()) + return nil + }, + sys.PPC64) + + addF("runtime/internal/atomic", "Xchg", + func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { + v := s.newValue3(ssa.OpAtomicExchange32, types.NewTuple(types.Types[types.TUINT32], types.TypeMem), args[0], args[1], s.mem()) + s.vars[memVar] = s.newValue1(ssa.OpSelect1, types.TypeMem, v) + return s.newValue1(ssa.OpSelect0, types.Types[types.TUINT32], v) + }, + sys.AMD64, sys.MIPS, sys.MIPS64, sys.PPC64, sys.RISCV64, sys.S390X) + addF("runtime/internal/atomic", "Xchg64", + func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { + v := s.newValue3(ssa.OpAtomicExchange64, types.NewTuple(types.Types[types.TUINT64], types.TypeMem), args[0], args[1], s.mem()) + s.vars[memVar] = s.newValue1(ssa.OpSelect1, types.TypeMem, v) + return s.newValue1(ssa.OpSelect0, types.Types[types.TUINT64], v) + }, + sys.AMD64, sys.MIPS64, sys.PPC64, sys.RISCV64, sys.S390X) + + type atomicOpEmitter func(s *state, n *ir.CallExpr, args []*ssa.Value, op ssa.Op, typ types.Kind) + + makeAtomicGuardedIntrinsicARM64 := func(op0, op1 ssa.Op, typ, rtyp types.Kind, emit atomicOpEmitter) intrinsicBuilder { + + return func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { + // Target Atomic feature is identified by dynamic detection + addr := s.entryNewValue1A(ssa.OpAddr, types.Types[types.TBOOL].PtrTo(), ir.Syms.ARM64HasATOMICS, s.sb) + v := s.load(types.Types[types.TBOOL], addr) + b := s.endBlock() + b.Kind = ssa.BlockIf + b.SetControl(v) + bTrue := s.f.NewBlock(ssa.BlockPlain) + bFalse := s.f.NewBlock(ssa.BlockPlain) + bEnd := s.f.NewBlock(ssa.BlockPlain) + b.AddEdgeTo(bTrue) + b.AddEdgeTo(bFalse) + b.Likely = ssa.BranchLikely + + // We have atomic instructions - use it directly. + s.startBlock(bTrue) + emit(s, n, args, op1, typ) + s.endBlock().AddEdgeTo(bEnd) + + // Use original instruction sequence. + s.startBlock(bFalse) + emit(s, n, args, op0, typ) + s.endBlock().AddEdgeTo(bEnd) + + // Merge results. + s.startBlock(bEnd) + if rtyp == types.TNIL { + return nil + } else { + return s.variable(n, types.Types[rtyp]) + } + } + } + + atomicXchgXaddEmitterARM64 := func(s *state, n *ir.CallExpr, args []*ssa.Value, op ssa.Op, typ types.Kind) { + v := s.newValue3(op, types.NewTuple(types.Types[typ], types.TypeMem), args[0], args[1], s.mem()) + s.vars[memVar] = s.newValue1(ssa.OpSelect1, types.TypeMem, v) + s.vars[n] = s.newValue1(ssa.OpSelect0, types.Types[typ], v) + } + addF("runtime/internal/atomic", "Xchg", + makeAtomicGuardedIntrinsicARM64(ssa.OpAtomicExchange32, ssa.OpAtomicExchange32Variant, types.TUINT32, types.TUINT32, atomicXchgXaddEmitterARM64), + sys.ARM64) + addF("runtime/internal/atomic", "Xchg64", + makeAtomicGuardedIntrinsicARM64(ssa.OpAtomicExchange64, ssa.OpAtomicExchange64Variant, types.TUINT64, types.TUINT64, atomicXchgXaddEmitterARM64), + sys.ARM64) + + addF("runtime/internal/atomic", "Xadd", + func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { + v := s.newValue3(ssa.OpAtomicAdd32, types.NewTuple(types.Types[types.TUINT32], types.TypeMem), args[0], args[1], s.mem()) + s.vars[memVar] = s.newValue1(ssa.OpSelect1, types.TypeMem, v) + return s.newValue1(ssa.OpSelect0, types.Types[types.TUINT32], v) + }, + sys.AMD64, sys.MIPS, sys.MIPS64, sys.PPC64, sys.RISCV64, sys.S390X) + addF("runtime/internal/atomic", "Xadd64", + func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { + v := s.newValue3(ssa.OpAtomicAdd64, types.NewTuple(types.Types[types.TUINT64], types.TypeMem), args[0], args[1], s.mem()) + s.vars[memVar] = s.newValue1(ssa.OpSelect1, types.TypeMem, v) + return s.newValue1(ssa.OpSelect0, types.Types[types.TUINT64], v) + }, + sys.AMD64, sys.MIPS64, sys.PPC64, sys.RISCV64, sys.S390X) + + addF("runtime/internal/atomic", "Xadd", + makeAtomicGuardedIntrinsicARM64(ssa.OpAtomicAdd32, ssa.OpAtomicAdd32Variant, types.TUINT32, types.TUINT32, atomicXchgXaddEmitterARM64), + sys.ARM64) + addF("runtime/internal/atomic", "Xadd64", + makeAtomicGuardedIntrinsicARM64(ssa.OpAtomicAdd64, ssa.OpAtomicAdd64Variant, types.TUINT64, types.TUINT64, atomicXchgXaddEmitterARM64), + sys.ARM64) + + addF("runtime/internal/atomic", "Cas", + func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { + v := s.newValue4(ssa.OpAtomicCompareAndSwap32, types.NewTuple(types.Types[types.TBOOL], types.TypeMem), args[0], args[1], args[2], s.mem()) + s.vars[memVar] = s.newValue1(ssa.OpSelect1, types.TypeMem, v) + return s.newValue1(ssa.OpSelect0, types.Types[types.TBOOL], v) + }, + sys.AMD64, sys.MIPS, sys.MIPS64, sys.PPC64, sys.RISCV64, sys.S390X) + addF("runtime/internal/atomic", "Cas64", + func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { + v := s.newValue4(ssa.OpAtomicCompareAndSwap64, types.NewTuple(types.Types[types.TBOOL], types.TypeMem), args[0], args[1], args[2], s.mem()) + s.vars[memVar] = s.newValue1(ssa.OpSelect1, types.TypeMem, v) + return s.newValue1(ssa.OpSelect0, types.Types[types.TBOOL], v) + }, + sys.AMD64, sys.MIPS64, sys.PPC64, sys.RISCV64, sys.S390X) + addF("runtime/internal/atomic", "CasRel", + func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { + v := s.newValue4(ssa.OpAtomicCompareAndSwap32, types.NewTuple(types.Types[types.TBOOL], types.TypeMem), args[0], args[1], args[2], s.mem()) + s.vars[memVar] = s.newValue1(ssa.OpSelect1, types.TypeMem, v) + return s.newValue1(ssa.OpSelect0, types.Types[types.TBOOL], v) + }, + sys.PPC64) + + atomicCasEmitterARM64 := func(s *state, n *ir.CallExpr, args []*ssa.Value, op ssa.Op, typ types.Kind) { + v := s.newValue4(op, types.NewTuple(types.Types[types.TBOOL], types.TypeMem), args[0], args[1], args[2], s.mem()) + s.vars[memVar] = s.newValue1(ssa.OpSelect1, types.TypeMem, v) + s.vars[n] = s.newValue1(ssa.OpSelect0, types.Types[typ], v) + } + + addF("runtime/internal/atomic", "Cas", + makeAtomicGuardedIntrinsicARM64(ssa.OpAtomicCompareAndSwap32, ssa.OpAtomicCompareAndSwap32Variant, types.TUINT32, types.TBOOL, atomicCasEmitterARM64), + sys.ARM64) + addF("runtime/internal/atomic", "Cas64", + makeAtomicGuardedIntrinsicARM64(ssa.OpAtomicCompareAndSwap64, ssa.OpAtomicCompareAndSwap64Variant, types.TUINT64, types.TBOOL, atomicCasEmitterARM64), + sys.ARM64) + + addF("runtime/internal/atomic", "And8", + func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { + s.vars[memVar] = s.newValue3(ssa.OpAtomicAnd8, types.TypeMem, args[0], args[1], s.mem()) + return nil + }, + sys.AMD64, sys.MIPS, sys.PPC64, sys.S390X) + addF("runtime/internal/atomic", "And", + func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { + s.vars[memVar] = s.newValue3(ssa.OpAtomicAnd32, types.TypeMem, args[0], args[1], s.mem()) + return nil + }, + sys.AMD64, sys.MIPS, sys.PPC64, sys.S390X) + addF("runtime/internal/atomic", "Or8", + func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { + s.vars[memVar] = s.newValue3(ssa.OpAtomicOr8, types.TypeMem, args[0], args[1], s.mem()) + return nil + }, + sys.AMD64, sys.ARM64, sys.MIPS, sys.PPC64, sys.S390X) + addF("runtime/internal/atomic", "Or", + func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { + s.vars[memVar] = s.newValue3(ssa.OpAtomicOr32, types.TypeMem, args[0], args[1], s.mem()) + return nil + }, + sys.AMD64, sys.MIPS, sys.PPC64, sys.S390X) + + atomicAndOrEmitterARM64 := func(s *state, n *ir.CallExpr, args []*ssa.Value, op ssa.Op, typ types.Kind) { + s.vars[memVar] = s.newValue3(op, types.TypeMem, args[0], args[1], s.mem()) + } + + addF("runtime/internal/atomic", "And8", + makeAtomicGuardedIntrinsicARM64(ssa.OpAtomicAnd8, ssa.OpAtomicAnd8Variant, types.TNIL, types.TNIL, atomicAndOrEmitterARM64), + sys.ARM64) + addF("runtime/internal/atomic", "And", + makeAtomicGuardedIntrinsicARM64(ssa.OpAtomicAnd32, ssa.OpAtomicAnd32Variant, types.TNIL, types.TNIL, atomicAndOrEmitterARM64), + sys.ARM64) + addF("runtime/internal/atomic", "Or8", + makeAtomicGuardedIntrinsicARM64(ssa.OpAtomicOr8, ssa.OpAtomicOr8Variant, types.TNIL, types.TNIL, atomicAndOrEmitterARM64), + sys.ARM64) + addF("runtime/internal/atomic", "Or", + makeAtomicGuardedIntrinsicARM64(ssa.OpAtomicOr32, ssa.OpAtomicOr32Variant, types.TNIL, types.TNIL, atomicAndOrEmitterARM64), + sys.ARM64) + + alias("runtime/internal/atomic", "Loadint64", "runtime/internal/atomic", "Load64", all...) + alias("runtime/internal/atomic", "Xaddint64", "runtime/internal/atomic", "Xadd64", all...) + alias("runtime/internal/atomic", "Loaduint", "runtime/internal/atomic", "Load", p4...) + alias("runtime/internal/atomic", "Loaduint", "runtime/internal/atomic", "Load64", p8...) + alias("runtime/internal/atomic", "Loaduintptr", "runtime/internal/atomic", "Load", p4...) + alias("runtime/internal/atomic", "Loaduintptr", "runtime/internal/atomic", "Load64", p8...) + alias("runtime/internal/atomic", "LoadAcq", "runtime/internal/atomic", "Load", lwatomics...) + alias("runtime/internal/atomic", "LoadAcq64", "runtime/internal/atomic", "Load64", lwatomics...) + alias("runtime/internal/atomic", "LoadAcquintptr", "runtime/internal/atomic", "LoadAcq", p4...) + alias("sync", "runtime_LoadAcquintptr", "runtime/internal/atomic", "LoadAcq", p4...) // linknamed + alias("runtime/internal/atomic", "LoadAcquintptr", "runtime/internal/atomic", "LoadAcq64", p8...) + alias("sync", "runtime_LoadAcquintptr", "runtime/internal/atomic", "LoadAcq64", p8...) // linknamed + alias("runtime/internal/atomic", "Storeuintptr", "runtime/internal/atomic", "Store", p4...) + alias("runtime/internal/atomic", "Storeuintptr", "runtime/internal/atomic", "Store64", p8...) + alias("runtime/internal/atomic", "StoreRel", "runtime/internal/atomic", "Store", lwatomics...) + alias("runtime/internal/atomic", "StoreRel64", "runtime/internal/atomic", "Store64", lwatomics...) + alias("runtime/internal/atomic", "StoreReluintptr", "runtime/internal/atomic", "StoreRel", p4...) + alias("sync", "runtime_StoreReluintptr", "runtime/internal/atomic", "StoreRel", p4...) // linknamed + alias("runtime/internal/atomic", "StoreReluintptr", "runtime/internal/atomic", "StoreRel64", p8...) + alias("sync", "runtime_StoreReluintptr", "runtime/internal/atomic", "StoreRel64", p8...) // linknamed + alias("runtime/internal/atomic", "Xchguintptr", "runtime/internal/atomic", "Xchg", p4...) + alias("runtime/internal/atomic", "Xchguintptr", "runtime/internal/atomic", "Xchg64", p8...) + alias("runtime/internal/atomic", "Xadduintptr", "runtime/internal/atomic", "Xadd", p4...) + alias("runtime/internal/atomic", "Xadduintptr", "runtime/internal/atomic", "Xadd64", p8...) + alias("runtime/internal/atomic", "Casuintptr", "runtime/internal/atomic", "Cas", p4...) + alias("runtime/internal/atomic", "Casuintptr", "runtime/internal/atomic", "Cas64", p8...) + alias("runtime/internal/atomic", "Casp1", "runtime/internal/atomic", "Cas", p4...) + alias("runtime/internal/atomic", "Casp1", "runtime/internal/atomic", "Cas64", p8...) + alias("runtime/internal/atomic", "CasRel", "runtime/internal/atomic", "Cas", lwatomics...) + + /******** math ********/ + addF("math", "Sqrt", + func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { + return s.newValue1(ssa.OpSqrt, types.Types[types.TFLOAT64], args[0]) + }, + sys.I386, sys.AMD64, sys.ARM, sys.ARM64, sys.MIPS, sys.MIPS64, sys.PPC64, sys.RISCV64, sys.S390X, sys.Wasm) + addF("math", "Trunc", + func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { + return s.newValue1(ssa.OpTrunc, types.Types[types.TFLOAT64], args[0]) + }, + sys.ARM64, sys.PPC64, sys.S390X, sys.Wasm) + addF("math", "Ceil", + func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { + return s.newValue1(ssa.OpCeil, types.Types[types.TFLOAT64], args[0]) + }, + sys.ARM64, sys.PPC64, sys.S390X, sys.Wasm) + addF("math", "Floor", + func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { + return s.newValue1(ssa.OpFloor, types.Types[types.TFLOAT64], args[0]) + }, + sys.ARM64, sys.PPC64, sys.S390X, sys.Wasm) + addF("math", "Round", + func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { + return s.newValue1(ssa.OpRound, types.Types[types.TFLOAT64], args[0]) + }, + sys.ARM64, sys.PPC64, sys.S390X) + addF("math", "RoundToEven", + func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { + return s.newValue1(ssa.OpRoundToEven, types.Types[types.TFLOAT64], args[0]) + }, + sys.ARM64, sys.S390X, sys.Wasm) + addF("math", "Abs", + func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { + return s.newValue1(ssa.OpAbs, types.Types[types.TFLOAT64], args[0]) + }, + sys.ARM64, sys.ARM, sys.PPC64, sys.Wasm) + addF("math", "Copysign", + func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { + return s.newValue2(ssa.OpCopysign, types.Types[types.TFLOAT64], args[0], args[1]) + }, + sys.PPC64, sys.Wasm) + addF("math", "FMA", + func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { + return s.newValue3(ssa.OpFMA, types.Types[types.TFLOAT64], args[0], args[1], args[2]) + }, + sys.ARM64, sys.PPC64, sys.S390X) + addF("math", "FMA", + func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { + if !s.config.UseFMA { + s.vars[n] = s.callResult(n, callNormal) // types.Types[TFLOAT64] + return s.variable(n, types.Types[types.TFLOAT64]) + } + v := s.entryNewValue0A(ssa.OpHasCPUFeature, types.Types[types.TBOOL], ir.Syms.X86HasFMA) + b := s.endBlock() + b.Kind = ssa.BlockIf + b.SetControl(v) + bTrue := s.f.NewBlock(ssa.BlockPlain) + bFalse := s.f.NewBlock(ssa.BlockPlain) + bEnd := s.f.NewBlock(ssa.BlockPlain) + b.AddEdgeTo(bTrue) + b.AddEdgeTo(bFalse) + b.Likely = ssa.BranchLikely // >= haswell cpus are common + + // We have the intrinsic - use it directly. + s.startBlock(bTrue) + s.vars[n] = s.newValue3(ssa.OpFMA, types.Types[types.TFLOAT64], args[0], args[1], args[2]) + s.endBlock().AddEdgeTo(bEnd) + + // Call the pure Go version. + s.startBlock(bFalse) + s.vars[n] = s.callResult(n, callNormal) // types.Types[TFLOAT64] + s.endBlock().AddEdgeTo(bEnd) + + // Merge results. + s.startBlock(bEnd) + return s.variable(n, types.Types[types.TFLOAT64]) + }, + sys.AMD64) + addF("math", "FMA", + func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { + if !s.config.UseFMA { + s.vars[n] = s.callResult(n, callNormal) // types.Types[TFLOAT64] + return s.variable(n, types.Types[types.TFLOAT64]) + } + addr := s.entryNewValue1A(ssa.OpAddr, types.Types[types.TBOOL].PtrTo(), ir.Syms.ARMHasVFPv4, s.sb) + v := s.load(types.Types[types.TBOOL], addr) + b := s.endBlock() + b.Kind = ssa.BlockIf + b.SetControl(v) + bTrue := s.f.NewBlock(ssa.BlockPlain) + bFalse := s.f.NewBlock(ssa.BlockPlain) + bEnd := s.f.NewBlock(ssa.BlockPlain) + b.AddEdgeTo(bTrue) + b.AddEdgeTo(bFalse) + b.Likely = ssa.BranchLikely + + // We have the intrinsic - use it directly. + s.startBlock(bTrue) + s.vars[n] = s.newValue3(ssa.OpFMA, types.Types[types.TFLOAT64], args[0], args[1], args[2]) + s.endBlock().AddEdgeTo(bEnd) + + // Call the pure Go version. + s.startBlock(bFalse) + s.vars[n] = s.callResult(n, callNormal) // types.Types[TFLOAT64] + s.endBlock().AddEdgeTo(bEnd) + + // Merge results. + s.startBlock(bEnd) + return s.variable(n, types.Types[types.TFLOAT64]) + }, + sys.ARM) + + makeRoundAMD64 := func(op ssa.Op) func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { + return func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { + v := s.entryNewValue0A(ssa.OpHasCPUFeature, types.Types[types.TBOOL], ir.Syms.X86HasSSE41) + b := s.endBlock() + b.Kind = ssa.BlockIf + b.SetControl(v) + bTrue := s.f.NewBlock(ssa.BlockPlain) + bFalse := s.f.NewBlock(ssa.BlockPlain) + bEnd := s.f.NewBlock(ssa.BlockPlain) + b.AddEdgeTo(bTrue) + b.AddEdgeTo(bFalse) + b.Likely = ssa.BranchLikely // most machines have sse4.1 nowadays + + // We have the intrinsic - use it directly. + s.startBlock(bTrue) + s.vars[n] = s.newValue1(op, types.Types[types.TFLOAT64], args[0]) + s.endBlock().AddEdgeTo(bEnd) + + // Call the pure Go version. + s.startBlock(bFalse) + s.vars[n] = s.callResult(n, callNormal) // types.Types[TFLOAT64] + s.endBlock().AddEdgeTo(bEnd) + + // Merge results. + s.startBlock(bEnd) + return s.variable(n, types.Types[types.TFLOAT64]) + } + } + addF("math", "RoundToEven", + makeRoundAMD64(ssa.OpRoundToEven), + sys.AMD64) + addF("math", "Floor", + makeRoundAMD64(ssa.OpFloor), + sys.AMD64) + addF("math", "Ceil", + makeRoundAMD64(ssa.OpCeil), + sys.AMD64) + addF("math", "Trunc", + makeRoundAMD64(ssa.OpTrunc), + sys.AMD64) + + /******** math/bits ********/ + addF("math/bits", "TrailingZeros64", + func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { + return s.newValue1(ssa.OpCtz64, types.Types[types.TINT], args[0]) + }, + sys.AMD64, sys.ARM64, sys.ARM, sys.S390X, sys.MIPS, sys.PPC64, sys.Wasm) + addF("math/bits", "TrailingZeros32", + func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { + return s.newValue1(ssa.OpCtz32, types.Types[types.TINT], args[0]) + }, + sys.AMD64, sys.ARM64, sys.ARM, sys.S390X, sys.MIPS, sys.PPC64, sys.Wasm) + addF("math/bits", "TrailingZeros16", + func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { + x := s.newValue1(ssa.OpZeroExt16to32, types.Types[types.TUINT32], args[0]) + c := s.constInt32(types.Types[types.TUINT32], 1<<16) + y := s.newValue2(ssa.OpOr32, types.Types[types.TUINT32], x, c) + return s.newValue1(ssa.OpCtz32, types.Types[types.TINT], y) + }, + sys.MIPS) + addF("math/bits", "TrailingZeros16", + func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { + return s.newValue1(ssa.OpCtz16, types.Types[types.TINT], args[0]) + }, + sys.AMD64, sys.I386, sys.ARM, sys.ARM64, sys.Wasm) + addF("math/bits", "TrailingZeros16", + func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { + x := s.newValue1(ssa.OpZeroExt16to64, types.Types[types.TUINT64], args[0]) + c := s.constInt64(types.Types[types.TUINT64], 1<<16) + y := s.newValue2(ssa.OpOr64, types.Types[types.TUINT64], x, c) + return s.newValue1(ssa.OpCtz64, types.Types[types.TINT], y) + }, + sys.S390X, sys.PPC64) + addF("math/bits", "TrailingZeros8", + func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { + x := s.newValue1(ssa.OpZeroExt8to32, types.Types[types.TUINT32], args[0]) + c := s.constInt32(types.Types[types.TUINT32], 1<<8) + y := s.newValue2(ssa.OpOr32, types.Types[types.TUINT32], x, c) + return s.newValue1(ssa.OpCtz32, types.Types[types.TINT], y) + }, + sys.MIPS) + addF("math/bits", "TrailingZeros8", + func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { + return s.newValue1(ssa.OpCtz8, types.Types[types.TINT], args[0]) + }, + sys.AMD64, sys.ARM, sys.ARM64, sys.Wasm) + addF("math/bits", "TrailingZeros8", + func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { + x := s.newValue1(ssa.OpZeroExt8to64, types.Types[types.TUINT64], args[0]) + c := s.constInt64(types.Types[types.TUINT64], 1<<8) + y := s.newValue2(ssa.OpOr64, types.Types[types.TUINT64], x, c) + return s.newValue1(ssa.OpCtz64, types.Types[types.TINT], y) + }, + sys.S390X) + alias("math/bits", "ReverseBytes64", "runtime/internal/sys", "Bswap64", all...) + alias("math/bits", "ReverseBytes32", "runtime/internal/sys", "Bswap32", all...) + // ReverseBytes inlines correctly, no need to intrinsify it. + // ReverseBytes16 lowers to a rotate, no need for anything special here. + addF("math/bits", "Len64", + func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { + return s.newValue1(ssa.OpBitLen64, types.Types[types.TINT], args[0]) + }, + sys.AMD64, sys.ARM64, sys.ARM, sys.S390X, sys.MIPS, sys.PPC64, sys.Wasm) + addF("math/bits", "Len32", + func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { + return s.newValue1(ssa.OpBitLen32, types.Types[types.TINT], args[0]) + }, + sys.AMD64, sys.ARM64) + addF("math/bits", "Len32", + func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { + if s.config.PtrSize == 4 { + return s.newValue1(ssa.OpBitLen32, types.Types[types.TINT], args[0]) + } + x := s.newValue1(ssa.OpZeroExt32to64, types.Types[types.TUINT64], args[0]) + return s.newValue1(ssa.OpBitLen64, types.Types[types.TINT], x) + }, + sys.ARM, sys.S390X, sys.MIPS, sys.PPC64, sys.Wasm) + addF("math/bits", "Len16", + func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { + if s.config.PtrSize == 4 { + x := s.newValue1(ssa.OpZeroExt16to32, types.Types[types.TUINT32], args[0]) + return s.newValue1(ssa.OpBitLen32, types.Types[types.TINT], x) + } + x := s.newValue1(ssa.OpZeroExt16to64, types.Types[types.TUINT64], args[0]) + return s.newValue1(ssa.OpBitLen64, types.Types[types.TINT], x) + }, + sys.ARM64, sys.ARM, sys.S390X, sys.MIPS, sys.PPC64, sys.Wasm) + addF("math/bits", "Len16", + func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { + return s.newValue1(ssa.OpBitLen16, types.Types[types.TINT], args[0]) + }, + sys.AMD64) + addF("math/bits", "Len8", + func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { + if s.config.PtrSize == 4 { + x := s.newValue1(ssa.OpZeroExt8to32, types.Types[types.TUINT32], args[0]) + return s.newValue1(ssa.OpBitLen32, types.Types[types.TINT], x) + } + x := s.newValue1(ssa.OpZeroExt8to64, types.Types[types.TUINT64], args[0]) + return s.newValue1(ssa.OpBitLen64, types.Types[types.TINT], x) + }, + sys.ARM64, sys.ARM, sys.S390X, sys.MIPS, sys.PPC64, sys.Wasm) + addF("math/bits", "Len8", + func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { + return s.newValue1(ssa.OpBitLen8, types.Types[types.TINT], args[0]) + }, + sys.AMD64) + addF("math/bits", "Len", + func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { + if s.config.PtrSize == 4 { + return s.newValue1(ssa.OpBitLen32, types.Types[types.TINT], args[0]) + } + return s.newValue1(ssa.OpBitLen64, types.Types[types.TINT], args[0]) + }, + sys.AMD64, sys.ARM64, sys.ARM, sys.S390X, sys.MIPS, sys.PPC64, sys.Wasm) + // LeadingZeros is handled because it trivially calls Len. + addF("math/bits", "Reverse64", + func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { + return s.newValue1(ssa.OpBitRev64, types.Types[types.TINT], args[0]) + }, + sys.ARM64) + addF("math/bits", "Reverse32", + func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { + return s.newValue1(ssa.OpBitRev32, types.Types[types.TINT], args[0]) + }, + sys.ARM64) + addF("math/bits", "Reverse16", + func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { + return s.newValue1(ssa.OpBitRev16, types.Types[types.TINT], args[0]) + }, + sys.ARM64) + addF("math/bits", "Reverse8", + func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { + return s.newValue1(ssa.OpBitRev8, types.Types[types.TINT], args[0]) + }, + sys.ARM64) + addF("math/bits", "Reverse", + func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { + if s.config.PtrSize == 4 { + return s.newValue1(ssa.OpBitRev32, types.Types[types.TINT], args[0]) + } + return s.newValue1(ssa.OpBitRev64, types.Types[types.TINT], args[0]) + }, + sys.ARM64) + addF("math/bits", "RotateLeft8", + func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { + return s.newValue2(ssa.OpRotateLeft8, types.Types[types.TUINT8], args[0], args[1]) + }, + sys.AMD64) + addF("math/bits", "RotateLeft16", + func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { + return s.newValue2(ssa.OpRotateLeft16, types.Types[types.TUINT16], args[0], args[1]) + }, + sys.AMD64) + addF("math/bits", "RotateLeft32", + func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { + return s.newValue2(ssa.OpRotateLeft32, types.Types[types.TUINT32], args[0], args[1]) + }, + sys.AMD64, sys.ARM, sys.ARM64, sys.S390X, sys.PPC64, sys.Wasm) + addF("math/bits", "RotateLeft64", + func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { + return s.newValue2(ssa.OpRotateLeft64, types.Types[types.TUINT64], args[0], args[1]) + }, + sys.AMD64, sys.ARM64, sys.S390X, sys.PPC64, sys.Wasm) + alias("math/bits", "RotateLeft", "math/bits", "RotateLeft64", p8...) + + makeOnesCountAMD64 := func(op64 ssa.Op, op32 ssa.Op) func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { + return func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { + v := s.entryNewValue0A(ssa.OpHasCPUFeature, types.Types[types.TBOOL], ir.Syms.X86HasPOPCNT) + b := s.endBlock() + b.Kind = ssa.BlockIf + b.SetControl(v) + bTrue := s.f.NewBlock(ssa.BlockPlain) + bFalse := s.f.NewBlock(ssa.BlockPlain) + bEnd := s.f.NewBlock(ssa.BlockPlain) + b.AddEdgeTo(bTrue) + b.AddEdgeTo(bFalse) + b.Likely = ssa.BranchLikely // most machines have popcnt nowadays + + // We have the intrinsic - use it directly. + s.startBlock(bTrue) + op := op64 + if s.config.PtrSize == 4 { + op = op32 + } + s.vars[n] = s.newValue1(op, types.Types[types.TINT], args[0]) + s.endBlock().AddEdgeTo(bEnd) + + // Call the pure Go version. + s.startBlock(bFalse) + s.vars[n] = s.callResult(n, callNormal) // types.Types[TINT] + s.endBlock().AddEdgeTo(bEnd) + + // Merge results. + s.startBlock(bEnd) + return s.variable(n, types.Types[types.TINT]) + } + } + addF("math/bits", "OnesCount64", + makeOnesCountAMD64(ssa.OpPopCount64, ssa.OpPopCount64), + sys.AMD64) + addF("math/bits", "OnesCount64", + func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { + return s.newValue1(ssa.OpPopCount64, types.Types[types.TINT], args[0]) + }, + sys.PPC64, sys.ARM64, sys.S390X, sys.Wasm) + addF("math/bits", "OnesCount32", + makeOnesCountAMD64(ssa.OpPopCount32, ssa.OpPopCount32), + sys.AMD64) + addF("math/bits", "OnesCount32", + func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { + return s.newValue1(ssa.OpPopCount32, types.Types[types.TINT], args[0]) + }, + sys.PPC64, sys.ARM64, sys.S390X, sys.Wasm) + addF("math/bits", "OnesCount16", + makeOnesCountAMD64(ssa.OpPopCount16, ssa.OpPopCount16), + sys.AMD64) + addF("math/bits", "OnesCount16", + func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { + return s.newValue1(ssa.OpPopCount16, types.Types[types.TINT], args[0]) + }, + sys.ARM64, sys.S390X, sys.PPC64, sys.Wasm) + addF("math/bits", "OnesCount8", + func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { + return s.newValue1(ssa.OpPopCount8, types.Types[types.TINT], args[0]) + }, + sys.S390X, sys.PPC64, sys.Wasm) + addF("math/bits", "OnesCount", + makeOnesCountAMD64(ssa.OpPopCount64, ssa.OpPopCount32), + sys.AMD64) + addF("math/bits", "Mul64", + func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { + return s.newValue2(ssa.OpMul64uhilo, types.NewTuple(types.Types[types.TUINT64], types.Types[types.TUINT64]), args[0], args[1]) + }, + sys.AMD64, sys.ARM64, sys.PPC64, sys.S390X, sys.MIPS64) + alias("math/bits", "Mul", "math/bits", "Mul64", sys.ArchAMD64, sys.ArchARM64, sys.ArchPPC64, sys.ArchS390X, sys.ArchMIPS64, sys.ArchMIPS64LE) + addF("math/bits", "Add64", + func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { + return s.newValue3(ssa.OpAdd64carry, types.NewTuple(types.Types[types.TUINT64], types.Types[types.TUINT64]), args[0], args[1], args[2]) + }, + sys.AMD64, sys.ARM64, sys.PPC64, sys.S390X) + alias("math/bits", "Add", "math/bits", "Add64", sys.ArchAMD64, sys.ArchARM64, sys.ArchPPC64, sys.ArchS390X) + addF("math/bits", "Sub64", + func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { + return s.newValue3(ssa.OpSub64borrow, types.NewTuple(types.Types[types.TUINT64], types.Types[types.TUINT64]), args[0], args[1], args[2]) + }, + sys.AMD64, sys.ARM64, sys.S390X) + alias("math/bits", "Sub", "math/bits", "Sub64", sys.ArchAMD64, sys.ArchARM64, sys.ArchS390X) + addF("math/bits", "Div64", + func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { + // check for divide-by-zero/overflow and panic with appropriate message + cmpZero := s.newValue2(s.ssaOp(ir.ONE, types.Types[types.TUINT64]), types.Types[types.TBOOL], args[2], s.zeroVal(types.Types[types.TUINT64])) + s.check(cmpZero, ir.Syms.Panicdivide) + cmpOverflow := s.newValue2(s.ssaOp(ir.OLT, types.Types[types.TUINT64]), types.Types[types.TBOOL], args[0], args[2]) + s.check(cmpOverflow, ir.Syms.Panicoverflow) + return s.newValue3(ssa.OpDiv128u, types.NewTuple(types.Types[types.TUINT64], types.Types[types.TUINT64]), args[0], args[1], args[2]) + }, + sys.AMD64) + alias("math/bits", "Div", "math/bits", "Div64", sys.ArchAMD64) + + alias("runtime/internal/sys", "Ctz8", "math/bits", "TrailingZeros8", all...) + alias("runtime/internal/sys", "TrailingZeros8", "math/bits", "TrailingZeros8", all...) + alias("runtime/internal/sys", "TrailingZeros64", "math/bits", "TrailingZeros64", all...) + alias("runtime/internal/sys", "Len8", "math/bits", "Len8", all...) + alias("runtime/internal/sys", "Len64", "math/bits", "Len64", all...) + alias("runtime/internal/sys", "OnesCount64", "math/bits", "OnesCount64", all...) + + /******** sync/atomic ********/ + + // Note: these are disabled by flag_race in findIntrinsic below. + alias("sync/atomic", "LoadInt32", "runtime/internal/atomic", "Load", all...) + alias("sync/atomic", "LoadInt64", "runtime/internal/atomic", "Load64", all...) + alias("sync/atomic", "LoadPointer", "runtime/internal/atomic", "Loadp", all...) + alias("sync/atomic", "LoadUint32", "runtime/internal/atomic", "Load", all...) + alias("sync/atomic", "LoadUint64", "runtime/internal/atomic", "Load64", all...) + alias("sync/atomic", "LoadUintptr", "runtime/internal/atomic", "Load", p4...) + alias("sync/atomic", "LoadUintptr", "runtime/internal/atomic", "Load64", p8...) + + alias("sync/atomic", "StoreInt32", "runtime/internal/atomic", "Store", all...) + alias("sync/atomic", "StoreInt64", "runtime/internal/atomic", "Store64", all...) + // Note: not StorePointer, that needs a write barrier. Same below for {CompareAnd}Swap. + alias("sync/atomic", "StoreUint32", "runtime/internal/atomic", "Store", all...) + alias("sync/atomic", "StoreUint64", "runtime/internal/atomic", "Store64", all...) + alias("sync/atomic", "StoreUintptr", "runtime/internal/atomic", "Store", p4...) + alias("sync/atomic", "StoreUintptr", "runtime/internal/atomic", "Store64", p8...) + + alias("sync/atomic", "SwapInt32", "runtime/internal/atomic", "Xchg", all...) + alias("sync/atomic", "SwapInt64", "runtime/internal/atomic", "Xchg64", all...) + alias("sync/atomic", "SwapUint32", "runtime/internal/atomic", "Xchg", all...) + alias("sync/atomic", "SwapUint64", "runtime/internal/atomic", "Xchg64", all...) + alias("sync/atomic", "SwapUintptr", "runtime/internal/atomic", "Xchg", p4...) + alias("sync/atomic", "SwapUintptr", "runtime/internal/atomic", "Xchg64", p8...) + + alias("sync/atomic", "CompareAndSwapInt32", "runtime/internal/atomic", "Cas", all...) + alias("sync/atomic", "CompareAndSwapInt64", "runtime/internal/atomic", "Cas64", all...) + alias("sync/atomic", "CompareAndSwapUint32", "runtime/internal/atomic", "Cas", all...) + alias("sync/atomic", "CompareAndSwapUint64", "runtime/internal/atomic", "Cas64", all...) + alias("sync/atomic", "CompareAndSwapUintptr", "runtime/internal/atomic", "Cas", p4...) + alias("sync/atomic", "CompareAndSwapUintptr", "runtime/internal/atomic", "Cas64", p8...) + + alias("sync/atomic", "AddInt32", "runtime/internal/atomic", "Xadd", all...) + alias("sync/atomic", "AddInt64", "runtime/internal/atomic", "Xadd64", all...) + alias("sync/atomic", "AddUint32", "runtime/internal/atomic", "Xadd", all...) + alias("sync/atomic", "AddUint64", "runtime/internal/atomic", "Xadd64", all...) + alias("sync/atomic", "AddUintptr", "runtime/internal/atomic", "Xadd", p4...) + alias("sync/atomic", "AddUintptr", "runtime/internal/atomic", "Xadd64", p8...) + + /******** math/big ********/ + add("math/big", "mulWW", + func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { + return s.newValue2(ssa.OpMul64uhilo, types.NewTuple(types.Types[types.TUINT64], types.Types[types.TUINT64]), args[0], args[1]) + }, + sys.ArchAMD64, sys.ArchARM64, sys.ArchPPC64LE, sys.ArchPPC64, sys.ArchS390X) +} + +// findIntrinsic returns a function which builds the SSA equivalent of the +// function identified by the symbol sym. If sym is not an intrinsic call, returns nil. +func findIntrinsic(sym *types.Sym) intrinsicBuilder { + if sym == nil || sym.Pkg == nil { + return nil + } + pkg := sym.Pkg.Path + if sym.Pkg == types.LocalPkg { + pkg = base.Ctxt.Pkgpath + } + if base.Flag.Race && pkg == "sync/atomic" { + // The race detector needs to be able to intercept these calls. + // We can't intrinsify them. + return nil + } + // Skip intrinsifying math functions (which may contain hard-float + // instructions) when soft-float + if Arch.SoftFloat && pkg == "math" { + return nil + } + + fn := sym.Name + if ssa.IntrinsicsDisable { + if pkg == "runtime" && (fn == "getcallerpc" || fn == "getcallersp" || fn == "getclosureptr") { + // These runtime functions don't have definitions, must be intrinsics. + } else { + return nil + } + } + return intrinsics[intrinsicKey{Arch.LinkArch.Arch, pkg, fn}] +} + +func IsIntrinsicCall(n *ir.CallExpr) bool { + if n == nil { + return false + } + name, ok := n.X.(*ir.Name) + if !ok { + return false + } + return findIntrinsic(name.Sym()) != nil +} + +// intrinsicCall converts a call to a recognized intrinsic function into the intrinsic SSA operation. +func (s *state) intrinsicCall(n *ir.CallExpr) *ssa.Value { + v := findIntrinsic(n.X.Sym())(s, n, s.intrinsicArgs(n)) + if ssa.IntrinsicsDebug > 0 { + x := v + if x == nil { + x = s.mem() + } + if x.Op == ssa.OpSelect0 || x.Op == ssa.OpSelect1 { + x = x.Args[0] + } + base.WarnfAt(n.Pos(), "intrinsic substitution for %v with %s", n.X.Sym().Name, x.LongString()) + } + return v +} + +// intrinsicArgs extracts args from n, evaluates them to SSA values, and returns them. +func (s *state) intrinsicArgs(n *ir.CallExpr) []*ssa.Value { + // Construct map of temps; see comments in s.call about the structure of n. + temps := map[ir.Node]*ssa.Value{} + for _, a := range n.Args { + if a.Op() != ir.OAS { + s.Fatalf("non-assignment as a temp function argument %v", a.Op()) + } + a := a.(*ir.AssignStmt) + l, r := a.X, a.Y + if l.Op() != ir.ONAME { + s.Fatalf("non-ONAME temp function argument %v", a.Op()) + } + // Evaluate and store to "temporary". + // Walk ensures these temporaries are dead outside of n. + temps[l] = s.expr(r) + } + args := make([]*ssa.Value, len(n.Rargs)) + for i, n := range n.Rargs { + // Store a value to an argument slot. + if x, ok := temps[n]; ok { + // This is a previously computed temporary. + args[i] = x + continue + } + // This is an explicit value; evaluate it. + args[i] = s.expr(n) + } + return args +} + +// openDeferRecord adds code to evaluate and store the args for an open-code defer +// call, and records info about the defer, so we can generate proper code on the +// exit paths. n is the sub-node of the defer node that is the actual function +// call. We will also record funcdata information on where the args are stored +// (as well as the deferBits variable), and this will enable us to run the proper +// defer calls during panics. +func (s *state) openDeferRecord(n *ir.CallExpr) { + // Do any needed expression evaluation for the args (including the + // receiver, if any). This may be evaluating something like 'autotmp_3 = + // once.mutex'. Such a statement will create a mapping in s.vars[] from + // the autotmp name to the evaluated SSA arg value, but won't do any + // stores to the stack. + s.stmtList(n.Args) + + var args []*ssa.Value + var argNodes []*ir.Name + + opendefer := &openDeferInfo{ + n: n, + } + fn := n.X + if n.Op() == ir.OCALLFUNC { + // We must always store the function value in a stack slot for the + // runtime panic code to use. But in the defer exit code, we will + // call the function directly if it is a static function. + closureVal := s.expr(fn) + closure := s.openDeferSave(nil, fn.Type(), closureVal) + opendefer.closureNode = closure.Aux.(*ir.Name) + if !(fn.Op() == ir.ONAME && fn.(*ir.Name).Class_ == ir.PFUNC) { + opendefer.closure = closure + } + } else if n.Op() == ir.OCALLMETH { + if fn.Op() != ir.ODOTMETH { + base.Fatalf("OCALLMETH: n.Left not an ODOTMETH: %v", fn) + } + fn := fn.(*ir.SelectorExpr) + closureVal := s.getMethodClosure(fn) + // We must always store the function value in a stack slot for the + // runtime panic code to use. But in the defer exit code, we will + // call the method directly. + closure := s.openDeferSave(nil, fn.Type(), closureVal) + opendefer.closureNode = closure.Aux.(*ir.Name) + } else { + if fn.Op() != ir.ODOTINTER { + base.Fatalf("OCALLINTER: n.Left not an ODOTINTER: %v", fn.Op()) + } + fn := fn.(*ir.SelectorExpr) + closure, rcvr := s.getClosureAndRcvr(fn) + opendefer.closure = s.openDeferSave(nil, closure.Type, closure) + // Important to get the receiver type correct, so it is recognized + // as a pointer for GC purposes. + opendefer.rcvr = s.openDeferSave(nil, fn.Type().Recv().Type, rcvr) + opendefer.closureNode = opendefer.closure.Aux.(*ir.Name) + opendefer.rcvrNode = opendefer.rcvr.Aux.(*ir.Name) + } + for _, argn := range n.Rargs { + var v *ssa.Value + if TypeOK(argn.Type()) { + v = s.openDeferSave(nil, argn.Type(), s.expr(argn)) + } else { + v = s.openDeferSave(argn, argn.Type(), nil) + } + args = append(args, v) + argNodes = append(argNodes, v.Aux.(*ir.Name)) + } + opendefer.argVals = args + opendefer.argNodes = argNodes + index := len(s.openDefers) + s.openDefers = append(s.openDefers, opendefer) + + // Update deferBits only after evaluation and storage to stack of + // args/receiver/interface is successful. + bitvalue := s.constInt8(types.Types[types.TUINT8], 1<= 0; i-- { + r := s.openDefers[i] + bCond := s.f.NewBlock(ssa.BlockPlain) + bEnd := s.f.NewBlock(ssa.BlockPlain) + + deferBits := s.variable(deferBitsVar, types.Types[types.TUINT8]) + // Generate code to check if the bit associated with the current + // defer is set. + bitval := s.constInt8(types.Types[types.TUINT8], 1< int64(4*types.PtrSize) { + // 4*Widthptr is an arbitrary constant. We want it + // to be at least 3*Widthptr so slices can be registerized. + // Too big and we'll introduce too much register pressure. + return false + } + switch t.Kind() { + case types.TARRAY: + // We can't do larger arrays because dynamic indexing is + // not supported on SSA variables. + // TODO: allow if all indexes are constant. + if t.NumElem() <= 1 { + return TypeOK(t.Elem()) + } + return false + case types.TSTRUCT: + if t.NumFields() > ssa.MaxStruct { + return false + } + for _, t1 := range t.Fields().Slice() { + if !TypeOK(t1.Type) { + return false + } + } + return true + default: + return true + } +} + +// exprPtr evaluates n to a pointer and nil-checks it. +func (s *state) exprPtr(n ir.Node, bounded bool, lineno src.XPos) *ssa.Value { + p := s.expr(n) + if bounded || n.NonNil() { + if s.f.Frontend().Debug_checknil() && lineno.Line() > 1 { + s.f.Warnl(lineno, "removed nil check") + } + return p + } + s.nilCheck(p) + return p +} + +// nilCheck generates nil pointer checking code. +// Used only for automatically inserted nil checks, +// not for user code like 'x != nil'. +func (s *state) nilCheck(ptr *ssa.Value) { + if base.Debug.DisableNil != 0 || s.curfn.NilCheckDisabled() { + return + } + s.newValue2(ssa.OpNilCheck, types.TypeVoid, ptr, s.mem()) +} + +// boundsCheck generates bounds checking code. Checks if 0 <= idx <[=] len, branches to exit if not. +// Starts a new block on return. +// On input, len must be converted to full int width and be nonnegative. +// Returns idx converted to full int width. +// If bounded is true then caller guarantees the index is not out of bounds +// (but boundsCheck will still extend the index to full int width). +func (s *state) boundsCheck(idx, len *ssa.Value, kind ssa.BoundsKind, bounded bool) *ssa.Value { + idx = s.extendIndex(idx, len, kind, bounded) + + if bounded || base.Flag.B != 0 { + // If bounded or bounds checking is flag-disabled, then no check necessary, + // just return the extended index. + // + // Here, bounded == true if the compiler generated the index itself, + // such as in the expansion of a slice initializer. These indexes are + // compiler-generated, not Go program variables, so they cannot be + // attacker-controlled, so we can omit Spectre masking as well. + // + // Note that we do not want to omit Spectre masking in code like: + // + // if 0 <= i && i < len(x) { + // use(x[i]) + // } + // + // Lucky for us, bounded==false for that code. + // In that case (handled below), we emit a bound check (and Spectre mask) + // and then the prove pass will remove the bounds check. + // In theory the prove pass could potentially remove certain + // Spectre masks, but it's very delicate and probably better + // to be conservative and leave them all in. + return idx + } + + bNext := s.f.NewBlock(ssa.BlockPlain) + bPanic := s.f.NewBlock(ssa.BlockExit) + + if !idx.Type.IsSigned() { + switch kind { + case ssa.BoundsIndex: + kind = ssa.BoundsIndexU + case ssa.BoundsSliceAlen: + kind = ssa.BoundsSliceAlenU + case ssa.BoundsSliceAcap: + kind = ssa.BoundsSliceAcapU + case ssa.BoundsSliceB: + kind = ssa.BoundsSliceBU + case ssa.BoundsSlice3Alen: + kind = ssa.BoundsSlice3AlenU + case ssa.BoundsSlice3Acap: + kind = ssa.BoundsSlice3AcapU + case ssa.BoundsSlice3B: + kind = ssa.BoundsSlice3BU + case ssa.BoundsSlice3C: + kind = ssa.BoundsSlice3CU + } + } + + var cmp *ssa.Value + if kind == ssa.BoundsIndex || kind == ssa.BoundsIndexU { + cmp = s.newValue2(ssa.OpIsInBounds, types.Types[types.TBOOL], idx, len) + } else { + cmp = s.newValue2(ssa.OpIsSliceInBounds, types.Types[types.TBOOL], idx, len) + } + b := s.endBlock() + b.Kind = ssa.BlockIf + b.SetControl(cmp) + b.Likely = ssa.BranchLikely + b.AddEdgeTo(bNext) + b.AddEdgeTo(bPanic) + + s.startBlock(bPanic) + if Arch.LinkArch.Family == sys.Wasm { + // TODO(khr): figure out how to do "register" based calling convention for bounds checks. + // Should be similar to gcWriteBarrier, but I can't make it work. + s.rtcall(BoundsCheckFunc[kind], false, nil, idx, len) + } else { + mem := s.newValue3I(ssa.OpPanicBounds, types.TypeMem, int64(kind), idx, len, s.mem()) + s.endBlock().SetControl(mem) + } + s.startBlock(bNext) + + // In Spectre index mode, apply an appropriate mask to avoid speculative out-of-bounds accesses. + if base.Flag.Cfg.SpectreIndex { + op := ssa.OpSpectreIndex + if kind != ssa.BoundsIndex && kind != ssa.BoundsIndexU { + op = ssa.OpSpectreSliceIndex + } + idx = s.newValue2(op, types.Types[types.TINT], idx, len) + } + + return idx +} + +// If cmp (a bool) is false, panic using the given function. +func (s *state) check(cmp *ssa.Value, fn *obj.LSym) { + b := s.endBlock() + b.Kind = ssa.BlockIf + b.SetControl(cmp) + b.Likely = ssa.BranchLikely + bNext := s.f.NewBlock(ssa.BlockPlain) + line := s.peekPos() + pos := base.Ctxt.PosTable.Pos(line) + fl := funcLine{f: fn, base: pos.Base(), line: pos.Line()} + bPanic := s.panics[fl] + if bPanic == nil { + bPanic = s.f.NewBlock(ssa.BlockPlain) + s.panics[fl] = bPanic + s.startBlock(bPanic) + // The panic call takes/returns memory to ensure that the right + // memory state is observed if the panic happens. + s.rtcall(fn, false, nil) + } + b.AddEdgeTo(bNext) + b.AddEdgeTo(bPanic) + s.startBlock(bNext) +} + +func (s *state) intDivide(n ir.Node, a, b *ssa.Value) *ssa.Value { + needcheck := true + switch b.Op { + case ssa.OpConst8, ssa.OpConst16, ssa.OpConst32, ssa.OpConst64: + if b.AuxInt != 0 { + needcheck = false + } + } + if needcheck { + // do a size-appropriate check for zero + cmp := s.newValue2(s.ssaOp(ir.ONE, n.Type()), types.Types[types.TBOOL], b, s.zeroVal(n.Type())) + s.check(cmp, ir.Syms.Panicdivide) + } + return s.newValue2(s.ssaOp(n.Op(), n.Type()), a.Type, a, b) +} + +// rtcall issues a call to the given runtime function fn with the listed args. +// Returns a slice of results of the given result types. +// The call is added to the end of the current block. +// If returns is false, the block is marked as an exit block. +func (s *state) rtcall(fn *obj.LSym, returns bool, results []*types.Type, args ...*ssa.Value) []*ssa.Value { + s.prevCall = nil + // Write args to the stack + off := base.Ctxt.FixedFrameSize() + testLateExpansion := ssa.LateCallExpansionEnabledWithin(s.f) + var ACArgs []ssa.Param + var ACResults []ssa.Param + var callArgs []*ssa.Value + + for _, arg := range args { + t := arg.Type + off = types.Rnd(off, t.Alignment()) + size := t.Size() + ACArgs = append(ACArgs, ssa.Param{Type: t, Offset: int32(off)}) + if testLateExpansion { + callArgs = append(callArgs, arg) + } else { + ptr := s.constOffPtrSP(t.PtrTo(), off) + s.store(t, ptr, arg) + } + off += size + } + off = types.Rnd(off, int64(types.RegSize)) + + // Accumulate results types and offsets + offR := off + for _, t := range results { + offR = types.Rnd(offR, t.Alignment()) + ACResults = append(ACResults, ssa.Param{Type: t, Offset: int32(offR)}) + offR += t.Size() + } + + // Issue call + var call *ssa.Value + aux := ssa.StaticAuxCall(fn, ACArgs, ACResults) + if testLateExpansion { + callArgs = append(callArgs, s.mem()) + call = s.newValue0A(ssa.OpStaticLECall, aux.LateExpansionResultType(), aux) + call.AddArgs(callArgs...) + s.vars[memVar] = s.newValue1I(ssa.OpSelectN, types.TypeMem, int64(len(ACResults)), call) + } else { + call = s.newValue1A(ssa.OpStaticCall, types.TypeMem, aux, s.mem()) + s.vars[memVar] = call + } + + if !returns { + // Finish block + b := s.endBlock() + b.Kind = ssa.BlockExit + b.SetControl(call) + call.AuxInt = off - base.Ctxt.FixedFrameSize() + if len(results) > 0 { + s.Fatalf("panic call can't have results") + } + return nil + } + + // Load results + res := make([]*ssa.Value, len(results)) + if testLateExpansion { + for i, t := range results { + off = types.Rnd(off, t.Alignment()) + if TypeOK(t) { + res[i] = s.newValue1I(ssa.OpSelectN, t, int64(i), call) + } else { + addr := s.newValue1I(ssa.OpSelectNAddr, types.NewPtr(t), int64(i), call) + res[i] = s.rawLoad(t, addr) + } + off += t.Size() + } + } else { + for i, t := range results { + off = types.Rnd(off, t.Alignment()) + ptr := s.constOffPtrSP(types.NewPtr(t), off) + res[i] = s.load(t, ptr) + off += t.Size() + } + } + off = types.Rnd(off, int64(types.PtrSize)) + + // Remember how much callee stack space we needed. + call.AuxInt = off + + return res +} + +// do *left = right for type t. +func (s *state) storeType(t *types.Type, left, right *ssa.Value, skip skipMask, leftIsStmt bool) { + s.instrument(t, left, instrumentWrite) + + if skip == 0 && (!t.HasPointers() || ssa.IsStackAddr(left)) { + // Known to not have write barrier. Store the whole type. + s.vars[memVar] = s.newValue3Apos(ssa.OpStore, types.TypeMem, t, left, right, s.mem(), leftIsStmt) + return + } + + // store scalar fields first, so write barrier stores for + // pointer fields can be grouped together, and scalar values + // don't need to be live across the write barrier call. + // TODO: if the writebarrier pass knows how to reorder stores, + // we can do a single store here as long as skip==0. + s.storeTypeScalars(t, left, right, skip) + if skip&skipPtr == 0 && t.HasPointers() { + s.storeTypePtrs(t, left, right) + } +} + +// do *left = right for all scalar (non-pointer) parts of t. +func (s *state) storeTypeScalars(t *types.Type, left, right *ssa.Value, skip skipMask) { + switch { + case t.IsBoolean() || t.IsInteger() || t.IsFloat() || t.IsComplex(): + s.store(t, left, right) + case t.IsPtrShaped(): + if t.IsPtr() && t.Elem().NotInHeap() { + s.store(t, left, right) // see issue 42032 + } + // otherwise, no scalar fields. + case t.IsString(): + if skip&skipLen != 0 { + return + } + len := s.newValue1(ssa.OpStringLen, types.Types[types.TINT], right) + lenAddr := s.newValue1I(ssa.OpOffPtr, s.f.Config.Types.IntPtr, s.config.PtrSize, left) + s.store(types.Types[types.TINT], lenAddr, len) + case t.IsSlice(): + if skip&skipLen == 0 { + len := s.newValue1(ssa.OpSliceLen, types.Types[types.TINT], right) + lenAddr := s.newValue1I(ssa.OpOffPtr, s.f.Config.Types.IntPtr, s.config.PtrSize, left) + s.store(types.Types[types.TINT], lenAddr, len) + } + if skip&skipCap == 0 { + cap := s.newValue1(ssa.OpSliceCap, types.Types[types.TINT], right) + capAddr := s.newValue1I(ssa.OpOffPtr, s.f.Config.Types.IntPtr, 2*s.config.PtrSize, left) + s.store(types.Types[types.TINT], capAddr, cap) + } + case t.IsInterface(): + // itab field doesn't need a write barrier (even though it is a pointer). + itab := s.newValue1(ssa.OpITab, s.f.Config.Types.BytePtr, right) + s.store(types.Types[types.TUINTPTR], left, itab) + case t.IsStruct(): + n := t.NumFields() + for i := 0; i < n; i++ { + ft := t.FieldType(i) + addr := s.newValue1I(ssa.OpOffPtr, ft.PtrTo(), t.FieldOff(i), left) + val := s.newValue1I(ssa.OpStructSelect, ft, int64(i), right) + s.storeTypeScalars(ft, addr, val, 0) + } + case t.IsArray() && t.NumElem() == 0: + // nothing + case t.IsArray() && t.NumElem() == 1: + s.storeTypeScalars(t.Elem(), left, s.newValue1I(ssa.OpArraySelect, t.Elem(), 0, right), 0) + default: + s.Fatalf("bad write barrier type %v", t) + } +} + +// do *left = right for all pointer parts of t. +func (s *state) storeTypePtrs(t *types.Type, left, right *ssa.Value) { + switch { + case t.IsPtrShaped(): + if t.IsPtr() && t.Elem().NotInHeap() { + break // see issue 42032 + } + s.store(t, left, right) + case t.IsString(): + ptr := s.newValue1(ssa.OpStringPtr, s.f.Config.Types.BytePtr, right) + s.store(s.f.Config.Types.BytePtr, left, ptr) + case t.IsSlice(): + elType := types.NewPtr(t.Elem()) + ptr := s.newValue1(ssa.OpSlicePtr, elType, right) + s.store(elType, left, ptr) + case t.IsInterface(): + // itab field is treated as a scalar. + idata := s.newValue1(ssa.OpIData, s.f.Config.Types.BytePtr, right) + idataAddr := s.newValue1I(ssa.OpOffPtr, s.f.Config.Types.BytePtrPtr, s.config.PtrSize, left) + s.store(s.f.Config.Types.BytePtr, idataAddr, idata) + case t.IsStruct(): + n := t.NumFields() + for i := 0; i < n; i++ { + ft := t.FieldType(i) + if !ft.HasPointers() { + continue + } + addr := s.newValue1I(ssa.OpOffPtr, ft.PtrTo(), t.FieldOff(i), left) + val := s.newValue1I(ssa.OpStructSelect, ft, int64(i), right) + s.storeTypePtrs(ft, addr, val) + } + case t.IsArray() && t.NumElem() == 0: + // nothing + case t.IsArray() && t.NumElem() == 1: + s.storeTypePtrs(t.Elem(), left, s.newValue1I(ssa.OpArraySelect, t.Elem(), 0, right)) + default: + s.Fatalf("bad write barrier type %v", t) + } +} + +// putArg evaluates n for the purpose of passing it as an argument to a function and returns the corresponding Param for the call. +// If forLateExpandedCall is true, it returns the argument value to pass to the call operation. +// If forLateExpandedCall is false, then the value is stored at the specified stack offset, and the returned value is nil. +func (s *state) putArg(n ir.Node, t *types.Type, off int64, forLateExpandedCall bool) (ssa.Param, *ssa.Value) { + var a *ssa.Value + if forLateExpandedCall { + if !TypeOK(t) { + a = s.newValue2(ssa.OpDereference, t, s.addr(n), s.mem()) + } else { + a = s.expr(n) + } + } else { + s.storeArgWithBase(n, t, s.sp, off) + } + return ssa.Param{Type: t, Offset: int32(off)}, a +} + +func (s *state) storeArgWithBase(n ir.Node, t *types.Type, base *ssa.Value, off int64) { + pt := types.NewPtr(t) + var addr *ssa.Value + if base == s.sp { + // Use special routine that avoids allocation on duplicate offsets. + addr = s.constOffPtrSP(pt, off) + } else { + addr = s.newValue1I(ssa.OpOffPtr, pt, off, base) + } + + if !TypeOK(t) { + a := s.addr(n) + s.move(t, addr, a) + return + } + + a := s.expr(n) + s.storeType(t, addr, a, 0, false) +} + +// slice computes the slice v[i:j:k] and returns ptr, len, and cap of result. +// i,j,k may be nil, in which case they are set to their default value. +// v may be a slice, string or pointer to an array. +func (s *state) slice(v, i, j, k *ssa.Value, bounded bool) (p, l, c *ssa.Value) { + t := v.Type + var ptr, len, cap *ssa.Value + switch { + case t.IsSlice(): + ptr = s.newValue1(ssa.OpSlicePtr, types.NewPtr(t.Elem()), v) + len = s.newValue1(ssa.OpSliceLen, types.Types[types.TINT], v) + cap = s.newValue1(ssa.OpSliceCap, types.Types[types.TINT], v) + case t.IsString(): + ptr = s.newValue1(ssa.OpStringPtr, types.NewPtr(types.Types[types.TUINT8]), v) + len = s.newValue1(ssa.OpStringLen, types.Types[types.TINT], v) + cap = len + case t.IsPtr(): + if !t.Elem().IsArray() { + s.Fatalf("bad ptr to array in slice %v\n", t) + } + s.nilCheck(v) + ptr = s.newValue1(ssa.OpCopy, types.NewPtr(t.Elem().Elem()), v) + len = s.constInt(types.Types[types.TINT], t.Elem().NumElem()) + cap = len + default: + s.Fatalf("bad type in slice %v\n", t) + } + + // Set default values + if i == nil { + i = s.constInt(types.Types[types.TINT], 0) + } + if j == nil { + j = len + } + three := true + if k == nil { + three = false + k = cap + } + + // Panic if slice indices are not in bounds. + // Make sure we check these in reverse order so that we're always + // comparing against a value known to be nonnegative. See issue 28797. + if three { + if k != cap { + kind := ssa.BoundsSlice3Alen + if t.IsSlice() { + kind = ssa.BoundsSlice3Acap + } + k = s.boundsCheck(k, cap, kind, bounded) + } + if j != k { + j = s.boundsCheck(j, k, ssa.BoundsSlice3B, bounded) + } + i = s.boundsCheck(i, j, ssa.BoundsSlice3C, bounded) + } else { + if j != k { + kind := ssa.BoundsSliceAlen + if t.IsSlice() { + kind = ssa.BoundsSliceAcap + } + j = s.boundsCheck(j, k, kind, bounded) + } + i = s.boundsCheck(i, j, ssa.BoundsSliceB, bounded) + } + + // Word-sized integer operations. + subOp := s.ssaOp(ir.OSUB, types.Types[types.TINT]) + mulOp := s.ssaOp(ir.OMUL, types.Types[types.TINT]) + andOp := s.ssaOp(ir.OAND, types.Types[types.TINT]) + + // Calculate the length (rlen) and capacity (rcap) of the new slice. + // For strings the capacity of the result is unimportant. However, + // we use rcap to test if we've generated a zero-length slice. + // Use length of strings for that. + rlen := s.newValue2(subOp, types.Types[types.TINT], j, i) + rcap := rlen + if j != k && !t.IsString() { + rcap = s.newValue2(subOp, types.Types[types.TINT], k, i) + } + + if (i.Op == ssa.OpConst64 || i.Op == ssa.OpConst32) && i.AuxInt == 0 { + // No pointer arithmetic necessary. + return ptr, rlen, rcap + } + + // Calculate the base pointer (rptr) for the new slice. + // + // Generate the following code assuming that indexes are in bounds. + // The masking is to make sure that we don't generate a slice + // that points to the next object in memory. We cannot just set + // the pointer to nil because then we would create a nil slice or + // string. + // + // rcap = k - i + // rlen = j - i + // rptr = ptr + (mask(rcap) & (i * stride)) + // + // Where mask(x) is 0 if x==0 and -1 if x>0 and stride is the width + // of the element type. + stride := s.constInt(types.Types[types.TINT], ptr.Type.Elem().Width) + + // The delta is the number of bytes to offset ptr by. + delta := s.newValue2(mulOp, types.Types[types.TINT], i, stride) + + // If we're slicing to the point where the capacity is zero, + // zero out the delta. + mask := s.newValue1(ssa.OpSlicemask, types.Types[types.TINT], rcap) + delta = s.newValue2(andOp, types.Types[types.TINT], delta, mask) + + // Compute rptr = ptr + delta. + rptr := s.newValue2(ssa.OpAddPtr, ptr.Type, ptr, delta) + + return rptr, rlen, rcap +} + +type u642fcvtTab struct { + leq, cvt2F, and, rsh, or, add ssa.Op + one func(*state, *types.Type, int64) *ssa.Value +} + +var u64_f64 = u642fcvtTab{ + leq: ssa.OpLeq64, + cvt2F: ssa.OpCvt64to64F, + and: ssa.OpAnd64, + rsh: ssa.OpRsh64Ux64, + or: ssa.OpOr64, + add: ssa.OpAdd64F, + one: (*state).constInt64, +} + +var u64_f32 = u642fcvtTab{ + leq: ssa.OpLeq64, + cvt2F: ssa.OpCvt64to32F, + and: ssa.OpAnd64, + rsh: ssa.OpRsh64Ux64, + or: ssa.OpOr64, + add: ssa.OpAdd32F, + one: (*state).constInt64, +} + +func (s *state) uint64Tofloat64(n ir.Node, x *ssa.Value, ft, tt *types.Type) *ssa.Value { + return s.uint64Tofloat(&u64_f64, n, x, ft, tt) +} + +func (s *state) uint64Tofloat32(n ir.Node, x *ssa.Value, ft, tt *types.Type) *ssa.Value { + return s.uint64Tofloat(&u64_f32, n, x, ft, tt) +} + +func (s *state) uint64Tofloat(cvttab *u642fcvtTab, n ir.Node, x *ssa.Value, ft, tt *types.Type) *ssa.Value { + // if x >= 0 { + // result = (floatY) x + // } else { + // y = uintX(x) ; y = x & 1 + // z = uintX(x) ; z = z >> 1 + // z = z >> 1 + // z = z | y + // result = floatY(z) + // result = result + result + // } + // + // Code borrowed from old code generator. + // What's going on: large 64-bit "unsigned" looks like + // negative number to hardware's integer-to-float + // conversion. However, because the mantissa is only + // 63 bits, we don't need the LSB, so instead we do an + // unsigned right shift (divide by two), convert, and + // double. However, before we do that, we need to be + // sure that we do not lose a "1" if that made the + // difference in the resulting rounding. Therefore, we + // preserve it, and OR (not ADD) it back in. The case + // that matters is when the eleven discarded bits are + // equal to 10000000001; that rounds up, and the 1 cannot + // be lost else it would round down if the LSB of the + // candidate mantissa is 0. + cmp := s.newValue2(cvttab.leq, types.Types[types.TBOOL], s.zeroVal(ft), x) + b := s.endBlock() + b.Kind = ssa.BlockIf + b.SetControl(cmp) + b.Likely = ssa.BranchLikely + + bThen := s.f.NewBlock(ssa.BlockPlain) + bElse := s.f.NewBlock(ssa.BlockPlain) + bAfter := s.f.NewBlock(ssa.BlockPlain) + + b.AddEdgeTo(bThen) + s.startBlock(bThen) + a0 := s.newValue1(cvttab.cvt2F, tt, x) + s.vars[n] = a0 + s.endBlock() + bThen.AddEdgeTo(bAfter) + + b.AddEdgeTo(bElse) + s.startBlock(bElse) + one := cvttab.one(s, ft, 1) + y := s.newValue2(cvttab.and, ft, x, one) + z := s.newValue2(cvttab.rsh, ft, x, one) + z = s.newValue2(cvttab.or, ft, z, y) + a := s.newValue1(cvttab.cvt2F, tt, z) + a1 := s.newValue2(cvttab.add, tt, a, a) + s.vars[n] = a1 + s.endBlock() + bElse.AddEdgeTo(bAfter) + + s.startBlock(bAfter) + return s.variable(n, n.Type()) +} + +type u322fcvtTab struct { + cvtI2F, cvtF2F ssa.Op +} + +var u32_f64 = u322fcvtTab{ + cvtI2F: ssa.OpCvt32to64F, + cvtF2F: ssa.OpCopy, +} + +var u32_f32 = u322fcvtTab{ + cvtI2F: ssa.OpCvt32to32F, + cvtF2F: ssa.OpCvt64Fto32F, +} + +func (s *state) uint32Tofloat64(n ir.Node, x *ssa.Value, ft, tt *types.Type) *ssa.Value { + return s.uint32Tofloat(&u32_f64, n, x, ft, tt) +} + +func (s *state) uint32Tofloat32(n ir.Node, x *ssa.Value, ft, tt *types.Type) *ssa.Value { + return s.uint32Tofloat(&u32_f32, n, x, ft, tt) +} + +func (s *state) uint32Tofloat(cvttab *u322fcvtTab, n ir.Node, x *ssa.Value, ft, tt *types.Type) *ssa.Value { + // if x >= 0 { + // result = floatY(x) + // } else { + // result = floatY(float64(x) + (1<<32)) + // } + cmp := s.newValue2(ssa.OpLeq32, types.Types[types.TBOOL], s.zeroVal(ft), x) + b := s.endBlock() + b.Kind = ssa.BlockIf + b.SetControl(cmp) + b.Likely = ssa.BranchLikely + + bThen := s.f.NewBlock(ssa.BlockPlain) + bElse := s.f.NewBlock(ssa.BlockPlain) + bAfter := s.f.NewBlock(ssa.BlockPlain) + + b.AddEdgeTo(bThen) + s.startBlock(bThen) + a0 := s.newValue1(cvttab.cvtI2F, tt, x) + s.vars[n] = a0 + s.endBlock() + bThen.AddEdgeTo(bAfter) + + b.AddEdgeTo(bElse) + s.startBlock(bElse) + a1 := s.newValue1(ssa.OpCvt32to64F, types.Types[types.TFLOAT64], x) + twoToThe32 := s.constFloat64(types.Types[types.TFLOAT64], float64(1<<32)) + a2 := s.newValue2(ssa.OpAdd64F, types.Types[types.TFLOAT64], a1, twoToThe32) + a3 := s.newValue1(cvttab.cvtF2F, tt, a2) + + s.vars[n] = a3 + s.endBlock() + bElse.AddEdgeTo(bAfter) + + s.startBlock(bAfter) + return s.variable(n, n.Type()) +} + +// referenceTypeBuiltin generates code for the len/cap builtins for maps and channels. +func (s *state) referenceTypeBuiltin(n *ir.UnaryExpr, x *ssa.Value) *ssa.Value { + if !n.X.Type().IsMap() && !n.X.Type().IsChan() { + s.Fatalf("node must be a map or a channel") + } + // if n == nil { + // return 0 + // } else { + // // len + // return *((*int)n) + // // cap + // return *(((*int)n)+1) + // } + lenType := n.Type() + nilValue := s.constNil(types.Types[types.TUINTPTR]) + cmp := s.newValue2(ssa.OpEqPtr, types.Types[types.TBOOL], x, nilValue) + b := s.endBlock() + b.Kind = ssa.BlockIf + b.SetControl(cmp) + b.Likely = ssa.BranchUnlikely + + bThen := s.f.NewBlock(ssa.BlockPlain) + bElse := s.f.NewBlock(ssa.BlockPlain) + bAfter := s.f.NewBlock(ssa.BlockPlain) + + // length/capacity of a nil map/chan is zero + b.AddEdgeTo(bThen) + s.startBlock(bThen) + s.vars[n] = s.zeroVal(lenType) + s.endBlock() + bThen.AddEdgeTo(bAfter) + + b.AddEdgeTo(bElse) + s.startBlock(bElse) + switch n.Op() { + case ir.OLEN: + // length is stored in the first word for map/chan + s.vars[n] = s.load(lenType, x) + case ir.OCAP: + // capacity is stored in the second word for chan + sw := s.newValue1I(ssa.OpOffPtr, lenType.PtrTo(), lenType.Width, x) + s.vars[n] = s.load(lenType, sw) + default: + s.Fatalf("op must be OLEN or OCAP") + } + s.endBlock() + bElse.AddEdgeTo(bAfter) + + s.startBlock(bAfter) + return s.variable(n, lenType) +} + +type f2uCvtTab struct { + ltf, cvt2U, subf, or ssa.Op + floatValue func(*state, *types.Type, float64) *ssa.Value + intValue func(*state, *types.Type, int64) *ssa.Value + cutoff uint64 +} + +var f32_u64 = f2uCvtTab{ + ltf: ssa.OpLess32F, + cvt2U: ssa.OpCvt32Fto64, + subf: ssa.OpSub32F, + or: ssa.OpOr64, + floatValue: (*state).constFloat32, + intValue: (*state).constInt64, + cutoff: 1 << 63, +} + +var f64_u64 = f2uCvtTab{ + ltf: ssa.OpLess64F, + cvt2U: ssa.OpCvt64Fto64, + subf: ssa.OpSub64F, + or: ssa.OpOr64, + floatValue: (*state).constFloat64, + intValue: (*state).constInt64, + cutoff: 1 << 63, +} + +var f32_u32 = f2uCvtTab{ + ltf: ssa.OpLess32F, + cvt2U: ssa.OpCvt32Fto32, + subf: ssa.OpSub32F, + or: ssa.OpOr32, + floatValue: (*state).constFloat32, + intValue: func(s *state, t *types.Type, v int64) *ssa.Value { return s.constInt32(t, int32(v)) }, + cutoff: 1 << 31, +} + +var f64_u32 = f2uCvtTab{ + ltf: ssa.OpLess64F, + cvt2U: ssa.OpCvt64Fto32, + subf: ssa.OpSub64F, + or: ssa.OpOr32, + floatValue: (*state).constFloat64, + intValue: func(s *state, t *types.Type, v int64) *ssa.Value { return s.constInt32(t, int32(v)) }, + cutoff: 1 << 31, +} + +func (s *state) float32ToUint64(n ir.Node, x *ssa.Value, ft, tt *types.Type) *ssa.Value { + return s.floatToUint(&f32_u64, n, x, ft, tt) +} +func (s *state) float64ToUint64(n ir.Node, x *ssa.Value, ft, tt *types.Type) *ssa.Value { + return s.floatToUint(&f64_u64, n, x, ft, tt) +} + +func (s *state) float32ToUint32(n ir.Node, x *ssa.Value, ft, tt *types.Type) *ssa.Value { + return s.floatToUint(&f32_u32, n, x, ft, tt) +} + +func (s *state) float64ToUint32(n ir.Node, x *ssa.Value, ft, tt *types.Type) *ssa.Value { + return s.floatToUint(&f64_u32, n, x, ft, tt) +} + +func (s *state) floatToUint(cvttab *f2uCvtTab, n ir.Node, x *ssa.Value, ft, tt *types.Type) *ssa.Value { + // cutoff:=1<<(intY_Size-1) + // if x < floatX(cutoff) { + // result = uintY(x) + // } else { + // y = x - floatX(cutoff) + // z = uintY(y) + // result = z | -(cutoff) + // } + cutoff := cvttab.floatValue(s, ft, float64(cvttab.cutoff)) + cmp := s.newValue2(cvttab.ltf, types.Types[types.TBOOL], x, cutoff) + b := s.endBlock() + b.Kind = ssa.BlockIf + b.SetControl(cmp) + b.Likely = ssa.BranchLikely + + bThen := s.f.NewBlock(ssa.BlockPlain) + bElse := s.f.NewBlock(ssa.BlockPlain) + bAfter := s.f.NewBlock(ssa.BlockPlain) + + b.AddEdgeTo(bThen) + s.startBlock(bThen) + a0 := s.newValue1(cvttab.cvt2U, tt, x) + s.vars[n] = a0 + s.endBlock() + bThen.AddEdgeTo(bAfter) + + b.AddEdgeTo(bElse) + s.startBlock(bElse) + y := s.newValue2(cvttab.subf, ft, x, cutoff) + y = s.newValue1(cvttab.cvt2U, tt, y) + z := cvttab.intValue(s, tt, int64(-cvttab.cutoff)) + a1 := s.newValue2(cvttab.or, tt, y, z) + s.vars[n] = a1 + s.endBlock() + bElse.AddEdgeTo(bAfter) + + s.startBlock(bAfter) + return s.variable(n, n.Type()) +} + +// dottype generates SSA for a type assertion node. +// commaok indicates whether to panic or return a bool. +// If commaok is false, resok will be nil. +func (s *state) dottype(n *ir.TypeAssertExpr, commaok bool) (res, resok *ssa.Value) { + iface := s.expr(n.X) // input interface + target := s.expr(n.Ntype) // target type + byteptr := s.f.Config.Types.BytePtr + + if n.Type().IsInterface() { + if n.Type().IsEmptyInterface() { + // Converting to an empty interface. + // Input could be an empty or nonempty interface. + if base.Debug.TypeAssert > 0 { + base.WarnfAt(n.Pos(), "type assertion inlined") + } + + // Get itab/type field from input. + itab := s.newValue1(ssa.OpITab, byteptr, iface) + // Conversion succeeds iff that field is not nil. + cond := s.newValue2(ssa.OpNeqPtr, types.Types[types.TBOOL], itab, s.constNil(byteptr)) + + if n.X.Type().IsEmptyInterface() && commaok { + // Converting empty interface to empty interface with ,ok is just a nil check. + return iface, cond + } + + // Branch on nilness. + b := s.endBlock() + b.Kind = ssa.BlockIf + b.SetControl(cond) + b.Likely = ssa.BranchLikely + bOk := s.f.NewBlock(ssa.BlockPlain) + bFail := s.f.NewBlock(ssa.BlockPlain) + b.AddEdgeTo(bOk) + b.AddEdgeTo(bFail) + + if !commaok { + // On failure, panic by calling panicnildottype. + s.startBlock(bFail) + s.rtcall(ir.Syms.Panicnildottype, false, nil, target) + + // On success, return (perhaps modified) input interface. + s.startBlock(bOk) + if n.X.Type().IsEmptyInterface() { + res = iface // Use input interface unchanged. + return + } + // Load type out of itab, build interface with existing idata. + off := s.newValue1I(ssa.OpOffPtr, byteptr, int64(types.PtrSize), itab) + typ := s.load(byteptr, off) + idata := s.newValue1(ssa.OpIData, byteptr, iface) + res = s.newValue2(ssa.OpIMake, n.Type(), typ, idata) + return + } + + s.startBlock(bOk) + // nonempty -> empty + // Need to load type from itab + off := s.newValue1I(ssa.OpOffPtr, byteptr, int64(types.PtrSize), itab) + s.vars[typVar] = s.load(byteptr, off) + s.endBlock() + + // itab is nil, might as well use that as the nil result. + s.startBlock(bFail) + s.vars[typVar] = itab + s.endBlock() + + // Merge point. + bEnd := s.f.NewBlock(ssa.BlockPlain) + bOk.AddEdgeTo(bEnd) + bFail.AddEdgeTo(bEnd) + s.startBlock(bEnd) + idata := s.newValue1(ssa.OpIData, byteptr, iface) + res = s.newValue2(ssa.OpIMake, n.Type(), s.variable(typVar, byteptr), idata) + resok = cond + delete(s.vars, typVar) + return + } + // converting to a nonempty interface needs a runtime call. + if base.Debug.TypeAssert > 0 { + base.WarnfAt(n.Pos(), "type assertion not inlined") + } + if n.X.Type().IsEmptyInterface() { + if commaok { + call := s.rtcall(ir.Syms.AssertE2I2, true, []*types.Type{n.Type(), types.Types[types.TBOOL]}, target, iface) + return call[0], call[1] + } + return s.rtcall(ir.Syms.AssertE2I, true, []*types.Type{n.Type()}, target, iface)[0], nil + } + if commaok { + call := s.rtcall(ir.Syms.AssertI2I2, true, []*types.Type{n.Type(), types.Types[types.TBOOL]}, target, iface) + return call[0], call[1] + } + return s.rtcall(ir.Syms.AssertI2I, true, []*types.Type{n.Type()}, target, iface)[0], nil + } + + if base.Debug.TypeAssert > 0 { + base.WarnfAt(n.Pos(), "type assertion inlined") + } + + // Converting to a concrete type. + direct := types.IsDirectIface(n.Type()) + itab := s.newValue1(ssa.OpITab, byteptr, iface) // type word of interface + if base.Debug.TypeAssert > 0 { + base.WarnfAt(n.Pos(), "type assertion inlined") + } + var targetITab *ssa.Value + if n.X.Type().IsEmptyInterface() { + // Looking for pointer to target type. + targetITab = target + } else { + // Looking for pointer to itab for target type and source interface. + targetITab = s.expr(n.Itab[0]) + } + + var tmp ir.Node // temporary for use with large types + var addr *ssa.Value // address of tmp + if commaok && !TypeOK(n.Type()) { + // unSSAable type, use temporary. + // TODO: get rid of some of these temporaries. + tmp = typecheck.TempAt(n.Pos(), s.curfn, n.Type()) + s.vars[memVar] = s.newValue1A(ssa.OpVarDef, types.TypeMem, tmp.(*ir.Name), s.mem()) + addr = s.addr(tmp) + } + + cond := s.newValue2(ssa.OpEqPtr, types.Types[types.TBOOL], itab, targetITab) + b := s.endBlock() + b.Kind = ssa.BlockIf + b.SetControl(cond) + b.Likely = ssa.BranchLikely + + bOk := s.f.NewBlock(ssa.BlockPlain) + bFail := s.f.NewBlock(ssa.BlockPlain) + b.AddEdgeTo(bOk) + b.AddEdgeTo(bFail) + + if !commaok { + // on failure, panic by calling panicdottype + s.startBlock(bFail) + taddr := s.expr(n.Ntype.(*ir.AddrExpr).Alloc) + if n.X.Type().IsEmptyInterface() { + s.rtcall(ir.Syms.PanicdottypeE, false, nil, itab, target, taddr) + } else { + s.rtcall(ir.Syms.PanicdottypeI, false, nil, itab, target, taddr) + } + + // on success, return data from interface + s.startBlock(bOk) + if direct { + return s.newValue1(ssa.OpIData, n.Type(), iface), nil + } + p := s.newValue1(ssa.OpIData, types.NewPtr(n.Type()), iface) + return s.load(n.Type(), p), nil + } + + // commaok is the more complicated case because we have + // a control flow merge point. + bEnd := s.f.NewBlock(ssa.BlockPlain) + // Note that we need a new valVar each time (unlike okVar where we can + // reuse the variable) because it might have a different type every time. + valVar := ssaMarker("val") + + // type assertion succeeded + s.startBlock(bOk) + if tmp == nil { + if direct { + s.vars[valVar] = s.newValue1(ssa.OpIData, n.Type(), iface) + } else { + p := s.newValue1(ssa.OpIData, types.NewPtr(n.Type()), iface) + s.vars[valVar] = s.load(n.Type(), p) + } + } else { + p := s.newValue1(ssa.OpIData, types.NewPtr(n.Type()), iface) + s.move(n.Type(), addr, p) + } + s.vars[okVar] = s.constBool(true) + s.endBlock() + bOk.AddEdgeTo(bEnd) + + // type assertion failed + s.startBlock(bFail) + if tmp == nil { + s.vars[valVar] = s.zeroVal(n.Type()) + } else { + s.zero(n.Type(), addr) + } + s.vars[okVar] = s.constBool(false) + s.endBlock() + bFail.AddEdgeTo(bEnd) + + // merge point + s.startBlock(bEnd) + if tmp == nil { + res = s.variable(valVar, n.Type()) + delete(s.vars, valVar) + } else { + res = s.load(n.Type(), addr) + s.vars[memVar] = s.newValue1A(ssa.OpVarKill, types.TypeMem, tmp.(*ir.Name), s.mem()) + } + resok = s.variable(okVar, types.Types[types.TBOOL]) + delete(s.vars, okVar) + return res, resok +} + +// variable returns the value of a variable at the current location. +func (s *state) variable(n ir.Node, t *types.Type) *ssa.Value { + v := s.vars[n] + if v != nil { + return v + } + v = s.fwdVars[n] + if v != nil { + return v + } + + if s.curBlock == s.f.Entry { + // No variable should be live at entry. + s.Fatalf("Value live at entry. It shouldn't be. func %s, node %v, value %v", s.f.Name, n, v) + } + // Make a FwdRef, which records a value that's live on block input. + // We'll find the matching definition as part of insertPhis. + v = s.newValue0A(ssa.OpFwdRef, t, fwdRefAux{N: n}) + s.fwdVars[n] = v + if n.Op() == ir.ONAME { + s.addNamedValue(n.(*ir.Name), v) + } + return v +} + +func (s *state) mem() *ssa.Value { + return s.variable(memVar, types.TypeMem) +} + +func (s *state) addNamedValue(n *ir.Name, v *ssa.Value) { + if n.Class_ == ir.Pxxx { + // Don't track our marker nodes (memVar etc.). + return + } + if ir.IsAutoTmp(n) { + // Don't track temporary variables. + return + } + if n.Class_ == ir.PPARAMOUT { + // Don't track named output values. This prevents return values + // from being assigned too early. See #14591 and #14762. TODO: allow this. + return + } + loc := ssa.LocalSlot{N: n.Name(), Type: n.Type(), Off: 0} + values, ok := s.f.NamedValues[loc] + if !ok { + s.f.Names = append(s.f.Names, loc) + } + s.f.NamedValues[loc] = append(values, v) +} + +// Generate a disconnected call to a runtime routine and a return. +func gencallret(pp *objw.Progs, sym *obj.LSym) *obj.Prog { + p := pp.Prog(obj.ACALL) + p.To.Type = obj.TYPE_MEM + p.To.Name = obj.NAME_EXTERN + p.To.Sym = sym + p = pp.Prog(obj.ARET) + return p +} + +// Branch is an unresolved branch. +type Branch struct { + P *obj.Prog // branch instruction + B *ssa.Block // target +} + +// State contains state needed during Prog generation. +type State struct { + pp *objw.Progs + + // Branches remembers all the branch instructions we've seen + // and where they would like to go. + Branches []Branch + + // bstart remembers where each block starts (indexed by block ID) + bstart []*obj.Prog + + // Some architectures require a 64-bit temporary for FP-related register shuffling. Examples include PPC and Sparc V8. + ScratchFpMem *ir.Name + + maxarg int64 // largest frame size for arguments to calls made by the function + + // Map from GC safe points to liveness index, generated by + // liveness analysis. + livenessMap liveness.Map + + // lineRunStart records the beginning of the current run of instructions + // within a single block sharing the same line number + // Used to move statement marks to the beginning of such runs. + lineRunStart *obj.Prog + + // wasm: The number of values on the WebAssembly stack. This is only used as a safeguard. + OnWasmStackSkipped int +} + +// Prog appends a new Prog. +func (s *State) Prog(as obj.As) *obj.Prog { + p := s.pp.Prog(as) + if ssa.LosesStmtMark(as) { + return p + } + // Float a statement start to the beginning of any same-line run. + // lineRunStart is reset at block boundaries, which appears to work well. + if s.lineRunStart == nil || s.lineRunStart.Pos.Line() != p.Pos.Line() { + s.lineRunStart = p + } else if p.Pos.IsStmt() == src.PosIsStmt { + s.lineRunStart.Pos = s.lineRunStart.Pos.WithIsStmt() + p.Pos = p.Pos.WithNotStmt() + } + return p +} + +// Pc returns the current Prog. +func (s *State) Pc() *obj.Prog { + return s.pp.Next +} + +// SetPos sets the current source position. +func (s *State) SetPos(pos src.XPos) { + s.pp.Pos = pos +} + +// Br emits a single branch instruction and returns the instruction. +// Not all architectures need the returned instruction, but otherwise +// the boilerplate is common to all. +func (s *State) Br(op obj.As, target *ssa.Block) *obj.Prog { + p := s.Prog(op) + p.To.Type = obj.TYPE_BRANCH + s.Branches = append(s.Branches, Branch{P: p, B: target}) + return p +} + +// DebugFriendlySetPosFrom adjusts Pos.IsStmt subject to heuristics +// that reduce "jumpy" line number churn when debugging. +// Spill/fill/copy instructions from the register allocator, +// phi functions, and instructions with a no-pos position +// are examples of instructions that can cause churn. +func (s *State) DebugFriendlySetPosFrom(v *ssa.Value) { + switch v.Op { + case ssa.OpPhi, ssa.OpCopy, ssa.OpLoadReg, ssa.OpStoreReg: + // These are not statements + s.SetPos(v.Pos.WithNotStmt()) + default: + p := v.Pos + if p != src.NoXPos { + // If the position is defined, update the position. + // Also convert default IsStmt to NotStmt; only + // explicit statement boundaries should appear + // in the generated code. + if p.IsStmt() != src.PosIsStmt { + p = p.WithNotStmt() + // Calls use the pos attached to v, but copy the statement mark from SSAGenState + } + s.SetPos(p) + } else { + s.SetPos(s.pp.Pos.WithNotStmt()) + } + } +} + +// byXoffset implements sort.Interface for []*ir.Name using Xoffset as the ordering. +type byXoffset []*ir.Name + +func (s byXoffset) Len() int { return len(s) } +func (s byXoffset) Less(i, j int) bool { return s[i].FrameOffset() < s[j].FrameOffset() } +func (s byXoffset) Swap(i, j int) { s[i], s[j] = s[j], s[i] } + +func emitStackObjects(e *ssafn, pp *objw.Progs) { + var vars []*ir.Name + for _, n := range e.curfn.Dcl { + if liveness.ShouldTrack(n) && n.Addrtaken() { + vars = append(vars, n) + } + } + if len(vars) == 0 { + return + } + + // Sort variables from lowest to highest address. + sort.Sort(byXoffset(vars)) + + // Populate the stack object data. + // Format must match runtime/stack.go:stackObjectRecord. + x := e.curfn.LSym.Func().StackObjects + off := 0 + off = objw.Uintptr(x, off, uint64(len(vars))) + for _, v := range vars { + // Note: arguments and return values have non-negative Xoffset, + // in which case the offset is relative to argp. + // Locals have a negative Xoffset, in which case the offset is relative to varp. + off = objw.Uintptr(x, off, uint64(v.FrameOffset())) + if !types.TypeSym(v.Type()).Siggen() { + e.Fatalf(v.Pos(), "stack object's type symbol not generated for type %s", v.Type()) + } + off = objw.SymPtr(x, off, reflectdata.WriteType(v.Type()), 0) + } + + // Emit a funcdata pointing at the stack object data. + p := pp.Prog(obj.AFUNCDATA) + p.From.SetConst(objabi.FUNCDATA_StackObjects) + p.To.Type = obj.TYPE_MEM + p.To.Name = obj.NAME_EXTERN + p.To.Sym = x + + if base.Flag.Live != 0 { + for _, v := range vars { + base.WarnfAt(v.Pos(), "stack object %v %s", v, v.Type().String()) + } + } +} + +// genssa appends entries to pp for each instruction in f. +func genssa(f *ssa.Func, pp *objw.Progs) { + var s State + + e := f.Frontend().(*ssafn) + + s.livenessMap = liveness.Compute(e.curfn, f, e.stkptrsize, pp) + emitStackObjects(e, pp) + + openDeferInfo := e.curfn.LSym.Func().OpenCodedDeferInfo + if openDeferInfo != nil { + // This function uses open-coded defers -- write out the funcdata + // info that we computed at the end of genssa. + p := pp.Prog(obj.AFUNCDATA) + p.From.SetConst(objabi.FUNCDATA_OpenCodedDeferInfo) + p.To.Type = obj.TYPE_MEM + p.To.Name = obj.NAME_EXTERN + p.To.Sym = openDeferInfo + } + + // Remember where each block starts. + s.bstart = make([]*obj.Prog, f.NumBlocks()) + s.pp = pp + var progToValue map[*obj.Prog]*ssa.Value + var progToBlock map[*obj.Prog]*ssa.Block + var valueToProgAfter []*obj.Prog // The first Prog following computation of a value v; v is visible at this point. + if f.PrintOrHtmlSSA { + progToValue = make(map[*obj.Prog]*ssa.Value, f.NumValues()) + progToBlock = make(map[*obj.Prog]*ssa.Block, f.NumBlocks()) + f.Logf("genssa %s\n", f.Name) + progToBlock[s.pp.Next] = f.Blocks[0] + } + + s.ScratchFpMem = e.scratchFpMem + + if base.Ctxt.Flag_locationlists { + if cap(f.Cache.ValueToProgAfter) < f.NumValues() { + f.Cache.ValueToProgAfter = make([]*obj.Prog, f.NumValues()) + } + valueToProgAfter = f.Cache.ValueToProgAfter[:f.NumValues()] + for i := range valueToProgAfter { + valueToProgAfter[i] = nil + } + } + + // If the very first instruction is not tagged as a statement, + // debuggers may attribute it to previous function in program. + firstPos := src.NoXPos + for _, v := range f.Entry.Values { + if v.Pos.IsStmt() == src.PosIsStmt { + firstPos = v.Pos + v.Pos = firstPos.WithDefaultStmt() + break + } + } + + // inlMarks has an entry for each Prog that implements an inline mark. + // It maps from that Prog to the global inlining id of the inlined body + // which should unwind to this Prog's location. + var inlMarks map[*obj.Prog]int32 + var inlMarkList []*obj.Prog + + // inlMarksByPos maps from a (column 1) source position to the set of + // Progs that are in the set above and have that source position. + var inlMarksByPos map[src.XPos][]*obj.Prog + + // Emit basic blocks + for i, b := range f.Blocks { + s.bstart[b.ID] = s.pp.Next + s.lineRunStart = nil + + // Attach a "default" liveness info. Normally this will be + // overwritten in the Values loop below for each Value. But + // for an empty block this will be used for its control + // instruction. We won't use the actual liveness map on a + // control instruction. Just mark it something that is + // preemptible, unless this function is "all unsafe". + s.pp.NextLive = objw.LivenessIndex{StackMapIndex: -1, IsUnsafePoint: liveness.IsUnsafe(f)} + + // Emit values in block + Arch.SSAMarkMoves(&s, b) + for _, v := range b.Values { + x := s.pp.Next + s.DebugFriendlySetPosFrom(v) + + switch v.Op { + case ssa.OpInitMem: + // memory arg needs no code + case ssa.OpArg: + // input args need no code + case ssa.OpSP, ssa.OpSB: + // nothing to do + case ssa.OpSelect0, ssa.OpSelect1: + // nothing to do + case ssa.OpGetG: + // nothing to do when there's a g register, + // and checkLower complains if there's not + case ssa.OpVarDef, ssa.OpVarLive, ssa.OpKeepAlive, ssa.OpVarKill: + // nothing to do; already used by liveness + case ssa.OpPhi: + CheckLoweredPhi(v) + case ssa.OpConvert: + // nothing to do; no-op conversion for liveness + if v.Args[0].Reg() != v.Reg() { + v.Fatalf("OpConvert should be a no-op: %s; %s", v.Args[0].LongString(), v.LongString()) + } + case ssa.OpInlMark: + p := Arch.Ginsnop(s.pp) + if inlMarks == nil { + inlMarks = map[*obj.Prog]int32{} + inlMarksByPos = map[src.XPos][]*obj.Prog{} + } + inlMarks[p] = v.AuxInt32() + inlMarkList = append(inlMarkList, p) + pos := v.Pos.AtColumn1() + inlMarksByPos[pos] = append(inlMarksByPos[pos], p) + + default: + // Attach this safe point to the next + // instruction. + s.pp.NextLive = s.livenessMap.Get(v) + + // Special case for first line in function; move it to the start. + if firstPos != src.NoXPos { + s.SetPos(firstPos) + firstPos = src.NoXPos + } + // let the backend handle it + Arch.SSAGenValue(&s, v) + } + + if base.Ctxt.Flag_locationlists { + valueToProgAfter[v.ID] = s.pp.Next + } + + if f.PrintOrHtmlSSA { + for ; x != s.pp.Next; x = x.Link { + progToValue[x] = v + } + } + } + // If this is an empty infinite loop, stick a hardware NOP in there so that debuggers are less confused. + if s.bstart[b.ID] == s.pp.Next && len(b.Succs) == 1 && b.Succs[0].Block() == b { + p := Arch.Ginsnop(s.pp) + p.Pos = p.Pos.WithIsStmt() + if b.Pos == src.NoXPos { + b.Pos = p.Pos // It needs a file, otherwise a no-file non-zero line causes confusion. See #35652. + if b.Pos == src.NoXPos { + b.Pos = pp.Text.Pos // Sometimes p.Pos is empty. See #35695. + } + } + b.Pos = b.Pos.WithBogusLine() // Debuggers are not good about infinite loops, force a change in line number + } + // Emit control flow instructions for block + var next *ssa.Block + if i < len(f.Blocks)-1 && base.Flag.N == 0 { + // If -N, leave next==nil so every block with successors + // ends in a JMP (except call blocks - plive doesn't like + // select{send,recv} followed by a JMP call). Helps keep + // line numbers for otherwise empty blocks. + next = f.Blocks[i+1] + } + x := s.pp.Next + s.SetPos(b.Pos) + Arch.SSAGenBlock(&s, b, next) + if f.PrintOrHtmlSSA { + for ; x != s.pp.Next; x = x.Link { + progToBlock[x] = b + } + } + } + if f.Blocks[len(f.Blocks)-1].Kind == ssa.BlockExit { + // We need the return address of a panic call to + // still be inside the function in question. So if + // it ends in a call which doesn't return, add a + // nop (which will never execute) after the call. + Arch.Ginsnop(pp) + } + if openDeferInfo != nil { + // When doing open-coded defers, generate a disconnected call to + // deferreturn and a return. This will be used to during panic + // recovery to unwind the stack and return back to the runtime. + s.pp.NextLive = s.livenessMap.DeferReturn + gencallret(pp, ir.Syms.Deferreturn) + } + + if inlMarks != nil { + // We have some inline marks. Try to find other instructions we're + // going to emit anyway, and use those instructions instead of the + // inline marks. + for p := pp.Text; p != nil; p = p.Link { + if p.As == obj.ANOP || p.As == obj.AFUNCDATA || p.As == obj.APCDATA || p.As == obj.ATEXT || p.As == obj.APCALIGN || Arch.LinkArch.Family == sys.Wasm { + // Don't use 0-sized instructions as inline marks, because we need + // to identify inline mark instructions by pc offset. + // (Some of these instructions are sometimes zero-sized, sometimes not. + // We must not use anything that even might be zero-sized.) + // TODO: are there others? + continue + } + if _, ok := inlMarks[p]; ok { + // Don't use inline marks themselves. We don't know + // whether they will be zero-sized or not yet. + continue + } + pos := p.Pos.AtColumn1() + s := inlMarksByPos[pos] + if len(s) == 0 { + continue + } + for _, m := range s { + // We found an instruction with the same source position as + // some of the inline marks. + // Use this instruction instead. + p.Pos = p.Pos.WithIsStmt() // promote position to a statement + pp.CurFunc.LSym.Func().AddInlMark(p, inlMarks[m]) + // Make the inline mark a real nop, so it doesn't generate any code. + m.As = obj.ANOP + m.Pos = src.NoXPos + m.From = obj.Addr{} + m.To = obj.Addr{} + } + delete(inlMarksByPos, pos) + } + // Any unmatched inline marks now need to be added to the inlining tree (and will generate a nop instruction). + for _, p := range inlMarkList { + if p.As != obj.ANOP { + pp.CurFunc.LSym.Func().AddInlMark(p, inlMarks[p]) + } + } + } + + if base.Ctxt.Flag_locationlists { + debugInfo := ssa.BuildFuncDebug(base.Ctxt, f, base.Debug.LocationLists > 1, StackOffset) + e.curfn.DebugInfo = debugInfo + bstart := s.bstart + // Note that at this moment, Prog.Pc is a sequence number; it's + // not a real PC until after assembly, so this mapping has to + // be done later. + debugInfo.GetPC = func(b, v ssa.ID) int64 { + switch v { + case ssa.BlockStart.ID: + if b == f.Entry.ID { + return 0 // Start at the very beginning, at the assembler-generated prologue. + // this should only happen for function args (ssa.OpArg) + } + return bstart[b].Pc + case ssa.BlockEnd.ID: + return e.curfn.LSym.Size + default: + return valueToProgAfter[v].Pc + } + } + } + + // Resolve branches, and relax DefaultStmt into NotStmt + for _, br := range s.Branches { + br.P.To.SetTarget(s.bstart[br.B.ID]) + if br.P.Pos.IsStmt() != src.PosIsStmt { + br.P.Pos = br.P.Pos.WithNotStmt() + } else if v0 := br.B.FirstPossibleStmtValue(); v0 != nil && v0.Pos.Line() == br.P.Pos.Line() && v0.Pos.IsStmt() == src.PosIsStmt { + br.P.Pos = br.P.Pos.WithNotStmt() + } + + } + + if e.log { // spew to stdout + filename := "" + for p := pp.Text; p != nil; p = p.Link { + if p.Pos.IsKnown() && p.InnermostFilename() != filename { + filename = p.InnermostFilename() + f.Logf("# %s\n", filename) + } + + var s string + if v, ok := progToValue[p]; ok { + s = v.String() + } else if b, ok := progToBlock[p]; ok { + s = b.String() + } else { + s = " " // most value and branch strings are 2-3 characters long + } + f.Logf(" %-6s\t%.5d (%s)\t%s\n", s, p.Pc, p.InnermostLineNumber(), p.InstructionString()) + } + } + if f.HTMLWriter != nil { // spew to ssa.html + var buf bytes.Buffer + buf.WriteString("") + buf.WriteString("
") + filename := "" + for p := pp.Text; p != nil; p = p.Link { + // Don't spam every line with the file name, which is often huge. + // Only print changes, and "unknown" is not a change. + if p.Pos.IsKnown() && p.InnermostFilename() != filename { + filename = p.InnermostFilename() + buf.WriteString("
") + buf.WriteString(html.EscapeString("# " + filename)) + buf.WriteString("
") + } + + buf.WriteString("
") + if v, ok := progToValue[p]; ok { + buf.WriteString(v.HTML()) + } else if b, ok := progToBlock[p]; ok { + buf.WriteString("" + b.HTML() + "") + } + buf.WriteString("
") + buf.WriteString("
") + buf.WriteString(fmt.Sprintf("%.5d (%s) %s", p.Pc, p.InnermostLineNumber(), p.InnermostLineNumberHTML(), html.EscapeString(p.InstructionString()))) + buf.WriteString("
") + } + buf.WriteString("
") + buf.WriteString("") + f.HTMLWriter.WriteColumn("genssa", "genssa", "ssa-prog", buf.String()) + } + + defframe(&s, e) + + f.HTMLWriter.Close() + f.HTMLWriter = nil +} + +func defframe(s *State, e *ssafn) { + pp := s.pp + + frame := types.Rnd(s.maxarg+e.stksize, int64(types.RegSize)) + if Arch.PadFrame != nil { + frame = Arch.PadFrame(frame) + } + + // Fill in argument and frame size. + pp.Text.To.Type = obj.TYPE_TEXTSIZE + pp.Text.To.Val = int32(types.Rnd(e.curfn.Type().ArgWidth(), int64(types.RegSize))) + pp.Text.To.Offset = frame + + // Insert code to zero ambiguously live variables so that the + // garbage collector only sees initialized values when it + // looks for pointers. + p := pp.Text + var lo, hi int64 + + // Opaque state for backend to use. Current backends use it to + // keep track of which helper registers have been zeroed. + var state uint32 + + // Iterate through declarations. They are sorted in decreasing Xoffset order. + for _, n := range e.curfn.Dcl { + if !n.Needzero() { + continue + } + if n.Class_ != ir.PAUTO { + e.Fatalf(n.Pos(), "needzero class %d", n.Class_) + } + if n.Type().Size()%int64(types.PtrSize) != 0 || n.FrameOffset()%int64(types.PtrSize) != 0 || n.Type().Size() == 0 { + e.Fatalf(n.Pos(), "var %L has size %d offset %d", n, n.Type().Size(), n.Offset_) + } + + if lo != hi && n.FrameOffset()+n.Type().Size() >= lo-int64(2*types.RegSize) { + // Merge with range we already have. + lo = n.FrameOffset() + continue + } + + // Zero old range + p = Arch.ZeroRange(pp, p, frame+lo, hi-lo, &state) + + // Set new range. + lo = n.FrameOffset() + hi = lo + n.Type().Size() + } + + // Zero final range. + Arch.ZeroRange(pp, p, frame+lo, hi-lo, &state) +} + +// For generating consecutive jump instructions to model a specific branching +type IndexJump struct { + Jump obj.As + Index int +} + +func (s *State) oneJump(b *ssa.Block, jump *IndexJump) { + p := s.Br(jump.Jump, b.Succs[jump.Index].Block()) + p.Pos = b.Pos +} + +// CombJump generates combinational instructions (2 at present) for a block jump, +// thereby the behaviour of non-standard condition codes could be simulated +func (s *State) CombJump(b, next *ssa.Block, jumps *[2][2]IndexJump) { + switch next { + case b.Succs[0].Block(): + s.oneJump(b, &jumps[0][0]) + s.oneJump(b, &jumps[0][1]) + case b.Succs[1].Block(): + s.oneJump(b, &jumps[1][0]) + s.oneJump(b, &jumps[1][1]) + default: + var q *obj.Prog + if b.Likely != ssa.BranchUnlikely { + s.oneJump(b, &jumps[1][0]) + s.oneJump(b, &jumps[1][1]) + q = s.Br(obj.AJMP, b.Succs[1].Block()) + } else { + s.oneJump(b, &jumps[0][0]) + s.oneJump(b, &jumps[0][1]) + q = s.Br(obj.AJMP, b.Succs[0].Block()) + } + q.Pos = b.Pos + } +} + +// AddAux adds the offset in the aux fields (AuxInt and Aux) of v to a. +func AddAux(a *obj.Addr, v *ssa.Value) { + AddAux2(a, v, v.AuxInt) +} +func AddAux2(a *obj.Addr, v *ssa.Value, offset int64) { + if a.Type != obj.TYPE_MEM && a.Type != obj.TYPE_ADDR { + v.Fatalf("bad AddAux addr %v", a) + } + // add integer offset + a.Offset += offset + + // If no additional symbol offset, we're done. + if v.Aux == nil { + return + } + // Add symbol's offset from its base register. + switch n := v.Aux.(type) { + case *ssa.AuxCall: + a.Name = obj.NAME_EXTERN + a.Sym = n.Fn + case *obj.LSym: + a.Name = obj.NAME_EXTERN + a.Sym = n + case *ir.Name: + if n.Class_ == ir.PPARAM || n.Class_ == ir.PPARAMOUT { + a.Name = obj.NAME_PARAM + a.Sym = ir.Orig(n).Sym().Linksym() + a.Offset += n.FrameOffset() + break + } + a.Name = obj.NAME_AUTO + a.Sym = n.Sym().Linksym() + a.Offset += n.FrameOffset() + default: + v.Fatalf("aux in %s not implemented %#v", v, v.Aux) + } +} + +// extendIndex extends v to a full int width. +// panic with the given kind if v does not fit in an int (only on 32-bit archs). +func (s *state) extendIndex(idx, len *ssa.Value, kind ssa.BoundsKind, bounded bool) *ssa.Value { + size := idx.Type.Size() + if size == s.config.PtrSize { + return idx + } + if size > s.config.PtrSize { + // truncate 64-bit indexes on 32-bit pointer archs. Test the + // high word and branch to out-of-bounds failure if it is not 0. + var lo *ssa.Value + if idx.Type.IsSigned() { + lo = s.newValue1(ssa.OpInt64Lo, types.Types[types.TINT], idx) + } else { + lo = s.newValue1(ssa.OpInt64Lo, types.Types[types.TUINT], idx) + } + if bounded || base.Flag.B != 0 { + return lo + } + bNext := s.f.NewBlock(ssa.BlockPlain) + bPanic := s.f.NewBlock(ssa.BlockExit) + hi := s.newValue1(ssa.OpInt64Hi, types.Types[types.TUINT32], idx) + cmp := s.newValue2(ssa.OpEq32, types.Types[types.TBOOL], hi, s.constInt32(types.Types[types.TUINT32], 0)) + if !idx.Type.IsSigned() { + switch kind { + case ssa.BoundsIndex: + kind = ssa.BoundsIndexU + case ssa.BoundsSliceAlen: + kind = ssa.BoundsSliceAlenU + case ssa.BoundsSliceAcap: + kind = ssa.BoundsSliceAcapU + case ssa.BoundsSliceB: + kind = ssa.BoundsSliceBU + case ssa.BoundsSlice3Alen: + kind = ssa.BoundsSlice3AlenU + case ssa.BoundsSlice3Acap: + kind = ssa.BoundsSlice3AcapU + case ssa.BoundsSlice3B: + kind = ssa.BoundsSlice3BU + case ssa.BoundsSlice3C: + kind = ssa.BoundsSlice3CU + } + } + b := s.endBlock() + b.Kind = ssa.BlockIf + b.SetControl(cmp) + b.Likely = ssa.BranchLikely + b.AddEdgeTo(bNext) + b.AddEdgeTo(bPanic) + + s.startBlock(bPanic) + mem := s.newValue4I(ssa.OpPanicExtend, types.TypeMem, int64(kind), hi, lo, len, s.mem()) + s.endBlock().SetControl(mem) + s.startBlock(bNext) + + return lo + } + + // Extend value to the required size + var op ssa.Op + if idx.Type.IsSigned() { + switch 10*size + s.config.PtrSize { + case 14: + op = ssa.OpSignExt8to32 + case 18: + op = ssa.OpSignExt8to64 + case 24: + op = ssa.OpSignExt16to32 + case 28: + op = ssa.OpSignExt16to64 + case 48: + op = ssa.OpSignExt32to64 + default: + s.Fatalf("bad signed index extension %s", idx.Type) + } + } else { + switch 10*size + s.config.PtrSize { + case 14: + op = ssa.OpZeroExt8to32 + case 18: + op = ssa.OpZeroExt8to64 + case 24: + op = ssa.OpZeroExt16to32 + case 28: + op = ssa.OpZeroExt16to64 + case 48: + op = ssa.OpZeroExt32to64 + default: + s.Fatalf("bad unsigned index extension %s", idx.Type) + } + } + return s.newValue1(op, types.Types[types.TINT], idx) +} + +// CheckLoweredPhi checks that regalloc and stackalloc correctly handled phi values. +// Called during ssaGenValue. +func CheckLoweredPhi(v *ssa.Value) { + if v.Op != ssa.OpPhi { + v.Fatalf("CheckLoweredPhi called with non-phi value: %v", v.LongString()) + } + if v.Type.IsMemory() { + return + } + f := v.Block.Func + loc := f.RegAlloc[v.ID] + for _, a := range v.Args { + if aloc := f.RegAlloc[a.ID]; aloc != loc { // TODO: .Equal() instead? + v.Fatalf("phi arg at different location than phi: %v @ %s, but arg %v @ %s\n%s\n", v, loc, a, aloc, v.Block.Func) + } + } +} + +// CheckLoweredGetClosurePtr checks that v is the first instruction in the function's entry block. +// The output of LoweredGetClosurePtr is generally hardwired to the correct register. +// That register contains the closure pointer on closure entry. +func CheckLoweredGetClosurePtr(v *ssa.Value) { + entry := v.Block.Func.Entry + if entry != v.Block || entry.Values[0] != v { + base.Fatalf("in %s, badly placed LoweredGetClosurePtr: %v %v", v.Block.Func.Name, v.Block, v) + } +} + +func AddrAuto(a *obj.Addr, v *ssa.Value) { + n, off := ssa.AutoVar(v) + a.Type = obj.TYPE_MEM + a.Sym = n.Sym().Linksym() + a.Reg = int16(Arch.REGSP) + a.Offset = n.FrameOffset() + off + if n.Class_ == ir.PPARAM || n.Class_ == ir.PPARAMOUT { + a.Name = obj.NAME_PARAM + } else { + a.Name = obj.NAME_AUTO + } +} + +func (s *State) AddrScratch(a *obj.Addr) { + if s.ScratchFpMem == nil { + panic("no scratch memory available; forgot to declare usesScratch for Op?") + } + a.Type = obj.TYPE_MEM + a.Name = obj.NAME_AUTO + a.Sym = s.ScratchFpMem.Sym().Linksym() + a.Reg = int16(Arch.REGSP) + a.Offset = s.ScratchFpMem.Offset_ +} + +// Call returns a new CALL instruction for the SSA value v. +// It uses PrepareCall to prepare the call. +func (s *State) Call(v *ssa.Value) *obj.Prog { + pPosIsStmt := s.pp.Pos.IsStmt() // The statement-ness fo the call comes from ssaGenState + s.PrepareCall(v) + + p := s.Prog(obj.ACALL) + if pPosIsStmt == src.PosIsStmt { + p.Pos = v.Pos.WithIsStmt() + } else { + p.Pos = v.Pos.WithNotStmt() + } + if sym, ok := v.Aux.(*ssa.AuxCall); ok && sym.Fn != nil { + p.To.Type = obj.TYPE_MEM + p.To.Name = obj.NAME_EXTERN + p.To.Sym = sym.Fn + } else { + // TODO(mdempsky): Can these differences be eliminated? + switch Arch.LinkArch.Family { + case sys.AMD64, sys.I386, sys.PPC64, sys.RISCV64, sys.S390X, sys.Wasm: + p.To.Type = obj.TYPE_REG + case sys.ARM, sys.ARM64, sys.MIPS, sys.MIPS64: + p.To.Type = obj.TYPE_MEM + default: + base.Fatalf("unknown indirect call family") + } + p.To.Reg = v.Args[0].Reg() + } + return p +} + +// PrepareCall prepares to emit a CALL instruction for v and does call-related bookkeeping. +// It must be called immediately before emitting the actual CALL instruction, +// since it emits PCDATA for the stack map at the call (calls are safe points). +func (s *State) PrepareCall(v *ssa.Value) { + idx := s.livenessMap.Get(v) + if !idx.StackMapValid() { + // See Liveness.hasStackMap. + if sym, ok := v.Aux.(*ssa.AuxCall); !ok || !(sym.Fn == ir.Syms.Typedmemclr || sym.Fn == ir.Syms.Typedmemmove) { + base.Fatalf("missing stack map index for %v", v.LongString()) + } + } + + call, ok := v.Aux.(*ssa.AuxCall) + + if ok && call.Fn == ir.Syms.Deferreturn { + // Deferred calls will appear to be returning to + // the CALL deferreturn(SB) that we are about to emit. + // However, the stack trace code will show the line + // of the instruction byte before the return PC. + // To avoid that being an unrelated instruction, + // insert an actual hardware NOP that will have the right line number. + // This is different from obj.ANOP, which is a virtual no-op + // that doesn't make it into the instruction stream. + Arch.Ginsnopdefer(s.pp) + } + + if ok { + // Record call graph information for nowritebarrierrec + // analysis. + if nowritebarrierrecCheck != nil { + nowritebarrierrecCheck.recordCall(s.pp.CurFunc, call.Fn, v.Pos) + } + } + + if s.maxarg < v.AuxInt { + s.maxarg = v.AuxInt + } +} + +// UseArgs records the fact that an instruction needs a certain amount of +// callee args space for its use. +func (s *State) UseArgs(n int64) { + if s.maxarg < n { + s.maxarg = n + } +} + +// fieldIdx finds the index of the field referred to by the ODOT node n. +func fieldIdx(n *ir.SelectorExpr) int { + t := n.X.Type() + f := n.Sel + if !t.IsStruct() { + panic("ODOT's LHS is not a struct") + } + + var i int + for _, t1 := range t.Fields().Slice() { + if t1.Sym != f { + i++ + continue + } + if t1.Offset != n.Offset { + panic("field offset doesn't match") + } + return i + } + panic(fmt.Sprintf("can't find field in expr %v\n", n)) + + // TODO: keep the result of this function somewhere in the ODOT Node + // so we don't have to recompute it each time we need it. +} + +// ssafn holds frontend information about a function that the backend is processing. +// It also exports a bunch of compiler services for the ssa backend. +type ssafn struct { + curfn *ir.Func + strings map[string]*obj.LSym // map from constant string to data symbols + scratchFpMem *ir.Name // temp for floating point register / memory moves on some architectures + stksize int64 // stack size for current frame + stkptrsize int64 // prefix of stack containing pointers + log bool // print ssa debug to the stdout +} + +// StringData returns a symbol which +// is the data component of a global string constant containing s. +func (e *ssafn) StringData(s string) *obj.LSym { + if aux, ok := e.strings[s]; ok { + return aux + } + if e.strings == nil { + e.strings = make(map[string]*obj.LSym) + } + data := staticdata.StringSym(e.curfn.Pos(), s) + e.strings[s] = data + return data +} + +func (e *ssafn) Auto(pos src.XPos, t *types.Type) *ir.Name { + return typecheck.TempAt(pos, e.curfn, t) // Note: adds new auto to e.curfn.Func.Dcl list +} + +func (e *ssafn) SplitString(name ssa.LocalSlot) (ssa.LocalSlot, ssa.LocalSlot) { + ptrType := types.NewPtr(types.Types[types.TUINT8]) + lenType := types.Types[types.TINT] + // Split this string up into two separate variables. + p := e.SplitSlot(&name, ".ptr", 0, ptrType) + l := e.SplitSlot(&name, ".len", ptrType.Size(), lenType) + return p, l +} + +func (e *ssafn) SplitInterface(name ssa.LocalSlot) (ssa.LocalSlot, ssa.LocalSlot) { + n := name.N + u := types.Types[types.TUINTPTR] + t := types.NewPtr(types.Types[types.TUINT8]) + // Split this interface up into two separate variables. + f := ".itab" + if n.Type().IsEmptyInterface() { + f = ".type" + } + c := e.SplitSlot(&name, f, 0, u) // see comment in plive.go:onebitwalktype1. + d := e.SplitSlot(&name, ".data", u.Size(), t) + return c, d +} + +func (e *ssafn) SplitSlice(name ssa.LocalSlot) (ssa.LocalSlot, ssa.LocalSlot, ssa.LocalSlot) { + ptrType := types.NewPtr(name.Type.Elem()) + lenType := types.Types[types.TINT] + p := e.SplitSlot(&name, ".ptr", 0, ptrType) + l := e.SplitSlot(&name, ".len", ptrType.Size(), lenType) + c := e.SplitSlot(&name, ".cap", ptrType.Size()+lenType.Size(), lenType) + return p, l, c +} + +func (e *ssafn) SplitComplex(name ssa.LocalSlot) (ssa.LocalSlot, ssa.LocalSlot) { + s := name.Type.Size() / 2 + var t *types.Type + if s == 8 { + t = types.Types[types.TFLOAT64] + } else { + t = types.Types[types.TFLOAT32] + } + r := e.SplitSlot(&name, ".real", 0, t) + i := e.SplitSlot(&name, ".imag", t.Size(), t) + return r, i +} + +func (e *ssafn) SplitInt64(name ssa.LocalSlot) (ssa.LocalSlot, ssa.LocalSlot) { + var t *types.Type + if name.Type.IsSigned() { + t = types.Types[types.TINT32] + } else { + t = types.Types[types.TUINT32] + } + if Arch.LinkArch.ByteOrder == binary.BigEndian { + return e.SplitSlot(&name, ".hi", 0, t), e.SplitSlot(&name, ".lo", t.Size(), types.Types[types.TUINT32]) + } + return e.SplitSlot(&name, ".hi", t.Size(), t), e.SplitSlot(&name, ".lo", 0, types.Types[types.TUINT32]) +} + +func (e *ssafn) SplitStruct(name ssa.LocalSlot, i int) ssa.LocalSlot { + st := name.Type + // Note: the _ field may appear several times. But + // have no fear, identically-named but distinct Autos are + // ok, albeit maybe confusing for a debugger. + return e.SplitSlot(&name, "."+st.FieldName(i), st.FieldOff(i), st.FieldType(i)) +} + +func (e *ssafn) SplitArray(name ssa.LocalSlot) ssa.LocalSlot { + n := name.N + at := name.Type + if at.NumElem() != 1 { + e.Fatalf(n.Pos(), "bad array size") + } + et := at.Elem() + return e.SplitSlot(&name, "[0]", 0, et) +} + +func (e *ssafn) DerefItab(it *obj.LSym, offset int64) *obj.LSym { + return reflectdata.ITabSym(it, offset) +} + +// SplitSlot returns a slot representing the data of parent starting at offset. +func (e *ssafn) SplitSlot(parent *ssa.LocalSlot, suffix string, offset int64, t *types.Type) ssa.LocalSlot { + node := parent.N + + if node.Class_ != ir.PAUTO || node.Name().Addrtaken() { + // addressed things and non-autos retain their parents (i.e., cannot truly be split) + return ssa.LocalSlot{N: node, Type: t, Off: parent.Off + offset} + } + + s := &types.Sym{Name: node.Sym().Name + suffix, Pkg: types.LocalPkg} + n := ir.NewNameAt(parent.N.Pos(), s) + s.Def = n + ir.AsNode(s.Def).Name().SetUsed(true) + n.SetType(t) + n.Class_ = ir.PAUTO + n.SetEsc(ir.EscNever) + n.Curfn = e.curfn + e.curfn.Dcl = append(e.curfn.Dcl, n) + types.CalcSize(t) + return ssa.LocalSlot{N: n, Type: t, Off: 0, SplitOf: parent, SplitOffset: offset} +} + +func (e *ssafn) CanSSA(t *types.Type) bool { + return TypeOK(t) +} + +func (e *ssafn) Line(pos src.XPos) string { + return base.FmtPos(pos) +} + +// Log logs a message from the compiler. +func (e *ssafn) Logf(msg string, args ...interface{}) { + if e.log { + fmt.Printf(msg, args...) + } +} + +func (e *ssafn) Log() bool { + return e.log +} + +// Fatal reports a compiler error and exits. +func (e *ssafn) Fatalf(pos src.XPos, msg string, args ...interface{}) { + base.Pos = pos + nargs := append([]interface{}{ir.FuncName(e.curfn)}, args...) + base.Fatalf("'%s': "+msg, nargs...) +} + +// Warnl reports a "warning", which is usually flag-triggered +// logging output for the benefit of tests. +func (e *ssafn) Warnl(pos src.XPos, fmt_ string, args ...interface{}) { + base.WarnfAt(pos, fmt_, args...) +} + +func (e *ssafn) Debug_checknil() bool { + return base.Debug.Nil != 0 +} + +func (e *ssafn) UseWriteBarrier() bool { + return base.Flag.WB +} + +func (e *ssafn) Syslook(name string) *obj.LSym { + switch name { + case "goschedguarded": + return ir.Syms.Goschedguarded + case "writeBarrier": + return ir.Syms.WriteBarrier + case "gcWriteBarrier": + return ir.Syms.GCWriteBarrier + case "typedmemmove": + return ir.Syms.Typedmemmove + case "typedmemclr": + return ir.Syms.Typedmemclr + } + e.Fatalf(src.NoXPos, "unknown Syslook func %v", name) + return nil +} + +func (e *ssafn) SetWBPos(pos src.XPos) { + e.curfn.SetWBPos(pos) +} + +func (e *ssafn) MyImportPath() string { + return base.Ctxt.Pkgpath +} + +func clobberBase(n ir.Node) ir.Node { + if n.Op() == ir.ODOT { + n := n.(*ir.SelectorExpr) + if n.X.Type().NumFields() == 1 { + return clobberBase(n.X) + } + } + if n.Op() == ir.OINDEX { + n := n.(*ir.IndexExpr) + if n.X.Type().IsArray() && n.X.Type().NumElem() == 1 { + return clobberBase(n.X) + } + } + return n +} + +// callTargetLSym determines the correct LSym for 'callee' when called +// from function 'caller'. There are a couple of different scenarios +// to contend with here: +// +// 1. if 'caller' is an ABI wrapper, then we always want to use the +// LSym from the Func for the callee. +// +// 2. if 'caller' is not an ABI wrapper, then we looked at the callee +// to see if it corresponds to a "known" ABI0 symbol (e.g. assembly +// routine defined in the current package); if so, we want the call to +// directly target the ABI0 symbol (effectively bypassing the +// ABIInternal->ABI0 wrapper for 'callee'). +// +// 3. in all other cases, want the regular ABIInternal linksym +// +func callTargetLSym(callee *types.Sym, callerLSym *obj.LSym) *obj.LSym { + lsym := callee.Linksym() + if !base.Flag.ABIWrap { + return lsym + } + if ir.AsNode(callee.Def) == nil { + return lsym + } + defn := ir.AsNode(callee.Def).Name().Defn + if defn == nil { + return lsym + } + ndclfunc := defn.(*ir.Func) + + // check for case 1 above + if callerLSym.ABIWrapper() { + if nlsym := ndclfunc.LSym; nlsym != nil { + lsym = nlsym + } + } else { + // check for case 2 above + nam := ndclfunc.Nname + defABI, hasDefABI := symabiDefs[nam.Sym().LinksymName()] + if hasDefABI && defABI == obj.ABI0 { + lsym = nam.Sym().LinksymABI0() + } + } + return lsym +} + +func min8(a, b int8) int8 { + if a < b { + return a + } + return b +} + +func max8(a, b int8) int8 { + if a > b { + return a + } + return b +} + +// deferstruct makes a runtime._defer structure, with additional space for +// stksize bytes of args. +func deferstruct(stksize int64) *types.Type { + makefield := func(name string, typ *types.Type) *types.Field { + // Unlike the global makefield function, this one needs to set Pkg + // because these types might be compared (in SSA CSE sorting). + // TODO: unify this makefield and the global one above. + sym := &types.Sym{Name: name, Pkg: types.LocalPkg} + return types.NewField(src.NoXPos, sym, typ) + } + argtype := types.NewArray(types.Types[types.TUINT8], stksize) + argtype.Width = stksize + argtype.Align = 1 + // These fields must match the ones in runtime/runtime2.go:_defer and + // cmd/compile/internal/gc/ssa.go:(*state).call. + fields := []*types.Field{ + makefield("siz", types.Types[types.TUINT32]), + makefield("started", types.Types[types.TBOOL]), + makefield("heap", types.Types[types.TBOOL]), + makefield("openDefer", types.Types[types.TBOOL]), + makefield("sp", types.Types[types.TUINTPTR]), + makefield("pc", types.Types[types.TUINTPTR]), + // Note: the types here don't really matter. Defer structures + // are always scanned explicitly during stack copying and GC, + // so we make them uintptr type even though they are real pointers. + makefield("fn", types.Types[types.TUINTPTR]), + makefield("_panic", types.Types[types.TUINTPTR]), + makefield("link", types.Types[types.TUINTPTR]), + makefield("framepc", types.Types[types.TUINTPTR]), + makefield("varp", types.Types[types.TUINTPTR]), + makefield("fd", types.Types[types.TUINTPTR]), + makefield("args", argtype), + } + + // build struct holding the above fields + s := types.NewStruct(types.NoPkg, fields) + s.SetNoalg(true) + types.CalcStructSize(s) + return s +} + +var ( + BoundsCheckFunc [ssa.BoundsKindCount]*obj.LSym + ExtendCheckFunc [ssa.BoundsKindCount]*obj.LSym +) + +// GCWriteBarrierReg maps from registers to gcWriteBarrier implementation LSyms. +var GCWriteBarrierReg map[int16]*obj.LSym diff --git a/src/cmd/compile/internal/wasm/ssa.go b/src/cmd/compile/internal/wasm/ssa.go index ee86fc62d2..e4ef9d7c6a 100644 --- a/src/cmd/compile/internal/wasm/ssa.go +++ b/src/cmd/compile/internal/wasm/ssa.go @@ -6,18 +6,18 @@ package wasm import ( "cmd/compile/internal/base" - "cmd/compile/internal/gc" "cmd/compile/internal/ir" "cmd/compile/internal/logopt" "cmd/compile/internal/objw" "cmd/compile/internal/ssa" + "cmd/compile/internal/ssagen" "cmd/compile/internal/types" "cmd/internal/obj" "cmd/internal/obj/wasm" "cmd/internal/objabi" ) -func Init(arch *gc.Arch) { +func Init(arch *ssagen.ArchInfo) { arch.LinkArch = &wasm.Linkwasm arch.REGSP = wasm.REG_SP arch.MAXWIDTH = 1 << 50 @@ -52,10 +52,10 @@ func ginsnop(pp *objw.Progs) *obj.Prog { return pp.Prog(wasm.ANop) } -func ssaMarkMoves(s *gc.SSAGenState, b *ssa.Block) { +func ssaMarkMoves(s *ssagen.State, b *ssa.Block) { } -func ssaGenBlock(s *gc.SSAGenState, b, next *ssa.Block) { +func ssaGenBlock(s *ssagen.State, b, next *ssa.Block) { switch b.Kind { case ssa.BlockPlain: if next != b.Succs[0].Block() { @@ -121,7 +121,7 @@ func ssaGenBlock(s *gc.SSAGenState, b, next *ssa.Block) { } } -func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { +func ssaGenValue(s *ssagen.State, v *ssa.Value) { switch v.Op { case ssa.OpWasmLoweredStaticCall, ssa.OpWasmLoweredClosureCall, ssa.OpWasmLoweredInterCall: s.PrepareCall(v) @@ -188,7 +188,7 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { getReg(s, wasm.REG_SP) getValue64(s, v.Args[0]) p := s.Prog(storeOp(v.Type)) - gc.AddrAuto(&p.To, v) + ssagen.AddrAuto(&p.To, v) default: if v.Type.IsMemory() { @@ -208,7 +208,7 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { } } -func ssaGenValueOnStack(s *gc.SSAGenState, v *ssa.Value, extend bool) { +func ssaGenValueOnStack(s *ssagen.State, v *ssa.Value, extend bool) { switch v.Op { case ssa.OpWasmLoweredGetClosurePtr: getReg(s, wasm.REG_CTXT) @@ -243,10 +243,10 @@ func ssaGenValueOnStack(s *gc.SSAGenState, v *ssa.Value, extend bool) { p.From.Type = obj.TYPE_ADDR switch v.Aux.(type) { case *obj.LSym: - gc.AddAux(&p.From, v) + ssagen.AddAux(&p.From, v) case *ir.Name: p.From.Reg = v.Args[0].Reg() - gc.AddAux(&p.From, v) + ssagen.AddAux(&p.From, v) default: panic("wasm: bad LoweredAddr") } @@ -363,7 +363,7 @@ func ssaGenValueOnStack(s *gc.SSAGenState, v *ssa.Value, extend bool) { case ssa.OpLoadReg: p := s.Prog(loadOp(v.Type)) - gc.AddrAuto(&p.From, v.Args[0]) + ssagen.AddrAuto(&p.From, v.Args[0]) case ssa.OpCopy: getValue64(s, v.Args[0]) @@ -385,7 +385,7 @@ func isCmp(v *ssa.Value) bool { } } -func getValue32(s *gc.SSAGenState, v *ssa.Value) { +func getValue32(s *ssagen.State, v *ssa.Value) { if v.OnWasmStack { s.OnWasmStackSkipped-- ssaGenValueOnStack(s, v, false) @@ -402,7 +402,7 @@ func getValue32(s *gc.SSAGenState, v *ssa.Value) { } } -func getValue64(s *gc.SSAGenState, v *ssa.Value) { +func getValue64(s *ssagen.State, v *ssa.Value) { if v.OnWasmStack { s.OnWasmStackSkipped-- ssaGenValueOnStack(s, v, true) @@ -416,32 +416,32 @@ func getValue64(s *gc.SSAGenState, v *ssa.Value) { } } -func i32Const(s *gc.SSAGenState, val int32) { +func i32Const(s *ssagen.State, val int32) { p := s.Prog(wasm.AI32Const) p.From = obj.Addr{Type: obj.TYPE_CONST, Offset: int64(val)} } -func i64Const(s *gc.SSAGenState, val int64) { +func i64Const(s *ssagen.State, val int64) { p := s.Prog(wasm.AI64Const) p.From = obj.Addr{Type: obj.TYPE_CONST, Offset: val} } -func f32Const(s *gc.SSAGenState, val float64) { +func f32Const(s *ssagen.State, val float64) { p := s.Prog(wasm.AF32Const) p.From = obj.Addr{Type: obj.TYPE_FCONST, Val: val} } -func f64Const(s *gc.SSAGenState, val float64) { +func f64Const(s *ssagen.State, val float64) { p := s.Prog(wasm.AF64Const) p.From = obj.Addr{Type: obj.TYPE_FCONST, Val: val} } -func getReg(s *gc.SSAGenState, reg int16) { +func getReg(s *ssagen.State, reg int16) { p := s.Prog(wasm.AGet) p.From = obj.Addr{Type: obj.TYPE_REG, Reg: reg} } -func setReg(s *gc.SSAGenState, reg int16) { +func setReg(s *ssagen.State, reg int16) { p := s.Prog(wasm.ASet) p.To = obj.Addr{Type: obj.TYPE_REG, Reg: reg} } diff --git a/src/cmd/compile/internal/x86/galign.go b/src/cmd/compile/internal/x86/galign.go index 7d628f9b7c..fc806f9119 100644 --- a/src/cmd/compile/internal/x86/galign.go +++ b/src/cmd/compile/internal/x86/galign.go @@ -6,14 +6,14 @@ package x86 import ( "cmd/compile/internal/base" - "cmd/compile/internal/gc" + "cmd/compile/internal/ssagen" "cmd/internal/obj/x86" "cmd/internal/objabi" "fmt" "os" ) -func Init(arch *gc.Arch) { +func Init(arch *ssagen.ArchInfo) { arch.LinkArch = &x86.Link386 arch.REGSP = x86.REGSP arch.SSAGenValue = ssaGenValue diff --git a/src/cmd/compile/internal/x86/ssa.go b/src/cmd/compile/internal/x86/ssa.go index d3d60591cc..00dfa07bf7 100644 --- a/src/cmd/compile/internal/x86/ssa.go +++ b/src/cmd/compile/internal/x86/ssa.go @@ -9,17 +9,17 @@ import ( "math" "cmd/compile/internal/base" - "cmd/compile/internal/gc" "cmd/compile/internal/ir" "cmd/compile/internal/logopt" "cmd/compile/internal/ssa" + "cmd/compile/internal/ssagen" "cmd/compile/internal/types" "cmd/internal/obj" "cmd/internal/obj/x86" ) // markMoves marks any MOVXconst ops that need to avoid clobbering flags. -func ssaMarkMoves(s *gc.SSAGenState, b *ssa.Block) { +func ssaMarkMoves(s *ssagen.State, b *ssa.Block) { flive := b.FlagsLiveAtEnd for _, c := range b.ControlValues() { flive = c.Type.IsFlags() || flive @@ -109,7 +109,7 @@ func moveByType(t *types.Type) obj.As { // dest := dest(To) op src(From) // and also returns the created obj.Prog so it // may be further adjusted (offset, scale, etc). -func opregreg(s *gc.SSAGenState, op obj.As, dest, src int16) *obj.Prog { +func opregreg(s *ssagen.State, op obj.As, dest, src int16) *obj.Prog { p := s.Prog(op) p.From.Type = obj.TYPE_REG p.To.Type = obj.TYPE_REG @@ -118,7 +118,7 @@ func opregreg(s *gc.SSAGenState, op obj.As, dest, src int16) *obj.Prog { return p } -func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { +func ssaGenValue(s *ssagen.State, v *ssa.Value) { switch v.Op { case ssa.Op386ADDL: r := v.Reg() @@ -406,14 +406,14 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { p.From.Type = obj.TYPE_MEM p.From.Reg = r p.From.Index = i - gc.AddAux(&p.From, v) + ssagen.AddAux(&p.From, v) p.To.Type = obj.TYPE_REG p.To.Reg = v.Reg() case ssa.Op386LEAL: p := s.Prog(x86.ALEAL) p.From.Type = obj.TYPE_MEM p.From.Reg = v.Args[0].Reg() - gc.AddAux(&p.From, v) + ssagen.AddAux(&p.From, v) p.To.Type = obj.TYPE_REG p.To.Reg = v.Reg() case ssa.Op386CMPL, ssa.Op386CMPW, ssa.Op386CMPB, @@ -439,7 +439,7 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { p := s.Prog(v.Op.Asm()) p.From.Type = obj.TYPE_MEM p.From.Reg = v.Args[0].Reg() - gc.AddAux(&p.From, v) + ssagen.AddAux(&p.From, v) p.To.Type = obj.TYPE_REG p.To.Reg = v.Args[1].Reg() case ssa.Op386CMPLconstload, ssa.Op386CMPWconstload, ssa.Op386CMPBconstload: @@ -447,7 +447,7 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { p := s.Prog(v.Op.Asm()) p.From.Type = obj.TYPE_MEM p.From.Reg = v.Args[0].Reg() - gc.AddAux2(&p.From, v, sc.Off()) + ssagen.AddAux2(&p.From, v, sc.Off()) p.To.Type = obj.TYPE_CONST p.To.Offset = sc.Val() case ssa.Op386MOVLconst: @@ -499,7 +499,7 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { p := s.Prog(v.Op.Asm()) p.From.Type = obj.TYPE_MEM p.From.Reg = v.Args[0].Reg() - gc.AddAux(&p.From, v) + ssagen.AddAux(&p.From, v) p.To.Type = obj.TYPE_REG p.To.Reg = v.Reg() case ssa.Op386MOVBloadidx1, ssa.Op386MOVWloadidx1, ssa.Op386MOVLloadidx1, ssa.Op386MOVSSloadidx1, ssa.Op386MOVSDloadidx1, @@ -523,7 +523,7 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { } p.From.Reg = r p.From.Index = i - gc.AddAux(&p.From, v) + ssagen.AddAux(&p.From, v) p.To.Type = obj.TYPE_REG p.To.Reg = v.Reg() case ssa.Op386ADDLloadidx4, ssa.Op386SUBLloadidx4, ssa.Op386MULLloadidx4, @@ -533,7 +533,7 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { p.From.Reg = v.Args[1].Reg() p.From.Index = v.Args[2].Reg() p.From.Scale = 4 - gc.AddAux(&p.From, v) + ssagen.AddAux(&p.From, v) p.To.Type = obj.TYPE_REG p.To.Reg = v.Reg() if v.Reg() != v.Args[0].Reg() { @@ -546,7 +546,7 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { p := s.Prog(v.Op.Asm()) p.From.Type = obj.TYPE_MEM p.From.Reg = v.Args[1].Reg() - gc.AddAux(&p.From, v) + ssagen.AddAux(&p.From, v) p.To.Type = obj.TYPE_REG p.To.Reg = v.Reg() if v.Reg() != v.Args[0].Reg() { @@ -559,7 +559,7 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { p.From.Reg = v.Args[1].Reg() p.To.Type = obj.TYPE_MEM p.To.Reg = v.Args[0].Reg() - gc.AddAux(&p.To, v) + ssagen.AddAux(&p.To, v) case ssa.Op386ADDLconstmodify: sc := v.AuxValAndOff() val := sc.Val() @@ -573,7 +573,7 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { off := sc.Off() p.To.Type = obj.TYPE_MEM p.To.Reg = v.Args[0].Reg() - gc.AddAux2(&p.To, v, off) + ssagen.AddAux2(&p.To, v, off) break } fallthrough @@ -586,7 +586,7 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { p.From.Offset = val p.To.Type = obj.TYPE_MEM p.To.Reg = v.Args[0].Reg() - gc.AddAux2(&p.To, v, off) + ssagen.AddAux2(&p.To, v, off) case ssa.Op386MOVBstoreidx1, ssa.Op386MOVWstoreidx1, ssa.Op386MOVLstoreidx1, ssa.Op386MOVSSstoreidx1, ssa.Op386MOVSDstoreidx1, ssa.Op386MOVSDstoreidx8, ssa.Op386MOVSSstoreidx4, ssa.Op386MOVLstoreidx4, ssa.Op386MOVWstoreidx2, ssa.Op386ADDLmodifyidx4, ssa.Op386SUBLmodifyidx4, ssa.Op386ANDLmodifyidx4, ssa.Op386ORLmodifyidx4, ssa.Op386XORLmodifyidx4: @@ -612,7 +612,7 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { } p.To.Reg = r p.To.Index = i - gc.AddAux(&p.To, v) + ssagen.AddAux(&p.To, v) case ssa.Op386MOVLstoreconst, ssa.Op386MOVWstoreconst, ssa.Op386MOVBstoreconst: p := s.Prog(v.Op.Asm()) p.From.Type = obj.TYPE_CONST @@ -620,7 +620,7 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { p.From.Offset = sc.Val() p.To.Type = obj.TYPE_MEM p.To.Reg = v.Args[0].Reg() - gc.AddAux2(&p.To, v, sc.Off()) + ssagen.AddAux2(&p.To, v, sc.Off()) case ssa.Op386ADDLconstmodifyidx4: sc := v.AuxValAndOff() val := sc.Val() @@ -636,7 +636,7 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { p.To.Reg = v.Args[0].Reg() p.To.Scale = 4 p.To.Index = v.Args[1].Reg() - gc.AddAux2(&p.To, v, off) + ssagen.AddAux2(&p.To, v, off) break } fallthrough @@ -663,7 +663,7 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { p.To.Type = obj.TYPE_MEM p.To.Reg = r p.To.Index = i - gc.AddAux2(&p.To, v, sc.Off()) + ssagen.AddAux2(&p.To, v, sc.Off()) case ssa.Op386MOVWLSX, ssa.Op386MOVBLSX, ssa.Op386MOVWLZX, ssa.Op386MOVBLZX, ssa.Op386CVTSL2SS, ssa.Op386CVTSL2SD, ssa.Op386CVTTSS2SL, ssa.Op386CVTTSD2SL, @@ -695,7 +695,7 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { return } p := s.Prog(loadByType(v.Type)) - gc.AddrAuto(&p.From, v.Args[0]) + ssagen.AddrAuto(&p.From, v.Args[0]) p.To.Type = obj.TYPE_REG p.To.Reg = v.Reg() @@ -707,10 +707,10 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { p := s.Prog(storeByType(v.Type)) p.From.Type = obj.TYPE_REG p.From.Reg = v.Args[0].Reg() - gc.AddrAuto(&p.To, v) + ssagen.AddrAuto(&p.To, v) case ssa.Op386LoweredGetClosurePtr: // Closure pointer is DX. - gc.CheckLoweredGetClosurePtr(v) + ssagen.CheckLoweredGetClosurePtr(v) case ssa.Op386LoweredGetG: r := v.Reg() // See the comments in cmd/internal/obj/x86/obj6.go @@ -766,14 +766,14 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { p := s.Prog(obj.ACALL) p.To.Type = obj.TYPE_MEM p.To.Name = obj.NAME_EXTERN - p.To.Sym = gc.BoundsCheckFunc[v.AuxInt] + p.To.Sym = ssagen.BoundsCheckFunc[v.AuxInt] s.UseArgs(8) // space used in callee args area by assembly stubs case ssa.Op386LoweredPanicExtendA, ssa.Op386LoweredPanicExtendB, ssa.Op386LoweredPanicExtendC: p := s.Prog(obj.ACALL) p.To.Type = obj.TYPE_MEM p.To.Name = obj.NAME_EXTERN - p.To.Sym = gc.ExtendCheckFunc[v.AuxInt] + p.To.Sym = ssagen.ExtendCheckFunc[v.AuxInt] s.UseArgs(12) // space used in callee args area by assembly stubs case ssa.Op386CALLstatic, ssa.Op386CALLclosure, ssa.Op386CALLinter: @@ -848,7 +848,7 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { p.From.Reg = x86.REG_AX p.To.Type = obj.TYPE_MEM p.To.Reg = v.Args[0].Reg() - gc.AddAux(&p.To, v) + ssagen.AddAux(&p.To, v) if logopt.Enabled() { logopt.LogOpt(v.Pos, "nilcheck", "genssa", v.Block.Func.Name) } @@ -861,7 +861,7 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { p.From.Offset = 0xdeaddead p.To.Type = obj.TYPE_MEM p.To.Reg = x86.REG_SP - gc.AddAux(&p.To, v) + ssagen.AddAux(&p.To, v) default: v.Fatalf("genValue not implemented: %s", v.LongString()) } @@ -886,22 +886,22 @@ var blockJump = [...]struct { ssa.Block386NAN: {x86.AJPS, x86.AJPC}, } -var eqfJumps = [2][2]gc.IndexJump{ +var eqfJumps = [2][2]ssagen.IndexJump{ {{Jump: x86.AJNE, Index: 1}, {Jump: x86.AJPS, Index: 1}}, // next == b.Succs[0] {{Jump: x86.AJNE, Index: 1}, {Jump: x86.AJPC, Index: 0}}, // next == b.Succs[1] } -var nefJumps = [2][2]gc.IndexJump{ +var nefJumps = [2][2]ssagen.IndexJump{ {{Jump: x86.AJNE, Index: 0}, {Jump: x86.AJPC, Index: 1}}, // next == b.Succs[0] {{Jump: x86.AJNE, Index: 0}, {Jump: x86.AJPS, Index: 0}}, // next == b.Succs[1] } -func ssaGenBlock(s *gc.SSAGenState, b, next *ssa.Block) { +func ssaGenBlock(s *ssagen.State, b, next *ssa.Block) { switch b.Kind { case ssa.BlockPlain: if b.Succs[0].Block() != next { p := s.Prog(obj.AJMP) p.To.Type = obj.TYPE_BRANCH - s.Branches = append(s.Branches, gc.Branch{P: p, B: b.Succs[0].Block()}) + s.Branches = append(s.Branches, ssagen.Branch{P: p, B: b.Succs[0].Block()}) } case ssa.BlockDefer: // defer returns in rax: @@ -914,11 +914,11 @@ func ssaGenBlock(s *gc.SSAGenState, b, next *ssa.Block) { p.To.Reg = x86.REG_AX p = s.Prog(x86.AJNE) p.To.Type = obj.TYPE_BRANCH - s.Branches = append(s.Branches, gc.Branch{P: p, B: b.Succs[1].Block()}) + s.Branches = append(s.Branches, ssagen.Branch{P: p, B: b.Succs[1].Block()}) if b.Succs[0].Block() != next { p := s.Prog(obj.AJMP) p.To.Type = obj.TYPE_BRANCH - s.Branches = append(s.Branches, gc.Branch{P: p, B: b.Succs[0].Block()}) + s.Branches = append(s.Branches, ssagen.Branch{P: p, B: b.Succs[0].Block()}) } case ssa.BlockExit: case ssa.BlockRet: diff --git a/src/cmd/compile/main.go b/src/cmd/compile/main.go index 5a33719d87..cb2f4e8cf4 100644 --- a/src/cmd/compile/main.go +++ b/src/cmd/compile/main.go @@ -15,6 +15,7 @@ import ( "cmd/compile/internal/ppc64" "cmd/compile/internal/riscv64" "cmd/compile/internal/s390x" + "cmd/compile/internal/ssagen" "cmd/compile/internal/wasm" "cmd/compile/internal/x86" "cmd/internal/objabi" @@ -23,7 +24,7 @@ import ( "os" ) -var archInits = map[string]func(*gc.Arch){ +var archInits = map[string]func(*ssagen.ArchInfo){ "386": x86.Init, "amd64": amd64.Init, "arm": arm.Init, -- cgit v1.2.3-54-g00ecf