cmd/compile: implement CMOV on amd64

This builds upon the branchelim pass, activating it for amd64 and
lowering CondSelect. Special care is made to FPU instructions for
NaN handling.

Benchmark results on Xeon E5630 (Westmere EP):

name                      old time/op    new time/op    delta
BinaryTree17-16              4.99s ± 9%     4.66s ± 2%     ~     (p=0.095 n=5+5)
Fannkuch11-16                4.93s ± 3%     5.04s ± 2%     ~     (p=0.548 n=5+5)
FmtFprintfEmpty-16          58.8ns ± 7%    61.4ns ±14%     ~     (p=0.579 n=5+5)
FmtFprintfString-16          114ns ± 2%     114ns ± 4%     ~     (p=0.603 n=5+5)
FmtFprintfInt-16             181ns ± 4%     125ns ± 3%  -30.90%  (p=0.008 n=5+5)
FmtFprintfIntInt-16          263ns ± 2%     217ns ± 2%  -17.34%  (p=0.008 n=5+5)
FmtFprintfPrefixedInt-16     230ns ± 1%     212ns ± 1%   -7.99%  (p=0.008 n=5+5)
FmtFprintfFloat-16           411ns ± 3%     344ns ± 5%  -16.43%  (p=0.008 n=5+5)
FmtManyArgs-16               828ns ± 4%     790ns ± 2%   -4.59%  (p=0.032 n=5+5)
GobDecode-16                10.9ms ± 4%    10.8ms ± 5%     ~     (p=0.548 n=5+5)
GobEncode-16                9.52ms ± 5%    9.46ms ± 2%     ~     (p=1.000 n=5+5)
Gzip-16                      334ms ± 2%     337ms ± 2%     ~     (p=0.548 n=5+5)
Gunzip-16                   64.4ms ± 1%    65.0ms ± 1%   +1.00%  (p=0.008 n=5+5)
HTTPClientServer-16          156µs ± 3%     155µs ± 3%     ~     (p=0.690 n=5+5)
JSONEncode-16               21.0ms ± 1%    21.8ms ± 0%   +3.76%  (p=0.016 n=5+4)
JSONDecode-16               95.1ms ± 0%    95.7ms ± 1%     ~     (p=0.151 n=5+5)
Mandelbrot200-16            6.38ms ± 1%    6.42ms ± 1%     ~     (p=0.095 n=5+5)
GoParse-16                  5.47ms ± 2%    5.36ms ± 1%   -1.95%  (p=0.016 n=5+5)
RegexpMatchEasy0_32-16       111ns ± 1%     111ns ± 1%     ~     (p=0.635 n=5+4)
RegexpMatchEasy0_1K-16       408ns ± 1%     411ns ± 2%     ~     (p=0.087 n=5+5)
RegexpMatchEasy1_32-16       103ns ± 1%     104ns ± 1%     ~     (p=0.484 n=5+5)
RegexpMatchEasy1_1K-16       659ns ± 2%     652ns ± 1%     ~     (p=0.571 n=5+5)
RegexpMatchMedium_32-16      176ns ± 2%     174ns ± 1%     ~     (p=0.476 n=5+5)
RegexpMatchMedium_1K-16     58.6µs ± 4%    57.7µs ± 4%     ~     (p=0.548 n=5+5)
RegexpMatchHard_32-16       3.07µs ± 3%    3.04µs ± 4%     ~     (p=0.421 n=5+5)
RegexpMatchHard_1K-16       89.2µs ± 1%    87.9µs ± 2%   -1.52%  (p=0.032 n=5+5)
Revcomp-16                   575ms ± 0%     587ms ± 2%   +2.12%  (p=0.032 n=4+5)
Template-16                  110ms ± 1%     107ms ± 3%   -3.00%  (p=0.032 n=5+5)
TimeParse-16                 463ns ± 0%     462ns ± 0%     ~     (p=0.810 n=5+4)
TimeFormat-16                538ns ± 0%     535ns ± 0%   -0.63%  (p=0.024 n=5+5)

name                      old speed      new speed      delta
GobDecode-16              70.7MB/s ± 4%  71.4MB/s ± 5%     ~     (p=0.452 n=5+5)
GobEncode-16              80.7MB/s ± 5%  81.2MB/s ± 2%     ~     (p=1.000 n=5+5)
Gzip-16                   58.2MB/s ± 2%  57.7MB/s ± 2%     ~     (p=0.452 n=5+5)
Gunzip-16                  302MB/s ± 1%   299MB/s ± 1%   -0.99%  (p=0.008 n=5+5)
JSONEncode-16             92.4MB/s ± 1%  89.1MB/s ± 0%   -3.63%  (p=0.016 n=5+4)
JSONDecode-16             20.4MB/s ± 0%  20.3MB/s ± 1%     ~     (p=0.135 n=5+5)
GoParse-16                10.6MB/s ± 2%  10.8MB/s ± 1%   +2.00%  (p=0.016 n=5+5)
RegexpMatchEasy0_32-16     286MB/s ± 1%   285MB/s ± 3%     ~     (p=1.000 n=5+5)
RegexpMatchEasy0_1K-16    2.51GB/s ± 1%  2.49GB/s ± 2%     ~     (p=0.095 n=5+5)
RegexpMatchEasy1_32-16     309MB/s ± 1%   307MB/s ± 1%     ~     (p=0.548 n=5+5)
RegexpMatchEasy1_1K-16    1.55GB/s ± 2%  1.57GB/s ± 1%     ~     (p=0.690 n=5+5)
RegexpMatchMedium_32-16   5.68MB/s ± 2%  5.73MB/s ± 1%     ~     (p=0.579 n=5+5)
RegexpMatchMedium_1K-16   17.5MB/s ± 4%  17.8MB/s ± 4%     ~     (p=0.500 n=5+5)
RegexpMatchHard_32-16     10.4MB/s ± 3%  10.5MB/s ± 4%     ~     (p=0.460 n=5+5)
RegexpMatchHard_1K-16     11.5MB/s ± 1%  11.7MB/s ± 2%   +1.57%  (p=0.032 n=5+5)
Revcomp-16                 442MB/s ± 0%   433MB/s ± 2%   -2.05%  (p=0.032 n=4+5)
Template-16               17.7MB/s ± 1%  18.2MB/s ± 3%   +3.12%  (p=0.032 n=5+5)

Change-Id: Ic7cb7374d07da031e771bdcbfdd832fd1b17159c
Reviewed-on: https://go-review.googlesource.com/98695
Reviewed-by: Ilya Tocar <ilya.tocar@intel.com>

7 files changed, 3483 insertions, 116 deletions

diff --git a/src/cmd/compile/internal/ssa/branchelim.go b/src/cmd/compile/internal/ssa/branchelim.go
index 54508985b3..75a6b8238c 100644
--- a/src/cmd/compile/internal/ssa/branchelim.go
+++ b/src/cmd/compile/internal/ssa/branchelim.go
@@ -19,7 +19,10 @@ package ssa
 // rewrite Phis in the postdominator as CondSelects.
 func branchelim(f *Func) {
 	// FIXME: add support for lowering CondSelects on more architectures
-	if f.Config.arch != "arm64" {
+	switch f.Config.arch {
+	case "arm64", "amd64":
+		// implemented
+	default:
 		return
 	}
 
@@ -32,10 +35,22 @@ func branchelim(f *Func) {
 	}
 }
 
-func canCondSelect(v *Value) bool {
+func canCondSelect(v *Value, arch string) bool {
 	// For now, stick to simple scalars that fit in registers
-	sz := v.Type.Size()
-	return sz <= v.Block.Func.Config.RegSize && (v.Type.IsInteger() || v.Type.IsPtrShaped())
+	switch {
+	case v.Type.Size() > v.Block.Func.Config.RegSize:
+		return false
+	case v.Type.IsPtrShaped():
+		return true
+	case v.Type.IsInteger():
+		if arch == "amd64" && v.Type.Size() < 2 {
+			// amd64 doesn't support CMOV with byte registers
+			return false
+		}
+		return true
+	default:
+		return false
+	}
 }
 
 func elimIf(f *Func, dom *Block) bool {
@@ -68,7 +83,7 @@ func elimIf(f *Func, dom *Block) bool {
 	for _, v := range post.Values {
 		if v.Op == OpPhi {
 			hasphis = true
-			if !canCondSelect(v) {
+			if !canCondSelect(v, f.Config.arch) {
 				return false
 			}
 		}
@@ -169,7 +184,7 @@ func elimIfElse(f *Func, b *Block) bool {
 	for _, v := range post.Values {
 		if v.Op == OpPhi {
 			hasphis = true
-			if !canCondSelect(v) {
+			if !canCondSelect(v, f.Config.arch) {
 				return false
 			}
 		}
diff --git a/src/cmd/compile/internal/ssa/branchelim_test.go b/src/cmd/compile/internal/ssa/branchelim_test.go
index 979ba1d961..30bb133f8e 100644
--- a/src/cmd/compile/internal/ssa/branchelim_test.go
+++ b/src/cmd/compile/internal/ssa/branchelim_test.go
@@ -11,128 +11,162 @@ import (
 
 // Test that a trivial 'if' is eliminated
 func TestBranchElimIf(t *testing.T) {
-	c := testConfig(t)
-	c.config.arch = "arm64" // FIXME
-	boolType := types.New(types.TBOOL)
-	intType := types.New(types.TINT32)
-	fun := c.Fun("entry",
-		Bloc("entry",
-			Valu("start", OpInitMem, types.TypeMem, 0, nil),
-			Valu("sb", OpSB, types.TypeInvalid, 0, nil),
-			Valu("const1", OpConst32, intType, 1, nil),
-			Valu("const2", OpConst32, intType, 2, nil),
-			Valu("addr", OpAddr, boolType.PtrTo(), 0, nil, "sb"),
-			Valu("cond", OpLoad, boolType, 0, nil, "addr", "start"),
-			If("cond", "b2", "b3")),
-		Bloc("b2",
-			Goto("b3")),
-		Bloc("b3",
-			Valu("phi", OpPhi, intType, 0, nil, "const1", "const2"),
-			Valu("retstore", OpStore, types.TypeMem, 0, nil, "phi", "sb", "start"),
-			Exit("retstore")))
+	var testData = []struct {
+		arch    string
+		intType string
+		ok      bool
+	}{
+		{"arm64", "int32", true},
+		{"amd64", "int32", true},
+		{"amd64", "int8", false},
+	}
 
-	CheckFunc(fun.f)
-	branchelim(fun.f)
-	CheckFunc(fun.f)
-	Deadcode(fun.f)
-	CheckFunc(fun.f)
+	for _, data := range testData {
+		t.Run(data.arch+"/"+data.intType, func(t *testing.T) {
+			c := testConfigArch(t, data.arch)
+			boolType := c.config.Types.Bool
+			var intType *types.Type
+			switch data.intType {
+			case "int32":
+				intType = c.config.Types.Int32
+			case "int8":
+				intType = c.config.Types.Int8
+			default:
+				t.Fatal("invalid integer type:", data.intType)
+			}
+			fun := c.Fun("entry",
+				Bloc("entry",
+					Valu("start", OpInitMem, types.TypeMem, 0, nil),
+					Valu("sb", OpSB, types.TypeInvalid, 0, nil),
+					Valu("const1", OpConst32, intType, 1, nil),
+					Valu("const2", OpConst32, intType, 2, nil),
+					Valu("addr", OpAddr, boolType.PtrTo(), 0, nil, "sb"),
+					Valu("cond", OpLoad, boolType, 0, nil, "addr", "start"),
+					If("cond", "b2", "b3")),
+				Bloc("b2",
+					Goto("b3")),
+				Bloc("b3",
+					Valu("phi", OpPhi, intType, 0, nil, "const1", "const2"),
+					Valu("retstore", OpStore, types.TypeMem, 0, nil, "phi", "sb", "start"),
+					Exit("retstore")))
 
-	if len(fun.f.Blocks) != 1 {
-		t.Errorf("expected 1 block after branchelim and deadcode; found %d", len(fun.f.Blocks))
-	}
-	if fun.values["phi"].Op != OpCondSelect {
-		t.Errorf("expected phi op to be CondSelect; found op %s", fun.values["phi"].Op)
-	}
-	if fun.values["phi"].Args[2] != fun.values["cond"] {
-		t.Errorf("expected CondSelect condition to be %s; found %s", fun.values["cond"], fun.values["phi"].Args[2])
-	}
-	if fun.blocks["entry"].Kind != BlockExit {
-		t.Errorf("expected entry to be BlockExit; found kind %s", fun.blocks["entry"].Kind.String())
+			CheckFunc(fun.f)
+			branchelim(fun.f)
+			CheckFunc(fun.f)
+			Deadcode(fun.f)
+			CheckFunc(fun.f)
+
+			if data.ok {
+
+				if len(fun.f.Blocks) != 1 {
+					t.Fatalf("expected 1 block after branchelim and deadcode; found %d", len(fun.f.Blocks))
+				}
+				if fun.values["phi"].Op != OpCondSelect {
+					t.Fatalf("expected phi op to be CondSelect; found op %s", fun.values["phi"].Op)
+				}
+				if fun.values["phi"].Args[2] != fun.values["cond"] {
+					t.Errorf("expected CondSelect condition to be %s; found %s", fun.values["cond"], fun.values["phi"].Args[2])
+				}
+				if fun.blocks["entry"].Kind != BlockExit {
+					t.Errorf("expected entry to be BlockExit; found kind %s", fun.blocks["entry"].Kind.String())
+				}
+			} else {
+				if len(fun.f.Blocks) != 3 {
+					t.Fatalf("expected 3 block after branchelim and deadcode; found %d", len(fun.f.Blocks))
+				}
+			}
+		})
 	}
 }
 
 // Test that a trivial if/else is eliminated
 func TestBranchElimIfElse(t *testing.T) {
-	c := testConfig(t)
-	c.config.arch = "arm64" // FIXME
-	boolType := types.New(types.TBOOL)
-	intType := types.New(types.TINT32)
-	fun := c.Fun("entry",
-		Bloc("entry",
-			Valu("start", OpInitMem, types.TypeMem, 0, nil),
-			Valu("sb", OpSB, types.TypeInvalid, 0, nil),
-			Valu("const1", OpConst32, intType, 1, nil),
-			Valu("const2", OpConst32, intType, 2, nil),
-			Valu("addr", OpAddr, boolType.PtrTo(), 0, nil, "sb"),
-			Valu("cond", OpLoad, boolType, 0, nil, "addr", "start"),
-			If("cond", "b2", "b3")),
-		Bloc("b2",
-			Goto("b4")),
-		Bloc("b3",
-			Goto("b4")),
-		Bloc("b4",
-			Valu("phi", OpPhi, intType, 0, nil, "const1", "const2"),
-			Valu("retstore", OpStore, types.TypeMem, 0, nil, "phi", "sb", "start"),
-			Exit("retstore")))
+	for _, arch := range []string{"arm64", "amd64"} {
+		t.Run(arch, func(t *testing.T) {
+			c := testConfigArch(t, arch)
+			boolType := c.config.Types.Bool
+			intType := c.config.Types.Int32
+			fun := c.Fun("entry",
+				Bloc("entry",
+					Valu("start", OpInitMem, types.TypeMem, 0, nil),
+					Valu("sb", OpSB, types.TypeInvalid, 0, nil),
+					Valu("const1", OpConst32, intType, 1, nil),
+					Valu("const2", OpConst32, intType, 2, nil),
+					Valu("addr", OpAddr, boolType.PtrTo(), 0, nil, "sb"),
+					Valu("cond", OpLoad, boolType, 0, nil, "addr", "start"),
+					If("cond", "b2", "b3")),
+				Bloc("b2",
+					Goto("b4")),
+				Bloc("b3",
+					Goto("b4")),
+				Bloc("b4",
+					Valu("phi", OpPhi, intType, 0, nil, "const1", "const2"),
+					Valu("retstore", OpStore, types.TypeMem, 0, nil, "phi", "sb", "start"),
+					Exit("retstore")))
 
-	CheckFunc(fun.f)
-	branchelim(fun.f)
-	CheckFunc(fun.f)
-	Deadcode(fun.f)
-	CheckFunc(fun.f)
+			CheckFunc(fun.f)
+			branchelim(fun.f)
+			CheckFunc(fun.f)
+			Deadcode(fun.f)
+			CheckFunc(fun.f)
 
-	if len(fun.f.Blocks) != 1 {
-		t.Errorf("expected 1 block after branchelim; found %d", len(fun.f.Blocks))
-	}
-	if fun.values["phi"].Op != OpCondSelect {
-		t.Errorf("expected phi op to be CondSelect; found op %s", fun.values["phi"].Op)
-	}
-	if fun.values["phi"].Args[2] != fun.values["cond"] {
-		t.Errorf("expected CondSelect condition to be %s; found %s", fun.values["cond"], fun.values["phi"].Args[2])
-	}
-	if fun.blocks["entry"].Kind != BlockExit {
-		t.Errorf("expected entry to be BlockExit; found kind %s", fun.blocks["entry"].Kind.String())
+			if len(fun.f.Blocks) != 1 {
+				t.Fatalf("expected 1 block after branchelim; found %d", len(fun.f.Blocks))
+			}
+			if fun.values["phi"].Op != OpCondSelect {
+				t.Fatalf("expected phi op to be CondSelect; found op %s", fun.values["phi"].Op)
+			}
+			if fun.values["phi"].Args[2] != fun.values["cond"] {
+				t.Errorf("expected CondSelect condition to be %s; found %s", fun.values["cond"], fun.values["phi"].Args[2])
+			}
+			if fun.blocks["entry"].Kind != BlockExit {
+				t.Errorf("expected entry to be BlockExit; found kind %s", fun.blocks["entry"].Kind.String())
+			}
+		})
 	}
 }
 
 // Test that an if/else CFG that loops back
 // into itself does *not* get eliminated.
 func TestNoBranchElimLoop(t *testing.T) {
-	c := testConfig(t)
-	c.config.arch = "arm64" // FIXME
-	boolType := types.New(types.TBOOL)
-	intType := types.New(types.TINT32)
+	for _, arch := range []string{"arm64", "amd64"} {
+		t.Run(arch, func(t *testing.T) {
+			c := testConfigArch(t, arch)
+			boolType := c.config.Types.Bool
+			intType := c.config.Types.Int32
 
-	// The control flow here is totally bogus,
-	// but a dead cycle seems like the only plausible
-	// way to arrive at a diamond CFG that is also a loop.
-	fun := c.Fun("entry",
-		Bloc("entry",
-			Valu("start", OpInitMem, types.TypeMem, 0, nil),
-			Valu("sb", OpSB, types.TypeInvalid, 0, nil),
-			Valu("const2", OpConst32, intType, 2, nil),
-			Valu("const3", OpConst32, intType, 3, nil),
-			Goto("b5")),
-		Bloc("b2",
-			Valu("addr", OpAddr, boolType.PtrTo(), 0, nil, "sb"),
-			Valu("cond", OpLoad, boolType, 0, nil, "addr", "start"),
-			Valu("phi", OpPhi, intType, 0, nil, "const2", "const3"),
-			If("cond", "b3", "b4")),
-		Bloc("b3",
-			Goto("b2")),
-		Bloc("b4",
-			Goto("b2")),
-		Bloc("b5",
-			Exit("start")))
+			// The control flow here is totally bogus,
+			// but a dead cycle seems like the only plausible
+			// way to arrive at a diamond CFG that is also a loop.
+			fun := c.Fun("entry",
+				Bloc("entry",
+					Valu("start", OpInitMem, types.TypeMem, 0, nil),
+					Valu("sb", OpSB, types.TypeInvalid, 0, nil),
+					Valu("const2", OpConst32, intType, 2, nil),
+					Valu("const3", OpConst32, intType, 3, nil),
+					Goto("b5")),
+				Bloc("b2",
+					Valu("addr", OpAddr, boolType.PtrTo(), 0, nil, "sb"),
+					Valu("cond", OpLoad, boolType, 0, nil, "addr", "start"),
+					Valu("phi", OpPhi, intType, 0, nil, "const2", "const3"),
+					If("cond", "b3", "b4")),
+				Bloc("b3",
+					Goto("b2")),
+				Bloc("b4",
+					Goto("b2")),
+				Bloc("b5",
+					Exit("start")))
 
-	CheckFunc(fun.f)
-	branchelim(fun.f)
-	CheckFunc(fun.f)
+			CheckFunc(fun.f)
+			branchelim(fun.f)
+			CheckFunc(fun.f)
 
-	if len(fun.f.Blocks) != 5 {
-		t.Errorf("expected 5 block after branchelim; found %d", len(fun.f.Blocks))
-	}
-	if fun.values["phi"].Op != OpPhi {
-		t.Errorf("expected phi op to be CondSelect; found op %s", fun.values["phi"].Op)
+			if len(fun.f.Blocks) != 5 {
+				t.Errorf("expected 5 block after branchelim; found %d", len(fun.f.Blocks))
+			}
+			if fun.values["phi"].Op != OpPhi {
+				t.Errorf("expected phi op to be CondSelect; found op %s", fun.values["phi"].Op)
+			}
+		})
 	}
 }
diff --git a/src/cmd/compile/internal/ssa/export_test.go b/src/cmd/compile/internal/ssa/export_test.go
index 1fe0bbe6ae..8d3bd74fa5 100644
--- a/src/cmd/compile/internal/ssa/export_test.go
+++ b/src/cmd/compile/internal/ssa/export_test.go
@@ -7,6 +7,7 @@ package ssa
 import (
 	"cmd/compile/internal/types"
 	"cmd/internal/obj"
+	"cmd/internal/obj/arm64"
 	"cmd/internal/obj/s390x"
 	"cmd/internal/obj/x86"
 	"cmd/internal/src"
@@ -22,6 +23,7 @@ var Copyelim = copyelim
 var testCtxts = map[string]*obj.Link{
 	"amd64": obj.Linknew(&x86.Linkamd64),
 	"s390x": obj.Linknew(&s390x.Links390x),
+	"arm64": obj.Linknew(&arm64.Linkarm64),
 }
 
 func testConfig(tb testing.TB) *Conf      { return testConfigArch(tb, "amd64") }
diff --git a/src/cmd/compile/internal/ssa/gen/AMD64.rules b/src/cmd/compile/internal/ssa/gen/AMD64.rules
index 647a5d9cd1..ffac45bf66 100644
--- a/src/cmd/compile/internal/ssa/gen/AMD64.rules
+++ b/src/cmd/compile/internal/ssa/gen/AMD64.rules
@@ -475,6 +475,52 @@
 (ClosureCall [argwid] entry closure mem) -> (CALLclosure [argwid] entry closure mem)
 (InterCall [argwid] entry mem) -> (CALLinter [argwid] entry mem)
 
+// Lowering conditional moves
+// If the condition is a SETxx, we can just run a CMOV from the comparison that was
+// setting the flags.
+// Legend: HI=unsigned ABOVE, CS=unsigned BELOW, CC=unsigned ABOVE EQUAL, LS=unsigned BELOW EQUAL
+(CondSelect <t> x y (SET(EQ|NE|L|G|LE|GE|A|B|AE|BE|EQF|NEF|GF|GEF) cond)) && (is64BitInt(t) || isPtr(t))
+    -> (CMOVQ(EQ|NE|LT|GT|LE|GE|HI|CS|CC|LS|EQF|NEF|GTF|GEF) y x cond)
+(CondSelect <t> x y (SET(EQ|NE|L|G|LE|GE|A|B|AE|BE|EQF|NEF|GF|GEF) cond)) && is32BitInt(t)
+    -> (CMOVL(EQ|NE|LT|GT|LE|GE|HI|CS|CC|LS|EQF|NEF|GTF|GEF) y x cond)
+(CondSelect <t> x y (SET(EQ|NE|L|G|LE|GE|A|B|AE|BE|EQF|NEF|GF|GEF) cond)) && is16BitInt(t)
+    -> (CMOVW(EQ|NE|LT|GT|LE|GE|HI|CS|CC|LS|EQF|NEF|GTF|GEF) y x cond)
+
+// If the condition does not set the flags, we need to generate a comparison.
+(CondSelect <t> x y check) && !check.Type.IsFlags() && check.Type.Size() == 1
+    -> (CondSelect <t> x y (MOVBQZX <typ.UInt64> check))
+(CondSelect <t> x y check) && !check.Type.IsFlags() && check.Type.Size() == 2
+    -> (CondSelect <t> x y (MOVWQZX <typ.UInt64> check))
+(CondSelect <t> x y check) && !check.Type.IsFlags() && check.Type.Size() == 4
+    -> (CondSelect <t> x y (MOVLQZX <typ.UInt64> check))
+
+(CondSelect <t> x y check) && !check.Type.IsFlags() && check.Type.Size() == 8 && (is64BitInt(t) || isPtr(t))
+    -> (CMOVQNE y x (CMPQconst [0] check))
+(CondSelect <t> x y check) && !check.Type.IsFlags() && check.Type.Size() == 8 && is32BitInt(t)
+    -> (CMOVLNE y x (CMPQconst [0] check))
+(CondSelect <t> x y check) && !check.Type.IsFlags() && check.Type.Size() == 8 && is16BitInt(t)
+    -> (CMOVWNE y x (CMPQconst [0] check))
+
+// Absorb InvertFlags
+(CMOVQ(EQ|NE|LT|GT|LE|GE|HI|CS|CC|LS) x y (InvertFlags cond))
+    -> (CMOVQ(EQ|NE|GT|LT|GE|LE|CS|HI|LS|CC) x y cond)
+(CMOVL(EQ|NE|LT|GT|LE|GE|HI|CS|CC|LS) x y (InvertFlags cond))
+    -> (CMOVL(EQ|NE|GT|LT|GE|LE|CS|HI|LS|CC) x y cond)
+(CMOVW(EQ|NE|LT|GT|LE|GE|HI|CS|CC|LS) x y (InvertFlags cond))
+    -> (CMOVW(EQ|NE|GT|LT|GE|LE|CS|HI|LS|CC) x y cond)
+
+// Absorb constants generated during lower
+(CMOVQ(EQ|LE|GE|CC|LS) _ x (FlagEQ)) -> x
+(CMOVQ(NE|LT|GT|CS|HI) y _ (FlagEQ)) -> y
+(CMOVQ(NE|GT|GE|HI|CC) _ x (FlagGT_UGT)) -> x
+(CMOVQ(EQ|LE|LT|LS|CS) y _ (FlagGT_UGT)) -> y
+(CMOVQ(NE|GT|GE|LS|CS) _ x (FlagGT_ULT)) -> x
+(CMOVQ(EQ|LE|LT|HI|CC) y _ (FlagGT_ULT)) -> y
+(CMOVQ(NE|LT|LE|CS|LS) _ x (FlagLT_ULT)) -> x
+(CMOVQ(EQ|GT|GE|HI|CC) y _ (FlagLT_ULT)) -> y
+(CMOVQ(NE|LT|LE|HI|CC) _ x (FlagLT_UGT)) -> x
+(CMOVQ(EQ|GT|GE|CS|LS) y _ (FlagLT_ULT)) -> y
+
 // Miscellaneous
 (Convert <t> x mem) && config.PtrSize == 8 -> (MOVQconvert <t> x mem)
 (Convert <t> x mem) && config.PtrSize == 4 -> (MOVLconvert <t> x mem)
@@ -1353,6 +1399,10 @@
 (CMPLconst x [0]) -> (TESTL x x)
 (CMPWconst x [0]) -> (TESTW x x)
 (CMPBconst x [0]) -> (TESTB x x)
+(TESTQconst [-1] x) -> (TESTQ x x)
+(TESTLconst [-1] x) -> (TESTL x x)
+(TESTWconst [-1] x) -> (TESTW x x)
+(TESTBconst [-1] x) -> (TESTB x x)
 
 // Combining byte loads into larger (unaligned) loads.
 // There are many ways these combinations could occur.  This is
diff --git a/src/cmd/compile/internal/ssa/gen/AMD64Ops.go b/src/cmd/compile/internal/ssa/gen/AMD64Ops.go
index c5c2e8ceac..9577890f9a 100644
--- a/src/cmd/compile/internal/ssa/gen/AMD64Ops.go
+++ b/src/cmd/compile/internal/ssa/gen/AMD64Ops.go
@@ -132,6 +132,7 @@ func init() {
 		gpload    = regInfo{inputs: []regMask{gpspsb, 0}, outputs: gponly}
 		gp21load  = regInfo{inputs: []regMask{gp, gpspsb, 0}, outputs: gponly}
 		gploadidx = regInfo{inputs: []regMask{gpspsb, gpsp, 0}, outputs: gponly}
+		gp21pax   = regInfo{inputs: []regMask{gp &^ ax, gp}, outputs: []regMask{gp &^ ax}, clobbers: ax}
 
 		gpstore         = regInfo{inputs: []regMask{gpspsb, gpsp, 0}}
 		gpstoreconst    = regInfo{inputs: []regMask{gpspsb, 0}}
@@ -340,10 +341,57 @@ func init() {
 		{name: "BSRQ", argLength: 1, reg: gp11flags, asm: "BSRQ", typ: "(UInt64,Flags)"}, // # of high-order zeroes in 64-bit arg
 		{name: "BSRL", argLength: 1, reg: gp11flags, asm: "BSRL", typ: "(UInt32,Flags)"}, // # of high-order zeroes in 32-bit arg
 
-		// Note ASM for ops moves whole register
-		//
-		{name: "CMOVQEQ", argLength: 3, reg: gp21, asm: "CMOVQEQ", resultInArg0: true}, // if arg2 encodes "equal" return arg1 else arg0
-		{name: "CMOVLEQ", argLength: 3, reg: gp21, asm: "CMOVLEQ", resultInArg0: true}, // if arg2 encodes "equal" return arg1 else arg0
+		// CMOV instructions: 64, 32 and 16-bit sizes.
+		// if arg2 encodes a true result, return arg1, else arg0
+		{name: "CMOVQEQ", argLength: 3, reg: gp21, asm: "CMOVQEQ", resultInArg0: true},
+		{name: "CMOVQNE", argLength: 3, reg: gp21, asm: "CMOVQNE", resultInArg0: true},
+		{name: "CMOVQLT", argLength: 3, reg: gp21, asm: "CMOVQLT", resultInArg0: true},
+		{name: "CMOVQGT", argLength: 3, reg: gp21, asm: "CMOVQGT", resultInArg0: true},
+		{name: "CMOVQLE", argLength: 3, reg: gp21, asm: "CMOVQLE", resultInArg0: true},
+		{name: "CMOVQGE", argLength: 3, reg: gp21, asm: "CMOVQGE", resultInArg0: true},
+		{name: "CMOVQLS", argLength: 3, reg: gp21, asm: "CMOVQLS", resultInArg0: true},
+		{name: "CMOVQHI", argLength: 3, reg: gp21, asm: "CMOVQHI", resultInArg0: true},
+		{name: "CMOVQCC", argLength: 3, reg: gp21, asm: "CMOVQCC", resultInArg0: true},
+		{name: "CMOVQCS", argLength: 3, reg: gp21, asm: "CMOVQCS", resultInArg0: true},
+
+		{name: "CMOVLEQ", argLength: 3, reg: gp21, asm: "CMOVLEQ", resultInArg0: true},
+		{name: "CMOVLNE", argLength: 3, reg: gp21, asm: "CMOVLNE", resultInArg0: true},
+		{name: "CMOVLLT", argLength: 3, reg: gp21, asm: "CMOVLLT", resultInArg0: true},
+		{name: "CMOVLGT", argLength: 3, reg: gp21, asm: "CMOVLGT", resultInArg0: true},
+		{name: "CMOVLLE", argLength: 3, reg: gp21, asm: "CMOVLLE", resultInArg0: true},
+		{name: "CMOVLGE", argLength: 3, reg: gp21, asm: "CMOVLGE", resultInArg0: true},
+		{name: "CMOVLLS", argLength: 3, reg: gp21, asm: "CMOVLLS", resultInArg0: true},
+		{name: "CMOVLHI", argLength: 3, reg: gp21, asm: "CMOVLHI", resultInArg0: true},
+		{name: "CMOVLCC", argLength: 3, reg: gp21, asm: "CMOVLCC", resultInArg0: true},
+		{name: "CMOVLCS", argLength: 3, reg: gp21, asm: "CMOVLCS", resultInArg0: true},
+
+		{name: "CMOVWEQ", argLength: 3, reg: gp21, asm: "CMOVWEQ", resultInArg0: true},
+		{name: "CMOVWNE", argLength: 3, reg: gp21, asm: "CMOVWNE", resultInArg0: true},
+		{name: "CMOVWLT", argLength: 3, reg: gp21, asm: "CMOVWLT", resultInArg0: true},
+		{name: "CMOVWGT", argLength: 3, reg: gp21, asm: "CMOVWGT", resultInArg0: true},
+		{name: "CMOVWLE", argLength: 3, reg: gp21, asm: "CMOVWLE", resultInArg0: true},
+		{name: "CMOVWGE", argLength: 3, reg: gp21, asm: "CMOVWGE", resultInArg0: true},
+		{name: "CMOVWLS", argLength: 3, reg: gp21, asm: "CMOVWLS", resultInArg0: true},
+		{name: "CMOVWHI", argLength: 3, reg: gp21, asm: "CMOVWHI", resultInArg0: true},
+		{name: "CMOVWCC", argLength: 3, reg: gp21, asm: "CMOVWCC", resultInArg0: true},
+		{name: "CMOVWCS", argLength: 3, reg: gp21, asm: "CMOVWCS", resultInArg0: true},
+
+		// CMOV with floating point instructions. We need separate pseudo-op to handle
+		// InvertFlags correctly, and to generate special code that handles NaN (unordered flag).
+		// NOTE: the fact that CMOV*EQF here is marked to generate CMOV*NE is not a bug. See
+		// code generation in amd64/ssa.go.
+		{name: "CMOVQEQF", argLength: 3, reg: gp21pax, asm: "CMOVQNE", resultInArg0: true},
+		{name: "CMOVQNEF", argLength: 3, reg: gp21, asm: "CMOVQNE", resultInArg0: true},
+		{name: "CMOVQGTF", argLength: 3, reg: gp21, asm: "CMOVQHI", resultInArg0: true},
+		{name: "CMOVQGEF", argLength: 3, reg: gp21, asm: "CMOVQCC", resultInArg0: true},
+		{name: "CMOVLEQF", argLength: 3, reg: gp21, asm: "CMOVLNE", resultInArg0: true},
+		{name: "CMOVLNEF", argLength: 3, reg: gp21, asm: "CMOVLNE", resultInArg0: true},
+		{name: "CMOVLGTF", argLength: 3, reg: gp21, asm: "CMOVLHI", resultInArg0: true},
+		{name: "CMOVLGEF", argLength: 3, reg: gp21, asm: "CMOVLCC", resultInArg0: true},
+		{name: "CMOVWEQF", argLength: 3, reg: gp21, asm: "CMOVWNE", resultInArg0: true},
+		{name: "CMOVWNEF", argLength: 3, reg: gp21, asm: "CMOVWNE", resultInArg0: true},
+		{name: "CMOVWGTF", argLength: 3, reg: gp21, asm: "CMOVWHI", resultInArg0: true},
+		{name: "CMOVWGEF", argLength: 3, reg: gp21, asm: "CMOVWCC", resultInArg0: true},
 
 		{name: "BSWAPQ", argLength: 1, reg: gp11, asm: "BSWAPQ", resultInArg0: true, clobberFlags: true}, // arg0 swap bytes
 		{name: "BSWAPL", argLength: 1, reg: gp11, asm: "BSWAPL", resultInArg0: true, clobberFlags: true}, // arg0 swap bytes
@@ -578,7 +626,6 @@ func init() {
 		{name: "LoweredGetCallerSP", reg: gp01, rematerializeable: true},
 		//arg0=ptr,arg1=mem, returns void.  Faults if ptr is nil.
 		{name: "LoweredNilCheck", argLength: 2, reg: regInfo{inputs: []regMask{gpsp}}, clobberFlags: true, nilCheck: true, faultOnNilArg0: true},
-
 		// LoweredWB invokes runtime.gcWriteBarrier. arg0=destptr, arg1=srcptr, arg2=mem, aux=runtime.gcWriteBarrier
 		// It saves all GP registers if necessary, but may clobber others.
 		{name: "LoweredWB", argLength: 3, reg: regInfo{inputs: []regMask{buildReg("DI"), ax}, clobbers: callerSave &^ gp}, clobberFlags: true, aux: "Sym", symEffect: "None"},
diff --git a/src/cmd/compile/internal/ssa/opGen.go b/src/cmd/compile/internal/ssa/opGen.go
index 48ec74b3d2..2d73208623 100644
--- a/src/cmd/compile/internal/ssa/opGen.go
+++ b/src/cmd/compile/internal/ssa/opGen.go
@@ -560,7 +560,47 @@ const (
 	OpAMD64BSRQ
 	OpAMD64BSRL
 	OpAMD64CMOVQEQ
+	OpAMD64CMOVQNE
+	OpAMD64CMOVQLT
+	OpAMD64CMOVQGT
+	OpAMD64CMOVQLE
+	OpAMD64CMOVQGE
+	OpAMD64CMOVQLS
+	OpAMD64CMOVQHI
+	OpAMD64CMOVQCC
+	OpAMD64CMOVQCS
 	OpAMD64CMOVLEQ
+	OpAMD64CMOVLNE
+	OpAMD64CMOVLLT
+	OpAMD64CMOVLGT
+	OpAMD64CMOVLLE
+	OpAMD64CMOVLGE
+	OpAMD64CMOVLLS
+	OpAMD64CMOVLHI
+	OpAMD64CMOVLCC
+	OpAMD64CMOVLCS
+	OpAMD64CMOVWEQ
+	OpAMD64CMOVWNE
+	OpAMD64CMOVWLT
+	OpAMD64CMOVWGT
+	OpAMD64CMOVWLE
+	OpAMD64CMOVWGE
+	OpAMD64CMOVWLS
+	OpAMD64CMOVWHI
+	OpAMD64CMOVWCC
+	OpAMD64CMOVWCS
+	OpAMD64CMOVQEQF
+	OpAMD64CMOVQNEF
+	OpAMD64CMOVQGTF
+	OpAMD64CMOVQGEF
+	OpAMD64CMOVLEQF
+	OpAMD64CMOVLNEF
+	OpAMD64CMOVLGTF
+	OpAMD64CMOVLGEF
+	OpAMD64CMOVWEQF
+	OpAMD64CMOVWNEF
+	OpAMD64CMOVWGTF
+	OpAMD64CMOVWGEF
 	OpAMD64BSWAPQ
 	OpAMD64BSWAPL
 	OpAMD64POPCNTQ
@@ -6804,6 +6844,141 @@ var opcodeTable = [...]opInfo{
 		},
 	},
 	{
+		name:         "CMOVQNE",
+		argLen:       3,
+		resultInArg0: true,
+		asm:          x86.ACMOVQNE,
+		reg: regInfo{
+			inputs: []inputInfo{
+				{0, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+				{1, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+			},
+			outputs: []outputInfo{
+				{0, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+			},
+		},
+	},
+	{
+		name:         "CMOVQLT",
+		argLen:       3,
+		resultInArg0: true,
+		asm:          x86.ACMOVQLT,
+		reg: regInfo{
+			inputs: []inputInfo{
+				{0, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+				{1, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+			},
+			outputs: []outputInfo{
+				{0, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+			},
+		},
+	},
+	{
+		name:         "CMOVQGT",
+		argLen:       3,
+		resultInArg0: true,
+		asm:          x86.ACMOVQGT,
+		reg: regInfo{
+			inputs: []inputInfo{
+				{0, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+				{1, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+			},
+			outputs: []outputInfo{
+				{0, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+			},
+		},
+	},
+	{
+		name:         "CMOVQLE",
+		argLen:       3,
+		resultInArg0: true,
+		asm:          x86.ACMOVQLE,
+		reg: regInfo{
+			inputs: []inputInfo{
+				{0, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+				{1, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+			},
+			outputs: []outputInfo{
+				{0, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+			},
+		},
+	},
+	{
+		name:         "CMOVQGE",
+		argLen:       3,
+		resultInArg0: true,
+		asm:          x86.ACMOVQGE,
+		reg: regInfo{
+			inputs: []inputInfo{
+				{0, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+				{1, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+			},
+			outputs: []outputInfo{
+				{0, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+			},
+		},
+	},
+	{
+		name:         "CMOVQLS",
+		argLen:       3,
+		resultInArg0: true,
+		asm:          x86.ACMOVQLS,
+		reg: regInfo{
+			inputs: []inputInfo{
+				{0, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+				{1, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+			},
+			outputs: []outputInfo{
+				{0, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+			},
+		},
+	},
+	{
+		name:         "CMOVQHI",
+		argLen:       3,
+		resultInArg0: true,
+		asm:          x86.ACMOVQHI,
+		reg: regInfo{
+			inputs: []inputInfo{
+				{0, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+				{1, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+			},
+			outputs: []outputInfo{
+				{0, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+			},
+		},
+	},
+	{
+		name:         "CMOVQCC",
+		argLen:       3,
+		resultInArg0: true,
+		asm:          x86.ACMOVQCC,
+		reg: regInfo{
+			inputs: []inputInfo{
+				{0, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+				{1, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+			},
+			outputs: []outputInfo{
+				{0, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+			},
+		},
+	},
+	{
+		name:         "CMOVQCS",
+		argLen:       3,
+		resultInArg0: true,
+		asm:          x86.ACMOVQCS,
+		reg: regInfo{
+			inputs: []inputInfo{
+				{0, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+				{1, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+			},
+			outputs: []outputInfo{
+				{0, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+			},
+		},
+	},
+	{
 		name:         "CMOVLEQ",
 		argLen:       3,
 		resultInArg0: true,
@@ -6819,6 +6994,472 @@ var opcodeTable = [...]opInfo{
 		},
 	},
 	{
+		name:         "CMOVLNE",
+		argLen:       3,
+		resultInArg0: true,
+		asm:          x86.ACMOVLNE,
+		reg: regInfo{
+			inputs: []inputInfo{
+				{0, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+				{1, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+			},
+			outputs: []outputInfo{
+				{0, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+			},
+		},
+	},
+	{
+		name:         "CMOVLLT",
+		argLen:       3,
+		resultInArg0: true,
+		asm:          x86.ACMOVLLT,
+		reg: regInfo{
+			inputs: []inputInfo{
+				{0, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+				{1, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+			},
+			outputs: []outputInfo{
+				{0, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+			},
+		},
+	},
+	{
+		name:         "CMOVLGT",
+		argLen:       3,
+		resultInArg0: true,
+		asm:          x86.ACMOVLGT,
+		reg: regInfo{
+			inputs: []inputInfo{
+				{0, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+				{1, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+			},
+			outputs: []outputInfo{
+				{0, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+			},
+		},
+	},
+	{
+		name:         "CMOVLLE",
+		argLen:       3,
+		resultInArg0: true,
+		asm:          x86.ACMOVLLE,
+		reg: regInfo{
+			inputs: []inputInfo{
+				{0, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+				{1, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+			},
+			outputs: []outputInfo{
+				{0, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+			},
+		},
+	},
+	{
+		name:         "CMOVLGE",
+		argLen:       3,
+		resultInArg0: true,
+		asm:          x86.ACMOVLGE,
+		reg: regInfo{
+			inputs: []inputInfo{
+				{0, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+				{1, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+			},
+			outputs: []outputInfo{
+				{0, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+			},
+		},
+	},
+	{
+		name:         "CMOVLLS",
+		argLen:       3,
+		resultInArg0: true,
+		asm:          x86.ACMOVLLS,
+		reg: regInfo{
+			inputs: []inputInfo{
+				{0, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+				{1, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+			},
+			outputs: []outputInfo{
+				{0, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+			},
+		},
+	},
+	{
+		name:         "CMOVLHI",
+		argLen:       3,
+		resultInArg0: true,
+		asm:          x86.ACMOVLHI,
+		reg: regInfo{
+			inputs: []inputInfo{
+				{0, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+				{1, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+			},
+			outputs: []outputInfo{
+				{0, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+			},
+		},
+	},
+	{
+		name:         "CMOVLCC",
+		argLen:       3,
+		resultInArg0: true,
+		asm:          x86.ACMOVLCC,
+		reg: regInfo{
+			inputs: []inputInfo{
+				{0, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+				{1, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+			},
+			outputs: []outputInfo{
+				{0, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+			},
+		},
+	},
+	{
+		name:         "CMOVLCS",
+		argLen:       3,
+		resultInArg0: true,
+		asm:          x86.ACMOVLCS,
+		reg: regInfo{
+			inputs: []inputInfo{
+				{0, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+				{1, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+			},
+			outputs: []outputInfo{
+				{0, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+			},
+		},
+	},
+	{
+		name:         "CMOVWEQ",
+		argLen:       3,
+		resultInArg0: true,
+		asm:          x86.ACMOVWEQ,
+		reg: regInfo{
+			inputs: []inputInfo{
+				{0, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+				{1, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+			},
+			outputs: []outputInfo{
+				{0, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+			},
+		},
+	},
+	{
+		name:         "CMOVWNE",
+		argLen:       3,
+		resultInArg0: true,
+		asm:          x86.ACMOVWNE,
+		reg: regInfo{
+			inputs: []inputInfo{
+				{0, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+				{1, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+			},
+			outputs: []outputInfo{
+				{0, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+			},
+		},
+	},
+	{
+		name:         "CMOVWLT",
+		argLen:       3,
+		resultInArg0: true,
+		asm:          x86.ACMOVWLT,
+		reg: regInfo{
+			inputs: []inputInfo{
+				{0, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+				{1, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+			},
+			outputs: []outputInfo{
+				{0, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+			},
+		},
+	},
+	{
+		name:         "CMOVWGT",
+		argLen:       3,
+		resultInArg0: true,
+		asm:          x86.ACMOVWGT,
+		reg: regInfo{
+			inputs: []inputInfo{
+				{0, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+				{1, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+			},
+			outputs: []outputInfo{
+				{0, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+			},
+		},
+	},
+	{
+		name:         "CMOVWLE",
+		argLen:       3,
+		resultInArg0: true,
+		asm:          x86.ACMOVWLE,
+		reg: regInfo{
+			inputs: []inputInfo{
+				{0, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+				{1, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+			},
+			outputs: []outputInfo{
+				{0, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+			},
+		},
+	},
+	{
+		name:         "CMOVWGE",
+		argLen:       3,
+		resultInArg0: true,
+		asm:          x86.ACMOVWGE,
+		reg: regInfo{
+			inputs: []inputInfo{
+				{0, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+				{1, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+			},
+			outputs: []outputInfo{
+				{0, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+			},
+		},
+	},
+	{
+		name:         "CMOVWLS",
+		argLen:       3,
+		resultInArg0: true,
+		asm:          x86.ACMOVWLS,
+		reg: regInfo{
+			inputs: []inputInfo{
+				{0, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+				{1, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+			},
+			outputs: []outputInfo{
+				{0, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+			},
+		},
+	},
+	{
+		name:         "CMOVWHI",
+		argLen:       3,
+		resultInArg0: true,
+		asm:          x86.ACMOVWHI,
+		reg: regInfo{
+			inputs: []inputInfo{
+				{0, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+				{1, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+			},
+			outputs: []outputInfo{
+				{0, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+			},
+		},
+	},
+	{
+		name:         "CMOVWCC",
+		argLen:       3,
+		resultInArg0: true,
+		asm:          x86.ACMOVWCC,
+		reg: regInfo{
+			inputs: []inputInfo{
+				{0, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+				{1, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+			},
+			outputs: []outputInfo{
+				{0, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+			},
+		},
+	},
+	{
+		name:         "CMOVWCS",
+		argLen:       3,
+		resultInArg0: true,
+		asm:          x86.ACMOVWCS,
+		reg: regInfo{
+			inputs: []inputInfo{
+				{0, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+				{1, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+			},
+			outputs: []outputInfo{
+				{0, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+			},
+		},
+	},
+	{
+		name:         "CMOVQEQF",
+		argLen:       3,
+		resultInArg0: true,
+		asm:          x86.ACMOVQNE,
+		reg: regInfo{
+			inputs: []inputInfo{
+				{0, 65518}, // CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+				{1, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+			},
+			clobbers: 1, // AX
+			outputs: []outputInfo{
+				{0, 65518}, // CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+			},
+		},
+	},
+	{
+		name:         "CMOVQNEF",
+		argLen:       3,
+		resultInArg0: true,
+		asm:          x86.ACMOVQNE,
+		reg: regInfo{
+			inputs: []inputInfo{
+				{0, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+				{1, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+			},
+			outputs: []outputInfo{
+				{0, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+			},
+		},
+	},
+	{
+		name:         "CMOVQGTF",
+		argLen:       3,
+		resultInArg0: true,
+		asm:          x86.ACMOVQHI,
+		reg: regInfo{
+			inputs: []inputInfo{
+				{0, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+				{1, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+			},
+			outputs: []outputInfo{
+				{0, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+			},
+		},
+	},
+	{
+		name:         "CMOVQGEF",
+		argLen:       3,
+		resultInArg0: true,
+		asm:          x86.ACMOVQCC,
+		reg: regInfo{
+			inputs: []inputInfo{
+				{0, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+				{1, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+			},
+			outputs: []outputInfo{
+				{0, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+			},
+		},
+	},
+	{
+		name:         "CMOVLEQF",
+		argLen:       3,
+		resultInArg0: true,
+		asm:          x86.ACMOVLNE,
+		reg: regInfo{
+			inputs: []inputInfo{
+				{0, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+				{1, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+			},
+			outputs: []outputInfo{
+				{0, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+			},
+		},
+	},
+	{
+		name:         "CMOVLNEF",
+		argLen:       3,
+		resultInArg0: true,
+		asm:          x86.ACMOVLNE,
+		reg: regInfo{
+			inputs: []inputInfo{
+				{0, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+				{1, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+			},
+			outputs: []outputInfo{
+				{0, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+			},
+		},
+	},
+	{
+		name:         "CMOVLGTF",
+		argLen:       3,
+		resultInArg0: true,
+		asm:          x86.ACMOVLHI,
+		reg: regInfo{
+			inputs: []inputInfo{
+				{0, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+				{1, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+			},
+			outputs: []outputInfo{
+				{0, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+			},
+		},
+	},
+	{
+		name:         "CMOVLGEF",
+		argLen:       3,
+		resultInArg0: true,
+		asm:          x86.ACMOVLCC,
+		reg: regInfo{
+			inputs: []inputInfo{
+				{0, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+				{1, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+			},
+			outputs: []outputInfo{
+				{0, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+			},
+		},
+	},
+	{
+		name:         "CMOVWEQF",
+		argLen:       3,
+		resultInArg0: true,
+		asm:          x86.ACMOVWNE,
+		reg: regInfo{
+			inputs: []inputInfo{
+				{0, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+				{1, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+			},
+			outputs: []outputInfo{
+				{0, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+			},
+		},
+	},
+	{
+		name:         "CMOVWNEF",
+		argLen:       3,
+		resultInArg0: true,
+		asm:          x86.ACMOVWNE,
+		reg: regInfo{
+			inputs: []inputInfo{
+				{0, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+				{1, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+			},
+			outputs: []outputInfo{
+				{0, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+			},
+		},
+	},
+	{
+		name:         "CMOVWGTF",
+		argLen:       3,
+		resultInArg0: true,
+		asm:          x86.ACMOVWHI,
+		reg: regInfo{
+			inputs: []inputInfo{
+				{0, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+				{1, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+			},
+			outputs: []outputInfo{
+				{0, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+			},
+		},
+	},
+	{
+		name:         "CMOVWGEF",
+		argLen:       3,
+		resultInArg0: true,
+		asm:          x86.ACMOVWCC,
+		reg: regInfo{
+			inputs: []inputInfo{
+				{0, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+				{1, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+			},
+			outputs: []outputInfo{
+				{0, 65519}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+			},
+		},
+	},
+	{
 		name:         "BSWAPQ",
 		argLen:       1,
 		resultInArg0: true,
diff --git a/src/cmd/compile/internal/ssa/rewriteAMD64.go b/src/cmd/compile/internal/ssa/rewriteAMD64.go
index c500c757ef..eb2489ac77 100644
--- a/src/cmd/compile/internal/ssa/rewriteAMD64.go
+++ b/src/cmd/compile/internal/ssa/rewriteAMD64.go
@@ -55,8 +55,66 @@ func rewriteValueAMD64(v *Value) bool {
 		return rewriteValueAMD64_OpAMD64BSFQ_0(v)
 	case OpAMD64BTQconst:
 		return rewriteValueAMD64_OpAMD64BTQconst_0(v)
+	case OpAMD64CMOVLCC:
+		return rewriteValueAMD64_OpAMD64CMOVLCC_0(v)
+	case OpAMD64CMOVLCS:
+		return rewriteValueAMD64_OpAMD64CMOVLCS_0(v)
+	case OpAMD64CMOVLEQ:
+		return rewriteValueAMD64_OpAMD64CMOVLEQ_0(v)
+	case OpAMD64CMOVLGE:
+		return rewriteValueAMD64_OpAMD64CMOVLGE_0(v)
+	case OpAMD64CMOVLGT:
+		return rewriteValueAMD64_OpAMD64CMOVLGT_0(v)
+	case OpAMD64CMOVLHI:
+		return rewriteValueAMD64_OpAMD64CMOVLHI_0(v)
+	case OpAMD64CMOVLLE:
+		return rewriteValueAMD64_OpAMD64CMOVLLE_0(v)
+	case OpAMD64CMOVLLS:
+		return rewriteValueAMD64_OpAMD64CMOVLLS_0(v)
+	case OpAMD64CMOVLLT:
+		return rewriteValueAMD64_OpAMD64CMOVLLT_0(v)
+	case OpAMD64CMOVLNE:
+		return rewriteValueAMD64_OpAMD64CMOVLNE_0(v)
+	case OpAMD64CMOVQCC:
+		return rewriteValueAMD64_OpAMD64CMOVQCC_0(v)
+	case OpAMD64CMOVQCS:
+		return rewriteValueAMD64_OpAMD64CMOVQCS_0(v)
 	case OpAMD64CMOVQEQ:
 		return rewriteValueAMD64_OpAMD64CMOVQEQ_0(v)
+	case OpAMD64CMOVQGE:
+		return rewriteValueAMD64_OpAMD64CMOVQGE_0(v)
+	case OpAMD64CMOVQGT:
+		return rewriteValueAMD64_OpAMD64CMOVQGT_0(v)
+	case OpAMD64CMOVQHI:
+		return rewriteValueAMD64_OpAMD64CMOVQHI_0(v)
+	case OpAMD64CMOVQLE:
+		return rewriteValueAMD64_OpAMD64CMOVQLE_0(v)
+	case OpAMD64CMOVQLS:
+		return rewriteValueAMD64_OpAMD64CMOVQLS_0(v)
+	case OpAMD64CMOVQLT:
+		return rewriteValueAMD64_OpAMD64CMOVQLT_0(v)
+	case OpAMD64CMOVQNE:
+		return rewriteValueAMD64_OpAMD64CMOVQNE_0(v)
+	case OpAMD64CMOVWCC:
+		return rewriteValueAMD64_OpAMD64CMOVWCC_0(v)
+	case OpAMD64CMOVWCS:
+		return rewriteValueAMD64_OpAMD64CMOVWCS_0(v)
+	case OpAMD64CMOVWEQ:
+		return rewriteValueAMD64_OpAMD64CMOVWEQ_0(v)
+	case OpAMD64CMOVWGE:
+		return rewriteValueAMD64_OpAMD64CMOVWGE_0(v)
+	case OpAMD64CMOVWGT:
+		return rewriteValueAMD64_OpAMD64CMOVWGT_0(v)
+	case OpAMD64CMOVWHI:
+		return rewriteValueAMD64_OpAMD64CMOVWHI_0(v)
+	case OpAMD64CMOVWLE:
+		return rewriteValueAMD64_OpAMD64CMOVWLE_0(v)
+	case OpAMD64CMOVWLS:
+		return rewriteValueAMD64_OpAMD64CMOVWLS_0(v)
+	case OpAMD64CMOVWLT:
+		return rewriteValueAMD64_OpAMD64CMOVWLT_0(v)
+	case OpAMD64CMOVWNE:
+		return rewriteValueAMD64_OpAMD64CMOVWNE_0(v)
 	case OpAMD64CMPB:
 		return rewriteValueAMD64_OpAMD64CMPB_0(v)
 	case OpAMD64CMPBconst:
@@ -391,12 +449,20 @@ func rewriteValueAMD64(v *Value) bool {
 		return rewriteValueAMD64_OpAMD64SUBSSmem_0(v)
 	case OpAMD64TESTB:
 		return rewriteValueAMD64_OpAMD64TESTB_0(v)
+	case OpAMD64TESTBconst:
+		return rewriteValueAMD64_OpAMD64TESTBconst_0(v)
 	case OpAMD64TESTL:
 		return rewriteValueAMD64_OpAMD64TESTL_0(v)
+	case OpAMD64TESTLconst:
+		return rewriteValueAMD64_OpAMD64TESTLconst_0(v)
 	case OpAMD64TESTQ:
 		return rewriteValueAMD64_OpAMD64TESTQ_0(v)
+	case OpAMD64TESTQconst:
+		return rewriteValueAMD64_OpAMD64TESTQconst_0(v)
 	case OpAMD64TESTW:
 		return rewriteValueAMD64_OpAMD64TESTW_0(v)
+	case OpAMD64TESTWconst:
+		return rewriteValueAMD64_OpAMD64TESTWconst_0(v)
 	case OpAMD64XADDLlock:
 		return rewriteValueAMD64_OpAMD64XADDLlock_0(v)
 	case OpAMD64XADDQlock:
@@ -493,6 +559,8 @@ func rewriteValueAMD64(v *Value) bool {
 		return rewriteValueAMD64_OpCom64_0(v)
 	case OpCom8:
 		return rewriteValueAMD64_OpCom8_0(v)
+	case OpCondSelect:
+		return rewriteValueAMD64_OpCondSelect_0(v) || rewriteValueAMD64_OpCondSelect_10(v) || rewriteValueAMD64_OpCondSelect_20(v) || rewriteValueAMD64_OpCondSelect_30(v) || rewriteValueAMD64_OpCondSelect_40(v)
 	case OpConst16:
 		return rewriteValueAMD64_OpConst16_0(v)
 	case OpConst32:
@@ -3285,7 +3353,502 @@ func rewriteValueAMD64_OpAMD64BTQconst_0(v *Value) bool {
 	}
 	return false
 }
+func rewriteValueAMD64_OpAMD64CMOVLCC_0(v *Value) bool {
+	// match: (CMOVLCC x y (InvertFlags cond))
+	// cond:
+	// result: (CMOVLLS x y cond)
+	for {
+		_ = v.Args[2]
+		x := v.Args[0]
+		y := v.Args[1]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64InvertFlags {
+			break
+		}
+		cond := v_2.Args[0]
+		v.reset(OpAMD64CMOVLLS)
+		v.AddArg(x)
+		v.AddArg(y)
+		v.AddArg(cond)
+		return true
+	}
+	return false
+}
+func rewriteValueAMD64_OpAMD64CMOVLCS_0(v *Value) bool {
+	// match: (CMOVLCS x y (InvertFlags cond))
+	// cond:
+	// result: (CMOVLHI x y cond)
+	for {
+		_ = v.Args[2]
+		x := v.Args[0]
+		y := v.Args[1]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64InvertFlags {
+			break
+		}
+		cond := v_2.Args[0]
+		v.reset(OpAMD64CMOVLHI)
+		v.AddArg(x)
+		v.AddArg(y)
+		v.AddArg(cond)
+		return true
+	}
+	return false
+}
+func rewriteValueAMD64_OpAMD64CMOVLEQ_0(v *Value) bool {
+	// match: (CMOVLEQ x y (InvertFlags cond))
+	// cond:
+	// result: (CMOVLEQ x y cond)
+	for {
+		_ = v.Args[2]
+		x := v.Args[0]
+		y := v.Args[1]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64InvertFlags {
+			break
+		}
+		cond := v_2.Args[0]
+		v.reset(OpAMD64CMOVLEQ)
+		v.AddArg(x)
+		v.AddArg(y)
+		v.AddArg(cond)
+		return true
+	}
+	return false
+}
+func rewriteValueAMD64_OpAMD64CMOVLGE_0(v *Value) bool {
+	// match: (CMOVLGE x y (InvertFlags cond))
+	// cond:
+	// result: (CMOVLLE x y cond)
+	for {
+		_ = v.Args[2]
+		x := v.Args[0]
+		y := v.Args[1]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64InvertFlags {
+			break
+		}
+		cond := v_2.Args[0]
+		v.reset(OpAMD64CMOVLLE)
+		v.AddArg(x)
+		v.AddArg(y)
+		v.AddArg(cond)
+		return true
+	}
+	return false
+}
+func rewriteValueAMD64_OpAMD64CMOVLGT_0(v *Value) bool {
+	// match: (CMOVLGT x y (InvertFlags cond))
+	// cond:
+	// result: (CMOVLLT x y cond)
+	for {
+		_ = v.Args[2]
+		x := v.Args[0]
+		y := v.Args[1]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64InvertFlags {
+			break
+		}
+		cond := v_2.Args[0]
+		v.reset(OpAMD64CMOVLLT)
+		v.AddArg(x)
+		v.AddArg(y)
+		v.AddArg(cond)
+		return true
+	}
+	return false
+}
+func rewriteValueAMD64_OpAMD64CMOVLHI_0(v *Value) bool {
+	// match: (CMOVLHI x y (InvertFlags cond))
+	// cond:
+	// result: (CMOVLCS x y cond)
+	for {
+		_ = v.Args[2]
+		x := v.Args[0]
+		y := v.Args[1]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64InvertFlags {
+			break
+		}
+		cond := v_2.Args[0]
+		v.reset(OpAMD64CMOVLCS)
+		v.AddArg(x)
+		v.AddArg(y)
+		v.AddArg(cond)
+		return true
+	}
+	return false
+}
+func rewriteValueAMD64_OpAMD64CMOVLLE_0(v *Value) bool {
+	// match: (CMOVLLE x y (InvertFlags cond))
+	// cond:
+	// result: (CMOVLGE x y cond)
+	for {
+		_ = v.Args[2]
+		x := v.Args[0]
+		y := v.Args[1]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64InvertFlags {
+			break
+		}
+		cond := v_2.Args[0]
+		v.reset(OpAMD64CMOVLGE)
+		v.AddArg(x)
+		v.AddArg(y)
+		v.AddArg(cond)
+		return true
+	}
+	return false
+}
+func rewriteValueAMD64_OpAMD64CMOVLLS_0(v *Value) bool {
+	// match: (CMOVLLS x y (InvertFlags cond))
+	// cond:
+	// result: (CMOVLCC x y cond)
+	for {
+		_ = v.Args[2]
+		x := v.Args[0]
+		y := v.Args[1]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64InvertFlags {
+			break
+		}
+		cond := v_2.Args[0]
+		v.reset(OpAMD64CMOVLCC)
+		v.AddArg(x)
+		v.AddArg(y)
+		v.AddArg(cond)
+		return true
+	}
+	return false
+}
+func rewriteValueAMD64_OpAMD64CMOVLLT_0(v *Value) bool {
+	// match: (CMOVLLT x y (InvertFlags cond))
+	// cond:
+	// result: (CMOVLGT x y cond)
+	for {
+		_ = v.Args[2]
+		x := v.Args[0]
+		y := v.Args[1]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64InvertFlags {
+			break
+		}
+		cond := v_2.Args[0]
+		v.reset(OpAMD64CMOVLGT)
+		v.AddArg(x)
+		v.AddArg(y)
+		v.AddArg(cond)
+		return true
+	}
+	return false
+}
+func rewriteValueAMD64_OpAMD64CMOVLNE_0(v *Value) bool {
+	// match: (CMOVLNE x y (InvertFlags cond))
+	// cond:
+	// result: (CMOVLNE x y cond)
+	for {
+		_ = v.Args[2]
+		x := v.Args[0]
+		y := v.Args[1]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64InvertFlags {
+			break
+		}
+		cond := v_2.Args[0]
+		v.reset(OpAMD64CMOVLNE)
+		v.AddArg(x)
+		v.AddArg(y)
+		v.AddArg(cond)
+		return true
+	}
+	return false
+}
+func rewriteValueAMD64_OpAMD64CMOVQCC_0(v *Value) bool {
+	// match: (CMOVQCC x y (InvertFlags cond))
+	// cond:
+	// result: (CMOVQLS x y cond)
+	for {
+		_ = v.Args[2]
+		x := v.Args[0]
+		y := v.Args[1]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64InvertFlags {
+			break
+		}
+		cond := v_2.Args[0]
+		v.reset(OpAMD64CMOVQLS)
+		v.AddArg(x)
+		v.AddArg(y)
+		v.AddArg(cond)
+		return true
+	}
+	// match: (CMOVQCC _ x (FlagEQ))
+	// cond:
+	// result: x
+	for {
+		_ = v.Args[2]
+		x := v.Args[1]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64FlagEQ {
+			break
+		}
+		v.reset(OpCopy)
+		v.Type = x.Type
+		v.AddArg(x)
+		return true
+	}
+	// match: (CMOVQCC _ x (FlagGT_UGT))
+	// cond:
+	// result: x
+	for {
+		_ = v.Args[2]
+		x := v.Args[1]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64FlagGT_UGT {
+			break
+		}
+		v.reset(OpCopy)
+		v.Type = x.Type
+		v.AddArg(x)
+		return true
+	}
+	// match: (CMOVQCC y _ (FlagGT_ULT))
+	// cond:
+	// result: y
+	for {
+		_ = v.Args[2]
+		y := v.Args[0]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64FlagGT_ULT {
+			break
+		}
+		v.reset(OpCopy)
+		v.Type = y.Type
+		v.AddArg(y)
+		return true
+	}
+	// match: (CMOVQCC y _ (FlagLT_ULT))
+	// cond:
+	// result: y
+	for {
+		_ = v.Args[2]
+		y := v.Args[0]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64FlagLT_ULT {
+			break
+		}
+		v.reset(OpCopy)
+		v.Type = y.Type
+		v.AddArg(y)
+		return true
+	}
+	// match: (CMOVQCC _ x (FlagLT_UGT))
+	// cond:
+	// result: x
+	for {
+		_ = v.Args[2]
+		x := v.Args[1]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64FlagLT_UGT {
+			break
+		}
+		v.reset(OpCopy)
+		v.Type = x.Type
+		v.AddArg(x)
+		return true
+	}
+	return false
+}
+func rewriteValueAMD64_OpAMD64CMOVQCS_0(v *Value) bool {
+	// match: (CMOVQCS x y (InvertFlags cond))
+	// cond:
+	// result: (CMOVQHI x y cond)
+	for {
+		_ = v.Args[2]
+		x := v.Args[0]
+		y := v.Args[1]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64InvertFlags {
+			break
+		}
+		cond := v_2.Args[0]
+		v.reset(OpAMD64CMOVQHI)
+		v.AddArg(x)
+		v.AddArg(y)
+		v.AddArg(cond)
+		return true
+	}
+	// match: (CMOVQCS y _ (FlagEQ))
+	// cond:
+	// result: y
+	for {
+		_ = v.Args[2]
+		y := v.Args[0]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64FlagEQ {
+			break
+		}
+		v.reset(OpCopy)
+		v.Type = y.Type
+		v.AddArg(y)
+		return true
+	}
+	// match: (CMOVQCS y _ (FlagGT_UGT))
+	// cond:
+	// result: y
+	for {
+		_ = v.Args[2]
+		y := v.Args[0]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64FlagGT_UGT {
+			break
+		}
+		v.reset(OpCopy)
+		v.Type = y.Type
+		v.AddArg(y)
+		return true
+	}
+	// match: (CMOVQCS _ x (FlagGT_ULT))
+	// cond:
+	// result: x
+	for {
+		_ = v.Args[2]
+		x := v.Args[1]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64FlagGT_ULT {
+			break
+		}
+		v.reset(OpCopy)
+		v.Type = x.Type
+		v.AddArg(x)
+		return true
+	}
+	// match: (CMOVQCS _ x (FlagLT_ULT))
+	// cond:
+	// result: x
+	for {
+		_ = v.Args[2]
+		x := v.Args[1]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64FlagLT_ULT {
+			break
+		}
+		v.reset(OpCopy)
+		v.Type = x.Type
+		v.AddArg(x)
+		return true
+	}
+	// match: (CMOVQCS y _ (FlagLT_ULT))
+	// cond:
+	// result: y
+	for {
+		_ = v.Args[2]
+		y := v.Args[0]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64FlagLT_ULT {
+			break
+		}
+		v.reset(OpCopy)
+		v.Type = y.Type
+		v.AddArg(y)
+		return true
+	}
+	return false
+}
 func rewriteValueAMD64_OpAMD64CMOVQEQ_0(v *Value) bool {
+	// match: (CMOVQEQ x y (InvertFlags cond))
+	// cond:
+	// result: (CMOVQEQ x y cond)
+	for {
+		_ = v.Args[2]
+		x := v.Args[0]
+		y := v.Args[1]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64InvertFlags {
+			break
+		}
+		cond := v_2.Args[0]
+		v.reset(OpAMD64CMOVQEQ)
+		v.AddArg(x)
+		v.AddArg(y)
+		v.AddArg(cond)
+		return true
+	}
+	// match: (CMOVQEQ _ x (FlagEQ))
+	// cond:
+	// result: x
+	for {
+		_ = v.Args[2]
+		x := v.Args[1]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64FlagEQ {
+			break
+		}
+		v.reset(OpCopy)
+		v.Type = x.Type
+		v.AddArg(x)
+		return true
+	}
+	// match: (CMOVQEQ y _ (FlagGT_UGT))
+	// cond:
+	// result: y
+	for {
+		_ = v.Args[2]
+		y := v.Args[0]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64FlagGT_UGT {
+			break
+		}
+		v.reset(OpCopy)
+		v.Type = y.Type
+		v.AddArg(y)
+		return true
+	}
+	// match: (CMOVQEQ y _ (FlagGT_ULT))
+	// cond:
+	// result: y
+	for {
+		_ = v.Args[2]
+		y := v.Args[0]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64FlagGT_ULT {
+			break
+		}
+		v.reset(OpCopy)
+		v.Type = y.Type
+		v.AddArg(y)
+		return true
+	}
+	// match: (CMOVQEQ y _ (FlagLT_ULT))
+	// cond:
+	// result: y
+	for {
+		_ = v.Args[2]
+		y := v.Args[0]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64FlagLT_ULT {
+			break
+		}
+		v.reset(OpCopy)
+		v.Type = y.Type
+		v.AddArg(y)
+		return true
+	}
+	// match: (CMOVQEQ y _ (FlagLT_ULT))
+	// cond:
+	// result: y
+	for {
+		_ = v.Args[2]
+		y := v.Args[0]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64FlagLT_ULT {
+			break
+		}
+		v.reset(OpCopy)
+		v.Type = y.Type
+		v.AddArg(y)
+		return true
+	}
 	// match: (CMOVQEQ x _ (Select1 (BSFQ (ORQconst [c] _))))
 	// cond: c != 0
 	// result: x
@@ -3315,6 +3878,888 @@ func rewriteValueAMD64_OpAMD64CMOVQEQ_0(v *Value) bool {
 	}
 	return false
 }
+func rewriteValueAMD64_OpAMD64CMOVQGE_0(v *Value) bool {
+	// match: (CMOVQGE x y (InvertFlags cond))
+	// cond:
+	// result: (CMOVQLE x y cond)
+	for {
+		_ = v.Args[2]
+		x := v.Args[0]
+		y := v.Args[1]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64InvertFlags {
+			break
+		}
+		cond := v_2.Args[0]
+		v.reset(OpAMD64CMOVQLE)
+		v.AddArg(x)
+		v.AddArg(y)
+		v.AddArg(cond)
+		return true
+	}
+	// match: (CMOVQGE _ x (FlagEQ))
+	// cond:
+	// result: x
+	for {
+		_ = v.Args[2]
+		x := v.Args[1]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64FlagEQ {
+			break
+		}
+		v.reset(OpCopy)
+		v.Type = x.Type
+		v.AddArg(x)
+		return true
+	}
+	// match: (CMOVQGE _ x (FlagGT_UGT))
+	// cond:
+	// result: x
+	for {
+		_ = v.Args[2]
+		x := v.Args[1]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64FlagGT_UGT {
+			break
+		}
+		v.reset(OpCopy)
+		v.Type = x.Type
+		v.AddArg(x)
+		return true
+	}
+	// match: (CMOVQGE _ x (FlagGT_ULT))
+	// cond:
+	// result: x
+	for {
+		_ = v.Args[2]
+		x := v.Args[1]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64FlagGT_ULT {
+			break
+		}
+		v.reset(OpCopy)
+		v.Type = x.Type
+		v.AddArg(x)
+		return true
+	}
+	// match: (CMOVQGE y _ (FlagLT_ULT))
+	// cond:
+	// result: y
+	for {
+		_ = v.Args[2]
+		y := v.Args[0]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64FlagLT_ULT {
+			break
+		}
+		v.reset(OpCopy)
+		v.Type = y.Type
+		v.AddArg(y)
+		return true
+	}
+	// match: (CMOVQGE y _ (FlagLT_ULT))
+	// cond:
+	// result: y
+	for {
+		_ = v.Args[2]
+		y := v.Args[0]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64FlagLT_ULT {
+			break
+		}
+		v.reset(OpCopy)
+		v.Type = y.Type
+		v.AddArg(y)
+		return true
+	}
+	return false
+}
+func rewriteValueAMD64_OpAMD64CMOVQGT_0(v *Value) bool {
+	// match: (CMOVQGT x y (InvertFlags cond))
+	// cond:
+	// result: (CMOVQLT x y cond)
+	for {
+		_ = v.Args[2]
+		x := v.Args[0]
+		y := v.Args[1]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64InvertFlags {
+			break
+		}
+		cond := v_2.Args[0]
+		v.reset(OpAMD64CMOVQLT)
+		v.AddArg(x)
+		v.AddArg(y)
+		v.AddArg(cond)
+		return true
+	}
+	// match: (CMOVQGT y _ (FlagEQ))
+	// cond:
+	// result: y
+	for {
+		_ = v.Args[2]
+		y := v.Args[0]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64FlagEQ {
+			break
+		}
+		v.reset(OpCopy)
+		v.Type = y.Type
+		v.AddArg(y)
+		return true
+	}
+	// match: (CMOVQGT _ x (FlagGT_UGT))
+	// cond:
+	// result: x
+	for {
+		_ = v.Args[2]
+		x := v.Args[1]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64FlagGT_UGT {
+			break
+		}
+		v.reset(OpCopy)
+		v.Type = x.Type
+		v.AddArg(x)
+		return true
+	}
+	// match: (CMOVQGT _ x (FlagGT_ULT))
+	// cond:
+	// result: x
+	for {
+		_ = v.Args[2]
+		x := v.Args[1]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64FlagGT_ULT {
+			break
+		}
+		v.reset(OpCopy)
+		v.Type = x.Type
+		v.AddArg(x)
+		return true
+	}
+	// match: (CMOVQGT y _ (FlagLT_ULT))
+	// cond:
+	// result: y
+	for {
+		_ = v.Args[2]
+		y := v.Args[0]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64FlagLT_ULT {
+			break
+		}
+		v.reset(OpCopy)
+		v.Type = y.Type
+		v.AddArg(y)
+		return true
+	}
+	// match: (CMOVQGT y _ (FlagLT_ULT))
+	// cond:
+	// result: y
+	for {
+		_ = v.Args[2]
+		y := v.Args[0]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64FlagLT_ULT {
+			break
+		}
+		v.reset(OpCopy)
+		v.Type = y.Type
+		v.AddArg(y)
+		return true
+	}
+	return false
+}
+func rewriteValueAMD64_OpAMD64CMOVQHI_0(v *Value) bool {
+	// match: (CMOVQHI x y (InvertFlags cond))
+	// cond:
+	// result: (CMOVQCS x y cond)
+	for {
+		_ = v.Args[2]
+		x := v.Args[0]
+		y := v.Args[1]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64InvertFlags {
+			break
+		}
+		cond := v_2.Args[0]
+		v.reset(OpAMD64CMOVQCS)
+		v.AddArg(x)
+		v.AddArg(y)
+		v.AddArg(cond)
+		return true
+	}
+	// match: (CMOVQHI y _ (FlagEQ))
+	// cond:
+	// result: y
+	for {
+		_ = v.Args[2]
+		y := v.Args[0]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64FlagEQ {
+			break
+		}
+		v.reset(OpCopy)
+		v.Type = y.Type
+		v.AddArg(y)
+		return true
+	}
+	// match: (CMOVQHI _ x (FlagGT_UGT))
+	// cond:
+	// result: x
+	for {
+		_ = v.Args[2]
+		x := v.Args[1]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64FlagGT_UGT {
+			break
+		}
+		v.reset(OpCopy)
+		v.Type = x.Type
+		v.AddArg(x)
+		return true
+	}
+	// match: (CMOVQHI y _ (FlagGT_ULT))
+	// cond:
+	// result: y
+	for {
+		_ = v.Args[2]
+		y := v.Args[0]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64FlagGT_ULT {
+			break
+		}
+		v.reset(OpCopy)
+		v.Type = y.Type
+		v.AddArg(y)
+		return true
+	}
+	// match: (CMOVQHI y _ (FlagLT_ULT))
+	// cond:
+	// result: y
+	for {
+		_ = v.Args[2]
+		y := v.Args[0]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64FlagLT_ULT {
+			break
+		}
+		v.reset(OpCopy)
+		v.Type = y.Type
+		v.AddArg(y)
+		return true
+	}
+	// match: (CMOVQHI _ x (FlagLT_UGT))
+	// cond:
+	// result: x
+	for {
+		_ = v.Args[2]
+		x := v.Args[1]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64FlagLT_UGT {
+			break
+		}
+		v.reset(OpCopy)
+		v.Type = x.Type
+		v.AddArg(x)
+		return true
+	}
+	return false
+}
+func rewriteValueAMD64_OpAMD64CMOVQLE_0(v *Value) bool {
+	// match: (CMOVQLE x y (InvertFlags cond))
+	// cond:
+	// result: (CMOVQGE x y cond)
+	for {
+		_ = v.Args[2]
+		x := v.Args[0]
+		y := v.Args[1]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64InvertFlags {
+			break
+		}
+		cond := v_2.Args[0]
+		v.reset(OpAMD64CMOVQGE)
+		v.AddArg(x)
+		v.AddArg(y)
+		v.AddArg(cond)
+		return true
+	}
+	// match: (CMOVQLE _ x (FlagEQ))
+	// cond:
+	// result: x
+	for {
+		_ = v.Args[2]
+		x := v.Args[1]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64FlagEQ {
+			break
+		}
+		v.reset(OpCopy)
+		v.Type = x.Type
+		v.AddArg(x)
+		return true
+	}
+	// match: (CMOVQLE y _ (FlagGT_UGT))
+	// cond:
+	// result: y
+	for {
+		_ = v.Args[2]
+		y := v.Args[0]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64FlagGT_UGT {
+			break
+		}
+		v.reset(OpCopy)
+		v.Type = y.Type
+		v.AddArg(y)
+		return true
+	}
+	// match: (CMOVQLE y _ (FlagGT_ULT))
+	// cond:
+	// result: y
+	for {
+		_ = v.Args[2]
+		y := v.Args[0]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64FlagGT_ULT {
+			break
+		}
+		v.reset(OpCopy)
+		v.Type = y.Type
+		v.AddArg(y)
+		return true
+	}
+	// match: (CMOVQLE _ x (FlagLT_ULT))
+	// cond:
+	// result: x
+	for {
+		_ = v.Args[2]
+		x := v.Args[1]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64FlagLT_ULT {
+			break
+		}
+		v.reset(OpCopy)
+		v.Type = x.Type
+		v.AddArg(x)
+		return true
+	}
+	// match: (CMOVQLE _ x (FlagLT_UGT))
+	// cond:
+	// result: x
+	for {
+		_ = v.Args[2]
+		x := v.Args[1]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64FlagLT_UGT {
+			break
+		}
+		v.reset(OpCopy)
+		v.Type = x.Type
+		v.AddArg(x)
+		return true
+	}
+	return false
+}
+func rewriteValueAMD64_OpAMD64CMOVQLS_0(v *Value) bool {
+	// match: (CMOVQLS x y (InvertFlags cond))
+	// cond:
+	// result: (CMOVQCC x y cond)
+	for {
+		_ = v.Args[2]
+		x := v.Args[0]
+		y := v.Args[1]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64InvertFlags {
+			break
+		}
+		cond := v_2.Args[0]
+		v.reset(OpAMD64CMOVQCC)
+		v.AddArg(x)
+		v.AddArg(y)
+		v.AddArg(cond)
+		return true
+	}
+	// match: (CMOVQLS _ x (FlagEQ))
+	// cond:
+	// result: x
+	for {
+		_ = v.Args[2]
+		x := v.Args[1]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64FlagEQ {
+			break
+		}
+		v.reset(OpCopy)
+		v.Type = x.Type
+		v.AddArg(x)
+		return true
+	}
+	// match: (CMOVQLS y _ (FlagGT_UGT))
+	// cond:
+	// result: y
+	for {
+		_ = v.Args[2]
+		y := v.Args[0]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64FlagGT_UGT {
+			break
+		}
+		v.reset(OpCopy)
+		v.Type = y.Type
+		v.AddArg(y)
+		return true
+	}
+	// match: (CMOVQLS _ x (FlagGT_ULT))
+	// cond:
+	// result: x
+	for {
+		_ = v.Args[2]
+		x := v.Args[1]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64FlagGT_ULT {
+			break
+		}
+		v.reset(OpCopy)
+		v.Type = x.Type
+		v.AddArg(x)
+		return true
+	}
+	// match: (CMOVQLS _ x (FlagLT_ULT))
+	// cond:
+	// result: x
+	for {
+		_ = v.Args[2]
+		x := v.Args[1]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64FlagLT_ULT {
+			break
+		}
+		v.reset(OpCopy)
+		v.Type = x.Type
+		v.AddArg(x)
+		return true
+	}
+	// match: (CMOVQLS y _ (FlagLT_ULT))
+	// cond:
+	// result: y
+	for {
+		_ = v.Args[2]
+		y := v.Args[0]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64FlagLT_ULT {
+			break
+		}
+		v.reset(OpCopy)
+		v.Type = y.Type
+		v.AddArg(y)
+		return true
+	}
+	return false
+}
+func rewriteValueAMD64_OpAMD64CMOVQLT_0(v *Value) bool {
+	// match: (CMOVQLT x y (InvertFlags cond))
+	// cond:
+	// result: (CMOVQGT x y cond)
+	for {
+		_ = v.Args[2]
+		x := v.Args[0]
+		y := v.Args[1]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64InvertFlags {
+			break
+		}
+		cond := v_2.Args[0]
+		v.reset(OpAMD64CMOVQGT)
+		v.AddArg(x)
+		v.AddArg(y)
+		v.AddArg(cond)
+		return true
+	}
+	// match: (CMOVQLT y _ (FlagEQ))
+	// cond:
+	// result: y
+	for {
+		_ = v.Args[2]
+		y := v.Args[0]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64FlagEQ {
+			break
+		}
+		v.reset(OpCopy)
+		v.Type = y.Type
+		v.AddArg(y)
+		return true
+	}
+	// match: (CMOVQLT y _ (FlagGT_UGT))
+	// cond:
+	// result: y
+	for {
+		_ = v.Args[2]
+		y := v.Args[0]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64FlagGT_UGT {
+			break
+		}
+		v.reset(OpCopy)
+		v.Type = y.Type
+		v.AddArg(y)
+		return true
+	}
+	// match: (CMOVQLT y _ (FlagGT_ULT))
+	// cond:
+	// result: y
+	for {
+		_ = v.Args[2]
+		y := v.Args[0]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64FlagGT_ULT {
+			break
+		}
+		v.reset(OpCopy)
+		v.Type = y.Type
+		v.AddArg(y)
+		return true
+	}
+	// match: (CMOVQLT _ x (FlagLT_ULT))
+	// cond:
+	// result: x
+	for {
+		_ = v.Args[2]
+		x := v.Args[1]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64FlagLT_ULT {
+			break
+		}
+		v.reset(OpCopy)
+		v.Type = x.Type
+		v.AddArg(x)
+		return true
+	}
+	// match: (CMOVQLT _ x (FlagLT_UGT))
+	// cond:
+	// result: x
+	for {
+		_ = v.Args[2]
+		x := v.Args[1]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64FlagLT_UGT {
+			break
+		}
+		v.reset(OpCopy)
+		v.Type = x.Type
+		v.AddArg(x)
+		return true
+	}
+	return false
+}
+func rewriteValueAMD64_OpAMD64CMOVQNE_0(v *Value) bool {
+	// match: (CMOVQNE x y (InvertFlags cond))
+	// cond:
+	// result: (CMOVQNE x y cond)
+	for {
+		_ = v.Args[2]
+		x := v.Args[0]
+		y := v.Args[1]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64InvertFlags {
+			break
+		}
+		cond := v_2.Args[0]
+		v.reset(OpAMD64CMOVQNE)
+		v.AddArg(x)
+		v.AddArg(y)
+		v.AddArg(cond)
+		return true
+	}
+	// match: (CMOVQNE y _ (FlagEQ))
+	// cond:
+	// result: y
+	for {
+		_ = v.Args[2]
+		y := v.Args[0]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64FlagEQ {
+			break
+		}
+		v.reset(OpCopy)
+		v.Type = y.Type
+		v.AddArg(y)
+		return true
+	}
+	// match: (CMOVQNE _ x (FlagGT_UGT))
+	// cond:
+	// result: x
+	for {
+		_ = v.Args[2]
+		x := v.Args[1]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64FlagGT_UGT {
+			break
+		}
+		v.reset(OpCopy)
+		v.Type = x.Type
+		v.AddArg(x)
+		return true
+	}
+	// match: (CMOVQNE _ x (FlagGT_ULT))
+	// cond:
+	// result: x
+	for {
+		_ = v.Args[2]
+		x := v.Args[1]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64FlagGT_ULT {
+			break
+		}
+		v.reset(OpCopy)
+		v.Type = x.Type
+		v.AddArg(x)
+		return true
+	}
+	// match: (CMOVQNE _ x (FlagLT_ULT))
+	// cond:
+	// result: x
+	for {
+		_ = v.Args[2]
+		x := v.Args[1]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64FlagLT_ULT {
+			break
+		}
+		v.reset(OpCopy)
+		v.Type = x.Type
+		v.AddArg(x)
+		return true
+	}
+	// match: (CMOVQNE _ x (FlagLT_UGT))
+	// cond:
+	// result: x
+	for {
+		_ = v.Args[2]
+		x := v.Args[1]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64FlagLT_UGT {
+			break
+		}
+		v.reset(OpCopy)
+		v.Type = x.Type
+		v.AddArg(x)
+		return true
+	}
+	return false
+}
+func rewriteValueAMD64_OpAMD64CMOVWCC_0(v *Value) bool {
+	// match: (CMOVWCC x y (InvertFlags cond))
+	// cond:
+	// result: (CMOVWLS x y cond)
+	for {
+		_ = v.Args[2]
+		x := v.Args[0]
+		y := v.Args[1]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64InvertFlags {
+			break
+		}
+		cond := v_2.Args[0]
+		v.reset(OpAMD64CMOVWLS)
+		v.AddArg(x)
+		v.AddArg(y)
+		v.AddArg(cond)
+		return true
+	}
+	return false
+}
+func rewriteValueAMD64_OpAMD64CMOVWCS_0(v *Value) bool {
+	// match: (CMOVWCS x y (InvertFlags cond))
+	// cond:
+	// result: (CMOVWHI x y cond)
+	for {
+		_ = v.Args[2]
+		x := v.Args[0]
+		y := v.Args[1]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64InvertFlags {
+			break
+		}
+		cond := v_2.Args[0]
+		v.reset(OpAMD64CMOVWHI)
+		v.AddArg(x)
+		v.AddArg(y)
+		v.AddArg(cond)
+		return true
+	}
+	return false
+}
+func rewriteValueAMD64_OpAMD64CMOVWEQ_0(v *Value) bool {
+	// match: (CMOVWEQ x y (InvertFlags cond))
+	// cond:
+	// result: (CMOVWEQ x y cond)
+	for {
+		_ = v.Args[2]
+		x := v.Args[0]
+		y := v.Args[1]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64InvertFlags {
+			break
+		}
+		cond := v_2.Args[0]
+		v.reset(OpAMD64CMOVWEQ)
+		v.AddArg(x)
+		v.AddArg(y)
+		v.AddArg(cond)
+		return true
+	}
+	return false
+}
+func rewriteValueAMD64_OpAMD64CMOVWGE_0(v *Value) bool {
+	// match: (CMOVWGE x y (InvertFlags cond))
+	// cond:
+	// result: (CMOVWLE x y cond)
+	for {
+		_ = v.Args[2]
+		x := v.Args[0]
+		y := v.Args[1]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64InvertFlags {
+			break
+		}
+		cond := v_2.Args[0]
+		v.reset(OpAMD64CMOVWLE)
+		v.AddArg(x)
+		v.AddArg(y)
+		v.AddArg(cond)
+		return true
+	}
+	return false
+}
+func rewriteValueAMD64_OpAMD64CMOVWGT_0(v *Value) bool {
+	// match: (CMOVWGT x y (InvertFlags cond))
+	// cond:
+	// result: (CMOVWLT x y cond)
+	for {
+		_ = v.Args[2]
+		x := v.Args[0]
+		y := v.Args[1]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64InvertFlags {
+			break
+		}
+		cond := v_2.Args[0]
+		v.reset(OpAMD64CMOVWLT)
+		v.AddArg(x)
+		v.AddArg(y)
+		v.AddArg(cond)
+		return true
+	}
+	return false
+}
+func rewriteValueAMD64_OpAMD64CMOVWHI_0(v *Value) bool {
+	// match: (CMOVWHI x y (InvertFlags cond))
+	// cond:
+	// result: (CMOVWCS x y cond)
+	for {
+		_ = v.Args[2]
+		x := v.Args[0]
+		y := v.Args[1]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64InvertFlags {
+			break
+		}
+		cond := v_2.Args[0]
+		v.reset(OpAMD64CMOVWCS)
+		v.AddArg(x)
+		v.AddArg(y)
+		v.AddArg(cond)
+		return true
+	}
+	return false
+}
+func rewriteValueAMD64_OpAMD64CMOVWLE_0(v *Value) bool {
+	// match: (CMOVWLE x y (InvertFlags cond))
+	// cond:
+	// result: (CMOVWGE x y cond)
+	for {
+		_ = v.Args[2]
+		x := v.Args[0]
+		y := v.Args[1]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64InvertFlags {
+			break
+		}
+		cond := v_2.Args[0]
+		v.reset(OpAMD64CMOVWGE)
+		v.AddArg(x)
+		v.AddArg(y)
+		v.AddArg(cond)
+		return true
+	}
+	return false
+}
+func rewriteValueAMD64_OpAMD64CMOVWLS_0(v *Value) bool {
+	// match: (CMOVWLS x y (InvertFlags cond))
+	// cond:
+	// result: (CMOVWCC x y cond)
+	for {
+		_ = v.Args[2]
+		x := v.Args[0]
+		y := v.Args[1]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64InvertFlags {
+			break
+		}
+		cond := v_2.Args[0]
+		v.reset(OpAMD64CMOVWCC)
+		v.AddArg(x)
+		v.AddArg(y)
+		v.AddArg(cond)
+		return true
+	}
+	return false
+}
+func rewriteValueAMD64_OpAMD64CMOVWLT_0(v *Value) bool {
+	// match: (CMOVWLT x y (InvertFlags cond))
+	// cond:
+	// result: (CMOVWGT x y cond)
+	for {
+		_ = v.Args[2]
+		x := v.Args[0]
+		y := v.Args[1]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64InvertFlags {
+			break
+		}
+		cond := v_2.Args[0]
+		v.reset(OpAMD64CMOVWGT)
+		v.AddArg(x)
+		v.AddArg(y)
+		v.AddArg(cond)
+		return true
+	}
+	return false
+}
+func rewriteValueAMD64_OpAMD64CMOVWNE_0(v *Value) bool {
+	// match: (CMOVWNE x y (InvertFlags cond))
+	// cond:
+	// result: (CMOVWNE x y cond)
+	for {
+		_ = v.Args[2]
+		x := v.Args[0]
+		y := v.Args[1]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64InvertFlags {
+			break
+		}
+		cond := v_2.Args[0]
+		v.reset(OpAMD64CMOVWNE)
+		v.AddArg(x)
+		v.AddArg(y)
+		v.AddArg(cond)
+		return true
+	}
+	return false
+}
 func rewriteValueAMD64_OpAMD64CMPB_0(v *Value) bool {
 	b := v.Block
 	_ = b
@@ -42931,6 +44376,22 @@ func rewriteValueAMD64_OpAMD64TESTB_0(v *Value) bool {
 	}
 	return false
 }
+func rewriteValueAMD64_OpAMD64TESTBconst_0(v *Value) bool {
+	// match: (TESTBconst [-1] x)
+	// cond:
+	// result: (TESTB x x)
+	for {
+		if v.AuxInt != -1 {
+			break
+		}
+		x := v.Args[0]
+		v.reset(OpAMD64TESTB)
+		v.AddArg(x)
+		v.AddArg(x)
+		return true
+	}
+	return false
+}
 func rewriteValueAMD64_OpAMD64TESTL_0(v *Value) bool {
 	b := v.Block
 	_ = b
@@ -43024,6 +44485,22 @@ func rewriteValueAMD64_OpAMD64TESTL_0(v *Value) bool {
 	}
 	return false
 }
+func rewriteValueAMD64_OpAMD64TESTLconst_0(v *Value) bool {
+	// match: (TESTLconst [-1] x)
+	// cond:
+	// result: (TESTL x x)
+	for {
+		if v.AuxInt != -1 {
+			break
+		}
+		x := v.Args[0]
+		v.reset(OpAMD64TESTL)
+		v.AddArg(x)
+		v.AddArg(x)
+		return true
+	}
+	return false
+}
 func rewriteValueAMD64_OpAMD64TESTQ_0(v *Value) bool {
 	b := v.Block
 	_ = b
@@ -43123,6 +44600,22 @@ func rewriteValueAMD64_OpAMD64TESTQ_0(v *Value) bool {
 	}
 	return false
 }
+func rewriteValueAMD64_OpAMD64TESTQconst_0(v *Value) bool {
+	// match: (TESTQconst [-1] x)
+	// cond:
+	// result: (TESTQ x x)
+	for {
+		if v.AuxInt != -1 {
+			break
+		}
+		x := v.Args[0]
+		v.reset(OpAMD64TESTQ)
+		v.AddArg(x)
+		v.AddArg(x)
+		return true
+	}
+	return false
+}
 func rewriteValueAMD64_OpAMD64TESTW_0(v *Value) bool {
 	b := v.Block
 	_ = b
@@ -43216,6 +44709,22 @@ func rewriteValueAMD64_OpAMD64TESTW_0(v *Value) bool {
 	}
 	return false
 }
+func rewriteValueAMD64_OpAMD64TESTWconst_0(v *Value) bool {
+	// match: (TESTWconst [-1] x)
+	// cond:
+	// result: (TESTW x x)
+	for {
+		if v.AuxInt != -1 {
+			break
+		}
+		x := v.Args[0]
+		v.reset(OpAMD64TESTW)
+		v.AddArg(x)
+		v.AddArg(x)
+		return true
+	}
+	return false
+}
 func rewriteValueAMD64_OpAMD64XADDLlock_0(v *Value) bool {
 	// match: (XADDLlock [off1] {sym} val (ADDQconst [off2] ptr) mem)
 	// cond: is32Bit(off1+off2)
@@ -44939,6 +46448,1075 @@ func rewriteValueAMD64_OpCom8_0(v *Value) bool {
 		return true
 	}
 }
+func rewriteValueAMD64_OpCondSelect_0(v *Value) bool {
+	// match: (CondSelect <t> x y (SETEQ cond))
+	// cond: (is64BitInt(t) || isPtr(t))
+	// result: (CMOVQEQ y x cond)
+	for {
+		t := v.Type
+		_ = v.Args[2]
+		x := v.Args[0]
+		y := v.Args[1]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64SETEQ {
+			break
+		}
+		cond := v_2.Args[0]
+		if !(is64BitInt(t) || isPtr(t)) {
+			break
+		}
+		v.reset(OpAMD64CMOVQEQ)
+		v.AddArg(y)
+		v.AddArg(x)
+		v.AddArg(cond)
+		return true
+	}
+	// match: (CondSelect <t> x y (SETNE cond))
+	// cond: (is64BitInt(t) || isPtr(t))
+	// result: (CMOVQNE y x cond)
+	for {
+		t := v.Type
+		_ = v.Args[2]
+		x := v.Args[0]
+		y := v.Args[1]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64SETNE {
+			break
+		}
+		cond := v_2.Args[0]
+		if !(is64BitInt(t) || isPtr(t)) {
+			break
+		}
+		v.reset(OpAMD64CMOVQNE)
+		v.AddArg(y)
+		v.AddArg(x)
+		v.AddArg(cond)
+		return true
+	}
+	// match: (CondSelect <t> x y (SETL cond))
+	// cond: (is64BitInt(t) || isPtr(t))
+	// result: (CMOVQLT y x cond)
+	for {
+		t := v.Type
+		_ = v.Args[2]
+		x := v.Args[0]
+		y := v.Args[1]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64SETL {
+			break
+		}
+		cond := v_2.Args[0]
+		if !(is64BitInt(t) || isPtr(t)) {
+			break
+		}
+		v.reset(OpAMD64CMOVQLT)
+		v.AddArg(y)
+		v.AddArg(x)
+		v.AddArg(cond)
+		return true
+	}
+	// match: (CondSelect <t> x y (SETG cond))
+	// cond: (is64BitInt(t) || isPtr(t))
+	// result: (CMOVQGT y x cond)
+	for {
+		t := v.Type
+		_ = v.Args[2]
+		x := v.Args[0]
+		y := v.Args[1]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64SETG {
+			break
+		}
+		cond := v_2.Args[0]
+		if !(is64BitInt(t) || isPtr(t)) {
+			break
+		}
+		v.reset(OpAMD64CMOVQGT)
+		v.AddArg(y)
+		v.AddArg(x)
+		v.AddArg(cond)
+		return true
+	}
+	// match: (CondSelect <t> x y (SETLE cond))
+	// cond: (is64BitInt(t) || isPtr(t))
+	// result: (CMOVQLE y x cond)
+	for {
+		t := v.Type
+		_ = v.Args[2]
+		x := v.Args[0]
+		y := v.Args[1]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64SETLE {
+			break
+		}
+		cond := v_2.Args[0]
+		if !(is64BitInt(t) || isPtr(t)) {
+			break
+		}
+		v.reset(OpAMD64CMOVQLE)
+		v.AddArg(y)
+		v.AddArg(x)
+		v.AddArg(cond)
+		return true
+	}
+	// match: (CondSelect <t> x y (SETGE cond))
+	// cond: (is64BitInt(t) || isPtr(t))
+	// result: (CMOVQGE y x cond)
+	for {
+		t := v.Type
+		_ = v.Args[2]
+		x := v.Args[0]
+		y := v.Args[1]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64SETGE {
+			break
+		}
+		cond := v_2.Args[0]
+		if !(is64BitInt(t) || isPtr(t)) {
+			break
+		}
+		v.reset(OpAMD64CMOVQGE)
+		v.AddArg(y)
+		v.AddArg(x)
+		v.AddArg(cond)
+		return true
+	}
+	// match: (CondSelect <t> x y (SETA cond))
+	// cond: (is64BitInt(t) || isPtr(t))
+	// result: (CMOVQHI y x cond)
+	for {
+		t := v.Type
+		_ = v.Args[2]
+		x := v.Args[0]
+		y := v.Args[1]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64SETA {
+			break
+		}
+		cond := v_2.Args[0]
+		if !(is64BitInt(t) || isPtr(t)) {
+			break
+		}
+		v.reset(OpAMD64CMOVQHI)
+		v.AddArg(y)
+		v.AddArg(x)
+		v.AddArg(cond)
+		return true
+	}
+	// match: (CondSelect <t> x y (SETB cond))
+	// cond: (is64BitInt(t) || isPtr(t))
+	// result: (CMOVQCS y x cond)
+	for {
+		t := v.Type
+		_ = v.Args[2]
+		x := v.Args[0]
+		y := v.Args[1]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64SETB {
+			break
+		}
+		cond := v_2.Args[0]
+		if !(is64BitInt(t) || isPtr(t)) {
+			break
+		}
+		v.reset(OpAMD64CMOVQCS)
+		v.AddArg(y)
+		v.AddArg(x)
+		v.AddArg(cond)
+		return true
+	}
+	// match: (CondSelect <t> x y (SETAE cond))
+	// cond: (is64BitInt(t) || isPtr(t))
+	// result: (CMOVQCC y x cond)
+	for {
+		t := v.Type
+		_ = v.Args[2]
+		x := v.Args[0]
+		y := v.Args[1]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64SETAE {
+			break
+		}
+		cond := v_2.Args[0]
+		if !(is64BitInt(t) || isPtr(t)) {
+			break
+		}
+		v.reset(OpAMD64CMOVQCC)
+		v.AddArg(y)
+		v.AddArg(x)
+		v.AddArg(cond)
+		return true
+	}
+	// match: (CondSelect <t> x y (SETBE cond))
+	// cond: (is64BitInt(t) || isPtr(t))
+	// result: (CMOVQLS y x cond)
+	for {
+		t := v.Type
+		_ = v.Args[2]
+		x := v.Args[0]
+		y := v.Args[1]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64SETBE {
+			break
+		}
+		cond := v_2.Args[0]
+		if !(is64BitInt(t) || isPtr(t)) {
+			break
+		}
+		v.reset(OpAMD64CMOVQLS)
+		v.AddArg(y)
+		v.AddArg(x)
+		v.AddArg(cond)
+		return true
+	}
+	return false
+}
+func rewriteValueAMD64_OpCondSelect_10(v *Value) bool {
+	// match: (CondSelect <t> x y (SETEQF cond))
+	// cond: (is64BitInt(t) || isPtr(t))
+	// result: (CMOVQEQF y x cond)
+	for {
+		t := v.Type
+		_ = v.Args[2]
+		x := v.Args[0]
+		y := v.Args[1]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64SETEQF {
+			break
+		}
+		cond := v_2.Args[0]
+		if !(is64BitInt(t) || isPtr(t)) {
+			break
+		}
+		v.reset(OpAMD64CMOVQEQF)
+		v.AddArg(y)
+		v.AddArg(x)
+		v.AddArg(cond)
+		return true
+	}
+	// match: (CondSelect <t> x y (SETNEF cond))
+	// cond: (is64BitInt(t) || isPtr(t))
+	// result: (CMOVQNEF y x cond)
+	for {
+		t := v.Type
+		_ = v.Args[2]
+		x := v.Args[0]
+		y := v.Args[1]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64SETNEF {
+			break
+		}
+		cond := v_2.Args[0]
+		if !(is64BitInt(t) || isPtr(t)) {
+			break
+		}
+		v.reset(OpAMD64CMOVQNEF)
+		v.AddArg(y)
+		v.AddArg(x)
+		v.AddArg(cond)
+		return true
+	}
+	// match: (CondSelect <t> x y (SETGF cond))
+	// cond: (is64BitInt(t) || isPtr(t))
+	// result: (CMOVQGTF y x cond)
+	for {
+		t := v.Type
+		_ = v.Args[2]
+		x := v.Args[0]
+		y := v.Args[1]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64SETGF {
+			break
+		}
+		cond := v_2.Args[0]
+		if !(is64BitInt(t) || isPtr(t)) {
+			break
+		}
+		v.reset(OpAMD64CMOVQGTF)
+		v.AddArg(y)
+		v.AddArg(x)
+		v.AddArg(cond)
+		return true
+	}
+	// match: (CondSelect <t> x y (SETGEF cond))
+	// cond: (is64BitInt(t) || isPtr(t))
+	// result: (CMOVQGEF y x cond)
+	for {
+		t := v.Type
+		_ = v.Args[2]
+		x := v.Args[0]
+		y := v.Args[1]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64SETGEF {
+			break
+		}
+		cond := v_2.Args[0]
+		if !(is64BitInt(t) || isPtr(t)) {
+			break
+		}
+		v.reset(OpAMD64CMOVQGEF)
+		v.AddArg(y)
+		v.AddArg(x)
+		v.AddArg(cond)
+		return true
+	}
+	// match: (CondSelect <t> x y (SETEQ cond))
+	// cond: is32BitInt(t)
+	// result: (CMOVLEQ y x cond)
+	for {
+		t := v.Type
+		_ = v.Args[2]
+		x := v.Args[0]
+		y := v.Args[1]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64SETEQ {
+			break
+		}
+		cond := v_2.Args[0]
+		if !(is32BitInt(t)) {
+			break
+		}
+		v.reset(OpAMD64CMOVLEQ)
+		v.AddArg(y)
+		v.AddArg(x)
+		v.AddArg(cond)
+		return true
+	}
+	// match: (CondSelect <t> x y (SETNE cond))
+	// cond: is32BitInt(t)
+	// result: (CMOVLNE y x cond)
+	for {
+		t := v.Type
+		_ = v.Args[2]
+		x := v.Args[0]
+		y := v.Args[1]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64SETNE {
+			break
+		}
+		cond := v_2.Args[0]
+		if !(is32BitInt(t)) {
+			break
+		}
+		v.reset(OpAMD64CMOVLNE)
+		v.AddArg(y)
+		v.AddArg(x)
+		v.AddArg(cond)
+		return true
+	}
+	// match: (CondSelect <t> x y (SETL cond))
+	// cond: is32BitInt(t)
+	// result: (CMOVLLT y x cond)
+	for {
+		t := v.Type
+		_ = v.Args[2]
+		x := v.Args[0]
+		y := v.Args[1]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64SETL {
+			break
+		}
+		cond := v_2.Args[0]
+		if !(is32BitInt(t)) {
+			break
+		}
+		v.reset(OpAMD64CMOVLLT)
+		v.AddArg(y)
+		v.AddArg(x)
+		v.AddArg(cond)
+		return true
+	}
+	// match: (CondSelect <t> x y (SETG cond))
+	// cond: is32BitInt(t)
+	// result: (CMOVLGT y x cond)
+	for {
+		t := v.Type
+		_ = v.Args[2]
+		x := v.Args[0]
+		y := v.Args[1]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64SETG {
+			break
+		}
+		cond := v_2.Args[0]
+		if !(is32BitInt(t)) {
+			break
+		}
+		v.reset(OpAMD64CMOVLGT)
+		v.AddArg(y)
+		v.AddArg(x)
+		v.AddArg(cond)
+		return true
+	}
+	// match: (CondSelect <t> x y (SETLE cond))
+	// cond: is32BitInt(t)
+	// result: (CMOVLLE y x cond)
+	for {
+		t := v.Type
+		_ = v.Args[2]
+		x := v.Args[0]
+		y := v.Args[1]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64SETLE {
+			break
+		}
+		cond := v_2.Args[0]
+		if !(is32BitInt(t)) {
+			break
+		}
+		v.reset(OpAMD64CMOVLLE)
+		v.AddArg(y)
+		v.AddArg(x)
+		v.AddArg(cond)
+		return true
+	}
+	// match: (CondSelect <t> x y (SETGE cond))
+	// cond: is32BitInt(t)
+	// result: (CMOVLGE y x cond)
+	for {
+		t := v.Type
+		_ = v.Args[2]
+		x := v.Args[0]
+		y := v.Args[1]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64SETGE {
+			break
+		}
+		cond := v_2.Args[0]
+		if !(is32BitInt(t)) {
+			break
+		}
+		v.reset(OpAMD64CMOVLGE)
+		v.AddArg(y)
+		v.AddArg(x)
+		v.AddArg(cond)
+		return true
+	}
+	return false
+}
+func rewriteValueAMD64_OpCondSelect_20(v *Value) bool {
+	// match: (CondSelect <t> x y (SETA cond))
+	// cond: is32BitInt(t)
+	// result: (CMOVLHI y x cond)
+	for {
+		t := v.Type
+		_ = v.Args[2]
+		x := v.Args[0]
+		y := v.Args[1]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64SETA {
+			break
+		}
+		cond := v_2.Args[0]
+		if !(is32BitInt(t)) {
+			break
+		}
+		v.reset(OpAMD64CMOVLHI)
+		v.AddArg(y)
+		v.AddArg(x)
+		v.AddArg(cond)
+		return true
+	}
+	// match: (CondSelect <t> x y (SETB cond))
+	// cond: is32BitInt(t)
+	// result: (CMOVLCS y x cond)
+	for {
+		t := v.Type
+		_ = v.Args[2]
+		x := v.Args[0]
+		y := v.Args[1]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64SETB {
+			break
+		}
+		cond := v_2.Args[0]
+		if !(is32BitInt(t)) {
+			break
+		}
+		v.reset(OpAMD64CMOVLCS)
+		v.AddArg(y)
+		v.AddArg(x)
+		v.AddArg(cond)
+		return true
+	}
+	// match: (CondSelect <t> x y (SETAE cond))
+	// cond: is32BitInt(t)
+	// result: (CMOVLCC y x cond)
+	for {
+		t := v.Type
+		_ = v.Args[2]
+		x := v.Args[0]
+		y := v.Args[1]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64SETAE {
+			break
+		}
+		cond := v_2.Args[0]
+		if !(is32BitInt(t)) {
+			break
+		}
+		v.reset(OpAMD64CMOVLCC)
+		v.AddArg(y)
+		v.AddArg(x)
+		v.AddArg(cond)
+		return true
+	}
+	// match: (CondSelect <t> x y (SETBE cond))
+	// cond: is32BitInt(t)
+	// result: (CMOVLLS y x cond)
+	for {
+		t := v.Type
+		_ = v.Args[2]
+		x := v.Args[0]
+		y := v.Args[1]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64SETBE {
+			break
+		}
+		cond := v_2.Args[0]
+		if !(is32BitInt(t)) {
+			break
+		}
+		v.reset(OpAMD64CMOVLLS)
+		v.AddArg(y)
+		v.AddArg(x)
+		v.AddArg(cond)
+		return true
+	}
+	// match: (CondSelect <t> x y (SETEQF cond))
+	// cond: is32BitInt(t)
+	// result: (CMOVLEQF y x cond)
+	for {
+		t := v.Type
+		_ = v.Args[2]
+		x := v.Args[0]
+		y := v.Args[1]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64SETEQF {
+			break
+		}
+		cond := v_2.Args[0]
+		if !(is32BitInt(t)) {
+			break
+		}
+		v.reset(OpAMD64CMOVLEQF)
+		v.AddArg(y)
+		v.AddArg(x)
+		v.AddArg(cond)
+		return true
+	}
+	// match: (CondSelect <t> x y (SETNEF cond))
+	// cond: is32BitInt(t)
+	// result: (CMOVLNEF y x cond)
+	for {
+		t := v.Type
+		_ = v.Args[2]
+		x := v.Args[0]
+		y := v.Args[1]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64SETNEF {
+			break
+		}
+		cond := v_2.Args[0]
+		if !(is32BitInt(t)) {
+			break
+		}
+		v.reset(OpAMD64CMOVLNEF)
+		v.AddArg(y)
+		v.AddArg(x)
+		v.AddArg(cond)
+		return true
+	}
+	// match: (CondSelect <t> x y (SETGF cond))
+	// cond: is32BitInt(t)
+	// result: (CMOVLGTF y x cond)
+	for {
+		t := v.Type
+		_ = v.Args[2]
+		x := v.Args[0]
+		y := v.Args[1]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64SETGF {
+			break
+		}
+		cond := v_2.Args[0]
+		if !(is32BitInt(t)) {
+			break
+		}
+		v.reset(OpAMD64CMOVLGTF)
+		v.AddArg(y)
+		v.AddArg(x)
+		v.AddArg(cond)
+		return true
+	}
+	// match: (CondSelect <t> x y (SETGEF cond))
+	// cond: is32BitInt(t)
+	// result: (CMOVLGEF y x cond)
+	for {
+		t := v.Type
+		_ = v.Args[2]
+		x := v.Args[0]
+		y := v.Args[1]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64SETGEF {
+			break
+		}
+		cond := v_2.Args[0]
+		if !(is32BitInt(t)) {
+			break
+		}
+		v.reset(OpAMD64CMOVLGEF)
+		v.AddArg(y)
+		v.AddArg(x)
+		v.AddArg(cond)
+		return true
+	}
+	// match: (CondSelect <t> x y (SETEQ cond))
+	// cond: is16BitInt(t)
+	// result: (CMOVWEQ y x cond)
+	for {
+		t := v.Type
+		_ = v.Args[2]
+		x := v.Args[0]
+		y := v.Args[1]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64SETEQ {
+			break
+		}
+		cond := v_2.Args[0]
+		if !(is16BitInt(t)) {
+			break
+		}
+		v.reset(OpAMD64CMOVWEQ)
+		v.AddArg(y)
+		v.AddArg(x)
+		v.AddArg(cond)
+		return true
+	}
+	// match: (CondSelect <t> x y (SETNE cond))
+	// cond: is16BitInt(t)
+	// result: (CMOVWNE y x cond)
+	for {
+		t := v.Type
+		_ = v.Args[2]
+		x := v.Args[0]
+		y := v.Args[1]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64SETNE {
+			break
+		}
+		cond := v_2.Args[0]
+		if !(is16BitInt(t)) {
+			break
+		}
+		v.reset(OpAMD64CMOVWNE)
+		v.AddArg(y)
+		v.AddArg(x)
+		v.AddArg(cond)
+		return true
+	}
+	return false
+}
+func rewriteValueAMD64_OpCondSelect_30(v *Value) bool {
+	// match: (CondSelect <t> x y (SETL cond))
+	// cond: is16BitInt(t)
+	// result: (CMOVWLT y x cond)
+	for {
+		t := v.Type
+		_ = v.Args[2]
+		x := v.Args[0]
+		y := v.Args[1]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64SETL {
+			break
+		}
+		cond := v_2.Args[0]
+		if !(is16BitInt(t)) {
+			break
+		}
+		v.reset(OpAMD64CMOVWLT)
+		v.AddArg(y)
+		v.AddArg(x)
+		v.AddArg(cond)
+		return true
+	}
+	// match: (CondSelect <t> x y (SETG cond))
+	// cond: is16BitInt(t)
+	// result: (CMOVWGT y x cond)
+	for {
+		t := v.Type
+		_ = v.Args[2]
+		x := v.Args[0]
+		y := v.Args[1]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64SETG {
+			break
+		}
+		cond := v_2.Args[0]
+		if !(is16BitInt(t)) {
+			break
+		}
+		v.reset(OpAMD64CMOVWGT)
+		v.AddArg(y)
+		v.AddArg(x)
+		v.AddArg(cond)
+		return true
+	}
+	// match: (CondSelect <t> x y (SETLE cond))
+	// cond: is16BitInt(t)
+	// result: (CMOVWLE y x cond)
+	for {
+		t := v.Type
+		_ = v.Args[2]
+		x := v.Args[0]
+		y := v.Args[1]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64SETLE {
+			break
+		}
+		cond := v_2.Args[0]
+		if !(is16BitInt(t)) {
+			break
+		}
+		v.reset(OpAMD64CMOVWLE)
+		v.AddArg(y)
+		v.AddArg(x)
+		v.AddArg(cond)
+		return true
+	}
+	// match: (CondSelect <t> x y (SETGE cond))
+	// cond: is16BitInt(t)
+	// result: (CMOVWGE y x cond)
+	for {
+		t := v.Type
+		_ = v.Args[2]
+		x := v.Args[0]
+		y := v.Args[1]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64SETGE {
+			break
+		}
+		cond := v_2.Args[0]
+		if !(is16BitInt(t)) {
+			break
+		}
+		v.reset(OpAMD64CMOVWGE)
+		v.AddArg(y)
+		v.AddArg(x)
+		v.AddArg(cond)
+		return true
+	}
+	// match: (CondSelect <t> x y (SETA cond))
+	// cond: is16BitInt(t)
+	// result: (CMOVWHI y x cond)
+	for {
+		t := v.Type
+		_ = v.Args[2]
+		x := v.Args[0]
+		y := v.Args[1]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64SETA {
+			break
+		}
+		cond := v_2.Args[0]
+		if !(is16BitInt(t)) {
+			break
+		}
+		v.reset(OpAMD64CMOVWHI)
+		v.AddArg(y)
+		v.AddArg(x)
+		v.AddArg(cond)
+		return true
+	}
+	// match: (CondSelect <t> x y (SETB cond))
+	// cond: is16BitInt(t)
+	// result: (CMOVWCS y x cond)
+	for {
+		t := v.Type
+		_ = v.Args[2]
+		x := v.Args[0]
+		y := v.Args[1]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64SETB {
+			break
+		}
+		cond := v_2.Args[0]
+		if !(is16BitInt(t)) {
+			break
+		}
+		v.reset(OpAMD64CMOVWCS)
+		v.AddArg(y)
+		v.AddArg(x)
+		v.AddArg(cond)
+		return true
+	}
+	// match: (CondSelect <t> x y (SETAE cond))
+	// cond: is16BitInt(t)
+	// result: (CMOVWCC y x cond)
+	for {
+		t := v.Type
+		_ = v.Args[2]
+		x := v.Args[0]
+		y := v.Args[1]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64SETAE {
+			break
+		}
+		cond := v_2.Args[0]
+		if !(is16BitInt(t)) {
+			break
+		}
+		v.reset(OpAMD64CMOVWCC)
+		v.AddArg(y)
+		v.AddArg(x)
+		v.AddArg(cond)
+		return true
+	}
+	// match: (CondSelect <t> x y (SETBE cond))
+	// cond: is16BitInt(t)
+	// result: (CMOVWLS y x cond)
+	for {
+		t := v.Type
+		_ = v.Args[2]
+		x := v.Args[0]
+		y := v.Args[1]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64SETBE {
+			break
+		}
+		cond := v_2.Args[0]
+		if !(is16BitInt(t)) {
+			break
+		}
+		v.reset(OpAMD64CMOVWLS)
+		v.AddArg(y)
+		v.AddArg(x)
+		v.AddArg(cond)
+		return true
+	}
+	// match: (CondSelect <t> x y (SETEQF cond))
+	// cond: is16BitInt(t)
+	// result: (CMOVWEQF y x cond)
+	for {
+		t := v.Type
+		_ = v.Args[2]
+		x := v.Args[0]
+		y := v.Args[1]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64SETEQF {
+			break
+		}
+		cond := v_2.Args[0]
+		if !(is16BitInt(t)) {
+			break
+		}
+		v.reset(OpAMD64CMOVWEQF)
+		v.AddArg(y)
+		v.AddArg(x)
+		v.AddArg(cond)
+		return true
+	}
+	// match: (CondSelect <t> x y (SETNEF cond))
+	// cond: is16BitInt(t)
+	// result: (CMOVWNEF y x cond)
+	for {
+		t := v.Type
+		_ = v.Args[2]
+		x := v.Args[0]
+		y := v.Args[1]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64SETNEF {
+			break
+		}
+		cond := v_2.Args[0]
+		if !(is16BitInt(t)) {
+			break
+		}
+		v.reset(OpAMD64CMOVWNEF)
+		v.AddArg(y)
+		v.AddArg(x)
+		v.AddArg(cond)
+		return true
+	}
+	return false
+}
+func rewriteValueAMD64_OpCondSelect_40(v *Value) bool {
+	b := v.Block
+	_ = b
+	typ := &b.Func.Config.Types
+	_ = typ
+	// match: (CondSelect <t> x y (SETGF cond))
+	// cond: is16BitInt(t)
+	// result: (CMOVWGTF y x cond)
+	for {
+		t := v.Type
+		_ = v.Args[2]
+		x := v.Args[0]
+		y := v.Args[1]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64SETGF {
+			break
+		}
+		cond := v_2.Args[0]
+		if !(is16BitInt(t)) {
+			break
+		}
+		v.reset(OpAMD64CMOVWGTF)
+		v.AddArg(y)
+		v.AddArg(x)
+		v.AddArg(cond)
+		return true
+	}
+	// match: (CondSelect <t> x y (SETGEF cond))
+	// cond: is16BitInt(t)
+	// result: (CMOVWGEF y x cond)
+	for {
+		t := v.Type
+		_ = v.Args[2]
+		x := v.Args[0]
+		y := v.Args[1]
+		v_2 := v.Args[2]
+		if v_2.Op != OpAMD64SETGEF {
+			break
+		}
+		cond := v_2.Args[0]
+		if !(is16BitInt(t)) {
+			break
+		}
+		v.reset(OpAMD64CMOVWGEF)
+		v.AddArg(y)
+		v.AddArg(x)
+		v.AddArg(cond)
+		return true
+	}
+	// match: (CondSelect <t> x y check)
+	// cond: !check.Type.IsFlags() && check.Type.Size() == 1
+	// result: (CondSelect <t> x y (MOVBQZX <typ.UInt64> check))
+	for {
+		t := v.Type
+		_ = v.Args[2]
+		x := v.Args[0]
+		y := v.Args[1]
+		check := v.Args[2]
+		if !(!check.Type.IsFlags() && check.Type.Size() == 1) {
+			break
+		}
+		v.reset(OpCondSelect)
+		v.Type = t
+		v.AddArg(x)
+		v.AddArg(y)
+		v0 := b.NewValue0(v.Pos, OpAMD64MOVBQZX, typ.UInt64)
+		v0.AddArg(check)
+		v.AddArg(v0)
+		return true
+	}
+	// match: (CondSelect <t> x y check)
+	// cond: !check.Type.IsFlags() && check.Type.Size() == 2
+	// result: (CondSelect <t> x y (MOVWQZX <typ.UInt64> check))
+	for {
+		t := v.Type
+		_ = v.Args[2]
+		x := v.Args[0]
+		y := v.Args[1]
+		check := v.Args[2]
+		if !(!check.Type.IsFlags() && check.Type.Size() == 2) {
+			break
+		}
+		v.reset(OpCondSelect)
+		v.Type = t
+		v.AddArg(x)
+		v.AddArg(y)
+		v0 := b.NewValue0(v.Pos, OpAMD64MOVWQZX, typ.UInt64)
+		v0.AddArg(check)
+		v.AddArg(v0)
+		return true
+	}
+	// match: (CondSelect <t> x y check)
+	// cond: !check.Type.IsFlags() && check.Type.Size() == 4
+	// result: (CondSelect <t> x y (MOVLQZX <typ.UInt64> check))
+	for {
+		t := v.Type
+		_ = v.Args[2]
+		x := v.Args[0]
+		y := v.Args[1]
+		check := v.Args[2]
+		if !(!check.Type.IsFlags() && check.Type.Size() == 4) {
+			break
+		}
+		v.reset(OpCondSelect)
+		v.Type = t
+		v.AddArg(x)
+		v.AddArg(y)
+		v0 := b.NewValue0(v.Pos, OpAMD64MOVLQZX, typ.UInt64)
+		v0.AddArg(check)
+		v.AddArg(v0)
+		return true
+	}
+	// match: (CondSelect <t> x y check)
+	// cond: !check.Type.IsFlags() && check.Type.Size() == 8 && (is64BitInt(t) || isPtr(t))
+	// result: (CMOVQNE y x (CMPQconst [0] check))
+	for {
+		t := v.Type
+		_ = v.Args[2]
+		x := v.Args[0]
+		y := v.Args[1]
+		check := v.Args[2]
+		if !(!check.Type.IsFlags() && check.Type.Size() == 8 && (is64BitInt(t) || isPtr(t))) {
+			break
+		}
+		v.reset(OpAMD64CMOVQNE)
+		v.AddArg(y)
+		v.AddArg(x)
+		v0 := b.NewValue0(v.Pos, OpAMD64CMPQconst, types.TypeFlags)
+		v0.AuxInt = 0
+		v0.AddArg(check)
+		v.AddArg(v0)
+		return true
+	}
+	// match: (CondSelect <t> x y check)
+	// cond: !check.Type.IsFlags() && check.Type.Size() == 8 && is32BitInt(t)
+	// result: (CMOVLNE y x (CMPQconst [0] check))
+	for {
+		t := v.Type
+		_ = v.Args[2]
+		x := v.Args[0]
+		y := v.Args[1]
+		check := v.Args[2]
+		if !(!check.Type.IsFlags() && check.Type.Size() == 8 && is32BitInt(t)) {
+			break
+		}
+		v.reset(OpAMD64CMOVLNE)
+		v.AddArg(y)
+		v.AddArg(x)
+		v0 := b.NewValue0(v.Pos, OpAMD64CMPQconst, types.TypeFlags)
+		v0.AuxInt = 0
+		v0.AddArg(check)
+		v.AddArg(v0)
+		return true
+	}
+	// match: (CondSelect <t> x y check)
+	// cond: !check.Type.IsFlags() && check.Type.Size() == 8 && is16BitInt(t)
+	// result: (CMOVWNE y x (CMPQconst [0] check))
+	for {
+		t := v.Type
+		_ = v.Args[2]
+		x := v.Args[0]
+		y := v.Args[1]
+		check := v.Args[2]
+		if !(!check.Type.IsFlags() && check.Type.Size() == 8 && is16BitInt(t)) {
+			break
+		}
+		v.reset(OpAMD64CMOVWNE)
+		v.AddArg(y)
+		v.AddArg(x)
+		v0 := b.NewValue0(v.Pos, OpAMD64CMPQconst, types.TypeFlags)
+		v0.AuxInt = 0
+		v0.AddArg(check)
+		v.AddArg(v0)
+		return true
+	}
+	return false
+}
 func rewriteValueAMD64_OpConst16_0(v *Value) bool {
 	// match: (Const16 [val])
 	// cond: