cmd/compile: restrict bit test rewrite rules

The {AND,OR,XOR}const ops can only take an int32 as an argument.
Make sure that when rewriting a BTx op to one of these, the result
has no high-order bits.

Fixes #38746

Change-Id: Ia7c5f76952329f60974bc033c29a5433610f3b28
Reviewed-on: https://go-review.googlesource.com/c/go/+/231977
Run-TryBot: Keith Randall <khr@golang.org>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Josh Bleecher Snyder <josharian@gmail.com>

1 files changed, 111 insertions, 75 deletions

diff --git a/src/cmd/compile/internal/ssa/rewriteAMD64.go b/src/cmd/compile/internal/ssa/rewriteAMD64.go
index 5f3d4e5b90..20eab05e9c 100644
--- a/src/cmd/compile/internal/ssa/rewriteAMD64.go
+++ b/src/cmd/compile/internal/ssa/rewriteAMD64.go
@@ -2797,28 +2797,28 @@ func rewriteValueAMD64_OpAMD64ANDLconst(v *Value) bool {
 	// match: (ANDLconst [c] (ANDLconst [d] x))
 	// result: (ANDLconst [c & d] x)
 	for {
-		c := v.AuxInt
+		c := auxIntToInt32(v.AuxInt)
 		if v_0.Op != OpAMD64ANDLconst {
 			break
 		}
-		d := v_0.AuxInt
+		d := auxIntToInt32(v_0.AuxInt)
 		x := v_0.Args[0]
 		v.reset(OpAMD64ANDLconst)
-		v.AuxInt = c & d
+		v.AuxInt = int32ToAuxInt(c & d)
 		v.AddArg(x)
 		return true
 	}
 	// match: (ANDLconst [c] (BTRLconst [d] x))
 	// result: (ANDLconst [c &^ (1<<uint32(d))] x)
 	for {
-		c := v.AuxInt
+		c := auxIntToInt32(v.AuxInt)
 		if v_0.Op != OpAMD64BTRLconst {
 			break
 		}
-		d := v_0.AuxInt
+		d := auxIntToInt8(v_0.AuxInt)
 		x := v_0.Args[0]
 		v.reset(OpAMD64ANDLconst)
-		v.AuxInt = c &^ (1 << uint32(d))
+		v.AuxInt = int32ToAuxInt(c &^ (1 << uint32(d)))
 		v.AddArg(x)
 		return true
 	}
@@ -3176,28 +3176,32 @@ func rewriteValueAMD64_OpAMD64ANDQconst(v *Value) bool {
 	// match: (ANDQconst [c] (ANDQconst [d] x))
 	// result: (ANDQconst [c & d] x)
 	for {
-		c := v.AuxInt
+		c := auxIntToInt32(v.AuxInt)
 		if v_0.Op != OpAMD64ANDQconst {
 			break
 		}
-		d := v_0.AuxInt
+		d := auxIntToInt32(v_0.AuxInt)
 		x := v_0.Args[0]
 		v.reset(OpAMD64ANDQconst)
-		v.AuxInt = c & d
+		v.AuxInt = int32ToAuxInt(c & d)
 		v.AddArg(x)
 		return true
 	}
 	// match: (ANDQconst [c] (BTRQconst [d] x))
+	// cond: is32Bit(int64(c) &^ (1<<uint32(d)))
 	// result: (ANDQconst [c &^ (1<<uint32(d))] x)
 	for {
-		c := v.AuxInt
+		c := auxIntToInt32(v.AuxInt)
 		if v_0.Op != OpAMD64BTRQconst {
 			break
 		}
-		d := v_0.AuxInt
+		d := auxIntToInt8(v_0.AuxInt)
 		x := v_0.Args[0]
+		if !(is32Bit(int64(c) &^ (1 << uint32(d)))) {
+			break
+		}
 		v.reset(OpAMD64ANDQconst)
-		v.AuxInt = c &^ (1 << uint32(d))
+		v.AuxInt = int32ToAuxInt(c &^ (1 << uint32(d)))
 		v.AddArg(x)
 		return true
 	}
@@ -3494,28 +3498,28 @@ func rewriteValueAMD64_OpAMD64BTCLconst(v *Value) bool {
 	// match: (BTCLconst [c] (XORLconst [d] x))
 	// result: (XORLconst [d ^ 1<<uint32(c)] x)
 	for {
-		c := v.AuxInt
+		c := auxIntToInt8(v.AuxInt)
 		if v_0.Op != OpAMD64XORLconst {
 			break
 		}
-		d := v_0.AuxInt
+		d := auxIntToInt32(v_0.AuxInt)
 		x := v_0.Args[0]
 		v.reset(OpAMD64XORLconst)
-		v.AuxInt = d ^ 1<<uint32(c)
+		v.AuxInt = int32ToAuxInt(d ^ 1<<uint32(c))
 		v.AddArg(x)
 		return true
 	}
 	// match: (BTCLconst [c] (BTCLconst [d] x))
-	// result: (XORLconst [1<<uint32(c) ^ 1<<uint32(d)] x)
+	// result: (XORLconst [1<<uint32(c) | 1<<uint32(d)] x)
 	for {
-		c := v.AuxInt
+		c := auxIntToInt8(v.AuxInt)
 		if v_0.Op != OpAMD64BTCLconst {
 			break
 		}
-		d := v_0.AuxInt
+		d := auxIntToInt8(v_0.AuxInt)
 		x := v_0.Args[0]
 		v.reset(OpAMD64XORLconst)
-		v.AuxInt = 1<<uint32(c) ^ 1<<uint32(d)
+		v.AuxInt = int32ToAuxInt(1<<uint32(c) | 1<<uint32(d))
 		v.AddArg(x)
 		return true
 	}
@@ -3635,30 +3639,38 @@ func rewriteValueAMD64_OpAMD64BTCLmodify(v *Value) bool {
 func rewriteValueAMD64_OpAMD64BTCQconst(v *Value) bool {
 	v_0 := v.Args[0]
 	// match: (BTCQconst [c] (XORQconst [d] x))
+	// cond: is32Bit(int64(d) ^ 1<<uint32(c))
 	// result: (XORQconst [d ^ 1<<uint32(c)] x)
 	for {
-		c := v.AuxInt
+		c := auxIntToInt8(v.AuxInt)
 		if v_0.Op != OpAMD64XORQconst {
 			break
 		}
-		d := v_0.AuxInt
+		d := auxIntToInt32(v_0.AuxInt)
 		x := v_0.Args[0]
+		if !(is32Bit(int64(d) ^ 1<<uint32(c))) {
+			break
+		}
 		v.reset(OpAMD64XORQconst)
-		v.AuxInt = d ^ 1<<uint32(c)
+		v.AuxInt = int32ToAuxInt(d ^ 1<<uint32(c))
 		v.AddArg(x)
 		return true
 	}
 	// match: (BTCQconst [c] (BTCQconst [d] x))
+	// cond: is32Bit(1<<uint32(c) ^ 1<<uint32(d))
 	// result: (XORQconst [1<<uint32(c) ^ 1<<uint32(d)] x)
 	for {
-		c := v.AuxInt
+		c := auxIntToInt8(v.AuxInt)
 		if v_0.Op != OpAMD64BTCQconst {
 			break
 		}
-		d := v_0.AuxInt
+		d := auxIntToInt8(v_0.AuxInt)
 		x := v_0.Args[0]
+		if !(is32Bit(1<<uint32(c) ^ 1<<uint32(d))) {
+			break
+		}
 		v.reset(OpAMD64XORQconst)
-		v.AuxInt = 1<<uint32(c) ^ 1<<uint32(d)
+		v.AuxInt = int32ToAuxInt(1<<uint32(c) ^ 1<<uint32(d))
 		v.AddArg(x)
 		return true
 	}
@@ -3970,28 +3982,28 @@ func rewriteValueAMD64_OpAMD64BTRLconst(v *Value) bool {
 	// match: (BTRLconst [c] (ANDLconst [d] x))
 	// result: (ANDLconst [d &^ (1<<uint32(c))] x)
 	for {
-		c := v.AuxInt
+		c := auxIntToInt8(v.AuxInt)
 		if v_0.Op != OpAMD64ANDLconst {
 			break
 		}
-		d := v_0.AuxInt
+		d := auxIntToInt32(v_0.AuxInt)
 		x := v_0.Args[0]
 		v.reset(OpAMD64ANDLconst)
-		v.AuxInt = d &^ (1 << uint32(c))
+		v.AuxInt = int32ToAuxInt(d &^ (1 << uint32(c)))
 		v.AddArg(x)
 		return true
 	}
 	// match: (BTRLconst [c] (BTRLconst [d] x))
 	// result: (ANDLconst [^(1<<uint32(c) | 1<<uint32(d))] x)
 	for {
-		c := v.AuxInt
+		c := auxIntToInt8(v.AuxInt)
 		if v_0.Op != OpAMD64BTRLconst {
 			break
 		}
-		d := v_0.AuxInt
+		d := auxIntToInt8(v_0.AuxInt)
 		x := v_0.Args[0]
 		v.reset(OpAMD64ANDLconst)
-		v.AuxInt = ^(1<<uint32(c) | 1<<uint32(d))
+		v.AuxInt = int32ToAuxInt(^(1<<uint32(c) | 1<<uint32(d)))
 		v.AddArg(x)
 		return true
 	}
@@ -4137,30 +4149,38 @@ func rewriteValueAMD64_OpAMD64BTRQconst(v *Value) bool {
 		return true
 	}
 	// match: (BTRQconst [c] (ANDQconst [d] x))
+	// cond: is32Bit(int64(d) &^ (1<<uint32(c)))
 	// result: (ANDQconst [d &^ (1<<uint32(c))] x)
 	for {
-		c := v.AuxInt
+		c := auxIntToInt8(v.AuxInt)
 		if v_0.Op != OpAMD64ANDQconst {
 			break
 		}
-		d := v_0.AuxInt
+		d := auxIntToInt32(v_0.AuxInt)
 		x := v_0.Args[0]
+		if !(is32Bit(int64(d) &^ (1 << uint32(c)))) {
+			break
+		}
 		v.reset(OpAMD64ANDQconst)
-		v.AuxInt = d &^ (1 << uint32(c))
+		v.AuxInt = int32ToAuxInt(d &^ (1 << uint32(c)))
 		v.AddArg(x)
 		return true
 	}
 	// match: (BTRQconst [c] (BTRQconst [d] x))
+	// cond: is32Bit(^(1<<uint32(c) | 1<<uint32(d)))
 	// result: (ANDQconst [^(1<<uint32(c) | 1<<uint32(d))] x)
 	for {
-		c := v.AuxInt
+		c := auxIntToInt8(v.AuxInt)
 		if v_0.Op != OpAMD64BTRQconst {
 			break
 		}
-		d := v_0.AuxInt
+		d := auxIntToInt8(v_0.AuxInt)
 		x := v_0.Args[0]
+		if !(is32Bit(^(1<<uint32(c) | 1<<uint32(d)))) {
+			break
+		}
 		v.reset(OpAMD64ANDQconst)
-		v.AuxInt = ^(1<<uint32(c) | 1<<uint32(d))
+		v.AuxInt = int32ToAuxInt(^(1<<uint32(c) | 1<<uint32(d)))
 		v.AddArg(x)
 		return true
 	}
@@ -4308,28 +4328,28 @@ func rewriteValueAMD64_OpAMD64BTSLconst(v *Value) bool {
 	// match: (BTSLconst [c] (ORLconst [d] x))
 	// result: (ORLconst [d | 1<<uint32(c)] x)
 	for {
-		c := v.AuxInt
+		c := auxIntToInt8(v.AuxInt)
 		if v_0.Op != OpAMD64ORLconst {
 			break
 		}
-		d := v_0.AuxInt
+		d := auxIntToInt32(v_0.AuxInt)
 		x := v_0.Args[0]
 		v.reset(OpAMD64ORLconst)
-		v.AuxInt = d | 1<<uint32(c)
+		v.AuxInt = int32ToAuxInt(d | 1<<uint32(c))
 		v.AddArg(x)
 		return true
 	}
 	// match: (BTSLconst [c] (BTSLconst [d] x))
-	// result: (ORLconst [1<<uint32(d) | 1<<uint32(c)] x)
+	// result: (ORLconst [1<<uint32(c) | 1<<uint32(d)] x)
 	for {
-		c := v.AuxInt
+		c := auxIntToInt8(v.AuxInt)
 		if v_0.Op != OpAMD64BTSLconst {
 			break
 		}
-		d := v_0.AuxInt
+		d := auxIntToInt8(v_0.AuxInt)
 		x := v_0.Args[0]
 		v.reset(OpAMD64ORLconst)
-		v.AuxInt = 1<<uint32(d) | 1<<uint32(c)
+		v.AuxInt = int32ToAuxInt(1<<uint32(c) | 1<<uint32(d))
 		v.AddArg(x)
 		return true
 	}
@@ -4475,30 +4495,38 @@ func rewriteValueAMD64_OpAMD64BTSQconst(v *Value) bool {
 		return true
 	}
 	// match: (BTSQconst [c] (ORQconst [d] x))
+	// cond: is32Bit(int64(d) | 1<<uint32(c))
 	// result: (ORQconst [d | 1<<uint32(c)] x)
 	for {
-		c := v.AuxInt
+		c := auxIntToInt8(v.AuxInt)
 		if v_0.Op != OpAMD64ORQconst {
 			break
 		}
-		d := v_0.AuxInt
+		d := auxIntToInt32(v_0.AuxInt)
 		x := v_0.Args[0]
+		if !(is32Bit(int64(d) | 1<<uint32(c))) {
+			break
+		}
 		v.reset(OpAMD64ORQconst)
-		v.AuxInt = d | 1<<uint32(c)
+		v.AuxInt = int32ToAuxInt(d | 1<<uint32(c))
 		v.AddArg(x)
 		return true
 	}
 	// match: (BTSQconst [c] (BTSQconst [d] x))
-	// result: (ORQconst [1<<uint32(d) | 1<<uint32(c)] x)
+	// cond: is32Bit(1<<uint32(c) | 1<<uint32(d))
+	// result: (ORQconst [1<<uint32(c) | 1<<uint32(d)] x)
 	for {
-		c := v.AuxInt
+		c := auxIntToInt8(v.AuxInt)
 		if v_0.Op != OpAMD64BTSQconst {
 			break
 		}
-		d := v_0.AuxInt
+		d := auxIntToInt8(v_0.AuxInt)
 		x := v_0.Args[0]
+		if !(is32Bit(1<<uint32(c) | 1<<uint32(d))) {
+			break
+		}
 		v.reset(OpAMD64ORQconst)
-		v.AuxInt = 1<<uint32(d) | 1<<uint32(c)
+		v.AuxInt = int32ToAuxInt(1<<uint32(c) | 1<<uint32(d))
 		v.AddArg(x)
 		return true
 	}
@@ -18127,28 +18155,28 @@ func rewriteValueAMD64_OpAMD64ORLconst(v *Value) bool {
 	// match: (ORLconst [c] (ORLconst [d] x))
 	// result: (ORLconst [c | d] x)
 	for {
-		c := v.AuxInt
+		c := auxIntToInt32(v.AuxInt)
 		if v_0.Op != OpAMD64ORLconst {
 			break
 		}
-		d := v_0.AuxInt
+		d := auxIntToInt32(v_0.AuxInt)
 		x := v_0.Args[0]
 		v.reset(OpAMD64ORLconst)
-		v.AuxInt = c | d
+		v.AuxInt = int32ToAuxInt(c | d)
 		v.AddArg(x)
 		return true
 	}
 	// match: (ORLconst [c] (BTSLconst [d] x))
 	// result: (ORLconst [c | 1<<uint32(d)] x)
 	for {
-		c := v.AuxInt
+		c := auxIntToInt32(v.AuxInt)
 		if v_0.Op != OpAMD64BTSLconst {
 			break
 		}
-		d := v_0.AuxInt
+		d := auxIntToInt8(v_0.AuxInt)
 		x := v_0.Args[0]
 		v.reset(OpAMD64ORLconst)
-		v.AuxInt = c | 1<<uint32(d)
+		v.AuxInt = int32ToAuxInt(c | 1<<uint32(d))
 		v.AddArg(x)
 		return true
 	}
@@ -19754,28 +19782,32 @@ func rewriteValueAMD64_OpAMD64ORQconst(v *Value) bool {
 	// match: (ORQconst [c] (ORQconst [d] x))
 	// result: (ORQconst [c | d] x)
 	for {
-		c := v.AuxInt
+		c := auxIntToInt32(v.AuxInt)
 		if v_0.Op != OpAMD64ORQconst {
 			break
 		}
-		d := v_0.AuxInt
+		d := auxIntToInt32(v_0.AuxInt)
 		x := v_0.Args[0]
 		v.reset(OpAMD64ORQconst)
-		v.AuxInt = c | d
+		v.AuxInt = int32ToAuxInt(c | d)
 		v.AddArg(x)
 		return true
 	}
 	// match: (ORQconst [c] (BTSQconst [d] x))
+	// cond: is32Bit(int64(c) | 1<<uint32(d))
 	// result: (ORQconst [c | 1<<uint32(d)] x)
 	for {
-		c := v.AuxInt
+		c := auxIntToInt32(v.AuxInt)
 		if v_0.Op != OpAMD64BTSQconst {
 			break
 		}
-		d := v_0.AuxInt
+		d := auxIntToInt8(v_0.AuxInt)
 		x := v_0.Args[0]
+		if !(is32Bit(int64(c) | 1<<uint32(d))) {
+			break
+		}
 		v.reset(OpAMD64ORQconst)
-		v.AuxInt = c | 1<<uint32(d)
+		v.AuxInt = int32ToAuxInt(c | 1<<uint32(d))
 		v.AddArg(x)
 		return true
 	}
@@ -27785,28 +27817,28 @@ func rewriteValueAMD64_OpAMD64XORLconst(v *Value) bool {
 	// match: (XORLconst [c] (XORLconst [d] x))
 	// result: (XORLconst [c ^ d] x)
 	for {
-		c := v.AuxInt
+		c := auxIntToInt32(v.AuxInt)
 		if v_0.Op != OpAMD64XORLconst {
 			break
 		}
-		d := v_0.AuxInt
+		d := auxIntToInt32(v_0.AuxInt)
 		x := v_0.Args[0]
 		v.reset(OpAMD64XORLconst)
-		v.AuxInt = c ^ d
+		v.AuxInt = int32ToAuxInt(c ^ d)
 		v.AddArg(x)
 		return true
 	}
 	// match: (XORLconst [c] (BTCLconst [d] x))
 	// result: (XORLconst [c ^ 1<<uint32(d)] x)
 	for {
-		c := v.AuxInt
+		c := auxIntToInt32(v.AuxInt)
 		if v_0.Op != OpAMD64BTCLconst {
 			break
 		}
-		d := v_0.AuxInt
+		d := auxIntToInt8(v_0.AuxInt)
 		x := v_0.Args[0]
 		v.reset(OpAMD64XORLconst)
-		v.AuxInt = c ^ 1<<uint32(d)
+		v.AuxInt = int32ToAuxInt(c ^ 1<<uint32(d))
 		v.AddArg(x)
 		return true
 	}
@@ -28151,28 +28183,32 @@ func rewriteValueAMD64_OpAMD64XORQconst(v *Value) bool {
 	// match: (XORQconst [c] (XORQconst [d] x))
 	// result: (XORQconst [c ^ d] x)
 	for {
-		c := v.AuxInt
+		c := auxIntToInt32(v.AuxInt)
 		if v_0.Op != OpAMD64XORQconst {
 			break
 		}
-		d := v_0.AuxInt
+		d := auxIntToInt32(v_0.AuxInt)
 		x := v_0.Args[0]
 		v.reset(OpAMD64XORQconst)
-		v.AuxInt = c ^ d
+		v.AuxInt = int32ToAuxInt(c ^ d)
 		v.AddArg(x)
 		return true
 	}
 	// match: (XORQconst [c] (BTCQconst [d] x))
+	// cond: is32Bit(int64(c) ^ 1<<uint32(d))
 	// result: (XORQconst [c ^ 1<<uint32(d)] x)
 	for {
-		c := v.AuxInt
+		c := auxIntToInt32(v.AuxInt)
 		if v_0.Op != OpAMD64BTCQconst {
 			break
 		}
-		d := v_0.AuxInt
+		d := auxIntToInt8(v_0.AuxInt)
 		x := v_0.Args[0]
+		if !(is32Bit(int64(c) ^ 1<<uint32(d))) {
+			break
+		}
 		v.reset(OpAMD64XORQconst)
-		v.AuxInt = c ^ 1<<uint32(d)
+		v.AuxInt = int32ToAuxInt(c ^ 1<<uint32(d))
 		v.AddArg(x)
 		return true
 	}