diff options
66 files changed, 8594 insertions, 107 deletions
diff --git a/api/except.txt b/api/except.txt index 14fe7785fa5..b9972c121cf 100644 --- a/api/except.txt +++ b/api/except.txt @@ -492,6 +492,7 @@ pkg syscall (windows-amd64), type CertRevocationInfo struct, OidSpecificInfo uin pkg syscall (windows-amd64), type CertSimpleChain struct, TrustListInfo uintptr pkg syscall (windows-amd64), type RawSockaddrAny struct, Pad [96]int8 pkg testing, func MainStart(func(string, string) (bool, error), []InternalTest, []InternalBenchmark, []InternalExample) *M +pkg testing, func MainStart(testDeps, []InternalTest, []InternalBenchmark, []InternalExample) *M pkg testing, func RegisterCover(Cover) pkg text/scanner, const GoTokens = 1012 pkg text/template/parse, type DotNode bool diff --git a/api/next.txt b/api/next.txt index 3eb7f3f797f..b1b9c1d7b16 100644 --- a/api/next.txt +++ b/api/next.txt @@ -106,3 +106,39 @@ pkg syscall (windows-386), func WSASendtoInet4(Handle, *WSABuf, uint32, *uint32, pkg syscall (windows-386), func WSASendtoInet6(Handle, *WSABuf, uint32, *uint32, uint32, SockaddrInet6, *Overlapped, *uint8) error pkg syscall (windows-amd64), func WSASendtoInet4(Handle, *WSABuf, uint32, *uint32, uint32, SockaddrInet4, *Overlapped, *uint8) error pkg syscall (windows-amd64), func WSASendtoInet6(Handle, *WSABuf, uint32, *uint32, uint32, SockaddrInet6, *Overlapped, *uint8) error +pkg testing, func Fuzz(func(*F)) FuzzResult +pkg testing, func MainStart(testDeps, []InternalTest, []InternalBenchmark, []InternalFuzzTarget, []InternalExample) *M +pkg testing, func RunFuzzTargets(func(string, string) (bool, error), []InternalFuzzTarget) bool +pkg testing, func RunFuzzing(func(string, string) (bool, error), []InternalFuzzTarget) bool +pkg testing, method (*B) Setenv(string, string) +pkg testing, method (*F) Add(...interface{}) +pkg testing, method (*F) Cleanup(func()) +pkg testing, method (*F) Error(...interface{}) +pkg testing, method (*F) Errorf(string, ...interface{}) +pkg testing, method (*F) Fail() +pkg testing, method (*F) FailNow() +pkg testing, method (*F) Failed() bool +pkg testing, method (*F) Fatal(...interface{}) +pkg testing, method (*F) Fatalf(string, ...interface{}) +pkg testing, method (*F) Fuzz(interface{}) +pkg testing, method (*F) Helper() +pkg testing, method (*F) Log(...interface{}) +pkg testing, method (*F) Logf(string, ...interface{}) +pkg testing, method (*F) Name() string +pkg testing, method (*F) Setenv(string, string) +pkg testing, method (*F) Skip(...interface{}) +pkg testing, method (*F) SkipNow() +pkg testing, method (*F) Skipf(string, ...interface{}) +pkg testing, method (*F) Skipped() bool +pkg testing, method (*F) TempDir() string +pkg testing, method (*T) Setenv(string, string) +pkg testing, method (FuzzResult) String() string +pkg testing, type F struct +pkg testing, type FuzzResult struct +pkg testing, type FuzzResult struct, Crasher entry +pkg testing, type FuzzResult struct, Error error +pkg testing, type FuzzResult struct, N int +pkg testing, type FuzzResult struct, T time.Duration +pkg testing, type InternalFuzzTarget struct +pkg testing, type InternalFuzzTarget struct, Fn func(*F) +pkg testing, type InternalFuzzTarget struct, Name string diff --git a/codereview.cfg b/codereview.cfg index 77a74f108ea..bed9bcf7eea 100644 --- a/codereview.cfg +++ b/codereview.cfg @@ -1 +1,2 @@ -branch: master +branch: dev.fuzz +parent-branch: master
\ No newline at end of file diff --git a/src/cmd/go/alldocs.go b/src/cmd/go/alldocs.go index 9753ebba3e6..74522691abd 100644 --- a/src/cmd/go/alldocs.go +++ b/src/cmd/go/alldocs.go @@ -53,6 +53,7 @@ // private configuration for downloading non-public code // testflag testing flags // testfunc testing functions +// fuzz fuzzing // vcs controlling version control with GOVCS // // Use "go help <topic>" for more information about that topic. @@ -292,6 +293,8 @@ // download cache, including unpacked source code of versioned // dependencies. // +// The -fuzzcache flag causes clean to remove values used for fuzz testing. +// // For more about build flags, see 'go help build'. // // For more about specifying packages, see 'go help packages'. @@ -1488,8 +1491,8 @@ // // 'Go test' recompiles each package along with any files with names matching // the file pattern "*_test.go". -// These additional files can contain test functions, benchmark functions, and -// example functions. See 'go help testfunc' for more. +// These additional files can contain test functions, benchmark functions, fuzz +// targets and example functions. See 'go help testfunc' for more. // Each listed package causes the execution of a separate test binary. // Files whose names begin with "_" (including "_test.go") or "." are ignored. // @@ -1559,6 +1562,8 @@ // in no time at all,so a successful package test result will be cached and // reused regardless of -timeout setting. // +// Run 'go help fuzz' for details around how the go command handles fuzz targets. +// // In addition to the build flags, the flags handled by 'go test' itself are: // // -args @@ -2728,7 +2733,8 @@ // (for example, -benchtime 100x). // // -count n -// Run each test and benchmark n times (default 1). +// Run each test, benchmark, and fuzz targets' seed corpora n times +// (default 1). // If -cpu is set, run n times for each GOMAXPROCS value. // Examples are always run once. // @@ -2757,36 +2763,55 @@ // Sets -cover. // // -cpu 1,2,4 -// Specify a list of GOMAXPROCS values for which the tests or -// benchmarks should be executed. The default is the current value +// Specify a list of GOMAXPROCS values for which the tests, benchmarks or +// fuzz targets should be executed. The default is the current value // of GOMAXPROCS. // // -failfast // Do not start new tests after the first test failure. // +// -fuzz name +// Run the fuzz target with the given regexp. Must match exactly one fuzz +// target. This is an experimental feature. +// +// -fuzztime t +// Run enough iterations of the fuzz test to take t, specified as a +// time.Duration (for example, -fuzztime 1h30s). The default is to run +// forever. +// The special syntax Nx means to run the fuzz test N times +// (for example, -fuzztime 100x). +// // -json // Log verbose output and test results in JSON. This presents the // same information as the -v flag in a machine-readable format. // +// -keepfuzzing +// Keep running the fuzz target if a crasher is found. +// // -list regexp -// List tests, benchmarks, or examples matching the regular expression. -// No tests, benchmarks or examples will be run. This will only -// list top-level tests. No subtest or subbenchmarks will be shown. +// List tests, benchmarks, fuzz targets, or examples matching the regular +// expression. No tests, benchmarks, fuzz targets, or examples will be run. +// This will only list top-level tests. No subtest or subbenchmarks will be +// shown. // // -parallel n -// Allow parallel execution of test functions that call t.Parallel. +// Allow parallel execution of test functions that call t.Parallel, and +// f.Fuzz functions that call t.Parallel when running the seed corpus. // The value of this flag is the maximum number of tests to run -// simultaneously; by default, it is set to the value of GOMAXPROCS. +// simultaneously. While fuzzing, the value of this flag is the +// maximum number of workers to run the fuzz function simultaneously, +// regardless of whether t.Parallel has been called; by default, it is set +// to the value of GOMAXPROCS. // Note that -parallel only applies within a single test binary. // The 'go test' command may run tests for different packages // in parallel as well, according to the setting of the -p flag // (see 'go help build'). // // -run regexp -// Run only those tests and examples matching the regular expression. -// For tests, the regular expression is split by unbracketed slash (/) -// characters into a sequence of regular expressions, and each part -// of a test's identifier must match the corresponding element in +// Run only those tests, examples, and fuzz targets matching the regular +// expression. For tests, the regular expression is split by unbracketed +// slash (/) characters into a sequence of regular expressions, and each +// part of a test's identifier must match the corresponding element in // the sequence, if any. Note that possible parents of matches are // run too, so that -run=X/Y matches and runs and reports the result // of all tests matching X, even those without sub-tests matching Y, @@ -2953,6 +2978,10 @@ // // func BenchmarkXxx(b *testing.B) { ... } // +// A fuzz target is one named FuzzXxx and should have the signature, +// +// func FuzzXxx(f *testing.F) { ... } +// // An example function is similar to a test function but, instead of using // *testing.T to report success or failure, prints output to os.Stdout. // If the last comment in the function starts with "Output:" then the output @@ -2992,11 +3021,30 @@ // // The entire test file is presented as the example when it contains a single // example function, at least one other function, type, variable, or constant -// declaration, and no test or benchmark functions. +// declaration, and no fuzz targets or test or benchmark functions. // // See the documentation of the testing package for more information. // // +// Fuzzing +// +// By default, go test will build and run the fuzz targets using the target's seed +// corpus only. Any generated corpora in $GOCACHE that were previously written by +// the fuzzing engine will not be run by default. +// +// When -fuzz is set, the binary will be instrumented for coverage. After all +// tests, examples, benchmark functions, and the seed corpora for all fuzz targets +// have been run, go test will begin to fuzz the specified fuzz target. +// Note that this feature is experimental. +// +// -run can be used for testing a single seed corpus entry for a fuzz target. The +// regular expression value of -run can be in the form $target/$name, where $target +// is the name of the fuzz target, and $name is the name of the file (ignoring file +// extensions) to run. For example, -run=FuzzFoo/497b6f87. +// +// See https://golang.org/s/draft-fuzzing-design for more details. +// +// // Controlling version control with GOVCS // // The 'go get' command can run version control commands like git diff --git a/src/cmd/go/internal/cache/cache.go b/src/cmd/go/internal/cache/cache.go index d592d704978..596f22e8fc1 100644 --- a/src/cmd/go/internal/cache/cache.go +++ b/src/cmd/go/internal/cache/cache.go @@ -533,3 +533,13 @@ func (c *Cache) copyFile(file io.ReadSeeker, out OutputID, size int64) error { return nil } + +// FuzzDir returns a subdirectory within the cache for storing fuzzing data. +// The subdirectory may not exist. +// +// This directory is managed by the internal/fuzz package. Files in this +// directory aren't removed by the 'go clean -cache' command or by Trim. +// They may be removed with 'go clean -fuzzcache'. +func (c *Cache) FuzzDir() string { + return filepath.Join(c.dir, "fuzz") +} diff --git a/src/cmd/go/internal/cfg/cfg.go b/src/cmd/go/internal/cfg/cfg.go index 5f4465e06b1..dd0e8cbbd66 100644 --- a/src/cmd/go/internal/cfg/cfg.go +++ b/src/cmd/go/internal/cfg/cfg.go @@ -60,6 +60,10 @@ var ( func defaultContext() build.Context { ctxt := build.Default + + // TODO(#47037): remove this tag before merging to master. + ctxt.BuildTags = []string{"gofuzzbeta"} + ctxt.JoinPath = filepath.Join // back door to say "do not use go command" ctxt.GOROOT = findGOROOT() diff --git a/src/cmd/go/internal/clean/clean.go b/src/cmd/go/internal/clean/clean.go index 1089211f0ce..518473c9149 100644 --- a/src/cmd/go/internal/clean/clean.go +++ b/src/cmd/go/internal/clean/clean.go @@ -75,6 +75,8 @@ The -modcache flag causes clean to remove the entire module download cache, including unpacked source code of versioned dependencies. +The -fuzzcache flag causes clean to remove values used for fuzz testing. + For more about build flags, see 'go help build'. For more about specifying packages, see 'go help packages'. @@ -85,6 +87,7 @@ var ( cleanI bool // clean -i flag cleanR bool // clean -r flag cleanCache bool // clean -cache flag + cleanFuzzcache bool // clean -fuzzcache flag cleanModcache bool // clean -modcache flag cleanTestcache bool // clean -testcache flag ) @@ -96,6 +99,7 @@ func init() { CmdClean.Flag.BoolVar(&cleanI, "i", false, "") CmdClean.Flag.BoolVar(&cleanR, "r", false, "") CmdClean.Flag.BoolVar(&cleanCache, "cache", false, "") + CmdClean.Flag.BoolVar(&cleanFuzzcache, "fuzzcache", false, "") CmdClean.Flag.BoolVar(&cleanModcache, "modcache", false, "") CmdClean.Flag.BoolVar(&cleanTestcache, "testcache", false, "") @@ -112,7 +116,7 @@ func runClean(ctx context.Context, cmd *base.Command, args []string) { // or no other target (such as a cache) was requested to be cleaned. cleanPkg := len(args) > 0 || cleanI || cleanR if (!modload.Enabled() || modload.HasModRoot()) && - !cleanCache && !cleanModcache && !cleanTestcache { + !cleanCache && !cleanModcache && !cleanTestcache && !cleanFuzzcache { cleanPkg = true } @@ -206,6 +210,18 @@ func runClean(ctx context.Context, cmd *base.Command, args []string) { } } } + + if cleanFuzzcache { + fuzzDir := cache.Default().FuzzDir() + if cfg.BuildN || cfg.BuildX { + b.Showcmd("", "rm -rf %s", fuzzDir) + } + if !cfg.BuildN { + if err := os.RemoveAll(fuzzDir); err != nil { + base.Errorf("go clean -fuzzcache: %v", err) + } + } + } } var cleaned = map[*load.Package]bool{} diff --git a/src/cmd/go/internal/load/flag.go b/src/cmd/go/internal/load/flag.go index 4e0cb5bc19e..24670524fc1 100644 --- a/src/cmd/go/internal/load/flag.go +++ b/src/cmd/go/internal/load/flag.go @@ -22,8 +22,9 @@ var ( // that allows specifying different effective flags for different packages. // See 'go help build' for more details about per-package flags. type PerPackageFlag struct { - present bool - values []ppfValue + present bool + values []ppfValue + seenPackages map[*Package]bool // the packages for which the flags have already been set } // A ppfValue is a single <pattern>=<flags> per-package flag value. diff --git a/src/cmd/go/internal/load/pkg.go b/src/cmd/go/internal/load/pkg.go index 4013330bc47..317053d9182 100644 --- a/src/cmd/go/internal/load/pkg.go +++ b/src/cmd/go/internal/load/pkg.go @@ -2630,10 +2630,20 @@ func (e *mainPackageError) ImportPath() string { func setToolFlags(pkgs ...*Package) { for _, p := range PackageList(pkgs) { - p.Internal.Asmflags = BuildAsmflags.For(p) - p.Internal.Gcflags = BuildGcflags.For(p) - p.Internal.Ldflags = BuildLdflags.For(p) - p.Internal.Gccgoflags = BuildGccgoflags.For(p) + appendFlags(p, &p.Internal.Asmflags, &BuildAsmflags) + appendFlags(p, &p.Internal.Gcflags, &BuildGcflags) + appendFlags(p, &p.Internal.Ldflags, &BuildLdflags) + appendFlags(p, &p.Internal.Gccgoflags, &BuildGccgoflags) + } +} + +func appendFlags(p *Package, flags *[]string, packageFlag *PerPackageFlag) { + if !packageFlag.seenPackages[p] { + if packageFlag.seenPackages == nil { + packageFlag.seenPackages = make(map[*Package]bool) + } + packageFlag.seenPackages[p] = true + *flags = append(*flags, packageFlag.For(p)...) } } diff --git a/src/cmd/go/internal/load/test.go b/src/cmd/go/internal/load/test.go index 42eefe37ba5..da6d1cb21d1 100644 --- a/src/cmd/go/internal/load/test.go +++ b/src/cmd/go/internal/load/test.go @@ -555,6 +555,7 @@ func formatTestmain(t *testFuncs) ([]byte, error) { type testFuncs struct { Tests []testFunc Benchmarks []testFunc + FuzzTargets []testFunc Examples []testFunc TestMain *testFunc Package *Package @@ -653,6 +654,13 @@ func (t *testFuncs) load(filename, pkg string, doImport, seen *bool) error { } t.Benchmarks = append(t.Benchmarks, testFunc{pkg, name, "", false}) *doImport, *seen = true, true + case isTest(name, "Fuzz"): + err := checkTestFunc(n, "F") + if err != nil { + return err + } + t.FuzzTargets = append(t.FuzzTargets, testFunc{pkg, name, "", false}) + *doImport, *seen = true, true } } ex := doc.Examples(f) @@ -716,6 +724,12 @@ var benchmarks = []testing.InternalBenchmark{ {{end}} } +var fuzzTargets = []testing.InternalFuzzTarget{ +{{range .FuzzTargets}} + {"{{.Name}}", {{.Package}}.{{.Name}}}, +{{end}} +} + var examples = []testing.InternalExample{ {{range .Examples}} {"{{.Name}}", {{.Package}}.{{.Name}}, {{.Output | printf "%q"}}, {{.Unordered}}}, @@ -774,7 +788,7 @@ func main() { CoveredPackages: {{printf "%q" .Covered}}, }) {{end}} - m := testing.MainStart(testdeps.TestDeps{}, tests, benchmarks, examples) + m := testing.MainStart(testdeps.TestDeps{}, tests, benchmarks, fuzzTargets, examples) {{with .TestMain}} {{.Package}}.{{.Name}}(m) os.Exit(int(reflect.ValueOf(m).Elem().FieldByName("exitCode").Int())) diff --git a/src/cmd/go/internal/test/flagdefs.go b/src/cmd/go/internal/test/flagdefs.go index 37ac81c2678..3148074d574 100644 --- a/src/cmd/go/internal/test/flagdefs.go +++ b/src/cmd/go/internal/test/flagdefs.go @@ -19,6 +19,9 @@ var passFlagToTest = map[string]bool{ "cpu": true, "cpuprofile": true, "failfast": true, + "fuzz": true, + "fuzzminimizetime": true, + "fuzztime": true, "list": true, "memprofile": true, "memprofilerate": true, diff --git a/src/cmd/go/internal/test/flagdefs_test.go b/src/cmd/go/internal/test/flagdefs_test.go index ab5440b3801..f238fc7d335 100644 --- a/src/cmd/go/internal/test/flagdefs_test.go +++ b/src/cmd/go/internal/test/flagdefs_test.go @@ -17,7 +17,7 @@ func TestPassFlagToTestIncludesAllTestFlags(t *testing.T) { } name := strings.TrimPrefix(f.Name, "test.") switch name { - case "testlogfile", "paniconexit0": + case "testlogfile", "paniconexit0", "fuzzcachedir", "fuzzworker": // These are internal flags. default: if !passFlagToTest[name] { diff --git a/src/cmd/go/internal/test/genflags.go b/src/cmd/go/internal/test/genflags.go index 9277de7fee8..645aae68b17 100644 --- a/src/cmd/go/internal/test/genflags.go +++ b/src/cmd/go/internal/test/genflags.go @@ -64,7 +64,7 @@ func testFlags() []string { name := strings.TrimPrefix(f.Name, "test.") switch name { - case "testlogfile", "paniconexit0": + case "testlogfile", "paniconexit0", "fuzzcachedir", "fuzzworker": // These flags are only for use by cmd/go. default: names = append(names, name) diff --git a/src/cmd/go/internal/test/test.go b/src/cmd/go/internal/test/test.go index 198afbf4c3d..8f5d57eff14 100644 --- a/src/cmd/go/internal/test/test.go +++ b/src/cmd/go/internal/test/test.go @@ -61,8 +61,8 @@ followed by detailed output for each failed package. 'Go test' recompiles each package along with any files with names matching the file pattern "*_test.go". -These additional files can contain test functions, benchmark functions, and -example functions. See 'go help testfunc' for more. +These additional files can contain test functions, benchmark functions, fuzz +targets and example functions. See 'go help testfunc' for more. Each listed package causes the execution of a separate test binary. Files whose names begin with "_" (including "_test.go") or "." are ignored. @@ -132,6 +132,8 @@ variables are unchanged. A cached test result is treated as executing in no time at all,so a successful package test result will be cached and reused regardless of -timeout setting. +Run 'go help fuzz' for details around how the go command handles fuzz targets. + In addition to the build flags, the flags handled by 'go test' itself are: -args @@ -208,7 +210,8 @@ control the execution of any test: (for example, -benchtime 100x). -count n - Run each test and benchmark n times (default 1). + Run each test, benchmark, and fuzz targets' seed corpora n times + (default 1). If -cpu is set, run n times for each GOMAXPROCS value. Examples are always run once. @@ -237,36 +240,55 @@ control the execution of any test: Sets -cover. -cpu 1,2,4 - Specify a list of GOMAXPROCS values for which the tests or - benchmarks should be executed. The default is the current value + Specify a list of GOMAXPROCS values for which the tests, benchmarks or + fuzz targets should be executed. The default is the current value of GOMAXPROCS. -failfast Do not start new tests after the first test failure. + -fuzz name + Run the fuzz target with the given regexp. Must match exactly one fuzz + target. This is an experimental feature. + + -fuzztime t + Run enough iterations of the fuzz test to take t, specified as a + time.Duration (for example, -fuzztime 1h30s). The default is to run + forever. + The special syntax Nx means to run the fuzz test N times + (for example, -fuzztime 100x). + -json Log verbose output and test results in JSON. This presents the same information as the -v flag in a machine-readable format. + -keepfuzzing + Keep running the fuzz target if a crasher is found. + -list regexp - List tests, benchmarks, or examples matching the regular expression. - No tests, benchmarks or examples will be run. This will only - list top-level tests. No subtest or subbenchmarks will be shown. + List tests, benchmarks, fuzz targets, or examples matching the regular + expression. No tests, benchmarks, fuzz targets, or examples will be run. + This will only list top-level tests. No subtest or subbenchmarks will be + shown. -parallel n - Allow parallel execution of test functions that call t.Parallel. + Allow parallel execution of test functions that call t.Parallel, and + f.Fuzz functions that call t.Parallel when running the seed corpus. The value of this flag is the maximum number of tests to run - simultaneously; by default, it is set to the value of GOMAXPROCS. + simultaneously. While fuzzing, the value of this flag is the + maximum number of workers to run the fuzz function simultaneously, + regardless of whether t.Parallel has been called; by default, it is set + to the value of GOMAXPROCS. Note that -parallel only applies within a single test binary. The 'go test' command may run tests for different packages in parallel as well, according to the setting of the -p flag (see 'go help build'). -run regexp - Run only those tests and examples matching the regular expression. - For tests, the regular expression is split by unbracketed slash (/) - characters into a sequence of regular expressions, and each part - of a test's identifier must match the corresponding element in + Run only those tests, examples, and fuzz targets matching the regular + expression. For tests, the regular expression is split by unbracketed + slash (/) characters into a sequence of regular expressions, and each + part of a test's identifier must match the corresponding element in the sequence, if any. Note that possible parents of matches are run too, so that -run=X/Y matches and runs and reports the result of all tests matching X, even those without sub-tests matching Y, @@ -436,6 +458,10 @@ A benchmark function is one named BenchmarkXxx and should have the signature, func BenchmarkXxx(b *testing.B) { ... } +A fuzz target is one named FuzzXxx and should have the signature, + + func FuzzXxx(f *testing.F) { ... } + An example function is similar to a test function but, instead of using *testing.T to report success or failure, prints output to os.Stdout. If the last comment in the function starts with "Output:" then the output @@ -475,12 +501,34 @@ Here is another example where the ordering of the output is ignored: The entire test file is presented as the example when it contains a single example function, at least one other function, type, variable, or constant -declaration, and no test or benchmark functions. +declaration, and no fuzz targets or test or benchmark functions. See the documentation of the testing package for more information. `, } +var HelpFuzz = &base.Command{ + UsageLine: "fuzz", + Short: "fuzzing", + Long: ` +By default, go test will build and run the fuzz targets using the target's seed +corpus only. Any generated corpora in $GOCACHE that were previously written by +the fuzzing engine will not be run by default. + +When -fuzz is set, the binary will be instrumented for coverage. After all +tests, examples, benchmark functions, and the seed corpora for all fuzz targets +have been run, go test will begin to fuzz the specified fuzz target. +Note that this feature is experimental. + +-run can be used for testing a single seed corpus entry for a fuzz target. The +regular expression value of -run can be in the form $target/$name, where $target +is the name of the fuzz target, and $name is the name of the file (ignoring file +extensions) to run. For example, -run=FuzzFoo/497b6f87. + +See https://golang.org/s/draft-fuzzing-design for more details. +`, +} + var ( testBench string // -bench flag testC bool // -c flag @@ -489,6 +537,7 @@ var ( testCoverPaths []string // -coverpkg flag testCoverPkgs []*load.Package // -coverpkg flag testCoverProfile string // -coverprofile flag + testFuzz string // -fuzz flag testJSON bool // -json flag testList string // -list flag testO string // -o flag @@ -622,6 +671,9 @@ func runTest(ctx context.Context, cmd *base.Command, args []string) { if testO != "" && len(pkgs) != 1 { base.Fatalf("cannot use -o flag with multiple packages") } + if testFuzz != "" && len(pkgs) != 1 { + base.Fatalf("cannot use -fuzz flag with multiple packages") + } if testProfile() != "" && len(pkgs) != 1 { base.Fatalf("cannot use %s flag with multiple packages", testProfile()) } @@ -632,7 +684,9 @@ func runTest(ctx context.Context, cmd *base.Command, args []string) { // to that timeout plus one minute. This is a backup alarm in case // the test wedges with a goroutine spinning and its background // timer does not get a chance to fire. - if testTimeout > 0 { + // Don't set this if fuzzing, since it should be able to run + // indefinitely. + if testTimeout > 0 && testFuzz == "" { testKillTimeout = testTimeout + 1*time.Minute } @@ -782,6 +836,32 @@ func runTest(ctx context.Context, cmd *base.Command, args []string) { } } + // Inform the compiler that it should instrument the binary at + // build-time when fuzzing is enabled. + fuzzFlags := work.FuzzInstrumentFlags() + if testFuzz != "" && fuzzFlags != nil { + // Don't instrument packages which may affect coverage guidance but are + // unlikely to be useful. Most of these are used by the testing or + // internal/fuzz concurrently with fuzzing. + var fuzzNoInstrument = map[string]bool{ + "context": true, + "internal/fuzz": true, + "reflect": true, + "runtime": true, + "sync": true, + "sync/atomic": true, + "syscall": true, + "testing": true, + "time": true, + } + for _, p := range load.TestPackageList(ctx, pkgOpts, pkgs) { + if fuzzNoInstrument[p.ImportPath] { + continue + } + p.Internal.Gcflags = append(p.Internal.Gcflags, fuzzFlags...) + } + } + // Prepare build + run + print actions for all packages being tested. for _, p := range pkgs { // sync/atomic import is inserted by the cover tool. See #18486 @@ -1087,6 +1167,8 @@ func declareCoverVars(p *load.Package, files ...string) map[string]*load.CoverVa } var noTestsToRun = []byte("\ntesting: warning: no tests to run\n") +var noTargetsToFuzz = []byte("\ntesting: warning: no targets to fuzz\n") +var tooManyTargetsToFuzz = []byte("\ntesting: warning: -fuzz matches more than one target, won't fuzz\n") type runCache struct { disableCache bool // cache should be disabled for this run @@ -1134,10 +1216,10 @@ func (c *runCache) builderRunTest(b *work.Builder, ctx context.Context, a *work. } var buf bytes.Buffer - if len(pkgArgs) == 0 || (testBench != "") { + if len(pkgArgs) == 0 || testBench != "" || testFuzz != "" { // Stream test output (no buffering) when no package has // been given on the command line (implicit current directory) - // or when benchmarking. + // or when benchmarking or fuzzing. // No change to stdout. } else { // If we're only running a single package under test or if parallelism is @@ -1190,7 +1272,12 @@ func (c *runCache) builderRunTest(b *work.Builder, ctx context.Context, a *work. testlogArg = []string{"-test.testlogfile=" + a.Objdir + "testlog.txt"} } panicArg := "-test.paniconexit0" - args := str.StringList(execCmd, a.Deps[0].BuiltTarget(), testlogArg, panicArg, testArgs) + fuzzArg := []string{} + if testFuzz != "" { + fuzzCacheDir := filepath.Join(cache.Default().FuzzDir(), a.Package.ImportPath) + fuzzArg = []string{"-test.fuzzcachedir=" + fuzzCacheDir} + } + args := str.StringList(execCmd, a.Deps[0].BuiltTarget(), testlogArg, panicArg, fuzzArg, testArgs) if testCoverProfile != "" { // Write coverage to temporary profile, for merging later. @@ -1283,6 +1370,12 @@ func (c *runCache) builderRunTest(b *work.Builder, ctx context.Context, a *work. if bytes.HasPrefix(out, noTestsToRun[1:]) || bytes.Contains(out, noTestsToRun) { norun = " [no tests to run]" } + if bytes.HasPrefix(out, noTargetsToFuzz[1:]) || bytes.Contains(out, noTargetsToFuzz) { + norun = " [no targets to fuzz]" + } + if bytes.HasPrefix(out, tooManyTargetsToFuzz[1:]) || bytes.Contains(out, tooManyTargetsToFuzz) { + norun = " [will not fuzz, -fuzz matches more than one target]" + } fmt.Fprintf(cmd.Stdout, "ok \t%s\t%s%s%s\n", a.Package.ImportPath, t, coveragePercentage(out), norun) c.saveOutput(a) } else { diff --git a/src/cmd/go/internal/test/testflag.go b/src/cmd/go/internal/test/testflag.go index e0a3e010fae..cb3543884a4 100644 --- a/src/cmd/go/internal/test/testflag.go +++ b/src/cmd/go/internal/test/testflag.go @@ -57,6 +57,7 @@ func init() { cf.String("cpu", "", "") cf.StringVar(&testCPUProfile, "cpuprofile", "", "") cf.Bool("failfast", false, "") + cf.StringVar(&testFuzz, "fuzz", "", "") cf.StringVar(&testList, "list", "", "") cf.StringVar(&testMemProfile, "memprofile", "", "") cf.String("memprofilerate", "", "") @@ -67,6 +68,8 @@ func init() { cf.String("run", "", "") cf.Bool("short", false, "") cf.DurationVar(&testTimeout, "timeout", 10*time.Minute, "") + cf.String("fuzztime", "", "") + cf.String("fuzzminimizetime", "", "") cf.StringVar(&testTrace, "trace", "", "") cf.BoolVar(&testV, "v", false, "") cf.Var(&testShuffle, "shuffle", "") diff --git a/src/cmd/go/internal/work/init.go b/src/cmd/go/internal/work/init.go index 7aa8dfe55fd..2a605e73ee1 100644 --- a/src/cmd/go/internal/work/init.go +++ b/src/cmd/go/internal/work/init.go @@ -60,6 +60,14 @@ func BuildInit() { } } +func FuzzInstrumentFlags() []string { + if cfg.Goarch != "amd64" && cfg.Goarch != "arm64" { + // Instrumentation is only supported on 64-bit architectures. + return nil + } + return []string{"-d=libfuzzer"} +} + func instrumentInit() { if !cfg.BuildRace && !cfg.BuildMSan { return diff --git a/src/cmd/go/main.go b/src/cmd/go/main.go index 16361e02ca7..11ff750affa 100644 --- a/src/cmd/go/main.go +++ b/src/cmd/go/main.go @@ -80,6 +80,7 @@ func init() { modfetch.HelpPrivate, test.HelpTestflag, test.HelpTestfunc, + test.HelpFuzz, modget.HelpVCS, } } diff --git a/src/cmd/go/testdata/script/test_fuzz.txt b/src/cmd/go/testdata/script/test_fuzz.txt new file mode 100644 index 00000000000..b1a02f46ebc --- /dev/null +++ b/src/cmd/go/testdata/script/test_fuzz.txt @@ -0,0 +1,442 @@ +# TODO(jayconrod): support shared memory on more platforms. +[!darwin] [!linux] [!windows] skip + +# Test that running a fuzz target that returns without failing or calling +# f.Fuzz fails and causes a non-zero exit status. +! go test noop_fuzz_test.go +! stdout ^ok +stdout FAIL + +# Test that fuzzing a fuzz target that returns without failing or calling +# f.Fuzz fails and causes a non-zero exit status. +! go test -fuzz=Fuzz -fuzztime=1x noop_fuzz_test.go +! stdout ^ok +stdout FAIL + +# Test that calling f.Error in a fuzz target causes a non-zero exit status. +! go test -fuzz=Fuzz -fuzztime=1x error_fuzz_test.go +! stdout ^ok +stdout FAIL + +# Test that calling f.Fatal in a fuzz target causes a non-zero exit status. +! go test fatal_fuzz_test.go +! stdout ^ok +stdout FAIL + +# Test that successful test exits cleanly. +go test success_fuzz_test.go +stdout ^ok +! stdout FAIL + +# Test that successful fuzzing exits cleanly. +go test -fuzz=Fuzz -fuzztime=1x success_fuzz_test.go +stdout ok +! stdout FAIL + +# Test that calling f.Fatal while fuzzing causes a non-zero exit status. +! go test -fuzz=Fuzz -fuzztime=1x fatal_fuzz_test.go +! stdout ^ok +stdout FAIL + +# Test error with seed corpus in f.Fuzz +! go test -run FuzzError fuzz_add_test.go +! stdout ^ok +stdout FAIL +stdout 'error here' + +[short] stop + +# Test that calling panic(nil) in a fuzz target causes a non-zero exit status. +! go test panic_fuzz_test.go +! stdout ^ok +stdout FAIL + +# Test that skipped test exits cleanly. +go test skipped_fuzz_test.go +stdout ok +! stdout FAIL + +# Test that f.Fatal within f.Fuzz panics +! go test fatal_fuzz_fn_fuzz_test.go +! stdout ^ok +! stdout 'fatal here' +stdout FAIL +stdout 'f.Fuzz function' + +# Test that f.Error within f.Fuzz panics +! go test error_fuzz_fn_fuzz_test.go +! stdout ^ok +! stdout 'error here' +stdout FAIL +stdout 'f.Fuzz function' + +# Test that f.Skip within f.Fuzz panics +! go test skip_fuzz_fn_fuzz_test.go +! stdout ^ok +! stdout 'skip here' +stdout FAIL +stdout 'f.Fuzz function' + +# Test that a call to f.Fatal after the Fuzz func is never executed. +go test fatal_after_fuzz_func_fuzz_test.go +stdout ok +! stdout FAIL + +# Test that missing *T in f.Fuzz causes a non-zero exit status. +! go test incomplete_fuzz_call_fuzz_test.go +! stdout ^ok +stdout FAIL + +# Test that a panic in the Cleanup func is executed. +! go test cleanup_fuzz_test.go +! stdout ^ok +stdout FAIL +stdout 'failed some precondition' + +# Test success with seed corpus in f.Fuzz +go test -run FuzzPass fuzz_add_test.go +stdout ok +! stdout FAIL +! stdout 'off by one error' + +# Test fatal with seed corpus in f.Fuzz +! go test -run FuzzFatal fuzz_add_test.go +! stdout ^ok +stdout FAIL +stdout 'fatal here' + +# Test panic with seed corpus in f.Fuzz +! go test -run FuzzPanic fuzz_add_test.go +! stdout ^ok +stdout FAIL +stdout 'off by one error' + +# Test panic(nil) with seed corpus in f.Fuzz +! go test -run FuzzNilPanic fuzz_add_test.go +! stdout ^ok +stdout FAIL + +# Test panic with unsupported seed corpus +! go test -run FuzzUnsupported fuzz_add_test.go +! stdout ^ok +stdout FAIL + +# Test panic with different number of args to f.Add +! go test -run FuzzAddDifferentNumber fuzz_add_test.go +! stdout ^ok +stdout FAIL + +# Test panic with different type of args to f.Add +! go test -run FuzzAddDifferentType fuzz_add_test.go +! stdout ^ok +stdout FAIL + +# Test that the wrong type given with f.Add will fail. +! go test -run FuzzWrongType fuzz_add_test.go +! stdout ^ok +stdout FAIL + +# Test fatal with testdata seed corpus +! go test -run FuzzFail corpustesting/fuzz_testdata_corpus_test.go +! stdout ^ok +stdout FAIL +stdout 'fatal here' + +# Test pass with testdata seed corpus +go test -run FuzzPass corpustesting/fuzz_testdata_corpus_test.go +stdout ok +! stdout FAIL +! stdout 'fatal here' + +# Test pass with testdata and f.Add seed corpus +go test -run FuzzPassString corpustesting/fuzz_testdata_corpus_test.go +stdout ok +! stdout FAIL + +# Fuzzing pass with testdata and f.Add seed corpus (skip running tests first) +go test -run=None -fuzz=FuzzPassString corpustesting/fuzz_testdata_corpus_test.go -fuzztime=10x +stdout ok +! stdout FAIL + +# Fuzzing pass with testdata and f.Add seed corpus +go test -run=FuzzPassString -fuzz=FuzzPassString corpustesting/fuzz_testdata_corpus_test.go -fuzztime=10x +stdout ok +! stdout FAIL + +# Test panic with malformed seed corpus +! go test -run FuzzFail corpustesting/fuzz_testdata_corpus_test.go +! stdout ^ok +stdout FAIL + +# Test pass with file in other nested testdata directory +go test -run FuzzInNestedDir corpustesting/fuzz_testdata_corpus_test.go +stdout ok +! stdout FAIL +! stdout 'fatal here' + +# Test fails with file containing wrong type +! go test -run FuzzWrongType corpustesting/fuzz_testdata_corpus_test.go +! stdout ^ok +stdout FAIL + +-- noop_fuzz_test.go -- +package noop_fuzz + +import "testing" + +func Fuzz(f *testing.F) {} + +-- error_fuzz_test.go -- +package error_fuzz + +import "testing" + +func Fuzz(f *testing.F) { + f.Error("error in target") +} + +-- fatal_fuzz_test.go -- +package fatal_fuzz + +import "testing" + +func Fuzz(f *testing.F) { + f.Fatal("fatal in target") +} + +-- panic_fuzz_test.go -- +package panic_fuzz + +import "testing" + +func FuzzPanic(f *testing.F) { + panic(nil) +} + +-- success_fuzz_test.go -- +package success_fuzz + +import "testing" + +func Fuzz(f *testing.F) { + f.Fuzz(func (*testing.T, []byte) {}) +} + +-- skipped_fuzz_test.go -- +package skipped_fuzz + +import "testing" + +func Fuzz(f *testing.F) { + f.Skip() +} + +-- fatal_fuzz_fn_fuzz_test.go -- +package fatal_fuzz_fn_fuzz + +import "testing" + +func Fuzz(f *testing.F) { + f.Add([]byte("aa")) + f.Fuzz(func(t *testing.T, b []byte) { + f.Fatal("fatal here") + }) +} + +-- error_fuzz_fn_fuzz_test.go -- +package error_fuzz_fn_fuzz + +import "testing" + +func Fuzz(f *testing.F) { + f.Add([]byte("aa")) + f.Fuzz(func(t *testing.T, b []byte) { + f.Error("error here") + }) +} + +-- skip_fuzz_fn_fuzz_test.go -- +package skip_fuzz_fn_fuzz + +import "testing" + +func Fuzz(f *testing.F) { + f.Add([]byte("aa")) + f.Fuzz(func(t *testing.T, b []byte) { + f.Skip("skip here") + }) +} + +-- fatal_after_fuzz_func_fuzz_test.go -- +package fatal_after_fuzz_func_fuzz + +import "testing" + +func Fuzz(f *testing.F) { + f.Fuzz(func(t *testing.T, b []byte) { + // no-op + }) + f.Fatal("this shouldn't be called") +} + +-- incomplete_fuzz_call_fuzz_test.go -- +package incomplete_fuzz_call_fuzz + +import "testing" + +func Fuzz(f *testing.F) { + f.Fuzz(func(b []byte) { + // this is missing *testing.T as the first param, so should panic + }) +} + +-- cleanup_fuzz_test.go -- +package cleanup_fuzz_test + +import "testing" + +func Fuzz(f *testing.F) { + f.Cleanup(func() { + panic("failed some precondition") + }) + f.Fuzz(func(t *testing.T, b []byte) { + // no-op + }) +} + +-- fuzz_add_test.go -- +package fuzz_add + +import "testing" + +func add(f *testing.F) { + f.Helper() + f.Add([]byte("123")) + f.Add([]byte("12345")) + f.Add([]byte("")) +} + +func FuzzPass(f *testing.F) { + add(f) + f.Fuzz(func(t *testing.T, b []byte) { + if len(b) == -1 { + t.Fatal("fatal here") // will not be executed + } + }) +} + +func FuzzError(f *testing.F) { + add(f) + f.Fuzz(func(t *testing.T, b []byte) { + if len(b) == 3 { + t.Error("error here") + } + }) +} + +func FuzzFatal(f *testing.F) { + add(f) + f.Fuzz(func(t *testing.T, b []byte) { + if len(b) == 0 { + t.Fatal("fatal here") + } + }) +} + +func FuzzPanic(f *testing.F) { + add(f) + f.Fuzz(func(t *testing.T, b []byte) { + if len(b) == 5 { + panic("off by one error") + } + }) +} + +func FuzzNilPanic(f *testing.F) { + add(f) + f.Fuzz(func(t *testing.T, b []byte) { + if len(b) == 3 { + panic(nil) + } + }) +} + +func FuzzUnsupported(f *testing.F) { + m := make(map[string]bool) + f.Add(m) + f.Fuzz(func(*testing.T, []byte) {}) +} + +func FuzzAddDifferentNumber(f *testing.F) { + f.Add([]byte("a")) + f.Add([]byte("a"), []byte("b")) + f.Fuzz(func(*testing.T, []byte) {}) +} + +func FuzzAddDifferentType(f *testing.F) { + f.Add(false) + f.Add(1234) + f.Fuzz(func(*testing.T, []byte) {}) +} + +func FuzzWrongType(f *testing.F) { + f.Add("hello") + f.Fuzz(func(*testing.T, []byte) {}) +} + +-- corpustesting/fuzz_testdata_corpus_test.go -- +package fuzz_testdata_corpus + +import "testing" + +func fuzzFn(f *testing.F) { + f.Helper() + f.Fuzz(func(t *testing.T, b []byte) { + if string(b) == "12345" { + t.Fatal("fatal here") + } + }) +} + +func FuzzFail(f *testing.F) { + fuzzFn(f) +} + +func FuzzPass(f *testing.F) { + fuzzFn(f) +} + +func FuzzPassString(f *testing.F) { + f.Add("some seed corpus") + f.Fuzz(func(*testing.T, string) {}) +} + +func FuzzPanic(f *testing.F) { + f.Fuzz(func(t *testing.T, b []byte) {}) +} + +func FuzzInNestedDir(f *testing.F) { + f.Fuzz(func(t *testing.T, b []byte) {}) +} + +func FuzzWrongType(f *testing.F) { + f.Fuzz(func(t *testing.T, b []byte) {}) +} + +-- corpustesting/testdata/fuzz/FuzzFail/1 -- +go test fuzz v1 +[]byte("12345") +-- corpustesting/testdata/fuzz/FuzzPass/1 -- +go test fuzz v1 +[]byte("00000") +-- corpustesting/testdata/fuzz/FuzzPassString/1 -- +go test fuzz v1 +string("hello") +-- corpustesting/testdata/fuzz/FuzzPanic/1 -- +malformed +-- corpustesting/testdata/fuzz/FuzzInNestedDir/anotherdir/1 -- +go test fuzz v1 +[]byte("12345") +-- corpustesting/testdata/fuzz/FuzzWrongType/1 -- +go test fuzz v1 +int("00000")
\ No newline at end of file diff --git a/src/cmd/go/testdata/script/test_fuzz_cache.txt b/src/cmd/go/testdata/script/test_fuzz_cache.txt new file mode 100644 index 00000000000..10e4c2926f4 --- /dev/null +++ b/src/cmd/go/testdata/script/test_fuzz_cache.txt @@ -0,0 +1,81 @@ +# TODO(jayconrod): support shared memory on more platforms. +[!darwin] [!linux] [!windows] skip + +[short] skip +env GOCACHE=$WORK/cache + +# Fuzz cache should not exist after a regular test run. +go test . +exists $GOCACHE +! exists $GOCACHE/fuzz + +# Fuzzing should write interesting values to the cache. +go test -fuzz=FuzzY -fuzztime=100x . +go run ./contains_files $GOCACHE/fuzz/example.com/y/FuzzY + +# 'go clean -cache' should not delete the fuzz cache. +go clean -cache +exists $GOCACHE/fuzz + +# 'go clean -fuzzcache' should delete the fuzz cache but not the build cache. +go list -f {{.Stale}} ./empty +stdout true +go install ./empty +go list -f {{.Stale}} ./empty +stdout false +go clean -fuzzcache +! exists $GOCACHE/fuzz +go list -f {{.Stale}} ./empty +stdout false + +-- go.mod -- +module example.com/y + +go 1.16 +-- y_test.go -- +package y + +import ( + "io" + "testing" +) + +func FuzzY(f *testing.F) { + f.Add([]byte("y")) + f.Fuzz(func(t *testing.T, b []byte) { Y(io.Discard, b) }) +} +-- y.go -- +package y + +import ( + "bytes" + "io" +) + +func Y(w io.Writer, b []byte) { + if !bytes.Equal(b, []byte("y")) { + w.Write([]byte("not equal")) + } +} +-- empty/empty.go -- +package empty +-- contains_files/contains_files.go -- +package main + +import ( + "fmt" + "path/filepath" + "io/ioutil" + "os" +) + +func main() { + infos, err := ioutil.ReadDir(filepath.Clean(os.Args[1])) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + if len(infos) == 0 { + os.Exit(1) + } +} diff --git a/src/cmd/go/testdata/script/test_fuzz_chatty.txt b/src/cmd/go/testdata/script/test_fuzz_chatty.txt new file mode 100644 index 00000000000..9ebd480c901 --- /dev/null +++ b/src/cmd/go/testdata/script/test_fuzz_chatty.txt @@ -0,0 +1,106 @@ +# TODO(jayconrod): support shared memory on more platforms. +[!darwin] [!linux] [!windows] skip + +[short] skip + +# Run chatty fuzz targets with an error. +! go test -v chatty_error_fuzz_test.go +! stdout '^ok' +stdout 'FAIL' +stdout 'error in target' + +# Run chatty fuzz targets with a fatal. +! go test -v chatty_fatal_fuzz_test.go +! stdout '^ok' +stdout 'FAIL' +stdout 'fatal in target' + +# Run chatty fuzz target with a panic +! go test -v chatty_panic_fuzz_test.go +! stdout ^ok +stdout FAIL +stdout 'this is bad' + +# Run skipped chatty fuzz targets. +go test -v chatty_skipped_fuzz_test.go +stdout ok +stdout SKIP +! stdout FAIL + +# Run successful chatty fuzz targets. +go test -v chatty_fuzz_test.go +stdout ok +stdout PASS +stdout 'all good here' +! stdout FAIL + +# Fuzz successful chatty fuzz target that includes a separate unit test. +go test -v chatty_with_test_fuzz_test.go -fuzz=Fuzz -fuzztime=1x +stdout ok +stdout PASS +! stdout FAIL +# TODO: It's currently the case that it's logged twice. Fix that, and change +# this check to verify it. +stdout 'all good here' +# Verify that the unit test is only run once. +! stdout '(?s)logged foo.*logged foo' + +-- chatty_error_fuzz_test.go -- +package chatty_error_fuzz + +import "testing" + +func Fuzz(f *testing.F) { + f.Error("error in target") +} + +-- chatty_fatal_fuzz_test.go -- +package chatty_fatal_fuzz + +import "testing" + +func Fuzz(f *testing.F) { + f.Fatal("fatal in target") +} + +-- chatty_panic_fuzz_test.go -- +package chatty_panic_fuzz + +import "testing" + +func Fuzz(f *testing.F) { + panic("this is bad") +} + +-- chatty_skipped_fuzz_test.go -- +package chatty_skipped_fuzz + +import "testing" + +func Fuzz(f *testing.F) { + f.Skip() +} + +-- chatty_fuzz_test.go -- +package chatty_fuzz + +import "testing" + +func Fuzz(f *testing.F) { + f.Log("all good here") + f.Fuzz(func(*testing.T, []byte) {}) +} + +-- chatty_with_test_fuzz_test.go -- +package chatty_with_test_fuzz + +import "testing" + +func TestFoo(t *testing.T) { + t.Log("logged foo") +} + +func Fuzz(f *testing.F) { + f.Log("all good here") + f.Fuzz(func(*testing.T, []byte) {}) +} diff --git a/src/cmd/go/testdata/script/test_fuzz_cleanup.txt b/src/cmd/go/testdata/script/test_fuzz_cleanup.txt new file mode 100644 index 00000000000..88625916bab --- /dev/null +++ b/src/cmd/go/testdata/script/test_fuzz_cleanup.txt @@ -0,0 +1,67 @@ +# TODO(jayconrod): support shared memory on more platforms. +[!darwin] [!linux] [!windows] skip +[short] skip + +# Cleanup should run after F.Skip. +go test -run=FuzzTargetSkip +stdout cleanup + +# Cleanup should run after F.Fatal. +! go test -run=FuzzTargetFatal +stdout cleanup + +# Cleanup should run after an unexpected runtime.Goexit. +! go test -run=FuzzTargetGoexit +stdout cleanup + +# Cleanup should run after panic. +! go test -run=FuzzTargetPanic +stdout cleanup + +# Cleanup should run in fuzz function on seed corpus. +go test -v -run=FuzzFunction +stdout '(?s)inner.*outer' + +# TODO(jayconrod): test cleanup while fuzzing. For now, the worker process's +# stdout and stderr is connected to the coordinator's, but it should eventually +# be connected to os.DevNull, so we wouldn't see t.Log output. + +-- go.mod -- +module cleanup + +go 1.15 +-- cleanup_test.go -- +package cleanup + +import ( + "runtime" + "testing" +) + +func FuzzTargetSkip(f *testing.F) { + f.Cleanup(func() { f.Log("cleanup") }) + f.Skip() +} + +func FuzzTargetFatal(f *testing.F) { + f.Cleanup(func() { f.Log("cleanup") }) + f.Fatal() +} + +func FuzzTargetGoexit(f *testing.F) { + f.Cleanup(func() { f.Log("cleanup") }) + runtime.Goexit() +} + +func FuzzTargetPanic(f *testing.F) { + f.Cleanup(func() { f.Log("cleanup") }) + panic("oh no") +} + +func FuzzFunction(f *testing.F) { + f.Add([]byte{0}) + f.Cleanup(func() { f.Log("outer") }) + f.Fuzz(func(t *testing.T, b []byte) { + t.Cleanup(func() { t.Logf("inner") }) + }) +} diff --git a/src/cmd/go/testdata/script/test_fuzz_deadline.txt b/src/cmd/go/testdata/script/test_fuzz_deadline.txt new file mode 100644 index 00000000000..12f1054f614 --- /dev/null +++ b/src/cmd/go/testdata/script/test_fuzz_deadline.txt @@ -0,0 +1,37 @@ +# TODO(jayconrod): support shared memory on more platforms. +[!darwin] [!linux] [!windows] skip + +[short] skip + +# The fuzz function should be able to detect whether -timeout +# was set with T.Deadline. Note there is no F.Deadline, and +# there is no timeout while fuzzing, even if -fuzztime is set. +go test -run=FuzzDeadline -wantdeadline=true # -timeout defaults to 10m +go test -run=FuzzDeadline -timeout=0 -wantdeadline=false +! go test -run=FuzzDeadline -timeout=1s -wantdeadline=false +go test -run=FuzzDeadline -timeout=1s -wantdeadline=true +go test -fuzz=FuzzDeadline -timeout=0 -fuzztime=1s -wantdeadline=false +go test -fuzz=FuzzDeadline -timeout=0 -fuzztime=100x -wantdeadline=false + +-- go.mod -- +module fuzz + +go 1.16 +-- fuzz_deadline_test.go -- +package fuzz_test + +import ( + "flag" + "testing" +) + +var wantDeadline = flag.Bool("wantdeadline", false, "whether the test should have a deadline") + +func FuzzDeadline(f *testing.F) { + f.Add("run once") + f.Fuzz(func (t *testing.T, _ string) { + if _, hasDeadline := t.Deadline(); hasDeadline != *wantDeadline { + t.Fatalf("function got %v; want %v", hasDeadline, *wantDeadline) + } + }) +} diff --git a/src/cmd/go/testdata/script/test_fuzz_fuzztime.txt b/src/cmd/go/testdata/script/test_fuzz_fuzztime.txt new file mode 100644 index 00000000000..7d644b4d132 --- /dev/null +++ b/src/cmd/go/testdata/script/test_fuzz_fuzztime.txt @@ -0,0 +1,82 @@ +# TODO(jayconrod): support shared memory on more platforms. +[!darwin] [!linux] [!windows] skip + +[short] skip + +# There are no seed values, so 'go test' should finish quickly. +go test + +# Fuzzing should exit 0 after fuzztime, even if timeout is short. +go test -timeout=10ms -fuzz=FuzzFast -fuzztime=5s + +# We should see the same behavior when invoking the test binary directly. +go test -c +exec ./fuzz.test$GOEXE -test.timeout=10ms -test.fuzz=FuzzFast -test.fuzztime=5s -test.parallel=1 -test.fuzzcachedir=$WORK/cache + +# Timeout should not cause inputs to be written as crashers. +! exists testdata/fuzz + +# When we use fuzztime with an "x" suffix, it runs a specific number of times. +# This fuzz function creates a file with a unique name ($pid.$count) on each run. +# We count the files to find the number of runs. +mkdir count +env GOCACHE=$WORK/tmp +go test -fuzz=FuzzCount -fuzztime=1000x -fuzzminimizetime=1x +go run check_file_count.go 1000 + +-- go.mod -- +module fuzz + +go 1.16 +-- fuzz_fast_test.go -- +package fuzz_test + +import "testing" + +func FuzzFast(f *testing.F) { + f.Fuzz(func (*testing.T, []byte) {}) +} +-- fuzz_count_test.go -- +package fuzz + +import ( + "fmt" + "os" + "testing" +) + +func FuzzCount(f *testing.F) { + pid := os.Getpid() + n := 0 + f.Fuzz(func(t *testing.T, _ []byte) { + name := fmt.Sprintf("count/%v.%d", pid, n) + if err := os.WriteFile(name, nil, 0666); err != nil { + t.Fatal(err) + } + n++ + }) +} +-- check_file_count.go -- +// +build ignore + +package main + +import ( + "fmt" + "os" + "strconv" +) + +func main() { + dir, err := os.ReadDir("count") + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + got := len(dir) + want, _ := strconv.Atoi(os.Args[1]) + if got != want { + fmt.Fprintf(os.Stderr, "got %d files; want %d\n", got, want) + os.Exit(1) + } +} diff --git a/src/cmd/go/testdata/script/test_fuzz_io_error.txt b/src/cmd/go/testdata/script/test_fuzz_io_error.txt new file mode 100644 index 00000000000..4c7ab4c1523 --- /dev/null +++ b/src/cmd/go/testdata/script/test_fuzz_io_error.txt @@ -0,0 +1,101 @@ +# Test that when the coordinator experiences an I/O error communicating +# with a worker, the coordinator stops the worker and reports the error. +# The coordinator should not record a crasher. +# +# We simulate an I/O error in the test by writing garbage to fuzz_out. +# This is unlikely, but possible. It's difficult to simulate interruptions +# due to ^C and EOF errors which are more common. We don't report those. +[short] skip +[!darwin] [!linux] [!windows] skip + +# If the I/O error occurs before F.Fuzz is called, the coordinator should +# stop the worker and say that. +! go test -fuzz=FuzzClosePipeBefore -parallel=1 +stdout '\s*fuzzing process terminated without fuzzing:' +! stdout 'communicating with fuzzing process' +! exists testdata + +# If the I/O error occurs after F.Fuzz is called (unlikely), just exit. +# It's hard to distinguish this case from the worker being interrupted by ^C +# or exiting with status 0 (which it should do when interrupted by ^C). +! go test -fuzz=FuzzClosePipeAfter -parallel=1 +stdout '^\s*communicating with fuzzing process: invalid character ''!'' looking for beginning of value$' +! exists testdata + +-- go.mod -- +module test + +go 1.17 +-- io_error_test.go -- +package io_error + +import ( + "flag" + "testing" + "time" +) + +func isWorker() bool { + f := flag.Lookup("test.fuzzworker") + if f == nil { + return false + } + get, ok := f.Value.(flag.Getter) + if !ok { + return false + } + return get.Get() == interface{}(true) +} + +func FuzzClosePipeBefore(f *testing.F) { + if isWorker() { + sendGarbageToCoordinator(f) + time.Sleep(3600 * time.Second) // pause until coordinator terminates the process + } + f.Fuzz(func(*testing.T, []byte) {}) +} + +func FuzzClosePipeAfter(f *testing.F) { + f.Fuzz(func(t *testing.T, _ []byte) { + if isWorker() { + sendGarbageToCoordinator(t) + time.Sleep(3600 * time.Second) // pause until coordinator terminates the process + } + }) +} +-- io_error_windows_test.go -- +package io_error + +import ( + "fmt" + "os" + "testing" +) + +func sendGarbageToCoordinator(tb testing.TB) { + v := os.Getenv("GO_TEST_FUZZ_WORKER_HANDLES") + var fuzzInFD, fuzzOutFD uintptr + if _, err := fmt.Sscanf(v, "%x,%x", &fuzzInFD, &fuzzOutFD); err != nil { + tb.Fatalf("parsing GO_TEST_FUZZ_WORKER_HANDLES: %v", err) + } + f := os.NewFile(fuzzOutFD, "fuzz_out") + if _, err := f.Write([]byte("!!")); err != nil { + tb.Fatalf("writing fuzz_out: %v", err) + } +} +-- io_error_notwindows_test.go -- +// +build !windows + +package io_error + +import ( + "os" + "testing" +) + +func sendGarbageToCoordinator(tb testing.TB) { + f := os.NewFile(4, "fuzz_out") + if _, err := f.Write([]byte("!!")); err != nil { + tb.Fatalf("writing fuzz_out: %v", err) + } +} diff --git a/src/cmd/go/testdata/script/test_fuzz_match.txt b/src/cmd/go/testdata/script/test_fuzz_match.txt new file mode 100644 index 00000000000..3a2ca631add --- /dev/null +++ b/src/cmd/go/testdata/script/test_fuzz_match.txt @@ -0,0 +1,39 @@ +# TODO(jayconrod): support shared memory on more platforms. +[!darwin] [!linux] [!windows] skip + +# Matches only fuzz targets to test. +go test standalone_fuzz_test.go +! stdout '^ok.*\[no tests to run\]' +stdout '^ok' + +# Matches only for fuzzing. +go test -fuzz Fuzz -fuzztime 1x standalone_fuzz_test.go +! stdout '^ok.*\[no tests to run\]' +stdout '^ok' + +# Matches none for fuzzing but will run the fuzz target as a test. +go test -fuzz ThisWillNotMatch -fuzztime 1x standalone_fuzz_test.go +! stdout '^ok.*no tests to run' +stdout '^ok' +stdout 'no targets to fuzz' + +[short] stop + +# Matches only fuzz targets to test with -run. +go test -run Fuzz standalone_fuzz_test.go +! stdout '^ok.*\[no tests to run\]' +stdout '^ok' + +# Matches no fuzz targets. +go test -run ThisWillNotMatch standalone_fuzz_test.go +stdout '^ok.*no tests to run' +! stdout 'no targets to fuzz' + +-- standalone_fuzz_test.go -- +package standalone_fuzz + +import "testing" + +func Fuzz(f *testing.F) { + f.Fuzz(func (*testing.T, []byte) {}) +} diff --git a/src/cmd/go/testdata/script/test_fuzz_minimize.txt b/src/cmd/go/testdata/script/test_fuzz_minimize.txt new file mode 100644 index 00000000000..f0adb9ec3e5 --- /dev/null +++ b/src/cmd/go/testdata/script/test_fuzz_minimize.txt @@ -0,0 +1,200 @@ +# TODO(jayconrod): support shared memory on more platforms. +[!darwin] [!linux] [!windows] skip + +[short] skip + +# We clean the fuzz cache during this test. Don't clean the user's cache. +env GOCACHE=$WORK/gocache + +# Test that fuzzminimizetime cannot be negative seconds +! go test -fuzz=FuzzMinimizerRecoverable -run=FuzzMinimizerRecoverable -fuzztime=10000x -fuzzminimizetime=-1ms minimizer_test.go +! stdout '^ok' +! stdout 'contains a non-zero byte' +stdout 'invalid duration' +stdout FAIL + +# Test that fuzzminimizetime cannot be negative times +! go test -fuzz=FuzzMinimizerRecoverable -run=FuzzMinimizerRecoverable -fuzztime=10000x -fuzzminimizetime=-1x minimizer_test.go +! stdout '^ok' +! stdout 'contains a non-zero byte' +stdout 'invalid count' +stdout FAIL + +# Test that fuzzminimizetime can be zero seconds, and minimization is disabled +! go test -fuzz=FuzzMinimizeZeroDurationSet -run=FuzzMinimizeZeroDurationSet -fuzztime=10000x -fuzzminimizetime=0s minimizer_test.go +! stdout '^ok' +! stdout 'minimizing' +stdout 'there was an Error' +stdout FAIL + +# Test that fuzzminimizetime can be zero times, and minimization is disabled +! go test -fuzz=FuzzMinimizeZeroLimitSet -run=FuzzMinimizeZeroLimitSet -fuzztime=10000x -fuzzminimizetime=0x minimizer_test.go +! stdout '^ok' +! stdout 'minimizing' +stdout 'there was an Error' +stdout FAIL + +# Test that minimization is working for recoverable errors. +! go test -fuzz=FuzzMinimizerRecoverable -run=FuzzMinimizerRecoverable -fuzztime=10000x minimizer_test.go +! stdout '^ok' +stdout 'got the minimum size!' +stdout 'contains a non-zero byte' +stdout FAIL + +# Check that the bytes written to testdata are of length 50 (the minimum size) +go run check_testdata.go FuzzMinimizerRecoverable 50 + +# Test that re-running the minimized value causes a crash. +! go test -run=FuzzMinimizerRecoverable minimizer_test.go +rm testdata + +# Test that minimization is working for non-recoverable errors. +! go test -fuzz=FuzzMinimizerNonrecoverable -run=FuzzMinimizerNonrecoverable -fuzztime=10000x minimizer_test.go +! stdout '^ok' +stdout 'minimizing' +stdout 'fuzzing process terminated unexpectedly: exit status 99' +stdout FAIL + +# Check that re-running the value causes a crash. +! go test -run=FuzzMinimizerNonrecoverable minimizer_test.go +rm testdata + +# Clear the fuzzing cache. There may already be minimized inputs that would +# interfere with the next stage of the test. +go clean -fuzzcache + +# Test that minimization can be cancelled by fuzzminimizetime and the latest +# crash will still be logged and written to testdata. +! go test -fuzz=FuzzMinimizerRecoverable -run=FuzzMinimizerRecoverable -fuzztime=100x -fuzzminimizetime=1x minimizer_test.go +! stdout '^ok' +stdout 'testdata[/\\]fuzz[/\\]FuzzMinimizerRecoverable[/\\]' +! stdout 'got the minimum size!' # it shouldn't have had enough time to minimize it +stdout FAIL + +# Test that re-running the unminimized value causes a crash. +! go test -run=FuzzMinimizerRecoverable minimizer_test.go + +# TODO(jayconrod,katiehockman): add a test which verifies that the right bytes +# are written to testdata in the case of an interrupt during minimization. + +-- go.mod -- +module m + +go 1.16 +-- minimizer_test.go -- +package fuzz_test + +import ( + "os" + "testing" +) + +func FuzzMinimizeZeroDurationSet(f *testing.F) { + f.Fuzz(func(t *testing.T, b []byte) { + if len(b) > 5 { + t.Errorf("there was an Error") + } + }) +} + +func FuzzMinimizeZeroLimitSet(f *testing.F) { + f.Fuzz(func(t *testing.T, b []byte) { + if len(b) > 5 { + t.Errorf("there was an Error") + } + }) +} + +func FuzzMinimizerRecoverable(f *testing.F) { + f.Add(make([]byte, 100)) + f.Fuzz(func(t *testing.T, b []byte) { + if len(b) < 50 { + // Make sure that b is large enough that it can be minimized + return + } + // Given the randomness of the mutations, this should allow the + // minimizer to trim down the value a bit. + for _, n := range b { + if n != 0 { + if len(b) == 50 { + t.Log("got the minimum size!") + } + t.Fatal("contains a non-zero byte") + } + } + }) +} + +func FuzzMinimizerNonrecoverable(f *testing.F) { + f.Add(make([]byte, 100)) + f.Fuzz(func(t *testing.T, b []byte) { + if len(b) < 50 { + // Make sure that b is large enough that it can be minimized + return + } + // Given the randomness of the mutations, this should allow the + // minimizer to trim down the value a bit. + for _, n := range b { + if n != 0 { + t.Log("contains a non-zero byte") + os.Exit(99) + } + } + }) +} +-- check_testdata.go -- +// +build ignore + +package main + +import ( + "bytes" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strconv" +) + +func main() { + target := os.Args[1] + numBytes, err := strconv.Atoi(os.Args[2]) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + + // Open the file in testdata (there should only be one) + dir := fmt.Sprintf("testdata/fuzz/%s", target) + files, err := ioutil.ReadDir(dir) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + if len(files) != 1 { + fmt.Fprintf(os.Stderr, "expected one file, got %d", len(files)) + os.Exit(1) + } + got, err := ioutil.ReadFile(filepath.Join(dir, files[0].Name())) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + + // Trim the newline at the end of the file + got = bytes.TrimSpace(got) + + // Make sure that there were exactly 100 bytes written to the corpus entry + prefix := []byte("[]byte(") + i := bytes.Index(got, prefix) + gotBytes := got[i+len(prefix) : len(got)-1] + s, err := strconv.Unquote(string(gotBytes)) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + if want, got := numBytes, len(s); want != got { + fmt.Fprintf(os.Stderr, "want %d bytes, got %d\n", want, got) + os.Exit(1) + } +} diff --git a/src/cmd/go/testdata/script/test_fuzz_minimize_interesting.txt b/src/cmd/go/testdata/script/test_fuzz_minimize_interesting.txt new file mode 100644 index 00000000000..5e1d90d8d9b --- /dev/null +++ b/src/cmd/go/testdata/script/test_fuzz_minimize_interesting.txt @@ -0,0 +1,112 @@ +# TODO(jayconrod): support shared memory on more platforms. +[!darwin] [!linux] [!windows] skip + +# Instrumentation only supported on 64-bit architectures. +[!amd64] [!arm64] skip + +# Test that when an interesting value is discovered (one that expands coverage), +# the fuzzing engine minimizes it before writing it to the cache. +# +# The program below starts with a seed value of length 100, but more coverage +# will be found for any value other than the seed. We should end with a value +# in the cache of length 1 (the minimizer currently does not produce empty +# strings). check_cache.go confirms that. +# +# We would like to verify that ALL values in the cache were minimized to a +# length of 1, but this isn't always possible when new coverage is found in +# functions called by testing or internal/fuzz in the background. + +go test -c -fuzz=. # Build using shared build cache for speed. +env GOCACHE=$WORK/gocache +exec ./fuzz.test$GOEXE -test.fuzzcachedir=$GOCACHE/fuzz -test.fuzz=. -test.fuzztime=1000x +go run check_cache.go $GOCACHE/fuzz/FuzzMin + +-- go.mod -- +module fuzz + +go 1.17 +-- fuzz_test.go -- +package fuzz + +import ( + "bytes" + "testing" +) + +func FuzzMin(f *testing.F) { + seed := bytes.Repeat([]byte("a"), 20) + f.Add(seed) + f.Fuzz(func(t *testing.T, buf []byte) { + if bytes.Equal(buf, seed) { + return + } + if n := sum(buf); n < 0 { + t.Error("sum cannot be negative") + } + }) +} + +func sum(buf []byte) int { + n := 0 + for _, b := range buf { + n += int(b) + } + return n +} +-- check_cache.go -- +//go:build ignore +// +build ignore + +// check_cache.go checks that each file in the cached corpus has a []byte +// of length at most 1. This verifies that at least one cached input is minimized. +package main + +import ( + "bytes" + "fmt" + "os" + "path/filepath" + "regexp" + "strconv" +) + +func main() { + dir := os.Args[1] + ents, err := os.ReadDir(dir) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + for _, ent := range ents { + name := filepath.Join(dir, ent.Name()) + if good, err := checkCacheFile(name); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } else if good { + os.Exit(0) + } + } + fmt.Fprintln(os.Stderr, "no cached inputs were minimized") + os.Exit(1) +} + +func checkCacheFile(name string) (good bool, err error) { + data, err := os.ReadFile(name) + if err != nil { + return false, err + } + for _, line := range bytes.Split(data, []byte("\n")) { + m := valRe.FindSubmatch(line) + if m == nil { + continue + } + if s, err := strconv.Unquote(string(m[1])); err != nil { + return false, err + } else if len(s) <= 1 { + return true, nil + } + } + return false, nil +} + +var valRe = regexp.MustCompile(`^\[\]byte\(([^)]+)\)$`) diff --git a/src/cmd/go/testdata/script/test_fuzz_multiple.txt b/src/cmd/go/testdata/script/test_fuzz_multiple.txt new file mode 100644 index 00000000000..6a7732f5149 --- /dev/null +++ b/src/cmd/go/testdata/script/test_fuzz_multiple.txt @@ -0,0 +1,51 @@ +# This test checks that 'go test' prints a reasonable error when fuzzing is +# enabled, and multiple package or multiple fuzz targets match. +# TODO(#46312): support fuzzing multiple targets in multiple packages. + +# TODO(jayconrod): support shared memory on more platforms. +[!darwin] [!linux] [!windows] skip + +[short] skip + +# With fuzzing disabled, multiple targets can be tested. +go test ./... + +# With fuzzing enabled, at most one package may be tested, +# even if only one package contains fuzz targets. +! go test -fuzz=. ./... +stderr '^cannot use -fuzz flag with multiple packages$' +! go test -fuzz=. ./zero ./one +stderr '^cannot use -fuzz flag with multiple packages$' +go test -fuzz=. -fuzztime=1x ./one + +# With fuzzing enabled, at most one target in the same package may match. +! go test -fuzz=. ./two +stdout '^testing: will not fuzz, -fuzz matches more than one target: \[FuzzOne FuzzTwo\]$' +go test -fuzz=FuzzTwo -fuzztime=1x ./two + +-- go.mod -- +module fuzz + +go 1.18 +-- zero/zero.go -- +package zero +-- one/one_test.go -- +package one + +import "testing" + +func FuzzOne(f *testing.F) { + f.Fuzz(func(*testing.T, []byte) {}) +} +-- two/two_test.go -- +package two + +import "testing" + +func FuzzOne(f *testing.F) { + f.Fuzz(func(*testing.T, []byte) {}) +} + +func FuzzTwo(f *testing.F) { + f.Fuzz(func(*testing.T, []byte) {}) +} diff --git a/src/cmd/go/testdata/script/test_fuzz_mutate_crash.txt b/src/cmd/go/testdata/script/test_fuzz_mutate_crash.txt new file mode 100644 index 00000000000..1b8b79b3dde --- /dev/null +++ b/src/cmd/go/testdata/script/test_fuzz_mutate_crash.txt @@ -0,0 +1,295 @@ +# TODO(jayconrod): support shared memory on more platforms. +[!darwin] [!linux] [!windows] skip + +# Tests that a crash caused by a mutator-discovered input writes the bad input +# to testdata, and fails+reports correctly. This tests the end-to-end behavior +# of the mutator finding a crash while fuzzing, adding it as a regression test +# to the seed corpus in testdata, and failing the next time the test is run. + +[short] skip + +# Running the seed corpus for all of the targets should pass the first +# time, since nothing in the seed corpus will cause a crash. +go test + +# Running the fuzzer should find a crashing input quickly. +! go test -fuzz=FuzzWithBug -fuzztime=100x -fuzzminimizetime=1000x +stdout 'testdata[/\\]fuzz[/\\]FuzzWithBug[/\\]' +stdout 'this input caused a crash!' +go run check_testdata.go FuzzWithBug + +# Now, the failing bytes should have been added to the seed corpus for +# the target, and should fail when run without fuzzing. +! go test +stdout 'testdata[/\\]fuzz[/\\]FuzzWithBug[/\\][a-f0-9]{64}' +stdout 'this input caused a crash!' + +! go test -run=FuzzWithNilPanic -fuzz=FuzzWithNilPanic -fuzztime=100x -fuzzminimizetime=1000x +stdout 'testdata[/\\]fuzz[/\\]FuzzWithNilPanic[/\\]' +stdout 'runtime.Goexit' +go run check_testdata.go FuzzWithNilPanic + +! go test -run=FuzzWithFail -fuzz=FuzzWithFail -fuzztime=100x -fuzzminimizetime=1000x +stdout 'testdata[/\\]fuzz[/\\]FuzzWithFail[/\\]' +go run check_testdata.go FuzzWithFail + +! go test -run=FuzzWithLogFail -fuzz=FuzzWithLogFail -fuzztime=100x -fuzzminimizetime=1000x +stdout 'testdata[/\\]fuzz[/\\]FuzzWithLogFail[/\\]' +stdout 'logged something' +go run check_testdata.go FuzzWithLogFail + +! go test -run=FuzzWithErrorf -fuzz=FuzzWithErrorf -fuzztime=100x -fuzzminimizetime=1000x +stdout 'testdata[/\\]fuzz[/\\]FuzzWithErrorf[/\\]' +stdout 'errorf was called here' +go run check_testdata.go FuzzWithErrorf + +! go test -run=FuzzWithFatalf -fuzz=FuzzWithFatalf -fuzztime=100x -fuzzminimizetime=1000x +stdout 'testdata[/\\]fuzz[/\\]FuzzWithFatalf[/\\]' +stdout 'fatalf was called here' +go run check_testdata.go FuzzWithFatalf + +! go test -run=FuzzWithBadExit -fuzz=FuzzWithBadExit -fuzztime=100x -fuzzminimizetime=1000x +stdout 'testdata[/\\]fuzz[/\\]FuzzWithBadExit[/\\]' +stdout 'unexpectedly' +go run check_testdata.go FuzzWithBadExit + +# Running the fuzzer should find a crashing input quickly for fuzzing two types. +! go test -run=FuzzWithTwoTypes -fuzz=FuzzWithTwoTypes -fuzztime=100x -fuzzminimizetime=1000x +stdout 'testdata[/\\]fuzz[/\\]FuzzWithTwoTypes[/\\]' +stdout 'these inputs caused a crash!' +go run check_testdata.go FuzzWithTwoTypes + +# Running the fuzzer should find a crashing input quickly for an integer. +! go test -run=FuzzInt -fuzz=FuzzInt -fuzztime=100x -fuzzminimizetime=1000x +stdout 'testdata[/\\]fuzz[/\\]FuzzInt[/\\]' +stdout 'this input caused a crash!' +go run check_testdata.go FuzzInt + +! go test -run=FuzzUint -fuzz=FuzzUint -fuzztime=100x -fuzzminimizetime=1000x +stdout 'testdata[/\\]fuzz[/\\]FuzzUint[/\\]' +stdout 'this input caused a crash!' +go run check_testdata.go FuzzUint + +# Running the fuzzer should find a crashing input quickly for a bool. +! go test -run=FuzzBool -fuzz=FuzzBool -fuzztime=100x -fuzzminimizetime=1000x +stdout 'testdata[/\\]fuzz[/\\]FuzzBool[/\\]' +stdout 'this input caused a crash!' +go run check_testdata.go FuzzBool + +# Running the fuzzer should find a crashing input quickly for a float. +! go test -run=FuzzFloat -fuzz=FuzzFloat -fuzztime=100x -fuzzminimizetime=1000x +stdout 'testdata[/\\]fuzz[/\\]FuzzFloat[/\\]' +stdout 'this input caused a crash!' +go run check_testdata.go FuzzFloat + +# Running the fuzzer should find a crashing input quickly for a byte. +! go test -run=FuzzByte -fuzz=FuzzByte -fuzztime=100x -fuzzminimizetime=1000x +stdout 'testdata[/\\]fuzz[/\\]FuzzByte[/\\]' +stdout 'this input caused a crash!' +go run check_testdata.go FuzzByte + +# Running the fuzzer should find a crashing input quickly for a rune. +! go test -run=FuzzRune -fuzz=FuzzRune -fuzztime=100x -fuzzminimizetime=1000x +stdout 'testdata[/\\]fuzz[/\\]FuzzRune[/\\]' +stdout 'this input caused a crash!' +go run check_testdata.go FuzzRune + +# Running the fuzzer should find a crashing input quickly for a string. +! go test -run=FuzzString -fuzz=FuzzString -fuzztime=100x -fuzzminimizetime=1000x +stdout 'testdata[/\\]fuzz[/\\]FuzzString[/\\]' +stdout 'this input caused a crash!' +go run check_testdata.go FuzzString + +-- go.mod -- +module m + +go 1.16 +-- fuzz_crash_test.go -- +package fuzz_crash + +import ( + "os" + "testing" +) + +func FuzzWithBug(f *testing.F) { + f.Add([]byte("aa")) + f.Fuzz(func(t *testing.T, b []byte) { + if string(b) != "aa" { + panic("this input caused a crash!") + } + }) +} + +func FuzzWithNilPanic(f *testing.F) { + f.Add([]byte("aa")) + f.Fuzz(func(t *testing.T, b []byte) { + if string(b) != "aa" { + panic(nil) + } + }) +} + +func FuzzWithFail(f *testing.F) { + f.Add([]byte("aa")) + f.Fuzz(func(t *testing.T, b []byte) { + if string(b) != "aa" { + t.Fail() + } + }) +} + +func FuzzWithLogFail(f *testing.F) { + f.Add([]byte("aa")) + f.Fuzz(func(t *testing.T, b []byte) { + if string(b) != "aa" { + t.Log("logged something") + t.Fail() + } + }) +} + +func FuzzWithErrorf(f *testing.F) { + f.Add([]byte("aa")) + f.Fuzz(func(t *testing.T, b []byte) { + if string(b) != "aa" { + t.Errorf("errorf was called here") + } + }) +} + +func FuzzWithFatalf(f *testing.F) { + f.Add([]byte("aa")) + f.Fuzz(func(t *testing.T, b []byte) { + if string(b) != "aa" { + t.Fatalf("fatalf was called here") + } + }) +} + +func FuzzWithBadExit(f *testing.F) { + f.Add([]byte("aa")) + f.Fuzz(func(t *testing.T, b []byte) { + if string(b) != "aa" { + os.Exit(1) + } + }) +} + +func FuzzWithTwoTypes(f *testing.F) { + f.Fuzz(func(t *testing.T, a, b []byte) { + if len(a) > 0 && len(b) > 0 { + panic("these inputs caused a crash!") + } + }) +} + +func FuzzInt(f *testing.F) { + f.Add(0) + f.Fuzz(func(t *testing.T, a int) { + if a != 0 { + panic("this input caused a crash!") + } + }) +} + +func FuzzUint(f *testing.F) { + f.Add(uint(0)) + f.Fuzz(func(t *testing.T, a uint) { + if a != 0 { + panic("this input caused a crash!") + } + }) +} + +func FuzzBool(f *testing.F) { + f.Add(false) + f.Fuzz(func(t *testing.T, a bool) { + if a { + panic("this input caused a crash!") + } + }) +} + +func FuzzFloat(f *testing.F) { + f.Fuzz(func(t *testing.T, a float64) { + if a != float64(int64(a)) { + // It has a decimal, so it was mutated by division + panic("this input caused a crash!") + } + }) +} + +func FuzzByte(f *testing.F) { + f.Add(byte(0)) + f.Fuzz(func(t *testing.T, a byte) { + if a != 0 { + panic("this input caused a crash!") + } + }) +} + +func FuzzRune(f *testing.F) { + f.Add(rune(0)) + f.Fuzz(func(t *testing.T, a rune) { + if a != 0 { + panic("this input caused a crash!") + } + }) +} + +func FuzzString(f *testing.F) { + f.Add("") + f.Fuzz(func(t *testing.T, a string) { + if a != "" { + panic("this input caused a crash!") + } + }) +} + +-- check_testdata.go -- +// +build ignore + +package main + +import ( + "bytes" + "crypto/sha256" + "fmt" + "io/ioutil" + "os" + "path/filepath" +) + +func main() { + target := os.Args[1] + dir := filepath.Join("testdata/fuzz", target) + + files, err := ioutil.ReadDir(dir) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + + if len(files) == 0 { + fmt.Fprintf(os.Stderr, "expect at least one new mutation to be written to testdata\n") + os.Exit(1) + } + + fname := files[0].Name() + contents, err := ioutil.ReadFile(filepath.Join(dir, fname)) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + if bytes.Equal(contents, []byte("aa")) { + fmt.Fprintf(os.Stderr, "newly written testdata entry was not mutated\n") + os.Exit(1) + } + // The hash of the bytes in the file should match the filename. + h := []byte(fmt.Sprintf("%x", sha256.Sum256(contents))) + if !bytes.Equal([]byte(fname), h) { + fmt.Fprintf(os.Stderr, "hash of bytes %q does not match filename %q\n", h, fname) + os.Exit(1) + } +} diff --git a/src/cmd/go/testdata/script/test_fuzz_mutate_fail.txt b/src/cmd/go/testdata/script/test_fuzz_mutate_fail.txt new file mode 100644 index 00000000000..935c22a05e2 --- /dev/null +++ b/src/cmd/go/testdata/script/test_fuzz_mutate_fail.txt @@ -0,0 +1,103 @@ +# TODO(jayconrod): support shared memory on more platforms. +[!darwin] [!linux] [!windows] skip + +# Check that if a worker does not call F.Fuzz or calls F.Fail first, +# 'go test' exits non-zero and no crasher is recorded. + +[short] skip + +! go test -fuzz=FuzzReturn +! exists testdata + +! go test -fuzz=FuzzSkip +! exists testdata + +! go test -fuzz=FuzzFail +! exists testdata + +! go test -fuzz=FuzzPanic +! exists testdata + +! go test -fuzz=FuzzNilPanic +! exists testdata + +! go test -fuzz=FuzzGoexit +! exists testdata + +! go test -fuzz=FuzzExit +! exists testdata + +-- go.mod -- +module m + +go 1.17 +-- fuzz_fail_test.go -- +package fuzz_fail + +import ( + "flag" + "os" + "runtime" + "testing" +) + +func isWorker() bool { + f := flag.Lookup("test.fuzzworker") + if f == nil { + return false + } + get, ok := f.Value.(flag.Getter) + if !ok { + return false + } + return get.Get() == interface{}(true) +} + +func FuzzReturn(f *testing.F) { + if isWorker() { + return + } + f.Fuzz(func(*testing.T, []byte) {}) +} + +func FuzzSkip(f *testing.F) { + if isWorker() { + f.Skip() + } + f.Fuzz(func(*testing.T, []byte) {}) +} + +func FuzzFail(f *testing.F) { + if isWorker() { + f.Fail() + } + f.Fuzz(func(*testing.T, []byte) {}) +} + +func FuzzPanic(f *testing.F) { + if isWorker() { + panic("nope") + } + f.Fuzz(func(*testing.T, []byte) {}) +} + +func FuzzNilPanic(f *testing.F) { + if isWorker() { + panic(nil) + } + f.Fuzz(func(*testing.T, []byte) {}) +} + +func FuzzGoexit(f *testing.F) { + if isWorker() { + runtime.Goexit() + } + f.Fuzz(func(*testing.T, []byte) {}) +} + +func FuzzExit(f *testing.F) { + if isWorker() { + os.Exit(99) + } + f.Fuzz(func(*testing.T, []byte) {}) +} diff --git a/src/cmd/go/testdata/script/test_fuzz_mutator.txt b/src/cmd/go/testdata/script/test_fuzz_mutator.txt new file mode 100644 index 00000000000..9d0738e1693 --- /dev/null +++ b/src/cmd/go/testdata/script/test_fuzz_mutator.txt @@ -0,0 +1,166 @@ +# TODO(jayconrod): support shared memory on more platforms. +[!darwin] [!linux] [!windows] skip + +# Test basic fuzzing mutator behavior. +# +# fuzz_test.go has two fuzz targets (FuzzA, FuzzB) which both add a seed value. +# Each fuzz function writes the input to a log file. The coordinator and worker +# use separate log files. check_logs.go verifies that the coordinator only +# tests seed values and the worker tests mutated values on the fuzz target. + +[short] skip + +go test -fuzz=FuzzA -fuzztime=100x -parallel=1 -log=fuzz +go run check_logs.go fuzz fuzz.worker + +# TODO(b/181800488): remove -parallel=1, here and below. For now, when a +# crash is found, all workers keep running, wasting resources and reducing +# the number of executions available to the minimizer, increasing flakiness. + +# Test that the mutator is good enough to find several unique mutations. +! go test -fuzz=FuzzMutator -parallel=1 -fuzztime=100x mutator_test.go +! stdout '^ok' +stdout FAIL +stdout 'mutator found enough unique mutations' + +-- go.mod -- +module m + +go 1.16 +-- fuzz_test.go -- +package fuzz_test + +import ( + "flag" + "fmt" + "os" + "testing" +) + +var ( + logPath = flag.String("log", "", "path to log file") + logFile *os.File +) + +func TestMain(m *testing.M) { + flag.Parse() + var err error + logFile, err = os.OpenFile(*logPath, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0666) + if os.IsExist(err) { + *logPath += ".worker" + logFile, err = os.OpenFile(*logPath, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0666) + } + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + os.Exit(m.Run()) +} + +func FuzzA(f *testing.F) { + f.Add([]byte("seed")) + f.Fuzz(func(t *testing.T, b []byte) { + fmt.Fprintf(logFile, "FuzzA %q\n", b) + }) +} + +func FuzzB(f *testing.F) { + f.Add([]byte("seed")) + f.Fuzz(func(t *testing.T, b []byte) { + fmt.Fprintf(logFile, "FuzzB %q\n", b) + }) +} + +-- check_logs.go -- +// +build ignore + +package main + +import ( + "bufio" + "bytes" + "fmt" + "io" + "os" + "strings" +) + +func main() { + coordPath, workerPath := os.Args[1], os.Args[2] + + coordLog, err := os.Open(coordPath) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + defer coordLog.Close() + if err := checkCoordLog(coordLog); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + + workerLog, err := os.Open(workerPath) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + defer workerLog.Close() + if err := checkWorkerLog(workerLog); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + +func checkCoordLog(r io.Reader) error { + b, err := io.ReadAll(r) + if err != nil { + return err + } + if string(bytes.TrimSpace(b)) != `FuzzB "seed"` { + return fmt.Errorf("coordinator: did not test FuzzB seed") + } + return nil +} + +func checkWorkerLog(r io.Reader) error { + scan := bufio.NewScanner(r) + var sawAMutant bool + for scan.Scan() { + line := scan.Text() + if !strings.HasPrefix(line, "FuzzA ") { + return fmt.Errorf("worker: tested something other than target: %s", line) + } + if strings.TrimPrefix(line, "FuzzA ") != `"seed"` { + sawAMutant = true + } + } + if err := scan.Err(); err != nil && err != bufio.ErrTooLong { + return err + } + if !sawAMutant { + return fmt.Errorf("worker: did not test any mutants") + } + return nil +} +-- mutator_test.go -- +package fuzz_test + +import ( + "testing" +) + +// TODO(katiehockman): re-work this test once we have a better fuzzing engine +// (ie. more mutations, and compiler instrumentation) +func FuzzMutator(f *testing.F) { + // TODO(katiehockman): simplify this once we can dedupe crashes (e.g. + // replace map with calls to panic, and simply count the number of crashes + // that were added to testdata) + crashes := make(map[string]bool) + // No seed corpus initiated + f.Fuzz(func(t *testing.T, b []byte) { + crashes[string(b)] = true + if len(crashes) >= 10 { + panic("mutator found enough unique mutations") + } + }) +} diff --git a/src/cmd/go/testdata/script/test_fuzz_mutator_repeat.txt b/src/cmd/go/testdata/script/test_fuzz_mutator_repeat.txt new file mode 100644 index 00000000000..0924ed37e6c --- /dev/null +++ b/src/cmd/go/testdata/script/test_fuzz_mutator_repeat.txt @@ -0,0 +1,66 @@ +# TODO(jayconrod): support shared memory on more platforms. +[!darwin] [!linux] [!windows] skip + +# Verify that the fuzzing engine records the actual crashing input, even when +# a worker process terminates without communicating the crashing input back +# to the coordinator. + +[short] skip + +# Start fuzzing. The worker crashes after ~100 iterations. +# The fuzz function writes the crashing input to "want" before exiting. +# The fuzzing engine reconstructs the crashing input and saves it to testdata. +! exists want +! go test -fuzz=. -parallel=1 +stdout 'fuzzing process terminated unexpectedly' +stdout 'Crash written to testdata' + +# Run the fuzz target without fuzzing. The fuzz function is called with the +# crashing input in testdata. The test passes if that input is identical to +# the one saved in "want". +exists want +go test -want=want + +-- go.mod -- +module fuzz + +go 1.17 +-- fuzz_test.go -- +package fuzz + +import ( + "bytes" + "flag" + "os" + "testing" +) + +var wantFlag = flag.String("want", "", "file containing previous crashing input") + +func FuzzRepeat(f *testing.F) { + i := 0 + f.Fuzz(func(t *testing.T, b []byte) { + i++ + if i == 100 { + f, err := os.OpenFile("want", os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0666) + if err != nil { + // Couldn't create the file, probably because it already exists, + // and we're minimizing now. Return without crashing. + return + } + f.Write(b) + f.Close() + os.Exit(1) // crash without communicating + } + + if *wantFlag != "" { + want, err := os.ReadFile(*wantFlag) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(want, b) { + t.Fatalf("inputs are not equal!\n got: %q\nwant:%q", b, want) + } + } + }) +} diff --git a/src/cmd/go/testdata/script/test_fuzz_non_crash_signal.txt b/src/cmd/go/testdata/script/test_fuzz_non_crash_signal.txt new file mode 100644 index 00000000000..1568757de75 --- /dev/null +++ b/src/cmd/go/testdata/script/test_fuzz_non_crash_signal.txt @@ -0,0 +1,55 @@ +# NOTE: this test is skipped on Windows, since there's no concept of signals. +# When a process terminates another process, it provides an exit code. +# TODO(jayconrod): support shared memory on more platforms. +[!darwin] [!freebsd] [!linux] skip +[short] skip + +# FuzzNonCrash sends itself a signal that does not appear to be a crash. +# We should not save a crasher. +! go test -fuzz=FuzzNonCrash +! exists testdata +! stdout unreachable +! stderr unreachable +stdout 'fuzzing process terminated by unexpected signal; no crash will be recorded: signal: killed' + +# FuzzCrash sends itself a signal that looks like a crash. +# We should save a crasher. +! go test -fuzz=FuzzCrash +exists testdata/fuzz/FuzzCrash +stdout 'fuzzing process terminated unexpectedly' + +-- go.mod -- +module test + +go 1.17 +-- fuzz_posix_test.go -- +// +build darwin freebsd linux + +package fuzz + +import ( + "syscall" + "testing" +) + +func FuzzNonCrash(f *testing.F) { + f.Fuzz(func(*testing.T, bool) { + pid := syscall.Getpid() + if err := syscall.Kill(pid, syscall.SIGKILL); err != nil { + panic(err) + } + // signal may not be received immediately. Wait for it. + select{} + }) +} + +func FuzzCrash(f *testing.F) { + f.Fuzz(func(*testing.T, bool) { + pid := syscall.Getpid() + if err := syscall.Kill(pid, syscall.SIGILL); err != nil { + panic(err) + } + // signal may not be received immediately. Wait for it. + select{} + }) +} diff --git a/src/cmd/go/testdata/script/test_fuzz_parallel.txt b/src/cmd/go/testdata/script/test_fuzz_parallel.txt new file mode 100644 index 00000000000..a49f30a27f9 --- /dev/null +++ b/src/cmd/go/testdata/script/test_fuzz_parallel.txt @@ -0,0 +1,61 @@ +# TODO(jayconrod): support shared memory on more platforms. +[!darwin] [!linux] [!windows] skip + +[short] skip + +# When running seed inputs, T.Parallel should let multiple inputs run in +# parallel. +go test -run=FuzzSeed + +# When fuzzing, T.Parallel should be safe to call, but it should have no effect. +# We just check that it doesn't hang, which would be the most obvious +# failure mode. +# TODO(jayconrod): check for the string "after T.Parallel". It's not printed +# by 'go test', so we can't distinguish that crasher from some other panic. +! go test -run=FuzzMutate -fuzz=FuzzMutate +exists testdata/fuzz/FuzzMutate + +-- go.mod -- +module fuzz_parallel + +go 1.17 +-- fuzz_parallel_test.go -- +package fuzz_parallel + +import ( + "sort" + "sync" + "testing" +) + +func FuzzSeed(f *testing.F) { + for _, v := range [][]byte{{'a'}, {'b'}, {'c'}} { + f.Add(v) + } + + var mu sync.Mutex + var before, after []byte + f.Cleanup(func() { + sort.Slice(after, func(i, j int) bool { return after[i] < after[j] }) + got := string(before) + string(after) + want := "abcabc" + if got != want { + f.Fatalf("got %q; want %q", got, want) + } + }) + + f.Fuzz(func(t *testing.T, b []byte) { + before = append(before, b...) + t.Parallel() + mu.Lock() + after = append(after, b...) + mu.Unlock() + }) +} + +func FuzzMutate(f *testing.F) { + f.Fuzz(func(t *testing.T, _ []byte) { + t.Parallel() + t.Error("after T.Parallel") + }) +} diff --git a/src/cmd/go/testdata/script/test_fuzz_seed_corpus.txt b/src/cmd/go/testdata/script/test_fuzz_seed_corpus.txt new file mode 100644 index 00000000000..016b101d725 --- /dev/null +++ b/src/cmd/go/testdata/script/test_fuzz_seed_corpus.txt @@ -0,0 +1,168 @@ +# TODO(jayconrod): support shared memory on more platforms. +[!darwin] [!linux] [!windows] skip + +[short] skip +env GOCACHE=$WORK/cache + +# Test that fuzzing a target with a failure in f.Add prints the crash +# and doesn't write anything to testdata/fuzz +! go test -fuzz=FuzzWithAdd -run=FuzzWithAdd -fuzztime=1x +! stdout ^ok +! stdout 'Crash written to testdata[/\\]fuzz[/\\]FuzzWithAdd[/\\]' +stdout FAIL + +# Test that fuzzing a target with a sucess in f.Add and a fuzztime of only +# 1 does not produce a crash. +go test -fuzz=FuzzWithGoodAdd -run=FuzzWithGoodAdd -fuzztime=1x +stdout ok +! stdout FAIL + +# Test that fuzzing a target with a failure in testdata/fuzz prints the crash +# and doesn't write anything to testdata/fuzz +! go test -fuzz=FuzzWithTestdata -run=FuzzWithTestdata -fuzztime=1x +! stdout ^ok +! stdout 'Crash written to testdata[/\\]fuzz[/\\]FuzzWithTestdata[/\\]' +stdout FAIL + +# Test that fuzzing a target with no seed corpus or cache finds a crash, prints +# it, and write it to testdata +! go test -fuzz=FuzzWithNoCache -run=FuzzWithNoCache -fuzztime=1x +! stdout ^ok +stdout 'Crash written to testdata[/\\]fuzz[/\\]FuzzWithNoCache[/\\]' +stdout FAIL + +# Write a crashing input to the cache +mkdir $GOCACHE/fuzz/example.com/x/FuzzWithCache +cp cache-file $GOCACHE/fuzz/example.com/x/FuzzWithCache/1 + +# Test that fuzzing a target with a failure in the cache prints the crash +# and writes this as a "new" crash to testdata/fuzz +! go test -fuzz=FuzzWithCache -run=FuzzWithCache -fuzztime=1x +! stdout ^ok +stdout 'Crash written to testdata[/\\]fuzz[/\\]FuzzWithCache[/\\]' +stdout FAIL + +# Clear the fuzz cache and make sure it's gone +go clean -fuzzcache +! exists $GOCACHE/fuzz + +# The tests below should operate the exact same as the previous tests. If -fuzz +# is enabled, then whatever target is going to be fuzzed shouldn't be run by +# anything other than the workers. + +# Test that fuzzing a target (with -run=None set) with a failure in f.Add prints +# the crash and doesn't write anything to testdata/fuzz -fuzztime=1x +! go test -fuzz=FuzzWithAdd -run=None +! stdout ^ok +! stdout 'Crash written to testdata[/\\]fuzz[/\\]FuzzWithAdd[/\\]' +stdout FAIL + +# Test that fuzzing a target (with -run=None set) with a sucess in f.Add and a +# fuzztime of only 1 does not produce a crash. +go test -fuzz=FuzzWithGoodAdd -run=None -fuzztime=1x +stdout ok +! stdout FAIL + +# Test that fuzzing a target (with -run=None set) with a failure in +# testdata/fuzz prints the crash and doesn't write anything to testdata/fuzz +! go test -fuzz=FuzzWithTestdata -run=None -fuzztime=1x +! stdout ^ok +! stdout 'Crash written to testdata[/\\]fuzz[/\\]FuzzWithTestdata[/\\]' +stdout FAIL + +# Write a crashing input to the cache +mkdir $GOCACHE/fuzz/example.com/x/FuzzRunNoneWithCache +cp cache-file $GOCACHE/fuzz/example.com/x/FuzzRunNoneWithCache/1 + +# Test that fuzzing a target (with -run=None set) with a failure in the cache +# prints the crash and writes this as a "new" crash to testdata/fuzz +! go test -fuzz=FuzzRunNoneWithCache -run=None -fuzztime=1x +! stdout ^ok +stdout 'Crash written to testdata[/\\]fuzz[/\\]FuzzRunNoneWithCache[/\\]' +stdout FAIL + +# Clear the fuzz cache and make sure it's gone +go clean -fuzzcache +! exists $GOCACHE/fuzz + +# The tests below should operate the exact same way for the previous tests with +# a seed corpus (namely, they should still fail). However, the binary is built +# without instrumentation, so this should be a "testing only" run which executes +# the seed corpus before attempting to fuzz. + +go test -c +! exec ./x.test$GOEXE -test.fuzz=FuzzWithAdd -test.run=FuzzWithAdd -test.fuzztime=1x -test.fuzzcachedir=$WORK/cache +! stdout ^ok +! stdout 'Crash written to testdata[/\\]fuzz[/\\]FuzzWithAdd[/\\]' +stdout FAIL +stderr warning + +go test -c +! exec ./x.test$GOEXE -test.fuzz=FuzzWithTestdata -test.run=FuzzWithTestdata -test.fuzztime=1x -test.fuzzcachedir=$WORK/cache +! stdout ^ok +! stdout 'Crash written to testdata[/\\]fuzz[/\\]FuzzWithTestdata[/\\]' +stdout FAIL +stderr warning + +-- go.mod -- +module example.com/x + +go 1.16 +-- x_test.go -- +package x + +import "testing" + +func FuzzWithAdd(f *testing.F) { + f.Add(10) + f.Fuzz(func(t *testing.T, i int) { + if i == 10 { + t.Error("bad thing here") + } + }) +} + +func FuzzWithGoodAdd(f *testing.F) { + f.Add(10) + f.Fuzz(func(t *testing.T, i int) { + if i != 10 { + t.Error("bad thing here") + } + }) +} + +func FuzzWithTestdata(f *testing.F) { + f.Fuzz(func(t *testing.T, i int) { + if i == 10 { + t.Error("bad thing here") + } + }) +} + +func FuzzWithNoCache(f *testing.F) { + f.Fuzz(func(t *testing.T, i int) { + t.Error("bad thing here") + }) +} + +func FuzzWithCache(f *testing.F) { + f.Fuzz(func(t *testing.T, i int) { + if i == 10 { + t.Error("bad thing here") + } + }) +} + +func FuzzRunNoneWithCache(f *testing.F) { + f.Fuzz(func(t *testing.T, i int) { + if i == 10 { + t.Error("bad thing here") + } + }) +} +-- testdata/fuzz/FuzzWithTestdata/1 -- +go test fuzz v1 +int(10) +-- cache-file -- +go test fuzz v1 +int(10)
\ No newline at end of file diff --git a/src/cmd/go/testdata/script/test_fuzz_setenv.txt b/src/cmd/go/testdata/script/test_fuzz_setenv.txt new file mode 100644 index 00000000000..9738697a91e --- /dev/null +++ b/src/cmd/go/testdata/script/test_fuzz_setenv.txt @@ -0,0 +1,45 @@ +[short] skip +[!darwin] [!linux] [!windows] skip + +go test -fuzz=FuzzA -fuzztime=100x fuzz_setenv_test.go + +-- fuzz_setenv_test.go -- +package fuzz + +import ( + "flag" + "os" + "testing" +) + +func FuzzA(f *testing.F) { + if s := os.Getenv("TEST_FUZZ_SETENV_A"); isWorker() && s == "" { + f.Fatal("environment variable not set") + } else if !isWorker() && s != "" { + f.Fatal("environment variable already set") + } + f.Setenv("TEST_FUZZ_SETENV_A", "A") + if os.Getenv("TEST_FUZZ_SETENV_A") == "" { + f.Fatal("Setenv did not set environment variable") + } + f.Fuzz(func(*testing.T, []byte) {}) +} + +func FuzzB(f *testing.F) { + if os.Getenv("TEST_FUZZ_SETENV_A") != "" { + f.Fatal("environment variable not cleared after FuzzA") + } + f.Skip() +} + +func isWorker() bool { + f := flag.Lookup("test.fuzzworker") + if f == nil { + return false + } + get, ok := f.Value.(flag.Getter) + if !ok { + return false + } + return get.Get() == interface{}(true) +} diff --git a/src/cmd/go/testdata/script/test_fuzz_tag.txt b/src/cmd/go/testdata/script/test_fuzz_tag.txt new file mode 100644 index 00000000000..07ed5d6d612 --- /dev/null +++ b/src/cmd/go/testdata/script/test_fuzz_tag.txt @@ -0,0 +1,31 @@ +# Check that the gofuzzbeta tag is enabled by default and can be disabled. +# TODO(jayconrod,katiehockman): before merging to master, restore the old +# default and delete this test. + +[short] skip + +go test -list=. +stdout Test +stdout Fuzz + +go test -tags= + +-- go.mod -- +module fuzz + +go 1.17 +-- fuzz_test.go -- +// +build gofuzzbeta + +package fuzz + +import "testing" + +func Fuzz(f *testing.F) { + f.Add([]byte(nil)) + f.Fuzz(func(*testing.T, []byte) {}) +} + +func Test(*testing.T) {} +-- empty_test.go -- +package fuzz diff --git a/src/cmd/link/internal/ld/data.go b/src/cmd/link/internal/ld/data.go index 43a0e06e901..1898ee020cb 100644 --- a/src/cmd/link/internal/ld/data.go +++ b/src/cmd/link/internal/ld/data.go @@ -1782,7 +1782,9 @@ func (state *dodataState) allocateDataSections(ctxt *Link) { // Coverage instrumentation counters for libfuzzer. if len(state.data[sym.SLIBFUZZER_EXTRA_COUNTER]) > 0 { - state.allocateNamedSectionAndAssignSyms(&Segdata, "__libfuzzer_extra_counters", sym.SLIBFUZZER_EXTRA_COUNTER, sym.Sxxx, 06) + sect := state.allocateNamedSectionAndAssignSyms(&Segdata, "__libfuzzer_extra_counters", sym.SLIBFUZZER_EXTRA_COUNTER, sym.Sxxx, 06) + ldr.SetSymSect(ldr.LookupOrCreateSym("internal/fuzz._counters", 0), sect) + ldr.SetSymSect(ldr.LookupOrCreateSym("internal/fuzz._ecounters", 0), sect) } if len(state.data[sym.STLSBSS]) > 0 { @@ -2522,6 +2524,7 @@ func (ctxt *Link) address() []*sym.Segment { var noptr *sym.Section var bss *sym.Section var noptrbss *sym.Section + var fuzzCounters *sym.Section for i, s := range Segdata.Sections { if (ctxt.IsELF || ctxt.HeadType == objabi.Haix) && s.Name == ".tbss" { continue @@ -2533,17 +2536,17 @@ func (ctxt *Link) address() []*sym.Segment { s.Vaddr = va va += uint64(vlen) Segdata.Length = va - Segdata.Vaddr - if s.Name == ".data" { + switch s.Name { + case ".data": data = s - } - if s.Name == ".noptrdata" { + case ".noptrdata": noptr = s - } - if s.Name == ".bss" { + case ".bss": bss = s - } - if s.Name == ".noptrbss" { + case ".noptrbss": noptrbss = s + case "__libfuzzer_extra_counters": + fuzzCounters = s } } @@ -2660,6 +2663,11 @@ func (ctxt *Link) address() []*sym.Segment { ctxt.xdefine("runtime.enoptrbss", sym.SNOPTRBSS, int64(noptrbss.Vaddr+noptrbss.Length)) ctxt.xdefine("runtime.end", sym.SBSS, int64(Segdata.Vaddr+Segdata.Length)) + if fuzzCounters != nil { + ctxt.xdefine("internal/fuzz._counters", sym.SLIBFUZZER_EXTRA_COUNTER, int64(fuzzCounters.Vaddr)) + ctxt.xdefine("internal/fuzz._ecounters", sym.SLIBFUZZER_EXTRA_COUNTER, int64(fuzzCounters.Vaddr+fuzzCounters.Length)) + } + if ctxt.IsSolaris() { // On Solaris, in the runtime it sets the external names of the // end symbols. Unset them and define separate symbols, so we diff --git a/src/go/build/deps_test.go b/src/go/build/deps_test.go index cbc77cfe72f..a9939dfcf37 100644 --- a/src/go/build/deps_test.go +++ b/src/go/build/deps_test.go @@ -513,7 +513,10 @@ var depsRules = ` FMT, flag, runtime/debug, runtime/trace, internal/sysinfo, math/rand < testing; - internal/testlog, runtime/pprof, regexp + FMT, crypto/sha256, encoding/json, go/ast, go/parser, go/token, math/rand, encoding/hex, crypto/sha256 + < internal/fuzz; + + internal/fuzz, internal/testlog, runtime/pprof, regexp < testing/internal/testdeps; OS, flag, testing, internal/cfg diff --git a/src/go/doc/example.go b/src/go/doc/example.go index 274000cecb7..fbbd8463546 100644 --- a/src/go/doc/example.go +++ b/src/go/doc/example.go @@ -44,13 +44,13 @@ type Example struct { // identifiers from other packages (or predeclared identifiers, such as // "int") and the test file does not include a dot import. // - The entire test file is the example: the file contains exactly one -// example function, zero test or benchmark functions, and at least one -// top-level function, type, variable, or constant declaration other -// than the example function. +// example function, zero test, fuzz target, or benchmark function, and at +// least one top-level function, type, variable, or constant declaration +// other than the example function. func Examples(testFiles ...*ast.File) []*Example { var list []*Example for _, file := range testFiles { - hasTests := false // file contains tests or benchmarks + hasTests := false // file contains tests, fuzz targets, or benchmarks numDecl := 0 // number of non-import declarations in the file var flist []*Example for _, decl := range file.Decls { @@ -64,7 +64,7 @@ func Examples(testFiles ...*ast.File) []*Example { } numDecl++ name := f.Name.Name - if isTest(name, "Test") || isTest(name, "Benchmark") { + if isTest(name, "Test") || isTest(name, "Benchmark") || isTest(name, "Fuzz") { hasTests = true continue } @@ -133,9 +133,9 @@ func exampleOutput(b *ast.BlockStmt, comments []*ast.CommentGroup) (output strin return "", false, false // no suitable comment found } -// isTest tells whether name looks like a test, example, or benchmark. -// It is a Test (say) if there is a character after Test that is not a -// lower-case letter. (We don't want Testiness.) +// isTest tells whether name looks like a test, example, fuzz target, or +// benchmark. It is a Test (say) if there is a character after Test that is not +// a lower-case letter. (We don't want Testiness.) func isTest(name, prefix string) bool { if !strings.HasPrefix(name, prefix) { return false diff --git a/src/go/doc/example_test.go b/src/go/doc/example_test.go index cf1b702549e..21b71290f7d 100644 --- a/src/go/doc/example_test.go +++ b/src/go/doc/example_test.go @@ -307,6 +307,9 @@ func (X) TestBlah() { func (X) BenchmarkFoo() { } +func (X) FuzzFoo() { +} + func Example() { fmt.Println("Hello, world!") // Output: Hello, world! @@ -326,6 +329,9 @@ func (X) TestBlah() { func (X) BenchmarkFoo() { } +func (X) FuzzFoo() { +} + func main() { fmt.Println("Hello, world!") } diff --git a/src/internal/fuzz/coverage.go b/src/internal/fuzz/coverage.go new file mode 100644 index 00000000000..71d0132e212 --- /dev/null +++ b/src/internal/fuzz/coverage.go @@ -0,0 +1,115 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package fuzz + +import ( + "fmt" + "internal/unsafeheader" + "math/bits" + "unsafe" +) + +// coverage returns a []byte containing unique 8-bit counters for each edge of +// the instrumented source code. This coverage data will only be generated if +// `-d=libfuzzer` is set at build time. This can be used to understand the code +// coverage of a test execution. +func coverage() []byte { + addr := unsafe.Pointer(&_counters) + size := uintptr(unsafe.Pointer(&_ecounters)) - uintptr(addr) + + var res []byte + *(*unsafeheader.Slice)(unsafe.Pointer(&res)) = unsafeheader.Slice{ + Data: addr, + Len: int(size), + Cap: int(size), + } + return res +} + +// ResetCovereage sets all of the counters for each edge of the instrumented +// source code to 0. +func ResetCoverage() { + cov := coverage() + for i := range cov { + cov[i] = 0 + } +} + +// SnapshotCoverage copies the current counter values into coverageSnapshot, +// preserving them for later inspection. SnapshotCoverage also rounds each +// counter down to the nearest power of two. This lets the coordinator store +// multiple values for each counter by OR'ing them together. +func SnapshotCoverage() { + cov := coverage() + for i, b := range cov { + b |= b >> 1 + b |= b >> 2 + b |= b >> 4 + b -= b >> 1 + coverageSnapshot[i] = b + } +} + +// diffCoverage returns a set of bits set in snapshot but not in base. +// If there are no new bits set, diffCoverage returns nil. +func diffCoverage(base, snapshot []byte) []byte { + if len(base) != len(snapshot) { + panic(fmt.Sprintf("the number of coverage bits changed: before=%d, after=%d", len(base), len(snapshot))) + } + found := false + for i := range snapshot { + if snapshot[i]&^base[i] != 0 { + found = true + break + } + } + if !found { + return nil + } + diff := make([]byte, len(snapshot)) + for i := range diff { + diff[i] = snapshot[i] &^ base[i] + } + return diff +} + +// countNewCoverageBits returns the number of bits set in snapshot that are not +// set in base. +func countNewCoverageBits(base, snapshot []byte) int { + n := 0 + for i := range snapshot { + n += bits.OnesCount8(snapshot[i] &^ base[i]) + } + return n +} + +// hasCoverageBit returns true if snapshot has at least one bit set that is +// also set in base. +func hasCoverageBit(base, snapshot []byte) bool { + for i := range snapshot { + if snapshot[i]&base[i] != 0 { + return true + } + } + return false +} + +func countBits(cov []byte) int { + n := 0 + for _, c := range cov { + n += bits.OnesCount8(c) + } + return n +} + +var ( + coverageEnabled = len(coverage()) > 0 + coverageSnapshot = make([]byte, len(coverage())) + + // _counters and _ecounters mark the start and end, respectively, of where + // the 8-bit coverage counters reside in memory. They're known to cmd/link, + // which specially assigns their addresses for this purpose. + _counters, _ecounters [0]byte +) diff --git a/src/internal/fuzz/encoding.go b/src/internal/fuzz/encoding.go new file mode 100644 index 00000000000..d3f24c3e6c8 --- /dev/null +++ b/src/internal/fuzz/encoding.go @@ -0,0 +1,240 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package fuzz + +import ( + "bytes" + "fmt" + "go/ast" + "go/parser" + "go/token" + "strconv" +) + +// encVersion1 will be the first line of a file with version 1 encoding. +var encVersion1 = "go test fuzz v1" + +// marshalCorpusFile encodes an arbitrary number of arguments into the file format for the +// corpus. +func marshalCorpusFile(vals ...interface{}) []byte { + if len(vals) == 0 { + panic("must have at least one value to marshal") + } + b := bytes.NewBuffer([]byte(encVersion1 + "\n")) + // TODO(katiehockman): keep uint8 and int32 encoding where applicable, + // instead of changing to byte and rune respectively. + for _, val := range vals { + switch t := val.(type) { + case int, int8, int16, int64, uint, uint16, uint32, uint64, float32, float64, bool: + fmt.Fprintf(b, "%T(%v)\n", t, t) + case string: + fmt.Fprintf(b, "string(%q)\n", t) + case rune: // int32 + fmt.Fprintf(b, "rune(%q)\n", t) + case byte: // uint8 + fmt.Fprintf(b, "byte(%q)\n", t) + case []byte: // []uint8 + fmt.Fprintf(b, "[]byte(%q)\n", t) + default: + panic(fmt.Sprintf("unsupported type: %T", t)) + } + } + return b.Bytes() +} + +// unmarshalCorpusFile decodes corpus bytes into their respective values. +func unmarshalCorpusFile(b []byte) ([]interface{}, error) { + if len(b) == 0 { + return nil, fmt.Errorf("cannot unmarshal empty string") + } + lines := bytes.Split(b, []byte("\n")) + if len(lines) < 2 { + return nil, fmt.Errorf("must include version and at least one value") + } + if string(lines[0]) != encVersion1 { + return nil, fmt.Errorf("unknown encoding version: %s", lines[0]) + } + var vals []interface{} + for _, line := range lines[1:] { + line = bytes.TrimSpace(line) + if len(line) == 0 { + continue + } + v, err := parseCorpusValue(line) + if err != nil { + return nil, fmt.Errorf("malformed line %q: %v", line, err) + } + vals = append(vals, v) + } + return vals, nil +} + +func parseCorpusValue(line []byte) (interface{}, error) { + fs := token.NewFileSet() + expr, err := parser.ParseExprFrom(fs, "(test)", line, 0) + if err != nil { + return nil, err + } + call, ok := expr.(*ast.CallExpr) + if !ok { + return nil, fmt.Errorf("expected call expression") + } + if len(call.Args) != 1 { + return nil, fmt.Errorf("expected call expression with 1 argument; got %d", len(call.Args)) + } + arg := call.Args[0] + + if arrayType, ok := call.Fun.(*ast.ArrayType); ok { + if arrayType.Len != nil { + return nil, fmt.Errorf("expected []byte or primitive type") + } + elt, ok := arrayType.Elt.(*ast.Ident) + if !ok || elt.Name != "byte" { + return nil, fmt.Errorf("expected []byte") + } + lit, ok := arg.(*ast.BasicLit) + if !ok || lit.Kind != token.STRING { + return nil, fmt.Errorf("string literal required for type []byte") + } + s, err := strconv.Unquote(lit.Value) + if err != nil { + return nil, err + } + return []byte(s), nil + } + + idType, ok := call.Fun.(*ast.Ident) + if !ok { + return nil, fmt.Errorf("expected []byte or primitive type") + } + if idType.Name == "bool" { + id, ok := arg.(*ast.Ident) + if !ok { + return nil, fmt.Errorf("malformed bool") + } + if id.Name == "true" { + return true, nil + } else if id.Name == "false" { + return false, nil + } else { + return nil, fmt.Errorf("true or false required for type bool") + } + } + var ( + val string + kind token.Token + ) + if op, ok := arg.(*ast.UnaryExpr); ok { + // Special case for negative numbers. + lit, ok := op.X.(*ast.BasicLit) + if !ok || (lit.Kind != token.INT && lit.Kind != token.FLOAT) { + return nil, fmt.Errorf("expected operation on int or float type") + } + if op.Op != token.SUB { + return nil, fmt.Errorf("unsupported operation on int: %v", op.Op) + } + val = op.Op.String() + lit.Value // e.g. "-" + "124" + kind = lit.Kind + } else { + lit, ok := arg.(*ast.BasicLit) + if !ok { + return nil, fmt.Errorf("literal value required for primitive type") + } + val, kind = lit.Value, lit.Kind + } + + switch typ := idType.Name; typ { + case "string": + if kind != token.STRING { + return nil, fmt.Errorf("string literal value required for type string") + } + return strconv.Unquote(val) + case "byte", "rune": + if kind != token.CHAR { + return nil, fmt.Errorf("character literal required for byte/rune types") + } + n := len(val) + if n < 2 { + return nil, fmt.Errorf("malformed character literal, missing single quotes") + } + code, _, _, err := strconv.UnquoteChar(val[1:n-1], '\'') + if err != nil { + return nil, err + } + if typ == "rune" { + return code, nil + } + if code >= 256 { + return nil, fmt.Errorf("can only encode single byte to a byte type") + } + return byte(code), nil + case "int", "int8", "int16", "int32", "int64": + if kind != token.INT { + return nil, fmt.Errorf("integer literal required for int types") + } + return parseInt(val, typ) + case "uint", "uint8", "uint16", "uint32", "uint64": + if kind != token.INT { + return nil, fmt.Errorf("integer literal required for uint types") + } + return parseUint(val, typ) + case "float32": + if kind != token.FLOAT && kind != token.INT { + return nil, fmt.Errorf("float or integer literal required for float32 type") + } + v, err := strconv.ParseFloat(val, 32) + return float32(v), err + case "float64": + if kind != token.FLOAT && kind != token.INT { + return nil, fmt.Errorf("float or integer literal required for float64 type") + } + return strconv.ParseFloat(val, 64) + default: + return nil, fmt.Errorf("expected []byte or primitive type") + } +} + +// parseInt returns an integer of value val and type typ. +func parseInt(val, typ string) (interface{}, error) { + switch typ { + case "int": + return strconv.Atoi(val) + case "int8": + i, err := strconv.ParseInt(val, 10, 8) + return int8(i), err + case "int16": + i, err := strconv.ParseInt(val, 10, 16) + return int16(i), err + case "int32": + i, err := strconv.ParseInt(val, 10, 32) + return int32(i), err + case "int64": + return strconv.ParseInt(val, 10, 64) + default: + panic("unreachable") + } +} + +// parseInt returns an unsigned integer of value val and type typ. +func parseUint(val, typ string) (interface{}, error) { + switch typ { + case "uint": + i, err := strconv.ParseUint(val, 10, 0) + return uint(i), err + case "uint8": + i, err := strconv.ParseUint(val, 10, 8) + return uint8(i), err + case "uint16": + i, err := strconv.ParseUint(val, 10, 16) + return uint16(i), err + case "uint32": + i, err := strconv.ParseUint(val, 10, 32) + return uint32(i), err + case "uint64": + return strconv.ParseUint(val, 10, 64) + default: + panic("unreachable") + } +} diff --git a/src/internal/fuzz/encoding_test.go b/src/internal/fuzz/encoding_test.go new file mode 100644 index 00000000000..b429d429c67 --- /dev/null +++ b/src/internal/fuzz/encoding_test.go @@ -0,0 +1,172 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package fuzz + +import ( + "strconv" + "strings" + "testing" +) + +func TestUnmarshalMarshal(t *testing.T) { + var tests = []struct { + in string + ok bool + }{ + { + in: "int(1234)", + ok: false, // missing version + }, + { + in: `go test fuzz v1 +string("a"bcad")`, + ok: false, // malformed + }, + { + in: `go test fuzz v1 +int()`, + ok: false, // empty value + }, + { + in: `go test fuzz v1 +uint(-32)`, + ok: false, // invalid negative uint + }, + { + in: `go test fuzz v1 +int8(1234456)`, + ok: false, // int8 too large + }, + { + in: `go test fuzz v1 +int(20*5)`, + ok: false, // expression in int value + }, + { + in: `go test fuzz v1 +int(--5)`, + ok: false, // expression in int value + }, + { + in: `go test fuzz v1 +bool(0)`, + ok: false, // malformed bool + }, + { + in: `go test fuzz v1 +byte('aa)`, + ok: false, // malformed byte + }, + { + in: `go test fuzz v1 +byte('☃')`, + ok: false, // byte out of range + }, + { + in: `go test fuzz v1 +string("has final newline") +`, + ok: true, // has final newline + }, + { + in: `go test fuzz v1 +string("extra") +[]byte("spacing") + `, + ok: true, // extra spaces in the final newline + }, + { + in: `go test fuzz v1 +float64(0) +float32(0)`, + ok: true, // will be an integer literal since there is no decimal + }, + { + in: `go test fuzz v1 +int(-23) +int8(-2) +int64(2342425) +uint(1) +uint16(234) +uint32(352342) +uint64(123) +rune('œ') +byte('K') +byte('ÿ') +[]byte("hello¿") +[]byte("a") +bool(true) +string("hello\\xbd\\xb2=\\xbc ⌘") +float64(-12.5) +float32(2.5)`, + ok: true, + }, + } + for _, test := range tests { + t.Run(test.in, func(t *testing.T) { + vals, err := unmarshalCorpusFile([]byte(test.in)) + if test.ok && err != nil { + t.Fatalf("unmarshal unexpected error: %v", err) + } else if !test.ok && err == nil { + t.Fatalf("unmarshal unexpected success") + } + if !test.ok { + return // skip the rest of the test + } + newB := marshalCorpusFile(vals...) + if err != nil { + t.Fatalf("marshal unexpected error: %v", err) + } + if newB[len(newB)-1] != '\n' { + t.Error("didn't write final newline to corpus file") + } + before, after := strings.TrimSpace(test.in), strings.TrimSpace(string(newB)) + if before != after { + t.Errorf("values changed after unmarshal then marshal\nbefore: %q\nafter: %q", before, after) + } + }) + } +} + +// BenchmarkMarshalCorpusFile measures the time it takes to serialize byte +// slices of various sizes to a corpus file. The slice contains a repeating +// sequence of bytes 0-255 to mix escaped and non-escaped characters. +func BenchmarkMarshalCorpusFile(b *testing.B) { + buf := make([]byte, 1024*1024) + for i := 0; i < len(buf); i++ { + buf[i] = byte(i) + } + + for sz := 1; sz <= len(buf); sz <<= 1 { + sz := sz + b.Run(strconv.Itoa(sz), func(b *testing.B) { + for i := 0; i < b.N; i++ { + b.SetBytes(int64(sz)) + marshalCorpusFile(buf[:sz]) + } + }) + } +} + +// BenchmarkUnmarshalCorpusfile measures the time it takes to deserialize +// files encoding byte slices of various sizes. The slice contains a repeating +// sequence of bytes 0-255 to mix escaped and non-escaped characters. +func BenchmarkUnmarshalCorpusFile(b *testing.B) { + buf := make([]byte, 1024*1024) + for i := 0; i < len(buf); i++ { + buf[i] = byte(i) + } + + for sz := 1; sz <= len(buf); sz <<= 1 { + sz := sz + data := marshalCorpusFile(buf[:sz]) + b.Run(strconv.Itoa(sz), func(b *testing.B) { + for i := 0; i < b.N; i++ { + b.SetBytes(int64(sz)) + unmarshalCorpusFile(data) + } + }) + } +} diff --git a/src/internal/fuzz/fuzz.go b/src/internal/fuzz/fuzz.go new file mode 100644 index 00000000000..2cd7ebb4728 --- /dev/null +++ b/src/internal/fuzz/fuzz.go @@ -0,0 +1,1020 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package fuzz provides common fuzzing functionality for tests built with +// "go test" and for programs that use fuzzing functionality in the testing +// package. +package fuzz + +import ( + "context" + "crypto/sha256" + "errors" + "fmt" + "io" + "io/ioutil" + "math/bits" + "os" + "path/filepath" + "reflect" + "runtime" + "strings" + "sync" + "time" +) + +// CoordinateFuzzingOpts is a set of arguments for CoordinateFuzzing. +// The zero value is valid for each field unless specified otherwise. +type CoordinateFuzzingOpts struct { + // Log is a writer for logging progress messages and warnings. + // If nil, io.Discard will be used instead. + Log io.Writer + + // Timeout is the amount of wall clock time to spend fuzzing after the corpus + // has loaded. If zero, there will be no time limit. + Timeout time.Duration + + // Limit is the number of random values to generate and test. If zero, + // there will be no limit on the number of generated values. + Limit int64 + + // MinimizeTimeout is the amount of wall clock time to spend minimizing + // after discovering a crasher. If zero, there will be no time limit. If + // MinimizeTimeout and MinimizeLimit are both zero, then minimization will + // be disabled. + MinimizeTimeout time.Duration + + // MinimizeLimit is the maximum number of calls to the fuzz function to be + // made while minimizing after finding a crash. If zero, there will be no + // limit. Calls to the fuzz function made when minimizing also count toward + // Limit. If MinimizeTimeout and MinimizeLimit are both zero, then + // minimization will be disabled. + MinimizeLimit int64 + + // parallel is the number of worker processes to run in parallel. If zero, + // CoordinateFuzzing will run GOMAXPROCS workers. + Parallel int + + // Seed is a list of seed values added by the fuzz target with testing.F.Add + // and in testdata. + Seed []CorpusEntry + + // Types is the list of types which make up a corpus entry. + // Types must be set and must match values in Seed. + Types []reflect.Type + + // CorpusDir is a directory where files containing values that crash the + // code being tested may be written. CorpusDir must be set. + CorpusDir string + + // CacheDir is a directory containing additional "interesting" values. + // The fuzzer may derive new values from these, and may write new values here. + CacheDir string +} + +// CoordinateFuzzing creates several worker processes and communicates with +// them to test random inputs that could trigger crashes and expose bugs. +// The worker processes run the same binary in the same directory with the +// same environment variables as the coordinator process. Workers also run +// with the same arguments as the coordinator, except with the -test.fuzzworker +// flag prepended to the argument list. +// +// If a crash occurs, the function will return an error containing information +// about the crash, which can be reported to the user. +func CoordinateFuzzing(ctx context.Context, opts CoordinateFuzzingOpts) (err error) { + if err := ctx.Err(); err != nil { + return err + } + if opts.Log == nil { + opts.Log = io.Discard + } + if opts.Parallel == 0 { + opts.Parallel = runtime.GOMAXPROCS(0) + } + if opts.Limit > 0 && int64(opts.Parallel) > opts.Limit { + // Don't start more workers than we need. + opts.Parallel = int(opts.Limit) + } + + c, err := newCoordinator(opts) + if err != nil { + return err + } + + if opts.Timeout > 0 { + var cancel func() + ctx, cancel = context.WithTimeout(ctx, opts.Timeout) + defer cancel() + } + + // fuzzCtx is used to stop workers, for example, after finding a crasher. + fuzzCtx, cancelWorkers := context.WithCancel(ctx) + defer cancelWorkers() + doneC := ctx.Done() + + // stop is called when a worker encounters a fatal error. + var fuzzErr error + stopping := false + stop := func(err error) { + if err == fuzzCtx.Err() || isInterruptError(err) { + // Suppress cancellation errors and terminations due to SIGINT. + // The messages are not helpful since either the user triggered the error + // (with ^C) or another more helpful message will be printed (a crasher). + err = nil + } + if err != nil && (fuzzErr == nil || fuzzErr == ctx.Err()) { + fuzzErr = err + } + if stopping { + return + } + stopping = true + cancelWorkers() + doneC = nil + } + + // Ensure that any crash we find is written to the corpus, even if an error + // or interruption occurs while minimizing it. + var crashMinimizing *fuzzResult + crashWritten := false + defer func() { + if crashMinimizing == nil || crashWritten { + return + } + fileName, werr := writeToCorpus(crashMinimizing.entry.Data, opts.CorpusDir) + if werr != nil { + err = fmt.Errorf("%w\n%v", err, werr) + return + } + if err == nil { + err = &crashError{ + name: filepath.Base(fileName), + err: errors.New(crashMinimizing.crasherMsg), + } + } + }() + + // Start workers. + // TODO(jayconrod): do we want to support fuzzing different binaries? + dir := "" // same as self + binPath := os.Args[0] + args := append([]string{"-test.fuzzworker"}, os.Args[1:]...) + env := os.Environ() // same as self + + errC := make(chan error) + workers := make([]*worker, opts.Parallel) + for i := range workers { + var err error + workers[i], err = newWorker(c, dir, binPath, args, env) + if err != nil { + return err + } + } + for i := range workers { + w := workers[i] + go func() { + err := w.coordinate(fuzzCtx) + if fuzzCtx.Err() != nil || isInterruptError(err) { + err = nil + } + cleanErr := w.cleanup() + if err == nil { + err = cleanErr + } + errC <- err + }() + } + + // Main event loop. + // Do not return until all workers have terminated. We avoid a deadlock by + // receiving messages from workers even after ctx is cancelled. + activeWorkers := len(workers) + statTicker := time.NewTicker(3 * time.Second) + defer statTicker.Stop() + defer c.logStats() + + c.logStats() + for { + var inputC chan fuzzInput + input, ok := c.peekInput() + if ok && crashMinimizing == nil && !stopping { + inputC = c.inputC + } + + var minimizeC chan fuzzMinimizeInput + minimizeInput, ok := c.peekMinimizeInput() + if ok && !stopping { + minimizeC = c.minimizeC + } + + select { + case <-doneC: + // Interrupted, cancelled, or timed out. + // stop sets doneC to nil so we don't busy wait here. + stop(ctx.Err()) + + case err := <-errC: + // A worker terminated, possibly after encountering a fatal error. + stop(err) + activeWorkers-- + if activeWorkers == 0 { + return fuzzErr + } + + case result := <-c.resultC: + // Received response from worker. + if stopping { + break + } + c.updateStats(result) + if c.opts.Limit > 0 && c.count >= c.opts.Limit { + stop(nil) + } + + if result.crasherMsg != "" { + if c.warmupRun() && result.entry.IsSeed { + fmt.Fprintf(c.opts.Log, "found a crash while testing seed corpus entry: %q\n", result.entry.Parent) + stop(errors.New(result.crasherMsg)) + break + } + if c.canMinimize() && !result.minimizeAttempted { + if crashMinimizing != nil { + // This crash is not minimized, and another crash is being minimized. + // Ignore this one and wait for the other one to finish. + break + } + // Found a crasher but haven't yet attempted to minimize it. + // Send it back to a worker for minimization. Disable inputC so + // other workers don't continue fuzzing. + crashMinimizing = &result + fmt.Fprintf(c.opts.Log, "fuzz: found a %d-byte crash input; minimizing...\n", len(result.entry.Data)) + c.queueForMinimization(result, nil) + } else if !crashWritten { + // Found a crasher that's either minimized or not minimizable. + // Write to corpus and stop. + fileName, err := writeToCorpus(result.entry.Data, opts.CorpusDir) + if err == nil { + crashWritten = true + err = &crashError{ + name: filepath.Base(fileName), + err: errors.New(result.crasherMsg), + } + } + if printDebugInfo() { + fmt.Fprintf( + c.opts.Log, + "DEBUG new crasher, elapsed: %s, id: %s, parent: %s, gen: %d, size: %d, exec time: %s\n", + c.elapsed(), + fileName, + result.entry.Parent, + result.entry.Generation, + len(result.entry.Data), + result.entryDuration, + ) + } + stop(err) + } + } else if result.coverageData != nil { + if c.warmupRun() { + if printDebugInfo() { + fmt.Fprintf( + c.opts.Log, + "DEBUG processed an initial input, elapsed: %s, id: %s, new bits: %d, size: %d, exec time: %s\n", + c.elapsed(), + result.entry.Parent, + countBits(diffCoverage(c.coverageMask, result.coverageData)), + len(result.entry.Data), + result.entryDuration, + ) + } + c.updateCoverage(result.coverageData) + c.warmupInputCount-- + if printDebugInfo() && c.warmupInputCount == 0 { + fmt.Fprintf( + c.opts.Log, + "DEBUG finished processing input corpus, elapsed: %s, entries: %d, initial coverage bits: %d\n", + c.elapsed(), + len(c.corpus.entries), + countBits(c.coverageMask), + ) + } + } else if keepCoverage := diffCoverage(c.coverageMask, result.coverageData); keepCoverage != nil { + // Found a value that expanded coverage. + // It's not a crasher, but we may want to add it to the on-disk + // corpus and prioritize it for future fuzzing. + // TODO(jayconrod, katiehockman): Prioritize fuzzing these + // values which expanded coverage, perhaps based on the + // number of new edges that this result expanded. + // TODO(jayconrod, katiehockman): Don't write a value that's already + // in the corpus. + if !result.minimizeAttempted && crashMinimizing == nil && c.canMinimize() { + // Send back to workers to find a smaller value that preserves + // at least one new coverage bit. + c.queueForMinimization(result, keepCoverage) + } else { + // Update the coordinator's coverage mask and save the value. + inputSize := len(result.entry.Data) + if opts.CacheDir != "" { + filename, err := writeToCorpus(result.entry.Data, opts.CacheDir) + if err != nil { + stop(err) + } + result.entry.Data = nil + result.entry.Name = filename + } + c.updateCoverage(keepCoverage) + c.corpus.entries = append(c.corpus.entries, result.entry) + c.inputQueue.enqueue(result.entry) + c.interestingCount++ + if printDebugInfo() { + fmt.Fprintf( + c.opts.Log, + "DEBUG new interesting input, elapsed: %s, id: %s, parent: %s, gen: %d, new bits: %d, total bits: %d, size: %d, exec time: %s\n", + c.elapsed(), + result.entry.Name, + result.entry.Parent, + result.entry.Generation, + countBits(keepCoverage), + countBits(c.coverageMask), + inputSize, + result.entryDuration, + ) + } + } + } else { + if printDebugInfo() { + fmt.Fprintf( + c.opts.Log, + "DEBUG worker reported interesting input that doesn't expand coverage, elapsed: %s, id: %s, parent: %s, minimized: %t\n", + c.elapsed(), + result.entry.Name, + result.entry.Parent, + result.minimizeAttempted, + ) + } + } + } else if c.warmupRun() { + // No error or coverage data was reported for this input during + // warmup, so continue processing results. + c.warmupInputCount-- + if printDebugInfo() && c.warmupInputCount == 0 { + fmt.Fprintf( + c.opts.Log, + "DEBUG finished testing-only phase, elapsed: %s, entries: %d\n", + time.Since(c.startTime), + len(c.corpus.entries), + ) + } + } + + case inputC <- input: + // Sent the next input to a worker. + c.sentInput(input) + + case minimizeC <- minimizeInput: + // Sent the next input for minimization to a worker. + c.sentMinimizeInput(minimizeInput) + + case <-statTicker.C: + c.logStats() + } + } + + // TODO(jayconrod,katiehockman): if a crasher can't be written to the corpus, + // write to the cache instead. +} + +// crashError wraps a crasher written to the seed corpus. It saves the name +// of the file where the input causing the crasher was saved. The testing +// framework uses this to report a command to re-run that specific input. +type crashError struct { + name string + err error +} + +func (e *crashError) Error() string { + return e.err.Error() +} + +func (e *crashError) Unwrap() error { + return e.err +} + +func (e *crashError) CrashName() string { + return e.name +} + +type corpus struct { + entries []CorpusEntry +} + +// CorpusEntry represents an individual input for fuzzing. +// +// We must use an equivalent type in the testing and testing/internal/testdeps +// packages, but testing can't import this package directly, and we don't want +// to export this type from testing. Instead, we use the same struct type and +// use a type alias (not a defined type) for convenience. +type CorpusEntry = struct { + Parent string + + // Name is the name of the corpus file, if the entry was loaded from the + // seed corpus. It can be used with -run. For entries added with f.Add and + // entries generated by the mutator, Name is empty and Data is populated. + Name string + + // Data is the raw input data. Data should only be populated for initial + // seed values added with f.Add. For on-disk corpus files, Data will + // be nil. + Data []byte + + // Values is the unmarshaled values from a corpus file. + Values []interface{} + + Generation int + + // IsSeed indicates whether this entry is part of the seed corpus. + IsSeed bool +} + +// Data returns the raw input bytes, either from the data struct field, +// or from disk. +func CorpusEntryData(ce CorpusEntry) ([]byte, error) { + if ce.Data != nil { + return ce.Data, nil + } + + return os.ReadFile(ce.Name) +} + +type fuzzInput struct { + // entry is the value to test initially. The worker will randomly mutate + // values from this starting point. + entry CorpusEntry + + // timeout is the time to spend fuzzing variations of this input, + // not including starting or cleaning up. + timeout time.Duration + + // limit is the maximum number of calls to the fuzz function the worker may + // make. The worker may make fewer calls, for example, if it finds an + // error early. If limit is zero, there is no limit on calls to the + // fuzz function. + limit int64 + + // warmup indicates whether this is a warmup input before fuzzing begins. If + // true, the input should not be fuzzed. + warmup bool + + // coverageData reflects the coordinator's current coverageMask. + coverageData []byte +} + +type fuzzResult struct { + // entry is an interesting value or a crasher. + entry CorpusEntry + + // crasherMsg is an error message from a crash. It's "" if no crash was found. + crasherMsg string + + // minimizeAttempted is true if the worker attempted to minimize this input. + // The worker may or may not have succeeded. + minimizeAttempted bool + + // coverageData is set if the worker found new coverage. + coverageData []byte + + // limit is the number of values the coordinator asked the worker + // to test. 0 if there was no limit. + limit int64 + + // count is the number of values the worker actually tested. + count int64 + + // totalDuration is the time the worker spent testing inputs. + totalDuration time.Duration + + // entryDuration is the time the worker spent execution an interesting result + entryDuration time.Duration +} + +type fuzzMinimizeInput struct { + // entry is an interesting value or crasher to minimize. + entry CorpusEntry + + // crasherMsg is an error message from a crash. It's "" if no crash was found. + // If set, the worker will attempt to find a smaller input that also produces + // an error, though not necessarily the same error. + crasherMsg string + + // limit is the maximum number of calls to the fuzz function the worker may + // make. The worker may make fewer calls, for example, if it can't reproduce + // an error. If limit is zero, there is no limit on calls to the fuzz function. + limit int64 + + // timeout is the time to spend minimizing this input. + // A zero timeout means no limit. + timeout time.Duration + + // keepCoverage is a set of coverage bits that entry found that were not in + // the coordinator's combined set. When minimizing, the worker should find an + // input that preserves at least one of these bits. keepCoverage is nil for + // crashing inputs. + keepCoverage []byte +} + +// coordinator holds channels that workers can use to communicate with +// the coordinator. +type coordinator struct { + opts CoordinateFuzzingOpts + + // startTime is the time we started the workers after loading the corpus. + // Used for logging. + startTime time.Time + + // inputC is sent values to fuzz by the coordinator. Any worker may receive + // values from this channel. Workers send results to resultC. + inputC chan fuzzInput + + // minimizeC is sent values to minimize by the coordinator. Any worker may + // receive values from this channel. Workers send results to resultC. + minimizeC chan fuzzMinimizeInput + + // resultC is sent results of fuzzing by workers. The coordinator + // receives these. Multiple types of messages are allowed. + resultC chan fuzzResult + + // count is the number of values fuzzed so far. + count int64 + + // interestingCount is the number of unique interesting values which have + // been found this execution. + interestingCount int64 + + // warmupInputCount is the number of entries in the corpus which still need + // to be received from workers to run once during warmup, but not fuzz. This + // could be for coverage data, or only for the purposes of verifying that + // the seed corpus doesn't have any crashers. See warmupRun. + warmupInputCount int + + // duration is the time spent fuzzing inside workers, not counting time + // starting up or tearing down. + duration time.Duration + + // countWaiting is the number of fuzzing executions the coordinator is + // waiting on workers to complete. + countWaiting int64 + + // corpus is a set of interesting values, including the seed corpus and + // generated values that workers reported as interesting. + corpus corpus + + // minimizationAllowed is true if one or more of the types of fuzz + // function's parameters can be minimized, and either the limit or duration + // for minimization is non-zero. + minimizationAllowed bool + + // inputQueue is a queue of inputs that workers should try fuzzing. This is + // initially populated from the seed corpus and cached inputs. More inputs + // may be added as new coverage is discovered. + inputQueue queue + + // minimizeQueue is a queue of inputs that caused errors or exposed new + // coverage. Workers should attempt to find smaller inputs that do the + // same thing. + minimizeQueue queue + + // coverageMask aggregates coverage that was found for all inputs in the + // corpus. Each byte represents a single basic execution block. Each set bit + // within the byte indicates that an input has triggered that block at least + // 1 << n times, where n is the position of the bit in the byte. For example, a + // value of 12 indicates that separate inputs have triggered this block + // between 4-7 times and 8-15 times. + coverageMask []byte +} + +func newCoordinator(opts CoordinateFuzzingOpts) (*coordinator, error) { + // Make sure all of the seed corpus given by f.Add has marshalled data. + for i := range opts.Seed { + if opts.Seed[i].Data == nil && opts.Seed[i].Values != nil { + opts.Seed[i].Data = marshalCorpusFile(opts.Seed[i].Values...) + } + } + corpus, err := readCache(opts.Seed, opts.Types, opts.CacheDir) + if err != nil { + return nil, err + } + c := &coordinator{ + opts: opts, + startTime: time.Now(), + inputC: make(chan fuzzInput), + minimizeC: make(chan fuzzMinimizeInput), + resultC: make(chan fuzzResult), + corpus: corpus, + } + if opts.MinimizeLimit > 0 || opts.MinimizeTimeout > 0 { + for _, t := range opts.Types { + if isMinimizable(t) { + c.minimizationAllowed = true + break + } + } + } + + covSize := len(coverage()) + if covSize == 0 { + fmt.Fprintf(c.opts.Log, "warning: the test binary was not built with coverage instrumentation, so fuzzing will run without coverage guidance and may be inefficient\n") + // Even though a coverage-only run won't occur, we should still run all + // of the seed corpus to make sure there are no existing failures before + // we start fuzzing. + c.warmupInputCount = len(c.opts.Seed) + for _, e := range c.opts.Seed { + c.inputQueue.enqueue(e) + } + } else { + c.warmupInputCount = len(c.corpus.entries) + for _, e := range c.corpus.entries { + c.inputQueue.enqueue(e) + } + // Set c.coverageMask to a clean []byte full of zeros. + c.coverageMask = make([]byte, covSize) + } + + if len(c.corpus.entries) == 0 { + fmt.Fprintf(c.opts.Log, "warning: starting with empty corpus\n") + var vals []interface{} + for _, t := range opts.Types { + vals = append(vals, zeroValue(t)) + } + data := marshalCorpusFile(vals...) + h := sha256.Sum256(data) + name := fmt.Sprintf("%x", h[:4]) + c.corpus.entries = append(c.corpus.entries, CorpusEntry{Name: name, Data: data}) + } + + return c, nil +} + +func (c *coordinator) updateStats(result fuzzResult) { + c.count += result.count + c.countWaiting -= result.limit + c.duration += result.totalDuration +} + +func (c *coordinator) logStats() { + elapsed := c.elapsed() + if c.warmupRun() { + if coverageEnabled { + fmt.Fprintf(c.opts.Log, "gathering baseline coverage, elapsed: %s, workers: %d, left: %d\n", elapsed, c.opts.Parallel, c.warmupInputCount) + } else { + fmt.Fprintf(c.opts.Log, "testing seed corpus, elapsed: %s, workers: %d, left: %d\n", elapsed, c.opts.Parallel, c.warmupInputCount) + } + } else { + rate := float64(c.count) / time.Since(c.startTime).Seconds() // be more precise here + fmt.Fprintf(c.opts.Log, "fuzz: elapsed: %s, execs: %d (%.0f/sec), workers: %d, interesting: %d\n", elapsed, c.count, rate, c.opts.Parallel, c.interestingCount) + } +} + +// peekInput returns the next value that should be sent to workers. +// If the number of executions is limited, the returned value includes +// a limit for one worker. If there are no executions left, peekInput returns +// a zero value and false. +// +// peekInput doesn't actually remove the input from the queue. The caller +// must call sentInput after sending the input. +// +// If the input queue is empty and the coverage/testing-only run has completed, +// queue refills it from the corpus. +func (c *coordinator) peekInput() (fuzzInput, bool) { + if c.opts.Limit > 0 && c.count+c.countWaiting >= c.opts.Limit { + // Already making the maximum number of calls to the fuzz function. + // Don't send more inputs right now. + return fuzzInput{}, false + } + if c.inputQueue.len == 0 { + if c.warmupInputCount > 0 { + // Wait for coverage/testing-only run to finish before sending more + // inputs. + return fuzzInput{}, false + } + c.refillInputQueue() + } + + entry, ok := c.inputQueue.peek() + if !ok { + panic("input queue empty after refill") + } + input := fuzzInput{ + entry: entry.(CorpusEntry), + timeout: workerFuzzDuration, + warmup: c.warmupRun(), + } + if c.coverageMask != nil { + input.coverageData = make([]byte, len(c.coverageMask)) + copy(input.coverageData, c.coverageMask) + } + if input.warmup { + // No fuzzing will occur, but it should count toward the limit set by + // -fuzztime. + input.limit = 1 + return input, true + } + + if c.opts.Limit > 0 { + input.limit = c.opts.Limit / int64(c.opts.Parallel) + if c.opts.Limit%int64(c.opts.Parallel) > 0 { + input.limit++ + } + remaining := c.opts.Limit - c.count - c.countWaiting + if input.limit > remaining { + input.limit = remaining + } + } + return input, true +} + +// sentInput updates internal counters after an input is sent to c.inputC. +func (c *coordinator) sentInput(input fuzzInput) { + c.inputQueue.dequeue() + c.countWaiting += input.limit +} + +// refillInputQueue refills the input queue from the corpus after it becomes +// empty. +func (c *coordinator) refillInputQueue() { + for _, e := range c.corpus.entries { + c.inputQueue.enqueue(e) + } +} + +// queueForMinimization creates a fuzzMinimizeInput from result and adds it +// to the minimization queue to be sent to workers. +func (c *coordinator) queueForMinimization(result fuzzResult, keepCoverage []byte) { + if result.crasherMsg != "" { + c.minimizeQueue.clear() + } + + input := fuzzMinimizeInput{ + entry: result.entry, + crasherMsg: result.crasherMsg, + keepCoverage: keepCoverage, + } + c.minimizeQueue.enqueue(input) +} + +// peekMinimizeInput returns the next input that should be sent to workers for +// minimization. +func (c *coordinator) peekMinimizeInput() (fuzzMinimizeInput, bool) { + if !c.canMinimize() { + // Already making the maximum number of calls to the fuzz function. + // Don't send more inputs right now. + return fuzzMinimizeInput{}, false + } + v, ok := c.minimizeQueue.peek() + if !ok { + return fuzzMinimizeInput{}, false + } + input := v.(fuzzMinimizeInput) + + if c.opts.MinimizeTimeout > 0 { + input.timeout = c.opts.MinimizeTimeout + } + if c.opts.MinimizeLimit > 0 { + input.limit = c.opts.MinimizeLimit + } else if c.opts.Limit > 0 { + if input.crasherMsg != "" { + input.limit = c.opts.Limit + } else { + input.limit = c.opts.Limit / int64(c.opts.Parallel) + if c.opts.Limit%int64(c.opts.Parallel) > 0 { + input.limit++ + } + } + } + remaining := c.opts.Limit - c.count - c.countWaiting + if input.limit > remaining { + input.limit = remaining + } + return input, true +} + +// sentMinimizeInput removes an input from the minimization queue after it's +// sent to minimizeC. +func (c *coordinator) sentMinimizeInput(input fuzzMinimizeInput) { + c.minimizeQueue.dequeue() + c.countWaiting += input.limit +} + +// warmupRun returns true while the coordinator is running inputs without +// mutating them as a warmup before fuzzing. This could be to gather baseline +// coverage data for entries in the corpus, or to test all of the seed corpus +// for errors before fuzzing begins. +// +// The coordinator doesn't store coverage data in the cache with each input +// because that data would be invalid when counter offsets in the test binary +// change. +// +// When gathering coverage, the coordinator sends each entry to a worker to +// gather coverage for that entry only, without fuzzing or minimizing. This +// phase ends when all workers have finished, and the coordinator has a combined +// coverage map. +func (c *coordinator) warmupRun() bool { + return c.warmupInputCount > 0 +} + +// updateCoverage sets bits in c.coverageMask that are set in newCoverage. +// updateCoverage returns the number of newly set bits. See the comment on +// coverageMask for the format. +func (c *coordinator) updateCoverage(newCoverage []byte) int { + if len(newCoverage) != len(c.coverageMask) { + panic(fmt.Sprintf("number of coverage counters changed at runtime: %d, expected %d", len(newCoverage), len(c.coverageMask))) + } + newBitCount := 0 + for i := range newCoverage { + diff := newCoverage[i] &^ c.coverageMask[i] + newBitCount += bits.OnesCount8(diff) + c.coverageMask[i] |= newCoverage[i] + } + return newBitCount +} + +// canMinimize returns whether the coordinator should attempt to find smaller +// inputs that reproduce a crash or new coverage. It shouldn't do this if it +// is in the warmup phase. +func (c *coordinator) canMinimize() bool { + return c.minimizationAllowed && + (c.opts.Limit == 0 || c.count+c.countWaiting < c.opts.Limit) && + !c.warmupRun() +} + +func (c *coordinator) elapsed() time.Duration { + return time.Since(c.startTime).Round(1 * time.Second) +} + +// readCache creates a combined corpus from seed values and values in the cache +// (in GOCACHE/fuzz). +// +// TODO(fuzzing): need a mechanism that can remove values that +// aren't useful anymore, for example, because they have the wrong type. +func readCache(seed []CorpusEntry, types []reflect.Type, cacheDir string) (corpus, error) { + var c corpus + c.entries = append(c.entries, seed...) + entries, err := ReadCorpus(cacheDir, types) + if err != nil { + if _, ok := err.(*MalformedCorpusError); !ok { + // It's okay if some files in the cache directory are malformed and + // are not included in the corpus, but fail if it's an I/O error. + return corpus{}, err + } + // TODO(jayconrod,katiehockman): consider printing some kind of warning + // indicating the number of files which were skipped because they are + // malformed. + } + c.entries = append(c.entries, entries...) + return c, nil +} + +// MalformedCorpusError is an error found while reading the corpus from the +// filesystem. All of the errors are stored in the errs list. The testing +// framework uses this to report malformed files in testdata. +type MalformedCorpusError struct { + errs []error +} + +func (e *MalformedCorpusError) Error() string { + var msgs []string + for _, s := range e.errs { + msgs = append(msgs, s.Error()) + } + return strings.Join(msgs, "\n") +} + +// ReadCorpus reads the corpus from the provided dir. The returned corpus +// entries are guaranteed to match the given types. Any malformed files will +// be saved in a MalformedCorpusError and returned, along with the most recent +// error. +func ReadCorpus(dir string, types []reflect.Type) ([]CorpusEntry, error) { + files, err := ioutil.ReadDir(dir) + if os.IsNotExist(err) { + return nil, nil // No corpus to read + } else if err != nil { + return nil, fmt.Errorf("reading seed corpus from testdata: %v", err) + } + var corpus []CorpusEntry + var errs []error + for _, file := range files { + // TODO(jayconrod,katiehockman): determine when a file is a fuzzing input + // based on its name. We should only read files created by writeToCorpus. + // If we read ALL files, we won't be able to change the file format by + // changing the extension. We also won't be able to add files like + // README.txt explaining why the directory exists. + if file.IsDir() { + continue + } + filename := filepath.Join(dir, file.Name()) + data, err := ioutil.ReadFile(filename) + if err != nil { + return nil, fmt.Errorf("failed to read corpus file: %v", err) + } + var vals []interface{} + vals, err = readCorpusData(data, types) + if err != nil { + errs = append(errs, fmt.Errorf("%q: %v", filename, err)) + continue + } + corpus = append(corpus, CorpusEntry{Name: filename, Values: vals}) + } + if len(errs) > 0 { + return corpus, &MalformedCorpusError{errs: errs} + } + return corpus, nil +} + +func readCorpusData(data []byte, types []reflect.Type) ([]interface{}, error) { + vals, err := unmarshalCorpusFile(data) + if err != nil { + return nil, fmt.Errorf("unmarshal: %v", err) + } + if err = CheckCorpus(vals, types); err != nil { + return nil, err + } + return vals, nil +} + +// CheckCorpus verifies that the types in vals match the expected types +// provided. +func CheckCorpus(vals []interface{}, types []reflect.Type) error { + if len(vals) != len(types) { + return fmt.Errorf("wrong number of values in corpus entry: %d, want %d", len(vals), len(types)) + } + for i := range types { + if reflect.TypeOf(vals[i]) != types[i] { + return fmt.Errorf("mismatched types in corpus entry: %v, want %v", vals, types) + } + } + return nil +} + +// writeToCorpus atomically writes the given bytes to a new file in testdata. +// If the directory does not exist, it will create one. If the file already +// exists, writeToCorpus will not rewrite it. writeToCorpus returns the +// file's name, or an error if it failed. +func writeToCorpus(b []byte, dir string) (name string, err error) { + sum := fmt.Sprintf("%x", sha256.Sum256(b)) + name = filepath.Join(dir, sum) + if err := os.MkdirAll(dir, 0777); err != nil { + return "", err + } + if err := ioutil.WriteFile(name, b, 0666); err != nil { + os.Remove(name) // remove partially written file + return "", err + } + return name, nil +} + +func zeroValue(t reflect.Type) interface{} { + for _, v := range zeroVals { + if reflect.TypeOf(v) == t { + return v + } + } + panic(fmt.Sprintf("unsupported type: %v", t)) +} + +var zeroVals []interface{} = []interface{}{ + []byte(""), + string(""), + false, + byte(0), + rune(0), + float32(0), + float64(0), + int(0), + int8(0), + int16(0), + int32(0), + int64(0), + uint(0), + uint8(0), + uint16(0), + uint32(0), + uint64(0), +} + +var ( + debugInfo bool + debugInfoOnce sync.Once +) + +func printDebugInfo() bool { + debugInfoOnce.Do(func() { + debug := strings.Split(os.Getenv("GODEBUG"), ",") + for _, f := range debug { + if f == "fuzzdebug=1" { + debugInfo = true + break + } + } + }) + return debugInfo +} diff --git a/src/internal/fuzz/mem.go b/src/internal/fuzz/mem.go new file mode 100644 index 00000000000..ccd4da24550 --- /dev/null +++ b/src/internal/fuzz/mem.go @@ -0,0 +1,134 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package fuzz + +import ( + "fmt" + "io/ioutil" + "os" + "unsafe" +) + +// sharedMem manages access to a region of virtual memory mapped from a file, +// shared between multiple processes. The region includes space for a header and +// a value of variable length. +// +// When fuzzing, the coordinator creates a sharedMem from a temporary file for +// each worker. This buffer is used to pass values to fuzz between processes. +// Care must be taken to manage access to shared memory across processes; +// sharedMem provides no synchronization on its own. See workerComm for an +// explanation. +type sharedMem struct { + // f is the file mapped into memory. + f *os.File + + // region is the mapped region of virtual memory for f. The content of f may + // be read or written through this slice. + region []byte + + // removeOnClose is true if the file should be deleted by Close. + removeOnClose bool + + // sys contains OS-specific information. + sys sharedMemSys +} + +// sharedMemHeader stores metadata in shared memory. +type sharedMemHeader struct { + // count is the number of times the worker has called the fuzz function. + // May be reset by coordinator. + count int64 + + // valueLen is the length of the value that was last fuzzed. + valueLen int + + // randState and randInc hold the state of a pseudo-random number generator. + randState, randInc uint64 +} + +// sharedMemSize returns the size needed for a shared memory buffer that can +// contain values of the given size. +func sharedMemSize(valueSize int) int { + // TODO(jayconrod): set a reasonable maximum size per platform. + return int(unsafe.Sizeof(sharedMemHeader{})) + valueSize +} + +// sharedMemTempFile creates a new temporary file of the given size, then maps +// it into memory. The file will be removed when the Close method is called. +func sharedMemTempFile(size int) (m *sharedMem, err error) { + // Create a temporary file. + f, err := ioutil.TempFile("", "fuzz-*") + if err != nil { + return nil, err + } + defer func() { + if err != nil { + f.Close() + os.Remove(f.Name()) + } + }() + + // Resize it to the correct size. + totalSize := sharedMemSize(size) + if err := f.Truncate(int64(totalSize)); err != nil { + return nil, err + } + + // Map the file into memory. + removeOnClose := true + return sharedMemMapFile(f, totalSize, removeOnClose) +} + +// header returns a pointer to metadata within the shared memory region. +func (m *sharedMem) header() *sharedMemHeader { + return (*sharedMemHeader)(unsafe.Pointer(&m.region[0])) +} + +// valueRef returns the value currently stored in shared memory. The returned +// slice points to shared memory; it is not a copy. +func (m *sharedMem) valueRef() []byte { + length := m.header().valueLen + valueOffset := int(unsafe.Sizeof(sharedMemHeader{})) + return m.region[valueOffset : valueOffset+length] +} + +// valueCopy returns a copy of the value stored in shared memory. +func (m *sharedMem) valueCopy() []byte { + ref := m.valueRef() + b := make([]byte, len(ref)) + copy(b, ref) + return b +} + +// setValue copies the data in b into the shared memory buffer and sets +// the length. len(b) must be less than or equal to the capacity of the buffer +// (as returned by cap(m.value())). +func (m *sharedMem) setValue(b []byte) { + v := m.valueRef() + if len(b) > cap(v) { + panic(fmt.Sprintf("value length %d larger than shared memory capacity %d", len(b), cap(v))) + } + m.header().valueLen = len(b) + copy(v[:cap(v)], b) +} + +// setValueLen sets the length of the shared memory buffer returned by valueRef +// to n, which may be at most the cap of that slice. +// +// Note that we can only store the length in the shared memory header. The full +// slice header contains a pointer, which is likely only valid for one process, +// since each process can map shared memory at a different virtual address. +func (m *sharedMem) setValueLen(n int) { + v := m.valueRef() + if n > cap(v) { + panic(fmt.Sprintf("length %d larger than shared memory capacity %d", n, cap(v))) + } + m.header().valueLen = n +} + +// TODO(jayconrod): add method to resize the buffer. We'll need that when the +// mutator can increase input length. Only the coordinator will be able to +// do it, since we'll need to send a message to the worker telling it to +// remap the file. diff --git a/src/internal/fuzz/minimize.go b/src/internal/fuzz/minimize.go new file mode 100644 index 00000000000..974df369eed --- /dev/null +++ b/src/internal/fuzz/minimize.go @@ -0,0 +1,116 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package fuzz + +import ( + "math" + "reflect" +) + +func isMinimizable(t reflect.Type) bool { + for _, v := range zeroVals { + if t == reflect.TypeOf(v) { + return true + } + } + return false +} + +func minimizeBytes(v []byte, try func(interface{}) bool, shouldStop func() bool) { + tmp := make([]byte, len(v)) + // If minimization was successful at any point during minimizeBytes, + // then the vals slice in (*workerServer).minimizeInput will point to + // tmp. Since tmp is altered while making new candidates, we need to + // make sure that it is equal to the correct value, v, before exiting + // this function. + defer copy(tmp, v) + + // First, try to cut the tail. + for n := 1024; n != 0; n /= 2 { + for len(v) > n { + if shouldStop() { + return + } + candidate := v[:len(v)-n] + if !try(candidate) { + break + } + // Set v to the new value to continue iterating. + v = candidate + } + } + + // Then, try to remove each individual byte. + for i := 0; i < len(v)-1; i++ { + if shouldStop() { + return + } + candidate := tmp[:len(v)-1] + copy(candidate[:i], v[:i]) + copy(candidate[i:], v[i+1:]) + if !try(candidate) { + continue + } + // Update v to delete the value at index i. + copy(v[i:], v[i+1:]) + v = v[:len(candidate)] + // v[i] is now different, so decrement i to redo this iteration + // of the loop with the new value. + i-- + } + + // Then, try to remove each possible subset of bytes. + for i := 0; i < len(v)-1; i++ { + copy(tmp, v[:i]) + for j := len(v); j > i+1; j-- { + if shouldStop() { + return + } + candidate := tmp[:len(v)-j+i] + copy(candidate[i:], v[j:]) + if !try(candidate) { + continue + } + // Update v and reset the loop with the new length. + copy(v[i:], v[j:]) + v = v[:len(candidate)] + j = len(v) + } + } +} + +func minimizeInteger(v uint, try func(interface{}) bool, shouldStop func() bool) { + // TODO(rolandshoemaker): another approach could be either unsetting/setting all bits + // (depending on signed-ness), or rotating bits? When operating on cast signed integers + // this would probably be more complex though. + for ; v != 0; v /= 10 { + if shouldStop() { + return + } + // We ignore the return value here because there is no point + // advancing the loop, since there is nothing after this check, + // and we don't return early because a smaller value could + // re-trigger the crash. + try(v) + } +} + +func minimizeFloat(v float64, try func(interface{}) bool, shouldStop func() bool) { + if math.IsNaN(v) { + return + } + minimized := float64(0) + for div := 10.0; minimized < v; div *= 10 { + if shouldStop() { + return + } + minimized = float64(int(v*div)) / div + if !try(minimized) { + // Since we are searching from least precision -> highest precision we + // can return early since we've already found the smallest value + return + } + } +} diff --git a/src/internal/fuzz/minimize_test.go b/src/internal/fuzz/minimize_test.go new file mode 100644 index 00000000000..410b78310be --- /dev/null +++ b/src/internal/fuzz/minimize_test.go @@ -0,0 +1,286 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build darwin || linux || windows +// +build darwin linux windows + +package fuzz + +import ( + "bytes" + "context" + "errors" + "fmt" + "reflect" + "testing" +) + +func TestMinimizeInput(t *testing.T) { + type testcase struct { + name string + fn func(CorpusEntry) error + input []interface{} + expected []interface{} + } + cases := []testcase{ + { + name: "ones_byte", + fn: func(e CorpusEntry) error { + b := e.Values[0].([]byte) + ones := 0 + for _, v := range b { + if v == 1 { + ones++ + } + } + if ones == 3 { + return fmt.Errorf("bad %v", e.Values[0]) + } + return nil + }, + input: []interface{}{[]byte{0, 0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}, + expected: []interface{}{[]byte{1, 1, 1}}, + }, + { + name: "single_bytes", + fn: func(e CorpusEntry) error { + b := e.Values[0].([]byte) + if len(b) < 2 { + return nil + } + if len(b) == 2 && b[0] == 1 && b[1] == 2 { + return nil + } + return fmt.Errorf("bad %v", e.Values[0]) + }, + input: []interface{}{[]byte{1, 2, 3, 4, 5}}, + expected: []interface{}{[]byte{2, 3}}, + }, + { + name: "set_of_bytes", + fn: func(e CorpusEntry) error { + b := e.Values[0].([]byte) + if len(b) < 3 { + return nil + } + if bytes.Equal(b, []byte{0, 1, 2, 3, 4, 5}) || bytes.Equal(b, []byte{0, 4, 5}) { + return fmt.Errorf("bad %v", e.Values[0]) + } + return nil + }, + input: []interface{}{[]byte{0, 1, 2, 3, 4, 5}}, + expected: []interface{}{[]byte{0, 4, 5}}, + }, + { + name: "ones_string", + fn: func(e CorpusEntry) error { + b := e.Values[0].(string) + ones := 0 + for _, v := range b { + if v == '1' { + ones++ + } + } + if ones == 3 { + return fmt.Errorf("bad %v", e.Values[0]) + } + return nil + }, + input: []interface{}{"001010001000000000000000000"}, + expected: []interface{}{"111"}, + }, + { + name: "int", + fn: func(e CorpusEntry) error { + i := e.Values[0].(int) + if i > 100 { + return fmt.Errorf("bad %v", e.Values[0]) + } + return nil + }, + input: []interface{}{123456}, + expected: []interface{}{123}, + }, + { + name: "int8", + fn: func(e CorpusEntry) error { + i := e.Values[0].(int8) + if i > 10 { + return fmt.Errorf("bad %v", e.Values[0]) + } + return nil + }, + input: []interface{}{int8(1<<7 - 1)}, + expected: []interface{}{int8(12)}, + }, + { + name: "int16", + fn: func(e CorpusEntry) error { + i := e.Values[0].(int16) + if i > 10 { + return fmt.Errorf("bad %v", e.Values[0]) + } + return nil + }, + input: []interface{}{int16(1<<15 - 1)}, + expected: []interface{}{int16(32)}, + }, + { + fn: func(e CorpusEntry) error { + i := e.Values[0].(int32) + if i > 10 { + return fmt.Errorf("bad %v", e.Values[0]) + } + return nil + }, + input: []interface{}{int32(1<<31 - 1)}, + expected: []interface{}{int32(21)}, + }, + { + name: "int32", + fn: func(e CorpusEntry) error { + i := e.Values[0].(uint) + if i > 10 { + return fmt.Errorf("bad %v", e.Values[0]) + } + return nil + }, + input: []interface{}{uint(123456)}, + expected: []interface{}{uint(12)}, + }, + { + name: "uint8", + fn: func(e CorpusEntry) error { + i := e.Values[0].(uint8) + if i > 10 { + return fmt.Errorf("bad %v", e.Values[0]) + } + return nil + }, + input: []interface{}{uint8(1<<8 - 1)}, + expected: []interface{}{uint8(25)}, + }, + { + name: "uint16", + fn: func(e CorpusEntry) error { + i := e.Values[0].(uint16) + if i > 10 { + return fmt.Errorf("bad %v", e.Values[0]) + } + return nil + }, + input: []interface{}{uint16(1<<16 - 1)}, + expected: []interface{}{uint16(65)}, + }, + { + name: "uint32", + fn: func(e CorpusEntry) error { + i := e.Values[0].(uint32) + if i > 10 { + return fmt.Errorf("bad %v", e.Values[0]) + } + return nil + }, + input: []interface{}{uint32(1<<32 - 1)}, + expected: []interface{}{uint32(42)}, + }, + { + name: "float32", + fn: func(e CorpusEntry) error { + if i := e.Values[0].(float32); i == 1.23 { + return nil + } + return fmt.Errorf("bad %v", e.Values[0]) + }, + input: []interface{}{float32(1.23456789)}, + expected: []interface{}{float32(1.2)}, + }, + { + name: "float64", + fn: func(e CorpusEntry) error { + if i := e.Values[0].(float64); i == 1.23 { + return nil + } + return fmt.Errorf("bad %v", e.Values[0]) + }, + input: []interface{}{float64(1.23456789)}, + expected: []interface{}{float64(1.2)}, + }, + } + + // If we are on a 64 bit platform add int64 and uint64 tests + if v := int64(1<<63 - 1); int64(int(v)) == v { + cases = append(cases, testcase{ + name: "int64", + fn: func(e CorpusEntry) error { + i := e.Values[0].(int64) + if i > 10 { + return fmt.Errorf("bad %v", e.Values[0]) + } + return nil + }, + input: []interface{}{int64(1<<63 - 1)}, + expected: []interface{}{int64(92)}, + }, testcase{ + name: "uint64", + fn: func(e CorpusEntry) error { + i := e.Values[0].(uint64) + if i > 10 { + return fmt.Errorf("bad %v", e.Values[0]) + } + return nil + }, + input: []interface{}{uint64(1<<64 - 1)}, + expected: []interface{}{uint64(18)}, + }) + } + + for _, tc := range cases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + ws := &workerServer{ + fuzzFn: tc.fn, + } + count := int64(0) + vals := tc.input + success, err := ws.minimizeInput(context.Background(), vals, &count, 0, nil) + if !success { + t.Errorf("minimizeInput did not succeed") + } + if err == nil { + t.Fatal("minimizeInput didn't provide an error") + } + if expected := fmt.Sprintf("bad %v", tc.expected[0]); err.Error() != expected { + t.Errorf("unexpected error: got %q, want %q", err, expected) + } + if !reflect.DeepEqual(vals, tc.expected) { + t.Errorf("unexpected results: got %v, want %v", vals, tc.expected) + } + }) + } +} + +// TestMinimizeInputCoverageError checks that if we're minimizing an interesting +// input (one that we don't expect to cause an error), and the fuzz function +// returns an error, minimizing fails, and we return the error quickly. +func TestMinimizeInputCoverageError(t *testing.T) { + errOhNo := errors.New("ohno") + ws := &workerServer{fuzzFn: func(e CorpusEntry) error { + return errOhNo + }} + keepCoverage := make([]byte, len(coverageSnapshot)) + count := int64(0) + vals := []interface{}{[]byte(nil)} + success, err := ws.minimizeInput(context.Background(), vals, &count, 0, keepCoverage) + if success { + t.Error("unexpected success") + } + if err != errOhNo { + t.Errorf("unexpected error: %v", err) + } + if count != 1 { + t.Errorf("count: got %d, want 1", count) + } +} diff --git a/src/internal/fuzz/mutator.go b/src/internal/fuzz/mutator.go new file mode 100644 index 00000000000..9aa56782b0b --- /dev/null +++ b/src/internal/fuzz/mutator.go @@ -0,0 +1,317 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package fuzz + +import ( + "encoding/binary" + "fmt" + "math" + "reflect" + "unsafe" +) + +type mutator struct { + r mutatorRand + scratch []byte // scratch slice to avoid additional allocations +} + +func newMutator() *mutator { + return &mutator{r: newPcgRand()} +} + +func (m *mutator) rand(n int) int { + return m.r.intn(n) +} + +func (m *mutator) randByteOrder() binary.ByteOrder { + if m.r.bool() { + return binary.LittleEndian + } + return binary.BigEndian +} + +// chooseLen chooses length of range mutation in range [1,n]. It gives +// preference to shorter ranges. +func (m *mutator) chooseLen(n int) int { + switch x := m.rand(100); { + case x < 90: + return m.rand(min(8, n)) + 1 + case x < 99: + return m.rand(min(32, n)) + 1 + default: + return m.rand(n) + 1 + } +} + +func min(a, b int) int { + if a < b { + return a + } + return b +} + +// mutate performs several mutations on the provided values. +func (m *mutator) mutate(vals []interface{}, maxBytes int) { + // TODO(katiehockman): pull some of these functions into helper methods and + // test that each case is working as expected. + // TODO(katiehockman): perform more types of mutations for []byte. + + // maxPerVal will represent the maximum number of bytes that each value be + // allowed after mutating, giving an equal amount of capacity to each line. + // Allow a little wiggle room for the encoding. + maxPerVal := maxBytes/len(vals) - 100 + + // Pick a random value to mutate. + // TODO: consider mutating more than one value at a time. + i := m.rand(len(vals)) + switch v := vals[i].(type) { + case int: + vals[i] = int(m.mutateInt(int64(v), maxInt)) + case int8: + vals[i] = int8(m.mutateInt(int64(v), math.MaxInt8)) + case int16: + vals[i] = int16(m.mutateInt(int64(v), math.MaxInt16)) + case int64: + vals[i] = m.mutateInt(v, maxInt) + case uint: + vals[i] = uint(m.mutateUInt(uint64(v), maxUint)) + case uint16: + vals[i] = uint16(m.mutateUInt(uint64(v), math.MaxUint16)) + case uint32: + vals[i] = uint32(m.mutateUInt(uint64(v), math.MaxUint32)) + case uint64: + vals[i] = m.mutateUInt(uint64(v), maxUint) + case float32: + vals[i] = float32(m.mutateFloat(float64(v), math.MaxFloat32)) + case float64: + vals[i] = m.mutateFloat(v, math.MaxFloat64) + case bool: + if m.rand(2) == 1 { + vals[i] = !v // 50% chance of flipping the bool + } + case rune: // int32 + vals[i] = rune(m.mutateInt(int64(v), math.MaxInt32)) + case byte: // uint8 + vals[i] = byte(m.mutateUInt(uint64(v), math.MaxUint8)) + case string: + if len(v) > maxPerVal { + panic(fmt.Sprintf("cannot mutate bytes of length %d", len(v))) + } + if cap(m.scratch) < maxPerVal { + m.scratch = append(make([]byte, 0, maxPerVal), v...) + } else { + m.scratch = m.scratch[:len(v)] + copy(m.scratch, v) + } + m.mutateBytes(&m.scratch) + var s string + shdr := (*reflect.StringHeader)(unsafe.Pointer(&s)) + bhdr := (*reflect.SliceHeader)(unsafe.Pointer(&m.scratch)) + shdr.Data = bhdr.Data + shdr.Len = bhdr.Len + vals[i] = s + case []byte: + if len(v) > maxPerVal { + panic(fmt.Sprintf("cannot mutate bytes of length %d", len(v))) + } + if cap(m.scratch) < maxPerVal { + m.scratch = append(make([]byte, 0, maxPerVal), v...) + } else { + m.scratch = m.scratch[:len(v)] + copy(m.scratch, v) + } + m.mutateBytes(&m.scratch) + vals[i] = m.scratch + default: + panic(fmt.Sprintf("type not supported for mutating: %T", vals[i])) + } +} + +func (m *mutator) mutateInt(v, maxValue int64) int64 { + numIters := 1 + m.r.exp2() + var max int64 + for iter := 0; iter < numIters; iter++ { + max = 100 + switch m.rand(2) { + case 0: + // Add a random number + if v >= maxValue { + iter-- + continue + } + if v > 0 && maxValue-v < max { + // Don't let v exceed maxValue + max = maxValue - v + } + v += int64(1 + m.rand(int(max))) + case 1: + // Subtract a random number + if v <= -maxValue { + iter-- + continue + } + if v < 0 && maxValue+v < max { + // Don't let v drop below -maxValue + max = maxValue + v + } + v -= int64(1 + m.rand(int(max))) + } + } + return v +} + +func (m *mutator) mutateUInt(v, maxValue uint64) uint64 { + numIters := 1 + m.r.exp2() + var max uint64 + for iter := 0; iter < numIters; iter++ { + max = 100 + switch m.rand(2) { + case 0: + // Add a random number + if v >= maxValue { + iter-- + continue + } + if v > 0 && maxValue-v < max { + // Don't let v exceed maxValue + max = maxValue - v + } + + v += uint64(1 + m.rand(int(max))) + case 1: + // Subtract a random number + if v <= 0 { + iter-- + continue + } + if v < max { + // Don't let v drop below 0 + max = v + } + v -= uint64(1 + m.rand(int(max))) + } + } + return v +} + +func (m *mutator) mutateFloat(v, maxValue float64) float64 { + numIters := 1 + m.r.exp2() + var max float64 + for iter := 0; iter < numIters; iter++ { + switch m.rand(4) { + case 0: + // Add a random number + if v >= maxValue { + iter-- + continue + } + max = 100 + if v > 0 && maxValue-v < max { + // Don't let v exceed maxValue + max = maxValue - v + } + v += float64(1 + m.rand(int(max))) + case 1: + // Subtract a random number + if v <= -maxValue { + iter-- + continue + } + max = 100 + if v < 0 && maxValue+v < max { + // Don't let v drop below -maxValue + max = maxValue + v + } + v -= float64(1 + m.rand(int(max))) + case 2: + // Multiply by a random number + absV := math.Abs(v) + if v == 0 || absV >= maxValue { + iter-- + continue + } + max = 10 + if maxValue/absV < max { + // Don't let v go beyond the minimum or maximum value + max = maxValue / absV + } + v *= float64(1 + m.rand(int(max))) + case 3: + // Divide by a random number + if v == 0 { + iter-- + continue + } + v /= float64(1 + m.rand(10)) + } + } + return v +} + +type byteSliceMutator func(*mutator, []byte) []byte + +var byteSliceMutators = []byteSliceMutator{ + byteSliceRemoveBytes, + byteSliceInsertRandomBytes, + byteSliceDuplicateBytes, + byteSliceOverwriteBytes, + byteSliceBitFlip, + byteSliceXORByte, + byteSliceSwapByte, + byteSliceArithmeticUint8, + byteSliceArithmeticUint16, + byteSliceArithmeticUint32, + byteSliceArithmeticUint64, + byteSliceOverwriteInterestingUint8, + byteSliceOverwriteInterestingUint16, + byteSliceOverwriteInterestingUint32, + byteSliceInsertConstantBytes, + byteSliceOverwriteConstantBytes, + byteSliceShuffleBytes, + byteSliceSwapBytes, +} + +func (m *mutator) mutateBytes(ptrB *[]byte) { + b := *ptrB + defer func() { + oldHdr := (*reflect.SliceHeader)(unsafe.Pointer(ptrB)) + newHdr := (*reflect.SliceHeader)(unsafe.Pointer(&b)) + if oldHdr.Data != newHdr.Data { + panic("data moved to new address") + } + *ptrB = b + }() + + numIters := 1 + m.r.exp2() + for iter := 0; iter < numIters; iter++ { + mut := byteSliceMutators[m.rand(len(byteSliceMutators))] + mutated := mut(m, b) + if mutated == nil { + iter-- + continue + } + b = mutated + } +} + +var ( + interesting8 = []int8{-128, -1, 0, 1, 16, 32, 64, 100, 127} + interesting16 = []int16{-32768, -129, 128, 255, 256, 512, 1000, 1024, 4096, 32767} + interesting32 = []int32{-2147483648, -100663046, -32769, 32768, 65535, 65536, 100663045, 2147483647} +) + +const ( + maxUint = uint64(^uint(0)) + maxInt = int64(maxUint >> 1) +) + +func init() { + for _, v := range interesting8 { + interesting16 = append(interesting16, int16(v)) + } + for _, v := range interesting16 { + interesting32 = append(interesting32, int32(v)) + } +} diff --git a/src/internal/fuzz/mutator_test.go b/src/internal/fuzz/mutator_test.go new file mode 100644 index 00000000000..ee2912dfd21 --- /dev/null +++ b/src/internal/fuzz/mutator_test.go @@ -0,0 +1,101 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package fuzz + +import ( + "fmt" + "os" + "strconv" + "testing" +) + +func BenchmarkMutatorBytes(b *testing.B) { + origEnv := os.Getenv("GODEBUG") + defer func() { os.Setenv("GODEBUG", origEnv) }() + os.Setenv("GODEBUG", fmt.Sprintf("%s,fuzzseed=123", origEnv)) + m := newMutator() + + for _, size := range []int{ + 1, + 10, + 100, + 1000, + 10000, + 100000, + } { + b.Run(strconv.Itoa(size), func(b *testing.B) { + buf := make([]byte, size) + b.ResetTimer() + + for i := 0; i < b.N; i++ { + // resize buffer to the correct shape and reset the PCG + buf = buf[0:size] + m.r = newPcgRand() + m.mutate([]interface{}{buf}, workerSharedMemSize) + } + }) + } +} + +func BenchmarkMutatorString(b *testing.B) { + origEnv := os.Getenv("GODEBUG") + defer func() { os.Setenv("GODEBUG", origEnv) }() + os.Setenv("GODEBUG", fmt.Sprintf("%s,fuzzseed=123", origEnv)) + m := newMutator() + + for _, size := range []int{ + 1, + 10, + 100, + 1000, + 10000, + 100000, + } { + b.Run(strconv.Itoa(size), func(b *testing.B) { + buf := make([]byte, size) + b.ResetTimer() + + for i := 0; i < b.N; i++ { + // resize buffer to the correct shape and reset the PCG + buf = buf[0:size] + m.r = newPcgRand() + m.mutate([]interface{}{string(buf)}, workerSharedMemSize) + } + }) + } +} + +func BenchmarkMutatorAllBasicTypes(b *testing.B) { + origEnv := os.Getenv("GODEBUG") + defer func() { os.Setenv("GODEBUG", origEnv) }() + os.Setenv("GODEBUG", fmt.Sprintf("%s,fuzzseed=123", origEnv)) + m := newMutator() + + types := []interface{}{ + []byte(""), + string(""), + false, + float32(0), + float64(0), + int(0), + int8(0), + int16(0), + int32(0), + int64(0), + uint8(0), + uint16(0), + uint32(0), + uint64(0), + } + + for _, t := range types { + b.Run(fmt.Sprintf("%T", t), func(b *testing.B) { + for i := 0; i < b.N; i++ { + m.r = newPcgRand() + m.mutate([]interface{}{t}, workerSharedMemSize) + } + }) + } +} diff --git a/src/internal/fuzz/mutators_byteslice.go b/src/internal/fuzz/mutators_byteslice.go new file mode 100644 index 00000000000..7c96b5920ea --- /dev/null +++ b/src/internal/fuzz/mutators_byteslice.go @@ -0,0 +1,301 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package fuzz + +// byteSliceRemoveBytes removes a random chunk of bytes from b. +func byteSliceRemoveBytes(m *mutator, b []byte) []byte { + if len(b) <= 1 { + return nil + } + pos0 := m.rand(len(b)) + pos1 := pos0 + m.chooseLen(len(b)-pos0) + copy(b[pos0:], b[pos1:]) + b = b[:len(b)-(pos1-pos0)] + return b +} + +// byteSliceInsertRandomBytes inserts a chunk of random bytes into b at a random +// position. +func byteSliceInsertRandomBytes(m *mutator, b []byte) []byte { + pos := m.rand(len(b) + 1) + n := m.chooseLen(1024) + if len(b)+n >= cap(b) { + return nil + } + b = b[:len(b)+n] + copy(b[pos+n:], b[pos:]) + for i := 0; i < n; i++ { + b[pos+i] = byte(m.rand(256)) + } + return b +} + +// byteSliceDuplicateBytes duplicates a chunk of bytes in b and inserts it into +// a random position. +func byteSliceDuplicateBytes(m *mutator, b []byte) []byte { + if len(b) <= 1 { + return nil + } + src := m.rand(len(b)) + dst := m.rand(len(b)) + for dst == src { + dst = m.rand(len(b)) + } + n := m.chooseLen(len(b) - src) + // Use the end of the slice as scratch space to avoid doing an + // allocation. If the slice is too small abort and try something + // else. + if len(b)+(n*2) >= cap(b) { + return nil + } + end := len(b) + // Increase the size of b to fit the duplicated block as well as + // some extra working space + b = b[:end+(n*2)] + // Copy the block of bytes we want to duplicate to the end of the + // slice + copy(b[end+n:], b[src:src+n]) + // Shift the bytes after the splice point n positions to the right + // to make room for the new block + copy(b[dst+n:end+n], b[dst:end]) + // Insert the duplicate block into the splice point + copy(b[dst:], b[end+n:]) + b = b[:end+n] + return b +} + +// byteSliceOverwriteBytes overwrites a chunk of b with another chunk of b. +func byteSliceOverwriteBytes(m *mutator, b []byte) []byte { + if len(b) <= 1 { + return nil + } + src := m.rand(len(b)) + dst := m.rand(len(b)) + for dst == src { + dst = m.rand(len(b)) + } + n := m.chooseLen(len(b) - src - 1) + copy(b[dst:], b[src:src+n]) + return b +} + +// byteSliceBitFlip flips a random bit in a random byte in b. +func byteSliceBitFlip(m *mutator, b []byte) []byte { + if len(b) == 0 { + return nil + } + pos := m.rand(len(b)) + b[pos] ^= 1 << uint(m.rand(8)) + return b +} + +// byteSliceXORByte XORs a random byte in b with a random value. +func byteSliceXORByte(m *mutator, b []byte) []byte { + if len(b) == 0 { + return nil + } + pos := m.rand(len(b)) + // In order to avoid a no-op (where the random value matches + // the existing value), use XOR instead of just setting to + // the random value. + b[pos] ^= byte(1 + m.rand(255)) + return b +} + +// byteSliceSwapByte swaps two random bytes in b. +func byteSliceSwapByte(m *mutator, b []byte) []byte { + if len(b) <= 1 { + return nil + } + src := m.rand(len(b)) + dst := m.rand(len(b)) + for dst == src { + dst = m.rand(len(b)) + } + b[src], b[dst] = b[dst], b[src] + return b +} + +// byteSliceArithmeticUint8 adds/subtracts from a random byte in b. +func byteSliceArithmeticUint8(m *mutator, b []byte) []byte { + if len(b) == 0 { + return nil + } + pos := m.rand(len(b)) + v := byte(m.rand(35) + 1) + if m.r.bool() { + b[pos] += v + } else { + b[pos] -= v + } + return b +} + +// byteSliceArithmeticUint16 adds/subtracts from a random uint16 in b. +func byteSliceArithmeticUint16(m *mutator, b []byte) []byte { + if len(b) < 2 { + return nil + } + v := uint16(m.rand(35) + 1) + if m.r.bool() { + v = 0 - v + } + pos := m.rand(len(b) - 1) + enc := m.randByteOrder() + enc.PutUint16(b[pos:], enc.Uint16(b[pos:])+v) + return b +} + +// byteSliceArithmeticUint32 adds/subtracts from a random uint32 in b. +func byteSliceArithmeticUint32(m *mutator, b []byte) []byte { + if len(b) < 4 { + return nil + } + v := uint32(m.rand(35) + 1) + if m.r.bool() { + v = 0 - v + } + pos := m.rand(len(b) - 3) + enc := m.randByteOrder() + enc.PutUint32(b[pos:], enc.Uint32(b[pos:])+v) + return b +} + +// byteSliceArithmeticUint64 adds/subtracts from a random uint64 in b. +func byteSliceArithmeticUint64(m *mutator, b []byte) []byte { + if len(b) < 8 { + return nil + } + v := uint64(m.rand(35) + 1) + if m.r.bool() { + v = 0 - v + } + pos := m.rand(len(b) - 7) + enc := m.randByteOrder() + enc.PutUint64(b[pos:], enc.Uint64(b[pos:])+v) + return b +} + +// byteSliceOverwriteInterestingUint8 overwrites a random byte in b with an interesting +// value. +func byteSliceOverwriteInterestingUint8(m *mutator, b []byte) []byte { + if len(b) == 0 { + return nil + } + pos := m.rand(len(b)) + b[pos] = byte(interesting8[m.rand(len(interesting8))]) + return b +} + +// byteSliceOverwriteInterestingUint16 overwrites a random uint16 in b with an interesting +// value. +func byteSliceOverwriteInterestingUint16(m *mutator, b []byte) []byte { + if len(b) < 2 { + return nil + } + pos := m.rand(len(b) - 1) + v := uint16(interesting16[m.rand(len(interesting16))]) + m.randByteOrder().PutUint16(b[pos:], v) + return b +} + +// byteSliceOverwriteInterestingUint32 overwrites a random uint16 in b with an interesting +// value. +func byteSliceOverwriteInterestingUint32(m *mutator, b []byte) []byte { + if len(b) < 4 { + return nil + } + pos := m.rand(len(b) - 3) + v := uint32(interesting32[m.rand(len(interesting32))]) + m.randByteOrder().PutUint32(b[pos:], v) + return b +} + +// byteSliceInsertConstantBytes inserts a chunk of constant bytes into a random position in b. +func byteSliceInsertConstantBytes(m *mutator, b []byte) []byte { + if len(b) <= 1 { + return nil + } + dst := m.rand(len(b)) + // TODO(rolandshoemaker,katiehockman): 4096 was mainly picked + // randomly. We may want to either pick a much larger value + // (AFL uses 32768, paired with a similar impl to chooseLen + // which biases towards smaller lengths that grow over time), + // or set the max based on characteristics of the corpus + // (libFuzzer sets a min/max based on the min/max size of + // entries in the corpus and then picks uniformly from + // that range). + n := m.chooseLen(4096) + if len(b)+n >= cap(b) { + return nil + } + b = b[:len(b)+n] + copy(b[dst+n:], b[dst:]) + rb := byte(m.rand(256)) + for i := dst; i < dst+n; i++ { + b[i] = rb + } + return b +} + +// byteSliceOverwriteConstantBytes overwrites a chunk of b with constant bytes. +func byteSliceOverwriteConstantBytes(m *mutator, b []byte) []byte { + if len(b) <= 1 { + return nil + } + dst := m.rand(len(b)) + n := m.chooseLen(len(b) - dst) + rb := byte(m.rand(256)) + for i := dst; i < dst+n; i++ { + b[i] = rb + } + return b +} + +// byteSliceShuffleBytes shuffles a chunk of bytes in b. +func byteSliceShuffleBytes(m *mutator, b []byte) []byte { + if len(b) <= 1 { + return nil + } + dst := m.rand(len(b)) + n := m.chooseLen(len(b) - dst) + if n <= 2 { + return nil + } + // Start at the end of the range, and iterate backwards + // to dst, swapping each element with another element in + // dst:dst+n (Fisher-Yates shuffle). + for i := n - 1; i > 0; i-- { + j := m.rand(i + 1) + b[dst+i], b[dst+j] = b[dst+j], b[dst+i] + } + return b +} + +// byteSliceSwapBytes swaps two chunks of bytes in b. +func byteSliceSwapBytes(m *mutator, b []byte) []byte { + if len(b) <= 1 { + return nil + } + src := m.rand(len(b)) + dst := m.rand(len(b)) + for dst == src { + dst = m.rand(len(b)) + } + n := m.chooseLen(len(b) - src - 1) + // Use the end of the slice as scratch space to avoid doing an + // allocation. If the slice is too small abort and try something + // else. + if len(b)+n >= cap(b) { + return nil + } + end := len(b) + b = b[:end+n] + copy(b[end:], b[dst:dst+n]) + copy(b[dst:], b[src:src+n]) + copy(b[src:], b[end:]) + b = b[:end] + return b +} diff --git a/src/internal/fuzz/mutators_byteslice_test.go b/src/internal/fuzz/mutators_byteslice_test.go new file mode 100644 index 00000000000..50a39a9a5b5 --- /dev/null +++ b/src/internal/fuzz/mutators_byteslice_test.go @@ -0,0 +1,179 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package fuzz + +import ( + "bytes" + "testing" +) + +type mockRand struct { + counter int + b bool +} + +func (mr *mockRand) uint32() uint32 { + c := mr.counter + mr.counter++ + return uint32(c) +} + +func (mr *mockRand) intn(n int) int { + c := mr.counter + mr.counter++ + return c % n +} + +func (mr *mockRand) uint32n(n uint32) uint32 { + c := mr.counter + mr.counter++ + return uint32(c) % n +} + +func (mr *mockRand) exp2() int { + c := mr.counter + mr.counter++ + return c +} + +func (mr *mockRand) bool() bool { + b := mr.b + mr.b = !mr.b + return b +} + +func (mr *mockRand) save(*uint64, *uint64) { + panic("unimplemented") +} + +func (mr *mockRand) restore(uint64, uint64) { + panic("unimplemented") +} + +func TestByteSliceMutators(t *testing.T) { + for _, tc := range []struct { + name string + mutator func(*mutator, []byte) []byte + input []byte + expected []byte + }{ + { + name: "byteSliceRemoveBytes", + mutator: byteSliceRemoveBytes, + input: []byte{1, 2, 3, 4}, + expected: []byte{4}, + }, + { + name: "byteSliceInsertRandomBytes", + mutator: byteSliceInsertRandomBytes, + input: make([]byte, 4, 8), + expected: []byte{3, 4, 5, 0, 0, 0, 0}, + }, + { + name: "byteSliceDuplicateBytes", + mutator: byteSliceDuplicateBytes, + input: append(make([]byte, 0, 13), []byte{1, 2, 3, 4}...), + expected: []byte{1, 1, 2, 3, 4, 2, 3, 4}, + }, + { + name: "byteSliceOverwriteBytes", + mutator: byteSliceOverwriteBytes, + input: []byte{1, 2, 3, 4}, + expected: []byte{1, 1, 3, 4}, + }, + { + name: "byteSliceBitFlip", + mutator: byteSliceBitFlip, + input: []byte{1, 2, 3, 4}, + expected: []byte{3, 2, 3, 4}, + }, + { + name: "byteSliceXORByte", + mutator: byteSliceXORByte, + input: []byte{1, 2, 3, 4}, + expected: []byte{3, 2, 3, 4}, + }, + { + name: "byteSliceSwapByte", + mutator: byteSliceSwapByte, + input: []byte{1, 2, 3, 4}, + expected: []byte{2, 1, 3, 4}, + }, + { + name: "byteSliceArithmeticUint8", + mutator: byteSliceArithmeticUint8, + input: []byte{1, 2, 3, 4}, + expected: []byte{255, 2, 3, 4}, + }, + { + name: "byteSliceArithmeticUint16", + mutator: byteSliceArithmeticUint16, + input: []byte{1, 2, 3, 4}, + expected: []byte{1, 3, 3, 4}, + }, + { + name: "byteSliceArithmeticUint32", + mutator: byteSliceArithmeticUint32, + input: []byte{1, 2, 3, 4}, + expected: []byte{2, 2, 3, 4}, + }, + { + name: "byteSliceArithmeticUint64", + mutator: byteSliceArithmeticUint64, + input: []byte{1, 2, 3, 4, 5, 6, 7, 8}, + expected: []byte{2, 2, 3, 4, 5, 6, 7, 8}, + }, + { + name: "byteSliceOverwriteInterestingUint8", + mutator: byteSliceOverwriteInterestingUint8, + input: []byte{1, 2, 3, 4}, + expected: []byte{255, 2, 3, 4}, + }, + { + name: "byteSliceOverwriteInterestingUint16", + mutator: byteSliceOverwriteInterestingUint16, + input: []byte{1, 2, 3, 4}, + expected: []byte{255, 127, 3, 4}, + }, + { + name: "byteSliceOverwriteInterestingUint32", + mutator: byteSliceOverwriteInterestingUint32, + input: []byte{1, 2, 3, 4}, + expected: []byte{250, 0, 0, 250}, + }, + { + name: "byteSliceInsertConstantBytes", + mutator: byteSliceInsertConstantBytes, + input: append(make([]byte, 0, 8), []byte{1, 2, 3, 4}...), + expected: []byte{3, 3, 3, 1, 2, 3, 4}, + }, + { + name: "byteSliceOverwriteConstantBytes", + mutator: byteSliceOverwriteConstantBytes, + input: []byte{1, 2, 3, 4}, + expected: []byte{3, 3, 3, 4}, + }, + { + name: "byteSliceShuffleBytes", + mutator: byteSliceShuffleBytes, + input: []byte{1, 2, 3, 4}, + expected: []byte{2, 3, 1, 4}, + }, + { + name: "byteSliceSwapBytes", + mutator: byteSliceSwapBytes, + input: append(make([]byte, 0, 9), []byte{1, 2, 3, 4}...), + expected: []byte{2, 1, 3, 4}, + }, + } { + t.Run(tc.name, func(t *testing.T) { + m := &mutator{r: &mockRand{}} + b := tc.mutator(m, tc.input) + if !bytes.Equal(b, tc.expected) { + t.Errorf("got %x, want %x", b, tc.expected) + } + }) + } +} diff --git a/src/internal/fuzz/pcg.go b/src/internal/fuzz/pcg.go new file mode 100644 index 00000000000..c9ea0afcf8c --- /dev/null +++ b/src/internal/fuzz/pcg.go @@ -0,0 +1,145 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package fuzz + +import ( + "math/bits" + "os" + "strconv" + "strings" + "sync/atomic" + "time" +) + +type mutatorRand interface { + uint32() uint32 + intn(int) int + uint32n(uint32) uint32 + exp2() int + bool() bool + + save(randState, randInc *uint64) + restore(randState, randInc uint64) +} + +// The functions in pcg implement a 32 bit PRNG with a 64 bit period: pcg xsh rr +// 64 32. See https://www.pcg-random.org/ for more information. This +// implementation is geared specifically towards the needs of fuzzing: Simple +// creation and use, no reproducibility, no concurrency safety, just the +// necessary methods, optimized for speed. + +var globalInc uint64 // PCG stream + +const multiplier uint64 = 6364136223846793005 + +// pcgRand is a PRNG. It should not be copied or shared. No Rand methods are +// concurrency safe. +type pcgRand struct { + noCopy noCopy // help avoid mistakes: ask vet to ensure that we don't make a copy + state uint64 + inc uint64 +} + +func godebugSeed() *int { + debug := strings.Split(os.Getenv("GODEBUG"), ",") + for _, f := range debug { + if strings.HasPrefix(f, "fuzzseed=") { + seed, err := strconv.Atoi(strings.TrimPrefix(f, "fuzzseed=")) + if err != nil { + panic("malformed fuzzseed") + } + return &seed + } + } + return nil +} + +// newPcgRand generates a new, seeded Rand, ready for use. +func newPcgRand() *pcgRand { + r := new(pcgRand) + now := uint64(time.Now().UnixNano()) + if seed := godebugSeed(); seed != nil { + now = uint64(*seed) + } + inc := atomic.AddUint64(&globalInc, 1) + r.state = now + r.inc = (inc << 1) | 1 + r.step() + r.state += now + r.step() + return r +} + +func (r *pcgRand) step() { + r.state *= multiplier + r.state += r.inc +} + +func (r *pcgRand) save(randState, randInc *uint64) { + *randState = r.state + *randInc = r.inc +} + +func (r *pcgRand) restore(randState, randInc uint64) { + r.state = randState + r.inc = randInc +} + +// uint32 returns a pseudo-random uint32. +func (r *pcgRand) uint32() uint32 { + x := r.state + r.step() + return bits.RotateLeft32(uint32(((x>>18)^x)>>27), -int(x>>59)) +} + +// intn returns a pseudo-random number in [0, n). +// n must fit in a uint32. +func (r *pcgRand) intn(n int) int { + if int(uint32(n)) != n { + panic("large Intn") + } + return int(r.uint32n(uint32(n))) +} + +// uint32n returns a pseudo-random number in [0, n). +// +// For implementation details, see: +// https://lemire.me/blog/2016/06/27/a-fast-alternative-to-the-modulo-reduction +// https://lemire.me/blog/2016/06/30/fast-random-shuffling +func (r *pcgRand) uint32n(n uint32) uint32 { + v := r.uint32() + prod := uint64(v) * uint64(n) + low := uint32(prod) + if low < n { + thresh := uint32(-int32(n)) % n + for low < thresh { + v = r.uint32() + prod = uint64(v) * uint64(n) + low = uint32(prod) + } + } + return uint32(prod >> 32) +} + +// exp2 generates n with probability 1/2^(n+1). +func (r *pcgRand) exp2() int { + return bits.TrailingZeros32(r.uint32()) +} + +// bool generates a random bool. +func (r *pcgRand) bool() bool { + return r.uint32()&1 == 0 +} + +// noCopy may be embedded into structs which must not be copied +// after the first use. +// +// See https://golang.org/issues/8005#issuecomment-190753527 +// for details. +type noCopy struct{} + +// lock is a no-op used by -copylocks checker from `go vet`. +func (*noCopy) lock() {} +func (*noCopy) unlock() {} diff --git a/src/internal/fuzz/queue.go b/src/internal/fuzz/queue.go new file mode 100644 index 00000000000..cf67a28ba72 --- /dev/null +++ b/src/internal/fuzz/queue.go @@ -0,0 +1,71 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package fuzz + +// queue holds a growable sequence of inputs for fuzzing and minimization. +// +// For now, this is a simple ring buffer +// (https://en.wikipedia.org/wiki/Circular_buffer). +// +// TODO(golang.org/issue/46224): use a priotization algorithm based on input +// size, previous duration, coverage, and any other metrics that seem useful. +type queue struct { + // elems holds a ring buffer. + // The queue is empty when begin = end. + // The queue is full (until grow is called) when end = begin + N - 1 (mod N) + // where N = cap(elems). + elems []interface{} + head, len int +} + +func (q *queue) cap() int { + return len(q.elems) +} + +func (q *queue) grow() { + oldCap := q.cap() + newCap := oldCap * 2 + if newCap == 0 { + newCap = 8 + } + newElems := make([]interface{}, newCap) + oldLen := q.len + for i := 0; i < oldLen; i++ { + newElems[i] = q.elems[(q.head+i)%oldCap] + } + q.elems = newElems + q.head = 0 +} + +func (q *queue) enqueue(e interface{}) { + if q.len+1 > q.cap() { + q.grow() + } + i := (q.head + q.len) % q.cap() + q.elems[i] = e + q.len++ +} + +func (q *queue) dequeue() (interface{}, bool) { + if q.len == 0 { + return nil, false + } + e := q.elems[q.head] + q.elems[q.head] = nil + q.head = (q.head + 1) % q.cap() + q.len-- + return e, true +} + +func (q *queue) peek() (interface{}, bool) { + if q.len == 0 { + return nil, false + } + return q.elems[q.head], true +} + +func (q *queue) clear() { + *q = queue{} +} diff --git a/src/internal/fuzz/queue_test.go b/src/internal/fuzz/queue_test.go new file mode 100644 index 00000000000..3b179afb573 --- /dev/null +++ b/src/internal/fuzz/queue_test.go @@ -0,0 +1,58 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package fuzz + +import "testing" + +func TestQueue(t *testing.T) { + // Zero valued queue should have 0 length and capacity. + var q queue + if n := q.len; n != 0 { + t.Fatalf("empty queue has len %d; want 0", n) + } + if n := q.cap(); n != 0 { + t.Fatalf("empty queue has cap %d; want 0", n) + } + + // As we add elements, len should grow. + N := 32 + for i := 0; i < N; i++ { + q.enqueue(i) + if n := q.len; n != i+1 { + t.Fatalf("after adding %d elements, queue has len %d", i, n) + } + if v, ok := q.peek(); !ok { + t.Fatalf("couldn't peek after adding %d elements", i) + } else if v.(int) != 0 { + t.Fatalf("after adding %d elements, peek is %d; want 0", i, v) + } + } + + // As we remove and add elements, len should shrink and grow. + // We should also remove elements in the same order they were added. + want := 0 + for _, r := range []int{1, 2, 3, 5, 8, 13, 21} { + s := make([]int, 0, r) + for i := 0; i < r; i++ { + if got, ok := q.dequeue(); !ok { + t.Fatalf("after removing %d of %d elements, could not dequeue", i+1, r) + } else if got != want { + t.Fatalf("after removing %d of %d elements, got %d; want %d", i+1, r, got, want) + } else { + s = append(s, got.(int)) + } + want = (want + 1) % N + if n := q.len; n != N-i-1 { + t.Fatalf("after removing %d of %d elements, len is %d; want %d", i+1, r, n, N-i-1) + } + } + for i, v := range s { + q.enqueue(v) + if n := q.len; n != N-r+i+1 { + t.Fatalf("after adding back %d of %d elements, len is %d; want %d", i+1, r, n, n-r+i+1) + } + } + } +} diff --git a/src/internal/fuzz/sys_posix.go b/src/internal/fuzz/sys_posix.go new file mode 100644 index 00000000000..2473274ecf2 --- /dev/null +++ b/src/internal/fuzz/sys_posix.go @@ -0,0 +1,131 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build darwin || linux +// +build darwin linux + +package fuzz + +import ( + "fmt" + "os" + "os/exec" + "syscall" +) + +type sharedMemSys struct{} + +func sharedMemMapFile(f *os.File, size int, removeOnClose bool) (*sharedMem, error) { + prot := syscall.PROT_READ | syscall.PROT_WRITE + flags := syscall.MAP_FILE | syscall.MAP_SHARED + region, err := syscall.Mmap(int(f.Fd()), 0, size, prot, flags) + if err != nil { + return nil, err + } + + return &sharedMem{f: f, region: region, removeOnClose: removeOnClose}, nil +} + +// Close unmaps the shared memory and closes the temporary file. If this +// sharedMem was created with sharedMemTempFile, Close also removes the file. +func (m *sharedMem) Close() error { + // Attempt all operations, even if we get an error for an earlier operation. + // os.File.Close may fail due to I/O errors, but we still want to delete + // the temporary file. + var errs []error + errs = append(errs, + syscall.Munmap(m.region), + m.f.Close()) + if m.removeOnClose { + errs = append(errs, os.Remove(m.f.Name())) + } + for _, err := range errs { + if err != nil { + return err + } + } + return nil +} + +// setWorkerComm configures communciation channels on the cmd that will +// run a worker process. +func setWorkerComm(cmd *exec.Cmd, comm workerComm) { + mem := <-comm.memMu + memFile := mem.f + comm.memMu <- mem + cmd.ExtraFiles = []*os.File{comm.fuzzIn, comm.fuzzOut, memFile} +} + +// getWorkerComm returns communication channels in the worker process. +func getWorkerComm() (comm workerComm, err error) { + fuzzIn := os.NewFile(3, "fuzz_in") + fuzzOut := os.NewFile(4, "fuzz_out") + memFile := os.NewFile(5, "fuzz_mem") + fi, err := memFile.Stat() + if err != nil { + return workerComm{}, err + } + size := int(fi.Size()) + if int64(size) != fi.Size() { + return workerComm{}, fmt.Errorf("fuzz temp file exceeds maximum size") + } + removeOnClose := false + mem, err := sharedMemMapFile(memFile, size, removeOnClose) + if err != nil { + return workerComm{}, err + } + memMu := make(chan *sharedMem, 1) + memMu <- mem + return workerComm{fuzzIn: fuzzIn, fuzzOut: fuzzOut, memMu: memMu}, nil +} + +// isInterruptError returns whether an error was returned by a process that +// was terminated by an interrupt signal (SIGINT). +func isInterruptError(err error) bool { + exitErr, ok := err.(*exec.ExitError) + if !ok || exitErr.ExitCode() >= 0 { + return false + } + status := exitErr.Sys().(syscall.WaitStatus) + return status.Signal() == syscall.SIGINT +} + +// terminationSignal checks if err is an exec.ExitError with a signal status. +// If it is, terminationSignal returns the signal and true. +// If not, -1 and false. +func terminationSignal(err error) (os.Signal, bool) { + exitErr, ok := err.(*exec.ExitError) + if !ok || exitErr.ExitCode() >= 0 { + return syscall.Signal(-1), false + } + status := exitErr.Sys().(syscall.WaitStatus) + return status.Signal(), status.Signaled() +} + +// isCrashSignal returns whether a signal was likely to have been caused by an +// error in the program that received it, triggered by a fuzz input. For +// example, SIGSEGV would be received after a nil pointer dereference. +// Other signals like SIGKILL or SIGHUP are more likely to have been sent by +// another process, and we shouldn't record a crasher if the worker process +// receives one of these. +// +// Note that Go installs its own signal handlers on startup, so some of these +// signals may only be received if signal handlers are changed. For example, +// SIGSEGV is normally transformed into a panic that causes the process to exit +// with status 2 if not recovered, which we handle as a crash. +func isCrashSignal(signal os.Signal) bool { + switch signal { + case + syscall.SIGILL, // illegal instruction + syscall.SIGTRAP, // breakpoint + syscall.SIGABRT, // abort() called + syscall.SIGBUS, // invalid memory access (e.g., misaligned address) + syscall.SIGFPE, // math error, e.g., integer divide by zero + syscall.SIGSEGV, // invalid memory access (e.g., write to read-only) + syscall.SIGPIPE: // sent data to closed pipe or socket + return true + default: + return false + } +} diff --git a/src/internal/fuzz/sys_unimplemented.go b/src/internal/fuzz/sys_unimplemented.go new file mode 100644 index 00000000000..827e36cf329 --- /dev/null +++ b/src/internal/fuzz/sys_unimplemented.go @@ -0,0 +1,44 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// TODO(jayconrod): support more platforms. +//go:build !darwin && !linux && !windows +// +build !darwin,!linux,!windows + +package fuzz + +import ( + "os" + "os/exec" +) + +type sharedMemSys struct{} + +func sharedMemMapFile(f *os.File, size int, removeOnClose bool) (*sharedMem, error) { + panic("not implemented") +} + +func (m *sharedMem) Close() error { + panic("not implemented") +} + +func setWorkerComm(cmd *exec.Cmd, comm workerComm) { + panic("not implemented") +} + +func getWorkerComm() (comm workerComm, err error) { + panic("not implemented") +} + +func isInterruptError(err error) bool { + panic("not implemented") +} + +func terminationSignal(err error) (os.Signal, bool) { + panic("not implemented") +} + +func isCrashSignal(signal os.Signal) bool { + panic("not implemented") +} diff --git a/src/internal/fuzz/sys_windows.go b/src/internal/fuzz/sys_windows.go new file mode 100644 index 00000000000..fabf954ba73 --- /dev/null +++ b/src/internal/fuzz/sys_windows.go @@ -0,0 +1,152 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package fuzz + +import ( + "fmt" + "os" + "os/exec" + "reflect" + "syscall" + "unsafe" +) + +type sharedMemSys struct { + mapObj syscall.Handle +} + +func sharedMemMapFile(f *os.File, size int, removeOnClose bool) (mem *sharedMem, err error) { + defer func() { + if err != nil { + err = fmt.Errorf("mapping temporary file %s: %w", f.Name(), err) + } + }() + + // Create a file mapping object. The object itself is not shared. + mapObj, err := syscall.CreateFileMapping( + syscall.Handle(f.Fd()), // fhandle + nil, // sa + syscall.PAGE_READWRITE, // prot + 0, // maxSizeHigh + 0, // maxSizeLow + nil, // name + ) + if err != nil { + return nil, err + } + + // Create a view from the file mapping object. + access := uint32(syscall.FILE_MAP_READ | syscall.FILE_MAP_WRITE) + addr, err := syscall.MapViewOfFile( + mapObj, // handle + access, // access + 0, // offsetHigh + 0, // offsetLow + uintptr(size), // length + ) + if err != nil { + syscall.CloseHandle(mapObj) + return nil, err + } + + var region []byte + header := (*reflect.SliceHeader)(unsafe.Pointer(®ion)) + header.Data = addr + header.Len = size + header.Cap = size + return &sharedMem{ + f: f, + region: region, + removeOnClose: removeOnClose, + sys: sharedMemSys{mapObj: mapObj}, + }, nil +} + +// Close unmaps the shared memory and closes the temporary file. If this +// sharedMem was created with sharedMemTempFile, Close also removes the file. +func (m *sharedMem) Close() error { + // Attempt all operations, even if we get an error for an earlier operation. + // os.File.Close may fail due to I/O errors, but we still want to delete + // the temporary file. + var errs []error + errs = append(errs, + syscall.UnmapViewOfFile(uintptr(unsafe.Pointer(&m.region[0]))), + syscall.CloseHandle(m.sys.mapObj), + m.f.Close()) + if m.removeOnClose { + errs = append(errs, os.Remove(m.f.Name())) + } + for _, err := range errs { + if err != nil { + return err + } + } + return nil +} + +// setWorkerComm configures communciation channels on the cmd that will +// run a worker process. +func setWorkerComm(cmd *exec.Cmd, comm workerComm) { + mem := <-comm.memMu + memName := mem.f.Name() + comm.memMu <- mem + syscall.SetHandleInformation(syscall.Handle(comm.fuzzIn.Fd()), syscall.HANDLE_FLAG_INHERIT, 1) + syscall.SetHandleInformation(syscall.Handle(comm.fuzzOut.Fd()), syscall.HANDLE_FLAG_INHERIT, 1) + cmd.Env = append(cmd.Env, fmt.Sprintf("GO_TEST_FUZZ_WORKER_HANDLES=%x,%x,%q", comm.fuzzIn.Fd(), comm.fuzzOut.Fd(), memName)) + cmd.SysProcAttr = &syscall.SysProcAttr{AdditionalInheritedHandles: []syscall.Handle{syscall.Handle(comm.fuzzIn.Fd()), syscall.Handle(comm.fuzzOut.Fd())}} +} + +// getWorkerComm returns communication channels in the worker process. +func getWorkerComm() (comm workerComm, err error) { + v := os.Getenv("GO_TEST_FUZZ_WORKER_HANDLES") + if v == "" { + return workerComm{}, fmt.Errorf("GO_TEST_FUZZ_WORKER_HANDLES not set") + } + var fuzzInFD, fuzzOutFD uintptr + var memName string + if _, err := fmt.Sscanf(v, "%x,%x,%q", &fuzzInFD, &fuzzOutFD, &memName); err != nil { + return workerComm{}, fmt.Errorf("parsing GO_TEST_FUZZ_WORKER_HANDLES=%s: %v", v, err) + } + + fuzzIn := os.NewFile(fuzzInFD, "fuzz_in") + fuzzOut := os.NewFile(fuzzOutFD, "fuzz_out") + tmpFile, err := os.OpenFile(memName, os.O_RDWR, 0) + if err != nil { + return workerComm{}, fmt.Errorf("worker opening temp file: %w", err) + } + fi, err := tmpFile.Stat() + if err != nil { + return workerComm{}, fmt.Errorf("worker checking temp file size: %w", err) + } + size := int(fi.Size()) + if int64(size) != fi.Size() { + return workerComm{}, fmt.Errorf("fuzz temp file exceeds maximum size") + } + removeOnClose := false + mem, err := sharedMemMapFile(tmpFile, size, removeOnClose) + if err != nil { + return workerComm{}, err + } + memMu := make(chan *sharedMem, 1) + memMu <- mem + + return workerComm{fuzzIn: fuzzIn, fuzzOut: fuzzOut, memMu: memMu}, nil +} + +func isInterruptError(err error) bool { + // On Windows, we can't tell whether the process was interrupted by the error + // returned by Wait. It looks like an ExitError with status 1. + return false +} + +// terminationSignal returns -1 and false because Windows doesn't have signals. +func terminationSignal(err error) (os.Signal, bool) { + return syscall.Signal(-1), false +} + +// isCrashSignal is not implemented because Windows doesn't have signals. +func isCrashSignal(signal os.Signal) bool { + panic("not implemented: no signals on windows") +} diff --git a/src/internal/fuzz/trace.go b/src/internal/fuzz/trace.go new file mode 100644 index 00000000000..f70b1a6f5ba --- /dev/null +++ b/src/internal/fuzz/trace.go @@ -0,0 +1,29 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !libfuzzer + +package fuzz + +import _ "unsafe" // for go:linkname + +//go:linkname libfuzzerTraceCmp1 runtime.libfuzzerTraceCmp1 +//go:linkname libfuzzerTraceCmp2 runtime.libfuzzerTraceCmp2 +//go:linkname libfuzzerTraceCmp4 runtime.libfuzzerTraceCmp4 +//go:linkname libfuzzerTraceCmp8 runtime.libfuzzerTraceCmp8 + +//go:linkname libfuzzerTraceConstCmp1 runtime.libfuzzerTraceConstCmp1 +//go:linkname libfuzzerTraceConstCmp2 runtime.libfuzzerTraceConstCmp2 +//go:linkname libfuzzerTraceConstCmp4 runtime.libfuzzerTraceConstCmp4 +//go:linkname libfuzzerTraceConstCmp8 runtime.libfuzzerTraceConstCmp8 + +func libfuzzerTraceCmp1(arg0, arg1 uint8) {} +func libfuzzerTraceCmp2(arg0, arg1 uint16) {} +func libfuzzerTraceCmp4(arg0, arg1 uint32) {} +func libfuzzerTraceCmp8(arg0, arg1 uint64) {} + +func libfuzzerTraceConstCmp1(arg0, arg1 uint8) {} +func libfuzzerTraceConstCmp2(arg0, arg1 uint16) {} +func libfuzzerTraceConstCmp4(arg0, arg1 uint32) {} +func libfuzzerTraceConstCmp8(arg0, arg1 uint64) {} diff --git a/src/internal/fuzz/worker.go b/src/internal/fuzz/worker.go new file mode 100644 index 00000000000..da82a95fa18 --- /dev/null +++ b/src/internal/fuzz/worker.go @@ -0,0 +1,1160 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package fuzz + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/json" + "errors" + "fmt" + "io" + "io/ioutil" + "os" + "os/exec" + "runtime" + "sync" + "time" +) + +const ( + // workerFuzzDuration is the amount of time a worker can spend testing random + // variations of an input given by the coordinator. + workerFuzzDuration = 100 * time.Millisecond + + // workerTimeoutDuration is the amount of time a worker can go without + // responding to the coordinator before being stopped. + workerTimeoutDuration = 1 * time.Second + + // workerExitCode is used as an exit code by fuzz worker processes after an internal error. + // This distinguishes internal errors from uncontrolled panics and other crashes. + // Keep in sync with internal/fuzz.workerExitCode. + workerExitCode = 70 + + // workerSharedMemSize is the maximum size of the shared memory file used to + // communicate with workers. This limits the size of fuzz inputs. + workerSharedMemSize = 100 << 20 // 100 MB +) + +// worker manages a worker process running a test binary. The worker object +// exists only in the coordinator (the process started by 'go test -fuzz'). +// workerClient is used by the coordinator to send RPCs to the worker process, +// which handles them with workerServer. +type worker struct { + dir string // working directory, same as package directory + binPath string // path to test executable + args []string // arguments for test executable + env []string // environment for test executable + + coordinator *coordinator + + memMu chan *sharedMem // mutex guarding shared memory with worker; persists across processes. + + cmd *exec.Cmd // current worker process + client *workerClient // used to communicate with worker process + waitErr error // last error returned by wait, set before termC is closed. + interrupted bool // true after stop interrupts a running worker. + termC chan struct{} // closed by wait when worker process terminates +} + +func newWorker(c *coordinator, dir, binPath string, args, env []string) (*worker, error) { + mem, err := sharedMemTempFile(workerSharedMemSize) + if err != nil { + return nil, err + } + memMu := make(chan *sharedMem, 1) + memMu <- mem + return &worker{ + dir: dir, + binPath: binPath, + args: args, + env: env[:len(env):len(env)], // copy on append to ensure workers don't overwrite each other. + coordinator: c, + memMu: memMu, + }, nil +} + +// cleanup releases persistent resources associated with the worker. +func (w *worker) cleanup() error { + mem := <-w.memMu + if mem == nil { + return nil + } + close(w.memMu) + return mem.Close() +} + +// coordinate runs the test binary to perform fuzzing. +// +// coordinate loops until ctx is cancelled or a fatal error is encountered. +// If a test process terminates unexpectedly while fuzzing, coordinate will +// attempt to restart and continue unless the termination can be attributed +// to an interruption (from a timer or the user). +// +// While looping, coordinate receives inputs from the coordinator, passes +// those inputs to the worker process, then passes the results back to +// the coordinator. +func (w *worker) coordinate(ctx context.Context) error { + // Main event loop. + for { + // Start or restart the worker if it's not running. + if !w.isRunning() { + if err := w.startAndPing(ctx); err != nil { + return err + } + } + + select { + case <-ctx.Done(): + // Worker was told to stop. + err := w.stop() + if err != nil && !w.interrupted && !isInterruptError(err) { + return err + } + return ctx.Err() + + case <-w.termC: + // Worker process terminated unexpectedly while waiting for input. + err := w.stop() + if w.interrupted { + panic("worker interrupted after unexpected termination") + } + if err == nil || isInterruptError(err) { + // Worker stopped, either by exiting with status 0 or after being + // interrupted with a signal that was not sent by the coordinator. + // + // When the user presses ^C, on POSIX platforms, SIGINT is delivered to + // all processes in the group concurrently, and the worker may see it + // before the coordinator. The worker should exit 0 gracefully (in + // theory). + // + // This condition is probably intended by the user, so suppress + // the error. + return nil + } + if exitErr, ok := err.(*exec.ExitError); ok && exitErr.ExitCode() == workerExitCode { + // Worker exited with a code indicating F.Fuzz was not called correctly, + // for example, F.Fail was called first. + return fmt.Errorf("fuzzing process exited unexpectedly due to an internal failure: %w", err) + } + // Worker exited non-zero or was terminated by a non-interrupt signal + // (for example, SIGSEGV) while fuzzing. + return fmt.Errorf("fuzzing process terminated unexpectedly: %w", err) + // TODO(jayconrod,katiehockman): if -keepfuzzing, restart worker. + + case input := <-w.coordinator.inputC: + // Received input from coordinator. + args := fuzzArgs{ + Limit: input.limit, + Timeout: input.timeout, + Warmup: input.warmup, + CoverageData: input.coverageData, + } + entry, resp, err := w.client.fuzz(ctx, input.entry, args) + if err != nil { + // Error communicating with worker. + w.stop() + if ctx.Err() != nil { + // Timeout or interruption. + return ctx.Err() + } + if w.interrupted { + // Communication error before we stopped the worker. + // Report an error, but don't record a crasher. + return fmt.Errorf("communicating with fuzzing process: %v", err) + } + if w.waitErr == nil || isInterruptError(w.waitErr) { + // Worker stopped, either by exiting with status 0 or after being + // interrupted with a signal (not sent by coordinator). See comment in + // termC case above. + // + // Since we expect I/O errors around interrupts, ignore this error. + return nil + } + if sig, ok := terminationSignal(w.waitErr); ok && !isCrashSignal(sig) { + // Worker terminated by a signal that probably wasn't caused by a + // specific input to the fuzz function. For example, on Linux, + // the kernel (OOM killer) may send SIGKILL to a process using a lot + // of memory. Or the shell might send SIGHUP when the terminal + // is closed. Don't record a crasher. + return fmt.Errorf("fuzzing process terminated by unexpected signal; no crash will be recorded: %v", w.waitErr) + } + // Unexpected termination. Set error message and fall through. + // We'll restart the worker on the next iteration. + resp.Err = fmt.Sprintf("fuzzing process terminated unexpectedly: %v", w.waitErr) + } + result := fuzzResult{ + limit: input.limit, + count: resp.Count, + totalDuration: resp.TotalDuration, + entryDuration: resp.InterestingDuration, + entry: entry, + crasherMsg: resp.Err, + coverageData: resp.CoverageData, + } + w.coordinator.resultC <- result + + case input := <-w.coordinator.minimizeC: + // Received input to minimize from coordinator. + result, err := w.minimize(ctx, input) + if err != nil { + // Error minimizing. Send back the original input. If it didn't cause + // an error before, report it as causing an error now. + // TODO: double-check this is handled correctly when + // implementing -keepfuzzing. + result = fuzzResult{ + entry: input.entry, + crasherMsg: input.crasherMsg, + minimizeAttempted: true, + limit: input.limit, + } + if result.crasherMsg == "" { + result.crasherMsg = err.Error() + } + } + w.coordinator.resultC <- result + } + } +} + +// minimize tells a worker process to attempt to find a smaller value that +// either causes an error (if we started minimizing because we found an input +// that causes an error) or preserves new coverage (if we started minimizing +// because we found an input that expands coverage). +func (w *worker) minimize(ctx context.Context, input fuzzMinimizeInput) (min fuzzResult, err error) { + if w.coordinator.opts.MinimizeTimeout != 0 { + var cancel func() + ctx, cancel = context.WithTimeout(ctx, w.coordinator.opts.MinimizeTimeout) + defer cancel() + } + + args := minimizeArgs{ + Limit: input.limit, + Timeout: input.timeout, + KeepCoverage: input.keepCoverage, + } + entry, resp, err := w.client.minimize(ctx, input.entry, args) + if err != nil { + // Error communicating with worker. + w.stop() + if ctx.Err() != nil || w.interrupted || isInterruptError(w.waitErr) { + // Worker was interrupted, possibly by the user pressing ^C. + // Normally, workers can handle interrupts and timeouts gracefully and + // will return without error. An error here indicates the worker + // may not have been in a good state, but the error won't be meaningful + // to the user. Just return the original crasher without logging anything. + return fuzzResult{ + entry: input.entry, + crasherMsg: input.crasherMsg, + coverageData: input.keepCoverage, + minimizeAttempted: true, + limit: input.limit, + }, nil + } + return fuzzResult{}, fmt.Errorf("fuzzing process terminated unexpectedly while minimizing: %w", w.waitErr) + } + + if input.crasherMsg != "" && resp.Err == "" && !resp.Success { + return fuzzResult{}, fmt.Errorf("attempted to minimize but could not reproduce") + } + + return fuzzResult{ + entry: entry, + crasherMsg: resp.Err, + coverageData: resp.CoverageData, + minimizeAttempted: true, + limit: input.limit, + count: resp.Count, + totalDuration: resp.Duration, + }, nil +} + +func (w *worker) isRunning() bool { + return w.cmd != nil +} + +// startAndPing starts the worker process and sends it a message to make sure it +// can communicate. +// +// startAndPing returns an error if any part of this didn't work, including if +// the context is expired or the worker process was interrupted before it +// responded. Errors that happen after start but before the ping response +// likely indicate that the worker did not call F.Fuzz or called F.Fail first. +// We don't record crashers for these errors. +func (w *worker) startAndPing(ctx context.Context) error { + if ctx.Err() != nil { + return ctx.Err() + } + if err := w.start(); err != nil { + return err + } + if err := w.client.ping(ctx); err != nil { + w.stop() + if ctx.Err() != nil { + return ctx.Err() + } + if isInterruptError(err) { + // User may have pressed ^C before worker responded. + return err + } + // TODO: record and return stderr. + return fmt.Errorf("fuzzing process terminated without fuzzing: %w", err) + } + return nil +} + +// start runs a new worker process. +// +// If the process couldn't be started, start returns an error. Start won't +// return later termination errors from the process if they occur. +// +// If the process starts successfully, start returns nil. stop must be called +// once later to clean up, even if the process terminates on its own. +// +// When the process terminates, w.waitErr is set to the error (if any), and +// w.termC is closed. +func (w *worker) start() (err error) { + if w.isRunning() { + panic("worker already started") + } + w.waitErr = nil + w.interrupted = false + w.termC = nil + + cmd := exec.Command(w.binPath, w.args...) + cmd.Dir = w.dir + cmd.Env = w.env[:len(w.env):len(w.env)] // copy on append to ensure workers don't overwrite each other. + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + // Create the "fuzz_in" and "fuzz_out" pipes so we can communicate with + // the worker. We don't use stdin and stdout, since the test binary may + // do something else with those. + // + // Each pipe has a reader and a writer. The coordinator writes to fuzzInW + // and reads from fuzzOutR. The worker inherits fuzzInR and fuzzOutW. + // The coordinator closes fuzzInR and fuzzOutW after starting the worker, + // since we have no further need of them. + fuzzInR, fuzzInW, err := os.Pipe() + if err != nil { + return err + } + defer fuzzInR.Close() + fuzzOutR, fuzzOutW, err := os.Pipe() + if err != nil { + fuzzInW.Close() + return err + } + defer fuzzOutW.Close() + setWorkerComm(cmd, workerComm{fuzzIn: fuzzInR, fuzzOut: fuzzOutW, memMu: w.memMu}) + + // Start the worker process. + if err := cmd.Start(); err != nil { + fuzzInW.Close() + fuzzOutR.Close() + return err + } + + // Worker started successfully. + // After this, w.client owns fuzzInW and fuzzOutR, so w.client.Close must be + // called later by stop. + w.cmd = cmd + w.termC = make(chan struct{}) + comm := workerComm{fuzzIn: fuzzInW, fuzzOut: fuzzOutR, memMu: w.memMu} + m := newMutator() + w.client = newWorkerClient(comm, m) + + go func() { + w.waitErr = w.cmd.Wait() + close(w.termC) + }() + + return nil +} + +// stop tells the worker process to exit by closing w.client, then blocks until +// it terminates. If the worker doesn't terminate after a short time, stop +// signals it with os.Interrupt (where supported), then os.Kill. +// +// stop returns the error the process terminated with, if any (same as +// w.waitErr). +// +// stop must be called at least once after start returns successfully, even if +// the worker process terminates unexpectedly. +func (w *worker) stop() error { + if w.termC == nil { + panic("worker was not started successfully") + } + select { + case <-w.termC: + // Worker already terminated. + if w.client == nil { + // stop already called. + return w.waitErr + } + // Possible unexpected termination. + w.client.Close() + w.cmd = nil + w.client = nil + return w.waitErr + default: + // Worker still running. + } + + // Tell the worker to stop by closing fuzz_in. It won't actually stop until it + // finishes with earlier calls. + closeC := make(chan struct{}) + go func() { + w.client.Close() + close(closeC) + }() + + sig := os.Interrupt + if runtime.GOOS == "windows" { + // Per https://golang.org/pkg/os/#Signal, “Interrupt is not implemented on + // Windows; using it with os.Process.Signal will return an error.” + // Fall back to Kill instead. + sig = os.Kill + } + + t := time.NewTimer(workerTimeoutDuration) + for { + select { + case <-w.termC: + // Worker terminated. + t.Stop() + <-closeC + w.cmd = nil + w.client = nil + return w.waitErr + + case <-t.C: + // Timer fired before worker terminated. + w.interrupted = true + switch sig { + case os.Interrupt: + // Try to stop the worker with SIGINT and wait a little longer. + w.cmd.Process.Signal(sig) + sig = os.Kill + t.Reset(workerTimeoutDuration) + + case os.Kill: + // Try to stop the worker with SIGKILL and keep waiting. + w.cmd.Process.Signal(sig) + sig = nil + t.Reset(workerTimeoutDuration) + + case nil: + // Still waiting. Print a message to let the user know why. + fmt.Fprintf(w.coordinator.opts.Log, "waiting for fuzzing process to terminate...\n") + } + } + } +} + +// RunFuzzWorker is called in a worker process to communicate with the +// coordinator process in order to fuzz random inputs. RunFuzzWorker loops +// until the coordinator tells it to stop. +// +// fn is a wrapper on the fuzz function. It may return an error to indicate +// a given input "crashed". The coordinator will also record a crasher if +// the function times out or terminates the process. +// +// RunFuzzWorker returns an error if it could not communicate with the +// coordinator process. +func RunFuzzWorker(ctx context.Context, fn func(CorpusEntry) error) error { + comm, err := getWorkerComm() + if err != nil { + return err + } + srv := &workerServer{ + workerComm: comm, + fuzzFn: fn, + m: newMutator(), + } + return srv.serve(ctx) +} + +// call is serialized and sent from the coordinator on fuzz_in. It acts as +// a minimalist RPC mechanism. Exactly one of its fields must be set to indicate +// which method to call. +type call struct { + Ping *pingArgs + Fuzz *fuzzArgs + Minimize *minimizeArgs +} + +// minimizeArgs contains arguments to workerServer.minimize. The value to +// minimize is already in shared memory. +type minimizeArgs struct { + // Timeout is the time to spend minimizing. This may include time to start up, + // especially if the input causes the worker process to terminated, requiring + // repeated restarts. + Timeout time.Duration + + // Limit is the maximum number of values to test, without spending more time + // than Duration. 0 indicates no limit. + Limit int64 + + // KeepCoverage is a set of coverage counters the worker should attempt to + // keep in minimized values. When provided, the worker will reject inputs that + // don't cause at least one of these bits to be set. + KeepCoverage []byte +} + +// minimizeResponse contains results from workerServer.minimize. +type minimizeResponse struct { + // Success is true if the worker found a smaller input, stored in shared + // memory, that was "interesting" for the same reason as the original input. + // If minimizeArgs.KeepCoverage was set, the minimized input preserved at + // least one coverage bit and did not cause an error. Otherwise, the + // minimized input caused some error, recorded in Err. + Success bool + + // Err is the error string caused by the value in shared memory, if any. + Err string + + // CoverageData is the set of coverage bits activated by the minimized value + // in shared memory. When set, it contains at least one bit from KeepCoverage. + // CoverageData will be nil if Err is set or if minimization failed. + CoverageData []byte + + // Duration is the time spent minimizing, not including starting or cleaning up. + Duration time.Duration + + // Count is the number of values tested. + Count int64 +} + +// fuzzArgs contains arguments to workerServer.fuzz. The value to fuzz is +// passed in shared memory. +type fuzzArgs struct { + // Timeout is the time to spend fuzzing, not including starting or + // cleaning up. + Timeout time.Duration + + // Limit is the maximum number of values to test, without spending more time + // than Duration. 0 indicates no limit. + Limit int64 + + // Warmup indicates whether this is part of a warmup run, meaning that + // fuzzing should not occur. If coverageEnabled is true, then coverage data + // should be reported. + Warmup bool + + // CoverageData is the coverage data. If set, the worker should update its + // local coverage data prior to fuzzing. + CoverageData []byte +} + +// fuzzResponse contains results from workerServer.fuzz. +type fuzzResponse struct { + // Duration is the time spent fuzzing, not including starting or cleaning up. + TotalDuration time.Duration + InterestingDuration time.Duration + + // Count is the number of values tested. + Count int64 + + // CoverageData is set if the value in shared memory expands coverage + // and therefore may be interesting to the coordinator. + CoverageData []byte + + // Err is the error string caused by the value in shared memory, which is + // non-empty if the value in shared memory caused a crash. + Err string +} + +// pingArgs contains arguments to workerServer.ping. +type pingArgs struct{} + +// pingResponse contains results from workerServer.ping. +type pingResponse struct{} + +// workerComm holds pipes and shared memory used for communication +// between the coordinator process (client) and a worker process (server). +// These values are unique to each worker; they are shared only with the +// coordinator, not with other workers. +// +// Access to shared memory is synchronized implicitly over the RPC protocol +// implemented in workerServer and workerClient. During a call, the client +// (worker) has exclusive access to shared memory; at other times, the server +// (coordinator) has exclusive access. +type workerComm struct { + fuzzIn, fuzzOut *os.File + memMu chan *sharedMem // mutex guarding shared memory +} + +// workerServer is a minimalist RPC server, run by fuzz worker processes. +// It allows the coordinator process (using workerClient) to call methods in a +// worker process. This system allows the coordinator to run multiple worker +// processes in parallel and to collect inputs that caused crashes from shared +// memory after a worker process terminates unexpectedly. +type workerServer struct { + workerComm + m *mutator + + // coverageMask is the local coverage data for the worker. It is + // periodically updated to reflect the data in the coordinator when new + // coverage is found. + coverageMask []byte + + // fuzzFn runs the worker's fuzz function on the given input and returns + // an error if it finds a crasher (the process may also exit or crash). + fuzzFn func(CorpusEntry) error +} + +// serve reads serialized RPC messages on fuzzIn. When serve receives a message, +// it calls the corresponding method, then sends the serialized result back +// on fuzzOut. +// +// serve handles RPC calls synchronously; it will not attempt to read a message +// until the previous call has finished. +// +// serve returns errors that occurred when communicating over pipes. serve +// does not return errors from method calls; those are passed through serialized +// responses. +func (ws *workerServer) serve(ctx context.Context) error { + enc := json.NewEncoder(ws.fuzzOut) + dec := json.NewDecoder(&contextReader{ctx: ctx, r: ws.fuzzIn}) + for { + var c call + if err := dec.Decode(&c); err != nil { + if err == io.EOF || err == ctx.Err() { + return nil + } else { + return err + } + } + + var resp interface{} + switch { + case c.Fuzz != nil: + resp = ws.fuzz(ctx, *c.Fuzz) + case c.Minimize != nil: + resp = ws.minimize(ctx, *c.Minimize) + case c.Ping != nil: + resp = ws.ping(ctx, *c.Ping) + default: + return errors.New("no arguments provided for any call") + } + + if err := enc.Encode(resp); err != nil { + return err + } + } +} + +// fuzz runs the test function on random variations of the input value in shared +// memory for a limited duration or number of iterations. +// +// fuzz returns early if it finds an input that crashes the fuzz function (with +// fuzzResponse.Err set) or an input that expands coverage (with +// fuzzResponse.InterestingDuration set). +// +// fuzz does not modify the input in shared memory. Instead, it saves the +// initial PRNG state in shared memory and increments a counter in shared +// memory before each call to the test function. The caller may reconstruct +// the crashing input with this information, since the PRNG is deterministic. +func (ws *workerServer) fuzz(ctx context.Context, args fuzzArgs) (resp fuzzResponse) { + if args.CoverageData != nil { + if ws.coverageMask != nil && len(args.CoverageData) != len(ws.coverageMask) { + panic(fmt.Sprintf("unexpected size for CoverageData: got %d, expected %d", len(args.CoverageData), len(ws.coverageMask))) + } + ws.coverageMask = args.CoverageData + } + start := time.Now() + defer func() { resp.TotalDuration = time.Since(start) }() + + if args.Timeout != 0 { + var cancel func() + ctx, cancel = context.WithTimeout(ctx, args.Timeout) + defer cancel() + } + mem := <-ws.memMu + ws.m.r.save(&mem.header().randState, &mem.header().randInc) + defer func() { + resp.Count = mem.header().count + ws.memMu <- mem + }() + if args.Limit > 0 && mem.header().count >= args.Limit { + panic(fmt.Sprintf("mem.header().count %d already exceeds args.Limit %d", mem.header().count, args.Limit)) + } + + vals, err := unmarshalCorpusFile(mem.valueCopy()) + if err != nil { + panic(err) + } + + shouldStop := func() bool { + return args.Limit > 0 && mem.header().count >= args.Limit + } + fuzzOnce := func(entry CorpusEntry) (dur time.Duration, cov []byte, errMsg string) { + mem.header().count++ + start := time.Now() + err := ws.fuzzFn(entry) + dur = time.Since(start) + if err != nil { + errMsg = err.Error() + if errMsg == "" { + errMsg = "fuzz function failed with no input" + } + return dur, nil, errMsg + } + if ws.coverageMask != nil && countNewCoverageBits(ws.coverageMask, coverageSnapshot) > 0 { + return dur, coverageSnapshot, "" + } + return dur, nil, "" + } + + if args.Warmup { + dur, _, errMsg := fuzzOnce(CorpusEntry{Values: vals}) + if errMsg != "" { + resp.Err = errMsg + return resp + } + resp.InterestingDuration = dur + if coverageEnabled { + resp.CoverageData = coverageSnapshot + } + return resp + } + + for { + select { + case <-ctx.Done(): + return resp + + default: + ws.m.mutate(vals, cap(mem.valueRef())) + entry := CorpusEntry{Values: vals} + dur, cov, errMsg := fuzzOnce(entry) + if errMsg != "" { + resp.Err = errMsg + return resp + } + if cov != nil { + // Found new coverage. Before reporting to the coordinator, + // run the same values once more to deflake. + if !shouldStop() { + dur, cov, errMsg = fuzzOnce(entry) + if errMsg != "" { + resp.Err = errMsg + return resp + } + } + if cov != nil { + resp.CoverageData = cov + resp.InterestingDuration = dur + return resp + } + } + if shouldStop() { + return resp + } + } + } +} + +func (ws *workerServer) minimize(ctx context.Context, args minimizeArgs) (resp minimizeResponse) { + start := time.Now() + defer func() { resp.Duration = time.Now().Sub(start) }() + mem := <-ws.memMu + defer func() { ws.memMu <- mem }() + vals, err := unmarshalCorpusFile(mem.valueCopy()) + if err != nil { + panic(err) + } + if args.Timeout != 0 { + var cancel func() + ctx, cancel = context.WithTimeout(ctx, args.Timeout) + defer cancel() + } + + // Minimize the values in vals, then write to shared memory. We only write + // to shared memory after completing minimization. If the worker terminates + // unexpectedly before then, the coordinator will use the original input. + resp.Success, err = ws.minimizeInput(ctx, vals, &mem.header().count, args.Limit, args.KeepCoverage) + if resp.Success { + writeToMem(vals, mem) + } + if err != nil { + resp.Err = err.Error() + } else if resp.Success { + resp.CoverageData = coverageSnapshot + } + return resp +} + +// minimizeInput applies a series of minimizing transformations on the provided +// vals, ensuring that each minimization still causes an error in fuzzFn. Before +// every call to fuzzFn, it marshals the new vals and writes it to the provided +// mem just in case an unrecoverable error occurs. It uses the context to +// determine how long to run, stopping once closed. It returns a bool +// indicating whether minimization was successful and an error if one was found. +func (ws *workerServer) minimizeInput(ctx context.Context, vals []interface{}, count *int64, limit int64, keepCoverage []byte) (success bool, retErr error) { + wantError := keepCoverage == nil + shouldStop := func() bool { + return ctx.Err() != nil || + (limit > 0 && *count >= limit) || + (retErr != nil && !wantError) + } + if shouldStop() { + return false, nil + } + + // Check that the original value preserves coverage or causes an error. + // If not, then whatever caused us to think the value was interesting may + // have been a flake, and we can't minimize it. + *count++ + if retErr = ws.fuzzFn(CorpusEntry{Values: vals}); retErr == nil && wantError { + return false, nil + } else if retErr != nil && !wantError { + return false, retErr + } else if keepCoverage != nil && !hasCoverageBit(keepCoverage, coverageSnapshot) { + return false, nil + } + + var valI int + // tryMinimized runs the fuzz function with candidate replacing the value + // at index valI. tryMinimized returns whether the input with candidate is + // interesting for the same reason as the original input: it returns + // an error if one was expected, or it preserves coverage. + tryMinimized := func(candidate interface{}) bool { + prev := vals[valI] + // Set vals[valI] to the candidate after it has been + // properly cast. We know that candidate must be of + // the same type as prev, so use that as a reference. + switch c := candidate.(type) { + case float64: + switch prev.(type) { + case float32: + vals[valI] = float32(c) + case float64: + vals[valI] = c + default: + panic("impossible") + } + case uint: + switch prev.(type) { + case uint: + vals[valI] = c + case uint8: + vals[valI] = uint8(c) + case uint16: + vals[valI] = uint16(c) + case uint32: + vals[valI] = uint32(c) + case uint64: + vals[valI] = uint64(c) + case int: + vals[valI] = int(c) + case int8: + vals[valI] = int8(c) + case int16: + vals[valI] = int16(c) + case int32: + vals[valI] = int32(c) + case int64: + vals[valI] = int64(c) + default: + panic("impossible") + } + case []byte: + switch prev.(type) { + case []byte: + vals[valI] = c + case string: + vals[valI] = string(c) + default: + panic("impossible") + } + default: + panic("impossible") + } + *count++ + err := ws.fuzzFn(CorpusEntry{Values: vals}) + if err != nil { + retErr = err + return wantError + } + if keepCoverage != nil && hasCoverageBit(keepCoverage, coverageSnapshot) { + return true + } + vals[valI] = prev + return false + } + + for valI = range vals { + if shouldStop() { + break + } + switch v := vals[valI].(type) { + case bool: + continue // can't minimize + case float32: + minimizeFloat(float64(v), tryMinimized, shouldStop) + case float64: + minimizeFloat(v, tryMinimized, shouldStop) + case uint: + minimizeInteger(v, tryMinimized, shouldStop) + case uint8: + minimizeInteger(uint(v), tryMinimized, shouldStop) + case uint16: + minimizeInteger(uint(v), tryMinimized, shouldStop) + case uint32: + minimizeInteger(uint(v), tryMinimized, shouldStop) + case uint64: + if uint64(uint(v)) != v { + // Skip minimizing a uint64 on 32 bit platforms, since we'll truncate the + // value when casting + continue + } + minimizeInteger(uint(v), tryMinimized, shouldStop) + case int: + minimizeInteger(uint(v), tryMinimized, shouldStop) + case int8: + minimizeInteger(uint(v), tryMinimized, shouldStop) + case int16: + minimizeInteger(uint(v), tryMinimized, shouldStop) + case int32: + minimizeInteger(uint(v), tryMinimized, shouldStop) + case int64: + if int64(int(v)) != v { + // Skip minimizing a int64 on 32 bit platforms, since we'll truncate the + // value when casting + continue + } + minimizeInteger(uint(v), tryMinimized, shouldStop) + case string: + minimizeBytes([]byte(v), tryMinimized, shouldStop) + case []byte: + minimizeBytes(v, tryMinimized, shouldStop) + default: + panic("unreachable") + } + } + return (wantError || retErr == nil), retErr +} + +func writeToMem(vals []interface{}, mem *sharedMem) { + b := marshalCorpusFile(vals...) + mem.setValue(b) +} + +// ping does nothing. The coordinator calls this method to ensure the worker +// has called F.Fuzz and can communicate. +func (ws *workerServer) ping(ctx context.Context, args pingArgs) pingResponse { + return pingResponse{} +} + +// workerClient is a minimalist RPC client. The coordinator process uses a +// workerClient to call methods in each worker process (handled by +// workerServer). +type workerClient struct { + workerComm + mu sync.Mutex + m *mutator +} + +func newWorkerClient(comm workerComm, m *mutator) *workerClient { + return &workerClient{workerComm: comm, m: m} +} + +// Close shuts down the connection to the RPC server (the worker process) by +// closing fuzz_in. Close drains fuzz_out (avoiding a SIGPIPE in the worker), +// and closes it after the worker process closes the other end. +func (wc *workerClient) Close() error { + wc.mu.Lock() + defer wc.mu.Unlock() + + // Close fuzzIn. This signals to the server that there are no more calls, + // and it should exit. + if err := wc.fuzzIn.Close(); err != nil { + wc.fuzzOut.Close() + return err + } + + // Drain fuzzOut and close it. When the server exits, the kernel will close + // its end of fuzzOut, and we'll get EOF. + if _, err := io.Copy(ioutil.Discard, wc.fuzzOut); err != nil { + wc.fuzzOut.Close() + return err + } + return wc.fuzzOut.Close() +} + +// errSharedMemClosed is returned by workerClient methods that cannot access +// shared memory because it was closed and unmapped by another goroutine. That +// can happen when worker.cleanup is called in the worker goroutine while a +// workerClient.fuzz call runs concurrently. +// +// This error should not be reported. It indicates the operation was +// interrupted. +var errSharedMemClosed = errors.New("internal error: shared memory was closed and unmapped") + +// minimize tells the worker to call the minimize method. See +// workerServer.minimize. +func (wc *workerClient) minimize(ctx context.Context, entryIn CorpusEntry, args minimizeArgs) (entryOut CorpusEntry, resp minimizeResponse, err error) { + wc.mu.Lock() + defer wc.mu.Unlock() + + mem, ok := <-wc.memMu + if !ok { + return CorpusEntry{}, minimizeResponse{}, errSharedMemClosed + } + mem.header().count = 0 + inp, err := CorpusEntryData(entryIn) + if err != nil { + return CorpusEntry{}, minimizeResponse{}, err + } + mem.setValue(inp) + wc.memMu <- mem + + c := call{Minimize: &args} + callErr := wc.callLocked(ctx, c, &resp) + mem, ok = <-wc.memMu + if !ok { + return CorpusEntry{}, minimizeResponse{}, errSharedMemClosed + } + defer func() { wc.memMu <- mem }() + resp.Count = mem.header().count + if resp.Success { + entryOut.Data = mem.valueCopy() + entryOut.Values, err = unmarshalCorpusFile(entryOut.Data) + h := sha256.Sum256(entryOut.Data) + name := fmt.Sprintf("%x", h[:4]) + entryOut.Name = name + entryOut.Parent = entryIn.Parent + entryOut.Generation = entryIn.Generation + if err != nil { + panic(fmt.Sprintf("workerClient.minimize unmarshaling minimized value: %v", err)) + } + } else { + // Did not minimize, but the original input may still be interesting, + // for example, if there was an error. + entryOut = entryIn + } + + return entryOut, resp, callErr +} + +// fuzz tells the worker to call the fuzz method. See workerServer.fuzz. +func (wc *workerClient) fuzz(ctx context.Context, entryIn CorpusEntry, args fuzzArgs) (entryOut CorpusEntry, resp fuzzResponse, err error) { + wc.mu.Lock() + defer wc.mu.Unlock() + + mem, ok := <-wc.memMu + if !ok { + return CorpusEntry{}, fuzzResponse{}, errSharedMemClosed + } + mem.header().count = 0 + inp, err := CorpusEntryData(entryIn) + if err != nil { + return CorpusEntry{}, fuzzResponse{}, err + } + mem.setValue(inp) + wc.memMu <- mem + + c := call{Fuzz: &args} + callErr := wc.callLocked(ctx, c, &resp) + mem, ok = <-wc.memMu + if !ok { + return CorpusEntry{}, fuzzResponse{}, errSharedMemClosed + } + defer func() { wc.memMu <- mem }() + resp.Count = mem.header().count + + if !bytes.Equal(inp, mem.valueRef()) { + panic("workerServer.fuzz modified input") + } + needEntryOut := callErr != nil || resp.Err != "" || + (!args.Warmup && resp.CoverageData != nil) + if needEntryOut { + valuesOut, err := unmarshalCorpusFile(inp) + if err != nil { + panic(fmt.Sprintf("unmarshaling fuzz input value after call: %v", err)) + } + wc.m.r.restore(mem.header().randState, mem.header().randInc) + if !args.Warmup { + // Only mutate the valuesOut if fuzzing actually occurred. + for i := int64(0); i < mem.header().count; i++ { + wc.m.mutate(valuesOut, cap(mem.valueRef())) + } + } + dataOut := marshalCorpusFile(valuesOut...) + + h := sha256.Sum256(dataOut) + name := fmt.Sprintf("%x", h[:4]) + entryOut = CorpusEntry{ + Name: name, + Parent: entryIn.Name, + Data: dataOut, + Generation: entryIn.Generation + 1, + } + if args.Warmup { + // The bytes weren't mutated, so if entryIn was a seed corpus value, + // then entryOut is too. + entryOut.IsSeed = entryIn.IsSeed + } + } + + return entryOut, resp, callErr +} + +// ping tells the worker to call the ping method. See workerServer.ping. +func (wc *workerClient) ping(ctx context.Context) error { + wc.mu.Lock() + defer wc.mu.Unlock() + c := call{Ping: &pingArgs{}} + var resp pingResponse + return wc.callLocked(ctx, c, &resp) +} + +// callLocked sends an RPC from the coordinator to the worker process and waits +// for the response. The callLocked may be cancelled with ctx. +func (wc *workerClient) callLocked(ctx context.Context, c call, resp interface{}) (err error) { + enc := json.NewEncoder(wc.fuzzIn) + dec := json.NewDecoder(&contextReader{ctx: ctx, r: wc.fuzzOut}) + if err := enc.Encode(c); err != nil { + return err + } + return dec.Decode(resp) +} + +// contextReader wraps a Reader with a Context. If the context is cancelled +// while the underlying reader is blocked, Read returns immediately. +// +// This is useful for reading from a pipe. Closing a pipe file descriptor does +// not unblock pending Reads on that file descriptor. All copies of the pipe's +// other file descriptor (the write end) must be closed in all processes that +// inherit it. This is difficult to do correctly in the situation we care about +// (process group termination). +type contextReader struct { + ctx context.Context + r io.Reader +} + +func (cr *contextReader) Read(b []byte) (n int, err error) { + if err := cr.ctx.Err(); err != nil { + return 0, err + } + done := make(chan struct{}) + + // This goroutine may stay blocked after Read returns because the underlying + // read is blocked. + go func() { + n, err = cr.r.Read(b) + close(done) + }() + + select { + case <-cr.ctx.Done(): + return 0, cr.ctx.Err() + case <-done: + return n, err + } +} diff --git a/src/internal/fuzz/worker_test.go b/src/internal/fuzz/worker_test.go new file mode 100644 index 00000000000..2369b4ce3f4 --- /dev/null +++ b/src/internal/fuzz/worker_test.go @@ -0,0 +1,147 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package fuzz + +import ( + "context" + "flag" + "fmt" + "io" + "os" + "os/signal" + "reflect" + "testing" +) + +var benchmarkWorkerFlag = flag.Bool("benchmarkworker", false, "") + +func TestMain(m *testing.M) { + flag.Parse() + if *benchmarkWorkerFlag { + runBenchmarkWorker() + return + } + os.Exit(m.Run()) +} + +func BenchmarkWorkerFuzzOverhead(b *testing.B) { + origEnv := os.Getenv("GODEBUG") + defer func() { os.Setenv("GODEBUG", origEnv) }() + os.Setenv("GODEBUG", fmt.Sprintf("%s,fuzzseed=123", origEnv)) + + ws := &workerServer{ + fuzzFn: func(_ CorpusEntry) error { return nil }, + workerComm: workerComm{memMu: make(chan *sharedMem, 1)}, + } + + mem, err := sharedMemTempFile(workerSharedMemSize) + if err != nil { + b.Fatalf("failed to create temporary shared memory file: %s", err) + } + defer func() { + if err := mem.Close(); err != nil { + b.Error(err) + } + }() + + initialVal := []interface{}{make([]byte, 32)} + encodedVals := marshalCorpusFile(initialVal...) + mem.setValue(encodedVals) + + ws.memMu <- mem + + b.ResetTimer() + for i := 0; i < b.N; i++ { + ws.m = newMutator() + mem.setValue(encodedVals) + mem.header().count = 0 + + ws.fuzz(context.Background(), fuzzArgs{Limit: 1}) + } +} + +// BenchmarkWorkerPing acts as the coordinator and measures the time it takes +// a worker to respond to N pings. This is a rough measure of our RPC latency. +func BenchmarkWorkerPing(b *testing.B) { + b.SetParallelism(1) + w := newWorkerForTest(b) + for i := 0; i < b.N; i++ { + if err := w.client.ping(context.Background()); err != nil { + b.Fatal(err) + } + } +} + +// BenchmarkWorkerFuzz acts as the coordinator and measures the time it takes +// a worker to mutate a given input and call a trivial fuzz function N times. +func BenchmarkWorkerFuzz(b *testing.B) { + b.SetParallelism(1) + w := newWorkerForTest(b) + entry := CorpusEntry{Values: []interface{}{[]byte(nil)}} + entry.Data = marshalCorpusFile(entry.Values...) + for i := int64(0); i < int64(b.N); { + args := fuzzArgs{ + Limit: int64(b.N) - i, + Timeout: workerFuzzDuration, + } + _, resp, err := w.client.fuzz(context.Background(), entry, args) + if err != nil { + b.Fatal(err) + } + if resp.Err != "" { + b.Fatal(resp.Err) + } + if resp.Count == 0 { + b.Fatal("worker did not make progress") + } + i += resp.Count + } +} + +// newWorkerForTest creates and starts a worker process for testing or +// benchmarking. The worker process calls RunFuzzWorker, which responds to +// RPC messages until it's stopped. The process is stopped and cleaned up +// automatically when the test is done. +func newWorkerForTest(tb testing.TB) *worker { + tb.Helper() + c, err := newCoordinator(CoordinateFuzzingOpts{ + Types: []reflect.Type{reflect.TypeOf([]byte(nil))}, + Log: io.Discard, + }) + if err != nil { + tb.Fatal(err) + } + dir := "" // same as self + binPath := os.Args[0] // same as self + args := append(os.Args[1:], "-benchmarkworker") + env := os.Environ() // same as self + w, err := newWorker(c, dir, binPath, args, env) + if err != nil { + tb.Fatal(err) + } + tb.Cleanup(func() { + if err := w.cleanup(); err != nil { + tb.Error(err) + } + }) + if err := w.startAndPing(context.Background()); err != nil { + tb.Fatal(err) + } + tb.Cleanup(func() { + if err := w.stop(); err != nil { + tb.Error(err) + } + }) + return w +} + +func runBenchmarkWorker() { + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt) + defer cancel() + fn := func(CorpusEntry) error { return nil } + if err := RunFuzzWorker(ctx, fn); err != nil && err != ctx.Err() { + panic(err) + } +} diff --git a/src/testing/benchmark.go b/src/testing/benchmark.go index 15b4426c5a5..30fa106dd47 100644 --- a/src/testing/benchmark.go +++ b/src/testing/benchmark.go @@ -32,35 +32,36 @@ var ( matchBenchmarks *string benchmarkMemory *bool - benchTime = benchTimeFlag{d: 1 * time.Second} // changed during test of testing package + benchTime = durationOrCountFlag{d: 1 * time.Second} // changed during test of testing package ) -type benchTimeFlag struct { - d time.Duration - n int +type durationOrCountFlag struct { + d time.Duration + n int + allowZero bool } -func (f *benchTimeFlag) String() string { +func (f *durationOrCountFlag) String() string { if f.n > 0 { return fmt.Sprintf("%dx", f.n) } return time.Duration(f.d).String() } -func (f *benchTimeFlag) Set(s string) error { +func (f *durationOrCountFlag) Set(s string) error { if strings.HasSuffix(s, "x") { n, err := strconv.ParseInt(s[:len(s)-1], 10, 0) - if err != nil || n <= 0 { + if err != nil || n < 0 || (!f.allowZero && n == 0) { return fmt.Errorf("invalid count") } - *f = benchTimeFlag{n: int(n)} + *f = durationOrCountFlag{n: int(n)} return nil } d, err := time.ParseDuration(s) - if err != nil || d <= 0 { + if err != nil || d < 0 || (!f.allowZero && d == 0) { return fmt.Errorf("invalid duration") } - *f = benchTimeFlag{d: d} + *f = durationOrCountFlag{d: d} return nil } @@ -98,7 +99,7 @@ type B struct { previousN int // number of iterations in the previous run previousDuration time.Duration // total duration of the previous run benchFunc func(b *B) - benchTime benchTimeFlag + benchTime durationOrCountFlag bytes int64 missingBytes bool // one of the subbenchmarks does not have bytes set. timerOn bool diff --git a/src/testing/fuzz.go b/src/testing/fuzz.go new file mode 100644 index 00000000000..57ea4180391 --- /dev/null +++ b/src/testing/fuzz.go @@ -0,0 +1,783 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package testing + +import ( + "errors" + "flag" + "fmt" + "io" + "os" + "path/filepath" + "reflect" + "runtime" + "sync/atomic" + "time" +) + +func initFuzzFlags() { + matchFuzz = flag.String("test.fuzz", "", "run the fuzz target matching `regexp`") + flag.Var(&fuzzDuration, "test.fuzztime", "time to spend fuzzing; default is to run indefinitely") + flag.Var(&minimizeDuration, "test.fuzzminimizetime", "time to spend minimizing a value after finding a crash") + fuzzCacheDir = flag.String("test.fuzzcachedir", "", "directory where interesting fuzzing inputs are stored") + isFuzzWorker = flag.Bool("test.fuzzworker", false, "coordinate with the parent process to fuzz random values") +} + +var ( + matchFuzz *string + fuzzDuration durationOrCountFlag + minimizeDuration = durationOrCountFlag{d: 60 * time.Second, allowZero: true} + fuzzCacheDir *string + isFuzzWorker *bool + + // corpusDir is the parent directory of the target's seed corpus within + // the package. + corpusDir = "testdata/fuzz" +) + +// fuzzWorkerExitCode is used as an exit code by fuzz worker processes after an internal error. +// This distinguishes internal errors from uncontrolled panics and other crashes. +// Keep in sync with internal/fuzz.workerExitCode. +const fuzzWorkerExitCode = 70 + +// InternalFuzzTarget is an internal type but exported because it is cross-package; +// it is part of the implementation of the "go test" command. +type InternalFuzzTarget struct { + Name string + Fn func(f *F) +} + +// F is a type passed to fuzz targets. +// +// A fuzz target may add seed corpus entries using F.Add or by storing files in +// the testdata/fuzz/<FuzzTargetName> directory. The fuzz target must then +// call F.Fuzz once to provide a fuzz function. See the testing package +// documentation for an example, and see the F.Fuzz and F.Add method +// documentation for details. +type F struct { + common + fuzzContext *fuzzContext + testContext *testContext + + // inFuzzFn is true when the fuzz function is running. Most F methods cannot + // be called when inFuzzFn is true. + inFuzzFn bool + + // corpus is a set of seed corpus entries, added with F.Add and loaded + // from testdata. + corpus []corpusEntry + + result FuzzResult + fuzzCalled bool +} + +var _ TB = (*F)(nil) + +// corpusEntry is an alias to the same type as internal/fuzz.CorpusEntry. +// We use a type alias because we don't want to export this type, and we can't +// import internal/fuzz from testing. +type corpusEntry = struct { + Parent string + Name string + Data []byte + Values []interface{} + Generation int + IsSeed bool +} + +// Cleanup registers a function to be called after the fuzz function has been +// called on all seed corpus entries, and after fuzzing completes (if enabled). +// Cleanup functions will be called in last added, first called order. +func (f *F) Cleanup(fn func()) { + if f.inFuzzFn { + panic("testing: f.Cleanup was called inside the f.Fuzz function, use t.Cleanup instead") + } + f.common.Helper() + f.common.Cleanup(fn) +} + +// Error is equivalent to Log followed by Fail. +func (f *F) Error(args ...interface{}) { + if f.inFuzzFn { + panic("testing: f.Error was called inside the f.Fuzz function, use t.Error instead") + } + f.common.Helper() + f.common.Error(args...) +} + +// Errorf is equivalent to Logf followed by Fail. +func (f *F) Errorf(format string, args ...interface{}) { + if f.inFuzzFn { + panic("testing: f.Errorf was called inside the f.Fuzz function, use t.Errorf instead") + } + f.common.Helper() + f.common.Errorf(format, args...) +} + +// Fail marks the function as having failed but continues execution. +func (f *F) Fail() { + if f.inFuzzFn { + panic("testing: f.Fail was called inside the f.Fuzz function, use t.Fail instead") + } + f.common.Helper() + f.common.Fail() +} + +// FailNow marks the function as having failed and stops its execution +// by calling runtime.Goexit (which then runs all deferred calls in the +// current goroutine). +// Execution will continue at the next test, benchmark, or fuzz function. +// FailNow must be called from the goroutine running the +// fuzz target, not from other goroutines +// created during the test. Calling FailNow does not stop +// those other goroutines. +func (f *F) FailNow() { + if f.inFuzzFn { + panic("testing: f.FailNow was called inside the f.Fuzz function, use t.FailNow instead") + } + f.common.Helper() + f.common.FailNow() +} + +// Fatal is equivalent to Log followed by FailNow. +func (f *F) Fatal(args ...interface{}) { + if f.inFuzzFn { + panic("testing: f.Fatal was called inside the f.Fuzz function, use t.Fatal instead") + } + f.common.Helper() + f.common.Fatal(args...) +} + +// Fatalf is equivalent to Logf followed by FailNow. +func (f *F) Fatalf(format string, args ...interface{}) { + if f.inFuzzFn { + panic("testing: f.Fatalf was called inside the f.Fuzz function, use t.Fatalf instead") + } + f.common.Helper() + f.common.Fatalf(format, args...) +} + +// Helper marks the calling function as a test helper function. +// When printing file and line information, that function will be skipped. +// Helper may be called simultaneously from multiple goroutines. +func (f *F) Helper() { + if f.inFuzzFn { + panic("testing: f.Helper was called inside the f.Fuzz function, use t.Helper instead") + } + + // common.Helper is inlined here. + // If we called it, it would mark F.Helper as the helper + // instead of the caller. + f.mu.Lock() + defer f.mu.Unlock() + if f.helperPCs == nil { + f.helperPCs = make(map[uintptr]struct{}) + } + // repeating code from callerName here to save walking a stack frame + var pc [1]uintptr + n := runtime.Callers(2, pc[:]) // skip runtime.Callers + Helper + if n == 0 { + panic("testing: zero callers found") + } + if _, found := f.helperPCs[pc[0]]; !found { + f.helperPCs[pc[0]] = struct{}{} + f.helperNames = nil // map will be recreated next time it is needed + } +} + +// Setenv calls os.Setenv(key, value) and uses Cleanup to restore the +// environment variable to its original value after the test. +// +// When fuzzing is enabled, the fuzzing engine spawns worker processes running +// the test binary. Each worker process inherits the environment of the parent +// process, including environment variables set with F.Setenv. +func (f *F) Setenv(key, value string) { + if f.inFuzzFn { + panic("testing: f.Setenv was called inside the f.Fuzz function, use t.Setenv instead") + } + f.common.Helper() + f.common.Setenv(key, value) +} + +// Skip is equivalent to Log followed by SkipNow. +func (f *F) Skip(args ...interface{}) { + if f.inFuzzFn { + panic("testing: f.Skip was called inside the f.Fuzz function, use t.Skip instead") + } + f.common.Helper() + f.common.Skip(args...) +} + +// SkipNow marks the test as having been skipped and stops its execution +// by calling runtime.Goexit. +// If a test fails (see Error, Errorf, Fail) and is then skipped, +// it is still considered to have failed. +// Execution will continue at the next test or benchmark. See also FailNow. +// SkipNow must be called from the goroutine running the test, not from +// other goroutines created during the test. Calling SkipNow does not stop +// those other goroutines. +func (f *F) SkipNow() { + if f.inFuzzFn { + panic("testing: f.SkipNow was called inside the f.Fuzz function, use t.SkipNow instead") + } + f.common.Helper() + f.common.SkipNow() +} + +// Skipf is equivalent to Logf followed by SkipNow. +func (f *F) Skipf(format string, args ...interface{}) { + if f.inFuzzFn { + panic("testing: f.Skipf was called inside the f.Fuzz function, use t.Skipf instead") + } + f.common.Helper() + f.common.Skipf(format, args...) +} + +// TempDir returns a temporary directory for the test to use. +// The directory is automatically removed by Cleanup when the test and +// all its subtests complete. +// Each subsequent call to t.TempDir returns a unique directory; +// if the directory creation fails, TempDir terminates the test by calling Fatal. +func (f *F) TempDir() string { + if f.inFuzzFn { + panic("testing: f.TempDir was called inside the f.Fuzz function, use t.TempDir instead") + } + f.common.Helper() + return f.common.TempDir() +} + +// Add will add the arguments to the seed corpus for the fuzz target. This will +// be a no-op if called after or within the Fuzz function. The args must match +// those in the Fuzz function. +func (f *F) Add(args ...interface{}) { + var values []interface{} + for i := range args { + if t := reflect.TypeOf(args[i]); !supportedTypes[t] { + panic(fmt.Sprintf("testing: unsupported type to Add %v", t)) + } + values = append(values, args[i]) + } + f.corpus = append(f.corpus, corpusEntry{Values: values, IsSeed: true, Name: fmt.Sprintf("seed#%d", len(f.corpus))}) +} + +// supportedTypes represents all of the supported types which can be fuzzed. +var supportedTypes = map[reflect.Type]bool{ + reflect.TypeOf(([]byte)("")): true, + reflect.TypeOf((string)("")): true, + reflect.TypeOf((bool)(false)): true, + reflect.TypeOf((byte)(0)): true, + reflect.TypeOf((rune)(0)): true, + reflect.TypeOf((float32)(0)): true, + reflect.TypeOf((float64)(0)): true, + reflect.TypeOf((int)(0)): true, + reflect.TypeOf((int8)(0)): true, + reflect.TypeOf((int16)(0)): true, + reflect.TypeOf((int32)(0)): true, + reflect.TypeOf((int64)(0)): true, + reflect.TypeOf((uint)(0)): true, + reflect.TypeOf((uint8)(0)): true, + reflect.TypeOf((uint16)(0)): true, + reflect.TypeOf((uint32)(0)): true, + reflect.TypeOf((uint64)(0)): true, +} + +// Fuzz runs the fuzz function, ff, for fuzz testing. If ff fails for a set of +// arguments, those arguments will be added to the seed corpus. +// +// ff must be a function with no return value whose first argument is *T and +// whose remaining arguments are the types to be fuzzed. +// For example: +// +// f.Fuzz(func(t *testing.T, b []byte, i int) { ... }) +// +// This function should be fast, deterministic, and stateless. +// None of the pointers to any input data should be retained between executions. +// +// This is a terminal function which will terminate the currently running fuzz +// target by calling runtime.Goexit. +// To run any code after fuzzing stops, use (*F).Cleanup. +func (f *F) Fuzz(ff interface{}) { + if f.fuzzCalled { + panic("testing: F.Fuzz called more than once") + } + f.fuzzCalled = true + if f.failed { + return + } + f.Helper() + + // ff should be in the form func(*testing.T, ...interface{}) + fn := reflect.ValueOf(ff) + fnType := fn.Type() + if fnType.Kind() != reflect.Func { + panic("testing: F.Fuzz must receive a function") + } + if fnType.NumIn() < 2 || fnType.In(0) != reflect.TypeOf((*T)(nil)) { + panic("testing: F.Fuzz function must receive at least two arguments, where the first argument is a *T") + } + + // Save the types of the function to compare against the corpus. + var types []reflect.Type + for i := 1; i < fnType.NumIn(); i++ { + t := fnType.In(i) + if !supportedTypes[t] { + panic(fmt.Sprintf("testing: unsupported type for fuzzing %v", t)) + } + types = append(types, t) + } + + // Load the testdata seed corpus. Check types of entries in the testdata + // corpus and entries declared with F.Add. + // + // Don't load the seed corpus if this is a worker process; we won't use it. + if f.fuzzContext.mode != fuzzWorker { + for _, c := range f.corpus { + if err := f.fuzzContext.deps.CheckCorpus(c.Values, types); err != nil { + // TODO(#48302): Report the source location of the F.Add call. + f.Fatal(err) + } + } + + // Load seed corpus + c, err := f.fuzzContext.deps.ReadCorpus(filepath.Join(corpusDir, f.name), types) + if err != nil { + f.Fatal(err) + } + for i := range c { + c[i].IsSeed = true // these are all seed corpus values + if f.fuzzContext.mode == fuzzCoordinator { + // If this is the coordinator process, zero the values, since we don't need + // to hold onto them. + c[i].Values = nil + } + } + + f.corpus = append(f.corpus, c...) + } + + // run calls fn on a given input, as a subtest with its own T. + // run is analogous to T.Run. The test filtering and cleanup works similarly. + // fn is called in its own goroutine. + run := func(e corpusEntry) error { + if e.Values == nil { + // Every code path should have already unmarshaled Data into Values. + // It's our fault if it didn't. + panic(fmt.Sprintf("corpus file %q was not unmarshaled", e.Name)) + } + if shouldFailFast() { + return nil + } + testName := f.common.name + if e.Name != "" { + testName = fmt.Sprintf("%s/%s", testName, e.Name) + } + + // Record the stack trace at the point of this call so that if the subtest + // function - which runs in a separate stack - is marked as a helper, we can + // continue walking the stack into the parent test. + var pc [maxStackLen]uintptr + n := runtime.Callers(2, pc[:]) + t := &T{ + common: common{ + barrier: make(chan bool), + signal: make(chan bool), + name: testName, + parent: &f.common, + level: f.level + 1, + creator: pc[:n], + chatty: f.chatty, + fuzzing: true, + }, + context: f.testContext, + } + t.w = indenter{&t.common} + if t.chatty != nil { + // TODO(#48132): adjust this to work with test2json. + t.chatty.Updatef(t.name, "=== RUN %s\n", t.name) + } + f.inFuzzFn = true + go tRunner(t, func(t *T) { + args := []reflect.Value{reflect.ValueOf(t)} + for _, v := range e.Values { + args = append(args, reflect.ValueOf(v)) + } + // Before reseting the current coverage, defer the snapshot so that we + // make sure it is called right before the tRunner function exits, + // regardless of whether it was executed cleanly, panicked, or if the + // fuzzFn called t.Fatal. + defer f.fuzzContext.deps.SnapshotCoverage() + f.fuzzContext.deps.ResetCoverage() + fn.Call(args) + }) + <-t.signal + f.inFuzzFn = false + if t.Failed() { + return errors.New(string(f.output)) + } + return nil + } + + switch f.fuzzContext.mode { + case fuzzCoordinator: + // Fuzzing is enabled, and this is the test process started by 'go test'. + // Act as the coordinator process, and coordinate workers to perform the + // actual fuzzing. + corpusTargetDir := filepath.Join(corpusDir, f.name) + cacheTargetDir := filepath.Join(*fuzzCacheDir, f.name) + err := f.fuzzContext.deps.CoordinateFuzzing( + fuzzDuration.d, + int64(fuzzDuration.n), + minimizeDuration.d, + int64(minimizeDuration.n), + *parallel, + f.corpus, + types, + corpusTargetDir, + cacheTargetDir) + if err != nil { + f.result = FuzzResult{Error: err} + f.Fail() + fmt.Fprintf(f.w, "%v\n", err) + if crashErr, ok := err.(fuzzCrashError); ok { + crashName := crashErr.CrashName() + fmt.Fprintf(f.w, "Crash written to %s\n", filepath.Join(corpusDir, f.name, crashName)) + fmt.Fprintf(f.w, "To re-run:\ngo test %s -run=%s/%s\n", f.fuzzContext.deps.ImportPath(), f.name, crashName) + } + } + // TODO(jayconrod,katiehockman): Aggregate statistics across workers + // and add to FuzzResult (ie. time taken, num iterations) + + case fuzzWorker: + // Fuzzing is enabled, and this is a worker process. Follow instructions + // from the coordinator. + if err := f.fuzzContext.deps.RunFuzzWorker(run); err != nil { + // Internal errors are marked with f.Fail; user code may call this too, before F.Fuzz. + // The worker will exit with fuzzWorkerExitCode, indicating this is a failure + // (and 'go test' should exit non-zero) but a crasher should not be recorded. + f.Errorf("communicating with fuzzing coordinator: %v", err) + } + + default: + // Fuzzing is not enabled, or will be done later. Only run the seed + // corpus now. + for _, e := range f.corpus { + run(e) + } + } + + // Record that the fuzz function (or coordinateFuzzing or runFuzzWorker) + // returned normally. This is used to distinguish runtime.Goexit below + // from panic(nil). + f.finished = true + + // Terminate the goroutine. F.Fuzz should not return. + // We cannot call runtime.Goexit from a deferred function: if there is a + // panic, that would replace the panic value with nil. + runtime.Goexit() +} + +func (f *F) report() { + if *isFuzzWorker || f.parent == nil { + return + } + dstr := fmtDuration(f.duration) + format := "--- %s: %s (%s)\n" + if f.Failed() { + f.flushToParent(f.name, format, "FAIL", f.name, dstr) + } else if f.chatty != nil { + if f.Skipped() { + f.flushToParent(f.name, format, "SKIP", f.name, dstr) + } else { + f.flushToParent(f.name, format, "PASS", f.name, dstr) + } + } +} + +// FuzzResult contains the results of a fuzz run. +type FuzzResult struct { + N int // The number of iterations. + T time.Duration // The total time taken. + Error error // Error is the error from the crash +} + +func (r FuzzResult) String() string { + s := "" + if r.Error == nil { + return s + } + s = fmt.Sprintf("%s", r.Error.Error()) + return s +} + +// fuzzCrashError is satisfied by a crash detected within the fuzz function. +// These errors are written to the seed corpus and can be re-run with 'go test'. +// Errors within the fuzzing framework (like I/O errors between coordinator +// and worker processes) don't satisfy this interface. +type fuzzCrashError interface { + error + Unwrap() error + + // CrashName returns the name of the subtest that corresponds to the saved + // crash input file in the seed corpus. The test can be re-run with + // go test $pkg -run=$target/$name where $pkg is the package's import path, + // $target is the fuzz target name, and $name is the string returned here. + CrashName() string +} + +// fuzzContext holds fields common to all fuzz targets. +type fuzzContext struct { + deps testDeps + mode fuzzMode +} + +type fuzzMode uint8 + +const ( + seedCorpusOnly fuzzMode = iota + fuzzCoordinator + fuzzWorker +) + +// runFuzzTargets runs the fuzz targets matching the pattern for -run. This will +// only run the f.Fuzz function for each seed corpus without using the fuzzing +// engine to generate or mutate inputs. +func runFuzzTargets(deps testDeps, fuzzTargets []InternalFuzzTarget, deadline time.Time) (ran, ok bool) { + ok = true + if len(fuzzTargets) == 0 || *isFuzzWorker { + return ran, ok + } + m := newMatcher(deps.MatchString, *match, "-test.run") + tctx := newTestContext(*parallel, m) + tctx.deadline = deadline + var mFuzz *matcher + if *matchFuzz != "" { + mFuzz = newMatcher(deps.MatchString, *matchFuzz, "-test.fuzz") + } + fctx := &fuzzContext{deps: deps, mode: seedCorpusOnly} + root := common{w: os.Stdout} // gather output in one place + if Verbose() { + root.chatty = newChattyPrinter(root.w) + } + for _, ft := range fuzzTargets { + if shouldFailFast() { + break + } + testName, matched, _ := tctx.match.fullName(nil, ft.Name) + if !matched { + continue + } + if mFuzz != nil { + if _, fuzzMatched, _ := mFuzz.fullName(nil, ft.Name); fuzzMatched { + // If this target will be fuzzed, then don't run the seed corpus + // right now. That will happen later. + continue + } + } + f := &F{ + common: common{ + signal: make(chan bool), + barrier: make(chan bool), + name: testName, + parent: &root, + level: root.level + 1, + chatty: root.chatty, + }, + testContext: tctx, + fuzzContext: fctx, + } + f.w = indenter{&f.common} + if f.chatty != nil { + // TODO(#48132): adjust this to work with test2json. + f.chatty.Updatef(f.name, "=== RUN %s\n", f.name) + } + + go fRunner(f, ft.Fn) + <-f.signal + } + return root.ran, !root.Failed() +} + +// runFuzzing runs the fuzz target matching the pattern for -fuzz. Only one such +// fuzz target must match. This will run the fuzzing engine to generate and +// mutate new inputs against the f.Fuzz function. +// +// If fuzzing is disabled (-test.fuzz is not set), runFuzzing +// returns immediately. +func runFuzzing(deps testDeps, fuzzTargets []InternalFuzzTarget) (ok bool) { + // TODO(katiehockman,jayconrod): Should we do something special to make sure + // we don't print f.Log statements again with runFuzzing, since we already + // would have printed them when we ran runFuzzTargets (ie. seed corpus run)? + if len(fuzzTargets) == 0 || *matchFuzz == "" { + return true + } + m := newMatcher(deps.MatchString, *matchFuzz, "-test.fuzz") + tctx := newTestContext(1, m) + fctx := &fuzzContext{ + deps: deps, + } + root := common{w: os.Stdout} + if *isFuzzWorker { + root.w = io.Discard + fctx.mode = fuzzWorker + } else { + fctx.mode = fuzzCoordinator + } + if Verbose() && !*isFuzzWorker { + root.chatty = newChattyPrinter(root.w) + } + var target *InternalFuzzTarget + var targetName string + var matched []string + for i := range fuzzTargets { + name, ok, _ := tctx.match.fullName(nil, fuzzTargets[i].Name) + if !ok { + continue + } + matched = append(matched, name) + target = &fuzzTargets[i] + targetName = name + } + if len(matched) == 0 { + fmt.Fprintln(os.Stderr, "testing: warning: no targets to fuzz") + return true + } + if len(matched) > 1 { + fmt.Fprintf(os.Stderr, "testing: will not fuzz, -fuzz matches more than one target: %v\n", matched) + return false + } + + f := &F{ + common: common{ + signal: make(chan bool), + barrier: nil, // T.Parallel has no effect when fuzzing. + name: targetName, + parent: &root, + level: root.level + 1, + chatty: root.chatty, + }, + fuzzContext: fctx, + testContext: tctx, + } + f.w = indenter{&f.common} + if f.chatty != nil { + // TODO(#48132): adjust this to work with test2json. + f.chatty.Updatef(f.name, "=== FUZZ %s\n", f.name) + } + go fRunner(f, target.Fn) + <-f.signal + return !f.failed +} + +// fRunner wraps a call to a fuzz target and ensures that cleanup functions are +// called and status flags are set. fRunner should be called in its own +// goroutine. To wait for its completion, receive from f.signal. +// +// fRunner is analogous to tRunner, which wraps subtests started with T.Run. +// Tests and fuzz targets work a little differently, so for now, these functions +// aren't consolidated. In particular, because there are no F.Run and F.Parallel +// methods, i.e., no fuzz sub-targets or parallel fuzz targets, a few +// simplifications are made. We also require that F.Fuzz, F.Skip, or F.Fail is +// called. +func fRunner(f *F, fn func(*F)) { + // When this goroutine is done, either because runtime.Goexit was called, + // a panic started, or fn returned normally, record the duration and send + // t.signal, indicating the fuzz target is done. + defer func() { + // Detect whether the fuzz target panicked or called runtime.Goexit without + // calling F.Fuzz, F.Fail, or F.Skip. If it did, panic (possibly replacing a + // nil panic value). Nothing should recover after fRunner unwinds, so this + // should crash the process and print stack. Unfortunately, recovering here + // adds stack frames, but the location of the original panic should still be + // clear. + if f.Failed() { + atomic.AddUint32(&numFailed, 1) + } + err := recover() + f.mu.RLock() + ok := f.skipped || f.failed || (f.fuzzCalled && f.finished) + f.mu.RUnlock() + if err == nil && !ok { + err = errNilPanicOrGoexit + } + + // Use a deferred call to ensure that we report that the test is + // complete even if a cleanup function calls t.FailNow. See issue 41355. + didPanic := false + defer func() { + if didPanic { + return + } + if err != nil { + panic(err) + } + // Only report that the test is complete if it doesn't panic, + // as otherwise the test binary can exit before the panic is + // reported to the user. See issue 41479. + f.signal <- true + }() + + // If we recovered a panic or inappropriate runtime.Goexit, fail the test, + // flush the output log up to the root, then panic. + doPanic := func(err interface{}) { + f.Fail() + if r := f.runCleanup(recoverAndReturnPanic); r != nil { + f.Logf("cleanup panicked with %v", r) + } + for root := &f.common; root.parent != nil; root = root.parent { + root.mu.Lock() + root.duration += time.Since(root.start) + d := root.duration + root.mu.Unlock() + root.flushToParent(root.name, "--- FAIL: %s (%s)\n", root.name, fmtDuration(d)) + } + didPanic = true + panic(err) + } + if err != nil { + doPanic(err) + } + + // No panic or inappropriate Goexit. + f.duration += time.Since(f.start) + + if len(f.sub) > 0 { + // Unblock inputs that called T.Parallel while running the seed corpus. + // T.Parallel has no effect while fuzzing, so this only affects fuzz + // targets run as normal tests. + close(f.barrier) + // Wait for the subtests to complete. + for _, sub := range f.sub { + <-sub.signal + } + cleanupStart := time.Now() + err := f.runCleanup(recoverAndReturnPanic) + f.duration += time.Since(cleanupStart) + if err != nil { + doPanic(err) + } + } + + // Report after all subtests have finished. + f.report() + f.done = true + f.setRan() + }() + defer func() { + if len(f.sub) == 0 { + f.runCleanup(normalPanic) + } + }() + + f.start = time.Now() + fn(f) + + // Code beyond this point is only executed if fn returned normally. + // That means fn did not call F.Fuzz or F.Skip. It should have called F.Fail. + f.mu.Lock() + defer f.mu.Unlock() + if !f.failed { + panic(f.name + " returned without calling F.Fuzz, F.Fail, or F.Skip") + } +} diff --git a/src/testing/internal/testdeps/deps.go b/src/testing/internal/testdeps/deps.go index 3608d332946..c612355a007 100644 --- a/src/testing/internal/testdeps/deps.go +++ b/src/testing/internal/testdeps/deps.go @@ -12,12 +12,18 @@ package testdeps import ( "bufio" + "context" + "internal/fuzz" "internal/testlog" "io" + "os" + "os/signal" + "reflect" "regexp" "runtime/pprof" "strings" "sync" + "time" ) // TestDeps is an implementation of the testing.testDeps interface, @@ -126,3 +132,68 @@ func (TestDeps) StopTestLog() error { func (TestDeps) SetPanicOnExit0(v bool) { testlog.SetPanicOnExit0(v) } + +func (TestDeps) CoordinateFuzzing( + timeout time.Duration, + limit int64, + minimizeTimeout time.Duration, + minimizeLimit int64, + parallel int, + seed []fuzz.CorpusEntry, + types []reflect.Type, + corpusDir, + cacheDir string) (err error) { + // Fuzzing may be interrupted with a timeout or if the user presses ^C. + // In either case, we'll stop worker processes gracefully and save + // crashers and interesting values. + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt) + defer cancel() + err = fuzz.CoordinateFuzzing(ctx, fuzz.CoordinateFuzzingOpts{ + Log: os.Stderr, + Timeout: timeout, + Limit: limit, + MinimizeTimeout: minimizeTimeout, + MinimizeLimit: minimizeLimit, + Parallel: parallel, + Seed: seed, + Types: types, + CorpusDir: corpusDir, + CacheDir: cacheDir, + }) + if err == ctx.Err() { + return nil + } + return err +} + +func (TestDeps) RunFuzzWorker(fn func(fuzz.CorpusEntry) error) error { + // Worker processes may or may not receive a signal when the user presses ^C + // On POSIX operating systems, a signal sent to a process group is delivered + // to all processes in that group. This is not the case on Windows. + // If the worker is interrupted, return quickly and without error. + // If only the coordinator process is interrupted, it tells each worker + // process to stop by closing its "fuzz_in" pipe. + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt) + defer cancel() + err := fuzz.RunFuzzWorker(ctx, fn) + if err == ctx.Err() { + return nil + } + return err +} + +func (TestDeps) ReadCorpus(dir string, types []reflect.Type) ([]fuzz.CorpusEntry, error) { + return fuzz.ReadCorpus(dir, types) +} + +func (TestDeps) CheckCorpus(vals []interface{}, types []reflect.Type) error { + return fuzz.CheckCorpus(vals, types) +} + +func (TestDeps) ResetCoverage() { + fuzz.ResetCoverage() +} + +func (TestDeps) SnapshotCoverage() { + fuzz.SnapshotCoverage() +} diff --git a/src/testing/sub_test.go b/src/testing/sub_test.go index 6c7d83aac2c..6a5add6f4e8 100644 --- a/src/testing/sub_test.go +++ b/src/testing/sub_test.go @@ -480,9 +480,10 @@ func TestTRun(t *T) { buf := &bytes.Buffer{} root := &T{ common: common{ - signal: make(chan bool), - name: "Test", - w: buf, + signal: make(chan bool), + barrier: make(chan bool), + name: "Test", + w: buf, }, context: ctx, } @@ -669,7 +670,7 @@ func TestBRun(t *T) { w: buf, }, benchFunc: func(b *B) { ok = b.Run("test", tc.f) }, // Use Run to catch failure. - benchTime: benchTimeFlag{d: 1 * time.Microsecond}, + benchTime: durationOrCountFlag{d: 1 * time.Microsecond}, } if tc.chatty { root.chatty = newChattyPrinter(root.w) diff --git a/src/testing/testing.go b/src/testing/testing.go index 2239e01e224..567eb0dfa39 100644 --- a/src/testing/testing.go +++ b/src/testing/testing.go @@ -34,7 +34,7 @@ // its -bench flag is provided. Benchmarks are run sequentially. // // For a description of the testing flags, see -// https://golang.org/cmd/go/#hdr-Testing_flags +// https://golang.org/cmd/go/#hdr-Testing_flags. // // A sample benchmark function looks like this: // func BenchmarkRandInt(b *testing.B) { @@ -132,6 +132,30 @@ // example function, at least one other function, type, variable, or constant // declaration, and no test or benchmark functions. // +// Fuzzing +// +// Functions of the form +// func FuzzXxx(*testing.F) +// are considered fuzz targets, and are executed by the "go test" command. When +// the -fuzz flag is provided, the functions will be fuzzed. +// +// For a description of the testing flags, see +// https://golang.org/cmd/go/#hdr-Testing_flags. +// +// For a description of fuzzing, see golang.org/s/draft-fuzzing-design. +// TODO(#48255): write and link to documentation that will be helpful to users +// who are unfamiliar with fuzzing. +// +// A sample fuzz target looks like this: +// +// func FuzzBytesCmp(f *testing.F) { +// f.Fuzz(func(t *testing.T, a, b []byte) { +// if bytes.HasPrefix(a, b) && !bytes.Contains(a, b) { +// t.Error("HasPrefix is true, but Contains is false") +// } +// }) +// } +// // Skipping // // Tests or benchmarks may be skipped at run time with a call to @@ -144,6 +168,21 @@ // ... // } // +// The Skip method of *T can be used in a fuzz target if the input is invalid, +// but should not be considered a crash. For example: +// +// func FuzzJSONMarshalling(f *testing.F) { +// f.Fuzz(func(t *testing.T, b []byte) { +// var v interface{} +// if err := json.Unmarshal(b, &v); err != nil { +// t.Skip() +// } +// if _, err := json.Marshal(v); err != nil { +// t.Error("Marshal: %v", err) +// } +// }) +// } +// // Subtests and Sub-benchmarks // // The Run methods of T and B allow defining subtests and sub-benchmarks, @@ -163,17 +202,25 @@ // of the top-level test and the sequence of names passed to Run, separated by // slashes, with an optional trailing sequence number for disambiguation. // -// The argument to the -run and -bench command-line flags is an unanchored regular +// The argument to the -run, -bench, and -fuzz command-line flags is an unanchored regular // expression that matches the test's name. For tests with multiple slash-separated // elements, such as subtests, the argument is itself slash-separated, with // expressions matching each name element in turn. Because it is unanchored, an // empty expression matches any string. // For example, using "matching" to mean "whose name contains": // -// go test -run '' # Run all tests. -// go test -run Foo # Run top-level tests matching "Foo", such as "TestFooBar". -// go test -run Foo/A= # For top-level tests matching "Foo", run subtests matching "A=". -// go test -run /A=1 # For all top-level tests, run subtests matching "A=1". +// go test -run '' # Run all tests. +// go test -run Foo # Run top-level tests matching "Foo", such as "TestFooBar". +// go test -run Foo/A= # For top-level tests matching "Foo", run subtests matching "A=". +// go test -run /A=1 # For all top-level tests, run subtests matching "A=1". +// go test -fuzz FuzzFoo # Fuzz the target matching "FuzzFoo" +// +// The -run argument can also be used to run a specific value in the seed +// corpus, for debugging. For example: +// go test -run=FuzzFoo/9ddb952d9814 +// +// The -fuzz and -run flags can both be set, in order to fuzz a target but +// skip the execution of all other tests. // // Subtests can also be used to control parallelism. A parent test will only // complete once all of its subtests complete. In this example, all tests are @@ -246,6 +293,7 @@ import ( "io" "math/rand" "os" + "reflect" "runtime" "runtime/debug" "runtime/trace" @@ -307,6 +355,7 @@ func Init() { shuffle = flag.String("test.shuffle", "off", "randomize the execution order of tests and benchmarks") initBenchmarkFlags() + initFuzzFlags() } var ( @@ -406,6 +455,7 @@ type common struct { chatty *chattyPrinter // A copy of chattyPrinter, if the chatty flag is set. bench bool // Whether the current test is a benchmark. + fuzzing bool // Whether the current test is a fuzzing target. hasSub int32 // Written atomically. raceErrors int // Number of races detected during test. runner string // Function name of tRunner running the test. @@ -538,6 +588,13 @@ func (c *common) frameSkip(skip int) runtime.Frame { // and inserts the final newline if needed and indentation spaces for formatting. // This function must be called with c.mu held. func (c *common) decorate(s string, skip int) string { + if c.helperNames == nil { + c.helperNames = make(map[string]struct{}) + for pc := range c.helperPCs { + c.helperNames[pcToName(pc)] = struct{}{} + } + } + frame := c.frameSkip(skip) file := frame.File line := frame.Line @@ -607,6 +664,17 @@ func (c *common) flushToParent(testName, format string, args ...interface{}) { } } +// isFuzzing returns whether the current context, or any of the parent contexts, +// are a fuzzing target +func (c *common) isFuzzing() bool { + for com := c; com != nil; com = com.parent { + if com.fuzzing { + return true + } + } + return false +} + type indenter struct { c *common } @@ -1083,6 +1151,12 @@ func (t *T) Parallel() { panic("testing: t.Parallel called after t.Setenv; cannot set environment variables in parallel tests") } t.isParallel = true + if t.parent.barrier == nil { + // T.Parallel has no effect when fuzzing. + // Multiple processes may run in parallel, but only one input can run at a + // time per process so we can attribute crashes to specific inputs. + return + } // We don't want to include the time we spend waiting for serial tests // in the test duration. Record the elapsed time thus far and reset the @@ -1155,10 +1229,25 @@ func tRunner(t *T, fn func(t *T)) { t.Errorf("race detected during execution of test") } - // If the test panicked, print any test output before dying. + // Check if the test panicked or Goexited inappropriately. + // + // If this happens in a normal test, print output but continue panicking. + // tRunner is called in its own goroutine, so this terminates the process. + // + // If this happens while fuzzing, recover from the panic and treat it like a + // normal failure. It's important that the process keeps running in order to + // find short inputs that cause panics. err := recover() signal := true + if err != nil && t.isFuzzing() { + t.Errorf("panic: %s\n%s\n", err, string(debug.Stack())) + t.mu.Lock() + t.finished = true + t.mu.Unlock() + err = nil + } + t.mu.RLock() finished := t.finished t.mu.RUnlock() @@ -1176,6 +1265,7 @@ func tRunner(t *T, fn func(t *T)) { } } } + // Use a deferred call to ensure that we report that the test is // complete even if a cleanup function calls t.FailNow. See issue 41355. didPanic := false @@ -1245,7 +1335,7 @@ func tRunner(t *T, fn func(t *T)) { t.report() // Report after all subtests have finished. // Do not lock t.done to allow race detector to detect race in case - // the user does not appropriately synchronizes a goroutine. + // the user does not appropriately synchronize a goroutine. t.done = true if t.parent != nil && atomic.LoadInt32(&t.hasSub) == 0 { t.setRan() @@ -1393,6 +1483,16 @@ func (f matchStringOnly) ImportPath() string { return " func (f matchStringOnly) StartTestLog(io.Writer) {} func (f matchStringOnly) StopTestLog() error { return errMain } func (f matchStringOnly) SetPanicOnExit0(bool) {} +func (f matchStringOnly) CoordinateFuzzing(time.Duration, int64, time.Duration, int64, int, []corpusEntry, []reflect.Type, string, string) error { + return errMain +} +func (f matchStringOnly) RunFuzzWorker(func(corpusEntry) error) error { return errMain } +func (f matchStringOnly) ReadCorpus(string, []reflect.Type) ([]corpusEntry, error) { + return nil, errMain +} +func (f matchStringOnly) CheckCorpus([]interface{}, []reflect.Type) error { return nil } +func (f matchStringOnly) ResetCoverage() {} +func (f matchStringOnly) SnapshotCoverage() {} // Main is an internal function, part of the implementation of the "go test" command. // It was exported because it is cross-package and predates "internal" packages. @@ -1401,15 +1501,16 @@ func (f matchStringOnly) SetPanicOnExit0(bool) {} // new functionality is added to the testing package. // Systems simulating "go test" should be updated to use MainStart. func Main(matchString func(pat, str string) (bool, error), tests []InternalTest, benchmarks []InternalBenchmark, examples []InternalExample) { - os.Exit(MainStart(matchStringOnly(matchString), tests, benchmarks, examples).Run()) + os.Exit(MainStart(matchStringOnly(matchString), tests, benchmarks, nil, examples).Run()) } // M is a type passed to a TestMain function to run the actual tests. type M struct { - deps testDeps - tests []InternalTest - benchmarks []InternalBenchmark - examples []InternalExample + deps testDeps + tests []InternalTest + benchmarks []InternalBenchmark + fuzzTargets []InternalFuzzTarget + examples []InternalExample timer *time.Timer afterOnce sync.Once @@ -1434,18 +1535,25 @@ type testDeps interface { StartTestLog(io.Writer) StopTestLog() error WriteProfileTo(string, io.Writer, int) error + CoordinateFuzzing(time.Duration, int64, time.Duration, int64, int, []corpusEntry, []reflect.Type, string, string) error + RunFuzzWorker(func(corpusEntry) error) error + ReadCorpus(string, []reflect.Type) ([]corpusEntry, error) + CheckCorpus([]interface{}, []reflect.Type) error + ResetCoverage() + SnapshotCoverage() } // MainStart is meant for use by tests generated by 'go test'. // It is not meant to be called directly and is not subject to the Go 1 compatibility document. // It may change signature from release to release. -func MainStart(deps testDeps, tests []InternalTest, benchmarks []InternalBenchmark, examples []InternalExample) *M { +func MainStart(deps testDeps, tests []InternalTest, benchmarks []InternalBenchmark, fuzzTargets []InternalFuzzTarget, examples []InternalExample) *M { Init() return &M{ - deps: deps, - tests: tests, - benchmarks: benchmarks, - examples: examples, + deps: deps, + tests: tests, + benchmarks: benchmarks, + fuzzTargets: fuzzTargets, + examples: examples, } } @@ -1472,9 +1580,15 @@ func (m *M) Run() (code int) { m.exitCode = 2 return } + if *matchFuzz != "" && *fuzzCacheDir == "" { + fmt.Fprintln(os.Stderr, "testing: -test.fuzzcachedir must be set if -test.fuzz is set") + flag.Usage() + m.exitCode = 2 + return + } if len(*matchList) != 0 { - listTests(m.deps.MatchString, m.tests, m.benchmarks, m.examples) + listTests(m.deps.MatchString, m.tests, m.benchmarks, m.fuzzTargets, m.examples) m.exitCode = 0 return } @@ -1502,22 +1616,42 @@ func (m *M) Run() (code int) { m.before() defer m.after() - deadline := m.startAlarm() - haveExamples = len(m.examples) > 0 - testRan, testOk := runTests(m.deps.MatchString, m.tests, deadline) - exampleRan, exampleOk := runExamples(m.deps.MatchString, m.examples) - m.stopAlarm() - if !testRan && !exampleRan && *matchBenchmarks == "" { - fmt.Fprintln(os.Stderr, "testing: warning: no tests to run") + + // Run tests, examples, and benchmarks unless this is a fuzz worker process. + // Workers start after this is done by their parent process, and they should + // not repeat this work. + if !*isFuzzWorker { + deadline := m.startAlarm() + haveExamples = len(m.examples) > 0 + testRan, testOk := runTests(m.deps.MatchString, m.tests, deadline) + fuzzTargetsRan, fuzzTargetsOk := runFuzzTargets(m.deps, m.fuzzTargets, deadline) + exampleRan, exampleOk := runExamples(m.deps.MatchString, m.examples) + m.stopAlarm() + if !testRan && !exampleRan && !fuzzTargetsRan && *matchBenchmarks == "" && *matchFuzz == "" { + fmt.Fprintln(os.Stderr, "testing: warning: no tests to run") + } + if !testOk || !exampleOk || !fuzzTargetsOk || !runBenchmarks(m.deps.ImportPath(), m.deps.MatchString, m.benchmarks) || race.Errors() > 0 { + fmt.Println("FAIL") + m.exitCode = 1 + return + } } - if !testOk || !exampleOk || !runBenchmarks(m.deps.ImportPath(), m.deps.MatchString, m.benchmarks) || race.Errors() > 0 { + + fuzzingOk := runFuzzing(m.deps, m.fuzzTargets) + if !fuzzingOk { fmt.Println("FAIL") - m.exitCode = 1 + if *isFuzzWorker { + m.exitCode = fuzzWorkerExitCode + } else { + m.exitCode = 1 + } return } - fmt.Println("PASS") m.exitCode = 0 + if !*isFuzzWorker { + fmt.Println("PASS") + } return } @@ -1538,7 +1672,7 @@ func (t *T) report() { } } -func listTests(matchString func(pat, str string) (bool, error), tests []InternalTest, benchmarks []InternalBenchmark, examples []InternalExample) { +func listTests(matchString func(pat, str string) (bool, error), tests []InternalTest, benchmarks []InternalBenchmark, fuzzTargets []InternalFuzzTarget, examples []InternalExample) { if _, err := matchString(*matchList, "non-empty"); err != nil { fmt.Fprintf(os.Stderr, "testing: invalid regexp in -test.list (%q): %s\n", *matchList, err) os.Exit(1) @@ -1554,6 +1688,11 @@ func listTests(matchString func(pat, str string) (bool, error), tests []Internal fmt.Println(bench.Name) } } + for _, fuzzTarget := range fuzzTargets { + if ok, _ := matchString(*matchList, fuzzTarget.Name); ok { + fmt.Println(fuzzTarget.Name) + } + } for _, example := range examples { if ok, _ := matchString(*matchList, example.Name); ok { fmt.Println(example.Name) |