diff options
Diffstat (limited to 'src/cmd/go')
34 files changed, 2563 insertions, 44 deletions
diff --git a/src/cmd/go/alldocs.go b/src/cmd/go/alldocs.go index 9753ebba3e..74522691ab 100644 --- a/src/cmd/go/alldocs.go +++ b/src/cmd/go/alldocs.go @@ -53,6 +53,7 @@ // private configuration for downloading non-public code // testflag testing flags // testfunc testing functions +// fuzz fuzzing // vcs controlling version control with GOVCS // // Use "go help <topic>" for more information about that topic. @@ -292,6 +293,8 @@ // download cache, including unpacked source code of versioned // dependencies. // +// The -fuzzcache flag causes clean to remove values used for fuzz testing. +// // For more about build flags, see 'go help build'. // // For more about specifying packages, see 'go help packages'. @@ -1488,8 +1491,8 @@ // // 'Go test' recompiles each package along with any files with names matching // the file pattern "*_test.go". -// These additional files can contain test functions, benchmark functions, and -// example functions. See 'go help testfunc' for more. +// These additional files can contain test functions, benchmark functions, fuzz +// targets and example functions. See 'go help testfunc' for more. // Each listed package causes the execution of a separate test binary. // Files whose names begin with "_" (including "_test.go") or "." are ignored. // @@ -1559,6 +1562,8 @@ // in no time at all,so a successful package test result will be cached and // reused regardless of -timeout setting. // +// Run 'go help fuzz' for details around how the go command handles fuzz targets. +// // In addition to the build flags, the flags handled by 'go test' itself are: // // -args @@ -2728,7 +2733,8 @@ // (for example, -benchtime 100x). // // -count n -// Run each test and benchmark n times (default 1). +// Run each test, benchmark, and fuzz targets' seed corpora n times +// (default 1). // If -cpu is set, run n times for each GOMAXPROCS value. // Examples are always run once. // @@ -2757,36 +2763,55 @@ // Sets -cover. // // -cpu 1,2,4 -// Specify a list of GOMAXPROCS values for which the tests or -// benchmarks should be executed. The default is the current value +// Specify a list of GOMAXPROCS values for which the tests, benchmarks or +// fuzz targets should be executed. The default is the current value // of GOMAXPROCS. // // -failfast // Do not start new tests after the first test failure. // +// -fuzz name +// Run the fuzz target with the given regexp. Must match exactly one fuzz +// target. This is an experimental feature. +// +// -fuzztime t +// Run enough iterations of the fuzz test to take t, specified as a +// time.Duration (for example, -fuzztime 1h30s). The default is to run +// forever. +// The special syntax Nx means to run the fuzz test N times +// (for example, -fuzztime 100x). +// // -json // Log verbose output and test results in JSON. This presents the // same information as the -v flag in a machine-readable format. // +// -keepfuzzing +// Keep running the fuzz target if a crasher is found. +// // -list regexp -// List tests, benchmarks, or examples matching the regular expression. -// No tests, benchmarks or examples will be run. This will only -// list top-level tests. No subtest or subbenchmarks will be shown. +// List tests, benchmarks, fuzz targets, or examples matching the regular +// expression. No tests, benchmarks, fuzz targets, or examples will be run. +// This will only list top-level tests. No subtest or subbenchmarks will be +// shown. // // -parallel n -// Allow parallel execution of test functions that call t.Parallel. +// Allow parallel execution of test functions that call t.Parallel, and +// f.Fuzz functions that call t.Parallel when running the seed corpus. // The value of this flag is the maximum number of tests to run -// simultaneously; by default, it is set to the value of GOMAXPROCS. +// simultaneously. While fuzzing, the value of this flag is the +// maximum number of workers to run the fuzz function simultaneously, +// regardless of whether t.Parallel has been called; by default, it is set +// to the value of GOMAXPROCS. // Note that -parallel only applies within a single test binary. // The 'go test' command may run tests for different packages // in parallel as well, according to the setting of the -p flag // (see 'go help build'). // // -run regexp -// Run only those tests and examples matching the regular expression. -// For tests, the regular expression is split by unbracketed slash (/) -// characters into a sequence of regular expressions, and each part -// of a test's identifier must match the corresponding element in +// Run only those tests, examples, and fuzz targets matching the regular +// expression. For tests, the regular expression is split by unbracketed +// slash (/) characters into a sequence of regular expressions, and each +// part of a test's identifier must match the corresponding element in // the sequence, if any. Note that possible parents of matches are // run too, so that -run=X/Y matches and runs and reports the result // of all tests matching X, even those without sub-tests matching Y, @@ -2953,6 +2978,10 @@ // // func BenchmarkXxx(b *testing.B) { ... } // +// A fuzz target is one named FuzzXxx and should have the signature, +// +// func FuzzXxx(f *testing.F) { ... } +// // An example function is similar to a test function but, instead of using // *testing.T to report success or failure, prints output to os.Stdout. // If the last comment in the function starts with "Output:" then the output @@ -2992,11 +3021,30 @@ // // The entire test file is presented as the example when it contains a single // example function, at least one other function, type, variable, or constant -// declaration, and no test or benchmark functions. +// declaration, and no fuzz targets or test or benchmark functions. // // See the documentation of the testing package for more information. // // +// Fuzzing +// +// By default, go test will build and run the fuzz targets using the target's seed +// corpus only. Any generated corpora in $GOCACHE that were previously written by +// the fuzzing engine will not be run by default. +// +// When -fuzz is set, the binary will be instrumented for coverage. After all +// tests, examples, benchmark functions, and the seed corpora for all fuzz targets +// have been run, go test will begin to fuzz the specified fuzz target. +// Note that this feature is experimental. +// +// -run can be used for testing a single seed corpus entry for a fuzz target. The +// regular expression value of -run can be in the form $target/$name, where $target +// is the name of the fuzz target, and $name is the name of the file (ignoring file +// extensions) to run. For example, -run=FuzzFoo/497b6f87. +// +// See https://golang.org/s/draft-fuzzing-design for more details. +// +// // Controlling version control with GOVCS // // The 'go get' command can run version control commands like git diff --git a/src/cmd/go/internal/cache/cache.go b/src/cmd/go/internal/cache/cache.go index d592d70497..596f22e8fc 100644 --- a/src/cmd/go/internal/cache/cache.go +++ b/src/cmd/go/internal/cache/cache.go @@ -533,3 +533,13 @@ func (c *Cache) copyFile(file io.ReadSeeker, out OutputID, size int64) error { return nil } + +// FuzzDir returns a subdirectory within the cache for storing fuzzing data. +// The subdirectory may not exist. +// +// This directory is managed by the internal/fuzz package. Files in this +// directory aren't removed by the 'go clean -cache' command or by Trim. +// They may be removed with 'go clean -fuzzcache'. +func (c *Cache) FuzzDir() string { + return filepath.Join(c.dir, "fuzz") +} diff --git a/src/cmd/go/internal/cfg/cfg.go b/src/cmd/go/internal/cfg/cfg.go index 5f4465e06b..dd0e8cbbd6 100644 --- a/src/cmd/go/internal/cfg/cfg.go +++ b/src/cmd/go/internal/cfg/cfg.go @@ -60,6 +60,10 @@ var ( func defaultContext() build.Context { ctxt := build.Default + + // TODO(#47037): remove this tag before merging to master. + ctxt.BuildTags = []string{"gofuzzbeta"} + ctxt.JoinPath = filepath.Join // back door to say "do not use go command" ctxt.GOROOT = findGOROOT() diff --git a/src/cmd/go/internal/clean/clean.go b/src/cmd/go/internal/clean/clean.go index 1089211f0c..518473c914 100644 --- a/src/cmd/go/internal/clean/clean.go +++ b/src/cmd/go/internal/clean/clean.go @@ -75,6 +75,8 @@ The -modcache flag causes clean to remove the entire module download cache, including unpacked source code of versioned dependencies. +The -fuzzcache flag causes clean to remove values used for fuzz testing. + For more about build flags, see 'go help build'. For more about specifying packages, see 'go help packages'. @@ -85,6 +87,7 @@ var ( cleanI bool // clean -i flag cleanR bool // clean -r flag cleanCache bool // clean -cache flag + cleanFuzzcache bool // clean -fuzzcache flag cleanModcache bool // clean -modcache flag cleanTestcache bool // clean -testcache flag ) @@ -96,6 +99,7 @@ func init() { CmdClean.Flag.BoolVar(&cleanI, "i", false, "") CmdClean.Flag.BoolVar(&cleanR, "r", false, "") CmdClean.Flag.BoolVar(&cleanCache, "cache", false, "") + CmdClean.Flag.BoolVar(&cleanFuzzcache, "fuzzcache", false, "") CmdClean.Flag.BoolVar(&cleanModcache, "modcache", false, "") CmdClean.Flag.BoolVar(&cleanTestcache, "testcache", false, "") @@ -112,7 +116,7 @@ func runClean(ctx context.Context, cmd *base.Command, args []string) { // or no other target (such as a cache) was requested to be cleaned. cleanPkg := len(args) > 0 || cleanI || cleanR if (!modload.Enabled() || modload.HasModRoot()) && - !cleanCache && !cleanModcache && !cleanTestcache { + !cleanCache && !cleanModcache && !cleanTestcache && !cleanFuzzcache { cleanPkg = true } @@ -206,6 +210,18 @@ func runClean(ctx context.Context, cmd *base.Command, args []string) { } } } + + if cleanFuzzcache { + fuzzDir := cache.Default().FuzzDir() + if cfg.BuildN || cfg.BuildX { + b.Showcmd("", "rm -rf %s", fuzzDir) + } + if !cfg.BuildN { + if err := os.RemoveAll(fuzzDir); err != nil { + base.Errorf("go clean -fuzzcache: %v", err) + } + } + } } var cleaned = map[*load.Package]bool{} diff --git a/src/cmd/go/internal/load/flag.go b/src/cmd/go/internal/load/flag.go index 4e0cb5bc19..24670524fc 100644 --- a/src/cmd/go/internal/load/flag.go +++ b/src/cmd/go/internal/load/flag.go @@ -22,8 +22,9 @@ var ( // that allows specifying different effective flags for different packages. // See 'go help build' for more details about per-package flags. type PerPackageFlag struct { - present bool - values []ppfValue + present bool + values []ppfValue + seenPackages map[*Package]bool // the packages for which the flags have already been set } // A ppfValue is a single <pattern>=<flags> per-package flag value. diff --git a/src/cmd/go/internal/load/pkg.go b/src/cmd/go/internal/load/pkg.go index 4013330bc4..317053d918 100644 --- a/src/cmd/go/internal/load/pkg.go +++ b/src/cmd/go/internal/load/pkg.go @@ -2630,10 +2630,20 @@ func (e *mainPackageError) ImportPath() string { func setToolFlags(pkgs ...*Package) { for _, p := range PackageList(pkgs) { - p.Internal.Asmflags = BuildAsmflags.For(p) - p.Internal.Gcflags = BuildGcflags.For(p) - p.Internal.Ldflags = BuildLdflags.For(p) - p.Internal.Gccgoflags = BuildGccgoflags.For(p) + appendFlags(p, &p.Internal.Asmflags, &BuildAsmflags) + appendFlags(p, &p.Internal.Gcflags, &BuildGcflags) + appendFlags(p, &p.Internal.Ldflags, &BuildLdflags) + appendFlags(p, &p.Internal.Gccgoflags, &BuildGccgoflags) + } +} + +func appendFlags(p *Package, flags *[]string, packageFlag *PerPackageFlag) { + if !packageFlag.seenPackages[p] { + if packageFlag.seenPackages == nil { + packageFlag.seenPackages = make(map[*Package]bool) + } + packageFlag.seenPackages[p] = true + *flags = append(*flags, packageFlag.For(p)...) } } diff --git a/src/cmd/go/internal/load/test.go b/src/cmd/go/internal/load/test.go index 42eefe37ba..da6d1cb21d 100644 --- a/src/cmd/go/internal/load/test.go +++ b/src/cmd/go/internal/load/test.go @@ -555,6 +555,7 @@ func formatTestmain(t *testFuncs) ([]byte, error) { type testFuncs struct { Tests []testFunc Benchmarks []testFunc + FuzzTargets []testFunc Examples []testFunc TestMain *testFunc Package *Package @@ -653,6 +654,13 @@ func (t *testFuncs) load(filename, pkg string, doImport, seen *bool) error { } t.Benchmarks = append(t.Benchmarks, testFunc{pkg, name, "", false}) *doImport, *seen = true, true + case isTest(name, "Fuzz"): + err := checkTestFunc(n, "F") + if err != nil { + return err + } + t.FuzzTargets = append(t.FuzzTargets, testFunc{pkg, name, "", false}) + *doImport, *seen = true, true } } ex := doc.Examples(f) @@ -716,6 +724,12 @@ var benchmarks = []testing.InternalBenchmark{ {{end}} } +var fuzzTargets = []testing.InternalFuzzTarget{ +{{range .FuzzTargets}} + {"{{.Name}}", {{.Package}}.{{.Name}}}, +{{end}} +} + var examples = []testing.InternalExample{ {{range .Examples}} {"{{.Name}}", {{.Package}}.{{.Name}}, {{.Output | printf "%q"}}, {{.Unordered}}}, @@ -774,7 +788,7 @@ func main() { CoveredPackages: {{printf "%q" .Covered}}, }) {{end}} - m := testing.MainStart(testdeps.TestDeps{}, tests, benchmarks, examples) + m := testing.MainStart(testdeps.TestDeps{}, tests, benchmarks, fuzzTargets, examples) {{with .TestMain}} {{.Package}}.{{.Name}}(m) os.Exit(int(reflect.ValueOf(m).Elem().FieldByName("exitCode").Int())) diff --git a/src/cmd/go/internal/test/flagdefs.go b/src/cmd/go/internal/test/flagdefs.go index 37ac81c267..3148074d57 100644 --- a/src/cmd/go/internal/test/flagdefs.go +++ b/src/cmd/go/internal/test/flagdefs.go @@ -19,6 +19,9 @@ var passFlagToTest = map[string]bool{ "cpu": true, "cpuprofile": true, "failfast": true, + "fuzz": true, + "fuzzminimizetime": true, + "fuzztime": true, "list": true, "memprofile": true, "memprofilerate": true, diff --git a/src/cmd/go/internal/test/flagdefs_test.go b/src/cmd/go/internal/test/flagdefs_test.go index ab5440b380..f238fc7d33 100644 --- a/src/cmd/go/internal/test/flagdefs_test.go +++ b/src/cmd/go/internal/test/flagdefs_test.go @@ -17,7 +17,7 @@ func TestPassFlagToTestIncludesAllTestFlags(t *testing.T) { } name := strings.TrimPrefix(f.Name, "test.") switch name { - case "testlogfile", "paniconexit0": + case "testlogfile", "paniconexit0", "fuzzcachedir", "fuzzworker": // These are internal flags. default: if !passFlagToTest[name] { diff --git a/src/cmd/go/internal/test/genflags.go b/src/cmd/go/internal/test/genflags.go index 9277de7fee..645aae68b1 100644 --- a/src/cmd/go/internal/test/genflags.go +++ b/src/cmd/go/internal/test/genflags.go @@ -64,7 +64,7 @@ func testFlags() []string { name := strings.TrimPrefix(f.Name, "test.") switch name { - case "testlogfile", "paniconexit0": + case "testlogfile", "paniconexit0", "fuzzcachedir", "fuzzworker": // These flags are only for use by cmd/go. default: names = append(names, name) diff --git a/src/cmd/go/internal/test/test.go b/src/cmd/go/internal/test/test.go index 198afbf4c3..8f5d57eff1 100644 --- a/src/cmd/go/internal/test/test.go +++ b/src/cmd/go/internal/test/test.go @@ -61,8 +61,8 @@ followed by detailed output for each failed package. 'Go test' recompiles each package along with any files with names matching the file pattern "*_test.go". -These additional files can contain test functions, benchmark functions, and -example functions. See 'go help testfunc' for more. +These additional files can contain test functions, benchmark functions, fuzz +targets and example functions. See 'go help testfunc' for more. Each listed package causes the execution of a separate test binary. Files whose names begin with "_" (including "_test.go") or "." are ignored. @@ -132,6 +132,8 @@ variables are unchanged. A cached test result is treated as executing in no time at all,so a successful package test result will be cached and reused regardless of -timeout setting. +Run 'go help fuzz' for details around how the go command handles fuzz targets. + In addition to the build flags, the flags handled by 'go test' itself are: -args @@ -208,7 +210,8 @@ control the execution of any test: (for example, -benchtime 100x). -count n - Run each test and benchmark n times (default 1). + Run each test, benchmark, and fuzz targets' seed corpora n times + (default 1). If -cpu is set, run n times for each GOMAXPROCS value. Examples are always run once. @@ -237,36 +240,55 @@ control the execution of any test: Sets -cover. -cpu 1,2,4 - Specify a list of GOMAXPROCS values for which the tests or - benchmarks should be executed. The default is the current value + Specify a list of GOMAXPROCS values for which the tests, benchmarks or + fuzz targets should be executed. The default is the current value of GOMAXPROCS. -failfast Do not start new tests after the first test failure. + -fuzz name + Run the fuzz target with the given regexp. Must match exactly one fuzz + target. This is an experimental feature. + + -fuzztime t + Run enough iterations of the fuzz test to take t, specified as a + time.Duration (for example, -fuzztime 1h30s). The default is to run + forever. + The special syntax Nx means to run the fuzz test N times + (for example, -fuzztime 100x). + -json Log verbose output and test results in JSON. This presents the same information as the -v flag in a machine-readable format. + -keepfuzzing + Keep running the fuzz target if a crasher is found. + -list regexp - List tests, benchmarks, or examples matching the regular expression. - No tests, benchmarks or examples will be run. This will only - list top-level tests. No subtest or subbenchmarks will be shown. + List tests, benchmarks, fuzz targets, or examples matching the regular + expression. No tests, benchmarks, fuzz targets, or examples will be run. + This will only list top-level tests. No subtest or subbenchmarks will be + shown. -parallel n - Allow parallel execution of test functions that call t.Parallel. + Allow parallel execution of test functions that call t.Parallel, and + f.Fuzz functions that call t.Parallel when running the seed corpus. The value of this flag is the maximum number of tests to run - simultaneously; by default, it is set to the value of GOMAXPROCS. + simultaneously. While fuzzing, the value of this flag is the + maximum number of workers to run the fuzz function simultaneously, + regardless of whether t.Parallel has been called; by default, it is set + to the value of GOMAXPROCS. Note that -parallel only applies within a single test binary. The 'go test' command may run tests for different packages in parallel as well, according to the setting of the -p flag (see 'go help build'). -run regexp - Run only those tests and examples matching the regular expression. - For tests, the regular expression is split by unbracketed slash (/) - characters into a sequence of regular expressions, and each part - of a test's identifier must match the corresponding element in + Run only those tests, examples, and fuzz targets matching the regular + expression. For tests, the regular expression is split by unbracketed + slash (/) characters into a sequence of regular expressions, and each + part of a test's identifier must match the corresponding element in the sequence, if any. Note that possible parents of matches are run too, so that -run=X/Y matches and runs and reports the result of all tests matching X, even those without sub-tests matching Y, @@ -436,6 +458,10 @@ A benchmark function is one named BenchmarkXxx and should have the signature, func BenchmarkXxx(b *testing.B) { ... } +A fuzz target is one named FuzzXxx and should have the signature, + + func FuzzXxx(f *testing.F) { ... } + An example function is similar to a test function but, instead of using *testing.T to report success or failure, prints output to os.Stdout. If the last comment in the function starts with "Output:" then the output @@ -475,12 +501,34 @@ Here is another example where the ordering of the output is ignored: The entire test file is presented as the example when it contains a single example function, at least one other function, type, variable, or constant -declaration, and no test or benchmark functions. +declaration, and no fuzz targets or test or benchmark functions. See the documentation of the testing package for more information. `, } +var HelpFuzz = &base.Command{ + UsageLine: "fuzz", + Short: "fuzzing", + Long: ` +By default, go test will build and run the fuzz targets using the target's seed +corpus only. Any generated corpora in $GOCACHE that were previously written by +the fuzzing engine will not be run by default. + +When -fuzz is set, the binary will be instrumented for coverage. After all +tests, examples, benchmark functions, and the seed corpora for all fuzz targets +have been run, go test will begin to fuzz the specified fuzz target. +Note that this feature is experimental. + +-run can be used for testing a single seed corpus entry for a fuzz target. The +regular expression value of -run can be in the form $target/$name, where $target +is the name of the fuzz target, and $name is the name of the file (ignoring file +extensions) to run. For example, -run=FuzzFoo/497b6f87. + +See https://golang.org/s/draft-fuzzing-design for more details. +`, +} + var ( testBench string // -bench flag testC bool // -c flag @@ -489,6 +537,7 @@ var ( testCoverPaths []string // -coverpkg flag testCoverPkgs []*load.Package // -coverpkg flag testCoverProfile string // -coverprofile flag + testFuzz string // -fuzz flag testJSON bool // -json flag testList string // -list flag testO string // -o flag @@ -622,6 +671,9 @@ func runTest(ctx context.Context, cmd *base.Command, args []string) { if testO != "" && len(pkgs) != 1 { base.Fatalf("cannot use -o flag with multiple packages") } + if testFuzz != "" && len(pkgs) != 1 { + base.Fatalf("cannot use -fuzz flag with multiple packages") + } if testProfile() != "" && len(pkgs) != 1 { base.Fatalf("cannot use %s flag with multiple packages", testProfile()) } @@ -632,7 +684,9 @@ func runTest(ctx context.Context, cmd *base.Command, args []string) { // to that timeout plus one minute. This is a backup alarm in case // the test wedges with a goroutine spinning and its background // timer does not get a chance to fire. - if testTimeout > 0 { + // Don't set this if fuzzing, since it should be able to run + // indefinitely. + if testTimeout > 0 && testFuzz == "" { testKillTimeout = testTimeout + 1*time.Minute } @@ -782,6 +836,32 @@ func runTest(ctx context.Context, cmd *base.Command, args []string) { } } + // Inform the compiler that it should instrument the binary at + // build-time when fuzzing is enabled. + fuzzFlags := work.FuzzInstrumentFlags() + if testFuzz != "" && fuzzFlags != nil { + // Don't instrument packages which may affect coverage guidance but are + // unlikely to be useful. Most of these are used by the testing or + // internal/fuzz concurrently with fuzzing. + var fuzzNoInstrument = map[string]bool{ + "context": true, + "internal/fuzz": true, + "reflect": true, + "runtime": true, + "sync": true, + "sync/atomic": true, + "syscall": true, + "testing": true, + "time": true, + } + for _, p := range load.TestPackageList(ctx, pkgOpts, pkgs) { + if fuzzNoInstrument[p.ImportPath] { + continue + } + p.Internal.Gcflags = append(p.Internal.Gcflags, fuzzFlags...) + } + } + // Prepare build + run + print actions for all packages being tested. for _, p := range pkgs { // sync/atomic import is inserted by the cover tool. See #18486 @@ -1087,6 +1167,8 @@ func declareCoverVars(p *load.Package, files ...string) map[string]*load.CoverVa } var noTestsToRun = []byte("\ntesting: warning: no tests to run\n") +var noTargetsToFuzz = []byte("\ntesting: warning: no targets to fuzz\n") +var tooManyTargetsToFuzz = []byte("\ntesting: warning: -fuzz matches more than one target, won't fuzz\n") type runCache struct { disableCache bool // cache should be disabled for this run @@ -1134,10 +1216,10 @@ func (c *runCache) builderRunTest(b *work.Builder, ctx context.Context, a *work. } var buf bytes.Buffer - if len(pkgArgs) == 0 || (testBench != "") { + if len(pkgArgs) == 0 || testBench != "" || testFuzz != "" { // Stream test output (no buffering) when no package has // been given on the command line (implicit current directory) - // or when benchmarking. + // or when benchmarking or fuzzing. // No change to stdout. } else { // If we're only running a single package under test or if parallelism is @@ -1190,7 +1272,12 @@ func (c *runCache) builderRunTest(b *work.Builder, ctx context.Context, a *work. testlogArg = []string{"-test.testlogfile=" + a.Objdir + "testlog.txt"} } panicArg := "-test.paniconexit0" - args := str.StringList(execCmd, a.Deps[0].BuiltTarget(), testlogArg, panicArg, testArgs) + fuzzArg := []string{} + if testFuzz != "" { + fuzzCacheDir := filepath.Join(cache.Default().FuzzDir(), a.Package.ImportPath) + fuzzArg = []string{"-test.fuzzcachedir=" + fuzzCacheDir} + } + args := str.StringList(execCmd, a.Deps[0].BuiltTarget(), testlogArg, panicArg, fuzzArg, testArgs) if testCoverProfile != "" { // Write coverage to temporary profile, for merging later. @@ -1283,6 +1370,12 @@ func (c *runCache) builderRunTest(b *work.Builder, ctx context.Context, a *work. if bytes.HasPrefix(out, noTestsToRun[1:]) || bytes.Contains(out, noTestsToRun) { norun = " [no tests to run]" } + if bytes.HasPrefix(out, noTargetsToFuzz[1:]) || bytes.Contains(out, noTargetsToFuzz) { + norun = " [no targets to fuzz]" + } + if bytes.HasPrefix(out, tooManyTargetsToFuzz[1:]) || bytes.Contains(out, tooManyTargetsToFuzz) { + norun = " [will not fuzz, -fuzz matches more than one target]" + } fmt.Fprintf(cmd.Stdout, "ok \t%s\t%s%s%s\n", a.Package.ImportPath, t, coveragePercentage(out), norun) c.saveOutput(a) } else { diff --git a/src/cmd/go/internal/test/testflag.go b/src/cmd/go/internal/test/testflag.go index e0a3e010fa..cb3543884a 100644 --- a/src/cmd/go/internal/test/testflag.go +++ b/src/cmd/go/internal/test/testflag.go @@ -57,6 +57,7 @@ func init() { cf.String("cpu", "", "") cf.StringVar(&testCPUProfile, "cpuprofile", "", "") cf.Bool("failfast", false, "") + cf.StringVar(&testFuzz, "fuzz", "", "") cf.StringVar(&testList, "list", "", "") cf.StringVar(&testMemProfile, "memprofile", "", "") cf.String("memprofilerate", "", "") @@ -67,6 +68,8 @@ func init() { cf.String("run", "", "") cf.Bool("short", false, "") cf.DurationVar(&testTimeout, "timeout", 10*time.Minute, "") + cf.String("fuzztime", "", "") + cf.String("fuzzminimizetime", "", "") cf.StringVar(&testTrace, "trace", "", "") cf.BoolVar(&testV, "v", false, "") cf.Var(&testShuffle, "shuffle", "") diff --git a/src/cmd/go/internal/work/init.go b/src/cmd/go/internal/work/init.go index 7aa8dfe55f..2a605e73ee 100644 --- a/src/cmd/go/internal/work/init.go +++ b/src/cmd/go/internal/work/init.go @@ -60,6 +60,14 @@ func BuildInit() { } } +func FuzzInstrumentFlags() []string { + if cfg.Goarch != "amd64" && cfg.Goarch != "arm64" { + // Instrumentation is only supported on 64-bit architectures. + return nil + } + return []string{"-d=libfuzzer"} +} + func instrumentInit() { if !cfg.BuildRace && !cfg.BuildMSan { return diff --git a/src/cmd/go/main.go b/src/cmd/go/main.go index 16361e02ca..11ff750aff 100644 --- a/src/cmd/go/main.go +++ b/src/cmd/go/main.go @@ -80,6 +80,7 @@ func init() { modfetch.HelpPrivate, test.HelpTestflag, test.HelpTestfunc, + test.HelpFuzz, modget.HelpVCS, } } diff --git a/src/cmd/go/testdata/script/test_fuzz.txt b/src/cmd/go/testdata/script/test_fuzz.txt new file mode 100644 index 0000000000..b1a02f46eb --- /dev/null +++ b/src/cmd/go/testdata/script/test_fuzz.txt @@ -0,0 +1,442 @@ +# TODO(jayconrod): support shared memory on more platforms. +[!darwin] [!linux] [!windows] skip + +# Test that running a fuzz target that returns without failing or calling +# f.Fuzz fails and causes a non-zero exit status. +! go test noop_fuzz_test.go +! stdout ^ok +stdout FAIL + +# Test that fuzzing a fuzz target that returns without failing or calling +# f.Fuzz fails and causes a non-zero exit status. +! go test -fuzz=Fuzz -fuzztime=1x noop_fuzz_test.go +! stdout ^ok +stdout FAIL + +# Test that calling f.Error in a fuzz target causes a non-zero exit status. +! go test -fuzz=Fuzz -fuzztime=1x error_fuzz_test.go +! stdout ^ok +stdout FAIL + +# Test that calling f.Fatal in a fuzz target causes a non-zero exit status. +! go test fatal_fuzz_test.go +! stdout ^ok +stdout FAIL + +# Test that successful test exits cleanly. +go test success_fuzz_test.go +stdout ^ok +! stdout FAIL + +# Test that successful fuzzing exits cleanly. +go test -fuzz=Fuzz -fuzztime=1x success_fuzz_test.go +stdout ok +! stdout FAIL + +# Test that calling f.Fatal while fuzzing causes a non-zero exit status. +! go test -fuzz=Fuzz -fuzztime=1x fatal_fuzz_test.go +! stdout ^ok +stdout FAIL + +# Test error with seed corpus in f.Fuzz +! go test -run FuzzError fuzz_add_test.go +! stdout ^ok +stdout FAIL +stdout 'error here' + +[short] stop + +# Test that calling panic(nil) in a fuzz target causes a non-zero exit status. +! go test panic_fuzz_test.go +! stdout ^ok +stdout FAIL + +# Test that skipped test exits cleanly. +go test skipped_fuzz_test.go +stdout ok +! stdout FAIL + +# Test that f.Fatal within f.Fuzz panics +! go test fatal_fuzz_fn_fuzz_test.go +! stdout ^ok +! stdout 'fatal here' +stdout FAIL +stdout 'f.Fuzz function' + +# Test that f.Error within f.Fuzz panics +! go test error_fuzz_fn_fuzz_test.go +! stdout ^ok +! stdout 'error here' +stdout FAIL +stdout 'f.Fuzz function' + +# Test that f.Skip within f.Fuzz panics +! go test skip_fuzz_fn_fuzz_test.go +! stdout ^ok +! stdout 'skip here' +stdout FAIL +stdout 'f.Fuzz function' + +# Test that a call to f.Fatal after the Fuzz func is never executed. +go test fatal_after_fuzz_func_fuzz_test.go +stdout ok +! stdout FAIL + +# Test that missing *T in f.Fuzz causes a non-zero exit status. +! go test incomplete_fuzz_call_fuzz_test.go +! stdout ^ok +stdout FAIL + +# Test that a panic in the Cleanup func is executed. +! go test cleanup_fuzz_test.go +! stdout ^ok +stdout FAIL +stdout 'failed some precondition' + +# Test success with seed corpus in f.Fuzz +go test -run FuzzPass fuzz_add_test.go +stdout ok +! stdout FAIL +! stdout 'off by one error' + +# Test fatal with seed corpus in f.Fuzz +! go test -run FuzzFatal fuzz_add_test.go +! stdout ^ok +stdout FAIL +stdout 'fatal here' + +# Test panic with seed corpus in f.Fuzz +! go test -run FuzzPanic fuzz_add_test.go +! stdout ^ok +stdout FAIL +stdout 'off by one error' + +# Test panic(nil) with seed corpus in f.Fuzz +! go test -run FuzzNilPanic fuzz_add_test.go +! stdout ^ok +stdout FAIL + +# Test panic with unsupported seed corpus +! go test -run FuzzUnsupported fuzz_add_test.go +! stdout ^ok +stdout FAIL + +# Test panic with different number of args to f.Add +! go test -run FuzzAddDifferentNumber fuzz_add_test.go +! stdout ^ok +stdout FAIL + +# Test panic with different type of args to f.Add +! go test -run FuzzAddDifferentType fuzz_add_test.go +! stdout ^ok +stdout FAIL + +# Test that the wrong type given with f.Add will fail. +! go test -run FuzzWrongType fuzz_add_test.go +! stdout ^ok +stdout FAIL + +# Test fatal with testdata seed corpus +! go test -run FuzzFail corpustesting/fuzz_testdata_corpus_test.go +! stdout ^ok +stdout FAIL +stdout 'fatal here' + +# Test pass with testdata seed corpus +go test -run FuzzPass corpustesting/fuzz_testdata_corpus_test.go +stdout ok +! stdout FAIL +! stdout 'fatal here' + +# Test pass with testdata and f.Add seed corpus +go test -run FuzzPassString corpustesting/fuzz_testdata_corpus_test.go +stdout ok +! stdout FAIL + +# Fuzzing pass with testdata and f.Add seed corpus (skip running tests first) +go test -run=None -fuzz=FuzzPassString corpustesting/fuzz_testdata_corpus_test.go -fuzztime=10x +stdout ok +! stdout FAIL + +# Fuzzing pass with testdata and f.Add seed corpus +go test -run=FuzzPassString -fuzz=FuzzPassString corpustesting/fuzz_testdata_corpus_test.go -fuzztime=10x +stdout ok +! stdout FAIL + +# Test panic with malformed seed corpus +! go test -run FuzzFail corpustesting/fuzz_testdata_corpus_test.go +! stdout ^ok +stdout FAIL + +# Test pass with file in other nested testdata directory +go test -run FuzzInNestedDir corpustesting/fuzz_testdata_corpus_test.go +stdout ok +! stdout FAIL +! stdout 'fatal here' + +# Test fails with file containing wrong type +! go test -run FuzzWrongType corpustesting/fuzz_testdata_corpus_test.go +! stdout ^ok +stdout FAIL + +-- noop_fuzz_test.go -- +package noop_fuzz + +import "testing" + +func Fuzz(f *testing.F) {} + +-- error_fuzz_test.go -- +package error_fuzz + +import "testing" + +func Fuzz(f *testing.F) { + f.Error("error in target") +} + +-- fatal_fuzz_test.go -- +package fatal_fuzz + +import "testing" + +func Fuzz(f *testing.F) { + f.Fatal("fatal in target") +} + +-- panic_fuzz_test.go -- +package panic_fuzz + +import "testing" + +func FuzzPanic(f *testing.F) { + panic(nil) +} + +-- success_fuzz_test.go -- +package success_fuzz + +import "testing" + +func Fuzz(f *testing.F) { + f.Fuzz(func (*testing.T, []byte) {}) +} + +-- skipped_fuzz_test.go -- +package skipped_fuzz + +import "testing" + +func Fuzz(f *testing.F) { + f.Skip() +} + +-- fatal_fuzz_fn_fuzz_test.go -- +package fatal_fuzz_fn_fuzz + +import "testing" + +func Fuzz(f *testing.F) { + f.Add([]byte("aa")) + f.Fuzz(func(t *testing.T, b []byte) { + f.Fatal("fatal here") + }) +} + +-- error_fuzz_fn_fuzz_test.go -- +package error_fuzz_fn_fuzz + +import "testing" + +func Fuzz(f *testing.F) { + f.Add([]byte("aa")) + f.Fuzz(func(t *testing.T, b []byte) { + f.Error("error here") + }) +} + +-- skip_fuzz_fn_fuzz_test.go -- +package skip_fuzz_fn_fuzz + +import "testing" + +func Fuzz(f *testing.F) { + f.Add([]byte("aa")) + f.Fuzz(func(t *testing.T, b []byte) { + f.Skip("skip here") + }) +} + +-- fatal_after_fuzz_func_fuzz_test.go -- +package fatal_after_fuzz_func_fuzz + +import "testing" + +func Fuzz(f *testing.F) { + f.Fuzz(func(t *testing.T, b []byte) { + // no-op + }) + f.Fatal("this shouldn't be called") +} + +-- incomplete_fuzz_call_fuzz_test.go -- +package incomplete_fuzz_call_fuzz + +import "testing" + +func Fuzz(f *testing.F) { + f.Fuzz(func(b []byte) { + // this is missing *testing.T as the first param, so should panic + }) +} + +-- cleanup_fuzz_test.go -- +package cleanup_fuzz_test + +import "testing" + +func Fuzz(f *testing.F) { + f.Cleanup(func() { + panic("failed some precondition") + }) + f.Fuzz(func(t *testing.T, b []byte) { + // no-op + }) +} + +-- fuzz_add_test.go -- +package fuzz_add + +import "testing" + +func add(f *testing.F) { + f.Helper() + f.Add([]byte("123")) + f.Add([]byte("12345")) + f.Add([]byte("")) +} + +func FuzzPass(f *testing.F) { + add(f) + f.Fuzz(func(t *testing.T, b []byte) { + if len(b) == -1 { + t.Fatal("fatal here") // will not be executed + } + }) +} + +func FuzzError(f *testing.F) { + add(f) + f.Fuzz(func(t *testing.T, b []byte) { + if len(b) == 3 { + t.Error("error here") + } + }) +} + +func FuzzFatal(f *testing.F) { + add(f) + f.Fuzz(func(t *testing.T, b []byte) { + if len(b) == 0 { + t.Fatal("fatal here") + } + }) +} + +func FuzzPanic(f *testing.F) { + add(f) + f.Fuzz(func(t *testing.T, b []byte) { + if len(b) == 5 { + panic("off by one error") + } + }) +} + +func FuzzNilPanic(f *testing.F) { + add(f) + f.Fuzz(func(t *testing.T, b []byte) { + if len(b) == 3 { + panic(nil) + } + }) +} + +func FuzzUnsupported(f *testing.F) { + m := make(map[string]bool) + f.Add(m) + f.Fuzz(func(*testing.T, []byte) {}) +} + +func FuzzAddDifferentNumber(f *testing.F) { + f.Add([]byte("a")) + f.Add([]byte("a"), []byte("b")) + f.Fuzz(func(*testing.T, []byte) {}) +} + +func FuzzAddDifferentType(f *testing.F) { + f.Add(false) + f.Add(1234) + f.Fuzz(func(*testing.T, []byte) {}) +} + +func FuzzWrongType(f *testing.F) { + f.Add("hello") + f.Fuzz(func(*testing.T, []byte) {}) +} + +-- corpustesting/fuzz_testdata_corpus_test.go -- +package fuzz_testdata_corpus + +import "testing" + +func fuzzFn(f *testing.F) { + f.Helper() + f.Fuzz(func(t *testing.T, b []byte) { + if string(b) == "12345" { + t.Fatal("fatal here") + } + }) +} + +func FuzzFail(f *testing.F) { + fuzzFn(f) +} + +func FuzzPass(f *testing.F) { + fuzzFn(f) +} + +func FuzzPassString(f *testing.F) { + f.Add("some seed corpus") + f.Fuzz(func(*testing.T, string) {}) +} + +func FuzzPanic(f *testing.F) { + f.Fuzz(func(t *testing.T, b []byte) {}) +} + +func FuzzInNestedDir(f *testing.F) { + f.Fuzz(func(t *testing.T, b []byte) {}) +} + +func FuzzWrongType(f *testing.F) { + f.Fuzz(func(t *testing.T, b []byte) {}) +} + +-- corpustesting/testdata/fuzz/FuzzFail/1 -- +go test fuzz v1 +[]byte("12345") +-- corpustesting/testdata/fuzz/FuzzPass/1 -- +go test fuzz v1 +[]byte("00000") +-- corpustesting/testdata/fuzz/FuzzPassString/1 -- +go test fuzz v1 +string("hello") +-- corpustesting/testdata/fuzz/FuzzPanic/1 -- +malformed +-- corpustesting/testdata/fuzz/FuzzInNestedDir/anotherdir/1 -- +go test fuzz v1 +[]byte("12345") +-- corpustesting/testdata/fuzz/FuzzWrongType/1 -- +go test fuzz v1 +int("00000")
\ No newline at end of file diff --git a/src/cmd/go/testdata/script/test_fuzz_cache.txt b/src/cmd/go/testdata/script/test_fuzz_cache.txt new file mode 100644 index 0000000000..10e4c2926f --- /dev/null +++ b/src/cmd/go/testdata/script/test_fuzz_cache.txt @@ -0,0 +1,81 @@ +# TODO(jayconrod): support shared memory on more platforms. +[!darwin] [!linux] [!windows] skip + +[short] skip +env GOCACHE=$WORK/cache + +# Fuzz cache should not exist after a regular test run. +go test . +exists $GOCACHE +! exists $GOCACHE/fuzz + +# Fuzzing should write interesting values to the cache. +go test -fuzz=FuzzY -fuzztime=100x . +go run ./contains_files $GOCACHE/fuzz/example.com/y/FuzzY + +# 'go clean -cache' should not delete the fuzz cache. +go clean -cache +exists $GOCACHE/fuzz + +# 'go clean -fuzzcache' should delete the fuzz cache but not the build cache. +go list -f {{.Stale}} ./empty +stdout true +go install ./empty +go list -f {{.Stale}} ./empty +stdout false +go clean -fuzzcache +! exists $GOCACHE/fuzz +go list -f {{.Stale}} ./empty +stdout false + +-- go.mod -- +module example.com/y + +go 1.16 +-- y_test.go -- +package y + +import ( + "io" + "testing" +) + +func FuzzY(f *testing.F) { + f.Add([]byte("y")) + f.Fuzz(func(t *testing.T, b []byte) { Y(io.Discard, b) }) +} +-- y.go -- +package y + +import ( + "bytes" + "io" +) + +func Y(w io.Writer, b []byte) { + if !bytes.Equal(b, []byte("y")) { + w.Write([]byte("not equal")) + } +} +-- empty/empty.go -- +package empty +-- contains_files/contains_files.go -- +package main + +import ( + "fmt" + "path/filepath" + "io/ioutil" + "os" +) + +func main() { + infos, err := ioutil.ReadDir(filepath.Clean(os.Args[1])) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + if len(infos) == 0 { + os.Exit(1) + } +} diff --git a/src/cmd/go/testdata/script/test_fuzz_chatty.txt b/src/cmd/go/testdata/script/test_fuzz_chatty.txt new file mode 100644 index 0000000000..9ebd480c90 --- /dev/null +++ b/src/cmd/go/testdata/script/test_fuzz_chatty.txt @@ -0,0 +1,106 @@ +# TODO(jayconrod): support shared memory on more platforms. +[!darwin] [!linux] [!windows] skip + +[short] skip + +# Run chatty fuzz targets with an error. +! go test -v chatty_error_fuzz_test.go +! stdout '^ok' +stdout 'FAIL' +stdout 'error in target' + +# Run chatty fuzz targets with a fatal. +! go test -v chatty_fatal_fuzz_test.go +! stdout '^ok' +stdout 'FAIL' +stdout 'fatal in target' + +# Run chatty fuzz target with a panic +! go test -v chatty_panic_fuzz_test.go +! stdout ^ok +stdout FAIL +stdout 'this is bad' + +# Run skipped chatty fuzz targets. +go test -v chatty_skipped_fuzz_test.go +stdout ok +stdout SKIP +! stdout FAIL + +# Run successful chatty fuzz targets. +go test -v chatty_fuzz_test.go +stdout ok +stdout PASS +stdout 'all good here' +! stdout FAIL + +# Fuzz successful chatty fuzz target that includes a separate unit test. +go test -v chatty_with_test_fuzz_test.go -fuzz=Fuzz -fuzztime=1x +stdout ok +stdout PASS +! stdout FAIL +# TODO: It's currently the case that it's logged twice. Fix that, and change +# this check to verify it. +stdout 'all good here' +# Verify that the unit test is only run once. +! stdout '(?s)logged foo.*logged foo' + +-- chatty_error_fuzz_test.go -- +package chatty_error_fuzz + +import "testing" + +func Fuzz(f *testing.F) { + f.Error("error in target") +} + +-- chatty_fatal_fuzz_test.go -- +package chatty_fatal_fuzz + +import "testing" + +func Fuzz(f *testing.F) { + f.Fatal("fatal in target") +} + +-- chatty_panic_fuzz_test.go -- +package chatty_panic_fuzz + +import "testing" + +func Fuzz(f *testing.F) { + panic("this is bad") +} + +-- chatty_skipped_fuzz_test.go -- +package chatty_skipped_fuzz + +import "testing" + +func Fuzz(f *testing.F) { + f.Skip() +} + +-- chatty_fuzz_test.go -- +package chatty_fuzz + +import "testing" + +func Fuzz(f *testing.F) { + f.Log("all good here") + f.Fuzz(func(*testing.T, []byte) {}) +} + +-- chatty_with_test_fuzz_test.go -- +package chatty_with_test_fuzz + +import "testing" + +func TestFoo(t *testing.T) { + t.Log("logged foo") +} + +func Fuzz(f *testing.F) { + f.Log("all good here") + f.Fuzz(func(*testing.T, []byte) {}) +} diff --git a/src/cmd/go/testdata/script/test_fuzz_cleanup.txt b/src/cmd/go/testdata/script/test_fuzz_cleanup.txt new file mode 100644 index 0000000000..88625916ba --- /dev/null +++ b/src/cmd/go/testdata/script/test_fuzz_cleanup.txt @@ -0,0 +1,67 @@ +# TODO(jayconrod): support shared memory on more platforms. +[!darwin] [!linux] [!windows] skip +[short] skip + +# Cleanup should run after F.Skip. +go test -run=FuzzTargetSkip +stdout cleanup + +# Cleanup should run after F.Fatal. +! go test -run=FuzzTargetFatal +stdout cleanup + +# Cleanup should run after an unexpected runtime.Goexit. +! go test -run=FuzzTargetGoexit +stdout cleanup + +# Cleanup should run after panic. +! go test -run=FuzzTargetPanic +stdout cleanup + +# Cleanup should run in fuzz function on seed corpus. +go test -v -run=FuzzFunction +stdout '(?s)inner.*outer' + +# TODO(jayconrod): test cleanup while fuzzing. For now, the worker process's +# stdout and stderr is connected to the coordinator's, but it should eventually +# be connected to os.DevNull, so we wouldn't see t.Log output. + +-- go.mod -- +module cleanup + +go 1.15 +-- cleanup_test.go -- +package cleanup + +import ( + "runtime" + "testing" +) + +func FuzzTargetSkip(f *testing.F) { + f.Cleanup(func() { f.Log("cleanup") }) + f.Skip() +} + +func FuzzTargetFatal(f *testing.F) { + f.Cleanup(func() { f.Log("cleanup") }) + f.Fatal() +} + +func FuzzTargetGoexit(f *testing.F) { + f.Cleanup(func() { f.Log("cleanup") }) + runtime.Goexit() +} + +func FuzzTargetPanic(f *testing.F) { + f.Cleanup(func() { f.Log("cleanup") }) + panic("oh no") +} + +func FuzzFunction(f *testing.F) { + f.Add([]byte{0}) + f.Cleanup(func() { f.Log("outer") }) + f.Fuzz(func(t *testing.T, b []byte) { + t.Cleanup(func() { t.Logf("inner") }) + }) +} diff --git a/src/cmd/go/testdata/script/test_fuzz_deadline.txt b/src/cmd/go/testdata/script/test_fuzz_deadline.txt new file mode 100644 index 0000000000..12f1054f61 --- /dev/null +++ b/src/cmd/go/testdata/script/test_fuzz_deadline.txt @@ -0,0 +1,37 @@ +# TODO(jayconrod): support shared memory on more platforms. +[!darwin] [!linux] [!windows] skip + +[short] skip + +# The fuzz function should be able to detect whether -timeout +# was set with T.Deadline. Note there is no F.Deadline, and +# there is no timeout while fuzzing, even if -fuzztime is set. +go test -run=FuzzDeadline -wantdeadline=true # -timeout defaults to 10m +go test -run=FuzzDeadline -timeout=0 -wantdeadline=false +! go test -run=FuzzDeadline -timeout=1s -wantdeadline=false +go test -run=FuzzDeadline -timeout=1s -wantdeadline=true +go test -fuzz=FuzzDeadline -timeout=0 -fuzztime=1s -wantdeadline=false +go test -fuzz=FuzzDeadline -timeout=0 -fuzztime=100x -wantdeadline=false + +-- go.mod -- +module fuzz + +go 1.16 +-- fuzz_deadline_test.go -- +package fuzz_test + +import ( + "flag" + "testing" +) + +var wantDeadline = flag.Bool("wantdeadline", false, "whether the test should have a deadline") + +func FuzzDeadline(f *testing.F) { + f.Add("run once") + f.Fuzz(func (t *testing.T, _ string) { + if _, hasDeadline := t.Deadline(); hasDeadline != *wantDeadline { + t.Fatalf("function got %v; want %v", hasDeadline, *wantDeadline) + } + }) +} diff --git a/src/cmd/go/testdata/script/test_fuzz_fuzztime.txt b/src/cmd/go/testdata/script/test_fuzz_fuzztime.txt new file mode 100644 index 0000000000..7d644b4d13 --- /dev/null +++ b/src/cmd/go/testdata/script/test_fuzz_fuzztime.txt @@ -0,0 +1,82 @@ +# TODO(jayconrod): support shared memory on more platforms. +[!darwin] [!linux] [!windows] skip + +[short] skip + +# There are no seed values, so 'go test' should finish quickly. +go test + +# Fuzzing should exit 0 after fuzztime, even if timeout is short. +go test -timeout=10ms -fuzz=FuzzFast -fuzztime=5s + +# We should see the same behavior when invoking the test binary directly. +go test -c +exec ./fuzz.test$GOEXE -test.timeout=10ms -test.fuzz=FuzzFast -test.fuzztime=5s -test.parallel=1 -test.fuzzcachedir=$WORK/cache + +# Timeout should not cause inputs to be written as crashers. +! exists testdata/fuzz + +# When we use fuzztime with an "x" suffix, it runs a specific number of times. +# This fuzz function creates a file with a unique name ($pid.$count) on each run. +# We count the files to find the number of runs. +mkdir count +env GOCACHE=$WORK/tmp +go test -fuzz=FuzzCount -fuzztime=1000x -fuzzminimizetime=1x +go run check_file_count.go 1000 + +-- go.mod -- +module fuzz + +go 1.16 +-- fuzz_fast_test.go -- +package fuzz_test + +import "testing" + +func FuzzFast(f *testing.F) { + f.Fuzz(func (*testing.T, []byte) {}) +} +-- fuzz_count_test.go -- +package fuzz + +import ( + "fmt" + "os" + "testing" +) + +func FuzzCount(f *testing.F) { + pid := os.Getpid() + n := 0 + f.Fuzz(func(t *testing.T, _ []byte) { + name := fmt.Sprintf("count/%v.%d", pid, n) + if err := os.WriteFile(name, nil, 0666); err != nil { + t.Fatal(err) + } + n++ + }) +} +-- check_file_count.go -- +// +build ignore + +package main + +import ( + "fmt" + "os" + "strconv" +) + +func main() { + dir, err := os.ReadDir("count") + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + got := len(dir) + want, _ := strconv.Atoi(os.Args[1]) + if got != want { + fmt.Fprintf(os.Stderr, "got %d files; want %d\n", got, want) + os.Exit(1) + } +} diff --git a/src/cmd/go/testdata/script/test_fuzz_io_error.txt b/src/cmd/go/testdata/script/test_fuzz_io_error.txt new file mode 100644 index 0000000000..4c7ab4c152 --- /dev/null +++ b/src/cmd/go/testdata/script/test_fuzz_io_error.txt @@ -0,0 +1,101 @@ +# Test that when the coordinator experiences an I/O error communicating +# with a worker, the coordinator stops the worker and reports the error. +# The coordinator should not record a crasher. +# +# We simulate an I/O error in the test by writing garbage to fuzz_out. +# This is unlikely, but possible. It's difficult to simulate interruptions +# due to ^C and EOF errors which are more common. We don't report those. +[short] skip +[!darwin] [!linux] [!windows] skip + +# If the I/O error occurs before F.Fuzz is called, the coordinator should +# stop the worker and say that. +! go test -fuzz=FuzzClosePipeBefore -parallel=1 +stdout '\s*fuzzing process terminated without fuzzing:' +! stdout 'communicating with fuzzing process' +! exists testdata + +# If the I/O error occurs after F.Fuzz is called (unlikely), just exit. +# It's hard to distinguish this case from the worker being interrupted by ^C +# or exiting with status 0 (which it should do when interrupted by ^C). +! go test -fuzz=FuzzClosePipeAfter -parallel=1 +stdout '^\s*communicating with fuzzing process: invalid character ''!'' looking for beginning of value$' +! exists testdata + +-- go.mod -- +module test + +go 1.17 +-- io_error_test.go -- +package io_error + +import ( + "flag" + "testing" + "time" +) + +func isWorker() bool { + f := flag.Lookup("test.fuzzworker") + if f == nil { + return false + } + get, ok := f.Value.(flag.Getter) + if !ok { + return false + } + return get.Get() == interface{}(true) +} + +func FuzzClosePipeBefore(f *testing.F) { + if isWorker() { + sendGarbageToCoordinator(f) + time.Sleep(3600 * time.Second) // pause until coordinator terminates the process + } + f.Fuzz(func(*testing.T, []byte) {}) +} + +func FuzzClosePipeAfter(f *testing.F) { + f.Fuzz(func(t *testing.T, _ []byte) { + if isWorker() { + sendGarbageToCoordinator(t) + time.Sleep(3600 * time.Second) // pause until coordinator terminates the process + } + }) +} +-- io_error_windows_test.go -- +package io_error + +import ( + "fmt" + "os" + "testing" +) + +func sendGarbageToCoordinator(tb testing.TB) { + v := os.Getenv("GO_TEST_FUZZ_WORKER_HANDLES") + var fuzzInFD, fuzzOutFD uintptr + if _, err := fmt.Sscanf(v, "%x,%x", &fuzzInFD, &fuzzOutFD); err != nil { + tb.Fatalf("parsing GO_TEST_FUZZ_WORKER_HANDLES: %v", err) + } + f := os.NewFile(fuzzOutFD, "fuzz_out") + if _, err := f.Write([]byte("!!")); err != nil { + tb.Fatalf("writing fuzz_out: %v", err) + } +} +-- io_error_notwindows_test.go -- +// +build !windows + +package io_error + +import ( + "os" + "testing" +) + +func sendGarbageToCoordinator(tb testing.TB) { + f := os.NewFile(4, "fuzz_out") + if _, err := f.Write([]byte("!!")); err != nil { + tb.Fatalf("writing fuzz_out: %v", err) + } +} diff --git a/src/cmd/go/testdata/script/test_fuzz_match.txt b/src/cmd/go/testdata/script/test_fuzz_match.txt new file mode 100644 index 0000000000..3a2ca631ad --- /dev/null +++ b/src/cmd/go/testdata/script/test_fuzz_match.txt @@ -0,0 +1,39 @@ +# TODO(jayconrod): support shared memory on more platforms. +[!darwin] [!linux] [!windows] skip + +# Matches only fuzz targets to test. +go test standalone_fuzz_test.go +! stdout '^ok.*\[no tests to run\]' +stdout '^ok' + +# Matches only for fuzzing. +go test -fuzz Fuzz -fuzztime 1x standalone_fuzz_test.go +! stdout '^ok.*\[no tests to run\]' +stdout '^ok' + +# Matches none for fuzzing but will run the fuzz target as a test. +go test -fuzz ThisWillNotMatch -fuzztime 1x standalone_fuzz_test.go +! stdout '^ok.*no tests to run' +stdout '^ok' +stdout 'no targets to fuzz' + +[short] stop + +# Matches only fuzz targets to test with -run. +go test -run Fuzz standalone_fuzz_test.go +! stdout '^ok.*\[no tests to run\]' +stdout '^ok' + +# Matches no fuzz targets. +go test -run ThisWillNotMatch standalone_fuzz_test.go +stdout '^ok.*no tests to run' +! stdout 'no targets to fuzz' + +-- standalone_fuzz_test.go -- +package standalone_fuzz + +import "testing" + +func Fuzz(f *testing.F) { + f.Fuzz(func (*testing.T, []byte) {}) +} diff --git a/src/cmd/go/testdata/script/test_fuzz_minimize.txt b/src/cmd/go/testdata/script/test_fuzz_minimize.txt new file mode 100644 index 0000000000..f0adb9ec3e --- /dev/null +++ b/src/cmd/go/testdata/script/test_fuzz_minimize.txt @@ -0,0 +1,200 @@ +# TODO(jayconrod): support shared memory on more platforms. +[!darwin] [!linux] [!windows] skip + +[short] skip + +# We clean the fuzz cache during this test. Don't clean the user's cache. +env GOCACHE=$WORK/gocache + +# Test that fuzzminimizetime cannot be negative seconds +! go test -fuzz=FuzzMinimizerRecoverable -run=FuzzMinimizerRecoverable -fuzztime=10000x -fuzzminimizetime=-1ms minimizer_test.go +! stdout '^ok' +! stdout 'contains a non-zero byte' +stdout 'invalid duration' +stdout FAIL + +# Test that fuzzminimizetime cannot be negative times +! go test -fuzz=FuzzMinimizerRecoverable -run=FuzzMinimizerRecoverable -fuzztime=10000x -fuzzminimizetime=-1x minimizer_test.go +! stdout '^ok' +! stdout 'contains a non-zero byte' +stdout 'invalid count' +stdout FAIL + +# Test that fuzzminimizetime can be zero seconds, and minimization is disabled +! go test -fuzz=FuzzMinimizeZeroDurationSet -run=FuzzMinimizeZeroDurationSet -fuzztime=10000x -fuzzminimizetime=0s minimizer_test.go +! stdout '^ok' +! stdout 'minimizing' +stdout 'there was an Error' +stdout FAIL + +# Test that fuzzminimizetime can be zero times, and minimization is disabled +! go test -fuzz=FuzzMinimizeZeroLimitSet -run=FuzzMinimizeZeroLimitSet -fuzztime=10000x -fuzzminimizetime=0x minimizer_test.go +! stdout '^ok' +! stdout 'minimizing' +stdout 'there was an Error' +stdout FAIL + +# Test that minimization is working for recoverable errors. +! go test -fuzz=FuzzMinimizerRecoverable -run=FuzzMinimizerRecoverable -fuzztime=10000x minimizer_test.go +! stdout '^ok' +stdout 'got the minimum size!' +stdout 'contains a non-zero byte' +stdout FAIL + +# Check that the bytes written to testdata are of length 50 (the minimum size) +go run check_testdata.go FuzzMinimizerRecoverable 50 + +# Test that re-running the minimized value causes a crash. +! go test -run=FuzzMinimizerRecoverable minimizer_test.go +rm testdata + +# Test that minimization is working for non-recoverable errors. +! go test -fuzz=FuzzMinimizerNonrecoverable -run=FuzzMinimizerNonrecoverable -fuzztime=10000x minimizer_test.go +! stdout '^ok' +stdout 'minimizing' +stdout 'fuzzing process terminated unexpectedly: exit status 99' +stdout FAIL + +# Check that re-running the value causes a crash. +! go test -run=FuzzMinimizerNonrecoverable minimizer_test.go +rm testdata + +# Clear the fuzzing cache. There may already be minimized inputs that would +# interfere with the next stage of the test. +go clean -fuzzcache + +# Test that minimization can be cancelled by fuzzminimizetime and the latest +# crash will still be logged and written to testdata. +! go test -fuzz=FuzzMinimizerRecoverable -run=FuzzMinimizerRecoverable -fuzztime=100x -fuzzminimizetime=1x minimizer_test.go +! stdout '^ok' +stdout 'testdata[/\\]fuzz[/\\]FuzzMinimizerRecoverable[/\\]' +! stdout 'got the minimum size!' # it shouldn't have had enough time to minimize it +stdout FAIL + +# Test that re-running the unminimized value causes a crash. +! go test -run=FuzzMinimizerRecoverable minimizer_test.go + +# TODO(jayconrod,katiehockman): add a test which verifies that the right bytes +# are written to testdata in the case of an interrupt during minimization. + +-- go.mod -- +module m + +go 1.16 +-- minimizer_test.go -- +package fuzz_test + +import ( + "os" + "testing" +) + +func FuzzMinimizeZeroDurationSet(f *testing.F) { + f.Fuzz(func(t *testing.T, b []byte) { + if len(b) > 5 { + t.Errorf("there was an Error") + } + }) +} + +func FuzzMinimizeZeroLimitSet(f *testing.F) { + f.Fuzz(func(t *testing.T, b []byte) { + if len(b) > 5 { + t.Errorf("there was an Error") + } + }) +} + +func FuzzMinimizerRecoverable(f *testing.F) { + f.Add(make([]byte, 100)) + f.Fuzz(func(t *testing.T, b []byte) { + if len(b) < 50 { + // Make sure that b is large enough that it can be minimized + return + } + // Given the randomness of the mutations, this should allow the + // minimizer to trim down the value a bit. + for _, n := range b { + if n != 0 { + if len(b) == 50 { + t.Log("got the minimum size!") + } + t.Fatal("contains a non-zero byte") + } + } + }) +} + +func FuzzMinimizerNonrecoverable(f *testing.F) { + f.Add(make([]byte, 100)) + f.Fuzz(func(t *testing.T, b []byte) { + if len(b) < 50 { + // Make sure that b is large enough that it can be minimized + return + } + // Given the randomness of the mutations, this should allow the + // minimizer to trim down the value a bit. + for _, n := range b { + if n != 0 { + t.Log("contains a non-zero byte") + os.Exit(99) + } + } + }) +} +-- check_testdata.go -- +// +build ignore + +package main + +import ( + "bytes" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strconv" +) + +func main() { + target := os.Args[1] + numBytes, err := strconv.Atoi(os.Args[2]) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + + // Open the file in testdata (there should only be one) + dir := fmt.Sprintf("testdata/fuzz/%s", target) + files, err := ioutil.ReadDir(dir) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + if len(files) != 1 { + fmt.Fprintf(os.Stderr, "expected one file, got %d", len(files)) + os.Exit(1) + } + got, err := ioutil.ReadFile(filepath.Join(dir, files[0].Name())) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + + // Trim the newline at the end of the file + got = bytes.TrimSpace(got) + + // Make sure that there were exactly 100 bytes written to the corpus entry + prefix := []byte("[]byte(") + i := bytes.Index(got, prefix) + gotBytes := got[i+len(prefix) : len(got)-1] + s, err := strconv.Unquote(string(gotBytes)) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + if want, got := numBytes, len(s); want != got { + fmt.Fprintf(os.Stderr, "want %d bytes, got %d\n", want, got) + os.Exit(1) + } +} diff --git a/src/cmd/go/testdata/script/test_fuzz_minimize_interesting.txt b/src/cmd/go/testdata/script/test_fuzz_minimize_interesting.txt new file mode 100644 index 0000000000..5e1d90d8d9 --- /dev/null +++ b/src/cmd/go/testdata/script/test_fuzz_minimize_interesting.txt @@ -0,0 +1,112 @@ +# TODO(jayconrod): support shared memory on more platforms. +[!darwin] [!linux] [!windows] skip + +# Instrumentation only supported on 64-bit architectures. +[!amd64] [!arm64] skip + +# Test that when an interesting value is discovered (one that expands coverage), +# the fuzzing engine minimizes it before writing it to the cache. +# +# The program below starts with a seed value of length 100, but more coverage +# will be found for any value other than the seed. We should end with a value +# in the cache of length 1 (the minimizer currently does not produce empty +# strings). check_cache.go confirms that. +# +# We would like to verify that ALL values in the cache were minimized to a +# length of 1, but this isn't always possible when new coverage is found in +# functions called by testing or internal/fuzz in the background. + +go test -c -fuzz=. # Build using shared build cache for speed. +env GOCACHE=$WORK/gocache +exec ./fuzz.test$GOEXE -test.fuzzcachedir=$GOCACHE/fuzz -test.fuzz=. -test.fuzztime=1000x +go run check_cache.go $GOCACHE/fuzz/FuzzMin + +-- go.mod -- +module fuzz + +go 1.17 +-- fuzz_test.go -- +package fuzz + +import ( + "bytes" + "testing" +) + +func FuzzMin(f *testing.F) { + seed := bytes.Repeat([]byte("a"), 20) + f.Add(seed) + f.Fuzz(func(t *testing.T, buf []byte) { + if bytes.Equal(buf, seed) { + return + } + if n := sum(buf); n < 0 { + t.Error("sum cannot be negative") + } + }) +} + +func sum(buf []byte) int { + n := 0 + for _, b := range buf { + n += int(b) + } + return n +} +-- check_cache.go -- +//go:build ignore +// +build ignore + +// check_cache.go checks that each file in the cached corpus has a []byte +// of length at most 1. This verifies that at least one cached input is minimized. +package main + +import ( + "bytes" + "fmt" + "os" + "path/filepath" + "regexp" + "strconv" +) + +func main() { + dir := os.Args[1] + ents, err := os.ReadDir(dir) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + for _, ent := range ents { + name := filepath.Join(dir, ent.Name()) + if good, err := checkCacheFile(name); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } else if good { + os.Exit(0) + } + } + fmt.Fprintln(os.Stderr, "no cached inputs were minimized") + os.Exit(1) +} + +func checkCacheFile(name string) (good bool, err error) { + data, err := os.ReadFile(name) + if err != nil { + return false, err + } + for _, line := range bytes.Split(data, []byte("\n")) { + m := valRe.FindSubmatch(line) + if m == nil { + continue + } + if s, err := strconv.Unquote(string(m[1])); err != nil { + return false, err + } else if len(s) <= 1 { + return true, nil + } + } + return false, nil +} + +var valRe = regexp.MustCompile(`^\[\]byte\(([^)]+)\)$`) diff --git a/src/cmd/go/testdata/script/test_fuzz_multiple.txt b/src/cmd/go/testdata/script/test_fuzz_multiple.txt new file mode 100644 index 0000000000..6a7732f514 --- /dev/null +++ b/src/cmd/go/testdata/script/test_fuzz_multiple.txt @@ -0,0 +1,51 @@ +# This test checks that 'go test' prints a reasonable error when fuzzing is +# enabled, and multiple package or multiple fuzz targets match. +# TODO(#46312): support fuzzing multiple targets in multiple packages. + +# TODO(jayconrod): support shared memory on more platforms. +[!darwin] [!linux] [!windows] skip + +[short] skip + +# With fuzzing disabled, multiple targets can be tested. +go test ./... + +# With fuzzing enabled, at most one package may be tested, +# even if only one package contains fuzz targets. +! go test -fuzz=. ./... +stderr '^cannot use -fuzz flag with multiple packages$' +! go test -fuzz=. ./zero ./one +stderr '^cannot use -fuzz flag with multiple packages$' +go test -fuzz=. -fuzztime=1x ./one + +# With fuzzing enabled, at most one target in the same package may match. +! go test -fuzz=. ./two +stdout '^testing: will not fuzz, -fuzz matches more than one target: \[FuzzOne FuzzTwo\]$' +go test -fuzz=FuzzTwo -fuzztime=1x ./two + +-- go.mod -- +module fuzz + +go 1.18 +-- zero/zero.go -- +package zero +-- one/one_test.go -- +package one + +import "testing" + +func FuzzOne(f *testing.F) { + f.Fuzz(func(*testing.T, []byte) {}) +} +-- two/two_test.go -- +package two + +import "testing" + +func FuzzOne(f *testing.F) { + f.Fuzz(func(*testing.T, []byte) {}) +} + +func FuzzTwo(f *testing.F) { + f.Fuzz(func(*testing.T, []byte) {}) +} diff --git a/src/cmd/go/testdata/script/test_fuzz_mutate_crash.txt b/src/cmd/go/testdata/script/test_fuzz_mutate_crash.txt new file mode 100644 index 0000000000..1b8b79b3dd --- /dev/null +++ b/src/cmd/go/testdata/script/test_fuzz_mutate_crash.txt @@ -0,0 +1,295 @@ +# TODO(jayconrod): support shared memory on more platforms. +[!darwin] [!linux] [!windows] skip + +# Tests that a crash caused by a mutator-discovered input writes the bad input +# to testdata, and fails+reports correctly. This tests the end-to-end behavior +# of the mutator finding a crash while fuzzing, adding it as a regression test +# to the seed corpus in testdata, and failing the next time the test is run. + +[short] skip + +# Running the seed corpus for all of the targets should pass the first +# time, since nothing in the seed corpus will cause a crash. +go test + +# Running the fuzzer should find a crashing input quickly. +! go test -fuzz=FuzzWithBug -fuzztime=100x -fuzzminimizetime=1000x +stdout 'testdata[/\\]fuzz[/\\]FuzzWithBug[/\\]' +stdout 'this input caused a crash!' +go run check_testdata.go FuzzWithBug + +# Now, the failing bytes should have been added to the seed corpus for +# the target, and should fail when run without fuzzing. +! go test +stdout 'testdata[/\\]fuzz[/\\]FuzzWithBug[/\\][a-f0-9]{64}' +stdout 'this input caused a crash!' + +! go test -run=FuzzWithNilPanic -fuzz=FuzzWithNilPanic -fuzztime=100x -fuzzminimizetime=1000x +stdout 'testdata[/\\]fuzz[/\\]FuzzWithNilPanic[/\\]' +stdout 'runtime.Goexit' +go run check_testdata.go FuzzWithNilPanic + +! go test -run=FuzzWithFail -fuzz=FuzzWithFail -fuzztime=100x -fuzzminimizetime=1000x +stdout 'testdata[/\\]fuzz[/\\]FuzzWithFail[/\\]' +go run check_testdata.go FuzzWithFail + +! go test -run=FuzzWithLogFail -fuzz=FuzzWithLogFail -fuzztime=100x -fuzzminimizetime=1000x +stdout 'testdata[/\\]fuzz[/\\]FuzzWithLogFail[/\\]' +stdout 'logged something' +go run check_testdata.go FuzzWithLogFail + +! go test -run=FuzzWithErrorf -fuzz=FuzzWithErrorf -fuzztime=100x -fuzzminimizetime=1000x +stdout 'testdata[/\\]fuzz[/\\]FuzzWithErrorf[/\\]' +stdout 'errorf was called here' +go run check_testdata.go FuzzWithErrorf + +! go test -run=FuzzWithFatalf -fuzz=FuzzWithFatalf -fuzztime=100x -fuzzminimizetime=1000x +stdout 'testdata[/\\]fuzz[/\\]FuzzWithFatalf[/\\]' +stdout 'fatalf was called here' +go run check_testdata.go FuzzWithFatalf + +! go test -run=FuzzWithBadExit -fuzz=FuzzWithBadExit -fuzztime=100x -fuzzminimizetime=1000x +stdout 'testdata[/\\]fuzz[/\\]FuzzWithBadExit[/\\]' +stdout 'unexpectedly' +go run check_testdata.go FuzzWithBadExit + +# Running the fuzzer should find a crashing input quickly for fuzzing two types. +! go test -run=FuzzWithTwoTypes -fuzz=FuzzWithTwoTypes -fuzztime=100x -fuzzminimizetime=1000x +stdout 'testdata[/\\]fuzz[/\\]FuzzWithTwoTypes[/\\]' +stdout 'these inputs caused a crash!' +go run check_testdata.go FuzzWithTwoTypes + +# Running the fuzzer should find a crashing input quickly for an integer. +! go test -run=FuzzInt -fuzz=FuzzInt -fuzztime=100x -fuzzminimizetime=1000x +stdout 'testdata[/\\]fuzz[/\\]FuzzInt[/\\]' +stdout 'this input caused a crash!' +go run check_testdata.go FuzzInt + +! go test -run=FuzzUint -fuzz=FuzzUint -fuzztime=100x -fuzzminimizetime=1000x +stdout 'testdata[/\\]fuzz[/\\]FuzzUint[/\\]' +stdout 'this input caused a crash!' +go run check_testdata.go FuzzUint + +# Running the fuzzer should find a crashing input quickly for a bool. +! go test -run=FuzzBool -fuzz=FuzzBool -fuzztime=100x -fuzzminimizetime=1000x +stdout 'testdata[/\\]fuzz[/\\]FuzzBool[/\\]' +stdout 'this input caused a crash!' +go run check_testdata.go FuzzBool + +# Running the fuzzer should find a crashing input quickly for a float. +! go test -run=FuzzFloat -fuzz=FuzzFloat -fuzztime=100x -fuzzminimizetime=1000x +stdout 'testdata[/\\]fuzz[/\\]FuzzFloat[/\\]' +stdout 'this input caused a crash!' +go run check_testdata.go FuzzFloat + +# Running the fuzzer should find a crashing input quickly for a byte. +! go test -run=FuzzByte -fuzz=FuzzByte -fuzztime=100x -fuzzminimizetime=1000x +stdout 'testdata[/\\]fuzz[/\\]FuzzByte[/\\]' +stdout 'this input caused a crash!' +go run check_testdata.go FuzzByte + +# Running the fuzzer should find a crashing input quickly for a rune. +! go test -run=FuzzRune -fuzz=FuzzRune -fuzztime=100x -fuzzminimizetime=1000x +stdout 'testdata[/\\]fuzz[/\\]FuzzRune[/\\]' +stdout 'this input caused a crash!' +go run check_testdata.go FuzzRune + +# Running the fuzzer should find a crashing input quickly for a string. +! go test -run=FuzzString -fuzz=FuzzString -fuzztime=100x -fuzzminimizetime=1000x +stdout 'testdata[/\\]fuzz[/\\]FuzzString[/\\]' +stdout 'this input caused a crash!' +go run check_testdata.go FuzzString + +-- go.mod -- +module m + +go 1.16 +-- fuzz_crash_test.go -- +package fuzz_crash + +import ( + "os" + "testing" +) + +func FuzzWithBug(f *testing.F) { + f.Add([]byte("aa")) + f.Fuzz(func(t *testing.T, b []byte) { + if string(b) != "aa" { + panic("this input caused a crash!") + } + }) +} + +func FuzzWithNilPanic(f *testing.F) { + f.Add([]byte("aa")) + f.Fuzz(func(t *testing.T, b []byte) { + if string(b) != "aa" { + panic(nil) + } + }) +} + +func FuzzWithFail(f *testing.F) { + f.Add([]byte("aa")) + f.Fuzz(func(t *testing.T, b []byte) { + if string(b) != "aa" { + t.Fail() + } + }) +} + +func FuzzWithLogFail(f *testing.F) { + f.Add([]byte("aa")) + f.Fuzz(func(t *testing.T, b []byte) { + if string(b) != "aa" { + t.Log("logged something") + t.Fail() + } + }) +} + +func FuzzWithErrorf(f *testing.F) { + f.Add([]byte("aa")) + f.Fuzz(func(t *testing.T, b []byte) { + if string(b) != "aa" { + t.Errorf("errorf was called here") + } + }) +} + +func FuzzWithFatalf(f *testing.F) { + f.Add([]byte("aa")) + f.Fuzz(func(t *testing.T, b []byte) { + if string(b) != "aa" { + t.Fatalf("fatalf was called here") + } + }) +} + +func FuzzWithBadExit(f *testing.F) { + f.Add([]byte("aa")) + f.Fuzz(func(t *testing.T, b []byte) { + if string(b) != "aa" { + os.Exit(1) + } + }) +} + +func FuzzWithTwoTypes(f *testing.F) { + f.Fuzz(func(t *testing.T, a, b []byte) { + if len(a) > 0 && len(b) > 0 { + panic("these inputs caused a crash!") + } + }) +} + +func FuzzInt(f *testing.F) { + f.Add(0) + f.Fuzz(func(t *testing.T, a int) { + if a != 0 { + panic("this input caused a crash!") + } + }) +} + +func FuzzUint(f *testing.F) { + f.Add(uint(0)) + f.Fuzz(func(t *testing.T, a uint) { + if a != 0 { + panic("this input caused a crash!") + } + }) +} + +func FuzzBool(f *testing.F) { + f.Add(false) + f.Fuzz(func(t *testing.T, a bool) { + if a { + panic("this input caused a crash!") + } + }) +} + +func FuzzFloat(f *testing.F) { + f.Fuzz(func(t *testing.T, a float64) { + if a != float64(int64(a)) { + // It has a decimal, so it was mutated by division + panic("this input caused a crash!") + } + }) +} + +func FuzzByte(f *testing.F) { + f.Add(byte(0)) + f.Fuzz(func(t *testing.T, a byte) { + if a != 0 { + panic("this input caused a crash!") + } + }) +} + +func FuzzRune(f *testing.F) { + f.Add(rune(0)) + f.Fuzz(func(t *testing.T, a rune) { + if a != 0 { + panic("this input caused a crash!") + } + }) +} + +func FuzzString(f *testing.F) { + f.Add("") + f.Fuzz(func(t *testing.T, a string) { + if a != "" { + panic("this input caused a crash!") + } + }) +} + +-- check_testdata.go -- +// +build ignore + +package main + +import ( + "bytes" + "crypto/sha256" + "fmt" + "io/ioutil" + "os" + "path/filepath" +) + +func main() { + target := os.Args[1] + dir := filepath.Join("testdata/fuzz", target) + + files, err := ioutil.ReadDir(dir) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + + if len(files) == 0 { + fmt.Fprintf(os.Stderr, "expect at least one new mutation to be written to testdata\n") + os.Exit(1) + } + + fname := files[0].Name() + contents, err := ioutil.ReadFile(filepath.Join(dir, fname)) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + if bytes.Equal(contents, []byte("aa")) { + fmt.Fprintf(os.Stderr, "newly written testdata entry was not mutated\n") + os.Exit(1) + } + // The hash of the bytes in the file should match the filename. + h := []byte(fmt.Sprintf("%x", sha256.Sum256(contents))) + if !bytes.Equal([]byte(fname), h) { + fmt.Fprintf(os.Stderr, "hash of bytes %q does not match filename %q\n", h, fname) + os.Exit(1) + } +} diff --git a/src/cmd/go/testdata/script/test_fuzz_mutate_fail.txt b/src/cmd/go/testdata/script/test_fuzz_mutate_fail.txt new file mode 100644 index 0000000000..935c22a05e --- /dev/null +++ b/src/cmd/go/testdata/script/test_fuzz_mutate_fail.txt @@ -0,0 +1,103 @@ +# TODO(jayconrod): support shared memory on more platforms. +[!darwin] [!linux] [!windows] skip + +# Check that if a worker does not call F.Fuzz or calls F.Fail first, +# 'go test' exits non-zero and no crasher is recorded. + +[short] skip + +! go test -fuzz=FuzzReturn +! exists testdata + +! go test -fuzz=FuzzSkip +! exists testdata + +! go test -fuzz=FuzzFail +! exists testdata + +! go test -fuzz=FuzzPanic +! exists testdata + +! go test -fuzz=FuzzNilPanic +! exists testdata + +! go test -fuzz=FuzzGoexit +! exists testdata + +! go test -fuzz=FuzzExit +! exists testdata + +-- go.mod -- +module m + +go 1.17 +-- fuzz_fail_test.go -- +package fuzz_fail + +import ( + "flag" + "os" + "runtime" + "testing" +) + +func isWorker() bool { + f := flag.Lookup("test.fuzzworker") + if f == nil { + return false + } + get, ok := f.Value.(flag.Getter) + if !ok { + return false + } + return get.Get() == interface{}(true) +} + +func FuzzReturn(f *testing.F) { + if isWorker() { + return + } + f.Fuzz(func(*testing.T, []byte) {}) +} + +func FuzzSkip(f *testing.F) { + if isWorker() { + f.Skip() + } + f.Fuzz(func(*testing.T, []byte) {}) +} + +func FuzzFail(f *testing.F) { + if isWorker() { + f.Fail() + } + f.Fuzz(func(*testing.T, []byte) {}) +} + +func FuzzPanic(f *testing.F) { + if isWorker() { + panic("nope") + } + f.Fuzz(func(*testing.T, []byte) {}) +} + +func FuzzNilPanic(f *testing.F) { + if isWorker() { + panic(nil) + } + f.Fuzz(func(*testing.T, []byte) {}) +} + +func FuzzGoexit(f *testing.F) { + if isWorker() { + runtime.Goexit() + } + f.Fuzz(func(*testing.T, []byte) {}) +} + +func FuzzExit(f *testing.F) { + if isWorker() { + os.Exit(99) + } + f.Fuzz(func(*testing.T, []byte) {}) +} diff --git a/src/cmd/go/testdata/script/test_fuzz_mutator.txt b/src/cmd/go/testdata/script/test_fuzz_mutator.txt new file mode 100644 index 0000000000..9d0738e169 --- /dev/null +++ b/src/cmd/go/testdata/script/test_fuzz_mutator.txt @@ -0,0 +1,166 @@ +# TODO(jayconrod): support shared memory on more platforms. +[!darwin] [!linux] [!windows] skip + +# Test basic fuzzing mutator behavior. +# +# fuzz_test.go has two fuzz targets (FuzzA, FuzzB) which both add a seed value. +# Each fuzz function writes the input to a log file. The coordinator and worker +# use separate log files. check_logs.go verifies that the coordinator only +# tests seed values and the worker tests mutated values on the fuzz target. + +[short] skip + +go test -fuzz=FuzzA -fuzztime=100x -parallel=1 -log=fuzz +go run check_logs.go fuzz fuzz.worker + +# TODO(b/181800488): remove -parallel=1, here and below. For now, when a +# crash is found, all workers keep running, wasting resources and reducing +# the number of executions available to the minimizer, increasing flakiness. + +# Test that the mutator is good enough to find several unique mutations. +! go test -fuzz=FuzzMutator -parallel=1 -fuzztime=100x mutator_test.go +! stdout '^ok' +stdout FAIL +stdout 'mutator found enough unique mutations' + +-- go.mod -- +module m + +go 1.16 +-- fuzz_test.go -- +package fuzz_test + +import ( + "flag" + "fmt" + "os" + "testing" +) + +var ( + logPath = flag.String("log", "", "path to log file") + logFile *os.File +) + +func TestMain(m *testing.M) { + flag.Parse() + var err error + logFile, err = os.OpenFile(*logPath, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0666) + if os.IsExist(err) { + *logPath += ".worker" + logFile, err = os.OpenFile(*logPath, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0666) + } + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + os.Exit(m.Run()) +} + +func FuzzA(f *testing.F) { + f.Add([]byte("seed")) + f.Fuzz(func(t *testing.T, b []byte) { + fmt.Fprintf(logFile, "FuzzA %q\n", b) + }) +} + +func FuzzB(f *testing.F) { + f.Add([]byte("seed")) + f.Fuzz(func(t *testing.T, b []byte) { + fmt.Fprintf(logFile, "FuzzB %q\n", b) + }) +} + +-- check_logs.go -- +// +build ignore + +package main + +import ( + "bufio" + "bytes" + "fmt" + "io" + "os" + "strings" +) + +func main() { + coordPath, workerPath := os.Args[1], os.Args[2] + + coordLog, err := os.Open(coordPath) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + defer coordLog.Close() + if err := checkCoordLog(coordLog); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + + workerLog, err := os.Open(workerPath) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + defer workerLog.Close() + if err := checkWorkerLog(workerLog); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + +func checkCoordLog(r io.Reader) error { + b, err := io.ReadAll(r) + if err != nil { + return err + } + if string(bytes.TrimSpace(b)) != `FuzzB "seed"` { + return fmt.Errorf("coordinator: did not test FuzzB seed") + } + return nil +} + +func checkWorkerLog(r io.Reader) error { + scan := bufio.NewScanner(r) + var sawAMutant bool + for scan.Scan() { + line := scan.Text() + if !strings.HasPrefix(line, "FuzzA ") { + return fmt.Errorf("worker: tested something other than target: %s", line) + } + if strings.TrimPrefix(line, "FuzzA ") != `"seed"` { + sawAMutant = true + } + } + if err := scan.Err(); err != nil && err != bufio.ErrTooLong { + return err + } + if !sawAMutant { + return fmt.Errorf("worker: did not test any mutants") + } + return nil +} +-- mutator_test.go -- +package fuzz_test + +import ( + "testing" +) + +// TODO(katiehockman): re-work this test once we have a better fuzzing engine +// (ie. more mutations, and compiler instrumentation) +func FuzzMutator(f *testing.F) { + // TODO(katiehockman): simplify this once we can dedupe crashes (e.g. + // replace map with calls to panic, and simply count the number of crashes + // that were added to testdata) + crashes := make(map[string]bool) + // No seed corpus initiated + f.Fuzz(func(t *testing.T, b []byte) { + crashes[string(b)] = true + if len(crashes) >= 10 { + panic("mutator found enough unique mutations") + } + }) +} diff --git a/src/cmd/go/testdata/script/test_fuzz_mutator_repeat.txt b/src/cmd/go/testdata/script/test_fuzz_mutator_repeat.txt new file mode 100644 index 0000000000..0924ed37e6 --- /dev/null +++ b/src/cmd/go/testdata/script/test_fuzz_mutator_repeat.txt @@ -0,0 +1,66 @@ +# TODO(jayconrod): support shared memory on more platforms. +[!darwin] [!linux] [!windows] skip + +# Verify that the fuzzing engine records the actual crashing input, even when +# a worker process terminates without communicating the crashing input back +# to the coordinator. + +[short] skip + +# Start fuzzing. The worker crashes after ~100 iterations. +# The fuzz function writes the crashing input to "want" before exiting. +# The fuzzing engine reconstructs the crashing input and saves it to testdata. +! exists want +! go test -fuzz=. -parallel=1 +stdout 'fuzzing process terminated unexpectedly' +stdout 'Crash written to testdata' + +# Run the fuzz target without fuzzing. The fuzz function is called with the +# crashing input in testdata. The test passes if that input is identical to +# the one saved in "want". +exists want +go test -want=want + +-- go.mod -- +module fuzz + +go 1.17 +-- fuzz_test.go -- +package fuzz + +import ( + "bytes" + "flag" + "os" + "testing" +) + +var wantFlag = flag.String("want", "", "file containing previous crashing input") + +func FuzzRepeat(f *testing.F) { + i := 0 + f.Fuzz(func(t *testing.T, b []byte) { + i++ + if i == 100 { + f, err := os.OpenFile("want", os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0666) + if err != nil { + // Couldn't create the file, probably because it already exists, + // and we're minimizing now. Return without crashing. + return + } + f.Write(b) + f.Close() + os.Exit(1) // crash without communicating + } + + if *wantFlag != "" { + want, err := os.ReadFile(*wantFlag) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(want, b) { + t.Fatalf("inputs are not equal!\n got: %q\nwant:%q", b, want) + } + } + }) +} diff --git a/src/cmd/go/testdata/script/test_fuzz_non_crash_signal.txt b/src/cmd/go/testdata/script/test_fuzz_non_crash_signal.txt new file mode 100644 index 0000000000..1568757de7 --- /dev/null +++ b/src/cmd/go/testdata/script/test_fuzz_non_crash_signal.txt @@ -0,0 +1,55 @@ +# NOTE: this test is skipped on Windows, since there's no concept of signals. +# When a process terminates another process, it provides an exit code. +# TODO(jayconrod): support shared memory on more platforms. +[!darwin] [!freebsd] [!linux] skip +[short] skip + +# FuzzNonCrash sends itself a signal that does not appear to be a crash. +# We should not save a crasher. +! go test -fuzz=FuzzNonCrash +! exists testdata +! stdout unreachable +! stderr unreachable +stdout 'fuzzing process terminated by unexpected signal; no crash will be recorded: signal: killed' + +# FuzzCrash sends itself a signal that looks like a crash. +# We should save a crasher. +! go test -fuzz=FuzzCrash +exists testdata/fuzz/FuzzCrash +stdout 'fuzzing process terminated unexpectedly' + +-- go.mod -- +module test + +go 1.17 +-- fuzz_posix_test.go -- +// +build darwin freebsd linux + +package fuzz + +import ( + "syscall" + "testing" +) + +func FuzzNonCrash(f *testing.F) { + f.Fuzz(func(*testing.T, bool) { + pid := syscall.Getpid() + if err := syscall.Kill(pid, syscall.SIGKILL); err != nil { + panic(err) + } + // signal may not be received immediately. Wait for it. + select{} + }) +} + +func FuzzCrash(f *testing.F) { + f.Fuzz(func(*testing.T, bool) { + pid := syscall.Getpid() + if err := syscall.Kill(pid, syscall.SIGILL); err != nil { + panic(err) + } + // signal may not be received immediately. Wait for it. + select{} + }) +} diff --git a/src/cmd/go/testdata/script/test_fuzz_parallel.txt b/src/cmd/go/testdata/script/test_fuzz_parallel.txt new file mode 100644 index 0000000000..a49f30a27f --- /dev/null +++ b/src/cmd/go/testdata/script/test_fuzz_parallel.txt @@ -0,0 +1,61 @@ +# TODO(jayconrod): support shared memory on more platforms. +[!darwin] [!linux] [!windows] skip + +[short] skip + +# When running seed inputs, T.Parallel should let multiple inputs run in +# parallel. +go test -run=FuzzSeed + +# When fuzzing, T.Parallel should be safe to call, but it should have no effect. +# We just check that it doesn't hang, which would be the most obvious +# failure mode. +# TODO(jayconrod): check for the string "after T.Parallel". It's not printed +# by 'go test', so we can't distinguish that crasher from some other panic. +! go test -run=FuzzMutate -fuzz=FuzzMutate +exists testdata/fuzz/FuzzMutate + +-- go.mod -- +module fuzz_parallel + +go 1.17 +-- fuzz_parallel_test.go -- +package fuzz_parallel + +import ( + "sort" + "sync" + "testing" +) + +func FuzzSeed(f *testing.F) { + for _, v := range [][]byte{{'a'}, {'b'}, {'c'}} { + f.Add(v) + } + + var mu sync.Mutex + var before, after []byte + f.Cleanup(func() { + sort.Slice(after, func(i, j int) bool { return after[i] < after[j] }) + got := string(before) + string(after) + want := "abcabc" + if got != want { + f.Fatalf("got %q; want %q", got, want) + } + }) + + f.Fuzz(func(t *testing.T, b []byte) { + before = append(before, b...) + t.Parallel() + mu.Lock() + after = append(after, b...) + mu.Unlock() + }) +} + +func FuzzMutate(f *testing.F) { + f.Fuzz(func(t *testing.T, _ []byte) { + t.Parallel() + t.Error("after T.Parallel") + }) +} diff --git a/src/cmd/go/testdata/script/test_fuzz_seed_corpus.txt b/src/cmd/go/testdata/script/test_fuzz_seed_corpus.txt new file mode 100644 index 0000000000..016b101d72 --- /dev/null +++ b/src/cmd/go/testdata/script/test_fuzz_seed_corpus.txt @@ -0,0 +1,168 @@ +# TODO(jayconrod): support shared memory on more platforms. +[!darwin] [!linux] [!windows] skip + +[short] skip +env GOCACHE=$WORK/cache + +# Test that fuzzing a target with a failure in f.Add prints the crash +# and doesn't write anything to testdata/fuzz +! go test -fuzz=FuzzWithAdd -run=FuzzWithAdd -fuzztime=1x +! stdout ^ok +! stdout 'Crash written to testdata[/\\]fuzz[/\\]FuzzWithAdd[/\\]' +stdout FAIL + +# Test that fuzzing a target with a sucess in f.Add and a fuzztime of only +# 1 does not produce a crash. +go test -fuzz=FuzzWithGoodAdd -run=FuzzWithGoodAdd -fuzztime=1x +stdout ok +! stdout FAIL + +# Test that fuzzing a target with a failure in testdata/fuzz prints the crash +# and doesn't write anything to testdata/fuzz +! go test -fuzz=FuzzWithTestdata -run=FuzzWithTestdata -fuzztime=1x +! stdout ^ok +! stdout 'Crash written to testdata[/\\]fuzz[/\\]FuzzWithTestdata[/\\]' +stdout FAIL + +# Test that fuzzing a target with no seed corpus or cache finds a crash, prints +# it, and write it to testdata +! go test -fuzz=FuzzWithNoCache -run=FuzzWithNoCache -fuzztime=1x +! stdout ^ok +stdout 'Crash written to testdata[/\\]fuzz[/\\]FuzzWithNoCache[/\\]' +stdout FAIL + +# Write a crashing input to the cache +mkdir $GOCACHE/fuzz/example.com/x/FuzzWithCache +cp cache-file $GOCACHE/fuzz/example.com/x/FuzzWithCache/1 + +# Test that fuzzing a target with a failure in the cache prints the crash +# and writes this as a "new" crash to testdata/fuzz +! go test -fuzz=FuzzWithCache -run=FuzzWithCache -fuzztime=1x +! stdout ^ok +stdout 'Crash written to testdata[/\\]fuzz[/\\]FuzzWithCache[/\\]' +stdout FAIL + +# Clear the fuzz cache and make sure it's gone +go clean -fuzzcache +! exists $GOCACHE/fuzz + +# The tests below should operate the exact same as the previous tests. If -fuzz +# is enabled, then whatever target is going to be fuzzed shouldn't be run by +# anything other than the workers. + +# Test that fuzzing a target (with -run=None set) with a failure in f.Add prints +# the crash and doesn't write anything to testdata/fuzz -fuzztime=1x +! go test -fuzz=FuzzWithAdd -run=None +! stdout ^ok +! stdout 'Crash written to testdata[/\\]fuzz[/\\]FuzzWithAdd[/\\]' +stdout FAIL + +# Test that fuzzing a target (with -run=None set) with a sucess in f.Add and a +# fuzztime of only 1 does not produce a crash. +go test -fuzz=FuzzWithGoodAdd -run=None -fuzztime=1x +stdout ok +! stdout FAIL + +# Test that fuzzing a target (with -run=None set) with a failure in +# testdata/fuzz prints the crash and doesn't write anything to testdata/fuzz +! go test -fuzz=FuzzWithTestdata -run=None -fuzztime=1x +! stdout ^ok +! stdout 'Crash written to testdata[/\\]fuzz[/\\]FuzzWithTestdata[/\\]' +stdout FAIL + +# Write a crashing input to the cache +mkdir $GOCACHE/fuzz/example.com/x/FuzzRunNoneWithCache +cp cache-file $GOCACHE/fuzz/example.com/x/FuzzRunNoneWithCache/1 + +# Test that fuzzing a target (with -run=None set) with a failure in the cache +# prints the crash and writes this as a "new" crash to testdata/fuzz +! go test -fuzz=FuzzRunNoneWithCache -run=None -fuzztime=1x +! stdout ^ok +stdout 'Crash written to testdata[/\\]fuzz[/\\]FuzzRunNoneWithCache[/\\]' +stdout FAIL + +# Clear the fuzz cache and make sure it's gone +go clean -fuzzcache +! exists $GOCACHE/fuzz + +# The tests below should operate the exact same way for the previous tests with +# a seed corpus (namely, they should still fail). However, the binary is built +# without instrumentation, so this should be a "testing only" run which executes +# the seed corpus before attempting to fuzz. + +go test -c +! exec ./x.test$GOEXE -test.fuzz=FuzzWithAdd -test.run=FuzzWithAdd -test.fuzztime=1x -test.fuzzcachedir=$WORK/cache +! stdout ^ok +! stdout 'Crash written to testdata[/\\]fuzz[/\\]FuzzWithAdd[/\\]' +stdout FAIL +stderr warning + +go test -c +! exec ./x.test$GOEXE -test.fuzz=FuzzWithTestdata -test.run=FuzzWithTestdata -test.fuzztime=1x -test.fuzzcachedir=$WORK/cache +! stdout ^ok +! stdout 'Crash written to testdata[/\\]fuzz[/\\]FuzzWithTestdata[/\\]' +stdout FAIL +stderr warning + +-- go.mod -- +module example.com/x + +go 1.16 +-- x_test.go -- +package x + +import "testing" + +func FuzzWithAdd(f *testing.F) { + f.Add(10) + f.Fuzz(func(t *testing.T, i int) { + if i == 10 { + t.Error("bad thing here") + } + }) +} + +func FuzzWithGoodAdd(f *testing.F) { + f.Add(10) + f.Fuzz(func(t *testing.T, i int) { + if i != 10 { + t.Error("bad thing here") + } + }) +} + +func FuzzWithTestdata(f *testing.F) { + f.Fuzz(func(t *testing.T, i int) { + if i == 10 { + t.Error("bad thing here") + } + }) +} + +func FuzzWithNoCache(f *testing.F) { + f.Fuzz(func(t *testing.T, i int) { + t.Error("bad thing here") + }) +} + +func FuzzWithCache(f *testing.F) { + f.Fuzz(func(t *testing.T, i int) { + if i == 10 { + t.Error("bad thing here") + } + }) +} + +func FuzzRunNoneWithCache(f *testing.F) { + f.Fuzz(func(t *testing.T, i int) { + if i == 10 { + t.Error("bad thing here") + } + }) +} +-- testdata/fuzz/FuzzWithTestdata/1 -- +go test fuzz v1 +int(10) +-- cache-file -- +go test fuzz v1 +int(10)
\ No newline at end of file diff --git a/src/cmd/go/testdata/script/test_fuzz_setenv.txt b/src/cmd/go/testdata/script/test_fuzz_setenv.txt new file mode 100644 index 0000000000..9738697a91 --- /dev/null +++ b/src/cmd/go/testdata/script/test_fuzz_setenv.txt @@ -0,0 +1,45 @@ +[short] skip +[!darwin] [!linux] [!windows] skip + +go test -fuzz=FuzzA -fuzztime=100x fuzz_setenv_test.go + +-- fuzz_setenv_test.go -- +package fuzz + +import ( + "flag" + "os" + "testing" +) + +func FuzzA(f *testing.F) { + if s := os.Getenv("TEST_FUZZ_SETENV_A"); isWorker() && s == "" { + f.Fatal("environment variable not set") + } else if !isWorker() && s != "" { + f.Fatal("environment variable already set") + } + f.Setenv("TEST_FUZZ_SETENV_A", "A") + if os.Getenv("TEST_FUZZ_SETENV_A") == "" { + f.Fatal("Setenv did not set environment variable") + } + f.Fuzz(func(*testing.T, []byte) {}) +} + +func FuzzB(f *testing.F) { + if os.Getenv("TEST_FUZZ_SETENV_A") != "" { + f.Fatal("environment variable not cleared after FuzzA") + } + f.Skip() +} + +func isWorker() bool { + f := flag.Lookup("test.fuzzworker") + if f == nil { + return false + } + get, ok := f.Value.(flag.Getter) + if !ok { + return false + } + return get.Get() == interface{}(true) +} diff --git a/src/cmd/go/testdata/script/test_fuzz_tag.txt b/src/cmd/go/testdata/script/test_fuzz_tag.txt new file mode 100644 index 0000000000..07ed5d6d61 --- /dev/null +++ b/src/cmd/go/testdata/script/test_fuzz_tag.txt @@ -0,0 +1,31 @@ +# Check that the gofuzzbeta tag is enabled by default and can be disabled. +# TODO(jayconrod,katiehockman): before merging to master, restore the old +# default and delete this test. + +[short] skip + +go test -list=. +stdout Test +stdout Fuzz + +go test -tags= + +-- go.mod -- +module fuzz + +go 1.17 +-- fuzz_test.go -- +// +build gofuzzbeta + +package fuzz + +import "testing" + +func Fuzz(f *testing.F) { + f.Add([]byte(nil)) + f.Fuzz(func(*testing.T, []byte) {}) +} + +func Test(*testing.T) {} +-- empty_test.go -- +package fuzz |