diff options
Diffstat (limited to 'src/cmd/go/testdata/script')
20 files changed, 2308 insertions, 0 deletions
diff --git a/src/cmd/go/testdata/script/test_fuzz.txt b/src/cmd/go/testdata/script/test_fuzz.txt new file mode 100644 index 0000000000..b1a02f46eb --- /dev/null +++ b/src/cmd/go/testdata/script/test_fuzz.txt @@ -0,0 +1,442 @@ +# TODO(jayconrod): support shared memory on more platforms. +[!darwin] [!linux] [!windows] skip + +# Test that running a fuzz target that returns without failing or calling +# f.Fuzz fails and causes a non-zero exit status. +! go test noop_fuzz_test.go +! stdout ^ok +stdout FAIL + +# Test that fuzzing a fuzz target that returns without failing or calling +# f.Fuzz fails and causes a non-zero exit status. +! go test -fuzz=Fuzz -fuzztime=1x noop_fuzz_test.go +! stdout ^ok +stdout FAIL + +# Test that calling f.Error in a fuzz target causes a non-zero exit status. +! go test -fuzz=Fuzz -fuzztime=1x error_fuzz_test.go +! stdout ^ok +stdout FAIL + +# Test that calling f.Fatal in a fuzz target causes a non-zero exit status. +! go test fatal_fuzz_test.go +! stdout ^ok +stdout FAIL + +# Test that successful test exits cleanly. +go test success_fuzz_test.go +stdout ^ok +! stdout FAIL + +# Test that successful fuzzing exits cleanly. +go test -fuzz=Fuzz -fuzztime=1x success_fuzz_test.go +stdout ok +! stdout FAIL + +# Test that calling f.Fatal while fuzzing causes a non-zero exit status. +! go test -fuzz=Fuzz -fuzztime=1x fatal_fuzz_test.go +! stdout ^ok +stdout FAIL + +# Test error with seed corpus in f.Fuzz +! go test -run FuzzError fuzz_add_test.go +! stdout ^ok +stdout FAIL +stdout 'error here' + +[short] stop + +# Test that calling panic(nil) in a fuzz target causes a non-zero exit status. +! go test panic_fuzz_test.go +! stdout ^ok +stdout FAIL + +# Test that skipped test exits cleanly. +go test skipped_fuzz_test.go +stdout ok +! stdout FAIL + +# Test that f.Fatal within f.Fuzz panics +! go test fatal_fuzz_fn_fuzz_test.go +! stdout ^ok +! stdout 'fatal here' +stdout FAIL +stdout 'f.Fuzz function' + +# Test that f.Error within f.Fuzz panics +! go test error_fuzz_fn_fuzz_test.go +! stdout ^ok +! stdout 'error here' +stdout FAIL +stdout 'f.Fuzz function' + +# Test that f.Skip within f.Fuzz panics +! go test skip_fuzz_fn_fuzz_test.go +! stdout ^ok +! stdout 'skip here' +stdout FAIL +stdout 'f.Fuzz function' + +# Test that a call to f.Fatal after the Fuzz func is never executed. +go test fatal_after_fuzz_func_fuzz_test.go +stdout ok +! stdout FAIL + +# Test that missing *T in f.Fuzz causes a non-zero exit status. +! go test incomplete_fuzz_call_fuzz_test.go +! stdout ^ok +stdout FAIL + +# Test that a panic in the Cleanup func is executed. +! go test cleanup_fuzz_test.go +! stdout ^ok +stdout FAIL +stdout 'failed some precondition' + +# Test success with seed corpus in f.Fuzz +go test -run FuzzPass fuzz_add_test.go +stdout ok +! stdout FAIL +! stdout 'off by one error' + +# Test fatal with seed corpus in f.Fuzz +! go test -run FuzzFatal fuzz_add_test.go +! stdout ^ok +stdout FAIL +stdout 'fatal here' + +# Test panic with seed corpus in f.Fuzz +! go test -run FuzzPanic fuzz_add_test.go +! stdout ^ok +stdout FAIL +stdout 'off by one error' + +# Test panic(nil) with seed corpus in f.Fuzz +! go test -run FuzzNilPanic fuzz_add_test.go +! stdout ^ok +stdout FAIL + +# Test panic with unsupported seed corpus +! go test -run FuzzUnsupported fuzz_add_test.go +! stdout ^ok +stdout FAIL + +# Test panic with different number of args to f.Add +! go test -run FuzzAddDifferentNumber fuzz_add_test.go +! stdout ^ok +stdout FAIL + +# Test panic with different type of args to f.Add +! go test -run FuzzAddDifferentType fuzz_add_test.go +! stdout ^ok +stdout FAIL + +# Test that the wrong type given with f.Add will fail. +! go test -run FuzzWrongType fuzz_add_test.go +! stdout ^ok +stdout FAIL + +# Test fatal with testdata seed corpus +! go test -run FuzzFail corpustesting/fuzz_testdata_corpus_test.go +! stdout ^ok +stdout FAIL +stdout 'fatal here' + +# Test pass with testdata seed corpus +go test -run FuzzPass corpustesting/fuzz_testdata_corpus_test.go +stdout ok +! stdout FAIL +! stdout 'fatal here' + +# Test pass with testdata and f.Add seed corpus +go test -run FuzzPassString corpustesting/fuzz_testdata_corpus_test.go +stdout ok +! stdout FAIL + +# Fuzzing pass with testdata and f.Add seed corpus (skip running tests first) +go test -run=None -fuzz=FuzzPassString corpustesting/fuzz_testdata_corpus_test.go -fuzztime=10x +stdout ok +! stdout FAIL + +# Fuzzing pass with testdata and f.Add seed corpus +go test -run=FuzzPassString -fuzz=FuzzPassString corpustesting/fuzz_testdata_corpus_test.go -fuzztime=10x +stdout ok +! stdout FAIL + +# Test panic with malformed seed corpus +! go test -run FuzzFail corpustesting/fuzz_testdata_corpus_test.go +! stdout ^ok +stdout FAIL + +# Test pass with file in other nested testdata directory +go test -run FuzzInNestedDir corpustesting/fuzz_testdata_corpus_test.go +stdout ok +! stdout FAIL +! stdout 'fatal here' + +# Test fails with file containing wrong type +! go test -run FuzzWrongType corpustesting/fuzz_testdata_corpus_test.go +! stdout ^ok +stdout FAIL + +-- noop_fuzz_test.go -- +package noop_fuzz + +import "testing" + +func Fuzz(f *testing.F) {} + +-- error_fuzz_test.go -- +package error_fuzz + +import "testing" + +func Fuzz(f *testing.F) { + f.Error("error in target") +} + +-- fatal_fuzz_test.go -- +package fatal_fuzz + +import "testing" + +func Fuzz(f *testing.F) { + f.Fatal("fatal in target") +} + +-- panic_fuzz_test.go -- +package panic_fuzz + +import "testing" + +func FuzzPanic(f *testing.F) { + panic(nil) +} + +-- success_fuzz_test.go -- +package success_fuzz + +import "testing" + +func Fuzz(f *testing.F) { + f.Fuzz(func (*testing.T, []byte) {}) +} + +-- skipped_fuzz_test.go -- +package skipped_fuzz + +import "testing" + +func Fuzz(f *testing.F) { + f.Skip() +} + +-- fatal_fuzz_fn_fuzz_test.go -- +package fatal_fuzz_fn_fuzz + +import "testing" + +func Fuzz(f *testing.F) { + f.Add([]byte("aa")) + f.Fuzz(func(t *testing.T, b []byte) { + f.Fatal("fatal here") + }) +} + +-- error_fuzz_fn_fuzz_test.go -- +package error_fuzz_fn_fuzz + +import "testing" + +func Fuzz(f *testing.F) { + f.Add([]byte("aa")) + f.Fuzz(func(t *testing.T, b []byte) { + f.Error("error here") + }) +} + +-- skip_fuzz_fn_fuzz_test.go -- +package skip_fuzz_fn_fuzz + +import "testing" + +func Fuzz(f *testing.F) { + f.Add([]byte("aa")) + f.Fuzz(func(t *testing.T, b []byte) { + f.Skip("skip here") + }) +} + +-- fatal_after_fuzz_func_fuzz_test.go -- +package fatal_after_fuzz_func_fuzz + +import "testing" + +func Fuzz(f *testing.F) { + f.Fuzz(func(t *testing.T, b []byte) { + // no-op + }) + f.Fatal("this shouldn't be called") +} + +-- incomplete_fuzz_call_fuzz_test.go -- +package incomplete_fuzz_call_fuzz + +import "testing" + +func Fuzz(f *testing.F) { + f.Fuzz(func(b []byte) { + // this is missing *testing.T as the first param, so should panic + }) +} + +-- cleanup_fuzz_test.go -- +package cleanup_fuzz_test + +import "testing" + +func Fuzz(f *testing.F) { + f.Cleanup(func() { + panic("failed some precondition") + }) + f.Fuzz(func(t *testing.T, b []byte) { + // no-op + }) +} + +-- fuzz_add_test.go -- +package fuzz_add + +import "testing" + +func add(f *testing.F) { + f.Helper() + f.Add([]byte("123")) + f.Add([]byte("12345")) + f.Add([]byte("")) +} + +func FuzzPass(f *testing.F) { + add(f) + f.Fuzz(func(t *testing.T, b []byte) { + if len(b) == -1 { + t.Fatal("fatal here") // will not be executed + } + }) +} + +func FuzzError(f *testing.F) { + add(f) + f.Fuzz(func(t *testing.T, b []byte) { + if len(b) == 3 { + t.Error("error here") + } + }) +} + +func FuzzFatal(f *testing.F) { + add(f) + f.Fuzz(func(t *testing.T, b []byte) { + if len(b) == 0 { + t.Fatal("fatal here") + } + }) +} + +func FuzzPanic(f *testing.F) { + add(f) + f.Fuzz(func(t *testing.T, b []byte) { + if len(b) == 5 { + panic("off by one error") + } + }) +} + +func FuzzNilPanic(f *testing.F) { + add(f) + f.Fuzz(func(t *testing.T, b []byte) { + if len(b) == 3 { + panic(nil) + } + }) +} + +func FuzzUnsupported(f *testing.F) { + m := make(map[string]bool) + f.Add(m) + f.Fuzz(func(*testing.T, []byte) {}) +} + +func FuzzAddDifferentNumber(f *testing.F) { + f.Add([]byte("a")) + f.Add([]byte("a"), []byte("b")) + f.Fuzz(func(*testing.T, []byte) {}) +} + +func FuzzAddDifferentType(f *testing.F) { + f.Add(false) + f.Add(1234) + f.Fuzz(func(*testing.T, []byte) {}) +} + +func FuzzWrongType(f *testing.F) { + f.Add("hello") + f.Fuzz(func(*testing.T, []byte) {}) +} + +-- corpustesting/fuzz_testdata_corpus_test.go -- +package fuzz_testdata_corpus + +import "testing" + +func fuzzFn(f *testing.F) { + f.Helper() + f.Fuzz(func(t *testing.T, b []byte) { + if string(b) == "12345" { + t.Fatal("fatal here") + } + }) +} + +func FuzzFail(f *testing.F) { + fuzzFn(f) +} + +func FuzzPass(f *testing.F) { + fuzzFn(f) +} + +func FuzzPassString(f *testing.F) { + f.Add("some seed corpus") + f.Fuzz(func(*testing.T, string) {}) +} + +func FuzzPanic(f *testing.F) { + f.Fuzz(func(t *testing.T, b []byte) {}) +} + +func FuzzInNestedDir(f *testing.F) { + f.Fuzz(func(t *testing.T, b []byte) {}) +} + +func FuzzWrongType(f *testing.F) { + f.Fuzz(func(t *testing.T, b []byte) {}) +} + +-- corpustesting/testdata/fuzz/FuzzFail/1 -- +go test fuzz v1 +[]byte("12345") +-- corpustesting/testdata/fuzz/FuzzPass/1 -- +go test fuzz v1 +[]byte("00000") +-- corpustesting/testdata/fuzz/FuzzPassString/1 -- +go test fuzz v1 +string("hello") +-- corpustesting/testdata/fuzz/FuzzPanic/1 -- +malformed +-- corpustesting/testdata/fuzz/FuzzInNestedDir/anotherdir/1 -- +go test fuzz v1 +[]byte("12345") +-- corpustesting/testdata/fuzz/FuzzWrongType/1 -- +go test fuzz v1 +int("00000")
\ No newline at end of file diff --git a/src/cmd/go/testdata/script/test_fuzz_cache.txt b/src/cmd/go/testdata/script/test_fuzz_cache.txt new file mode 100644 index 0000000000..10e4c2926f --- /dev/null +++ b/src/cmd/go/testdata/script/test_fuzz_cache.txt @@ -0,0 +1,81 @@ +# TODO(jayconrod): support shared memory on more platforms. +[!darwin] [!linux] [!windows] skip + +[short] skip +env GOCACHE=$WORK/cache + +# Fuzz cache should not exist after a regular test run. +go test . +exists $GOCACHE +! exists $GOCACHE/fuzz + +# Fuzzing should write interesting values to the cache. +go test -fuzz=FuzzY -fuzztime=100x . +go run ./contains_files $GOCACHE/fuzz/example.com/y/FuzzY + +# 'go clean -cache' should not delete the fuzz cache. +go clean -cache +exists $GOCACHE/fuzz + +# 'go clean -fuzzcache' should delete the fuzz cache but not the build cache. +go list -f {{.Stale}} ./empty +stdout true +go install ./empty +go list -f {{.Stale}} ./empty +stdout false +go clean -fuzzcache +! exists $GOCACHE/fuzz +go list -f {{.Stale}} ./empty +stdout false + +-- go.mod -- +module example.com/y + +go 1.16 +-- y_test.go -- +package y + +import ( + "io" + "testing" +) + +func FuzzY(f *testing.F) { + f.Add([]byte("y")) + f.Fuzz(func(t *testing.T, b []byte) { Y(io.Discard, b) }) +} +-- y.go -- +package y + +import ( + "bytes" + "io" +) + +func Y(w io.Writer, b []byte) { + if !bytes.Equal(b, []byte("y")) { + w.Write([]byte("not equal")) + } +} +-- empty/empty.go -- +package empty +-- contains_files/contains_files.go -- +package main + +import ( + "fmt" + "path/filepath" + "io/ioutil" + "os" +) + +func main() { + infos, err := ioutil.ReadDir(filepath.Clean(os.Args[1])) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + if len(infos) == 0 { + os.Exit(1) + } +} diff --git a/src/cmd/go/testdata/script/test_fuzz_chatty.txt b/src/cmd/go/testdata/script/test_fuzz_chatty.txt new file mode 100644 index 0000000000..9ebd480c90 --- /dev/null +++ b/src/cmd/go/testdata/script/test_fuzz_chatty.txt @@ -0,0 +1,106 @@ +# TODO(jayconrod): support shared memory on more platforms. +[!darwin] [!linux] [!windows] skip + +[short] skip + +# Run chatty fuzz targets with an error. +! go test -v chatty_error_fuzz_test.go +! stdout '^ok' +stdout 'FAIL' +stdout 'error in target' + +# Run chatty fuzz targets with a fatal. +! go test -v chatty_fatal_fuzz_test.go +! stdout '^ok' +stdout 'FAIL' +stdout 'fatal in target' + +# Run chatty fuzz target with a panic +! go test -v chatty_panic_fuzz_test.go +! stdout ^ok +stdout FAIL +stdout 'this is bad' + +# Run skipped chatty fuzz targets. +go test -v chatty_skipped_fuzz_test.go +stdout ok +stdout SKIP +! stdout FAIL + +# Run successful chatty fuzz targets. +go test -v chatty_fuzz_test.go +stdout ok +stdout PASS +stdout 'all good here' +! stdout FAIL + +# Fuzz successful chatty fuzz target that includes a separate unit test. +go test -v chatty_with_test_fuzz_test.go -fuzz=Fuzz -fuzztime=1x +stdout ok +stdout PASS +! stdout FAIL +# TODO: It's currently the case that it's logged twice. Fix that, and change +# this check to verify it. +stdout 'all good here' +# Verify that the unit test is only run once. +! stdout '(?s)logged foo.*logged foo' + +-- chatty_error_fuzz_test.go -- +package chatty_error_fuzz + +import "testing" + +func Fuzz(f *testing.F) { + f.Error("error in target") +} + +-- chatty_fatal_fuzz_test.go -- +package chatty_fatal_fuzz + +import "testing" + +func Fuzz(f *testing.F) { + f.Fatal("fatal in target") +} + +-- chatty_panic_fuzz_test.go -- +package chatty_panic_fuzz + +import "testing" + +func Fuzz(f *testing.F) { + panic("this is bad") +} + +-- chatty_skipped_fuzz_test.go -- +package chatty_skipped_fuzz + +import "testing" + +func Fuzz(f *testing.F) { + f.Skip() +} + +-- chatty_fuzz_test.go -- +package chatty_fuzz + +import "testing" + +func Fuzz(f *testing.F) { + f.Log("all good here") + f.Fuzz(func(*testing.T, []byte) {}) +} + +-- chatty_with_test_fuzz_test.go -- +package chatty_with_test_fuzz + +import "testing" + +func TestFoo(t *testing.T) { + t.Log("logged foo") +} + +func Fuzz(f *testing.F) { + f.Log("all good here") + f.Fuzz(func(*testing.T, []byte) {}) +} diff --git a/src/cmd/go/testdata/script/test_fuzz_cleanup.txt b/src/cmd/go/testdata/script/test_fuzz_cleanup.txt new file mode 100644 index 0000000000..88625916ba --- /dev/null +++ b/src/cmd/go/testdata/script/test_fuzz_cleanup.txt @@ -0,0 +1,67 @@ +# TODO(jayconrod): support shared memory on more platforms. +[!darwin] [!linux] [!windows] skip +[short] skip + +# Cleanup should run after F.Skip. +go test -run=FuzzTargetSkip +stdout cleanup + +# Cleanup should run after F.Fatal. +! go test -run=FuzzTargetFatal +stdout cleanup + +# Cleanup should run after an unexpected runtime.Goexit. +! go test -run=FuzzTargetGoexit +stdout cleanup + +# Cleanup should run after panic. +! go test -run=FuzzTargetPanic +stdout cleanup + +# Cleanup should run in fuzz function on seed corpus. +go test -v -run=FuzzFunction +stdout '(?s)inner.*outer' + +# TODO(jayconrod): test cleanup while fuzzing. For now, the worker process's +# stdout and stderr is connected to the coordinator's, but it should eventually +# be connected to os.DevNull, so we wouldn't see t.Log output. + +-- go.mod -- +module cleanup + +go 1.15 +-- cleanup_test.go -- +package cleanup + +import ( + "runtime" + "testing" +) + +func FuzzTargetSkip(f *testing.F) { + f.Cleanup(func() { f.Log("cleanup") }) + f.Skip() +} + +func FuzzTargetFatal(f *testing.F) { + f.Cleanup(func() { f.Log("cleanup") }) + f.Fatal() +} + +func FuzzTargetGoexit(f *testing.F) { + f.Cleanup(func() { f.Log("cleanup") }) + runtime.Goexit() +} + +func FuzzTargetPanic(f *testing.F) { + f.Cleanup(func() { f.Log("cleanup") }) + panic("oh no") +} + +func FuzzFunction(f *testing.F) { + f.Add([]byte{0}) + f.Cleanup(func() { f.Log("outer") }) + f.Fuzz(func(t *testing.T, b []byte) { + t.Cleanup(func() { t.Logf("inner") }) + }) +} diff --git a/src/cmd/go/testdata/script/test_fuzz_deadline.txt b/src/cmd/go/testdata/script/test_fuzz_deadline.txt new file mode 100644 index 0000000000..12f1054f61 --- /dev/null +++ b/src/cmd/go/testdata/script/test_fuzz_deadline.txt @@ -0,0 +1,37 @@ +# TODO(jayconrod): support shared memory on more platforms. +[!darwin] [!linux] [!windows] skip + +[short] skip + +# The fuzz function should be able to detect whether -timeout +# was set with T.Deadline. Note there is no F.Deadline, and +# there is no timeout while fuzzing, even if -fuzztime is set. +go test -run=FuzzDeadline -wantdeadline=true # -timeout defaults to 10m +go test -run=FuzzDeadline -timeout=0 -wantdeadline=false +! go test -run=FuzzDeadline -timeout=1s -wantdeadline=false +go test -run=FuzzDeadline -timeout=1s -wantdeadline=true +go test -fuzz=FuzzDeadline -timeout=0 -fuzztime=1s -wantdeadline=false +go test -fuzz=FuzzDeadline -timeout=0 -fuzztime=100x -wantdeadline=false + +-- go.mod -- +module fuzz + +go 1.16 +-- fuzz_deadline_test.go -- +package fuzz_test + +import ( + "flag" + "testing" +) + +var wantDeadline = flag.Bool("wantdeadline", false, "whether the test should have a deadline") + +func FuzzDeadline(f *testing.F) { + f.Add("run once") + f.Fuzz(func (t *testing.T, _ string) { + if _, hasDeadline := t.Deadline(); hasDeadline != *wantDeadline { + t.Fatalf("function got %v; want %v", hasDeadline, *wantDeadline) + } + }) +} diff --git a/src/cmd/go/testdata/script/test_fuzz_fuzztime.txt b/src/cmd/go/testdata/script/test_fuzz_fuzztime.txt new file mode 100644 index 0000000000..7d644b4d13 --- /dev/null +++ b/src/cmd/go/testdata/script/test_fuzz_fuzztime.txt @@ -0,0 +1,82 @@ +# TODO(jayconrod): support shared memory on more platforms. +[!darwin] [!linux] [!windows] skip + +[short] skip + +# There are no seed values, so 'go test' should finish quickly. +go test + +# Fuzzing should exit 0 after fuzztime, even if timeout is short. +go test -timeout=10ms -fuzz=FuzzFast -fuzztime=5s + +# We should see the same behavior when invoking the test binary directly. +go test -c +exec ./fuzz.test$GOEXE -test.timeout=10ms -test.fuzz=FuzzFast -test.fuzztime=5s -test.parallel=1 -test.fuzzcachedir=$WORK/cache + +# Timeout should not cause inputs to be written as crashers. +! exists testdata/fuzz + +# When we use fuzztime with an "x" suffix, it runs a specific number of times. +# This fuzz function creates a file with a unique name ($pid.$count) on each run. +# We count the files to find the number of runs. +mkdir count +env GOCACHE=$WORK/tmp +go test -fuzz=FuzzCount -fuzztime=1000x -fuzzminimizetime=1x +go run check_file_count.go 1000 + +-- go.mod -- +module fuzz + +go 1.16 +-- fuzz_fast_test.go -- +package fuzz_test + +import "testing" + +func FuzzFast(f *testing.F) { + f.Fuzz(func (*testing.T, []byte) {}) +} +-- fuzz_count_test.go -- +package fuzz + +import ( + "fmt" + "os" + "testing" +) + +func FuzzCount(f *testing.F) { + pid := os.Getpid() + n := 0 + f.Fuzz(func(t *testing.T, _ []byte) { + name := fmt.Sprintf("count/%v.%d", pid, n) + if err := os.WriteFile(name, nil, 0666); err != nil { + t.Fatal(err) + } + n++ + }) +} +-- check_file_count.go -- +// +build ignore + +package main + +import ( + "fmt" + "os" + "strconv" +) + +func main() { + dir, err := os.ReadDir("count") + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + got := len(dir) + want, _ := strconv.Atoi(os.Args[1]) + if got != want { + fmt.Fprintf(os.Stderr, "got %d files; want %d\n", got, want) + os.Exit(1) + } +} diff --git a/src/cmd/go/testdata/script/test_fuzz_io_error.txt b/src/cmd/go/testdata/script/test_fuzz_io_error.txt new file mode 100644 index 0000000000..4c7ab4c152 --- /dev/null +++ b/src/cmd/go/testdata/script/test_fuzz_io_error.txt @@ -0,0 +1,101 @@ +# Test that when the coordinator experiences an I/O error communicating +# with a worker, the coordinator stops the worker and reports the error. +# The coordinator should not record a crasher. +# +# We simulate an I/O error in the test by writing garbage to fuzz_out. +# This is unlikely, but possible. It's difficult to simulate interruptions +# due to ^C and EOF errors which are more common. We don't report those. +[short] skip +[!darwin] [!linux] [!windows] skip + +# If the I/O error occurs before F.Fuzz is called, the coordinator should +# stop the worker and say that. +! go test -fuzz=FuzzClosePipeBefore -parallel=1 +stdout '\s*fuzzing process terminated without fuzzing:' +! stdout 'communicating with fuzzing process' +! exists testdata + +# If the I/O error occurs after F.Fuzz is called (unlikely), just exit. +# It's hard to distinguish this case from the worker being interrupted by ^C +# or exiting with status 0 (which it should do when interrupted by ^C). +! go test -fuzz=FuzzClosePipeAfter -parallel=1 +stdout '^\s*communicating with fuzzing process: invalid character ''!'' looking for beginning of value$' +! exists testdata + +-- go.mod -- +module test + +go 1.17 +-- io_error_test.go -- +package io_error + +import ( + "flag" + "testing" + "time" +) + +func isWorker() bool { + f := flag.Lookup("test.fuzzworker") + if f == nil { + return false + } + get, ok := f.Value.(flag.Getter) + if !ok { + return false + } + return get.Get() == interface{}(true) +} + +func FuzzClosePipeBefore(f *testing.F) { + if isWorker() { + sendGarbageToCoordinator(f) + time.Sleep(3600 * time.Second) // pause until coordinator terminates the process + } + f.Fuzz(func(*testing.T, []byte) {}) +} + +func FuzzClosePipeAfter(f *testing.F) { + f.Fuzz(func(t *testing.T, _ []byte) { + if isWorker() { + sendGarbageToCoordinator(t) + time.Sleep(3600 * time.Second) // pause until coordinator terminates the process + } + }) +} +-- io_error_windows_test.go -- +package io_error + +import ( + "fmt" + "os" + "testing" +) + +func sendGarbageToCoordinator(tb testing.TB) { + v := os.Getenv("GO_TEST_FUZZ_WORKER_HANDLES") + var fuzzInFD, fuzzOutFD uintptr + if _, err := fmt.Sscanf(v, "%x,%x", &fuzzInFD, &fuzzOutFD); err != nil { + tb.Fatalf("parsing GO_TEST_FUZZ_WORKER_HANDLES: %v", err) + } + f := os.NewFile(fuzzOutFD, "fuzz_out") + if _, err := f.Write([]byte("!!")); err != nil { + tb.Fatalf("writing fuzz_out: %v", err) + } +} +-- io_error_notwindows_test.go -- +// +build !windows + +package io_error + +import ( + "os" + "testing" +) + +func sendGarbageToCoordinator(tb testing.TB) { + f := os.NewFile(4, "fuzz_out") + if _, err := f.Write([]byte("!!")); err != nil { + tb.Fatalf("writing fuzz_out: %v", err) + } +} diff --git a/src/cmd/go/testdata/script/test_fuzz_match.txt b/src/cmd/go/testdata/script/test_fuzz_match.txt new file mode 100644 index 0000000000..3a2ca631ad --- /dev/null +++ b/src/cmd/go/testdata/script/test_fuzz_match.txt @@ -0,0 +1,39 @@ +# TODO(jayconrod): support shared memory on more platforms. +[!darwin] [!linux] [!windows] skip + +# Matches only fuzz targets to test. +go test standalone_fuzz_test.go +! stdout '^ok.*\[no tests to run\]' +stdout '^ok' + +# Matches only for fuzzing. +go test -fuzz Fuzz -fuzztime 1x standalone_fuzz_test.go +! stdout '^ok.*\[no tests to run\]' +stdout '^ok' + +# Matches none for fuzzing but will run the fuzz target as a test. +go test -fuzz ThisWillNotMatch -fuzztime 1x standalone_fuzz_test.go +! stdout '^ok.*no tests to run' +stdout '^ok' +stdout 'no targets to fuzz' + +[short] stop + +# Matches only fuzz targets to test with -run. +go test -run Fuzz standalone_fuzz_test.go +! stdout '^ok.*\[no tests to run\]' +stdout '^ok' + +# Matches no fuzz targets. +go test -run ThisWillNotMatch standalone_fuzz_test.go +stdout '^ok.*no tests to run' +! stdout 'no targets to fuzz' + +-- standalone_fuzz_test.go -- +package standalone_fuzz + +import "testing" + +func Fuzz(f *testing.F) { + f.Fuzz(func (*testing.T, []byte) {}) +} diff --git a/src/cmd/go/testdata/script/test_fuzz_minimize.txt b/src/cmd/go/testdata/script/test_fuzz_minimize.txt new file mode 100644 index 0000000000..f0adb9ec3e --- /dev/null +++ b/src/cmd/go/testdata/script/test_fuzz_minimize.txt @@ -0,0 +1,200 @@ +# TODO(jayconrod): support shared memory on more platforms. +[!darwin] [!linux] [!windows] skip + +[short] skip + +# We clean the fuzz cache during this test. Don't clean the user's cache. +env GOCACHE=$WORK/gocache + +# Test that fuzzminimizetime cannot be negative seconds +! go test -fuzz=FuzzMinimizerRecoverable -run=FuzzMinimizerRecoverable -fuzztime=10000x -fuzzminimizetime=-1ms minimizer_test.go +! stdout '^ok' +! stdout 'contains a non-zero byte' +stdout 'invalid duration' +stdout FAIL + +# Test that fuzzminimizetime cannot be negative times +! go test -fuzz=FuzzMinimizerRecoverable -run=FuzzMinimizerRecoverable -fuzztime=10000x -fuzzminimizetime=-1x minimizer_test.go +! stdout '^ok' +! stdout 'contains a non-zero byte' +stdout 'invalid count' +stdout FAIL + +# Test that fuzzminimizetime can be zero seconds, and minimization is disabled +! go test -fuzz=FuzzMinimizeZeroDurationSet -run=FuzzMinimizeZeroDurationSet -fuzztime=10000x -fuzzminimizetime=0s minimizer_test.go +! stdout '^ok' +! stdout 'minimizing' +stdout 'there was an Error' +stdout FAIL + +# Test that fuzzminimizetime can be zero times, and minimization is disabled +! go test -fuzz=FuzzMinimizeZeroLimitSet -run=FuzzMinimizeZeroLimitSet -fuzztime=10000x -fuzzminimizetime=0x minimizer_test.go +! stdout '^ok' +! stdout 'minimizing' +stdout 'there was an Error' +stdout FAIL + +# Test that minimization is working for recoverable errors. +! go test -fuzz=FuzzMinimizerRecoverable -run=FuzzMinimizerRecoverable -fuzztime=10000x minimizer_test.go +! stdout '^ok' +stdout 'got the minimum size!' +stdout 'contains a non-zero byte' +stdout FAIL + +# Check that the bytes written to testdata are of length 50 (the minimum size) +go run check_testdata.go FuzzMinimizerRecoverable 50 + +# Test that re-running the minimized value causes a crash. +! go test -run=FuzzMinimizerRecoverable minimizer_test.go +rm testdata + +# Test that minimization is working for non-recoverable errors. +! go test -fuzz=FuzzMinimizerNonrecoverable -run=FuzzMinimizerNonrecoverable -fuzztime=10000x minimizer_test.go +! stdout '^ok' +stdout 'minimizing' +stdout 'fuzzing process terminated unexpectedly: exit status 99' +stdout FAIL + +# Check that re-running the value causes a crash. +! go test -run=FuzzMinimizerNonrecoverable minimizer_test.go +rm testdata + +# Clear the fuzzing cache. There may already be minimized inputs that would +# interfere with the next stage of the test. +go clean -fuzzcache + +# Test that minimization can be cancelled by fuzzminimizetime and the latest +# crash will still be logged and written to testdata. +! go test -fuzz=FuzzMinimizerRecoverable -run=FuzzMinimizerRecoverable -fuzztime=100x -fuzzminimizetime=1x minimizer_test.go +! stdout '^ok' +stdout 'testdata[/\\]fuzz[/\\]FuzzMinimizerRecoverable[/\\]' +! stdout 'got the minimum size!' # it shouldn't have had enough time to minimize it +stdout FAIL + +# Test that re-running the unminimized value causes a crash. +! go test -run=FuzzMinimizerRecoverable minimizer_test.go + +# TODO(jayconrod,katiehockman): add a test which verifies that the right bytes +# are written to testdata in the case of an interrupt during minimization. + +-- go.mod -- +module m + +go 1.16 +-- minimizer_test.go -- +package fuzz_test + +import ( + "os" + "testing" +) + +func FuzzMinimizeZeroDurationSet(f *testing.F) { + f.Fuzz(func(t *testing.T, b []byte) { + if len(b) > 5 { + t.Errorf("there was an Error") + } + }) +} + +func FuzzMinimizeZeroLimitSet(f *testing.F) { + f.Fuzz(func(t *testing.T, b []byte) { + if len(b) > 5 { + t.Errorf("there was an Error") + } + }) +} + +func FuzzMinimizerRecoverable(f *testing.F) { + f.Add(make([]byte, 100)) + f.Fuzz(func(t *testing.T, b []byte) { + if len(b) < 50 { + // Make sure that b is large enough that it can be minimized + return + } + // Given the randomness of the mutations, this should allow the + // minimizer to trim down the value a bit. + for _, n := range b { + if n != 0 { + if len(b) == 50 { + t.Log("got the minimum size!") + } + t.Fatal("contains a non-zero byte") + } + } + }) +} + +func FuzzMinimizerNonrecoverable(f *testing.F) { + f.Add(make([]byte, 100)) + f.Fuzz(func(t *testing.T, b []byte) { + if len(b) < 50 { + // Make sure that b is large enough that it can be minimized + return + } + // Given the randomness of the mutations, this should allow the + // minimizer to trim down the value a bit. + for _, n := range b { + if n != 0 { + t.Log("contains a non-zero byte") + os.Exit(99) + } + } + }) +} +-- check_testdata.go -- +// +build ignore + +package main + +import ( + "bytes" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strconv" +) + +func main() { + target := os.Args[1] + numBytes, err := strconv.Atoi(os.Args[2]) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + + // Open the file in testdata (there should only be one) + dir := fmt.Sprintf("testdata/fuzz/%s", target) + files, err := ioutil.ReadDir(dir) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + if len(files) != 1 { + fmt.Fprintf(os.Stderr, "expected one file, got %d", len(files)) + os.Exit(1) + } + got, err := ioutil.ReadFile(filepath.Join(dir, files[0].Name())) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + + // Trim the newline at the end of the file + got = bytes.TrimSpace(got) + + // Make sure that there were exactly 100 bytes written to the corpus entry + prefix := []byte("[]byte(") + i := bytes.Index(got, prefix) + gotBytes := got[i+len(prefix) : len(got)-1] + s, err := strconv.Unquote(string(gotBytes)) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + if want, got := numBytes, len(s); want != got { + fmt.Fprintf(os.Stderr, "want %d bytes, got %d\n", want, got) + os.Exit(1) + } +} diff --git a/src/cmd/go/testdata/script/test_fuzz_minimize_interesting.txt b/src/cmd/go/testdata/script/test_fuzz_minimize_interesting.txt new file mode 100644 index 0000000000..5e1d90d8d9 --- /dev/null +++ b/src/cmd/go/testdata/script/test_fuzz_minimize_interesting.txt @@ -0,0 +1,112 @@ +# TODO(jayconrod): support shared memory on more platforms. +[!darwin] [!linux] [!windows] skip + +# Instrumentation only supported on 64-bit architectures. +[!amd64] [!arm64] skip + +# Test that when an interesting value is discovered (one that expands coverage), +# the fuzzing engine minimizes it before writing it to the cache. +# +# The program below starts with a seed value of length 100, but more coverage +# will be found for any value other than the seed. We should end with a value +# in the cache of length 1 (the minimizer currently does not produce empty +# strings). check_cache.go confirms that. +# +# We would like to verify that ALL values in the cache were minimized to a +# length of 1, but this isn't always possible when new coverage is found in +# functions called by testing or internal/fuzz in the background. + +go test -c -fuzz=. # Build using shared build cache for speed. +env GOCACHE=$WORK/gocache +exec ./fuzz.test$GOEXE -test.fuzzcachedir=$GOCACHE/fuzz -test.fuzz=. -test.fuzztime=1000x +go run check_cache.go $GOCACHE/fuzz/FuzzMin + +-- go.mod -- +module fuzz + +go 1.17 +-- fuzz_test.go -- +package fuzz + +import ( + "bytes" + "testing" +) + +func FuzzMin(f *testing.F) { + seed := bytes.Repeat([]byte("a"), 20) + f.Add(seed) + f.Fuzz(func(t *testing.T, buf []byte) { + if bytes.Equal(buf, seed) { + return + } + if n := sum(buf); n < 0 { + t.Error("sum cannot be negative") + } + }) +} + +func sum(buf []byte) int { + n := 0 + for _, b := range buf { + n += int(b) + } + return n +} +-- check_cache.go -- +//go:build ignore +// +build ignore + +// check_cache.go checks that each file in the cached corpus has a []byte +// of length at most 1. This verifies that at least one cached input is minimized. +package main + +import ( + "bytes" + "fmt" + "os" + "path/filepath" + "regexp" + "strconv" +) + +func main() { + dir := os.Args[1] + ents, err := os.ReadDir(dir) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + for _, ent := range ents { + name := filepath.Join(dir, ent.Name()) + if good, err := checkCacheFile(name); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } else if good { + os.Exit(0) + } + } + fmt.Fprintln(os.Stderr, "no cached inputs were minimized") + os.Exit(1) +} + +func checkCacheFile(name string) (good bool, err error) { + data, err := os.ReadFile(name) + if err != nil { + return false, err + } + for _, line := range bytes.Split(data, []byte("\n")) { + m := valRe.FindSubmatch(line) + if m == nil { + continue + } + if s, err := strconv.Unquote(string(m[1])); err != nil { + return false, err + } else if len(s) <= 1 { + return true, nil + } + } + return false, nil +} + +var valRe = regexp.MustCompile(`^\[\]byte\(([^)]+)\)$`) diff --git a/src/cmd/go/testdata/script/test_fuzz_multiple.txt b/src/cmd/go/testdata/script/test_fuzz_multiple.txt new file mode 100644 index 0000000000..6a7732f514 --- /dev/null +++ b/src/cmd/go/testdata/script/test_fuzz_multiple.txt @@ -0,0 +1,51 @@ +# This test checks that 'go test' prints a reasonable error when fuzzing is +# enabled, and multiple package or multiple fuzz targets match. +# TODO(#46312): support fuzzing multiple targets in multiple packages. + +# TODO(jayconrod): support shared memory on more platforms. +[!darwin] [!linux] [!windows] skip + +[short] skip + +# With fuzzing disabled, multiple targets can be tested. +go test ./... + +# With fuzzing enabled, at most one package may be tested, +# even if only one package contains fuzz targets. +! go test -fuzz=. ./... +stderr '^cannot use -fuzz flag with multiple packages$' +! go test -fuzz=. ./zero ./one +stderr '^cannot use -fuzz flag with multiple packages$' +go test -fuzz=. -fuzztime=1x ./one + +# With fuzzing enabled, at most one target in the same package may match. +! go test -fuzz=. ./two +stdout '^testing: will not fuzz, -fuzz matches more than one target: \[FuzzOne FuzzTwo\]$' +go test -fuzz=FuzzTwo -fuzztime=1x ./two + +-- go.mod -- +module fuzz + +go 1.18 +-- zero/zero.go -- +package zero +-- one/one_test.go -- +package one + +import "testing" + +func FuzzOne(f *testing.F) { + f.Fuzz(func(*testing.T, []byte) {}) +} +-- two/two_test.go -- +package two + +import "testing" + +func FuzzOne(f *testing.F) { + f.Fuzz(func(*testing.T, []byte) {}) +} + +func FuzzTwo(f *testing.F) { + f.Fuzz(func(*testing.T, []byte) {}) +} diff --git a/src/cmd/go/testdata/script/test_fuzz_mutate_crash.txt b/src/cmd/go/testdata/script/test_fuzz_mutate_crash.txt new file mode 100644 index 0000000000..1b8b79b3dd --- /dev/null +++ b/src/cmd/go/testdata/script/test_fuzz_mutate_crash.txt @@ -0,0 +1,295 @@ +# TODO(jayconrod): support shared memory on more platforms. +[!darwin] [!linux] [!windows] skip + +# Tests that a crash caused by a mutator-discovered input writes the bad input +# to testdata, and fails+reports correctly. This tests the end-to-end behavior +# of the mutator finding a crash while fuzzing, adding it as a regression test +# to the seed corpus in testdata, and failing the next time the test is run. + +[short] skip + +# Running the seed corpus for all of the targets should pass the first +# time, since nothing in the seed corpus will cause a crash. +go test + +# Running the fuzzer should find a crashing input quickly. +! go test -fuzz=FuzzWithBug -fuzztime=100x -fuzzminimizetime=1000x +stdout 'testdata[/\\]fuzz[/\\]FuzzWithBug[/\\]' +stdout 'this input caused a crash!' +go run check_testdata.go FuzzWithBug + +# Now, the failing bytes should have been added to the seed corpus for +# the target, and should fail when run without fuzzing. +! go test +stdout 'testdata[/\\]fuzz[/\\]FuzzWithBug[/\\][a-f0-9]{64}' +stdout 'this input caused a crash!' + +! go test -run=FuzzWithNilPanic -fuzz=FuzzWithNilPanic -fuzztime=100x -fuzzminimizetime=1000x +stdout 'testdata[/\\]fuzz[/\\]FuzzWithNilPanic[/\\]' +stdout 'runtime.Goexit' +go run check_testdata.go FuzzWithNilPanic + +! go test -run=FuzzWithFail -fuzz=FuzzWithFail -fuzztime=100x -fuzzminimizetime=1000x +stdout 'testdata[/\\]fuzz[/\\]FuzzWithFail[/\\]' +go run check_testdata.go FuzzWithFail + +! go test -run=FuzzWithLogFail -fuzz=FuzzWithLogFail -fuzztime=100x -fuzzminimizetime=1000x +stdout 'testdata[/\\]fuzz[/\\]FuzzWithLogFail[/\\]' +stdout 'logged something' +go run check_testdata.go FuzzWithLogFail + +! go test -run=FuzzWithErrorf -fuzz=FuzzWithErrorf -fuzztime=100x -fuzzminimizetime=1000x +stdout 'testdata[/\\]fuzz[/\\]FuzzWithErrorf[/\\]' +stdout 'errorf was called here' +go run check_testdata.go FuzzWithErrorf + +! go test -run=FuzzWithFatalf -fuzz=FuzzWithFatalf -fuzztime=100x -fuzzminimizetime=1000x +stdout 'testdata[/\\]fuzz[/\\]FuzzWithFatalf[/\\]' +stdout 'fatalf was called here' +go run check_testdata.go FuzzWithFatalf + +! go test -run=FuzzWithBadExit -fuzz=FuzzWithBadExit -fuzztime=100x -fuzzminimizetime=1000x +stdout 'testdata[/\\]fuzz[/\\]FuzzWithBadExit[/\\]' +stdout 'unexpectedly' +go run check_testdata.go FuzzWithBadExit + +# Running the fuzzer should find a crashing input quickly for fuzzing two types. +! go test -run=FuzzWithTwoTypes -fuzz=FuzzWithTwoTypes -fuzztime=100x -fuzzminimizetime=1000x +stdout 'testdata[/\\]fuzz[/\\]FuzzWithTwoTypes[/\\]' +stdout 'these inputs caused a crash!' +go run check_testdata.go FuzzWithTwoTypes + +# Running the fuzzer should find a crashing input quickly for an integer. +! go test -run=FuzzInt -fuzz=FuzzInt -fuzztime=100x -fuzzminimizetime=1000x +stdout 'testdata[/\\]fuzz[/\\]FuzzInt[/\\]' +stdout 'this input caused a crash!' +go run check_testdata.go FuzzInt + +! go test -run=FuzzUint -fuzz=FuzzUint -fuzztime=100x -fuzzminimizetime=1000x +stdout 'testdata[/\\]fuzz[/\\]FuzzUint[/\\]' +stdout 'this input caused a crash!' +go run check_testdata.go FuzzUint + +# Running the fuzzer should find a crashing input quickly for a bool. +! go test -run=FuzzBool -fuzz=FuzzBool -fuzztime=100x -fuzzminimizetime=1000x +stdout 'testdata[/\\]fuzz[/\\]FuzzBool[/\\]' +stdout 'this input caused a crash!' +go run check_testdata.go FuzzBool + +# Running the fuzzer should find a crashing input quickly for a float. +! go test -run=FuzzFloat -fuzz=FuzzFloat -fuzztime=100x -fuzzminimizetime=1000x +stdout 'testdata[/\\]fuzz[/\\]FuzzFloat[/\\]' +stdout 'this input caused a crash!' +go run check_testdata.go FuzzFloat + +# Running the fuzzer should find a crashing input quickly for a byte. +! go test -run=FuzzByte -fuzz=FuzzByte -fuzztime=100x -fuzzminimizetime=1000x +stdout 'testdata[/\\]fuzz[/\\]FuzzByte[/\\]' +stdout 'this input caused a crash!' +go run check_testdata.go FuzzByte + +# Running the fuzzer should find a crashing input quickly for a rune. +! go test -run=FuzzRune -fuzz=FuzzRune -fuzztime=100x -fuzzminimizetime=1000x +stdout 'testdata[/\\]fuzz[/\\]FuzzRune[/\\]' +stdout 'this input caused a crash!' +go run check_testdata.go FuzzRune + +# Running the fuzzer should find a crashing input quickly for a string. +! go test -run=FuzzString -fuzz=FuzzString -fuzztime=100x -fuzzminimizetime=1000x +stdout 'testdata[/\\]fuzz[/\\]FuzzString[/\\]' +stdout 'this input caused a crash!' +go run check_testdata.go FuzzString + +-- go.mod -- +module m + +go 1.16 +-- fuzz_crash_test.go -- +package fuzz_crash + +import ( + "os" + "testing" +) + +func FuzzWithBug(f *testing.F) { + f.Add([]byte("aa")) + f.Fuzz(func(t *testing.T, b []byte) { + if string(b) != "aa" { + panic("this input caused a crash!") + } + }) +} + +func FuzzWithNilPanic(f *testing.F) { + f.Add([]byte("aa")) + f.Fuzz(func(t *testing.T, b []byte) { + if string(b) != "aa" { + panic(nil) + } + }) +} + +func FuzzWithFail(f *testing.F) { + f.Add([]byte("aa")) + f.Fuzz(func(t *testing.T, b []byte) { + if string(b) != "aa" { + t.Fail() + } + }) +} + +func FuzzWithLogFail(f *testing.F) { + f.Add([]byte("aa")) + f.Fuzz(func(t *testing.T, b []byte) { + if string(b) != "aa" { + t.Log("logged something") + t.Fail() + } + }) +} + +func FuzzWithErrorf(f *testing.F) { + f.Add([]byte("aa")) + f.Fuzz(func(t *testing.T, b []byte) { + if string(b) != "aa" { + t.Errorf("errorf was called here") + } + }) +} + +func FuzzWithFatalf(f *testing.F) { + f.Add([]byte("aa")) + f.Fuzz(func(t *testing.T, b []byte) { + if string(b) != "aa" { + t.Fatalf("fatalf was called here") + } + }) +} + +func FuzzWithBadExit(f *testing.F) { + f.Add([]byte("aa")) + f.Fuzz(func(t *testing.T, b []byte) { + if string(b) != "aa" { + os.Exit(1) + } + }) +} + +func FuzzWithTwoTypes(f *testing.F) { + f.Fuzz(func(t *testing.T, a, b []byte) { + if len(a) > 0 && len(b) > 0 { + panic("these inputs caused a crash!") + } + }) +} + +func FuzzInt(f *testing.F) { + f.Add(0) + f.Fuzz(func(t *testing.T, a int) { + if a != 0 { + panic("this input caused a crash!") + } + }) +} + +func FuzzUint(f *testing.F) { + f.Add(uint(0)) + f.Fuzz(func(t *testing.T, a uint) { + if a != 0 { + panic("this input caused a crash!") + } + }) +} + +func FuzzBool(f *testing.F) { + f.Add(false) + f.Fuzz(func(t *testing.T, a bool) { + if a { + panic("this input caused a crash!") + } + }) +} + +func FuzzFloat(f *testing.F) { + f.Fuzz(func(t *testing.T, a float64) { + if a != float64(int64(a)) { + // It has a decimal, so it was mutated by division + panic("this input caused a crash!") + } + }) +} + +func FuzzByte(f *testing.F) { + f.Add(byte(0)) + f.Fuzz(func(t *testing.T, a byte) { + if a != 0 { + panic("this input caused a crash!") + } + }) +} + +func FuzzRune(f *testing.F) { + f.Add(rune(0)) + f.Fuzz(func(t *testing.T, a rune) { + if a != 0 { + panic("this input caused a crash!") + } + }) +} + +func FuzzString(f *testing.F) { + f.Add("") + f.Fuzz(func(t *testing.T, a string) { + if a != "" { + panic("this input caused a crash!") + } + }) +} + +-- check_testdata.go -- +// +build ignore + +package main + +import ( + "bytes" + "crypto/sha256" + "fmt" + "io/ioutil" + "os" + "path/filepath" +) + +func main() { + target := os.Args[1] + dir := filepath.Join("testdata/fuzz", target) + + files, err := ioutil.ReadDir(dir) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + + if len(files) == 0 { + fmt.Fprintf(os.Stderr, "expect at least one new mutation to be written to testdata\n") + os.Exit(1) + } + + fname := files[0].Name() + contents, err := ioutil.ReadFile(filepath.Join(dir, fname)) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + if bytes.Equal(contents, []byte("aa")) { + fmt.Fprintf(os.Stderr, "newly written testdata entry was not mutated\n") + os.Exit(1) + } + // The hash of the bytes in the file should match the filename. + h := []byte(fmt.Sprintf("%x", sha256.Sum256(contents))) + if !bytes.Equal([]byte(fname), h) { + fmt.Fprintf(os.Stderr, "hash of bytes %q does not match filename %q\n", h, fname) + os.Exit(1) + } +} diff --git a/src/cmd/go/testdata/script/test_fuzz_mutate_fail.txt b/src/cmd/go/testdata/script/test_fuzz_mutate_fail.txt new file mode 100644 index 0000000000..935c22a05e --- /dev/null +++ b/src/cmd/go/testdata/script/test_fuzz_mutate_fail.txt @@ -0,0 +1,103 @@ +# TODO(jayconrod): support shared memory on more platforms. +[!darwin] [!linux] [!windows] skip + +# Check that if a worker does not call F.Fuzz or calls F.Fail first, +# 'go test' exits non-zero and no crasher is recorded. + +[short] skip + +! go test -fuzz=FuzzReturn +! exists testdata + +! go test -fuzz=FuzzSkip +! exists testdata + +! go test -fuzz=FuzzFail +! exists testdata + +! go test -fuzz=FuzzPanic +! exists testdata + +! go test -fuzz=FuzzNilPanic +! exists testdata + +! go test -fuzz=FuzzGoexit +! exists testdata + +! go test -fuzz=FuzzExit +! exists testdata + +-- go.mod -- +module m + +go 1.17 +-- fuzz_fail_test.go -- +package fuzz_fail + +import ( + "flag" + "os" + "runtime" + "testing" +) + +func isWorker() bool { + f := flag.Lookup("test.fuzzworker") + if f == nil { + return false + } + get, ok := f.Value.(flag.Getter) + if !ok { + return false + } + return get.Get() == interface{}(true) +} + +func FuzzReturn(f *testing.F) { + if isWorker() { + return + } + f.Fuzz(func(*testing.T, []byte) {}) +} + +func FuzzSkip(f *testing.F) { + if isWorker() { + f.Skip() + } + f.Fuzz(func(*testing.T, []byte) {}) +} + +func FuzzFail(f *testing.F) { + if isWorker() { + f.Fail() + } + f.Fuzz(func(*testing.T, []byte) {}) +} + +func FuzzPanic(f *testing.F) { + if isWorker() { + panic("nope") + } + f.Fuzz(func(*testing.T, []byte) {}) +} + +func FuzzNilPanic(f *testing.F) { + if isWorker() { + panic(nil) + } + f.Fuzz(func(*testing.T, []byte) {}) +} + +func FuzzGoexit(f *testing.F) { + if isWorker() { + runtime.Goexit() + } + f.Fuzz(func(*testing.T, []byte) {}) +} + +func FuzzExit(f *testing.F) { + if isWorker() { + os.Exit(99) + } + f.Fuzz(func(*testing.T, []byte) {}) +} diff --git a/src/cmd/go/testdata/script/test_fuzz_mutator.txt b/src/cmd/go/testdata/script/test_fuzz_mutator.txt new file mode 100644 index 0000000000..9d0738e169 --- /dev/null +++ b/src/cmd/go/testdata/script/test_fuzz_mutator.txt @@ -0,0 +1,166 @@ +# TODO(jayconrod): support shared memory on more platforms. +[!darwin] [!linux] [!windows] skip + +# Test basic fuzzing mutator behavior. +# +# fuzz_test.go has two fuzz targets (FuzzA, FuzzB) which both add a seed value. +# Each fuzz function writes the input to a log file. The coordinator and worker +# use separate log files. check_logs.go verifies that the coordinator only +# tests seed values and the worker tests mutated values on the fuzz target. + +[short] skip + +go test -fuzz=FuzzA -fuzztime=100x -parallel=1 -log=fuzz +go run check_logs.go fuzz fuzz.worker + +# TODO(b/181800488): remove -parallel=1, here and below. For now, when a +# crash is found, all workers keep running, wasting resources and reducing +# the number of executions available to the minimizer, increasing flakiness. + +# Test that the mutator is good enough to find several unique mutations. +! go test -fuzz=FuzzMutator -parallel=1 -fuzztime=100x mutator_test.go +! stdout '^ok' +stdout FAIL +stdout 'mutator found enough unique mutations' + +-- go.mod -- +module m + +go 1.16 +-- fuzz_test.go -- +package fuzz_test + +import ( + "flag" + "fmt" + "os" + "testing" +) + +var ( + logPath = flag.String("log", "", "path to log file") + logFile *os.File +) + +func TestMain(m *testing.M) { + flag.Parse() + var err error + logFile, err = os.OpenFile(*logPath, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0666) + if os.IsExist(err) { + *logPath += ".worker" + logFile, err = os.OpenFile(*logPath, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0666) + } + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + os.Exit(m.Run()) +} + +func FuzzA(f *testing.F) { + f.Add([]byte("seed")) + f.Fuzz(func(t *testing.T, b []byte) { + fmt.Fprintf(logFile, "FuzzA %q\n", b) + }) +} + +func FuzzB(f *testing.F) { + f.Add([]byte("seed")) + f.Fuzz(func(t *testing.T, b []byte) { + fmt.Fprintf(logFile, "FuzzB %q\n", b) + }) +} + +-- check_logs.go -- +// +build ignore + +package main + +import ( + "bufio" + "bytes" + "fmt" + "io" + "os" + "strings" +) + +func main() { + coordPath, workerPath := os.Args[1], os.Args[2] + + coordLog, err := os.Open(coordPath) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + defer coordLog.Close() + if err := checkCoordLog(coordLog); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + + workerLog, err := os.Open(workerPath) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + defer workerLog.Close() + if err := checkWorkerLog(workerLog); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + +func checkCoordLog(r io.Reader) error { + b, err := io.ReadAll(r) + if err != nil { + return err + } + if string(bytes.TrimSpace(b)) != `FuzzB "seed"` { + return fmt.Errorf("coordinator: did not test FuzzB seed") + } + return nil +} + +func checkWorkerLog(r io.Reader) error { + scan := bufio.NewScanner(r) + var sawAMutant bool + for scan.Scan() { + line := scan.Text() + if !strings.HasPrefix(line, "FuzzA ") { + return fmt.Errorf("worker: tested something other than target: %s", line) + } + if strings.TrimPrefix(line, "FuzzA ") != `"seed"` { + sawAMutant = true + } + } + if err := scan.Err(); err != nil && err != bufio.ErrTooLong { + return err + } + if !sawAMutant { + return fmt.Errorf("worker: did not test any mutants") + } + return nil +} +-- mutator_test.go -- +package fuzz_test + +import ( + "testing" +) + +// TODO(katiehockman): re-work this test once we have a better fuzzing engine +// (ie. more mutations, and compiler instrumentation) +func FuzzMutator(f *testing.F) { + // TODO(katiehockman): simplify this once we can dedupe crashes (e.g. + // replace map with calls to panic, and simply count the number of crashes + // that were added to testdata) + crashes := make(map[string]bool) + // No seed corpus initiated + f.Fuzz(func(t *testing.T, b []byte) { + crashes[string(b)] = true + if len(crashes) >= 10 { + panic("mutator found enough unique mutations") + } + }) +} diff --git a/src/cmd/go/testdata/script/test_fuzz_mutator_repeat.txt b/src/cmd/go/testdata/script/test_fuzz_mutator_repeat.txt new file mode 100644 index 0000000000..0924ed37e6 --- /dev/null +++ b/src/cmd/go/testdata/script/test_fuzz_mutator_repeat.txt @@ -0,0 +1,66 @@ +# TODO(jayconrod): support shared memory on more platforms. +[!darwin] [!linux] [!windows] skip + +# Verify that the fuzzing engine records the actual crashing input, even when +# a worker process terminates without communicating the crashing input back +# to the coordinator. + +[short] skip + +# Start fuzzing. The worker crashes after ~100 iterations. +# The fuzz function writes the crashing input to "want" before exiting. +# The fuzzing engine reconstructs the crashing input and saves it to testdata. +! exists want +! go test -fuzz=. -parallel=1 +stdout 'fuzzing process terminated unexpectedly' +stdout 'Crash written to testdata' + +# Run the fuzz target without fuzzing. The fuzz function is called with the +# crashing input in testdata. The test passes if that input is identical to +# the one saved in "want". +exists want +go test -want=want + +-- go.mod -- +module fuzz + +go 1.17 +-- fuzz_test.go -- +package fuzz + +import ( + "bytes" + "flag" + "os" + "testing" +) + +var wantFlag = flag.String("want", "", "file containing previous crashing input") + +func FuzzRepeat(f *testing.F) { + i := 0 + f.Fuzz(func(t *testing.T, b []byte) { + i++ + if i == 100 { + f, err := os.OpenFile("want", os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0666) + if err != nil { + // Couldn't create the file, probably because it already exists, + // and we're minimizing now. Return without crashing. + return + } + f.Write(b) + f.Close() + os.Exit(1) // crash without communicating + } + + if *wantFlag != "" { + want, err := os.ReadFile(*wantFlag) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(want, b) { + t.Fatalf("inputs are not equal!\n got: %q\nwant:%q", b, want) + } + } + }) +} diff --git a/src/cmd/go/testdata/script/test_fuzz_non_crash_signal.txt b/src/cmd/go/testdata/script/test_fuzz_non_crash_signal.txt new file mode 100644 index 0000000000..1568757de7 --- /dev/null +++ b/src/cmd/go/testdata/script/test_fuzz_non_crash_signal.txt @@ -0,0 +1,55 @@ +# NOTE: this test is skipped on Windows, since there's no concept of signals. +# When a process terminates another process, it provides an exit code. +# TODO(jayconrod): support shared memory on more platforms. +[!darwin] [!freebsd] [!linux] skip +[short] skip + +# FuzzNonCrash sends itself a signal that does not appear to be a crash. +# We should not save a crasher. +! go test -fuzz=FuzzNonCrash +! exists testdata +! stdout unreachable +! stderr unreachable +stdout 'fuzzing process terminated by unexpected signal; no crash will be recorded: signal: killed' + +# FuzzCrash sends itself a signal that looks like a crash. +# We should save a crasher. +! go test -fuzz=FuzzCrash +exists testdata/fuzz/FuzzCrash +stdout 'fuzzing process terminated unexpectedly' + +-- go.mod -- +module test + +go 1.17 +-- fuzz_posix_test.go -- +// +build darwin freebsd linux + +package fuzz + +import ( + "syscall" + "testing" +) + +func FuzzNonCrash(f *testing.F) { + f.Fuzz(func(*testing.T, bool) { + pid := syscall.Getpid() + if err := syscall.Kill(pid, syscall.SIGKILL); err != nil { + panic(err) + } + // signal may not be received immediately. Wait for it. + select{} + }) +} + +func FuzzCrash(f *testing.F) { + f.Fuzz(func(*testing.T, bool) { + pid := syscall.Getpid() + if err := syscall.Kill(pid, syscall.SIGILL); err != nil { + panic(err) + } + // signal may not be received immediately. Wait for it. + select{} + }) +} diff --git a/src/cmd/go/testdata/script/test_fuzz_parallel.txt b/src/cmd/go/testdata/script/test_fuzz_parallel.txt new file mode 100644 index 0000000000..a49f30a27f --- /dev/null +++ b/src/cmd/go/testdata/script/test_fuzz_parallel.txt @@ -0,0 +1,61 @@ +# TODO(jayconrod): support shared memory on more platforms. +[!darwin] [!linux] [!windows] skip + +[short] skip + +# When running seed inputs, T.Parallel should let multiple inputs run in +# parallel. +go test -run=FuzzSeed + +# When fuzzing, T.Parallel should be safe to call, but it should have no effect. +# We just check that it doesn't hang, which would be the most obvious +# failure mode. +# TODO(jayconrod): check for the string "after T.Parallel". It's not printed +# by 'go test', so we can't distinguish that crasher from some other panic. +! go test -run=FuzzMutate -fuzz=FuzzMutate +exists testdata/fuzz/FuzzMutate + +-- go.mod -- +module fuzz_parallel + +go 1.17 +-- fuzz_parallel_test.go -- +package fuzz_parallel + +import ( + "sort" + "sync" + "testing" +) + +func FuzzSeed(f *testing.F) { + for _, v := range [][]byte{{'a'}, {'b'}, {'c'}} { + f.Add(v) + } + + var mu sync.Mutex + var before, after []byte + f.Cleanup(func() { + sort.Slice(after, func(i, j int) bool { return after[i] < after[j] }) + got := string(before) + string(after) + want := "abcabc" + if got != want { + f.Fatalf("got %q; want %q", got, want) + } + }) + + f.Fuzz(func(t *testing.T, b []byte) { + before = append(before, b...) + t.Parallel() + mu.Lock() + after = append(after, b...) + mu.Unlock() + }) +} + +func FuzzMutate(f *testing.F) { + f.Fuzz(func(t *testing.T, _ []byte) { + t.Parallel() + t.Error("after T.Parallel") + }) +} diff --git a/src/cmd/go/testdata/script/test_fuzz_seed_corpus.txt b/src/cmd/go/testdata/script/test_fuzz_seed_corpus.txt new file mode 100644 index 0000000000..016b101d72 --- /dev/null +++ b/src/cmd/go/testdata/script/test_fuzz_seed_corpus.txt @@ -0,0 +1,168 @@ +# TODO(jayconrod): support shared memory on more platforms. +[!darwin] [!linux] [!windows] skip + +[short] skip +env GOCACHE=$WORK/cache + +# Test that fuzzing a target with a failure in f.Add prints the crash +# and doesn't write anything to testdata/fuzz +! go test -fuzz=FuzzWithAdd -run=FuzzWithAdd -fuzztime=1x +! stdout ^ok +! stdout 'Crash written to testdata[/\\]fuzz[/\\]FuzzWithAdd[/\\]' +stdout FAIL + +# Test that fuzzing a target with a sucess in f.Add and a fuzztime of only +# 1 does not produce a crash. +go test -fuzz=FuzzWithGoodAdd -run=FuzzWithGoodAdd -fuzztime=1x +stdout ok +! stdout FAIL + +# Test that fuzzing a target with a failure in testdata/fuzz prints the crash +# and doesn't write anything to testdata/fuzz +! go test -fuzz=FuzzWithTestdata -run=FuzzWithTestdata -fuzztime=1x +! stdout ^ok +! stdout 'Crash written to testdata[/\\]fuzz[/\\]FuzzWithTestdata[/\\]' +stdout FAIL + +# Test that fuzzing a target with no seed corpus or cache finds a crash, prints +# it, and write it to testdata +! go test -fuzz=FuzzWithNoCache -run=FuzzWithNoCache -fuzztime=1x +! stdout ^ok +stdout 'Crash written to testdata[/\\]fuzz[/\\]FuzzWithNoCache[/\\]' +stdout FAIL + +# Write a crashing input to the cache +mkdir $GOCACHE/fuzz/example.com/x/FuzzWithCache +cp cache-file $GOCACHE/fuzz/example.com/x/FuzzWithCache/1 + +# Test that fuzzing a target with a failure in the cache prints the crash +# and writes this as a "new" crash to testdata/fuzz +! go test -fuzz=FuzzWithCache -run=FuzzWithCache -fuzztime=1x +! stdout ^ok +stdout 'Crash written to testdata[/\\]fuzz[/\\]FuzzWithCache[/\\]' +stdout FAIL + +# Clear the fuzz cache and make sure it's gone +go clean -fuzzcache +! exists $GOCACHE/fuzz + +# The tests below should operate the exact same as the previous tests. If -fuzz +# is enabled, then whatever target is going to be fuzzed shouldn't be run by +# anything other than the workers. + +# Test that fuzzing a target (with -run=None set) with a failure in f.Add prints +# the crash and doesn't write anything to testdata/fuzz -fuzztime=1x +! go test -fuzz=FuzzWithAdd -run=None +! stdout ^ok +! stdout 'Crash written to testdata[/\\]fuzz[/\\]FuzzWithAdd[/\\]' +stdout FAIL + +# Test that fuzzing a target (with -run=None set) with a sucess in f.Add and a +# fuzztime of only 1 does not produce a crash. +go test -fuzz=FuzzWithGoodAdd -run=None -fuzztime=1x +stdout ok +! stdout FAIL + +# Test that fuzzing a target (with -run=None set) with a failure in +# testdata/fuzz prints the crash and doesn't write anything to testdata/fuzz +! go test -fuzz=FuzzWithTestdata -run=None -fuzztime=1x +! stdout ^ok +! stdout 'Crash written to testdata[/\\]fuzz[/\\]FuzzWithTestdata[/\\]' +stdout FAIL + +# Write a crashing input to the cache +mkdir $GOCACHE/fuzz/example.com/x/FuzzRunNoneWithCache +cp cache-file $GOCACHE/fuzz/example.com/x/FuzzRunNoneWithCache/1 + +# Test that fuzzing a target (with -run=None set) with a failure in the cache +# prints the crash and writes this as a "new" crash to testdata/fuzz +! go test -fuzz=FuzzRunNoneWithCache -run=None -fuzztime=1x +! stdout ^ok +stdout 'Crash written to testdata[/\\]fuzz[/\\]FuzzRunNoneWithCache[/\\]' +stdout FAIL + +# Clear the fuzz cache and make sure it's gone +go clean -fuzzcache +! exists $GOCACHE/fuzz + +# The tests below should operate the exact same way for the previous tests with +# a seed corpus (namely, they should still fail). However, the binary is built +# without instrumentation, so this should be a "testing only" run which executes +# the seed corpus before attempting to fuzz. + +go test -c +! exec ./x.test$GOEXE -test.fuzz=FuzzWithAdd -test.run=FuzzWithAdd -test.fuzztime=1x -test.fuzzcachedir=$WORK/cache +! stdout ^ok +! stdout 'Crash written to testdata[/\\]fuzz[/\\]FuzzWithAdd[/\\]' +stdout FAIL +stderr warning + +go test -c +! exec ./x.test$GOEXE -test.fuzz=FuzzWithTestdata -test.run=FuzzWithTestdata -test.fuzztime=1x -test.fuzzcachedir=$WORK/cache +! stdout ^ok +! stdout 'Crash written to testdata[/\\]fuzz[/\\]FuzzWithTestdata[/\\]' +stdout FAIL +stderr warning + +-- go.mod -- +module example.com/x + +go 1.16 +-- x_test.go -- +package x + +import "testing" + +func FuzzWithAdd(f *testing.F) { + f.Add(10) + f.Fuzz(func(t *testing.T, i int) { + if i == 10 { + t.Error("bad thing here") + } + }) +} + +func FuzzWithGoodAdd(f *testing.F) { + f.Add(10) + f.Fuzz(func(t *testing.T, i int) { + if i != 10 { + t.Error("bad thing here") + } + }) +} + +func FuzzWithTestdata(f *testing.F) { + f.Fuzz(func(t *testing.T, i int) { + if i == 10 { + t.Error("bad thing here") + } + }) +} + +func FuzzWithNoCache(f *testing.F) { + f.Fuzz(func(t *testing.T, i int) { + t.Error("bad thing here") + }) +} + +func FuzzWithCache(f *testing.F) { + f.Fuzz(func(t *testing.T, i int) { + if i == 10 { + t.Error("bad thing here") + } + }) +} + +func FuzzRunNoneWithCache(f *testing.F) { + f.Fuzz(func(t *testing.T, i int) { + if i == 10 { + t.Error("bad thing here") + } + }) +} +-- testdata/fuzz/FuzzWithTestdata/1 -- +go test fuzz v1 +int(10) +-- cache-file -- +go test fuzz v1 +int(10)
\ No newline at end of file diff --git a/src/cmd/go/testdata/script/test_fuzz_setenv.txt b/src/cmd/go/testdata/script/test_fuzz_setenv.txt new file mode 100644 index 0000000000..9738697a91 --- /dev/null +++ b/src/cmd/go/testdata/script/test_fuzz_setenv.txt @@ -0,0 +1,45 @@ +[short] skip +[!darwin] [!linux] [!windows] skip + +go test -fuzz=FuzzA -fuzztime=100x fuzz_setenv_test.go + +-- fuzz_setenv_test.go -- +package fuzz + +import ( + "flag" + "os" + "testing" +) + +func FuzzA(f *testing.F) { + if s := os.Getenv("TEST_FUZZ_SETENV_A"); isWorker() && s == "" { + f.Fatal("environment variable not set") + } else if !isWorker() && s != "" { + f.Fatal("environment variable already set") + } + f.Setenv("TEST_FUZZ_SETENV_A", "A") + if os.Getenv("TEST_FUZZ_SETENV_A") == "" { + f.Fatal("Setenv did not set environment variable") + } + f.Fuzz(func(*testing.T, []byte) {}) +} + +func FuzzB(f *testing.F) { + if os.Getenv("TEST_FUZZ_SETENV_A") != "" { + f.Fatal("environment variable not cleared after FuzzA") + } + f.Skip() +} + +func isWorker() bool { + f := flag.Lookup("test.fuzzworker") + if f == nil { + return false + } + get, ok := f.Value.(flag.Getter) + if !ok { + return false + } + return get.Get() == interface{}(true) +} diff --git a/src/cmd/go/testdata/script/test_fuzz_tag.txt b/src/cmd/go/testdata/script/test_fuzz_tag.txt new file mode 100644 index 0000000000..07ed5d6d61 --- /dev/null +++ b/src/cmd/go/testdata/script/test_fuzz_tag.txt @@ -0,0 +1,31 @@ +# Check that the gofuzzbeta tag is enabled by default and can be disabled. +# TODO(jayconrod,katiehockman): before merging to master, restore the old +# default and delete this test. + +[short] skip + +go test -list=. +stdout Test +stdout Fuzz + +go test -tags= + +-- go.mod -- +module fuzz + +go 1.17 +-- fuzz_test.go -- +// +build gofuzzbeta + +package fuzz + +import "testing" + +func Fuzz(f *testing.F) { + f.Add([]byte(nil)) + f.Fuzz(func(*testing.T, []byte) {}) +} + +func Test(*testing.T) {} +-- empty_test.go -- +package fuzz |