diff options
author | Robert Griesemer <gri@golang.org> | 2016-03-04 17:09:08 -0800 |
---|---|---|
committer | Matthew Dempsky <mdempsky@google.com> | 2016-08-18 21:33:38 +0000 |
commit | c8683ff7977c526fb48ae007971fed16ef32ff62 (patch) | |
tree | 408437654d4fd529b864b27ff66068f646bde746 /src/cmd/compile/internal/syntax/parser_test.go | |
parent | 3b967be4219c789ef9c47aa5e9607cab3005e1cd (diff) | |
download | go-c8683ff7977c526fb48ae007971fed16ef32ff62.tar.gz go-c8683ff7977c526fb48ae007971fed16ef32ff62.zip |
cmd/compile/internal/syntax: fast Go syntax trees, initial commit.
Syntax tree nodes, scanner, parser, basic printers.
Builds syntax trees for entire Go std lib at a rate of ~1.8M lines/s
in warmed up state (MacMini, 2.3 GHz Intel Core i7, 8GB RAM):
$ go test -run StdLib -fast
parsed 1074617 lines (2832 files) in 579.66364ms (1853863 lines/s)
allocated 282.212Mb (486.854Mb/s)
PASS
Change-Id: Ie26d9a7bf4e5ff07457aedfcc9b89f0eba72ae3f
Reviewed-on: https://go-review.googlesource.com/27195
Run-TryBot: Matthew Dempsky <mdempsky@google.com>
Reviewed-by: Robert Griesemer <gri@golang.org>
Diffstat (limited to 'src/cmd/compile/internal/syntax/parser_test.go')
-rw-r--r-- | src/cmd/compile/internal/syntax/parser_test.go | 157 |
1 files changed, 157 insertions, 0 deletions
diff --git a/src/cmd/compile/internal/syntax/parser_test.go b/src/cmd/compile/internal/syntax/parser_test.go new file mode 100644 index 0000000000..12fc019414 --- /dev/null +++ b/src/cmd/compile/internal/syntax/parser_test.go @@ -0,0 +1,157 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package syntax + +import ( + "bytes" + "flag" + "fmt" + "io/ioutil" + "path/filepath" + "runtime" + "strings" + "sync" + "testing" + "time" +) + +var fast = flag.Bool("fast", false, "parse package files in parallel") +var src = flag.String("src", "parser.go", "source file to parse") +var verify = flag.Bool("verify", false, "verify idempotent printing") + +func TestParse(t *testing.T) { + _, err := ReadFile(*src, nil, 0) + if err != nil { + t.Fatal(err) + } +} + +func TestStdLib(t *testing.T) { + if testing.Short() { + t.Skip("skipping test in short mode") + } + + var m1 runtime.MemStats + runtime.ReadMemStats(&m1) + start := time.Now() + + type parseResult struct { + filename string + lines int + } + + results := make(chan parseResult) + go func() { + for _, dir := range []string{ + runtime.GOROOT(), + //"/Users/gri/src", + } { + walkDirs(t, dir, func(filename string) { + if debug { + fmt.Printf("parsing %s\n", filename) + } + ast, err := ReadFile(filename, nil, 0) + if err != nil { + t.Fatal(err) + } + if *verify { + verifyPrint(filename, ast) + } + results <- parseResult{filename, ast.Lines} + }) + } + close(results) + }() + + var count, lines int + for res := range results { + count++ + lines += res.lines + if testing.Verbose() { + fmt.Printf("%5d %s (%d lines)\n", count, res.filename, res.lines) + } + } + + dt := time.Since(start) + var m2 runtime.MemStats + runtime.ReadMemStats(&m2) + dm := float64(m2.TotalAlloc-m1.TotalAlloc) / 1e6 + + fmt.Printf("parsed %d lines (%d files) in %v (%d lines/s)\n", lines, count, dt, int64(float64(lines)/dt.Seconds())) + fmt.Printf("allocated %.3fMb (%.3fMb/s)\n", dm, dm/dt.Seconds()) +} + +func walkDirs(t *testing.T, dir string, action func(string)) { + fis, err := ioutil.ReadDir(dir) + if err != nil { + t.Error(err) + return + } + + var files, dirs []string + for _, fi := range fis { + if fi.Mode().IsRegular() { + if strings.HasSuffix(fi.Name(), ".go") { + path := filepath.Join(dir, fi.Name()) + files = append(files, path) + } + } else if fi.IsDir() && fi.Name() != "testdata" { + path := filepath.Join(dir, fi.Name()) + if !strings.Contains(path, "go/test") { + dirs = append(dirs, path) + } + } + } + + if *fast { + var wg sync.WaitGroup + wg.Add(len(files)) + for _, filename := range files { + go func(filename string) { + defer wg.Done() + action(filename) + }(filename) + } + wg.Wait() + } else { + for _, filename := range files { + action(filename) + } + } + + for _, dir := range dirs { + walkDirs(t, dir, action) + } +} + +func verifyPrint(filename string, ast1 *File) { + var buf1 bytes.Buffer + _, err := Fprint(&buf1, ast1, true) + if err != nil { + panic(err) + } + + ast2, err := ReadBytes(buf1.Bytes(), nil, 0) + if err != nil { + panic(err) + } + + var buf2 bytes.Buffer + _, err = Fprint(&buf2, ast2, true) + if err != nil { + panic(err) + } + + if bytes.Compare(buf1.Bytes(), buf2.Bytes()) != 0 { + fmt.Printf("--- %s ---\n", filename) + fmt.Printf("%s\n", buf1.Bytes()) + fmt.Println() + + fmt.Printf("--- %s ---\n", filename) + fmt.Printf("%s\n", buf2.Bytes()) + fmt.Println() + panic("not equal") + } +} |