Merge pull request #589 from AudriusButkevicius/include

Add #include directive to .stignore (fixes #424)

20 files changed, 116 insertions, 51 deletions

diff --git a/build.go b/build.go
index 90784c48c..2957a80e4 100644
--- a/build.go
+++ b/build.go
@@ -163,7 +163,7 @@ func setup() {
 }
 
 func test(pkg string) {
-	runPrint("godep", "go", "test", pkg)
+	runPrint("godep", "go", "test", "-short", "-timeout", "10s", pkg)
 }
 
 func install(pkg string) {
diff --git a/scanner/testdata/.foo/bar b/scanner/testdata/.foo/bar
deleted file mode 100644
index e69de29bb..000000000
--- a/scanner/testdata/.foo/bar
+++ /dev/null
diff --git a/scanner/testdata/.stignore b/scanner/testdata/.stignore
index cf8fb0fe8..d8d9cd382 100644
--- a/scanner/testdata/.stignore
+++ b/scanner/testdata/.stignore
@@ -1,2 +1,6 @@
-.*
-quux
+#include excludes
+#include nonexistent-file
+
+bfile
+dir1/cfile
+
diff --git a/scanner/testdata/afile b/scanner/testdata/afile
new file mode 100644
index 000000000..257cc5642
--- /dev/null
+++ b/scanner/testdata/afile
@@ -0,0 +1 @@
+foo
diff --git a/scanner/testdata/bar b/scanner/testdata/bar
deleted file mode 100644
index b33c13891..000000000
--- a/scanner/testdata/bar
+++ /dev/null
@@ -1 +0,0 @@
-foobarbaz
diff --git a/scanner/testdata/baz/quux b/scanner/testdata/baz/quux
deleted file mode 100644
index 55976ea06..000000000
--- a/scanner/testdata/baz/quux
+++ /dev/null
@@ -1 +0,0 @@
-baazquux
diff --git a/scanner/testdata/bfile b/scanner/testdata/bfile
new file mode 100644
index 000000000..5716ca598
--- /dev/null
+++ b/scanner/testdata/bfile
@@ -0,0 +1 @@
+bar
diff --git a/scanner/testdata/dir1/cfile b/scanner/testdata/dir1/cfile
new file mode 100644
index 000000000..76018072e
--- /dev/null
+++ b/scanner/testdata/dir1/cfile
@@ -0,0 +1 @@
+baz
diff --git a/scanner/testdata/dir1/dfile b/scanner/testdata/dir1/dfile
new file mode 100644
index 000000000..d90bda0ff
--- /dev/null
+++ b/scanner/testdata/dir1/dfile
@@ -0,0 +1 @@
+quux
diff --git a/scanner/testdata/dir2/cfile b/scanner/testdata/dir2/cfile
new file mode 100644
index 000000000..76018072e
--- /dev/null
+++ b/scanner/testdata/dir2/cfile
@@ -0,0 +1 @@
+baz
diff --git a/scanner/testdata/dir2/dfile b/scanner/testdata/dir2/dfile
new file mode 100644
index 000000000..d90bda0ff
--- /dev/null
+++ b/scanner/testdata/dir2/dfile
@@ -0,0 +1 @@
+quux
diff --git a/scanner/testdata/dir3/cfile b/scanner/testdata/dir3/cfile
new file mode 100644
index 000000000..76018072e
--- /dev/null
+++ b/scanner/testdata/dir3/cfile
@@ -0,0 +1 @@
+baz
diff --git a/scanner/testdata/dir3/dfile b/scanner/testdata/dir3/dfile
new file mode 100644
index 000000000..d90bda0ff
--- /dev/null
+++ b/scanner/testdata/dir3/dfile
@@ -0,0 +1 @@
+quux
diff --git a/scanner/testdata/empty b/scanner/testdata/empty
deleted file mode 100644
index e69de29bb..000000000
--- a/scanner/testdata/empty
+++ /dev/null
diff --git a/scanner/testdata/excludes b/scanner/testdata/excludes
new file mode 100644
index 000000000..b6de3b738
--- /dev/null
+++ b/scanner/testdata/excludes
@@ -0,0 +1,4 @@
+dir2/dfile
+#include excludes
+#include further-excludes
+#include loop-excludes
diff --git a/scanner/testdata/foo b/scanner/testdata/foo
deleted file mode 100644
index 323fae03f..000000000
--- a/scanner/testdata/foo
+++ /dev/null
@@ -1 +0,0 @@
-foobar
diff --git a/scanner/testdata/further-excludes b/scanner/testdata/further-excludes
new file mode 100644
index 000000000..9e831d51b
--- /dev/null
+++ b/scanner/testdata/further-excludes
@@ -0,0 +1 @@
+dir3
diff --git a/scanner/testdata/loop-excludes b/scanner/testdata/loop-excludes
new file mode 100644
index 000000000..9337e0d49
--- /dev/null
+++ b/scanner/testdata/loop-excludes
@@ -0,0 +1 @@
+#include excludes
diff --git a/scanner/walk.go b/scanner/walk.go
index 8cb189725..1278490c0 100644
--- a/scanner/walk.go
+++ b/scanner/walk.go
@@ -99,7 +99,8 @@ func (w *Walker) loadIgnoreFiles(dir string, ignores *[]*regexp.Regexp) filepath
 
 		if pn, sn := filepath.Split(rn); sn == w.IgnoreFile {
 			pn := filepath.Clean(pn)
-			dirIgnores := loadIgnoreFile(p, pn)
+			filesSeen := make(map[string]map[string]bool)
+			dirIgnores := loadIgnoreFile(p, pn, filesSeen)
 			*ignores = append(*ignores, dirIgnores...)
 		}
 
@@ -107,16 +108,16 @@ func (w *Walker) loadIgnoreFiles(dir string, ignores *[]*regexp.Regexp) filepath
 	}
 }
 
-func loadIgnoreFile(ignFile, base string) []*regexp.Regexp {
+func loadIgnoreFile(ignFile, base string, filesSeen map[string]map[string]bool) []*regexp.Regexp {
 	fd, err := os.Open(ignFile)
 	if err != nil {
 		return nil
 	}
 	defer fd.Close()
-	return parseIgnoreFile(fd, base)
+	return parseIgnoreFile(fd, base, ignFile, filesSeen)
 }
 
-func parseIgnoreFile(fd io.Reader, base string) []*regexp.Regexp {
+func parseIgnoreFile(fd io.Reader, base, currentFile string, filesSeen map[string]map[string]bool) []*regexp.Regexp {
 	var exps []*regexp.Regexp
 	scanner := bufio.NewScanner(fd)
 	for scanner.Scan() {
@@ -148,6 +149,27 @@ func parseIgnoreFile(fd io.Reader, base string) []*regexp.Regexp {
 				continue
 			}
 			exps = append(exps, exp)
+		} else if strings.HasPrefix(line, "#include ") {
+			includeFile := filepath.Join(filepath.Dir(currentFile), strings.Replace(line, "#include ", "", 1))
+			if _, err := os.Stat(includeFile); os.IsNotExist(err) {
+				l.Infoln("Could not open ignore include file", includeFile)
+			} else {
+				seen := false
+				if seenByCurrent, ok := filesSeen[currentFile]; ok {
+					_, seen = seenByCurrent[includeFile]
+				}
+
+				if seen {
+					l.Warnf("Recursion detected while including %s from %s", includeFile, currentFile)
+				} else {
+					if filesSeen[currentFile] == nil {
+						filesSeen[currentFile] = make(map[string]bool)
+					}
+					filesSeen[currentFile][includeFile] = true
+					includes := loadIgnoreFile(includeFile, base, filesSeen)
+					exps = append(exps, includes...)
+				}
+			}
 		} else {
 			// Path name or pattern, add it so it matches files both in
 			// current directory and subdirs.
diff --git a/scanner/walk_test.go b/scanner/walk_test.go
index 943843b2a..7a9f705dc 100644
--- a/scanner/walk_test.go
+++ b/scanner/walk_test.go
@@ -8,32 +8,48 @@ import (
 	"bytes"
 	"fmt"
 	"path/filepath"
+	"reflect"
+	rdebug "runtime/debug"
 	"sort"
 	"testing"
-	"time"
 
 	"github.com/syncthing/syncthing/protocol"
 )
 
-var testdata = []struct {
+type testfile struct {
 	name string
 	size int
 	hash string
-}{
-	{"bar", 10, "2f72cc11a6fcd0271ecef8c61056ee1eb1243be3805bf9a9df98f92f7636b05c"},
-	{"baz", 0, ""},
-	{"empty", 0, "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"},
-	{"foo", 7, "aec070645fe53ee3b3763059376134f058cc337247c978add178b6ccdfb0019f"},
+}
+
+type testfileList []testfile
+
+var testdata = testfileList{
+	{"afile", 4, "b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c"},
+	{"dir1", 128, ""},
+	{filepath.Join("dir1", "dfile"), 5, "49ae93732fcf8d63fe1cce759664982dbd5b23161f007dba8561862adc96d063"},
+	{"dir2", 128, ""},
+	{filepath.Join("dir2", "cfile"), 4, "bf07a7fbb825fc0aae7bf4a1177b2b31fcf8a3feeaf7092761e18c859ee52a9c"},
+	{"excludes", 78, "1f5ac95d9e6fb2516629a029d788d27953c7bb2f4dc09184b660fdda0c8f2f04"},
+	{"further-excludes", 5, "7eb0a548094fa6295f7fd9200d69973e5f5ec5c04f2a86d998080ac43ecf89f1"},
+	{"loop-excludes", 18, "2db057aa82a8b8fe4b1367ccc875259ed4b8020255820d4e3d4bfe78f0dd3f2a"},
 }
 
 var correctIgnores = map[string][]string{
 	".": {".*", "quux"},
 }
 
+func init() {
+	// This test runs the risk of entering infinite recursion if it fails.
+	// Limit the stack size to 10 megs to creash early in that case instead of
+	// potentially taking down the box...
+	rdebug.SetMaxStack(10 * 1 << 20)
+}
+
 func TestWalkSub(t *testing.T) {
 	w := Walker{
 		Dir:        "testdata",
-		Sub:        "foo",
+		Sub:        "dir2",
 		BlockSize:  128 * 1024,
 		IgnoreFile: ".stignore",
 	}
@@ -46,11 +62,17 @@ func TestWalkSub(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	if len(files) != 1 {
-		t.Fatalf("Incorrect length %d != 1", len(files))
+	// The directory contains two files, where one is ignored from a higher
+	// level. We should see only the directory and one of the files.
+
+	if len(files) != 2 {
+		t.Fatalf("Incorrect length %d != 2", len(files))
+	}
+	if files[0].Name != "dir2" {
+		t.Errorf("Incorrect file %v != dir2", files[0])
 	}
-	if files[0].Name != "foo" {
-		t.Errorf("Incorrect file %v != foo", files[0])
+	if files[1].Name != filepath.Join("dir2", "cfile") {
+		t.Errorf("Incorrect file %v != dir2/cfile", files[1])
 	}
 }
 
@@ -60,39 +82,21 @@ func TestWalk(t *testing.T) {
 		BlockSize:  128 * 1024,
 		IgnoreFile: ".stignore",
 	}
-	fchan, err := w.Walk()
-	var files []protocol.FileInfo
-	for f := range fchan {
-		files = append(files, f)
-	}
-	sort.Sort(fileList(files))
 
+	fchan, err := w.Walk()
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	if l1, l2 := len(files), len(testdata); l1 != l2 {
-		t.Log(files)
-		t.Log(testdata)
-		t.Fatalf("Incorrect number of walked files %d != %d", l1, l2)
+	var tmp []protocol.FileInfo
+	for f := range fchan {
+		tmp = append(tmp, f)
 	}
+	sort.Sort(fileList(tmp))
+	files := fileList(tmp).testfiles()
 
-	for i := range testdata {
-		if n1, n2 := testdata[i].name, files[i].Name; n1 != n2 {
-			t.Errorf("Incorrect file name %q != %q for case #%d", n1, n2, i)
-		}
-
-		if testdata[i].hash != "" {
-			if h1, h2 := fmt.Sprintf("%x", files[i].Blocks[0].Hash), testdata[i].hash; h1 != h2 {
-				t.Errorf("Incorrect hash %q != %q for case #%d", h1, h2, i)
-			}
-		}
-
-		t0 := time.Date(2010, 1, 1, 0, 0, 0, 0, time.UTC).Unix()
-		t1 := time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC).Unix()
-		if mt := files[i].Modified; mt < t0 || mt > t1 {
-			t.Errorf("Unrealistic modtime %d for test %d", mt, i)
-		}
+	if !reflect.DeepEqual(files, testdata) {
+		t.Errorf("Walk returned unexpected data\nExpected: %v\nActual: %v", testdata, files)
 	}
 }
 
@@ -128,20 +132,20 @@ func TestIgnore(t *testing.T) {
 		*/other/test
 		**/deep
 	`)
-	patterns := parseIgnoreFile(patStr, "")
+	patterns := parseIgnoreFile(patStr, "", "", make(map[string]map[string]bool))
 
 	patStr = bytes.NewBufferString(`
 		bar
 		z*
 		q[abc]x
 	`)
-	patterns = append(patterns, parseIgnoreFile(patStr, "foo")...)
+	patterns = append(patterns, parseIgnoreFile(patStr, "foo", "", make(map[string]map[string]bool))...)
 
 	patStr = bytes.NewBufferString(`
 		quux
 		.*
 	`)
-	patterns = append(patterns, parseIgnoreFile(patStr, "foo/baz")...)
+	patterns = append(patterns, parseIgnoreFile(patStr, "foo/baz", "", make(map[string]map[string]bool))...)
 
 	var tests = []struct {
 		f string
@@ -194,3 +198,27 @@ func (f fileList) Less(a, b int) bool {
 func (f fileList) Swap(a, b int) {
 	f[a], f[b] = f[b], f[a]
 }
+
+func (l fileList) testfiles() testfileList {
+	testfiles := make(testfileList, len(l))
+	for i, f := range l {
+		if len(f.Blocks) > 1 {
+			panic("simple test case stuff only supports a single block per file")
+		}
+		testfiles[i] = testfile{name: f.Name, size: int(f.Size())}
+		if len(f.Blocks) == 1 {
+			testfiles[i].hash = fmt.Sprintf("%x", f.Blocks[0].Hash)
+		}
+	}
+	return testfiles
+}
+
+func (l testfileList) String() string {
+	var b bytes.Buffer
+	b.WriteString("{\n")
+	for _, f := range l {
+		fmt.Fprintf(&b, "  %s (%d bytes): %s\n", f.name, f.size, f.hash)
+	}
+	b.WriteString("}")
+	return b.String()
+}