notmuch: add strategies for multi-file messages

A single notmuch message can represent multiple files. As a result,
file-based operations like move, copy, and delete can be ambiguous. Add
a new account config option, multi-file-strategy, to tell aerc how to
handle these ambiguous cases. Also add options to relevant commands to
set the multi-file strategy on a per-invocation basis.

If no multi-file strategy is set, refuse to take file-based actions on
multi-file messages. This default behavior is mostly the same as aerc's
previous behavior, but a bit stricter in some cases which previously
tried to be smart about multi-file operations (e.g., move and delete).

Applying multi-file strategies to cross-account copy and move operations
is not implemented. These operations will proceed as they have in the
past -- aerc will copy/move a single file. However, for cross-account
move operations, aerc will refuse to delete multiple files to prevent
data loss as not all of the files are added to the destination account.

See the changes to aerc-notmuch(5) for details on the currently
supported multi-file strategies.

Changelog-added: Tell aerc how to handle file-based operations
 on multi-file notmuch messages with the account config option
 `multi-file-strategy` and the `-m` flag to `:archive`, `:copy`,
 `:delete`, and `:move`.
Signed-off-by: Jason Cox <me@jasoncarloscox.com>
Tested-by: Maarten Aertsen <maarten@nlnetlabs.nl>
Acked-by: Robin Jarry <robin@jarry.cc>

5 files changed, 447 insertions, 93 deletions

diff --git a/worker/notmuch/message.go b/worker/notmuch/message.go
index 09850d64..daaec7d0 100644
--- a/worker/notmuch/message.go
+++ b/worker/notmuch/message.go
@@ -17,6 +17,7 @@ import (
 	"git.sr.ht/~rjarry/aerc/models"
 	"git.sr.ht/~rjarry/aerc/worker/lib"
 	notmuch "git.sr.ht/~rjarry/aerc/worker/notmuch/lib"
+	"git.sr.ht/~rjarry/aerc/worker/types"
 )
 
 type Message struct {
@@ -173,87 +174,144 @@ func (m *Message) ModifyTags(add, remove []string) error {
 	return m.db.MsgModifyTags(m.key, add, remove)
 }
 
-func (m *Message) Remove(dir maildir.Dir) error {
-	filenames, err := m.db.MsgFilenames(m.key)
+func (m *Message) Remove(curDir maildir.Dir, mfs types.MultiFileStrategy) error {
+	rm, del, err := m.filenamesForStrategy(mfs, curDir)
 	if err != nil {
 		return err
 	}
-	for _, filename := range filenames {
-		if dirContains(dir, filename) {
-			err := m.db.DeleteMessage(filename)
-			if err != nil {
-				return err
-			}
 
-			if err := os.Remove(filename); err != nil {
-				return err
-			}
-
-			return nil
-		}
-	}
-
-	return fmt.Errorf("no matching message file found in %s", string(dir))
+	rm = append(rm, del...)
+	return m.deleteFiles(rm)
 }
 
-func (m *Message) Copy(target maildir.Dir) error {
-	filename, err := m.Filename()
+func (m *Message) Copy(curDir, destDir maildir.Dir, mfs types.MultiFileStrategy) error {
+	cp, del, err := m.filenamesForStrategy(mfs, curDir)
 	if err != nil {
 		return err
 	}
 
-	source, key := parseFilename(filename)
-	if key == "" {
-		return fmt.Errorf("failed to parse message filename: %s", filename)
-	}
+	for _, filename := range cp {
+		source, key := parseFilename(filename)
+		if key == "" {
+			return fmt.Errorf("failed to parse message filename: %s", filename)
+		}
 
-	newKey, err := source.Copy(target, key)
-	if err != nil {
-		return err
+		newKey, err := source.Copy(destDir, key)
+		if err != nil {
+			return err
+		}
+		newFilename, err := destDir.Filename(newKey)
+		if err != nil {
+			return err
+		}
+		_, err = m.db.IndexFile(newFilename)
+		if err != nil {
+			return err
+		}
 	}
-	newFilename, err := target.Filename(newKey)
+
+	return m.deleteFiles(del)
+}
+
+func (m *Message) Move(curDir, destDir maildir.Dir, mfs types.MultiFileStrategy) error {
+	move, del, err := m.filenamesForStrategy(mfs, curDir)
 	if err != nil {
 		return err
 	}
-	_, err = m.db.IndexFile(newFilename)
-	return err
-}
 
-func (m *Message) Move(srcDir, destDir maildir.Dir) error {
-	var src string
+	for _, filename := range move {
+		// Remove encoded UID information from the key to prevent sync issues
+		name := lib.StripUIDFromMessageFilename(filepath.Base(filename))
+		dest := filepath.Join(string(destDir), "cur", name)
 
-	filenames, err := m.db.MsgFilenames(m.key)
-	if err != nil {
-		return err
+		if err := os.Rename(filename, dest); err != nil {
+			return err
+		}
+
+		if _, err = m.db.IndexFile(dest); err != nil {
+			return err
+		}
+
+		if err := m.db.DeleteMessage(filename); err != nil {
+			return err
+		}
 	}
+
+	return m.deleteFiles(del)
+}
+
+func (m *Message) deleteFiles(filenames []string) error {
 	for _, filename := range filenames {
-		if dirContains(srcDir, filename) {
-			src = filename
-			break
+		if err := os.Remove(filename); err != nil {
+			return err
+		}
+
+		if err := m.db.DeleteMessage(filename); err != nil {
+			return err
 		}
 	}
 
-	if src == "" {
-		return fmt.Errorf("no matching message file found in %s", string(srcDir))
+	return nil
+}
+
+func (m *Message) filenamesForStrategy(strategy types.MultiFileStrategy,
+	curDir maildir.Dir,
+) (act, del []string, err error) {
+	filenames, err := m.db.MsgFilenames(m.key)
+	if err != nil {
+		return nil, nil, err
 	}
+	return filterForStrategy(filenames, strategy, curDir)
+}
 
-	// Remove encoded UID information from the key to prevent sync issues
-	name := lib.StripUIDFromMessageFilename(filepath.Base(src))
-	dest := filepath.Join(string(destDir), "cur", name)
+func filterForStrategy(filenames []string, strategy types.MultiFileStrategy,
+	curDir maildir.Dir,
+) (act, del []string, err error) {
+	if curDir == "" &&
+		(strategy == types.ActDir || strategy == types.ActDirDelRest) {
+		strategy = types.Refuse
+	}
 
-	if err := os.Rename(src, dest); err != nil {
-		return err
+	if len(filenames) < 2 {
+		return filenames, []string{}, nil
 	}
 
-	if _, err = m.db.IndexFile(dest); err != nil {
-		return err
+	act = []string{}
+	rest := []string{}
+	switch strategy {
+	case types.Refuse:
+		return nil, nil, fmt.Errorf("refusing to act on multiple files")
+	case types.ActAll:
+		act = filenames
+	case types.ActOne:
+		fallthrough
+	case types.ActOneDelRest:
+		act = filenames[:1]
+		rest = filenames[1:]
+	case types.ActDir:
+		fallthrough
+	case types.ActDirDelRest:
+		for _, filename := range filenames {
+			if filepath.Dir(filepath.Dir(filename)) == string(curDir) {
+				act = append(act, filename)
+			} else {
+				rest = append(rest, filename)
+			}
+		}
+	default:
+		return nil, nil, fmt.Errorf("invalid multi-file strategy %v", strategy)
 	}
 
-	if err := m.db.DeleteMessage(src); err != nil {
-		return err
+	switch strategy {
+	case types.ActOneDelRest:
+		fallthrough
+	case types.ActDirDelRest:
+		del = rest
+	default:
+		del = []string{}
 	}
 
-	return nil
+	return act, del, nil
 }
 
 func parseFilename(filename string) (maildir.Dir, string) {
@@ -270,13 +328,3 @@ func parseFilename(filename string) (maildir.Dir, string) {
 	key := split[0]
 	return maildir.Dir(dir), key
 }
-
-func dirContains(dir maildir.Dir, filename string) bool {
-	for _, sub := range []string{"cur", "new"} {
-		match, _ := filepath.Match(filepath.Join(string(dir), sub, "*"), filename)
-		if match {
-			return true
-		}
-	}
-	return false
-}
diff --git a/worker/notmuch/message_test.go b/worker/notmuch/message_test.go
new file mode 100644
index 00000000..51fcdb09
--- /dev/null
+++ b/worker/notmuch/message_test.go
@@ -0,0 +1,264 @@
+//go:build notmuch
+// +build notmuch
+
+package notmuch
+
+import (
+	"testing"
+
+	"git.sr.ht/~rjarry/aerc/worker/types"
+	"github.com/emersion/go-maildir"
+)
+
+func TestFilterForStrategy(t *testing.T) {
+	tests := []struct {
+		filenames   []string
+		strategy    types.MultiFileStrategy
+		curDir      string
+		expectedAct []string
+		expectedDel []string
+		expectedErr bool
+	}{
+		// if there's only one file, always act on it
+		{
+			filenames:   []string{"/h/j/m/A/cur/a.b.c:2,"},
+			strategy:    types.Refuse,
+			curDir:      "/h/j/m/B",
+			expectedAct: []string{"/h/j/m/A/cur/a.b.c:2,"},
+			expectedDel: []string{},
+		},
+		{
+			filenames:   []string{"/h/j/m/A/cur/a.b.c:2,"},
+			strategy:    types.ActAll,
+			curDir:      "/h/j/m/B",
+			expectedAct: []string{"/h/j/m/A/cur/a.b.c:2,"},
+			expectedDel: []string{},
+		},
+		{
+			filenames:   []string{"/h/j/m/A/cur/a.b.c:2,"},
+			strategy:    types.ActOne,
+			curDir:      "/h/j/m/B",
+			expectedAct: []string{"/h/j/m/A/cur/a.b.c:2,"},
+			expectedDel: []string{},
+		},
+		{
+			filenames:   []string{"/h/j/m/A/cur/a.b.c:2,"},
+			strategy:    types.ActOneDelRest,
+			curDir:      "/h/j/m/B",
+			expectedAct: []string{"/h/j/m/A/cur/a.b.c:2,"},
+			expectedDel: []string{},
+		},
+		{
+			filenames:   []string{"/h/j/m/A/cur/a.b.c:2,"},
+			strategy:    types.ActDir,
+			curDir:      "/h/j/m/B",
+			expectedAct: []string{"/h/j/m/A/cur/a.b.c:2,"},
+			expectedDel: []string{},
+		},
+		{
+			filenames:   []string{"/h/j/m/A/cur/a.b.c:2,"},
+			strategy:    types.ActDirDelRest,
+			curDir:      "/h/j/m/B",
+			expectedAct: []string{"/h/j/m/A/cur/a.b.c:2,"},
+			expectedDel: []string{},
+		},
+
+		// follow strategy for multiple files
+		{
+			filenames: []string{
+				"/h/j/m/A/cur/a.b.c:2,",
+				"/h/j/m/B/new/b.c.d",
+				"/h/j/m/B/cur/c.d.e:2,S",
+				"/h/j/m/C/new/d.e.f",
+			},
+			strategy:    types.Refuse,
+			curDir:      "/h/j/m/B",
+			expectedErr: true,
+		},
+		{
+			filenames: []string{
+				"/h/j/m/A/cur/a.b.c:2,",
+				"/h/j/m/B/new/b.c.d",
+				"/h/j/m/B/cur/c.d.e:2,S",
+				"/h/j/m/C/new/d.e.f",
+			},
+			strategy: types.ActAll,
+			curDir:   "/h/j/m/B",
+			expectedAct: []string{
+				"/h/j/m/A/cur/a.b.c:2,",
+				"/h/j/m/B/new/b.c.d",
+				"/h/j/m/B/cur/c.d.e:2,S",
+				"/h/j/m/C/new/d.e.f",
+			},
+			expectedDel: []string{},
+		},
+		{
+			filenames: []string{
+				"/h/j/m/A/cur/a.b.c:2,",
+				"/h/j/m/B/new/b.c.d",
+				"/h/j/m/B/cur/c.d.e:2,S",
+				"/h/j/m/C/new/d.e.f",
+			},
+			strategy:    types.ActOne,
+			curDir:      "/h/j/m/B",
+			expectedAct: []string{"/h/j/m/A/cur/a.b.c:2,"},
+			expectedDel: []string{},
+		},
+		{
+			filenames: []string{
+				"/h/j/m/A/cur/a.b.c:2,",
+				"/h/j/m/B/new/b.c.d",
+				"/h/j/m/B/cur/c.d.e:2,S",
+				"/h/j/m/C/new/d.e.f",
+			},
+			strategy:    types.ActOneDelRest,
+			curDir:      "/h/j/m/B",
+			expectedAct: []string{"/h/j/m/A/cur/a.b.c:2,"},
+			expectedDel: []string{
+				"/h/j/m/B/new/b.c.d",
+				"/h/j/m/B/cur/c.d.e:2,S",
+				"/h/j/m/C/new/d.e.f",
+			},
+		},
+		{
+			filenames: []string{
+				"/h/j/m/A/cur/a.b.c:2,",
+				"/h/j/m/B/new/b.c.d",
+				"/h/j/m/B/cur/c.d.e:2,S",
+				"/h/j/m/C/new/d.e.f",
+			},
+			strategy: types.ActDir,
+			curDir:   "/h/j/m/B",
+			expectedAct: []string{
+				"/h/j/m/B/new/b.c.d",
+				"/h/j/m/B/cur/c.d.e:2,S",
+			},
+			expectedDel: []string{},
+		},
+		{
+			filenames: []string{
+				"/h/j/m/A/cur/a.b.c:2,",
+				"/h/j/m/B/new/b.c.d",
+				"/h/j/m/B/cur/c.d.e:2,S",
+				"/h/j/m/C/new/d.e.f",
+			},
+			strategy: types.ActDirDelRest,
+			curDir:   "/h/j/m/B",
+			expectedAct: []string{
+				"/h/j/m/B/new/b.c.d",
+				"/h/j/m/B/cur/c.d.e:2,S",
+			},
+			expectedDel: []string{
+				"/h/j/m/A/cur/a.b.c:2,",
+				"/h/j/m/C/new/d.e.f",
+			},
+		},
+
+		// refuse to act on multiple files for ActDir and friends if
+		// no current dir is provided
+		{
+			filenames: []string{
+				"/h/j/m/A/cur/a.b.c:2,",
+				"/h/j/m/B/new/b.c.d",
+				"/h/j/m/B/cur/c.d.e:2,S",
+				"/h/j/m/C/new/d.e.f",
+			},
+			strategy:    types.ActDir,
+			curDir:      "",
+			expectedErr: true,
+		},
+		{
+			filenames: []string{
+				"/h/j/m/A/cur/a.b.c:2,",
+				"/h/j/m/B/new/b.c.d",
+				"/h/j/m/B/cur/c.d.e:2,S",
+				"/h/j/m/C/new/d.e.f",
+			},
+			strategy:    types.ActDirDelRest,
+			curDir:      "",
+			expectedErr: true,
+		},
+
+		// act on multiple files w/o current dir for other strategies
+		{
+			filenames: []string{
+				"/h/j/m/A/cur/a.b.c:2,",
+				"/h/j/m/B/new/b.c.d",
+				"/h/j/m/B/cur/c.d.e:2,S",
+				"/h/j/m/C/new/d.e.f",
+			},
+			strategy: types.ActAll,
+			curDir:   "",
+			expectedAct: []string{
+				"/h/j/m/A/cur/a.b.c:2,",
+				"/h/j/m/B/new/b.c.d",
+				"/h/j/m/B/cur/c.d.e:2,S",
+				"/h/j/m/C/new/d.e.f",
+			},
+			expectedDel: []string{},
+		},
+		{
+			filenames: []string{
+				"/h/j/m/A/cur/a.b.c:2,",
+				"/h/j/m/B/new/b.c.d",
+				"/h/j/m/B/cur/c.d.e:2,S",
+				"/h/j/m/C/new/d.e.f",
+			},
+			strategy:    types.ActOne,
+			curDir:      "",
+			expectedAct: []string{"/h/j/m/A/cur/a.b.c:2,"},
+			expectedDel: []string{},
+		},
+		{
+			filenames: []string{
+				"/h/j/m/A/cur/a.b.c:2,",
+				"/h/j/m/B/new/b.c.d",
+				"/h/j/m/B/cur/c.d.e:2,S",
+				"/h/j/m/C/new/d.e.f",
+			},
+			strategy:    types.ActOneDelRest,
+			curDir:      "",
+			expectedAct: []string{"/h/j/m/A/cur/a.b.c:2,"},
+			expectedDel: []string{
+				"/h/j/m/B/new/b.c.d",
+				"/h/j/m/B/cur/c.d.e:2,S",
+				"/h/j/m/C/new/d.e.f",
+			},
+		},
+	}
+
+	for i, test := range tests {
+		act, del, err := filterForStrategy(test.filenames, test.strategy,
+			maildir.Dir(test.curDir))
+
+		if test.expectedErr && err == nil {
+			t.Errorf("[test %d] got nil, expected error", i)
+		}
+
+		if !test.expectedErr && err != nil {
+			t.Errorf("[test %d] got %v, expected nil", i, err)
+		}
+
+		if !arrEq(act, test.expectedAct) {
+			t.Errorf("[test %d] got %v, expected %v", i, act, test.expectedAct)
+		}
+
+		if !arrEq(del, test.expectedDel) {
+			t.Errorf("[test %d] got %v, expected %v", i, del, test.expectedDel)
+		}
+	}
+}
+
+func arrEq(a, b []string) bool {
+	if len(a) != len(b) {
+		return false
+	}
+
+	for i := range a {
+		if a[i] != b[i] {
+			return false
+		}
+	}
+
+	return true
+}
diff --git a/worker/notmuch/worker.go b/worker/notmuch/worker.go
index aa2da391..fe41446e 100644
--- a/worker/notmuch/worker.go
+++ b/worker/notmuch/worker.go
@@ -55,6 +55,7 @@ type worker struct {
 	headers             []string
 	headersExclude      []string
 	state               uint64
+	mfs                 types.MultiFileStrategy
 }
 
 // NewWorker creates a new notmuch worker with the provided worker.
@@ -243,6 +244,16 @@ func (w *worker) handleConfigure(msg *types.Configure) error {
 	w.headers = msg.Config.Headers
 	w.headersExclude = msg.Config.HeadersExclude
 
+	mfs := msg.Config.Params["multi-file-strategy"]
+	if mfs != "" {
+		w.mfs, ok = types.StrToStrategy[mfs]
+		if !ok {
+			return fmt.Errorf("invalid multi-file strategy %s", mfs)
+		}
+	} else {
+		w.mfs = types.Refuse
+	}
+
 	return nil
 }
 
@@ -755,17 +766,12 @@ func (w *worker) handleDeleteMessages(msg *types.DeleteMessages) error {
 
 	var deleted []uint32
 
-	// With notmuch, two identical files can be referenced under
-	// the same index key, even if they exist in two different
-	// folders. So in order to remove the message from the right
-	// maildir folder we need to pass a hint to Remove() so it
-	// can purge the right file.
 	folders, _ := w.store.FolderMap()
-	path, ok := folders[w.currentQueryName]
-	if !ok {
-		w.err(msg, fmt.Errorf("Can only delete file from a maildir folder"))
-		w.done(msg)
-		return nil
+	curDir := folders[w.currentQueryName]
+
+	mfs := w.mfs
+	if msg.MultiFileStrategy != nil {
+		mfs = *msg.MultiFileStrategy
 	}
 
 	for _, uid := range msg.Uids {
@@ -775,7 +781,7 @@ func (w *worker) handleDeleteMessages(msg *types.DeleteMessages) error {
 			w.err(msg, err)
 			continue
 		}
-		if err := m.Remove(path); err != nil {
+		if err := m.Remove(curDir, mfs); err != nil {
 			w.w.Errorf("could not remove message: %v", err)
 			w.err(msg, err)
 			continue
@@ -804,13 +810,20 @@ func (w *worker) handleCopyMessages(msg *types.CopyMessages) error {
 		return fmt.Errorf("Can only copy file to a maildir folder")
 	}
 
+	curDir := folders[w.currentQueryName]
+
+	mfs := w.mfs
+	if msg.MultiFileStrategy != nil {
+		mfs = *msg.MultiFileStrategy
+	}
+
 	for _, uid := range msg.Uids {
 		m, err := w.msgFromUid(uid)
 		if err != nil {
 			w.w.Errorf("could not get message: %v", err)
 			return err
 		}
-		if err := m.Copy(dest); err != nil {
+		if err := m.Copy(curDir, dest, mfs); err != nil {
 			w.w.Errorf("could not copy message: %v", err)
 			return err
 		}
@@ -839,6 +852,13 @@ func (w *worker) handleMoveMessages(msg *types.MoveMessages) error {
 		return fmt.Errorf("Can only move file to a maildir folder")
 	}
 
+	curDir := folders[w.currentQueryName]
+
+	mfs := w.mfs
+	if msg.MultiFileStrategy != nil {
+		mfs = *msg.MultiFileStrategy
+	}
+
 	var err error
 	for _, uid := range msg.Uids {
 		m, err := w.msgFromUid(uid)
@@ -846,22 +866,8 @@ func (w *worker) handleMoveMessages(msg *types.MoveMessages) error {
 			w.w.Errorf("could not get message: %v", err)
 			break
 		}
-		filenames, err := m.db.MsgFilenames(m.key)
-		if err != nil {
-			return err
-		}
-		// In the future, it'd be nice if we could overload move with
-		// the possibility to affect some or all of the files
-		// corresponding to a message.
-		if len(filenames) > 1 {
-			return fmt.Errorf("Cannot move: message %d has multiple files", m.uid)
-		}
-		source, key := parseFilename(filenames[0])
-		if key == "" {
-			return fmt.Errorf("failed to parse message filename: %s", filenames[0])
-		}
-		if err := m.Move(source, dest); err != nil {
-			w.w.Errorf("could not copy message: %v", err)
+		if err := m.Move(curDir, dest, mfs); err != nil {
+			w.w.Errorf("could not move message: %v", err)
 			break
 		}
 		moved = append(moved, uid)
diff --git a/worker/types/messages.go b/worker/types/messages.go
index 90d3d7bb..bbc430ca 100644
--- a/worker/types/messages.go
+++ b/worker/types/messages.go
@@ -167,7 +167,8 @@ type FetchMessageFlags struct {
 
 type DeleteMessages struct {
 	Message
-	Uids []uint32
+	Uids              []uint32
+	MultiFileStrategy *MultiFileStrategy
 }
 
 // Flag messages with different mail types
@@ -186,14 +187,16 @@ type AnsweredMessages struct {
 
 type CopyMessages struct {
 	Message
-	Destination string
-	Uids        []uint32
+	Destination       string
+	Uids              []uint32
+	MultiFileStrategy *MultiFileStrategy
 }
 
 type MoveMessages struct {
 	Message
-	Destination string
-	Uids        []uint32
+	Destination       string
+	Uids              []uint32
+	MultiFileStrategy *MultiFileStrategy
 }
 
 type AppendMessage struct {
diff --git a/worker/types/mfs.go b/worker/types/mfs.go
new file mode 100644
index 00000000..071eda1d
--- /dev/null
+++ b/worker/types/mfs.go
@@ -0,0 +1,33 @@
+package types
+
+// MultiFileStrategy represents a strategy for taking file-based actions (e.g.,
+// move, copy, delete) on messages that are represented by more than one file.
+// These strategies are only used by the notmuch backend but are defined in this
+// package to prevent import cycles.
+type MultiFileStrategy uint
+
+const (
+	Refuse MultiFileStrategy = iota
+	ActAll
+	ActOne
+	ActOneDelRest
+	ActDir
+	ActDirDelRest
+)
+
+var StrToStrategy = map[string]MultiFileStrategy{
+	"refuse":              Refuse,
+	"act-all":             ActAll,
+	"act-one":             ActOne,
+	"act-one-delete-rest": ActOneDelRest,
+	"act-dir":             ActDir,
+	"act-dir-delete-rest": ActDirDelRest,
+}
+
+func StrategyStrs() []string {
+	strs := make([]string, len(StrToStrategy))
+	for s := range StrToStrategy {
+		strs = append(strs, s)
+	}
+	return strs
+}