dir: skip excluded directory with nested repo on prefix match

95c11ecc73 (Fix error-prone fill_directory() API; make it only return
matches, 2020-04-01) optimized away pathspec matching too eagerly and
cada7308ad (dir: check pathspecs before returning `path_excluded`,
2020-07-20) fixed that for files and symlinks.

A corner case remained unaddressed: Ignored directories that contain a
nested repository.  Make sure to run match_pathspec_with_flags() for it
if necessary.

Add a status test to check whether the pathspec is applied correctly as
well as a checkout test to exercise the changed code without a pathspec.

Reported-by: Sean Whitton <spwhitton@spwhitton.name>
Reported-by: Lester Longley <lester@ieee.org>
Helped-by: Junio C Hamano <gitster@pobox.com>
Signed-off-by: René Scharfe <l.s.r@web.de>
Signed-off-by: Junio C Hamano <gitster@pobox.com>

dir.c

diff --git a/dir.c b/dir.c
index 3243009..d5d6272 100644
--- a/dir.c
+++ b/dir.c
@@ -1930,6 +1930,16 @@ static enum exist_status directory_exists_in_index(struct index_state *istate,
 	return index_nonexistent;
 }
 
+static int dir_match(struct index_state *istate,
+		     const struct pathspec *pathspec,
+		     const char *dirname, int len)
+{
+	return match_pathspec_with_flags(istate, pathspec, dirname, len,
+					 0 /* prefix */,
+					 NULL /* seen */,
+					 DO_MATCH_LEADING_PATHSPEC);
+}
+
 /*
  * When we find a directory when traversing the filesystem, we
  * have three distinct cases:
@@ -1996,11 +2006,7 @@ static enum path_treatment treat_directory(struct dir_struct *dir,
 	 * for matching patterns.
 	 */
 	if (pathspec && !excluded) {
-		matches_how = match_pathspec_with_flags(istate, pathspec,
-							dirname, len,
-							0 /* prefix */,
-							NULL /* seen */,
-							DO_MATCH_LEADING_PATHSPEC);
+		matches_how = dir_match(istate, pathspec, dirname, len);
 		if (!matches_how)
 			return path_none;
 	}
@@ -2034,8 +2040,15 @@ static enum path_treatment treat_directory(struct dir_struct *dir,
 		strbuf_release(&sb);
 
 		if (nested_repo) {
-			if ((dir->flags & DIR_SKIP_NESTED_GIT) ||
-				(matches_how == MATCHED_RECURSIVELY_LEADING_PATHSPEC))
+			if (dir->flags & DIR_SKIP_NESTED_GIT)
+				return path_none;
+			if (pathspec && !matches_how) {
+				matches_how = dir_match(istate, pathspec,
+							dirname, len);
+				if (!matches_how)
+					return path_none;
+			}
+			if (matches_how == MATCHED_RECURSIVELY_LEADING_PATHSPEC)
 				return path_none;
 			return excluded ? path_excluded : path_untracked;
 		}

t/t2021-checkout-overwrite.sh

diff --git a/t/t2021-checkout-overwrite.sh b/t/t2021-checkout-overwrite.sh
index 38c41ae..2b7fff5 100755
--- a/t/t2021-checkout-overwrite.sh
+++ b/t/t2021-checkout-overwrite.sh
@@ -79,4 +79,11 @@
 	test_path_is_file some_dir
 '
 
+test_expect_success 'checkout must not overwrite untracked nested repo' '
+	git checkout -f start &&
+	rm -rf some_dir &&
+	git init some_dir &&
+	test_must_fail git checkout df_conflict
+'
+
 test_done

t/t7061-wtstatus-ignore.sh

diff --git a/t/t7061-wtstatus-ignore.sh b/t/t7061-wtstatus-ignore.sh
index 14ddaba..1829334 100755
--- a/t/t7061-wtstatus-ignore.sh
+++ b/t/t7061-wtstatus-ignore.sh
@@ -340,4 +340,9 @@
 	test_cmp expected actual
 '
 
+test_expect_success 'status omits excluded directory with submodule on prefix match' '
+	git status --porcelain --ignored -u tracke >actual &&
+	test_must_be_empty actual
+'
+
 test_done