Merge branch 'jt/partial-clone-submodule-1'
Prepare the internals for lazily fetching objects in submodules
from their promisor remotes.
* jt/partial-clone-submodule-1:
promisor-remote: teach lazy-fetch in any repo
run-command: refactor subprocess env preparation
submodule: refrain from filtering GIT_CONFIG_COUNT
promisor-remote: support per-repository config
repository: move global r_f_p_c to repo struct
diff --git a/Makefile b/Makefile
index c6801ef..c7c46c0 100644
--- a/Makefile
+++ b/Makefile
@@ -729,6 +729,7 @@
TEST_BUILTINS_OBJS += test-online-cpus.o
TEST_BUILTINS_OBJS += test-parse-options.o
TEST_BUILTINS_OBJS += test-parse-pathspec-file.o
+TEST_BUILTINS_OBJS += test-partial-clone.o
TEST_BUILTINS_OBJS += test-path-utils.o
TEST_BUILTINS_OBJS += test-pcre2-config.o
TEST_BUILTINS_OBJS += test-pkt-line.o
diff --git a/object-file.c b/object-file.c
index b9c3219..ecca5a8 100644
--- a/object-file.c
+++ b/object-file.c
@@ -1584,15 +1584,12 @@
}
/* Check if it is a missing object */
- if (fetch_if_missing && has_promisor_remote() &&
- !already_retried && r == the_repository &&
+ if (fetch_if_missing && repo_has_promisor_remote(r) &&
+ !already_retried &&
!(flags & OBJECT_INFO_SKIP_FETCH_OBJECT)) {
/*
* TODO Investigate checking promisor_remote_get_direct()
* TODO return value and stopping on error here.
- * TODO Pass a repository struct through
- * promisor_remote_get_direct(), such that arbitrary
- * repositories work.
*/
promisor_remote_get_direct(r, real, 1);
already_retried = 1;
diff --git a/promisor-remote.c b/promisor-remote.c
index d465377..db2ebdc 100644
--- a/promisor-remote.c
+++ b/promisor-remote.c
@@ -5,12 +5,10 @@
#include "transport.h"
#include "strvec.h"
-static char *repository_format_partial_clone;
-
-void set_repository_format_partial_clone(char *partial_clone)
-{
- repository_format_partial_clone = xstrdup_or_null(partial_clone);
-}
+struct promisor_remote_config {
+ struct promisor_remote *promisors;
+ struct promisor_remote **promisors_tail;
+};
static int fetch_objects(struct repository *repo,
const char *remote_name,
@@ -23,6 +21,8 @@
child.git_cmd = 1;
child.in = -1;
+ if (repo != the_repository)
+ prepare_other_repo_env(&child.env_array, repo->gitdir);
strvec_pushl(&child.args, "-c", "fetch.negotiationAlgorithm=noop",
"fetch", remote_name, "--no-tags",
"--no-write-fetch-head", "--recurse-submodules=no",
@@ -45,10 +45,8 @@
return finish_command(&child) ? -1 : 0;
}
-static struct promisor_remote *promisors;
-static struct promisor_remote **promisors_tail = &promisors;
-
-static struct promisor_remote *promisor_remote_new(const char *remote_name)
+static struct promisor_remote *promisor_remote_new(struct promisor_remote_config *config,
+ const char *remote_name)
{
struct promisor_remote *r;
@@ -60,18 +58,19 @@
FLEX_ALLOC_STR(r, name, remote_name);
- *promisors_tail = r;
- promisors_tail = &r->next;
+ *config->promisors_tail = r;
+ config->promisors_tail = &r->next;
return r;
}
-static struct promisor_remote *promisor_remote_lookup(const char *remote_name,
+static struct promisor_remote *promisor_remote_lookup(struct promisor_remote_config *config,
+ const char *remote_name,
struct promisor_remote **previous)
{
struct promisor_remote *r, *p;
- for (p = NULL, r = promisors; r; p = r, r = r->next)
+ for (p = NULL, r = config->promisors; r; p = r, r = r->next)
if (!strcmp(r->name, remote_name)) {
if (previous)
*previous = p;
@@ -81,7 +80,8 @@
return NULL;
}
-static void promisor_remote_move_to_tail(struct promisor_remote *r,
+static void promisor_remote_move_to_tail(struct promisor_remote_config *config,
+ struct promisor_remote *r,
struct promisor_remote *previous)
{
if (r->next == NULL)
@@ -90,14 +90,15 @@
if (previous)
previous->next = r->next;
else
- promisors = r->next ? r->next : r;
+ config->promisors = r->next ? r->next : r;
r->next = NULL;
- *promisors_tail = r;
- promisors_tail = &r->next;
+ *config->promisors_tail = r;
+ config->promisors_tail = &r->next;
}
static int promisor_remote_config(const char *var, const char *value, void *data)
{
+ struct promisor_remote_config *config = data;
const char *name;
size_t namelen;
const char *subkey;
@@ -113,8 +114,8 @@
remote_name = xmemdupz(name, namelen);
- if (!promisor_remote_lookup(remote_name, NULL))
- promisor_remote_new(remote_name);
+ if (!promisor_remote_lookup(config, remote_name, NULL))
+ promisor_remote_new(config, remote_name);
free(remote_name);
return 0;
@@ -123,9 +124,9 @@
struct promisor_remote *r;
char *remote_name = xmemdupz(name, namelen);
- r = promisor_remote_lookup(remote_name, NULL);
+ r = promisor_remote_lookup(config, remote_name, NULL);
if (!r)
- r = promisor_remote_new(remote_name);
+ r = promisor_remote_new(config, remote_name);
free(remote_name);
@@ -138,59 +139,63 @@
return 0;
}
-static int initialized;
-
-static void promisor_remote_init(void)
+static void promisor_remote_init(struct repository *r)
{
- if (initialized)
+ struct promisor_remote_config *config;
+
+ if (r->promisor_remote_config)
return;
- initialized = 1;
+ config = r->promisor_remote_config =
+ xcalloc(sizeof(*r->promisor_remote_config), 1);
+ config->promisors_tail = &config->promisors;
- git_config(promisor_remote_config, NULL);
+ repo_config(r, promisor_remote_config, config);
- if (repository_format_partial_clone) {
+ if (r->repository_format_partial_clone) {
struct promisor_remote *o, *previous;
- o = promisor_remote_lookup(repository_format_partial_clone,
+ o = promisor_remote_lookup(config,
+ r->repository_format_partial_clone,
&previous);
if (o)
- promisor_remote_move_to_tail(o, previous);
+ promisor_remote_move_to_tail(config, o, previous);
else
- promisor_remote_new(repository_format_partial_clone);
+ promisor_remote_new(config, r->repository_format_partial_clone);
}
}
-static void promisor_remote_clear(void)
+void promisor_remote_clear(struct promisor_remote_config *config)
{
- while (promisors) {
- struct promisor_remote *r = promisors;
- promisors = promisors->next;
+ while (config->promisors) {
+ struct promisor_remote *r = config->promisors;
+ config->promisors = config->promisors->next;
free(r);
}
- promisors_tail = &promisors;
+ config->promisors_tail = &config->promisors;
}
-void promisor_remote_reinit(void)
+void repo_promisor_remote_reinit(struct repository *r)
{
- initialized = 0;
- promisor_remote_clear();
- promisor_remote_init();
+ promisor_remote_clear(r->promisor_remote_config);
+ FREE_AND_NULL(r->promisor_remote_config);
+ promisor_remote_init(r);
}
-struct promisor_remote *promisor_remote_find(const char *remote_name)
+struct promisor_remote *repo_promisor_remote_find(struct repository *r,
+ const char *remote_name)
{
- promisor_remote_init();
+ promisor_remote_init(r);
if (!remote_name)
- return promisors;
+ return r->promisor_remote_config->promisors;
- return promisor_remote_lookup(remote_name, NULL);
+ return promisor_remote_lookup(r->promisor_remote_config, remote_name, NULL);
}
-int has_promisor_remote(void)
+int repo_has_promisor_remote(struct repository *r)
{
- return !!promisor_remote_find(NULL);
+ return !!repo_promisor_remote_find(r, NULL);
}
static int remove_fetched_oids(struct repository *repo,
@@ -238,9 +243,9 @@
if (oid_nr == 0)
return 0;
- promisor_remote_init();
+ promisor_remote_init(repo);
- for (r = promisors; r; r = r->next) {
+ for (r = repo->promisor_remote_config->promisors; r; r = r->next) {
if (fetch_objects(repo, r->name, remaining_oids, remaining_nr) < 0) {
if (remaining_nr == 1)
continue;
diff --git a/promisor-remote.h b/promisor-remote.h
index c7a1406..edc45ab 100644
--- a/promisor-remote.h
+++ b/promisor-remote.h
@@ -17,9 +17,25 @@
const char name[FLEX_ARRAY];
};
-void promisor_remote_reinit(void);
-struct promisor_remote *promisor_remote_find(const char *remote_name);
-int has_promisor_remote(void);
+void repo_promisor_remote_reinit(struct repository *r);
+static inline void promisor_remote_reinit(void)
+{
+ repo_promisor_remote_reinit(the_repository);
+}
+
+void promisor_remote_clear(struct promisor_remote_config *config);
+
+struct promisor_remote *repo_promisor_remote_find(struct repository *r, const char *remote_name);
+static inline struct promisor_remote *promisor_remote_find(const char *remote_name)
+{
+ return repo_promisor_remote_find(the_repository, remote_name);
+}
+
+int repo_has_promisor_remote(struct repository *r);
+static inline int has_promisor_remote(void)
+{
+ return repo_has_promisor_remote(the_repository);
+}
/*
* Fetches all requested objects from all promisor remotes, trying them one at
@@ -32,10 +48,4 @@
const struct object_id *oids,
int oid_nr);
-/*
- * This should be used only once from setup.c to set the value we got
- * from the extensions.partialclone config option.
- */
-void set_repository_format_partial_clone(char *partial_clone);
-
#endif /* PROMISOR_REMOTE_H */
diff --git a/repository.c b/repository.c
index 448cd55..b2bf44c 100644
--- a/repository.c
+++ b/repository.c
@@ -11,6 +11,7 @@
#include "lockfile.h"
#include "submodule-config.h"
#include "sparse-index.h"
+#include "promisor-remote.h"
/* The main repository */
static struct repository the_repo;
@@ -172,6 +173,10 @@
repo_set_hash_algo(repo, format.hash_algo);
+ /* take ownership of format.partial_clone */
+ repo->repository_format_partial_clone = format.partial_clone;
+ format.partial_clone = NULL;
+
if (worktree)
repo_set_worktree(repo, worktree);
@@ -258,6 +263,11 @@
if (repo->index != &the_index)
FREE_AND_NULL(repo->index);
}
+
+ if (repo->promisor_remote_config) {
+ promisor_remote_clear(repo->promisor_remote_config);
+ FREE_AND_NULL(repo->promisor_remote_config);
+ }
}
int repo_read_index(struct repository *repo)
diff --git a/repository.h b/repository.h
index a45f752..3740c93 100644
--- a/repository.h
+++ b/repository.h
@@ -10,6 +10,7 @@
struct pathspec;
struct raw_object_store;
struct submodule_cache;
+struct promisor_remote_config;
enum untracked_cache_setting {
UNTRACKED_CACHE_UNSET = -1,
@@ -139,6 +140,10 @@
/* True if commit-graph has been disabled within this process. */
int commit_graph_disabled;
+ /* Configurations related to promisor remotes. */
+ char *repository_format_partial_clone;
+ struct promisor_remote_config *promisor_remote_config;
+
/* Configurations */
/* Indicate if a repository has a different 'commondir' from 'gitdir' */
diff --git a/run-command.c b/run-command.c
index 8750df1..f72e72c 100644
--- a/run-command.c
+++ b/run-command.c
@@ -1891,3 +1891,15 @@
return run_command(&maint);
}
+
+void prepare_other_repo_env(struct strvec *env_array, const char *new_git_dir)
+{
+ const char * const *var;
+
+ for (var = local_repo_env; *var; var++) {
+ if (strcmp(*var, CONFIG_DATA_ENVIRONMENT) &&
+ strcmp(*var, CONFIG_COUNT_ENVIRONMENT))
+ strvec_push(env_array, *var);
+ }
+ strvec_pushf(env_array, "%s=%s", GIT_DIR_ENVIRONMENT, new_git_dir);
+}
diff --git a/run-command.h b/run-command.h
index 62a922d..af12967 100644
--- a/run-command.h
+++ b/run-command.h
@@ -486,4 +486,14 @@
task_finished_fn, void *pp_cb,
const char *tr2_category, const char *tr2_label);
+/**
+ * Convenience function which prepares env_array for a command to be run in a
+ * new repo. This adds all GIT_* environment variables to env_array with the
+ * exception of GIT_CONFIG_PARAMETERS and GIT_CONFIG_COUNT (which cause the
+ * corresponding environment variables to be unset in the subprocess) and adds
+ * an environment variable pointing to new_git_dir. See local_repo_env in
+ * cache.h for more information.
+ */
+void prepare_other_repo_env(struct strvec *env_array, const char *new_git_dir);
+
#endif
diff --git a/setup.c b/setup.c
index ead2f80..eb9367c 100644
--- a/setup.c
+++ b/setup.c
@@ -468,8 +468,6 @@
data->precious_objects = git_config_bool(var, value);
return EXTENSION_OK;
} else if (!strcmp(ext, "partialclone")) {
- if (!value)
- return config_error_nonbool(var);
data->partial_clone = xstrdup(value);
return EXTENSION_OK;
} else if (!strcmp(ext, "worktreeconfig")) {
@@ -566,7 +564,6 @@
}
repository_format_precious_objects = candidate->precious_objects;
- set_repository_format_partial_clone(candidate->partial_clone);
repository_format_worktree_config = candidate->worktree_config;
string_list_clear(&candidate->unknown_extensions, 0);
string_list_clear(&candidate->v1_only_extensions, 0);
@@ -1197,6 +1194,11 @@
return -1;
}
+ /* take ownership of candidate.partial_clone */
+ the_repository->repository_format_partial_clone =
+ candidate.partial_clone;
+ candidate.partial_clone = NULL;
+
clear_repository_format(&candidate);
return 0;
}
@@ -1304,8 +1306,13 @@
gitdir = DEFAULT_GIT_DIR_ENVIRONMENT;
setup_git_env(gitdir);
}
- if (startup_info->have_repository)
+ if (startup_info->have_repository) {
repo_set_hash_algo(the_repository, repo_fmt.hash_algo);
+ /* take ownership of repo_fmt.partial_clone */
+ the_repository->repository_format_partial_clone =
+ repo_fmt.partial_clone;
+ repo_fmt.partial_clone = NULL;
+ }
}
/*
* Since precompose_string_if_needed() needs to look at
@@ -1390,6 +1397,8 @@
check_repository_format_gently(get_git_dir(), fmt, NULL);
startup_info->have_repository = 1;
repo_set_hash_algo(the_repository, fmt->hash_algo);
+ the_repository->repository_format_partial_clone =
+ xstrdup_or_null(fmt->partial_clone);
clear_repository_format(&repo_fmt);
}
diff --git a/submodule.c b/submodule.c
index 0b1d9c1..8e611fe 100644
--- a/submodule.c
+++ b/submodule.c
@@ -484,27 +484,14 @@
strbuf_release(&sb);
}
-static void prepare_submodule_repo_env_no_git_dir(struct strvec *out)
-{
- const char * const *var;
-
- for (var = local_repo_env; *var; var++) {
- if (strcmp(*var, CONFIG_DATA_ENVIRONMENT))
- strvec_push(out, *var);
- }
-}
-
void prepare_submodule_repo_env(struct strvec *out)
{
- prepare_submodule_repo_env_no_git_dir(out);
- strvec_pushf(out, "%s=%s", GIT_DIR_ENVIRONMENT,
- DEFAULT_GIT_DIR_ENVIRONMENT);
+ prepare_other_repo_env(out, DEFAULT_GIT_DIR_ENVIRONMENT);
}
static void prepare_submodule_repo_env_in_gitdir(struct strvec *out)
{
- prepare_submodule_repo_env_no_git_dir(out);
- strvec_pushf(out, "%s=.", GIT_DIR_ENVIRONMENT);
+ prepare_other_repo_env(out, ".");
}
/*
diff --git a/t/helper/test-partial-clone.c b/t/helper/test-partial-clone.c
new file mode 100644
index 0000000..3f102cf
--- /dev/null
+++ b/t/helper/test-partial-clone.c
@@ -0,0 +1,43 @@
+#include "cache.h"
+#include "test-tool.h"
+#include "repository.h"
+#include "object-store.h"
+
+/*
+ * Prints the size of the object corresponding to the given hash in a specific
+ * gitdir. This is similar to "git -C gitdir cat-file -s", except that this
+ * exercises the code that accesses the object of an arbitrary repository that
+ * is not the_repository. ("git -C gitdir" makes it so that the_repository is
+ * the one in gitdir.)
+ */
+static void object_info(const char *gitdir, const char *oid_hex)
+{
+ struct repository r;
+ struct object_id oid;
+ unsigned long size;
+ struct object_info oi = {.sizep = &size};
+ const char *p;
+
+ if (repo_init(&r, gitdir, NULL))
+ die("could not init repo");
+ if (parse_oid_hex(oid_hex, &oid, &p))
+ die("could not parse oid");
+ if (oid_object_info_extended(&r, &oid, &oi, 0))
+ die("could not obtain object info");
+ printf("%d\n", (int) size);
+}
+
+int cmd__partial_clone(int argc, const char **argv)
+{
+ setup_git_directory();
+
+ if (argc < 4)
+ die("too few arguments");
+
+ if (!strcmp(argv[1], "object-info"))
+ object_info(argv[2], argv[3]);
+ else
+ die("invalid argument '%s'", argv[1]);
+
+ return 0;
+}
diff --git a/t/helper/test-tool.c b/t/helper/test-tool.c
index c5bd0c6..b21e8f1 100644
--- a/t/helper/test-tool.c
+++ b/t/helper/test-tool.c
@@ -46,6 +46,7 @@
{ "online-cpus", cmd__online_cpus },
{ "parse-options", cmd__parse_options },
{ "parse-pathspec-file", cmd__parse_pathspec_file },
+ { "partial-clone", cmd__partial_clone },
{ "path-utils", cmd__path_utils },
{ "pcre2-config", cmd__pcre2_config },
{ "pkt-line", cmd__pkt_line },
diff --git a/t/helper/test-tool.h b/t/helper/test-tool.h
index e8069a3..f845ced 100644
--- a/t/helper/test-tool.h
+++ b/t/helper/test-tool.h
@@ -35,6 +35,7 @@
int cmd__online_cpus(int argc, const char **argv);
int cmd__parse_options(int argc, const char **argv);
int cmd__parse_pathspec_file(int argc, const char** argv);
+int cmd__partial_clone(int argc, const char **argv);
int cmd__path_utils(int argc, const char **argv);
int cmd__pcre2_config(int argc, const char **argv);
int cmd__pkt_line(int argc, const char **argv);
diff --git a/t/t0410-partial-clone.sh b/t/t0410-partial-clone.sh
index 584a039..a211a66 100755
--- a/t/t0410-partial-clone.sh
+++ b/t/t0410-partial-clone.sh
@@ -604,6 +604,29 @@
git -C repo cherry-pick side1
'
+test_expect_success 'lazy-fetch when accessing object not in the_repository' '
+ rm -rf full partial.git &&
+ test_create_repo full &&
+ test_commit -C full create-a-file file.txt &&
+
+ test_config -C full uploadpack.allowfilter 1 &&
+ test_config -C full uploadpack.allowanysha1inwant 1 &&
+ git clone --filter=blob:none --bare "file://$(pwd)/full" partial.git &&
+ FILE_HASH=$(git -C full rev-parse HEAD:file.txt) &&
+
+ # Sanity check that the file is missing
+ git -C partial.git rev-list --objects --missing=print HEAD >out &&
+ grep "[?]$FILE_HASH" out &&
+
+ git -C full cat-file -s "$FILE_HASH" >expect &&
+ test-tool partial-clone object-info partial.git "$FILE_HASH" >actual &&
+ test_cmp expect actual &&
+
+ # Sanity check that the file is now present
+ git -C partial.git rev-list --objects --missing=print HEAD >out &&
+ ! grep "[?]$FILE_HASH" out
+'
+
. "$TEST_DIRECTORY"/lib-httpd.sh
start_httpd
