fetch-pack: accept absolute-path packfile URIs

Packfile URIs require a scheme and authority even when the pack and the
repository are served by the same HTTP(S) server.

Allow the server to advertise an absolute-path reference, as defined by
RFC 3986, section 4.2 [1]. Resolve it using the remote's scheme and
authority, following section 5.2.2 [2]. For example, /foo.pack from
https://example.com/repo.git resolves to https://example.com/foo.pack.
Packfile URIs do not support local paths, so there is no ambiguity with
a local file named /foo.pack.

Add an "absolute-path" token to the packfile-uris protocol list. Request
it only for HTTP(S) remotes whose scheme is allowed by
fetch.uriprotocols.
Without the token, keep the corresponding objects in the inline pack so
older clients can still fetch them.

Handle only references beginning with a single slash. Redact their paths
in packet traces, as we do for absolute URIs.

[1] https://www.rfc-editor.org/rfc/rfc3986.html#section-4.2
[2] https://www.rfc-editor.org/rfc/rfc3986.html#section-5.2.2

Signed-off-by: Friel <friel@openai.com>

Author: friel-openai

Documentation/gitprotocol-v2.adoc

@@ -493,6 +493,17 @@ header. Most sections are sent only when the packfile is sent.
 	* For each URI the server sends, it sends a hash of the pack's
 	  contents (as output by git index-pack) followed by the URI.
 
+	* The client may include `absolute-path` in its `packfile-uris`
+	  list to accept references beginning with a single `/`. These
+	  inherit the remote URL's scheme and authority, replacing its
+	  path, query, and fragment. For example, `/foo.pack` from
+	  `https://example.com/repo.git` resolves to
+	  `https://example.com/foo.pack`. The client MUST only request
+	  `absolute-path` when the remote uses an accepted HTTP or HTTPS
+	  scheme. The server MUST NOT send these references unless the
+	  client requests `absolute-path`. References beginning with `//`
+	  and relative paths without a leading `/` are not supported.
+
 	* The hashes are 40 hex characters long. When Git upgrades to a new
 	  hash algorithm, this might need to be updated. (It should match
 	  whatever index-pack outputs after "pack\t" or "keep\t".

builtin/fetch-pack.c

@@ -187,6 +187,7 @@ int cmd_fetch_pack(int argc,
 		dest = argv[i++];
 	else
 		usage(fetch_pack_usage);
+	args.url = dest;
 
 	/*
 	 * Copy refs from cmdline to growable list, then append any

builtin/pack-objects.c

@@ -1815,10 +1815,11 @@ static int want_object_in_pack_mtime(const struct object_id *oid,
 
 		if (ex) {
 			for (i = 0; i < uri_protocols.nr; i++) {
-				if (skip_prefix(ex->uri,
-						uri_protocols.items[i].string,
-						&p) &&
-				    *p == ':') {
+				const char *protocol = uri_protocols.items[i].string;
+
+				if ((ex->uri[0] == '/' && ex->uri[1] != '/' &&
+				     !strcmp(protocol, "absolute-path")) ||
+				    (skip_prefix(ex->uri, protocol, &p) && *p == ':')) {
 					oidset_insert(&excluded_by_config, oid);
 					return 0;
 				}

fetch-pack.c

@@ -37,6 +37,7 @@
 #include "mergesort.h"
 #include "prio-queue.h"
 #include "promisor-remote.h"
+#include "urlmatch.h"
 
 static int transfer_unpack_limit = -1;
 static int fetch_unpack_limit = -1;
@@ -1375,6 +1376,29 @@ static int add_haves(struct fetch_negotiator *negotiator,
 	return haves_added;
 }
 
+static char *get_packfile_uri_base(const char *url)
+{
+	struct url_info info;
+	char *base;
+
+	if (!url || !uri_protocols.nr)
+		return NULL;
+	base = url_normalize(url, &info);
+	if (!base)
+		return NULL;
+
+	/* An absolute path must not bypass the configured URI protocols. */
+	if ((starts_with(base, "http:") &&
+	     unsorted_string_list_has_string(&uri_protocols, "http")) ||
+	    (starts_with(base, "https:") &&
+	     unsorted_string_list_has_string(&uri_protocols, "https"))) {
+		base[info.path_off] = '\0';
+		return base;
+	}
+	free(base);
+	return NULL;
+}
+
 static int send_fetch_request(struct fetch_negotiator *negotiator, int fd_out,
 			      struct fetch_pack_args *args,
 			      const struct ref *wants, struct oidset *common,
@@ -1423,6 +1447,11 @@ static int send_fetch_request(struct fetch_negotiator *negotiator, int fd_out,
 			}
 		}
 		if (to_send.len) {
+			char *uri_base = get_packfile_uri_base(args->url);
+
+			if (uri_base)
+				strbuf_addstr(&to_send, ",absolute-path");
+			free(uri_base);
 			packet_buf_write(&req_buf, "packfile-uris %s",
 					 to_send.buf);
 			strbuf_release(&to_send);
@@ -1639,18 +1668,32 @@ static void receive_wanted_refs(struct packet_reader *reader,
 }
 
 static void receive_packfile_uris(struct packet_reader *reader,
-				  struct string_list *uris)
+				  struct string_list *uris,
+				  const char *url)
 {
+	char *uri_base = get_packfile_uri_base(url);
+
 	process_section_header(reader, "packfile-uris", 0);
 	while (packet_reader_read(reader) == PACKET_READ_NORMAL) {
-		if (reader->pktlen < the_hash_algo->hexsz ||
-		    reader->line[the_hash_algo->hexsz] != ' ')
+		struct object_id oid;
+		const char *end;
+
+		if (parse_oid_hex(reader->line, &oid, &end) || *end != ' ')
 			die("expected '<hash> <uri>', got: %s", reader->line);
 
-		string_list_append(uris, reader->line);
+		if (end[1] == '/') {
+			if (end[2] == '/' || !uri_base)
+				die("unexpected relative packfile URI");
+			string_list_append_nodup(uris,
+						 xstrfmt("%s %s%s", oid_to_hex(&oid),
+							 uri_base, end + 1));
+		} else {
+			string_list_append(uris, reader->line);
+		}
 	}
 	if (reader->status != PACKET_READ_DELIM)
 		die("expected DELIM");
+	free(uri_base);
 }
 
 enum fetch_state {
@@ -1826,7 +1869,8 @@ static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args,
 			if (git_env_bool("GIT_TRACE_REDACT", 1))
 				reader.options |= PACKET_READ_REDACT_URI_PATH;
 			if (process_section_header(&reader, "packfile-uris", 1))
-				receive_packfile_uris(&reader, &packfile_uris);
+				receive_packfile_uris(&reader, &packfile_uris,
+						      args->url);
 			/* We don't expect more URIs. Reset to avoid expensive URI check. */
 			reader.options &= ~PACKET_READ_REDACT_URI_PATH;
 

fetch-pack.h

@@ -10,6 +10,8 @@ struct oid_array;
 
 struct fetch_pack_args {
 	const char *uploadpack;
+	/* Remote URL used to resolve absolute-path packfile URIs. */
+	const char *url;
 	int unpacklimit;
 	int depth;
 	const char *deepen_since;

pkt-line.c

@@ -398,7 +398,11 @@ static const char *find_packfile_uri_path(const char *buffer)
 	if (!(len == 40 || len == 64) || buffer[len] != ' ')
 		return NULL; /* required "<hash>SP" not seen */
 
-	path = strstr(buffer + len + 1, URI_MARK);
+	buffer += len + 1;
+	if (buffer[0] == '/' && buffer[1] != '/')
+		return buffer + 1;
+
+	path = strstr(buffer, URI_MARK);
 	if (!path)
 		return NULL;
 

t/t5702-protocol-v2.sh

@@ -1217,10 +1217,79 @@ configure_exclusion () {
 	git -C "$1" pack-objects "$HTTPD_DOCUMENT_ROOT_PATH/mypack" <objh >packh &&
 	git -C "$1" config --add \
 		"uploadpack.blobpackfileuri" \
-		"$(cat objh) $(cat packh) $HTTPD_URL/dumb/mypack-$(cat packh).pack" &&
+		"$(cat objh) $(cat packh) ${3-$HTTPD_URL}/dumb/mypack-$(cat packh).pack" &&
 	cat objh
 }
 
+test_expect_success 'setup absolute-path packfile URIs' '
+	P="$HTTPD_DOCUMENT_ROOT_PATH/absolute-uri" &&
+	git init "$P" &&
+	git -C "$P" config uploadpack.allowsidebandall true &&
+	>absolute-pack-hashes &&
+	for name in one two
+	do
+		test_commit -C "$P" "$name" &&
+		configure_exclusion "$P" "$name.t" "" >/dev/null &&
+		cat packh >>absolute-pack-hashes || return 1
+	done
+'
+
+test_expect_success 'clone with absolute-path packfile URIs' '
+	test_when_finished "rm -rf absolute-child log" &&
+	GIT_TRACE_PACKET="$TRASH_DIRECTORY/log" GIT_TRACE_REDACT=0 \
+	GIT_TEST_SIDEBAND_ALL=1 \
+	git -c protocol.version=2 -c fetch.uriprotocols=http,https \
+		clone "$HTTPD_URL/smart/absolute-uri" absolute-child &&
+	while read hash
+	do
+		test_path_is_file \
+			"absolute-child/.git/objects/pack/pack-$hash.pack" &&
+		test_grep -F "clone< \\1$hash /dumb/mypack-$hash.pack" log ||
+		return 1
+	done <absolute-pack-hashes &&
+	test_grep "packfile-uris http,https,absolute-path" log &&
+	git -C absolute-child fsck
+'
+
+test_expect_success 'absolute-path packfile URI fetch redacts the path' '
+	test_when_finished "rm -rf absolute-child log" &&
+	git init absolute-child &&
+	GIT_TRACE_PACKET="$TRASH_DIRECTORY/log" GIT_TEST_SIDEBAND_ALL=1 \
+	git -C absolute-child -c protocol.version=2 \
+		-c fetch.uriprotocols=http,https \
+		fetch "$HTTPD_URL/smart/absolute-uri" &&
+	while read hash
+	do
+		test_grep -F "fetch< \\1$hash /<redacted>" log || return 1
+	done <absolute-pack-hashes &&
+	test_grep ! /dumb/mypack- log
+'
+
+test_expect_success 'absolute-path packfile URIs require an allowed HTTP scheme' '
+	test_when_finished "rm -rf absolute-child log" &&
+	case "$HTTPD_PROTO" in
+	http) other_protocol=https ;;
+	https) other_protocol=http ;;
+	esac &&
+	for url in "$HTTPD_URL/smart/absolute-uri" \
+		"file://$HTTPD_DOCUMENT_ROOT_PATH/absolute-uri"
+	do
+		GIT_TRACE_PACKET="$TRASH_DIRECTORY/log" GIT_TEST_SIDEBAND_ALL=1 \
+		git -c protocol.version=2 -c fetch.uriprotocols=$other_protocol \
+			clone "$url" absolute-child &&
+		test_grep "packfile-uris $other_protocol$" log &&
+		test_grep ! "packfile-uris .*absolute-path" log &&
+		while read hash
+		do
+			test_path_is_missing \
+				"absolute-child/.git/objects/pack/pack-$hash.pack" ||
+			return 1
+		done <absolute-pack-hashes &&
+		git -C absolute-child fsck &&
+		rm -rf absolute-child log || return 1
+	done
+'
+
 test_expect_success 'part of packfile response provided as URI' '
 	P="$HTTPD_DOCUMENT_ROOT_PATH/http_parent" &&
 	rm -rf "$P" http_child log &&

transport.c

@@ -487,6 +487,7 @@ static int fetch_refs_via_pack(struct transport *transport,
 
 	memset(&args, 0, sizeof(args));
 	args.uploadpack = data->options.uploadpack;
+	args.url = transport->url;
 	args.keep_pack = data->options.keep;
 	args.lock_pack = 1;
 	args.use_thin_pack = data->options.thin;