| | @@ -13,13 +13,25 @@ use std::{ |
| 13 | 13 | sync::{Arc, Mutex}, |
| 14 | 14 | }; |
| 15 | 15 | |
| 16 | | −use tokio::process::Command; |
| 16 | +use tokio::{ |
| 17 | + io::{AsyncBufReadExt, AsyncReadExt, BufReader}, |
| 18 | + process::{ChildStdout, Command}, |
| 19 | +}; |
| 17 | 20 | |
| 18 | 21 | use crate::{ |
| 19 | | − application::port::{GitStorage, GitStorageError}, |
| 22 | + application::port::{ |
| 23 | + GitMethod, GitProtocolError, GitProtocolServer, GitRequest, GitResponse, GitStorage, |
| 24 | + GitStorageError, |
| 25 | + }, |
| 20 | 26 | domain::{OrgName, RepoName}, |
| 21 | 27 | }; |
| 22 | 28 | |
| 29 | +/// The most CGI headers `git http-backend` will ever emit, with room to spare. |
| 30 | +/// |
| 31 | +/// A guard rather than a real expectation: the header block is read before anything is |
| 32 | +/// streamed, and an unbounded read of a subprocess's stdout is a hang waiting to happen. |
| 33 | +const MAX_CGI_HEADERS: usize = 64; |
| 34 | + |
| 23 | 35 | /// Environment variables that redirect where git reads and writes data. |
| 24 | 36 | /// |
| 25 | 37 | /// Steid's own environment must not reach into a repository's layout. These are set |
| | @@ -104,6 +116,28 @@ impl GitStorage for DiskGitStorage { |
| 104 | 116 | } |
| 105 | 117 | } |
| 106 | 118 | |
| 119 | +/// A `git` command isolated from the host. |
| 120 | +/// |
| 121 | +/// The one place that decides what git inherits: no ambient configuration, no |
| 122 | +/// redirected object storage. Both the lifecycle commands and the protocol backend |
| 123 | +/// build on this, which is the point — 0006 exists because these flags are exactly what |
| 124 | +/// drifts silently between call sites. |
| 125 | +fn git_command() -> Command { |
| 126 | + let mut command = Command::new("git"); |
| 127 | + |
| 128 | + // Host configuration must not leak into repositories Steid creates, for the same |
| 129 | + // reason `--initial-branch` is passed explicitly. |
| 130 | + command |
| 131 | + .env("GIT_CONFIG_GLOBAL", "/dev/null") |
| 132 | + .env("GIT_CONFIG_SYSTEM", "/dev/null"); |
| 133 | + |
| 134 | + for variable in REDIRECTING_VARS { |
| 135 | + command.env_remove(variable); |
| 136 | + } |
| 137 | + |
| 138 | + command |
| 139 | +} |
| 140 | + |
| 107 | 141 | /// Runs `git` and fails on a non-zero exit. |
| 108 | 142 | /// |
| 109 | 143 | /// The single place that decides how Steid invokes git, so every call site gets the |
| | @@ -115,18 +149,8 @@ where |
| 115 | 149 | I: IntoIterator<Item = S>, |
| 116 | 150 | S: AsRef<OsStr>, |
| 117 | 151 | { |
| 118 | | − let mut command = Command::new("git"); |
| 119 | | − command |
| 120 | | − .args(args) |
| 121 | | − // Host configuration must not leak into repositories Steid creates, for the |
| 122 | | − // same reason `--initial-branch` is passed explicitly. |
| 123 | | − .env("GIT_CONFIG_GLOBAL", "/dev/null") |
| 124 | | − .env("GIT_CONFIG_SYSTEM", "/dev/null") |
| 125 | | − .stdin(Stdio::null()); |
| 126 | | − |
| 127 | | − for variable in REDIRECTING_VARS { |
| 128 | | − command.env_remove(variable); |
| 129 | | − } |
| 152 | + let mut command = git_command(); |
| 153 | + command.args(args).stdin(Stdio::null()); |
| 130 | 154 | |
| 131 | 155 | // `output()` pipes stdout and stderr and waits without blocking the runtime. |
| 132 | 156 | let output = command |
| | @@ -207,6 +231,232 @@ impl GitStorage for InMemoryGitStorage { |
| 207 | 231 | } |
| 208 | 232 | } |
| 209 | 233 | |
| 234 | +/// The git smart-HTTP protocol, served by `git http-backend`. |
| 235 | +/// |
| 236 | +/// The binary is a CGI: it takes an environment and a request body on stdin, and writes |
| 237 | +/// CRLF-terminated headers, a blank line, then the response body. Its contract was |
| 238 | +/// probed rather than assumed — see `plans/progress.md` under Milestone 4a. |
| 239 | +#[derive(Debug, Clone)] |
| 240 | +pub struct GitHttpBackend { |
| 241 | + data_dir: PathBuf, |
| 242 | +} |
| 243 | + |
| 244 | +impl GitHttpBackend { |
| 245 | + pub fn new(data_dir: impl Into<PathBuf>) -> Self { |
| 246 | + Self { |
| 247 | + data_dir: data_dir.into(), |
| 248 | + } |
| 249 | + } |
| 250 | +} |
| 251 | + |
| 252 | +impl GitProtocolServer for GitHttpBackend { |
| 253 | + async fn serve(&self, request: GitRequest) -> Result<GitResponse, GitProtocolError> { |
| 254 | + let mut command = git_command(); |
| 255 | + command |
| 256 | + .arg("http-backend") |
| 257 | + .env("GIT_PROJECT_ROOT", &self.data_dir) |
| 258 | + // Steid decides visibility from the `repositories` table, in the use case. |
| 259 | + // Without this, git applies its own rule and refuses everything lacking a |
| 260 | + // `git-daemon-export-ok` marker file — a second source of truth for the same |
| 261 | + // question, free to drift from the first. |
| 262 | + .env("GIT_HTTP_EXPORT_ALL", "1") |
| 263 | + .env("PATH_INFO", &request.path_info) |
| 264 | + .env("QUERY_STRING", &request.query) |
| 265 | + .env("REQUEST_METHOD", request.method.as_str()) |
| 266 | + .stdin(Stdio::piped()) |
| 267 | + .stdout(Stdio::piped()) |
| 268 | + .stderr(Stdio::piped()); |
| 269 | + |
| 270 | + // CGI gives only Content-Type and Content-Length unprefixed names; every other |
| 271 | + // request header arrives `HTTP_`-prefixed. Passing `CONTENT_ENCODING` instead of |
| 272 | + // `HTTP_CONTENT_ENCODING` makes the backend hand a still-gzipped body to |
| 273 | + // upload-pack, and the client reports `expected 'packfile'` with nothing naming |
| 274 | + // the cause. Measured, not guessed. |
| 275 | + for (variable, value) in [ |
| 276 | + ("CONTENT_TYPE", &request.content_type), |
| 277 | + ("CONTENT_LENGTH", &request.content_length), |
| 278 | + ("HTTP_CONTENT_ENCODING", &request.content_encoding), |
| 279 | + ("HTTP_GIT_PROTOCOL", &request.git_protocol), |
| 280 | + ] { |
| 281 | + if let Some(value) = value { |
| 282 | + command.env(variable, value); |
| 283 | + } |
| 284 | + } |
| 285 | + |
| 286 | + let mut child = command |
| 287 | + .spawn() |
| 288 | + .map_err(|error| GitProtocolError::new(format!("could not run git: {error}")))?; |
| 289 | + |
| 290 | + let mut stdin = child.stdin.take().expect("stdin was piped"); |
| 291 | + let stdout = child.stdout.take().expect("stdout was piped"); |
| 292 | + let mut stderr = child.stderr.take().expect("stderr was piped"); |
| 293 | + let mut body = request.body; |
| 294 | + |
| 295 | + // The request body streams in while the response streams out; a push is far too |
| 296 | + // large to buffer, and a fetch would otherwise wait for a body it already has. |
| 297 | + // Dropping stdin closes the pipe, which is what tells the backend the request is |
| 298 | + // complete — an error here is the client having gone away, which the backend |
| 299 | + // then sees as EOF. |
| 300 | + tokio::spawn(async move { |
| 301 | + let _ = tokio::io::copy(&mut body, &mut stdin).await; |
| 302 | + }); |
| 303 | + |
| 304 | + // Reaps the child and surfaces its complaint. This cannot gate the response: a |
| 305 | + // protocol failure exits non-zero *after* a complete, successful-looking header |
| 306 | + // block has already been written, so by the time the status is known it has been |
| 307 | + // sent. Draining stderr is not optional either — an unread pipe fills and blocks |
| 308 | + // the backend mid-transfer. |
| 309 | + tokio::spawn(async move { |
| 310 | + let mut complaint = String::new(); |
| 311 | + let _ = stderr.read_to_string(&mut complaint).await; |
| 312 | + |
| 313 | + match child.wait().await { |
| 314 | + Ok(status) if status.success() => {} |
| 315 | + Ok(status) => eprintln!( |
| 316 | + "steid: git http-backend exited with {status}: {}", |
| 317 | + complaint.trim() |
| 318 | + ), |
| 319 | + Err(error) => eprintln!("steid: could not wait for git http-backend: {error}"), |
| 320 | + } |
| 321 | + }); |
| 322 | + |
| 323 | + // `BufReader` keeps whatever it read past the header block, and handing the |
| 324 | + // reader itself back as the body is what makes that safe — the first bytes of |
| 325 | + // the pack are already buffered inside it. |
| 326 | + let mut reader = BufReader::new(stdout); |
| 327 | + let (status, headers) = read_cgi_headers(&mut reader).await?; |
| 328 | + |
| 329 | + Ok(GitResponse { |
| 330 | + status, |
| 331 | + headers, |
| 332 | + body: Box::pin(reader), |
| 333 | + }) |
| 334 | + } |
| 335 | +} |
| 336 | + |
| 337 | +/// Reads the CGI header block, stopping at the blank line that ends it. |
| 338 | +/// |
| 339 | +/// `Status:` is git's way of reporting failure and appears only then, so its absence |
| 340 | +/// means 200. It is translated into the response status rather than forwarded as a |
| 341 | +/// header, which would be meaningless to a client. |
| 342 | +async fn read_cgi_headers( |
| 343 | + reader: &mut BufReader<ChildStdout>, |
| 344 | +) -> Result<(u16, Vec<(String, String)>), GitProtocolError> { |
| 345 | + let mut status = 200; |
| 346 | + let mut headers = Vec::new(); |
| 347 | + let mut line = Vec::new(); |
| 348 | + |
| 349 | + loop { |
| 350 | + line.clear(); |
| 351 | + |
| 352 | + let read = reader |
| 353 | + .read_until(b'\n', &mut line) |
| 354 | + .await |
| 355 | + .map_err(|error| GitProtocolError::new(format!("reading git's headers: {error}")))?; |
| 356 | + |
| 357 | + if read == 0 { |
| 358 | + return Err(GitProtocolError::new( |
| 359 | + "git http-backend produced no headers before closing", |
| 360 | + )); |
| 361 | + } |
| 362 | + |
| 363 | + // Tolerates a bare LF as well as the CRLF actually observed: a header reader |
| 364 | + // that hangs on an unexpected line ending is a bad way to find out. |
| 365 | + let text = String::from_utf8_lossy(&line); |
| 366 | + let text = text.trim_end_matches(['\r', '\n']); |
| 367 | + |
| 368 | + if text.is_empty() { |
| 369 | + return Ok((status, headers)); |
| 370 | + } |
| 371 | + |
| 372 | + let Some((name, value)) = text.split_once(": ") else { |
| 373 | + return Err(GitProtocolError::new(format!( |
| 374 | + "git http-backend wrote an unparseable header: {text:?}" |
| 375 | + ))); |
| 376 | + }; |
| 377 | + |
| 378 | + if name.eq_ignore_ascii_case("status") { |
| 379 | + status = value |
| 380 | + .split_whitespace() |
| 381 | + .next() |
| 382 | + .and_then(|code| code.parse().ok()) |
| 383 | + .ok_or_else(|| { |
| 384 | + GitProtocolError::new(format!("git http-backend wrote a bad status: {value:?}")) |
| 385 | + })?; |
| 386 | + } else { |
| 387 | + headers.push((name.to_owned(), value.to_owned())); |
| 388 | + } |
| 389 | + |
| 390 | + if headers.len() > MAX_CGI_HEADERS { |
| 391 | + return Err(GitProtocolError::new( |
| 392 | + "git http-backend wrote more headers than a CGI response can plausibly have", |
| 393 | + )); |
| 394 | + } |
| 395 | + } |
| 396 | +} |
| 397 | + |
| 398 | +/// What a [`InMemoryGitProtocol`] was asked for, minus the body. |
| 399 | +/// |
| 400 | +/// The body is a stream and comparing it would mean draining it; every rule worth |
| 401 | +/// asserting on lives in the metadata anyway. |
| 402 | +#[derive(Debug, Clone, PartialEq, Eq)] |
| 403 | +pub struct RecordedGitRequest { |
| 404 | + pub method: GitMethod, |
| 405 | + pub path_info: String, |
| 406 | + pub query: String, |
| 407 | + pub git_protocol: Option<String>, |
| 408 | + pub content_encoding: Option<String>, |
| 409 | +} |
| 410 | + |
| 411 | +/// A git protocol that records what it was asked and never runs git. |
| 412 | +/// |
| 413 | +/// The counterpart to [`GitHttpBackend`]. What it is really for is proving a negative: |
| 414 | +/// that a use case refused *before* reaching the protocol. `was_called` is how a test |
| 415 | +/// says "and no bytes flowed". |
| 416 | +#[derive(Debug, Default, Clone)] |
| 417 | +pub struct InMemoryGitProtocol { |
| 418 | + requests: Arc<Mutex<Vec<RecordedGitRequest>>>, |
| 419 | +} |
| 420 | + |
| 421 | +impl InMemoryGitProtocol { |
| 422 | + pub fn new() -> Self { |
| 423 | + Self::default() |
| 424 | + } |
| 425 | + |
| 426 | + pub fn requests(&self) -> Vec<RecordedGitRequest> { |
| 427 | + self.requests.lock().expect("lock poisoned").clone() |
| 428 | + } |
| 429 | + |
| 430 | + /// Whether the protocol was reached at all. |
| 431 | + pub fn was_called(&self) -> bool { |
| 432 | + !self.requests.lock().expect("lock poisoned").is_empty() |
| 433 | + } |
| 434 | +} |
| 435 | + |
| 436 | +impl GitProtocolServer for InMemoryGitProtocol { |
| 437 | + async fn serve(&self, request: GitRequest) -> Result<GitResponse, GitProtocolError> { |
| 438 | + self.requests |
| 439 | + .lock() |
| 440 | + .expect("lock poisoned") |
| 441 | + .push(RecordedGitRequest { |
| 442 | + method: request.method, |
| 443 | + path_info: request.path_info, |
| 444 | + query: request.query, |
| 445 | + git_protocol: request.git_protocol, |
| 446 | + content_encoding: request.content_encoding, |
| 447 | + }); |
| 448 | + |
| 449 | + Ok(GitResponse { |
| 450 | + status: 200, |
| 451 | + headers: vec![( |
| 452 | + "Content-Type".to_owned(), |
| 453 | + "application/x-git-upload-pack-advertisement".to_owned(), |
| 454 | + )], |
| 455 | + body: Box::pin(std::io::Cursor::new(b"0000".to_vec())), |
| 456 | + }) |
| 457 | + } |
| 458 | +} |
| 459 | + |
| 210 | 460 | #[cfg(test)] |
| 211 | 461 | mod tests { |
| 212 | 462 | use std::path::Path; |
| | @@ -488,4 +738,145 @@ mod tests { |
| 488 | 738 | "expected git's own words, got: {message}" |
| 489 | 739 | ); |
| 490 | 740 | } |
| 741 | + |
| 742 | + // --- GitHttpBackend -------------------------------------------------------- |
| 743 | + |
| 744 | + /// A data directory holding one bare repository at `acme/steid.git`. |
| 745 | + async fn backend() -> (TempDir, GitHttpBackend) { |
| 746 | + let dir = TempDir::new().expect("temp dir"); |
| 747 | + let storage = DiskGitStorage::new(dir.path()); |
| 748 | + |
| 749 | + let acme = OrgName::new("acme").expect("valid handle"); |
| 750 | + storage |
| 751 | + .init_bare(&acme, &repo_name("steid")) |
| 752 | + .await |
| 753 | + .expect("init bare"); |
| 754 | + |
| 755 | + let backend = GitHttpBackend::new(dir.path()); |
| 756 | + (dir, backend) |
| 757 | + } |
| 758 | + |
| 759 | + fn advertisement(path_info: &str) -> GitRequest { |
| 760 | + GitRequest { |
| 761 | + method: GitMethod::Get, |
| 762 | + path_info: path_info.to_owned(), |
| 763 | + query: "service=git-upload-pack".to_owned(), |
| 764 | + content_type: None, |
| 765 | + content_encoding: None, |
| 766 | + content_length: None, |
| 767 | + git_protocol: None, |
| 768 | + body: Box::pin(tokio::io::empty()), |
| 769 | + } |
| 770 | + } |
| 771 | + |
| 772 | + async fn drain(response: GitResponse) -> Vec<u8> { |
| 773 | + let mut body = response.body; |
| 774 | + let mut bytes = Vec::new(); |
| 775 | + body.read_to_end(&mut bytes).await.expect("read body"); |
| 776 | + bytes |
| 777 | + } |
| 778 | + |
| 779 | + #[tokio::test] |
| 780 | + async fn the_backend_advertises_refs() { |
| 781 | + let (_dir, backend) = backend().await; |
| 782 | + |
| 783 | + let response = backend |
| 784 | + .serve(advertisement("/acme/steid.git/info/refs")) |
| 785 | + .await |
| 786 | + .expect("should serve"); |
| 787 | + |
| 788 | + assert_eq!(response.status, 200); |
| 789 | + assert!( |
| 790 | + response |
| 791 | + .headers |
| 792 | + .iter() |
| 793 | + .any(|(name, value)| name == "Content-Type" |
| 794 | + && value == "application/x-git-upload-pack-advertisement"), |
| 795 | + "git sets its own content type and we forward it: {:?}", |
| 796 | + response.headers |
| 797 | + ); |
| 798 | + |
| 799 | + // The pkt-line the smart protocol opens with. Getting this from git rather than |
| 800 | + // writing it is the whole reason the backend is a subprocess. |
| 801 | + let body = drain(response).await; |
| 802 | + assert!( |
| 803 | + body.starts_with(b"001e# service=git-upload-pack\n"), |
| 804 | + "unexpected advertisement: {:?}", |
| 805 | + String::from_utf8_lossy(&body[..body.len().min(40)]) |
| 806 | + ); |
| 807 | + } |
| 808 | + |
| 809 | + #[tokio::test] |
| 810 | + async fn a_missing_repository_is_reported_as_404_not_as_a_failure() { |
| 811 | + // Failure arrives in the CGI stream, not the exit code: git exits 0 here and |
| 812 | + // says 404 in a header. Keying off the exit code instead would answer 200. |
| 813 | + let (_dir, backend) = backend().await; |
| 814 | + |
| 815 | + let response = backend |
| 816 | + .serve(advertisement("/acme/nothing-here.git/info/refs")) |
| 817 | + .await |
| 818 | + .expect("serving should not itself fail"); |
| 819 | + |
| 820 | + assert_eq!(response.status, 404); |
| 821 | + } |
| 822 | + |
| 823 | + #[tokio::test] |
| 824 | + async fn the_status_header_is_translated_rather_than_forwarded() { |
| 825 | + let (_dir, backend) = backend().await; |
| 826 | + |
| 827 | + let response = backend |
| 828 | + .serve(advertisement("/acme/nothing-here.git/info/refs")) |
| 829 | + .await |
| 830 | + .expect("should serve"); |
| 831 | + |
| 832 | + assert!( |
| 833 | + !response |
| 834 | + .headers |
| 835 | + .iter() |
| 836 | + .any(|(name, _)| name.eq_ignore_ascii_case("status")), |
| 837 | + "Status: is CGI's, and means nothing to an HTTP client: {:?}", |
| 838 | + response.headers |
| 839 | + ); |
| 840 | + } |
| 841 | + |
| 842 | + #[tokio::test] |
| 843 | + async fn the_protocol_version_reaches_upload_pack() { |
| 844 | + // Protocol v2 answers an advertisement with a capability list rather than refs. |
| 845 | + // If `HTTP_GIT_PROTOCOL` is dropped the exchange silently falls back to v0, which |
| 846 | + // still works — so nothing fails, it just quietly gets worse. |
| 847 | + let (_dir, backend) = backend().await; |
| 848 | + |
| 849 | + let mut request = advertisement("/acme/steid.git/info/refs"); |
| 850 | + request.git_protocol = Some("version=2".to_owned()); |
| 851 | + |
| 852 | + let body = drain(backend.serve(request).await.expect("should serve")).await; |
| 853 | + |
| 854 | + assert!( |
| 855 | + String::from_utf8_lossy(&body).contains("version 2"), |
| 856 | + "expected a v2 capability advertisement: {:?}", |
| 857 | + String::from_utf8_lossy(&body[..body.len().min(80)]) |
| 858 | + ); |
| 859 | + } |
| 860 | + |
| 861 | + #[tokio::test] |
| 862 | + async fn the_in_memory_protocol_records_what_it_was_asked() { |
| 863 | + let protocol = InMemoryGitProtocol::new(); |
| 864 | + |
| 865 | + protocol |
| 866 | + .serve(advertisement("/acme/steid.git/info/refs")) |
| 867 | + .await |
| 868 | + .expect("should serve"); |
| 869 | + |
| 870 | + assert_eq!( |
| 871 | + protocol.requests(), |
| 872 | + vec![RecordedGitRequest { |
| 873 | + method: GitMethod::Get, |
| 874 | + path_info: "/acme/steid.git/info/refs".to_owned(), |
| 875 | + query: "service=git-upload-pack".to_owned(), |
| 876 | + git_protocol: None, |
| 877 | + content_encoding: None, |
| 878 | + }] |
| 879 | + ); |
| 880 | + assert!(protocol.was_called()); |
| 881 | + } |
| 491 | 882 | } |