steid
Sign in

@jamesgill /

steid

steid/src/infrastructure/git/import.rs
Rust23.5 KBCode·Blame·History·Raw
1//! Mirroring a repository in from somewhere else.
2
3use std::{
4 ffi::OsStr,
5 path::PathBuf,
6 process::{Output, Stdio},
7 sync::{Arc, Mutex},
8};
9
10use crate::{
11 application::port::{GitImport, GitImportError},
12 domain::{ImportSource, OrgName, RepoName, is_public},
13 infrastructure::metrics::metrics,
14};
15
16use super::{count_spawn, git_command};
17
18// --- Importing from elsewhere -----------------------------------------------------
19
20/// How long a `git clone --mirror` may run before it is killed.
21///
22/// Ten minutes, far above the read timeout in `git_query.rs` and for the opposite
23/// reason: this is a network transfer of a whole repository with nobody waiting on a
24/// page, so the cost of being generous is a background task, not a held worker. It is
25/// bounded at all because a stalled TLS connection to a host that accepts and then says
26/// nothing would otherwise hold a task for the life of the process.
27const IMPORT_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(600);
28
29/// The port git connects to when a URL does not name one. Used only to resolve the
30/// host — nothing dials this socket directly.
31const HTTPS_PORT: u16 = 443;
32
33/// `git clone --mirror` into the same layout [`DiskGitStorage`] uses.
34///
35/// **This adapter is where the second half of the server-side request forgery check
36/// lives.** [`ImportSource`] refused a literal address that is not on the public
37/// internet, but `evil.example.com` resolving to `127.0.0.1` passes every syntactic
38/// rule there is. Only the thing about to open the connection can look, so it looks
39/// here, immediately before spawning git.
40#[derive(Debug, Clone)]
41pub struct DiskGitImport {
42 data_dir: PathBuf,
43}
44
45impl DiskGitImport {
46 pub fn new(data_dir: impl Into<PathBuf>) -> Self {
47 Self {
48 data_dir: data_dir.into(),
49 }
50 }
51
52 /// Where a repository lives, matching `DiskGitStorage`'s layout.
53 fn repo_path(&self, handle: &OrgName, name: &RepoName) -> PathBuf {
54 self.data_dir
55 .join(handle.as_str())
56 .join(format!("{name}.git"))
57 }
58}
59
60impl GitImport for DiskGitImport {
61 async fn mirror(
62 &self,
63 handle: &OrgName,
64 name: &RepoName,
65 source: &ImportSource,
66 ) -> Result<(), GitImportError> {
67 // Counted here rather than where the task is spawned, because this is the only
68 // place that sees an import's real outcome: `application::run_import` answers
69 // `Ok(())` for a *failed* import — the failure is data on the repository — so a
70 // counter on its return value would say every import succeeded.
71 //
72 // "Started" therefore means the clone was attempted. An import still waiting for
73 // a permit has not started, and is visible on `steid_active_imports` instead.
74 metrics().record_import_started();
75
76 let outcome = self.mirror_inner(handle, name, source).await;
77
78 match &outcome {
79 Ok(()) => metrics().record_import_finished(),
80 Err(_) => metrics().record_import_failed(),
81 }
82
83 outcome
84 }
85}
86
87impl DiskGitImport {
88 /// [`mirror`](GitImport::mirror) itself, with the counting split off so that every
89 /// early return is counted too — a `?` past a recording line records nothing.
90 async fn mirror_inner(
91 &self,
92 handle: &OrgName,
93 name: &RepoName,
94 source: &ImportSource,
95 ) -> Result<(), GitImportError> {
96 resolve_public_host(source.host()).await?;
97
98 let path = self.repo_path(handle, name);
99
100 // `git clone` refuses a non-empty destination, so an orphan left by an earlier
101 // attempt surfaces as a failed import rather than being adopted. Checked here
102 // as well so the message says something a person can act on.
103 if path.exists() {
104 return Err(GitImportError::new(
105 "something is already on disk at this repository's path",
106 ));
107 }
108
109 // `--mirror` is the whole reason this is one command: it fetches every ref into
110 // the same name it has on the source, sets `HEAD` to the source's default
111 // branch, and makes the result bare. A plain `--bare` clone would bring one
112 // branch and lose every tag.
113 run_import(
114 [
115 // **The other half of the redirect rule** — see `resolve_public_host`.
116 // git's own default is `initial`, which lets the *first* request be
117 // redirected, and it re-resolves whatever host it is sent to. So a
118 // perfectly public source answering `302 Location:
119 // http://169.254.169.254/…` walked the clone straight past the check
120 // above. Off entirely: a clone URL that needs a redirect is one whose
121 // real address the person can paste instead, and the failure says so.
122 OsStr::new("-c"),
123 OsStr::new("http.followRedirects=false"),
124 OsStr::new("clone"),
125 OsStr::new("--mirror"),
126 OsStr::new("--quiet"),
127 // Everything after this is data. The URL is already https-only and
128 // space-free, but a destination that could be read as a flag is exactly
129 // the class of bug `RepoName` exists to prevent, so it is closed here
130 // too where it costs nothing.
131 OsStr::new("--"),
132 OsStr::new(source.as_str()),
133 path.as_os_str(),
134 ],
135 None,
136 )
137 .await?;
138
139 finish_mirror(&path).await
140 }
141}
142
143/// Turns a fresh `--mirror` clone into an ordinary Steid repository.
144///
145/// Split out from [`DiskGitImport::mirror`] so it can be tested against a fixture: the
146/// clone itself needs a public HTTPS host, and everything that could plausibly be wrong
147/// is on this side of it.
148async fn finish_mirror(path: &std::path::Path) -> Result<(), GitImportError> {
149 // A mirror keeps its remote and its `remote.origin.mirror` flag, so the result
150 // would be a repository that fetches from somewhere else on the next `git remote
151 // update`. Removing it is what turns the clone into an ordinary Steid repository,
152 // and it takes the mirror refspec with it.
153 run_import(
154 [
155 OsStr::new("remote"),
156 OsStr::new("remove"),
157 OsStr::new("origin"),
158 ],
159 Some(path),
160 )
161 .await?;
162
163 // **`git clone` leaves no reflog either**, exactly as `git init --bare` does, and an
164 // import never goes through `init_bare` — so without this an imported repository
165 // records nobody as having pushed to it until the next boot sweep picks it up. Found
166 // by that sweep reporting six repositories on a development instance where every
167 // repository had been created by the current code: the six were the imports.
168 run_import(
169 [
170 OsStr::new("config"),
171 OsStr::new("core.logAllRefUpdates"),
172 OsStr::new("true"),
173 ],
174 Some(path),
175 )
176 .await?;
177
178 // `--mirror` puts branches under `refs/heads` already. This is the belt to that
179 // braces: a source whose own layout is unusual — itself a mirror of a mirror — can
180 // leave `refs/remotes`, and a branch nobody can see is worse than one moved to
181 // where every other page expects it.
182 promote_remote_refs(path).await
183}
184
185/// Moves anything under `refs/remotes/origin/` to `refs/heads/`, and drops the rest.
186///
187/// A no-op for every ordinary mirror, which is why it is written to cost one `git`
188/// process and stop: `for-each-ref` on a prefix with nothing under it exits 0 with no
189/// output.
190///
191/// Object ids are read alongside the names and written back as ids. `update-ref` takes
192/// a value, not a reference, and asking it to resolve one is how this was got wrong the
193/// first time.
194async fn promote_remote_refs(path: &std::path::Path) -> Result<(), GitImportError> {
195 let listed = run_import(
196 [
197 OsStr::new("for-each-ref"),
198 OsStr::new("--format=%(objectname) %(refname)"),
199 OsStr::new("refs/remotes/"),
200 ],
201 Some(path),
202 )
203 .await?;
204
205 let listed = String::from_utf8_lossy(&listed.stdout);
206
207 for line in listed.lines() {
208 let Some((id, reference)) = line.trim().split_once(' ') else {
209 continue;
210 };
211
212 // `origin/HEAD` is a symbolic pointer, not a branch, and `--mirror` has already
213 // set the real `HEAD` from the source. Anything under another remote's name is
214 // somebody else's tracking ref and means nothing here. Both are simply dropped.
215 if let Some(branch) = reference
216 .strip_prefix("refs/remotes/origin/")
217 .filter(|branch| *branch != "HEAD")
218 {
219 run_import(
220 [
221 OsStr::new("update-ref"),
222 OsStr::new(&format!("refs/heads/{branch}")),
223 OsStr::new(id),
224 ],
225 Some(path),
226 )
227 .await?;
228 }
229
230 run_import(
231 [
232 OsStr::new("update-ref"),
233 OsStr::new("-d"),
234 OsStr::new(reference),
235 ],
236 Some(path),
237 )
238 .await?;
239 }
240
241 Ok(())
242}
243
244/// Resolves a host and refuses it unless every address it answers with is on the public
245/// internet.
246///
247/// **Every address, not the first.** A host that answers with one public address and
248/// one loopback address is a host that can be made to connect to loopback, because
249/// which one git picks is not ours to decide.
250///
251/// It is only half the rule on its own: git would otherwise follow a redirect on its
252/// first request and re-resolve wherever it was sent, so the clone passes
253/// `http.followRedirects=false` — see [`DiskGitImport::mirror_inner`].
254///
255/// This leaves one hole, deliberately and knowingly: git resolves the name again when
256/// it connects, so a record whose TTL expires between the two lookups can change
257/// underneath us. Closing that means resolving here and handing git a literal address
258/// with the hostname forced for TLS and SNI, which is a larger change than this step —
259/// recorded rather than pretended away.
260async fn resolve_public_host(host: &str) -> Result<(), GitImportError> {
261 let refused = || {
262 GitImportError::new(format!(
263 "{host} resolves to an address that is not on the public internet"
264 ))
265 };
266
267 let addresses = tokio::net::lookup_host((host, HTTPS_PORT))
268 .await
269 .map_err(|error| GitImportError::new(format!("could not resolve {host}: {error}")))?;
270
271 let mut any = false;
272
273 for address in addresses {
274 any = true;
275
276 if !is_public(address.ip()) {
277 return Err(refused());
278 }
279 }
280
281 if !any {
282 return Err(GitImportError::new(format!("{host} resolves to nothing")));
283 }
284
285 Ok(())
286}
287
288/// Runs one `git` command on behalf of an import.
289///
290/// Separate from [`run_git`] for two reasons rather than one: the timeout is ten
291/// minutes instead of unbounded, and `GIT_TERMINAL_PROMPT=0` turns a private repository
292/// into a failed import instead of a subprocess sitting forever on a username prompt
293/// nobody can answer.
294async fn run_import<I, S>(args: I, repo: Option<&std::path::Path>) -> Result<Output, GitImportError>
295where
296 I: IntoIterator<Item = S>,
297 S: AsRef<OsStr>,
298{
299 let mut command = git_command();
300
301 if let Some(repo) = repo {
302 command.arg("-C").arg(repo);
303 }
304
305 command
306 .args(args)
307 .stdin(Stdio::null())
308 // Without this git blocks on a credential prompt for anything that asks for
309 // authentication — a private repository, or a typo'd path on a host that
310 // challenges rather than 404s.
311 .env("GIT_TERMINAL_PROMPT", "0")
312 // A killed clone must take the process with it, or a timed-out import keeps
313 // pulling a repository nobody is waiting for.
314 .kill_on_drop(true);
315 count_spawn(&command);
316
317 let output = match tokio::time::timeout(IMPORT_TIMEOUT, command.output()).await {
318 Ok(result) => {
319 result.map_err(|error| GitImportError::new(format!("could not run git: {error}")))?
320 }
321 Err(_elapsed) => {
322 return Err(GitImportError::new(format!(
323 "the import did not finish within {IMPORT_TIMEOUT:?}"
324 )));
325 }
326 };
327
328 if !output.status.success() {
329 // git's own words, trimmed to the last line: the earlier lines of a clone
330 // failure are progress, and the last one is the reason.
331 let stderr = String::from_utf8_lossy(&output.stderr);
332 let reason = stderr.lines().rev().find(|line| !line.trim().is_empty());
333
334 return Err(GitImportError::new(match reason {
335 Some(reason) => reason.trim().to_owned(),
336 None => format!("git exited with {}", output.status),
337 }));
338 }
339
340 Ok(output)
341}
342
343/// Imports recorded in memory, for testing use cases without a network.
344///
345/// The counterpart to [`DiskGitImport`], the way [`InMemoryGitStorage`] is to
346/// [`DiskGitStorage`]. It can be told to fail, because the failure path — recording the
347/// reason and removing the half-clone — is the half most worth testing.
348#[derive(Debug, Default, Clone)]
349pub struct InMemoryGitImport {
350 requests: Arc<Mutex<Vec<(String, String, String)>>>,
351 failure: Option<String>,
352}
353
354impl InMemoryGitImport {
355 pub fn new() -> Self {
356 Self::default()
357 }
358
359 /// One that always fails with `reason`.
360 pub fn failing(reason: impl Into<String>) -> Self {
361 Self {
362 requests: Arc::default(),
363 failure: Some(reason.into()),
364 }
365 }
366
367 /// What was asked for, as `(handle, name, source)`.
368 pub fn requests(&self) -> Vec<(String, String, String)> {
369 self.requests.lock().expect("lock poisoned").clone()
370 }
371}
372
373impl GitImport for InMemoryGitImport {
374 async fn mirror(
375 &self,
376 handle: &OrgName,
377 name: &RepoName,
378 source: &ImportSource,
379 ) -> Result<(), GitImportError> {
380 self.requests.lock().expect("lock poisoned").push((
381 handle.to_string(),
382 name.to_string(),
383 source.as_str().to_owned(),
384 ));
385
386 match &self.failure {
387 Some(reason) => Err(GitImportError::new(reason.clone())),
388 None => Ok(()),
389 }
390 }
391}
392
393#[cfg(test)]
394mod tests {
395 use tempfile::TempDir;
396
397 use super::*;
398 use crate::application::port::GitStorage;
399 use crate::infrastructure::git::fixture::{git_says, handle, repo_name, storage};
400
401 // --- Importing ------------------------------------------------------------------
402
403 /// A source repository on disk with two branches and an annotated tag, so a mirror
404 /// has something to lose.
405 fn source_repo() -> (TempDir, PathBuf) {
406 let dir = TempDir::new().expect("temp dir");
407 let path = dir.path().join("source");
408
409 std::fs::create_dir(&path).expect("create");
410
411 let run = |args: &[&str]| {
412 let output = std::process::Command::new("git")
413 .arg("-C")
414 .arg(&path)
415 .args(args)
416 .env("GIT_CONFIG_GLOBAL", "/dev/null")
417 .env("GIT_CONFIG_SYSTEM", "/dev/null")
418 .env("GIT_AUTHOR_NAME", "Ada")
419 .env("GIT_AUTHOR_EMAIL", "ada@example.com")
420 .env("GIT_COMMITTER_NAME", "Ada")
421 .env("GIT_COMMITTER_EMAIL", "ada@example.com")
422 .output()
423 .expect("git should be on PATH");
424 assert!(
425 output.status.success(),
426 "git {args:?} failed: {}",
427 String::from_utf8_lossy(&output.stderr)
428 );
429 };
430
431 run(&["init", "--quiet", "--initial-branch=main"]);
432 std::fs::write(path.join("README.md"), "# source\n").expect("write");
433 run(&["add", "README.md"]);
434 run(&["commit", "--quiet", "-m", "first"]);
435 run(&["tag", "-a", "v0.1.0", "-m", "first"]);
436 run(&["checkout", "--quiet", "-b", "side"]);
437 std::fs::write(path.join("more.md"), "more\n").expect("write");
438 run(&["add", "more.md"]);
439 run(&["commit", "--quiet", "-m", "second"]);
440 run(&["checkout", "--quiet", "main"]);
441
442 (dir, path)
443 }
444
445 /// The shape an import has to leave behind, exercised end to end minus the network.
446 ///
447 /// The clone is run against a local path because `ImportSource` allows only public
448 /// HTTPS, which cannot be stood up in a unit test. Everything the adapter decides
449 /// after the transfer — the refs, `HEAD`, and the remote — is the same either way,
450 /// and it is the half that could plausibly be wrong.
451 #[tokio::test]
452 async fn a_mirror_lands_as_an_ordinary_bare_repository() {
453 let (_source_dir, source) = source_repo();
454 let (dest_dir, _storage) = storage();
455 let destination = dest_dir.path().join("mirrored.git");
456
457 run_import(
458 [
459 OsStr::new("clone"),
460 OsStr::new("--mirror"),
461 OsStr::new("--quiet"),
462 OsStr::new("--"),
463 source.as_os_str(),
464 destination.as_os_str(),
465 ],
466 None,
467 )
468 .await
469 .expect("the clone should succeed");
470
471 finish_mirror(&destination)
472 .await
473 .expect("the mirror should finish");
474
475 // Every branch, under `refs/heads` — where every other page in Steid looks.
476 let branches = git_says(
477 &destination,
478 &["for-each-ref", "--format=%(refname)", "refs/heads/"],
479 );
480 assert_eq!(branches, "refs/heads/main\nrefs/heads/side");
481
482 // Tags too, annotated ones included.
483 assert_eq!(
484 git_says(
485 &destination,
486 &["for-each-ref", "--format=%(refname)", "refs/tags/"]
487 ),
488 "refs/tags/v0.1.0"
489 );
490
491 // Nothing left under `refs/remotes`, and no remote to fetch from.
492 assert_eq!(
493 git_says(
494 &destination,
495 &["for-each-ref", "--format=%(refname)", "refs/remotes/"]
496 ),
497 ""
498 );
499 assert_eq!(git_says(&destination, &["remote"]), "");
500
501 // `HEAD` follows the source's default branch, so the landing page opens on it.
502 assert_eq!(
503 git_says(&destination, &["symbolic-ref", "HEAD"]),
504 "refs/heads/main"
505 );
506 assert_eq!(
507 git_says(&destination, &["rev-parse", "--is-bare-repository"]),
508 "true"
509 );
510
511 // And a reflog, so the first push into an imported repository records who made
512 // it. `git clone` leaves this off exactly as `git init --bare` does, and an
513 // import never goes through `init_bare`.
514 assert_eq!(
515 git_says(&destination, &["config", "--get", "core.logAllRefUpdates"]),
516 "true"
517 );
518 }
519
520 /// A source whose branches arrived under `refs/remotes` — the case `--mirror`
521 /// normally makes impossible, which is exactly why nothing else would catch it.
522 #[tokio::test]
523 async fn branches_left_under_refs_remotes_are_moved_to_refs_heads() {
524 let (_source_dir, source) = source_repo();
525 let (dest_dir, _storage) = storage();
526 let destination = dest_dir.path().join("remotes.git");
527
528 // A plain `--bare` clone with a fetch refspec that writes remote-tracking refs,
529 // which is what a non-mirror clone of a mirror can leave.
530 run_import(
531 [
532 OsStr::new("clone"),
533 OsStr::new("--bare"),
534 OsStr::new("--quiet"),
535 OsStr::new("--"),
536 source.as_os_str(),
537 destination.as_os_str(),
538 ],
539 None,
540 )
541 .await
542 .expect("clone");
543
544 // Stage the shape by hand: a branch only under `refs/remotes`.
545 run_import(
546 [
547 OsStr::new("update-ref"),
548 OsStr::new("refs/remotes/origin/side"),
549 OsStr::new("refs/heads/main"),
550 ],
551 Some(&destination),
552 )
553 .await
554 .expect("stage a remote-tracking ref");
555 run_import(
556 [
557 OsStr::new("symbolic-ref"),
558 OsStr::new("refs/remotes/origin/HEAD"),
559 OsStr::new("refs/remotes/origin/side"),
560 ],
561 Some(&destination),
562 )
563 .await
564 .expect("stage origin/HEAD");
565
566 finish_mirror(&destination).await.expect("finish");
567
568 let branches = git_says(
569 &destination,
570 &["for-each-ref", "--format=%(refname)", "refs/heads/"],
571 );
572 assert!(
573 branches.contains("refs/heads/side"),
574 "expected the remote-tracking branch to become a branch, got {branches:?}"
575 );
576 assert_eq!(
577 git_says(
578 &destination,
579 &["for-each-ref", "--format=%(refname)", "refs/remotes/"]
580 ),
581 "",
582 "origin/HEAD is a pointer, not a branch, and must not survive either"
583 );
584 }
585
586 /// The DNS half of the server-side request forgery check, which `ImportSource`
587 /// cannot make: `localhost` is a name, and only resolving it says where it goes.
588 #[tokio::test]
589 async fn a_host_that_resolves_to_loopback_is_refused() {
590 let error = resolve_public_host("localhost")
591 .await
592 .expect_err("localhost must never be dialled");
593
594 assert!(
595 error.message().contains("not on the public internet"),
596 "{error}"
597 );
598 }
599
600 #[tokio::test]
601 async fn a_host_that_does_not_resolve_is_a_failed_import_not_a_panic() {
602 let error = resolve_public_host("no-such-host.invalid")
603 .await
604 .expect_err("`.invalid` never resolves");
605
606 assert!(error.message().contains("no-such-host.invalid"), "{error}");
607 }
608
609 /// git's last line is the reason; the lines before it are progress.
610 #[tokio::test]
611 async fn a_failed_git_command_carries_gits_own_last_word() {
612 let (dir, _storage) = storage();
613
614 let error = run_import(
615 [OsStr::new("rev-parse"), OsStr::new("HEAD")],
616 Some(dir.path()),
617 )
618 .await
619 .expect_err("an empty directory is not a repository");
620
621 assert!(
622 error.message().contains("not a git repository"),
623 "expected git's own words, got {error}"
624 );
625 }
626
627 #[tokio::test]
628 async fn an_import_refuses_a_path_that_is_already_occupied() {
629 let (dir, storage) = storage();
630 let importer = DiskGitImport::new(dir.path());
631
632 storage
633 .init_bare(&handle(), &repo_name("taken"))
634 .await
635 .expect("seed");
636
637 let error = importer
638 .mirror(
639 &handle(),
640 &repo_name("taken"),
641 &ImportSource::new("https://example.com/taken.git").expect("valid"),
642 )
643 .await
644 .expect_err("must not adopt what is already there");
645
646 assert!(error.message().contains("already on disk"), "{error}");
647 }
648
649 #[tokio::test]
650 async fn the_in_memory_importer_records_what_it_was_asked_for() {
651 let importer = InMemoryGitImport::new();
652 let source = ImportSource::new("https://example.com/thing.git").expect("valid");
653
654 importer
655 .mirror(&handle(), &repo_name("thing"), &source)
656 .await
657 .expect("succeeds");
658
659 assert_eq!(
660 importer.requests(),
661 vec![(
662 "jamesgill".to_owned(),
663 "thing".to_owned(),
664 "https://example.com/thing.git".to_owned()
665 )]
666 );
667 }
668}