steid
Sign in

@jamesgill /

steid

steid/src/infrastructure/web/import.rs
Rust25.1 KBCode·Blame·History·Raw
1//! Importing a repository from a URL — the second way to create one.
2//!
3//! The form lives on `/{handle}/repos/new` beside the create form, but everything it
4//! needs is here: its own POST route at `/{handle}/repos/import` (a static segment, and
5//! `RepoName` reserves `import` so the two agree), the background task, and the two
6//! panels the repository page shows while a copy is running or after it failed.
7
8use std::{
9 collections::HashSet,
10 sync::{Arc, Mutex},
11};
12
13use serde::Deserialize;
14use tokio::sync::{OwnedSemaphorePermit, Semaphore};
15use topcoat::{
16 Result,
17 context::{Cx, app_context},
18 icon::{icon, iconify::iconify_icon},
19 router::{StatusCode, content::Form, error::not_found, page},
20 view::{attributes, component, view},
21};
22
23use crate::{
24 application::{
25 AppConfig, Error, ImportStatus, NewImport, RepoView, Stores, import_failure_message,
26 import_repo, retry_import, run_import,
27 },
28 components::{
29 button::{ButtonSize, ButtonVariant, button},
30 flash::{FlashKind, flash},
31 input::input,
32 label::label,
33 },
34 domain::{DomainError, ImportSource, RepoName, Visibility},
35 infrastructure::{
36 git::{DiskGitImport, DiskGitStorage},
37 repository::SqliteRepoRepo,
38 },
39};
40
41use super::{
42 context::{current_actor, location, memberships, orgs, pool, repos, server_error},
43 csrf::{CsrfGuard, csrf_field, csrf_verified},
44 profile::profile_for,
45 repo::{Tab, clone_block, clone_url_for, new_repo_form, repo_for, repo_header},
46};
47
48/// How many clones may run at once.
49///
50/// **Two, chosen rather than derived.** One would serialise a small import behind a slow
51/// one and make the feature feel broken on the only instance that has ever run it — a
52/// personal one, where the second import is usually the interesting one. Much more than
53/// two stops being a limit: `git clone --mirror` is network-bound while it fetches and
54/// then CPU- and disk-bound while `index-pack` runs, so N clones of large repositories
55/// peak together on a box that is typically one or two vCPUs, and the wave-2 pass ran
56/// four at once on a laptop, which is evidence about a laptop. Two keeps a queue moving
57/// while bounding the peak to something a small VPS survives.
58///
59/// It bounds concurrency and nothing else. **There is still no size cap**: a single 10 GB
60/// source fills the disk exactly as before.
61pub const MAX_CONCURRENT_IMPORTS: usize = 2;
62
63/// The gate every import passes through, and the record of which ones are past it.
64///
65/// One per process, in app context, cloned into each spawned task — the `Arc`s are the
66/// whole of it, so cloning costs nothing.
67///
68/// The set is what lets the panel tell "queued" from "cloning". A semaphore can say how
69/// many permits are free but not *who* holds one, and the honest sentence on a repository
70/// page is about that repository rather than about the instance.
71#[derive(Debug, Clone)]
72pub struct ImportQueue {
73 permits: Arc<Semaphore>,
74 /// `{handle}/{name}` of every import currently holding a permit. A rename is refused
75 /// while an import is landing, so the pair is stable for as long as an entry lives.
76 running: Arc<Mutex<HashSet<String>>>,
77}
78
79impl Default for ImportQueue {
80 fn default() -> Self {
81 Self {
82 permits: Arc::new(Semaphore::new(MAX_CONCURRENT_IMPORTS)),
83 running: Arc::default(),
84 }
85 }
86}
87
88impl ImportQueue {
89 /// Waits for a permit, and marks the repository as cloning until the slot is dropped.
90 pub async fn enter(&self, handle: &str, name: &str) -> ImportSlot {
91 let key = format!("{handle}/{name}");
92
93 // `acquire_owned` on a closed semaphore is the only failure, and nothing closes
94 // this one — it lives as long as the process.
95 let permit = Arc::clone(&self.permits)
96 .acquire_owned()
97 .await
98 .expect("the import semaphore is never closed");
99
100 self.running
101 .lock()
102 .expect("lock poisoned")
103 .insert(key.clone());
104
105 ImportSlot {
106 _permit: permit,
107 running: Arc::clone(&self.running),
108 key,
109 }
110 }
111
112 /// How many imports hold a permit — cloning, as opposed to queued behind one.
113 ///
114 /// Exposed as `steid_active_imports`. Sampled at scrape time from the set itself
115 /// rather than mirrored into a counter, so it cannot drift from what the panels say;
116 /// a poisoned lock reports nothing rather than failing a scrape.
117 #[must_use]
118 pub fn active(&self) -> usize {
119 self.running.lock().map_or(0, |running| running.len())
120 }
121
122 /// Whether this repository's clone is actually running rather than waiting its turn.
123 pub fn is_running(&self, handle: &str, name: &str) -> bool {
124 self.running
125 .lock()
126 .expect("lock poisoned")
127 .contains(&format!("{handle}/{name}"))
128 }
129}
130
131/// A held permit. Dropping it releases the slot and unmarks the repository.
132///
133/// A guard rather than a pair of calls, so a clone that panics still gives its permit
134/// back — two lost permits would stop every future import on the instance with no error
135/// anywhere.
136pub struct ImportSlot {
137 _permit: OwnedSemaphorePermit,
138 running: Arc<Mutex<HashSet<String>>>,
139 key: String,
140}
141
142impl Drop for ImportSlot {
143 fn drop(&mut self) {
144 self.running
145 .lock()
146 .expect("lock poisoned")
147 .remove(&self.key);
148 }
149}
150
151#[derive(Debug, Deserialize)]
152struct ImportForm {
153 /// The hidden field every form carries; see [`csrf`](super::csrf).
154 ///
155 /// `default` so that a submission with no token at all is refused by
156 /// `csrf_verified` — a 403 saying what happened — rather than failing
157 /// deserialization into a bare 400 that says nothing.
158 #[serde(default)]
159 csrf_token: String,
160 name: String,
161 source: String,
162}
163
164/// Starts an import, then hands the work to a background task.
165///
166/// Success redirects to the repository, which already exists as a row and renders the
167/// importing panel — so the owner watches it from the page they would have landed on
168/// anyway rather than from a screen that exists only during an import.
169///
170/// **The task is spawned here, not in the use case.** The application layer has no
171/// business owning a runtime, and a use case that starts a task cannot be tested
172/// without waiting for one. What it needs is cloned in: a pool handle and the data
173/// directory, both cheap to clone and neither borrowing the request.
174#[page(POST "/{handle}/repos/import")]
175async fn start_import(cx: &Cx, Form(submitted): Form<ImportForm>) -> Result {
176 csrf_verified(cx, &submitted.csrf_token).await?;
177
178 let profile = profile_for(cx).await?;
179
180 let outcome = import_repo(
181 &current_actor(cx).await?,
182 &profile.handle,
183 &NewImport {
184 name: submitted.name.clone(),
185 source: submitted.source.clone(),
186 },
187 &Stores::new(&orgs(cx), &memberships(cx), &repos(cx)),
188 )
189 .await;
190
191 let repo = match outcome {
192 Ok(repo) => repo,
193 Err(Error::Domain(DomainError::Forbidden)) => return Err(not_found().into()),
194 Err(error) => {
195 let Some(message) = import_failure_message(&error, &submitted.name) else {
196 return Err(server_error(std::io::Error::other(error.to_string())));
197 };
198
199 // Re-rendered rather than redirected, so the URL that was typed is still in
200 // the box beside the reason it was refused.
201 let state = ImportBlock {
202 name: submitted.name,
203 source: submitted.source,
204 error: message,
205 };
206
207 return view! {
208 new_repo_form(
209 handle: profile.handle.as_str(),
210 name: "",
211 description: "",
212 visibility: Visibility::Public,
213 error: "",
214 import: &state,
215 )
216 };
217 }
218 };
219
220 spawn_import(cx, &profile.handle, &repo.id, &repo.name);
221
222 view! {
223 (StatusCode::SEE_OTHER)
224 (location(&format!("/{}/repos/{}", profile.handle, repo.name))?)
225 }
226}
227
228/// Starts a failed import over, and hands the work to the same background task.
229///
230/// Owner only — [`retry_import`] decides that, and a repository whose import did not fail
231/// answers `NotFound`, which is what the button's absence already said. The panel that
232/// carries the button renders only for the owner, so reaching here as anyone else means
233/// the URL was typed.
234#[page(POST "/{handle}/repos/{name}/import/retry")]
235async fn retry(cx: &Cx, Form(guard): Form<CsrfGuard>) -> Result {
236 csrf_verified(cx, &guard.csrf_token).await?;
237
238 let repo = repo_for(cx).await?;
239
240 let restarted = retry_import(
241 &current_actor(cx).await?,
242 &repo.handle,
243 &repo.name,
244 &Stores::new(&orgs(cx), &memberships(cx), &repos(cx)),
245 )
246 .await;
247
248 match restarted {
249 Ok(restarted) => spawn_import(cx, &repo.handle, &restarted.id, &restarted.name),
250 // The same 404 the settings pages give someone who does not own the repository:
251 // a 403 would confirm that a private one by this name exists.
252 Err(Error::Domain(DomainError::Forbidden | DomainError::NotFound { .. })) => {
253 return Err(not_found().into());
254 }
255 Err(other) => return Err(server_error(std::io::Error::other(other.to_string()))),
256 }
257
258 view! {
259 (StatusCode::SEE_OTHER)
260 (location(&format!("/{}/repos/{}", repo.handle, repo.name))?)
261 }
262}
263
264/// Runs one import in the background, behind the instance's concurrency gate.
265///
266/// **The task is spawned here, not in the use case.** The application layer has no
267/// business owning a runtime, and a use case that starts a task cannot be tested without
268/// waiting for one. What it needs is cloned in: a pool handle, the data directory and the
269/// queue, all cheap to clone and none borrowing the request.
270///
271/// The permit is taken *inside* the task rather than before spawning it, so the form
272/// answers immediately whether or not there is a slot free — a submission that blocked
273/// until a clone finished would look like a hung page. Waiting is a state the repository
274/// page renders, which is the point of the queue knowing who holds a permit.
275fn spawn_import(
276 cx: &Cx,
277 handle: &crate::domain::OrgName,
278 id: &crate::domain::RepoId,
279 name: &RepoName,
280) {
281 let data_dir = app_context::<AppConfig>(cx).data_dir.clone();
282 let queue = app_context::<ImportQueue>(cx).clone();
283 let pool = pool(cx).clone();
284 let handle = handle.clone();
285 let id = id.clone();
286 let name = name.clone();
287
288 tokio::spawn(async move {
289 // Held for the whole clone and released by the guard, whatever happens to it.
290 let _slot = queue.enter(handle.as_str(), name.as_str()).await;
291
292 if let Err(error) = run_import(
293 &id,
294 &handle,
295 &SqliteRepoRepo::new(pool),
296 &DiskGitImport::new(data_dir.clone()),
297 &DiskGitStorage::new(data_dir),
298 )
299 .await
300 {
301 // The import's own failure is recorded on the repository. Reaching here
302 // means the *recording* failed, which leaves a row stuck at `importing`
303 // until the next restart sweeps it — worth a log line, and nothing else can
304 // be done about it from in here.
305 eprintln!("steid: import of {handle}/{name} could not be recorded: {error}");
306 }
307 });
308}
309
310/// What the import block is showing.
311///
312/// Grouped rather than passed as three more parameters to
313/// [`new_repo_form`](super::repo::new_repo_form), which already carries the create
314/// form's own three: past about seven, an argument list stops being readable and starts
315/// being a place to put a value in the wrong slot.
316///
317/// Owned rather than borrowed because it is built once per render out of values with
318/// different lifetimes — a submitted form, or nothing at all.
319#[derive(Debug, Default, Clone)]
320pub(super) struct ImportBlock {
321 pub name: String,
322 pub source: String,
323 /// Empty when there is nothing to complain about.
324 pub error: String,
325}
326
327/// The second block on the new-repository page.
328///
329/// Clearly secondary: a rule above it, a quieter heading, and two fields instead of
330/// four. Creating an empty repository is what most visits are for, and an import that
331/// looked equally weighted would make the page read as a choice rather than a default
332/// with an alternative.
333///
334/// No JavaScript, so the name is not filled in from the URL as it is typed. The field
335/// may be left blank instead and the last path segment is used — which is the same
336/// answer a script would have given, arrived at on the server.
337#[component]
338pub(super) async fn import_block(handle: &str, state: &ImportBlock) -> Result {
339 let ImportBlock {
340 name,
341 source,
342 error,
343 } = state;
344 view! {
345 <section class="mt-10 border-t border-border pt-8">
346 <h2 class="text-sm font-medium">"Or import an existing repository"</h2>
347 <p class="mt-1 text-xs text-muted-foreground">
348 "Copies every branch and tag from a public "
349 <span class="font-mono">"https://"</span>
350 " repository. It runs in the background."
351 </p>
352
353 if !error.is_empty() {
354 <div class="mt-4">
355 flash(kind: FlashKind::Error, (error.as_str()))
356 </div>
357 }
358
359 <form method="post" action=(format!("/{handle}/repos/import")) class="mt-4 space-y-5">
360 csrf_field()
361 <div class="space-y-2">
362 label(attrs: attributes! { for="source" }, "URL")
363 input(attrs: attributes! {
364 id="source"
365 name="source"
366 type="url"
367 value=(source.as_str())
368 placeholder="https://github.com/owner/project.git"
369 required=(true)
370 maxlength=(ImportSource::MAX_LEN.to_string())
371 })
372 <p class="text-xs text-muted-foreground">
373 "Public repositories only. Steid will not send credentials."
374 </p>
375 </div>
376
377 <div class="space-y-2">
378 label(attrs: attributes! { for="import-name" }, "Name")
379 input(attrs: attributes! {
380 id="import-name"
381 name="name"
382 type="text"
383 value=(name.as_str())
384 placeholder="taken from the URL"
385 maxlength=(RepoName::MAX_LEN.to_string())
386 })
387 <p class="text-xs text-muted-foreground">
388 "Optional. Blank uses the last part of the URL, without "
389 <span class="font-mono">".git"</span> "."
390 </p>
391 </div>
392
393 // Outline, where the create form's is solid. Two identical buttons on
394 // one page make it read as a choice between equals; the heading and the
395 // rule above say this is the alternative, and the button should agree.
396 button(
397 attrs: attributes! { type="submit" },
398 variant: ButtonVariant::Outline,
399 "Import repository"
400 )
401 </form>
402 </section>
403 }
404}
405
406/// The repository page, for a repository whose contents have not arrived.
407///
408/// Stands where the file listing would, so the page is the same page — header, tabs,
409/// sidebar — with the one part that cannot be true yet replaced. A screen of its own
410/// would mean the owner watching a URL they then have to leave.
411#[component]
412pub(super) async fn import_state_panel(cx: &Cx, repo: &RepoView, status: &ImportStatus) -> Result {
413 let source = status
414 .source
415 .as_ref()
416 .map(ImportSource::as_str)
417 .unwrap_or("its source");
418
419 // A row that says `importing` is either being cloned or waiting for a slot, and only
420 // the queue knows which. Saying "importing" for a repository nothing has started on
421 // yet is the sentence that makes a busy instance look broken.
422 let queued = status.is_running()
423 && !app_context::<ImportQueue>(cx).is_running(repo.handle.as_str(), repo.name.as_str());
424
425 view! {
426 if status.is_running() {
427 <div class="rounded-lg border border-border px-4 py-5">
428 // The sentence is one child of the flex row, not three. As siblings the
429 // icon, the words and the address become flex items, and at 800px a
430 // long URL turns the line into three ragged columns.
431 <p class="flex items-start gap-2 text-sm">
432 // The one live thing on the page, so it carries the primary colour
433 // — the same rule as the latest-commit dot. A queued import is not
434 // live yet, so it gets the muted clock instead.
435 if queued {
436 icon(data: iconify_icon!("feather:clock"), attrs: attributes! {
437 class="mt-0.5 size-4 shrink-0 text-muted-foreground"
438 })
439 } else {
440 icon(data: iconify_icon!("feather:download-cloud"), attrs: attributes! {
441 class="mt-0.5 size-4 shrink-0 text-primary"
442 })
443 }
444 <span class="min-w-0">
445 (if queued { "Queued — waiting to import from " } else { "Importing from " })
446 // The ellipsis is inside the span so it sits against the
447 // address rather than a word-space away from it.
448 <span class="font-mono break-all">(format!("{source}…"))</span>
449 </span>
450 </p>
451 <p class="mt-2 text-xs text-muted-foreground">
452 if queued {
453 (format!(
454 "Steid copies {MAX_CONCURRENT_IMPORTS} repositories at a time, and \
455 this one is next in line. Refresh this page to check."
456 ))
457 } else {
458 "Every branch and tag is being copied. Refresh this page to check — \
459 there is nothing to wait on here."
460 }
461 </p>
462 </div>
463 } else {
464 <div class="rounded-lg border border-border px-4 py-5">
465 <p class="flex items-start gap-2 text-sm">
466 icon(data: iconify_icon!("feather:alert-triangle"), attrs: attributes! {
467 class="mt-0.5 size-4 shrink-0 text-muted-foreground"
468 })
469 <span class="min-w-0">
470 "The import from "
471 <span class="font-mono break-all">(source)</span>
472 " did not finish."
473 </span>
474 </p>
475 match &status.error {
476 // git's own words. The owner supplied the address, so the reason is
477 // theirs to read — usually "repository not found" or a name that
478 // does not resolve.
479 Some(reason) => <pre class="mt-3 overflow-x-auto rounded-lg border border-border bg-surface px-4 py-3 font-mono text-xs whitespace-pre-wrap">(reason)</pre>,
480 None => "",
481 }
482 if repo.viewer_is_owner {
483 // Retry first, because a failed import is usually a remote that was
484 // briefly unreachable rather than an address that was wrong. Deleting
485 // is the way out when it is not, and it stays a sentence rather than a
486 // second button: one destructive control beside one ordinary one reads
487 // as a choice between equals.
488 <div class="mt-4 flex flex-wrap items-center gap-3">
489 <form
490 method="post"
491 action=(format!("/{}/repos/{}/import/retry", repo.handle, repo.name))
492 >
493 csrf_field()
494 button(
495 attrs: attributes! { type="submit" },
496 variant: ButtonVariant::Outline,
497 size: ButtonSize::Sm,
498 "Retry import"
499 )
500 </form>
501 <p class="text-xs text-muted-foreground">
502 "Nothing was kept. Retrying copies from the same address again, or "
503 "delete this repository from its "
504 <a
505 href=(format!("/{}/repos/{}/settings", repo.handle, repo.name))
506 class="underline hover:text-foreground"
507 >"settings"</a>
508 " to free the name."
509 </p>
510 </div>
511 }
512 </div>
513 }
514 }
515}
516
517/// The About sidebar, for a repository that has no contents to count.
518///
519/// The same shape as the ready one — description, a definition list, the clone address
520/// — with every number an em dash. Leaving the rows out would make the sidebar change
521/// height when the import lands; a zero would be a claim that the repository is empty.
522#[component]
523pub(super) async fn import_about(repo: &RepoView, clone: &str) -> Result {
524 view! {
525 <section class="text-sm">
526 <h2 class="text-xs font-medium uppercase tracking-wider text-muted-foreground">
527 "About"
528 </h2>
529 match &repo.description {
530 Some(description) => <p class="mt-2 leading-relaxed">(description)</p>,
531 None => <p class="mt-2 text-muted-foreground">"No description."</p>,
532 }
533 </section>
534
535 <dl class="mt-4 space-y-1.5 border-t border-border pt-4 text-xs">
536 for term in ["Commits", "Branches", "Tags", "Pushed"] {
537 <div class="flex items-baseline justify-between gap-3">
538 <dt class="text-muted-foreground">(term)</dt>
539 <dd class="font-mono text-muted-foreground">"—"</dd>
540 </div>
541 }
542 </dl>
543
544 clone_block(
545 handle: repo.handle.as_str(),
546 name: repo.name.as_str(),
547 // No revision, so no archive links: there is nothing to pack yet. Empty is
548 // exactly what an empty repository passes, and `clone_block` already treats
549 // it that way.
550 rev: "",
551 url: clone,
552 )
553 }
554}
555
556/// The whole repository page while an import is running or after it failed.
557///
558/// A component rather than a branch inside `repo_page`, so the landing page keeps one
559/// shape: resolve the repository, and if it has not arrived, render this instead of
560/// asking git five questions about an empty directory.
561#[component]
562pub(super) async fn importing_page(cx: &Cx, repo: &RepoView, status: &ImportStatus) -> Result {
563 let clone = clone_url_for(cx, repo);
564
565 view! {
566 repo_header(repo: repo, rev: "", active: Tab::Code)
567
568 <div class="flex flex-col lg:flex-row lg:items-start lg:gap-6">
569 <div class="min-w-0 lg:flex-1">
570 import_state_panel(repo: repo, status: status)
571 </div>
572 <aside class="order-first mb-6 w-full shrink-0 border-b border-border pb-6 lg:order-none lg:sticky lg:top-6 lg:mb-0 lg:w-72 lg:border-b-0 lg:pb-0">
573 import_about(repo: repo, clone: clone.as_str())
574 </aside>
575 </div>
576 }
577}
578
579#[cfg(test)]
580mod tests {
581 use super::*;
582
583 #[tokio::test]
584 async fn a_slot_marks_the_repository_and_gives_the_permit_back_when_dropped() {
585 let queue = ImportQueue::default();
586
587 assert!(!queue.is_running("acme", "steid"));
588
589 {
590 let _slot = queue.enter("acme", "steid").await;
591 assert!(queue.is_running("acme", "steid"));
592 assert!(!queue.is_running("acme", "other"));
593 }
594
595 assert!(!queue.is_running("acme", "steid"));
596 }
597
598 #[tokio::test]
599 async fn a_third_import_waits_and_the_page_can_tell() {
600 let queue = ImportQueue::default();
601
602 let first = queue.enter("acme", "one").await;
603 let second = queue.enter("acme", "two").await;
604
605 // The third acquires nothing while the first two hold their permits, so its row
606 // reads as queued rather than as importing.
607 let waiting = tokio::spawn({
608 let queue = queue.clone();
609 async move { queue.enter("acme", "three").await }
610 });
611
612 tokio::task::yield_now().await;
613 assert!(!queue.is_running("acme", "three"));
614
615 drop(first);
616 let third = waiting.await.expect("the waiter is not cancelled");
617
618 assert!(queue.is_running("acme", "three"));
619 assert!(queue.is_running("acme", "two"));
620
621 drop(second);
622 drop(third);
623 assert!(!queue.is_running("acme", "three"));
624 }
625}