steid
Sign in

@jamesgill /

steid

steid/src/infrastructure/web/robots.rs
Rust1.6 KBCode·Blame·History·Raw
1//! `/robots.txt`.
2//!
3//! Well-behaved crawlers are the likeliest source of load on a public instance: every
4//! tag row links to an archive, every blob to a blame, every branch to a compare, and a
5//! crawler follows all of them. The paths listed here are the ones where a request
6//! spawns a `git` process or packs the repository. Browsing, issues and feeds stay
7//! crawlable — they are the portfolio.
8//!
9//! The name contains a dot, so no handle can ever collide with it: `OrgName` allows
10//! only `[a-z0-9-]`.
11
12use topcoat::{
13 Result,
14 context::Cx,
15 router::{StatusCode, route},
16};
17
18/// What a crawler is asked to leave alone. Prefix matches, per the convention.
19pub const ROBOTS: &str = "User-agent: *
20Disallow: /*/repos/*/archive/
21Disallow: /*/repos/*/search
22Disallow: /*/repos/*/blame/
23Disallow: /*/repos/*/compare
24Disallow: /*/repos/*/log/
25Disallow: /*/repos/*/commits/
26Disallow: /auth/
27";
28
29#[route(GET "/robots.txt")]
30async fn robots(_cx: &Cx) -> Result<(StatusCode, &'static str)> {
31 Ok((StatusCode::OK, ROBOTS))
32}
33
34#[cfg(test)]
35mod tests {
36 use super::*;
37
38 #[test]
39 fn every_expensive_path_is_disallowed_and_browsing_is_not() {
40 for path in [
41 "archive/", "search", "blame/", "compare", "log/", "commits/",
42 ] {
43 assert!(
44 ROBOTS.contains(&format!("Disallow: /*/repos/*/{path}")),
45 "{path}"
46 );
47 }
48 assert!(!ROBOTS.contains("Disallow: /*/repos/*/tree"));
49 assert!(!ROBOTS.contains("Disallow: /*/repos/*/issues"));
50 assert!(ROBOTS.starts_with("User-agent: *\n"));
51 }
52}