| | @@ -0,0 +1,330 @@ |
| 1 | +//! Syntax highlighting for the blob view. |
| 2 | +//! |
| 3 | +//! Classes, never colours. [`syntect`] can emit inline `style` attributes, and that |
| 4 | +//! would hardcode one palette into the markup — the theme would then follow neither a |
| 5 | +//! palette change nor the colour scheme, which is exactly what `styles.css` forbids |
| 6 | +//! everywhere else. So the output is `hl-`-prefixed classes and the colours live in |
| 7 | +//! `styles.css` beside every other token. |
| 8 | +//! |
| 9 | +//! **One HTML fragment per source line**, because the blob view is a table with a line |
| 10 | +//! number in one cell and the code in the other. `syntect`'s own |
| 11 | +//! `ClassedHTMLGenerator` produces a single blob whose `<span>`s cross line boundaries, |
| 12 | +//! which cannot be split across table rows — so this drives the parser a line at a time |
| 13 | +//! and closes every open span at the end of each line, reopening it at the start of the |
| 14 | +//! next. The DOM shape is unchanged: what was a text node is now the same text wrapped |
| 15 | +//! in spans. |
| 16 | +//! |
| 17 | +//! **Highlighting never turns a viewable file into an error.** Every failure path — |
| 18 | +//! no syntax for the file, a regex that gives up, a file past the cap — falls back to |
| 19 | +//! the plain text the page rendered before. |
| 20 | + |
| 21 | +use std::sync::OnceLock; |
| 22 | + |
| 23 | +use syntect::{ |
| 24 | + html::{ClassStyle, line_tokens_to_classed_spans}, |
| 25 | + parsing::{ParseState, ScopeStack, SyntaxReference, SyntaxSet}, |
| 26 | + util::LinesWithEndings, |
| 27 | +}; |
| 28 | + |
| 29 | +/// The class prefix, so a scope atom cannot collide with a Tailwind utility. |
| 30 | +const PREFIX: &str = "hl-"; |
| 31 | + |
| 32 | +const CLASS_STYLE: ClassStyle = ClassStyle::SpacedPrefixed { prefix: PREFIX }; |
| 33 | + |
| 34 | +/// Past this, a file is shown as plain text. |
| 35 | +/// |
| 36 | +/// Highlighting is regex matching over every line, and the fancy-regex engine is pure |
| 37 | +/// Rust rather than fast C — a generated or vendored file can be megabytes of one long |
| 38 | +/// line, and the page should still arrive. Half a megabyte is well above anything |
| 39 | +/// written by hand and far below `MAX_BLOB_BYTES`. |
| 40 | +pub const MAX_BYTES: usize = 512 * 1024; |
| 41 | + |
| 42 | +/// The other half of the cap: many short lines cost per-line work, not per-byte work. |
| 43 | +pub const MAX_LINES: usize = 10_000; |
| 44 | + |
| 45 | +/// A file, ready for the blob view's table: one entry per line, in the same order as |
| 46 | +/// `text.lines()`. |
| 47 | +#[derive(Debug, Clone, PartialEq, Eq)] |
| 48 | +pub struct Source { |
| 49 | + pub lines: Vec<SourceLine>, |
| 50 | + /// Whether highlighting was skipped because the file is past the caps. The page |
| 51 | + /// says so, because otherwise a large Rust file silently looks like an |
| 52 | + /// unsupported language. |
| 53 | + pub too_large: bool, |
| 54 | +} |
| 55 | + |
| 56 | +/// One line of a file. |
| 57 | +#[derive(Debug, Clone, PartialEq, Eq)] |
| 58 | +pub enum SourceLine { |
| 59 | + /// Text, to be escaped by the view like anything else. |
| 60 | + Plain(String), |
| 61 | + /// Markup this module wrote, to be emitted verbatim. |
| 62 | + Classed(String), |
| 63 | +} |
| 64 | + |
| 65 | +/// Splits a file into lines, highlighted where the language is known. |
| 66 | +/// |
| 67 | +/// Every fallback lands on [`SourceLine::Plain`]: past the caps, no syntax for the |
| 68 | +/// name, a parser that gave up. Highlighting can make a file prettier; it can never |
| 69 | +/// stop it being readable. |
| 70 | +pub fn source_lines(file_name: &str, text: &str) -> Source { |
| 71 | + let too_large = text.len() > MAX_BYTES || text.lines().count() > MAX_LINES; |
| 72 | + |
| 73 | + let classed = (!too_large) |
| 74 | + .then(|| { |
| 75 | + let syntaxes = syntaxes(); |
| 76 | + let syntax = syntax_for(syntaxes, file_name, text)?; |
| 77 | + classed_lines(syntaxes, syntax, text) |
| 78 | + }) |
| 79 | + .flatten() |
| 80 | + // A fragment per line is the contract the table relies on; anything else is a |
| 81 | + // bug in this module, and plain text is the safe reading of it. |
| 82 | + .filter(|lines| lines.len() == text.lines().count()); |
| 83 | + |
| 84 | + let lines = match classed { |
| 85 | + Some(classed) => text |
| 86 | + .lines() |
| 87 | + .zip(classed) |
| 88 | + .map(|(line, html)| match line.is_empty() { |
| 89 | + // An empty cell has no height, so a blank line would close the gap it |
| 90 | + // is there to make. The plain path has always spent a space on this. |
| 91 | + true => SourceLine::Plain(" ".to_owned()), |
| 92 | + false => SourceLine::Classed(html), |
| 93 | + }) |
| 94 | + .collect(), |
| 95 | + None => text |
| 96 | + .lines() |
| 97 | + .map(|line| match line.is_empty() { |
| 98 | + true => SourceLine::Plain(" ".to_owned()), |
| 99 | + false => SourceLine::Plain(line.to_owned()), |
| 100 | + }) |
| 101 | + .collect(), |
| 102 | + }; |
| 103 | + |
| 104 | + Source { lines, too_large } |
| 105 | +} |
| 106 | + |
| 107 | +/// The default syntax set, loaded once. |
| 108 | +/// |
| 109 | +/// Loading is a few megabytes of deserialization, so it happens on the first blob |
| 110 | +/// viewed and never again. `_newlines` is the variant whose rules expect a trailing |
| 111 | +/// newline on each line, which is what [`LinesWithEndings`] hands it. |
| 112 | +fn syntaxes() -> &'static SyntaxSet { |
| 113 | + static SYNTAXES: OnceLock<SyntaxSet> = OnceLock::new(); |
| 114 | + SYNTAXES.get_or_init(SyntaxSet::load_defaults_newlines) |
| 115 | +} |
| 116 | + |
| 117 | +/// The language, from the file name and then the first line. |
| 118 | +/// |
| 119 | +/// This is `SyntaxSet::find_syntax_for_file` without the filesystem: that method opens |
| 120 | +/// the path to read a shebang, and here the contents are already in memory. The whole |
| 121 | +/// name is tried as an extension first because that is how syntect registers names like |
| 122 | +/// `Makefile`; plain text is treated as no syntax at all, since wrapping every line in |
| 123 | +/// a span that carries no colour is pure markup for nothing. |
| 124 | +fn syntax_for<'a>( |
| 125 | + syntaxes: &'a SyntaxSet, |
| 126 | + file_name: &str, |
| 127 | + text: &str, |
| 128 | +) -> Option<&'a SyntaxReference> { |
| 129 | + let extension = file_name.rsplit_once('.').map(|(_, ext)| ext); |
| 130 | + |
| 131 | + let syntax = syntaxes |
| 132 | + .find_syntax_by_extension(file_name) |
| 133 | + .or_else(|| extension.and_then(|ext| syntaxes.find_syntax_by_extension(ext))) |
| 134 | + .or_else(|| syntaxes.find_syntax_by_first_line(text.lines().next().unwrap_or("")))?; |
| 135 | + |
| 136 | + (syntax.name != syntaxes.find_syntax_plain_text().name).then_some(syntax) |
| 137 | +} |
| 138 | + |
| 139 | +/// One line of HTML per line of source, or `None` if the parser gave up. |
| 140 | +/// |
| 141 | +/// The span stack is carried across lines — a block comment opens on one line and |
| 142 | +/// closes on another — but the *markup* is not: the scopes open at the end of a line |
| 143 | +/// are closed there and reopened on the next, so each fragment is balanced and can sit |
| 144 | +/// in its own table cell. |
| 145 | +fn classed_lines( |
| 146 | + syntaxes: &SyntaxSet, |
| 147 | + syntax: &SyntaxReference, |
| 148 | + text: &str, |
| 149 | +) -> Option<Vec<String>> { |
| 150 | + let mut state = ParseState::new(syntax); |
| 151 | + let mut stack = ScopeStack::new(); |
| 152 | + let mut lines = Vec::new(); |
| 153 | + |
| 154 | + for line in LinesWithEndings::from(text) { |
| 155 | + let ops = state.parse_line(line, syntaxes).ok()?; |
| 156 | + |
| 157 | + let mut html = String::with_capacity(line.len()); |
| 158 | + for scope in &stack.scopes { |
| 159 | + open(&mut html, &scope.build_string()); |
| 160 | + } |
| 161 | + |
| 162 | + let (body, _) = line_tokens_to_classed_spans(line, &ops, CLASS_STYLE, &mut stack).ok()?; |
| 163 | + // The line ending was handed to the parser because the `_newlines` syntaxes |
| 164 | + // expect it, but it must not reach the cell: `white-space: pre` would render it |
| 165 | + // as a second, empty line inside every row. It can be escaped anywhere in the |
| 166 | + // fragment — inside a span, or after one closes — so it is dropped by character |
| 167 | + // rather than trimmed off the end. |
| 168 | + html.extend(body.chars().filter(|c| *c != '\n' && *c != '\r')); |
| 169 | + |
| 170 | + // Whatever the line left open on the stack is open in the markup too, and is |
| 171 | + // closed here so the fragment stands alone in its own table cell. |
| 172 | + for _ in 0..stack.scopes.len() { |
| 173 | + html.push_str("</span>"); |
| 174 | + } |
| 175 | + |
| 176 | + lines.push(html); |
| 177 | + } |
| 178 | + |
| 179 | + Some(lines) |
| 180 | +} |
| 181 | + |
| 182 | +/// Opens one span for a scope, spelling its atoms the way syntect's own writer does. |
| 183 | +fn open(html: &mut String, scope: &str) { |
| 184 | + html.push_str("<span class=\""); |
| 185 | + for (index, atom) in scope.split('.').filter(|atom| !atom.is_empty()).enumerate() { |
| 186 | + if index != 0 { |
| 187 | + html.push(' '); |
| 188 | + } |
| 189 | + html.push_str(PREFIX); |
| 190 | + html.push_str(atom); |
| 191 | + } |
| 192 | + html.push_str("\">"); |
| 193 | +} |
| 194 | + |
| 195 | +#[cfg(test)] |
| 196 | +mod tests { |
| 197 | + use super::*; |
| 198 | + |
| 199 | + fn classes(line: &SourceLine) -> String { |
| 200 | + match line { |
| 201 | + SourceLine::Classed(html) => html.clone(), |
| 202 | + SourceLine::Plain(text) => text.clone(), |
| 203 | + } |
| 204 | + } |
| 205 | + |
| 206 | + #[test] |
| 207 | + fn rust_source_is_wrapped_in_prefixed_classes() { |
| 208 | + let source = source_lines("main.rs", "fn main() {\n // hi\n}\n"); |
| 209 | + |
| 210 | + assert!(!source.too_large); |
| 211 | + assert_eq!(source.lines.len(), 3); |
| 212 | + assert!(matches!(source.lines[0], SourceLine::Classed(_))); |
| 213 | + |
| 214 | + let first = classes(&source.lines[0]); |
| 215 | + assert!(first.contains("class=\"hl-"), "{first}"); |
| 216 | + // `fn` is `storage.type.function.rust`, so every atom becomes its own class. |
| 217 | + assert!(first.contains("hl-storage"), "{first}"); |
| 218 | + assert!(first.contains("hl-entity hl-name hl-function"), "{first}"); |
| 219 | + assert!(classes(&source.lines[1]).contains("hl-comment")); |
| 220 | + } |
| 221 | + |
| 222 | + #[test] |
| 223 | + fn a_line_carries_no_line_ending_and_closes_every_span_it_opens() { |
| 224 | + let source = source_lines("main.rs", "fn main() {\n let x = \"a\";\n}\n"); |
| 225 | + |
| 226 | + for line in &source.lines { |
| 227 | + let html = classes(line); |
| 228 | + assert!(!html.contains('\n'), "{html}"); |
| 229 | + assert_eq!( |
| 230 | + html.matches("<span").count(), |
| 231 | + html.matches("</span>").count(), |
| 232 | + "{html}" |
| 233 | + ); |
| 234 | + } |
| 235 | + } |
| 236 | + |
| 237 | + #[test] |
| 238 | + fn the_source_is_escaped_rather_than_passed_through() { |
| 239 | + let source = source_lines("main.rs", "// <script>alert(1)</script>\n"); |
| 240 | + |
| 241 | + let html = classes(&source.lines[0]); |
| 242 | + assert!(!html.contains("<script"), "{html}"); |
| 243 | + assert!(html.contains("<script"), "{html}"); |
| 244 | + } |
| 245 | + |
| 246 | + #[test] |
| 247 | + fn an_unknown_extension_stays_plain() { |
| 248 | + let source = source_lines("notes.wibble", "hello\nworld\n"); |
| 249 | + |
| 250 | + assert_eq!( |
| 251 | + source.lines, |
| 252 | + vec![ |
| 253 | + SourceLine::Plain("hello".to_owned()), |
| 254 | + SourceLine::Plain("world".to_owned()), |
| 255 | + ] |
| 256 | + ); |
| 257 | + } |
| 258 | + |
| 259 | + #[test] |
| 260 | + fn a_shebang_names_the_language_when_the_name_does_not() { |
| 261 | + let source = source_lines("run", "#!/bin/sh\necho hi\n"); |
| 262 | + |
| 263 | + assert!(matches!(source.lines[0], SourceLine::Classed(_))); |
| 264 | + } |
| 265 | + |
| 266 | + #[test] |
| 267 | + fn a_file_past_the_byte_cap_is_plain_and_says_so() { |
| 268 | + let text = format!("fn main() {{}}\n{}\n", "x".repeat(MAX_BYTES)); |
| 269 | + let source = source_lines("main.rs", &text); |
| 270 | + |
| 271 | + assert!(source.too_large); |
| 272 | + assert!( |
| 273 | + source |
| 274 | + .lines |
| 275 | + .iter() |
| 276 | + .all(|line| matches!(line, SourceLine::Plain(_))) |
| 277 | + ); |
| 278 | + } |
| 279 | + |
| 280 | + #[test] |
| 281 | + fn a_file_past_the_line_cap_is_plain_and_says_so() { |
| 282 | + let text = "let x = 1;\n".repeat(MAX_LINES + 1); |
| 283 | + let source = source_lines("main.rs", &text); |
| 284 | + |
| 285 | + assert!(source.too_large); |
| 286 | + assert_eq!(source.lines.len(), MAX_LINES + 1); |
| 287 | + assert!(matches!(source.lines[0], SourceLine::Plain(_))); |
| 288 | + } |
| 289 | + |
| 290 | + #[test] |
| 291 | + fn a_blank_line_keeps_its_height() { |
| 292 | + let source = source_lines("main.rs", "fn a() {}\n\nfn b() {}\n"); |
| 293 | + |
| 294 | + assert_eq!(source.lines[1], SourceLine::Plain(" ".to_owned())); |
| 295 | + } |
| 296 | + |
| 297 | + #[test] |
| 298 | + fn plain_text_is_not_dressed_in_spans_that_colour_nothing() { |
| 299 | + let source = source_lines("notes.txt", "hello\n"); |
| 300 | + |
| 301 | + assert_eq!(source.lines, vec![SourceLine::Plain("hello".to_owned())]); |
| 302 | + } |
| 303 | + |
| 304 | + /// The default syntax set is Sublime's, not a forge's, so the languages it lacks |
| 305 | + /// are worth knowing rather than discovering on a page. |
| 306 | + #[test] |
| 307 | + fn which_common_languages_the_default_set_knows() { |
| 308 | + for name in [ |
| 309 | + "main.rs", |
| 310 | + "Cargo.toml", |
| 311 | + "Dockerfile", |
| 312 | + "install.sh", |
| 313 | + "README.md", |
| 314 | + "app.py", |
| 315 | + "index.js", |
| 316 | + "app.ts", |
| 317 | + "main.go", |
| 318 | + "config.yaml", |
| 319 | + "data.json", |
| 320 | + "styles.css", |
| 321 | + "query.sql", |
| 322 | + "Makefile", |
| 323 | + "main.c", |
| 324 | + "App.jsx", |
| 325 | + ] { |
| 326 | + let known = syntax_for(syntaxes(), name, "").is_some(); |
| 327 | + println!("{name}: {}", if known { "highlighted" } else { "PLAIN" }); |
| 328 | + } |
| 329 | + } |
| 330 | +} |