steid
Sign in

@jamesgill /

steid

steid/src/infrastructure/highlight.rs
13.9 KBCode·Blame·Raw
1//! Syntax highlighting for the blob view.
2//!
3//! Classes, never colours. [`syntect`] can emit inline `style` attributes, and that
4//! would hardcode one palette into the markup — the theme would then follow neither a
5//! palette change nor the colour scheme, which is exactly what `styles.css` forbids
6//! everywhere else. So the output is `hl-`-prefixed classes and the colours live in
7//! `styles.css` beside every other token.
8//!
9//! **One HTML fragment per source line**, because the blob view is a table with a line
10//! number in one cell and the code in the other. `syntect`'s own
11//! `ClassedHTMLGenerator` produces a single blob whose `<span>`s cross line boundaries,
12//! which cannot be split across table rows — so this drives the parser a line at a time
13//! and closes every open span at the end of each line, reopening it at the start of the
14//! next. The DOM shape is unchanged: what was a text node is now the same text wrapped
15//! in spans.
16//!
17//! **Highlighting never turns a viewable file into an error.** Every failure path —
18//! no syntax for the file, a regex that gives up, a file past the cap — falls back to
19//! the plain text the page rendered before.
20
21use std::sync::OnceLock;
22
23use syntect::{
24 html::{ClassStyle, line_tokens_to_classed_spans},
25 parsing::{ParseState, ScopeStack, SyntaxReference, SyntaxSet},
26 util::LinesWithEndings,
27};
28
29/// The class prefix, so a scope atom cannot collide with a Tailwind utility.
30const PREFIX: &str = "hl-";
31
32const CLASS_STYLE: ClassStyle = ClassStyle::SpacedPrefixed { prefix: PREFIX };
33
34/// Past this, a file is shown as plain text.
35///
36/// Highlighting is regex matching over every line, and the fancy-regex engine is pure
37/// Rust rather than fast C — a generated or vendored file can be megabytes of one long
38/// line, and the page should still arrive. Half a megabyte is well above anything
39/// written by hand and far below `MAX_BLOB_BYTES`.
40pub const MAX_BYTES: usize = 512 * 1024;
41
42/// The other half of the cap: many short lines cost per-line work, not per-byte work.
43pub const MAX_LINES: usize = 10_000;
44
45/// A file, ready for the blob view's table: one entry per line, in the same order as
46/// `text.lines()`.
47#[derive(Debug, Clone, PartialEq, Eq)]
48pub struct Source {
49 pub lines: Vec<SourceLine>,
50 /// Whether highlighting was skipped because the file is past the caps. The page
51 /// says so, because otherwise a large Rust file silently looks like an
52 /// unsupported language.
53 pub too_large: bool,
54}
55
56/// One line of a file.
57#[derive(Debug, Clone, PartialEq, Eq)]
58pub enum SourceLine {
59 /// Text, to be escaped by the view like anything else.
60 Plain(String),
61 /// Markup this module wrote, to be emitted verbatim.
62 Classed(String),
63}
64
65/// Splits a file into lines, highlighted where the language is known.
66///
67/// Every fallback lands on [`SourceLine::Plain`]: past the caps, no syntax for the
68/// name, a parser that gave up. Highlighting can make a file prettier; it can never
69/// stop it being readable.
70pub fn source_lines(file_name: &str, text: &str) -> Source {
71 let too_large = text.len() > MAX_BYTES || text.lines().count() > MAX_LINES;
72
73 let classed = (!too_large)
74 .then(|| {
75 let syntaxes = syntaxes();
76 let syntax = syntax_for(syntaxes, file_name, text)?;
77 classed_lines(syntaxes, syntax, text)
78 })
79 .flatten()
80 // A fragment per line is the contract the table relies on; anything else is a
81 // bug in this module, and plain text is the safe reading of it.
82 .filter(|lines| lines.len() == text.lines().count());
83
84 let lines = match classed {
85 Some(classed) => text
86 .lines()
87 .zip(classed)
88 .map(|(line, html)| match line.is_empty() {
89 // An empty cell has no height, so a blank line would close the gap it
90 // is there to make. The plain path has always spent a space on this.
91 true => SourceLine::Plain(" ".to_owned()),
92 false => SourceLine::Classed(html),
93 })
94 .collect(),
95 None => text
96 .lines()
97 .map(|line| match line.is_empty() {
98 true => SourceLine::Plain(" ".to_owned()),
99 false => SourceLine::Plain(line.to_owned()),
100 })
101 .collect(),
102 };
103
104 Source { lines, too_large }
105}
106
107/// The syntax set, loaded once.
108///
109/// `two_face` rather than `SyntaxSet::load_defaults_newlines`: syntect's own defaults
110/// are Sublime's, which have no TOML, no Dockerfile and no TypeScript — the first two
111/// of which are in this repository. `two_face` is the same set plus the grammars `bat`
112/// curates, so the gap closes without teaching this module anything about individual
113/// languages. Its `syntect-fancy` feature pins it to the same pure-Rust regex engine,
114/// so no `onig` — and therefore no C — comes in behind it.
115///
116/// Loading is a few megabytes of deserialization, so it happens on the first blob
117/// viewed and never again. `_newlines` is the variant whose rules expect a trailing
118/// newline on each line, which is what [`LinesWithEndings`] hands it.
119fn syntaxes() -> &'static SyntaxSet {
120 static SYNTAXES: OnceLock<SyntaxSet> = OnceLock::new();
121 SYNTAXES.get_or_init(two_face::syntax::extra_newlines)
122}
123
124/// Extensions no grammar in the set claims, and the grammar to read them with anyway.
125///
126/// `two-face`'s JavaScript grammar registers `js` and `htc` and nothing else, so `.jsx`
127/// rendered as plain text while `.tsx` was highlighted, TypeScriptReact being its own
128/// grammar. JSX is JavaScript with element literals in it and the JavaScript grammar
129/// reads it well enough.
130///
131/// One entry, deliberately: every alias is a claim that two languages are close enough
132/// to read as one, and a wrong one is worse than no colour.
133const ALIASES: &[(&str, &str)] = &[("jsx", "js")];
134
135/// The language, from the file name and then the first line.
136///
137/// This is `SyntaxSet::find_syntax_for_file` without the filesystem: that method opens
138/// the path to read a shebang, and here the contents are already in memory. The whole
139/// name is tried as an extension first because that is how syntect registers names like
140/// `Makefile`; plain text is treated as no syntax at all, since wrapping every line in
141/// a span that carries no colour is pure markup for nothing.
142fn syntax_for<'a>(
143 syntaxes: &'a SyntaxSet,
144 file_name: &str,
145 text: &str,
146) -> Option<&'a SyntaxReference> {
147 let extension = file_name.rsplit_once('.').map(|(_, ext)| ext);
148
149 // After the set's own answer, never before it: an alias is what to do when no
150 // grammar claims the extension, not a way to overrule one that does.
151 let aliased = extension.and_then(|ext| {
152 ALIASES
153 .iter()
154 .find(|(from, _)| *from == ext)
155 .map(|(_, to)| *to)
156 });
157
158 let syntax = syntaxes
159 .find_syntax_by_extension(file_name)
160 .or_else(|| extension.and_then(|ext| syntaxes.find_syntax_by_extension(ext)))
161 .or_else(|| aliased.and_then(|to| syntaxes.find_syntax_by_extension(to)))
162 .or_else(|| syntaxes.find_syntax_by_first_line(text.lines().next().unwrap_or("")))?;
163
164 (syntax.name != syntaxes.find_syntax_plain_text().name).then_some(syntax)
165}
166
167/// One line of HTML per line of source, or `None` if the parser gave up.
168///
169/// The span stack is carried across lines — a block comment opens on one line and
170/// closes on another — but the *markup* is not: the scopes open at the end of a line
171/// are closed there and reopened on the next, so each fragment is balanced and can sit
172/// in its own table cell.
173fn classed_lines(
174 syntaxes: &SyntaxSet,
175 syntax: &SyntaxReference,
176 text: &str,
177) -> Option<Vec<String>> {
178 let mut state = ParseState::new(syntax);
179 let mut stack = ScopeStack::new();
180 let mut lines = Vec::new();
181
182 for line in LinesWithEndings::from(text) {
183 let ops = state.parse_line(line, syntaxes).ok()?;
184
185 let mut html = String::with_capacity(line.len());
186 for scope in &stack.scopes {
187 open(&mut html, &scope.build_string());
188 }
189
190 let (body, _) = line_tokens_to_classed_spans(line, &ops, CLASS_STYLE, &mut stack).ok()?;
191 // The line ending was handed to the parser because the `_newlines` syntaxes
192 // expect it, but it must not reach the cell: `white-space: pre` would render it
193 // as a second, empty line inside every row. It can be escaped anywhere in the
194 // fragment — inside a span, or after one closes — so it is dropped by character
195 // rather than trimmed off the end.
196 html.extend(body.chars().filter(|c| *c != '\n' && *c != '\r'));
197
198 // Whatever the line left open on the stack is open in the markup too, and is
199 // closed here so the fragment stands alone in its own table cell.
200 for _ in 0..stack.scopes.len() {
201 html.push_str("</span>");
202 }
203
204 lines.push(html);
205 }
206
207 Some(lines)
208}
209
210/// Opens one span for a scope, spelling its atoms the way syntect's own writer does.
211fn open(html: &mut String, scope: &str) {
212 html.push_str("<span class=\"");
213 for (index, atom) in scope.split('.').filter(|atom| !atom.is_empty()).enumerate() {
214 if index != 0 {
215 html.push(' ');
216 }
217 html.push_str(PREFIX);
218 html.push_str(atom);
219 }
220 html.push_str("\">");
221}
222
223#[cfg(test)]
224mod tests {
225 use super::*;
226
227 fn classes(line: &SourceLine) -> String {
228 match line {
229 SourceLine::Classed(html) => html.clone(),
230 SourceLine::Plain(text) => text.clone(),
231 }
232 }
233
234 #[test]
235 fn rust_source_is_wrapped_in_prefixed_classes() {
236 let source = source_lines("main.rs", "fn main() {\n // hi\n}\n");
237
238 assert!(!source.too_large);
239 assert_eq!(source.lines.len(), 3);
240 assert!(matches!(source.lines[0], SourceLine::Classed(_)));
241
242 let first = classes(&source.lines[0]);
243 assert!(first.contains("class=\"hl-"), "{first}");
244 // `fn` is `storage.type.function.rust`, so every atom becomes its own class.
245 assert!(first.contains("hl-storage"), "{first}");
246 assert!(first.contains("hl-entity hl-name hl-function"), "{first}");
247 assert!(classes(&source.lines[1]).contains("hl-comment"));
248 }
249
250 #[test]
251 fn a_line_carries_no_line_ending_and_closes_every_span_it_opens() {
252 let source = source_lines("main.rs", "fn main() {\n let x = \"a\";\n}\n");
253
254 for line in &source.lines {
255 let html = classes(line);
256 assert!(!html.contains('\n'), "{html}");
257 assert_eq!(
258 html.matches("<span").count(),
259 html.matches("</span>").count(),
260 "{html}"
261 );
262 }
263 }
264
265 #[test]
266 fn the_source_is_escaped_rather_than_passed_through() {
267 let source = source_lines("main.rs", "// <script>alert(1)</script>\n");
268
269 let html = classes(&source.lines[0]);
270 assert!(!html.contains("<script"), "{html}");
271 assert!(html.contains("&lt;script"), "{html}");
272 }
273
274 #[test]
275 fn an_unknown_extension_stays_plain() {
276 let source = source_lines("notes.wibble", "hello\nworld\n");
277
278 assert_eq!(
279 source.lines,
280 vec![
281 SourceLine::Plain("hello".to_owned()),
282 SourceLine::Plain("world".to_owned()),
283 ]
284 );
285 }
286
287 #[test]
288 fn a_shebang_names_the_language_when_the_name_does_not() {
289 let source = source_lines("run", "#!/bin/sh\necho hi\n");
290
291 assert!(matches!(source.lines[0], SourceLine::Classed(_)));
292 }
293
294 #[test]
295 fn a_file_past_the_byte_cap_is_plain_and_says_so() {
296 let text = format!("fn main() {{}}\n{}\n", "x".repeat(MAX_BYTES));
297 let source = source_lines("main.rs", &text);
298
299 assert!(source.too_large);
300 assert!(
301 source
302 .lines
303 .iter()
304 .all(|line| matches!(line, SourceLine::Plain(_)))
305 );
306 }
307
308 #[test]
309 fn a_file_past_the_line_cap_is_plain_and_says_so() {
310 let text = "let x = 1;\n".repeat(MAX_LINES + 1);
311 let source = source_lines("main.rs", &text);
312
313 assert!(source.too_large);
314 assert_eq!(source.lines.len(), MAX_LINES + 1);
315 assert!(matches!(source.lines[0], SourceLine::Plain(_)));
316 }
317
318 #[test]
319 fn a_blank_line_keeps_its_height() {
320 let source = source_lines("main.rs", "fn a() {}\n\nfn b() {}\n");
321
322 assert_eq!(source.lines[1], SourceLine::Plain(" ".to_owned()));
323 }
324
325 #[test]
326 fn plain_text_is_not_dressed_in_spans_that_colour_nothing() {
327 let source = source_lines("notes.txt", "hello\n");
328
329 assert_eq!(source.lines, vec![SourceLine::Plain("hello".to_owned())]);
330 }
331
332 /// The gap `two_face` was added to close, asserted rather than described: TOML,
333 /// Dockerfile and TypeScript rendered plain until it landed, and the first two are
334 /// in this repository. `.jsx` is here through [`ALIASES`] rather than through a
335 /// grammar of its own.
336 #[test]
337 fn the_languages_this_repository_uses_are_all_highlighted() {
338 for name in [
339 "main.rs",
340 "install.sh",
341 "Cargo.toml",
342 "Dockerfile",
343 "app.ts",
344 "App.tsx",
345 "README.md",
346 "styles.css",
347 "config.yaml",
348 "data.json",
349 "Makefile",
350 "app.py",
351 "index.js",
352 "App.jsx",
353 "main.go",
354 "query.sql",
355 "main.c",
356 ] {
357 assert!(
358 syntax_for(syntaxes(), name, "").is_some(),
359 "{name} has no syntax"
360 );
361 }
362 }
363
364 #[test]
365 fn toml_is_highlighted_and_not_merely_recognised() {
366 let source = source_lines("Cargo.toml", "[package]\nname = \"steid\"\n");
367
368 assert!(matches!(source.lines[1], SourceLine::Classed(_)));
369 let html = classes(&source.lines[1]);
370 assert!(html.contains("hl-string"), "{html}");
371 }
372}