steid
Sign in

@jamesgill /

steid

steid/src/infrastructure/highlight.rs
13.9 KBCode·Blame·Raw
979134bfeat: code reads as code, in classes rather than colours19h
1//! Syntax highlighting for the blob view.
2//!
3//! Classes, never colours. [`syntect`] can emit inline `style` attributes, and that
4//! would hardcode one palette into the markup — the theme would then follow neither a
5//! palette change nor the colour scheme, which is exactly what `styles.css` forbids
6//! everywhere else. So the output is `hl-`-prefixed classes and the colours live in
7//! `styles.css` beside every other token.
8//!
9//! **One HTML fragment per source line**, because the blob view is a table with a line
10//! number in one cell and the code in the other. `syntect`'s own
11//! `ClassedHTMLGenerator` produces a single blob whose `<span>`s cross line boundaries,
12//! which cannot be split across table rows — so this drives the parser a line at a time
13//! and closes every open span at the end of each line, reopening it at the start of the
14//! next. The DOM shape is unchanged: what was a text node is now the same text wrapped
15//! in spans.
16//!
17//! **Highlighting never turns a viewable file into an error.** Every failure path —
18//! no syntax for the file, a regex that gives up, a file past the cap — falls back to
19//! the plain text the page rendered before.
20
21use std::sync::OnceLock;
22
23use syntect::{
24 html::{ClassStyle, line_tokens_to_classed_spans},
25 parsing::{ParseState, ScopeStack, SyntaxReference, SyntaxSet},
26 util::LinesWithEndings,
27};
28
29/// The class prefix, so a scope atom cannot collide with a Tailwind utility.
30const PREFIX: &str = "hl-";
31
32const CLASS_STYLE: ClassStyle = ClassStyle::SpacedPrefixed { prefix: PREFIX };
33
34/// Past this, a file is shown as plain text.
35///
36/// Highlighting is regex matching over every line, and the fancy-regex engine is pure
37/// Rust rather than fast C — a generated or vendored file can be megabytes of one long
38/// line, and the page should still arrive. Half a megabyte is well above anything
39/// written by hand and far below `MAX_BLOB_BYTES`.
40pub const MAX_BYTES: usize = 512 * 1024;
41
42/// The other half of the cap: many short lines cost per-line work, not per-byte work.
43pub const MAX_LINES: usize = 10_000;
44
45/// A file, ready for the blob view's table: one entry per line, in the same order as
46/// `text.lines()`.
47#[derive(Debug, Clone, PartialEq, Eq)]
48pub struct Source {
49 pub lines: Vec<SourceLine>,
50 /// Whether highlighting was skipped because the file is past the caps. The page
51 /// says so, because otherwise a large Rust file silently looks like an
52 /// unsupported language.
53 pub too_large: bool,
54}
55
56/// One line of a file.
57#[derive(Debug, Clone, PartialEq, Eq)]
58pub enum SourceLine {
59 /// Text, to be escaped by the view like anything else.
60 Plain(String),
61 /// Markup this module wrote, to be emitted verbatim.
62 Classed(String),
63}
64
65/// Splits a file into lines, highlighted where the language is known.
66///
67/// Every fallback lands on [`SourceLine::Plain`]: past the caps, no syntax for the
68/// name, a parser that gave up. Highlighting can make a file prettier; it can never
69/// stop it being readable.
70pub fn source_lines(file_name: &str, text: &str) -> Source {
71 let too_large = text.len() > MAX_BYTES || text.lines().count() > MAX_LINES;
72
73 let classed = (!too_large)
74 .then(|| {
75 let syntaxes = syntaxes();
76 let syntax = syntax_for(syntaxes, file_name, text)?;
77 classed_lines(syntaxes, syntax, text)
78 })
79 .flatten()
80 // A fragment per line is the contract the table relies on; anything else is a
81 // bug in this module, and plain text is the safe reading of it.
82 .filter(|lines| lines.len() == text.lines().count());
83
84 let lines = match classed {
85 Some(classed) => text
86 .lines()
87 .zip(classed)
88 .map(|(line, html)| match line.is_empty() {
89 // An empty cell has no height, so a blank line would close the gap it
90 // is there to make. The plain path has always spent a space on this.
91 true => SourceLine::Plain(" ".to_owned()),
92 false => SourceLine::Classed(html),
93 })
94 .collect(),
95 None => text
96 .lines()
97 .map(|line| match line.is_empty() {
98 true => SourceLine::Plain(" ".to_owned()),
99 false => SourceLine::Plain(line.to_owned()),
100 })
101 .collect(),
102 };
103
104 Source { lines, too_large }
105}
106
8331d0cfeat: the languages this repository actually uses19h
107/// The syntax set, loaded once.
108///
109/// `two_face` rather than `SyntaxSet::load_defaults_newlines`: syntect's own defaults
110/// are Sublime's, which have no TOML, no Dockerfile and no TypeScript — the first two
111/// of which are in this repository. `two_face` is the same set plus the grammars `bat`
112/// curates, so the gap closes without teaching this module anything about individual
113/// languages. Its `syntect-fancy` feature pins it to the same pure-Rust regex engine,
114/// so no `onig` — and therefore no C — comes in behind it.
979134bfeat: code reads as code, in classes rather than colours19h
115///
116/// Loading is a few megabytes of deserialization, so it happens on the first blob
117/// viewed and never again. `_newlines` is the variant whose rules expect a trailing
118/// newline on each line, which is what [`LinesWithEndings`] hands it.
119fn syntaxes() -> &'static SyntaxSet {
120 static SYNTAXES: OnceLock<SyntaxSet> = OnceLock::new();
8331d0cfeat: the languages this repository actually uses19h
121 SYNTAXES.get_or_init(two_face::syntax::extra_newlines)
979134bfeat: code reads as code, in classes rather than colours19h
122}
123
610ab38feat: a .jsx file reads as JavaScript rather than as plain text17h
124/// Extensions no grammar in the set claims, and the grammar to read them with anyway.
125///
126/// `two-face`'s JavaScript grammar registers `js` and `htc` and nothing else, so `.jsx`
127/// rendered as plain text while `.tsx` was highlighted, TypeScriptReact being its own
128/// grammar. JSX is JavaScript with element literals in it and the JavaScript grammar
129/// reads it well enough.
130///
131/// One entry, deliberately: every alias is a claim that two languages are close enough
132/// to read as one, and a wrong one is worse than no colour.
133const ALIASES: &[(&str, &str)] = &[("jsx", "js")];
134
979134bfeat: code reads as code, in classes rather than colours19h
135/// The language, from the file name and then the first line.
136///
137/// This is `SyntaxSet::find_syntax_for_file` without the filesystem: that method opens
138/// the path to read a shebang, and here the contents are already in memory. The whole
139/// name is tried as an extension first because that is how syntect registers names like
140/// `Makefile`; plain text is treated as no syntax at all, since wrapping every line in
141/// a span that carries no colour is pure markup for nothing.
142fn syntax_for<'a>(
143 syntaxes: &'a SyntaxSet,
144 file_name: &str,
145 text: &str,
146) -> Option<&'a SyntaxReference> {
147 let extension = file_name.rsplit_once('.').map(|(_, ext)| ext);
148
610ab38feat: a .jsx file reads as JavaScript rather than as plain text17h
149 // After the set's own answer, never before it: an alias is what to do when no
150 // grammar claims the extension, not a way to overrule one that does.
151 let aliased = extension.and_then(|ext| {
152 ALIASES
153 .iter()
154 .find(|(from, _)| *from == ext)
155 .map(|(_, to)| *to)
156 });
157
979134bfeat: code reads as code, in classes rather than colours19h
158 let syntax = syntaxes
159 .find_syntax_by_extension(file_name)
160 .or_else(|| extension.and_then(|ext| syntaxes.find_syntax_by_extension(ext)))
610ab38feat: a .jsx file reads as JavaScript rather than as plain text17h
161 .or_else(|| aliased.and_then(|to| syntaxes.find_syntax_by_extension(to)))
979134bfeat: code reads as code, in classes rather than colours19h
162 .or_else(|| syntaxes.find_syntax_by_first_line(text.lines().next().unwrap_or("")))?;
163
164 (syntax.name != syntaxes.find_syntax_plain_text().name).then_some(syntax)
165}
166
167/// One line of HTML per line of source, or `None` if the parser gave up.
168///
169/// The span stack is carried across lines — a block comment opens on one line and
170/// closes on another — but the *markup* is not: the scopes open at the end of a line
171/// are closed there and reopened on the next, so each fragment is balanced and can sit
172/// in its own table cell.
173fn classed_lines(
174 syntaxes: &SyntaxSet,
175 syntax: &SyntaxReference,
176 text: &str,
177) -> Option<Vec<String>> {
178 let mut state = ParseState::new(syntax);
179 let mut stack = ScopeStack::new();
180 let mut lines = Vec::new();
181
182 for line in LinesWithEndings::from(text) {
183 let ops = state.parse_line(line, syntaxes).ok()?;
184
185 let mut html = String::with_capacity(line.len());
186 for scope in &stack.scopes {
187 open(&mut html, &scope.build_string());
188 }
189
190 let (body, _) = line_tokens_to_classed_spans(line, &ops, CLASS_STYLE, &mut stack).ok()?;
191 // The line ending was handed to the parser because the `_newlines` syntaxes
192 // expect it, but it must not reach the cell: `white-space: pre` would render it
193 // as a second, empty line inside every row. It can be escaped anywhere in the
194 // fragment — inside a span, or after one closes — so it is dropped by character
195 // rather than trimmed off the end.
196 html.extend(body.chars().filter(|c| *c != '\n' && *c != '\r'));
197
198 // Whatever the line left open on the stack is open in the markup too, and is
199 // closed here so the fragment stands alone in its own table cell.
200 for _ in 0..stack.scopes.len() {
201 html.push_str("</span>");
202 }
203
204 lines.push(html);
205 }
206
207 Some(lines)
208}
209
210/// Opens one span for a scope, spelling its atoms the way syntect's own writer does.
211fn open(html: &mut String, scope: &str) {
212 html.push_str("<span class=\"");
213 for (index, atom) in scope.split('.').filter(|atom| !atom.is_empty()).enumerate() {
214 if index != 0 {
215 html.push(' ');
216 }
217 html.push_str(PREFIX);
218 html.push_str(atom);
219 }
220 html.push_str("\">");
221}
222
223#[cfg(test)]
224mod tests {
225 use super::*;
226
227 fn classes(line: &SourceLine) -> String {
228 match line {
229 SourceLine::Classed(html) => html.clone(),
230 SourceLine::Plain(text) => text.clone(),
231 }
232 }
233
234 #[test]
235 fn rust_source_is_wrapped_in_prefixed_classes() {
236 let source = source_lines("main.rs", "fn main() {\n // hi\n}\n");
237
238 assert!(!source.too_large);
239 assert_eq!(source.lines.len(), 3);
240 assert!(matches!(source.lines[0], SourceLine::Classed(_)));
241
242 let first = classes(&source.lines[0]);
243 assert!(first.contains("class=\"hl-"), "{first}");
244 // `fn` is `storage.type.function.rust`, so every atom becomes its own class.
245 assert!(first.contains("hl-storage"), "{first}");
246 assert!(first.contains("hl-entity hl-name hl-function"), "{first}");
247 assert!(classes(&source.lines[1]).contains("hl-comment"));
248 }
249
250 #[test]
251 fn a_line_carries_no_line_ending_and_closes_every_span_it_opens() {
252 let source = source_lines("main.rs", "fn main() {\n let x = \"a\";\n}\n");
253
254 for line in &source.lines {
255 let html = classes(line);
256 assert!(!html.contains('\n'), "{html}");
257 assert_eq!(
258 html.matches("<span").count(),
259 html.matches("</span>").count(),
260 "{html}"
261 );
262 }
263 }
264
265 #[test]
266 fn the_source_is_escaped_rather_than_passed_through() {
267 let source = source_lines("main.rs", "// <script>alert(1)</script>\n");
268
269 let html = classes(&source.lines[0]);
270 assert!(!html.contains("<script"), "{html}");
271 assert!(html.contains("&lt;script"), "{html}");
272 }
273
274 #[test]
275 fn an_unknown_extension_stays_plain() {
276 let source = source_lines("notes.wibble", "hello\nworld\n");
277
278 assert_eq!(
279 source.lines,
280 vec![
281 SourceLine::Plain("hello".to_owned()),
282 SourceLine::Plain("world".to_owned()),
283 ]
284 );
285 }
286
287 #[test]
288 fn a_shebang_names_the_language_when_the_name_does_not() {
289 let source = source_lines("run", "#!/bin/sh\necho hi\n");
290
291 assert!(matches!(source.lines[0], SourceLine::Classed(_)));
292 }
293
294 #[test]
295 fn a_file_past_the_byte_cap_is_plain_and_says_so() {
296 let text = format!("fn main() {{}}\n{}\n", "x".repeat(MAX_BYTES));
297 let source = source_lines("main.rs", &text);
298
299 assert!(source.too_large);
300 assert!(
301 source
302 .lines
303 .iter()
304 .all(|line| matches!(line, SourceLine::Plain(_)))
305 );
306 }
307
308 #[test]
309 fn a_file_past_the_line_cap_is_plain_and_says_so() {
310 let text = "let x = 1;\n".repeat(MAX_LINES + 1);
311 let source = source_lines("main.rs", &text);
312
313 assert!(source.too_large);
314 assert_eq!(source.lines.len(), MAX_LINES + 1);
315 assert!(matches!(source.lines[0], SourceLine::Plain(_)));
316 }
317
318 #[test]
319 fn a_blank_line_keeps_its_height() {
320 let source = source_lines("main.rs", "fn a() {}\n\nfn b() {}\n");
321
322 assert_eq!(source.lines[1], SourceLine::Plain(" ".to_owned()));
323 }
324
325 #[test]
326 fn plain_text_is_not_dressed_in_spans_that_colour_nothing() {
327 let source = source_lines("notes.txt", "hello\n");
328
329 assert_eq!(source.lines, vec![SourceLine::Plain("hello".to_owned())]);
330 }
331
8331d0cfeat: the languages this repository actually uses19h
332 /// The gap `two_face` was added to close, asserted rather than described: TOML,
333 /// Dockerfile and TypeScript rendered plain until it landed, and the first two are
610ab38feat: a .jsx file reads as JavaScript rather than as plain text17h
334 /// in this repository. `.jsx` is here through [`ALIASES`] rather than through a
335 /// grammar of its own.
979134bfeat: code reads as code, in classes rather than colours19h
336 #[test]
8331d0cfeat: the languages this repository actually uses19h
337 fn the_languages_this_repository_uses_are_all_highlighted() {
979134bfeat: code reads as code, in classes rather than colours19h
338 for name in [
339 "main.rs",
8331d0cfeat: the languages this repository actually uses19h
340 "install.sh",
979134bfeat: code reads as code, in classes rather than colours19h
341 "Cargo.toml",
342 "Dockerfile",
8331d0cfeat: the languages this repository actually uses19h
343 "app.ts",
344 "App.tsx",
979134bfeat: code reads as code, in classes rather than colours19h
345 "README.md",
8331d0cfeat: the languages this repository actually uses19h
346 "styles.css",
347 "config.yaml",
348 "data.json",
349 "Makefile",
979134bfeat: code reads as code, in classes rather than colours19h
350 "app.py",
351 "index.js",
610ab38feat: a .jsx file reads as JavaScript rather than as plain text17h
352 "App.jsx",
979134bfeat: code reads as code, in classes rather than colours19h
353 "main.go",
354 "query.sql",
355 "main.c",
356 ] {
8331d0cfeat: the languages this repository actually uses19h
357 assert!(
358 syntax_for(syntaxes(), name, "").is_some(),
359 "{name} has no syntax"
360 );
979134bfeat: code reads as code, in classes rather than colours19h
361 }
362 }
8331d0cfeat: the languages this repository actually uses19h
363
364 #[test]
365 fn toml_is_highlighted_and_not_merely_recognised() {
366 let source = source_lines("Cargo.toml", "[package]\nname = \"steid\"\n");
367
368 assert!(matches!(source.lines[1], SourceLine::Classed(_)));
369 let html = classes(&source.lines[1]);
370 assert!(html.contains("hl-string"), "{html}");
371 }
979134bfeat: code reads as code, in classes rather than colours19h
372}