steid
Sign in

@jamesgill /

steid

steid/src/infrastructure/highlight.rs
12.9 KBCode·Blame·Raw
1//! Syntax highlighting for the blob view.
2//!
3//! Classes, never colours. [`syntect`] can emit inline `style` attributes, and that
4//! would hardcode one palette into the markup — the theme would then follow neither a
5//! palette change nor the colour scheme, which is exactly what `styles.css` forbids
6//! everywhere else. So the output is `hl-`-prefixed classes and the colours live in
7//! `styles.css` beside every other token.
8//!
9//! **One HTML fragment per source line**, because the blob view is a table with a line
10//! number in one cell and the code in the other. `syntect`'s own
11//! `ClassedHTMLGenerator` produces a single blob whose `<span>`s cross line boundaries,
12//! which cannot be split across table rows — so this drives the parser a line at a time
13//! and closes every open span at the end of each line, reopening it at the start of the
14//! next. The DOM shape is unchanged: what was a text node is now the same text wrapped
15//! in spans.
16//!
17//! **Highlighting never turns a viewable file into an error.** Every failure path —
18//! no syntax for the file, a regex that gives up, a file past the cap — falls back to
19//! the plain text the page rendered before.
20
21use std::sync::OnceLock;
22
23use syntect::{
24 html::{ClassStyle, line_tokens_to_classed_spans},
25 parsing::{ParseState, ScopeStack, SyntaxReference, SyntaxSet},
26 util::LinesWithEndings,
27};
28
29/// The class prefix, so a scope atom cannot collide with a Tailwind utility.
30const PREFIX: &str = "hl-";
31
32const CLASS_STYLE: ClassStyle = ClassStyle::SpacedPrefixed { prefix: PREFIX };
33
34/// Past this, a file is shown as plain text.
35///
36/// Highlighting is regex matching over every line, and the fancy-regex engine is pure
37/// Rust rather than fast C — a generated or vendored file can be megabytes of one long
38/// line, and the page should still arrive. Half a megabyte is well above anything
39/// written by hand and far below `MAX_BLOB_BYTES`.
40pub const MAX_BYTES: usize = 512 * 1024;
41
42/// The other half of the cap: many short lines cost per-line work, not per-byte work.
43pub const MAX_LINES: usize = 10_000;
44
45/// A file, ready for the blob view's table: one entry per line, in the same order as
46/// `text.lines()`.
47#[derive(Debug, Clone, PartialEq, Eq)]
48pub struct Source {
49 pub lines: Vec<SourceLine>,
50 /// Whether highlighting was skipped because the file is past the caps. The page
51 /// says so, because otherwise a large Rust file silently looks like an
52 /// unsupported language.
53 pub too_large: bool,
54}
55
56/// One line of a file.
57#[derive(Debug, Clone, PartialEq, Eq)]
58pub enum SourceLine {
59 /// Text, to be escaped by the view like anything else.
60 Plain(String),
61 /// Markup this module wrote, to be emitted verbatim.
62 Classed(String),
63}
64
65/// Splits a file into lines, highlighted where the language is known.
66///
67/// Every fallback lands on [`SourceLine::Plain`]: past the caps, no syntax for the
68/// name, a parser that gave up. Highlighting can make a file prettier; it can never
69/// stop it being readable.
70pub fn source_lines(file_name: &str, text: &str) -> Source {
71 let too_large = text.len() > MAX_BYTES || text.lines().count() > MAX_LINES;
72
73 let classed = (!too_large)
74 .then(|| {
75 let syntaxes = syntaxes();
76 let syntax = syntax_for(syntaxes, file_name, text)?;
77 classed_lines(syntaxes, syntax, text)
78 })
79 .flatten()
80 // A fragment per line is the contract the table relies on; anything else is a
81 // bug in this module, and plain text is the safe reading of it.
82 .filter(|lines| lines.len() == text.lines().count());
83
84 let lines = match classed {
85 Some(classed) => text
86 .lines()
87 .zip(classed)
88 .map(|(line, html)| match line.is_empty() {
89 // An empty cell has no height, so a blank line would close the gap it
90 // is there to make. The plain path has always spent a space on this.
91 true => SourceLine::Plain(" ".to_owned()),
92 false => SourceLine::Classed(html),
93 })
94 .collect(),
95 None => text
96 .lines()
97 .map(|line| match line.is_empty() {
98 true => SourceLine::Plain(" ".to_owned()),
99 false => SourceLine::Plain(line.to_owned()),
100 })
101 .collect(),
102 };
103
104 Source { lines, too_large }
105}
106
107/// The syntax set, loaded once.
108///
109/// `two_face` rather than `SyntaxSet::load_defaults_newlines`: syntect's own defaults
110/// are Sublime's, which have no TOML, no Dockerfile and no TypeScript — the first two
111/// of which are in this repository. `two_face` is the same set plus the grammars `bat`
112/// curates, so the gap closes without teaching this module anything about individual
113/// languages. Its `syntect-fancy` feature pins it to the same pure-Rust regex engine,
114/// so no `onig` — and therefore no C — comes in behind it.
115///
116/// Loading is a few megabytes of deserialization, so it happens on the first blob
117/// viewed and never again. `_newlines` is the variant whose rules expect a trailing
118/// newline on each line, which is what [`LinesWithEndings`] hands it.
119fn syntaxes() -> &'static SyntaxSet {
120 static SYNTAXES: OnceLock<SyntaxSet> = OnceLock::new();
121 SYNTAXES.get_or_init(two_face::syntax::extra_newlines)
122}
123
124/// The language, from the file name and then the first line.
125///
126/// This is `SyntaxSet::find_syntax_for_file` without the filesystem: that method opens
127/// the path to read a shebang, and here the contents are already in memory. The whole
128/// name is tried as an extension first because that is how syntect registers names like
129/// `Makefile`; plain text is treated as no syntax at all, since wrapping every line in
130/// a span that carries no colour is pure markup for nothing.
131fn syntax_for<'a>(
132 syntaxes: &'a SyntaxSet,
133 file_name: &str,
134 text: &str,
135) -> Option<&'a SyntaxReference> {
136 let extension = file_name.rsplit_once('.').map(|(_, ext)| ext);
137
138 let syntax = syntaxes
139 .find_syntax_by_extension(file_name)
140 .or_else(|| extension.and_then(|ext| syntaxes.find_syntax_by_extension(ext)))
141 .or_else(|| syntaxes.find_syntax_by_first_line(text.lines().next().unwrap_or("")))?;
142
143 (syntax.name != syntaxes.find_syntax_plain_text().name).then_some(syntax)
144}
145
146/// One line of HTML per line of source, or `None` if the parser gave up.
147///
148/// The span stack is carried across lines — a block comment opens on one line and
149/// closes on another — but the *markup* is not: the scopes open at the end of a line
150/// are closed there and reopened on the next, so each fragment is balanced and can sit
151/// in its own table cell.
152fn classed_lines(
153 syntaxes: &SyntaxSet,
154 syntax: &SyntaxReference,
155 text: &str,
156) -> Option<Vec<String>> {
157 let mut state = ParseState::new(syntax);
158 let mut stack = ScopeStack::new();
159 let mut lines = Vec::new();
160
161 for line in LinesWithEndings::from(text) {
162 let ops = state.parse_line(line, syntaxes).ok()?;
163
164 let mut html = String::with_capacity(line.len());
165 for scope in &stack.scopes {
166 open(&mut html, &scope.build_string());
167 }
168
169 let (body, _) = line_tokens_to_classed_spans(line, &ops, CLASS_STYLE, &mut stack).ok()?;
170 // The line ending was handed to the parser because the `_newlines` syntaxes
171 // expect it, but it must not reach the cell: `white-space: pre` would render it
172 // as a second, empty line inside every row. It can be escaped anywhere in the
173 // fragment — inside a span, or after one closes — so it is dropped by character
174 // rather than trimmed off the end.
175 html.extend(body.chars().filter(|c| *c != '\n' && *c != '\r'));
176
177 // Whatever the line left open on the stack is open in the markup too, and is
178 // closed here so the fragment stands alone in its own table cell.
179 for _ in 0..stack.scopes.len() {
180 html.push_str("</span>");
181 }
182
183 lines.push(html);
184 }
185
186 Some(lines)
187}
188
189/// Opens one span for a scope, spelling its atoms the way syntect's own writer does.
190fn open(html: &mut String, scope: &str) {
191 html.push_str("<span class=\"");
192 for (index, atom) in scope.split('.').filter(|atom| !atom.is_empty()).enumerate() {
193 if index != 0 {
194 html.push(' ');
195 }
196 html.push_str(PREFIX);
197 html.push_str(atom);
198 }
199 html.push_str("\">");
200}
201
202#[cfg(test)]
203mod tests {
204 use super::*;
205
206 fn classes(line: &SourceLine) -> String {
207 match line {
208 SourceLine::Classed(html) => html.clone(),
209 SourceLine::Plain(text) => text.clone(),
210 }
211 }
212
213 #[test]
214 fn rust_source_is_wrapped_in_prefixed_classes() {
215 let source = source_lines("main.rs", "fn main() {\n // hi\n}\n");
216
217 assert!(!source.too_large);
218 assert_eq!(source.lines.len(), 3);
219 assert!(matches!(source.lines[0], SourceLine::Classed(_)));
220
221 let first = classes(&source.lines[0]);
222 assert!(first.contains("class=\"hl-"), "{first}");
223 // `fn` is `storage.type.function.rust`, so every atom becomes its own class.
224 assert!(first.contains("hl-storage"), "{first}");
225 assert!(first.contains("hl-entity hl-name hl-function"), "{first}");
226 assert!(classes(&source.lines[1]).contains("hl-comment"));
227 }
228
229 #[test]
230 fn a_line_carries_no_line_ending_and_closes_every_span_it_opens() {
231 let source = source_lines("main.rs", "fn main() {\n let x = \"a\";\n}\n");
232
233 for line in &source.lines {
234 let html = classes(line);
235 assert!(!html.contains('\n'), "{html}");
236 assert_eq!(
237 html.matches("<span").count(),
238 html.matches("</span>").count(),
239 "{html}"
240 );
241 }
242 }
243
244 #[test]
245 fn the_source_is_escaped_rather_than_passed_through() {
246 let source = source_lines("main.rs", "// <script>alert(1)</script>\n");
247
248 let html = classes(&source.lines[0]);
249 assert!(!html.contains("<script"), "{html}");
250 assert!(html.contains("&lt;script"), "{html}");
251 }
252
253 #[test]
254 fn an_unknown_extension_stays_plain() {
255 let source = source_lines("notes.wibble", "hello\nworld\n");
256
257 assert_eq!(
258 source.lines,
259 vec![
260 SourceLine::Plain("hello".to_owned()),
261 SourceLine::Plain("world".to_owned()),
262 ]
263 );
264 }
265
266 #[test]
267 fn a_shebang_names_the_language_when_the_name_does_not() {
268 let source = source_lines("run", "#!/bin/sh\necho hi\n");
269
270 assert!(matches!(source.lines[0], SourceLine::Classed(_)));
271 }
272
273 #[test]
274 fn a_file_past_the_byte_cap_is_plain_and_says_so() {
275 let text = format!("fn main() {{}}\n{}\n", "x".repeat(MAX_BYTES));
276 let source = source_lines("main.rs", &text);
277
278 assert!(source.too_large);
279 assert!(
280 source
281 .lines
282 .iter()
283 .all(|line| matches!(line, SourceLine::Plain(_)))
284 );
285 }
286
287 #[test]
288 fn a_file_past_the_line_cap_is_plain_and_says_so() {
289 let text = "let x = 1;\n".repeat(MAX_LINES + 1);
290 let source = source_lines("main.rs", &text);
291
292 assert!(source.too_large);
293 assert_eq!(source.lines.len(), MAX_LINES + 1);
294 assert!(matches!(source.lines[0], SourceLine::Plain(_)));
295 }
296
297 #[test]
298 fn a_blank_line_keeps_its_height() {
299 let source = source_lines("main.rs", "fn a() {}\n\nfn b() {}\n");
300
301 assert_eq!(source.lines[1], SourceLine::Plain(" ".to_owned()));
302 }
303
304 #[test]
305 fn plain_text_is_not_dressed_in_spans_that_colour_nothing() {
306 let source = source_lines("notes.txt", "hello\n");
307
308 assert_eq!(source.lines, vec![SourceLine::Plain("hello".to_owned())]);
309 }
310
311 /// The gap `two_face` was added to close, asserted rather than described: TOML,
312 /// Dockerfile and TypeScript rendered plain until it landed, and the first two are
313 /// in this repository. `.jsx` is still absent — see the handover.
314 #[test]
315 fn the_languages_this_repository_uses_are_all_highlighted() {
316 for name in [
317 "main.rs",
318 "install.sh",
319 "Cargo.toml",
320 "Dockerfile",
321 "app.ts",
322 "App.tsx",
323 "README.md",
324 "styles.css",
325 "config.yaml",
326 "data.json",
327 "Makefile",
328 "app.py",
329 "index.js",
330 "main.go",
331 "query.sql",
332 "main.c",
333 ] {
334 assert!(
335 syntax_for(syntaxes(), name, "").is_some(),
336 "{name} has no syntax"
337 );
338 }
339 }
340
341 #[test]
342 fn toml_is_highlighted_and_not_merely_recognised() {
343 let source = source_lines("Cargo.toml", "[package]\nname = \"steid\"\n");
344
345 assert!(matches!(source.lines[1], SourceLine::Classed(_)));
346 let html = classes(&source.lines[1]);
347 assert!(html.contains("hl-string"), "{html}");
348 }
349}