steid
Sign in

@jamesgill /

steid

steid/src/infrastructure/highlight.rs
12.9 KBCode·Blame·Raw
979134bfeat: code reads as code, in classes rather than colours19h
1//! Syntax highlighting for the blob view.
2//!
3//! Classes, never colours. [`syntect`] can emit inline `style` attributes, and that
4//! would hardcode one palette into the markup — the theme would then follow neither a
5//! palette change nor the colour scheme, which is exactly what `styles.css` forbids
6//! everywhere else. So the output is `hl-`-prefixed classes and the colours live in
7//! `styles.css` beside every other token.
8//!
9//! **One HTML fragment per source line**, because the blob view is a table with a line
10//! number in one cell and the code in the other. `syntect`'s own
11//! `ClassedHTMLGenerator` produces a single blob whose `<span>`s cross line boundaries,
12//! which cannot be split across table rows — so this drives the parser a line at a time
13//! and closes every open span at the end of each line, reopening it at the start of the
14//! next. The DOM shape is unchanged: what was a text node is now the same text wrapped
15//! in spans.
16//!
17//! **Highlighting never turns a viewable file into an error.** Every failure path —
18//! no syntax for the file, a regex that gives up, a file past the cap — falls back to
19//! the plain text the page rendered before.
20
21use std::sync::OnceLock;
22
23use syntect::{
24 html::{ClassStyle, line_tokens_to_classed_spans},
25 parsing::{ParseState, ScopeStack, SyntaxReference, SyntaxSet},
26 util::LinesWithEndings,
27};
28
29/// The class prefix, so a scope atom cannot collide with a Tailwind utility.
30const PREFIX: &str = "hl-";
31
32const CLASS_STYLE: ClassStyle = ClassStyle::SpacedPrefixed { prefix: PREFIX };
33
34/// Past this, a file is shown as plain text.
35///
36/// Highlighting is regex matching over every line, and the fancy-regex engine is pure
37/// Rust rather than fast C — a generated or vendored file can be megabytes of one long
38/// line, and the page should still arrive. Half a megabyte is well above anything
39/// written by hand and far below `MAX_BLOB_BYTES`.
40pub const MAX_BYTES: usize = 512 * 1024;
41
42/// The other half of the cap: many short lines cost per-line work, not per-byte work.
43pub const MAX_LINES: usize = 10_000;
44
45/// A file, ready for the blob view's table: one entry per line, in the same order as
46/// `text.lines()`.
47#[derive(Debug, Clone, PartialEq, Eq)]
48pub struct Source {
49 pub lines: Vec<SourceLine>,
50 /// Whether highlighting was skipped because the file is past the caps. The page
51 /// says so, because otherwise a large Rust file silently looks like an
52 /// unsupported language.
53 pub too_large: bool,
54}
55
56/// One line of a file.
57#[derive(Debug, Clone, PartialEq, Eq)]
58pub enum SourceLine {
59 /// Text, to be escaped by the view like anything else.
60 Plain(String),
61 /// Markup this module wrote, to be emitted verbatim.
62 Classed(String),
63}
64
65/// Splits a file into lines, highlighted where the language is known.
66///
67/// Every fallback lands on [`SourceLine::Plain`]: past the caps, no syntax for the
68/// name, a parser that gave up. Highlighting can make a file prettier; it can never
69/// stop it being readable.
70pub fn source_lines(file_name: &str, text: &str) -> Source {
71 let too_large = text.len() > MAX_BYTES || text.lines().count() > MAX_LINES;
72
73 let classed = (!too_large)
74 .then(|| {
75 let syntaxes = syntaxes();
76 let syntax = syntax_for(syntaxes, file_name, text)?;
77 classed_lines(syntaxes, syntax, text)
78 })
79 .flatten()
80 // A fragment per line is the contract the table relies on; anything else is a
81 // bug in this module, and plain text is the safe reading of it.
82 .filter(|lines| lines.len() == text.lines().count());
83
84 let lines = match classed {
85 Some(classed) => text
86 .lines()
87 .zip(classed)
88 .map(|(line, html)| match line.is_empty() {
89 // An empty cell has no height, so a blank line would close the gap it
90 // is there to make. The plain path has always spent a space on this.
91 true => SourceLine::Plain(" ".to_owned()),
92 false => SourceLine::Classed(html),
93 })
94 .collect(),
95 None => text
96 .lines()
97 .map(|line| match line.is_empty() {
98 true => SourceLine::Plain(" ".to_owned()),
99 false => SourceLine::Plain(line.to_owned()),
100 })
101 .collect(),
102 };
103
104 Source { lines, too_large }
105}
106
8331d0cfeat: the languages this repository actually uses19h
107/// The syntax set, loaded once.
108///
109/// `two_face` rather than `SyntaxSet::load_defaults_newlines`: syntect's own defaults
110/// are Sublime's, which have no TOML, no Dockerfile and no TypeScript — the first two
111/// of which are in this repository. `two_face` is the same set plus the grammars `bat`
112/// curates, so the gap closes without teaching this module anything about individual
113/// languages. Its `syntect-fancy` feature pins it to the same pure-Rust regex engine,
114/// so no `onig` — and therefore no C — comes in behind it.
979134bfeat: code reads as code, in classes rather than colours19h
115///
116/// Loading is a few megabytes of deserialization, so it happens on the first blob
117/// viewed and never again. `_newlines` is the variant whose rules expect a trailing
118/// newline on each line, which is what [`LinesWithEndings`] hands it.
119fn syntaxes() -> &'static SyntaxSet {
120 static SYNTAXES: OnceLock<SyntaxSet> = OnceLock::new();
8331d0cfeat: the languages this repository actually uses19h
121 SYNTAXES.get_or_init(two_face::syntax::extra_newlines)
979134bfeat: code reads as code, in classes rather than colours19h
122}
123
124/// The language, from the file name and then the first line.
125///
126/// This is `SyntaxSet::find_syntax_for_file` without the filesystem: that method opens
127/// the path to read a shebang, and here the contents are already in memory. The whole
128/// name is tried as an extension first because that is how syntect registers names like
129/// `Makefile`; plain text is treated as no syntax at all, since wrapping every line in
130/// a span that carries no colour is pure markup for nothing.
131fn syntax_for<'a>(
132 syntaxes: &'a SyntaxSet,
133 file_name: &str,
134 text: &str,
135) -> Option<&'a SyntaxReference> {
136 let extension = file_name.rsplit_once('.').map(|(_, ext)| ext);
137
138 let syntax = syntaxes
139 .find_syntax_by_extension(file_name)
140 .or_else(|| extension.and_then(|ext| syntaxes.find_syntax_by_extension(ext)))
141 .or_else(|| syntaxes.find_syntax_by_first_line(text.lines().next().unwrap_or("")))?;
142
143 (syntax.name != syntaxes.find_syntax_plain_text().name).then_some(syntax)
144}
145
146/// One line of HTML per line of source, or `None` if the parser gave up.
147///
148/// The span stack is carried across lines — a block comment opens on one line and
149/// closes on another — but the *markup* is not: the scopes open at the end of a line
150/// are closed there and reopened on the next, so each fragment is balanced and can sit
151/// in its own table cell.
152fn classed_lines(
153 syntaxes: &SyntaxSet,
154 syntax: &SyntaxReference,
155 text: &str,
156) -> Option<Vec<String>> {
157 let mut state = ParseState::new(syntax);
158 let mut stack = ScopeStack::new();
159 let mut lines = Vec::new();
160
161 for line in LinesWithEndings::from(text) {
162 let ops = state.parse_line(line, syntaxes).ok()?;
163
164 let mut html = String::with_capacity(line.len());
165 for scope in &stack.scopes {
166 open(&mut html, &scope.build_string());
167 }
168
169 let (body, _) = line_tokens_to_classed_spans(line, &ops, CLASS_STYLE, &mut stack).ok()?;
170 // The line ending was handed to the parser because the `_newlines` syntaxes
171 // expect it, but it must not reach the cell: `white-space: pre` would render it
172 // as a second, empty line inside every row. It can be escaped anywhere in the
173 // fragment — inside a span, or after one closes — so it is dropped by character
174 // rather than trimmed off the end.
175 html.extend(body.chars().filter(|c| *c != '\n' && *c != '\r'));
176
177 // Whatever the line left open on the stack is open in the markup too, and is
178 // closed here so the fragment stands alone in its own table cell.
179 for _ in 0..stack.scopes.len() {
180 html.push_str("</span>");
181 }
182
183 lines.push(html);
184 }
185
186 Some(lines)
187}
188
189/// Opens one span for a scope, spelling its atoms the way syntect's own writer does.
190fn open(html: &mut String, scope: &str) {
191 html.push_str("<span class=\"");
192 for (index, atom) in scope.split('.').filter(|atom| !atom.is_empty()).enumerate() {
193 if index != 0 {
194 html.push(' ');
195 }
196 html.push_str(PREFIX);
197 html.push_str(atom);
198 }
199 html.push_str("\">");
200}
201
202#[cfg(test)]
203mod tests {
204 use super::*;
205
206 fn classes(line: &SourceLine) -> String {
207 match line {
208 SourceLine::Classed(html) => html.clone(),
209 SourceLine::Plain(text) => text.clone(),
210 }
211 }
212
213 #[test]
214 fn rust_source_is_wrapped_in_prefixed_classes() {
215 let source = source_lines("main.rs", "fn main() {\n // hi\n}\n");
216
217 assert!(!source.too_large);
218 assert_eq!(source.lines.len(), 3);
219 assert!(matches!(source.lines[0], SourceLine::Classed(_)));
220
221 let first = classes(&source.lines[0]);
222 assert!(first.contains("class=\"hl-"), "{first}");
223 // `fn` is `storage.type.function.rust`, so every atom becomes its own class.
224 assert!(first.contains("hl-storage"), "{first}");
225 assert!(first.contains("hl-entity hl-name hl-function"), "{first}");
226 assert!(classes(&source.lines[1]).contains("hl-comment"));
227 }
228
229 #[test]
230 fn a_line_carries_no_line_ending_and_closes_every_span_it_opens() {
231 let source = source_lines("main.rs", "fn main() {\n let x = \"a\";\n}\n");
232
233 for line in &source.lines {
234 let html = classes(line);
235 assert!(!html.contains('\n'), "{html}");
236 assert_eq!(
237 html.matches("<span").count(),
238 html.matches("</span>").count(),
239 "{html}"
240 );
241 }
242 }
243
244 #[test]
245 fn the_source_is_escaped_rather_than_passed_through() {
246 let source = source_lines("main.rs", "// <script>alert(1)</script>\n");
247
248 let html = classes(&source.lines[0]);
249 assert!(!html.contains("<script"), "{html}");
250 assert!(html.contains("&lt;script"), "{html}");
251 }
252
253 #[test]
254 fn an_unknown_extension_stays_plain() {
255 let source = source_lines("notes.wibble", "hello\nworld\n");
256
257 assert_eq!(
258 source.lines,
259 vec![
260 SourceLine::Plain("hello".to_owned()),
261 SourceLine::Plain("world".to_owned()),
262 ]
263 );
264 }
265
266 #[test]
267 fn a_shebang_names_the_language_when_the_name_does_not() {
268 let source = source_lines("run", "#!/bin/sh\necho hi\n");
269
270 assert!(matches!(source.lines[0], SourceLine::Classed(_)));
271 }
272
273 #[test]
274 fn a_file_past_the_byte_cap_is_plain_and_says_so() {
275 let text = format!("fn main() {{}}\n{}\n", "x".repeat(MAX_BYTES));
276 let source = source_lines("main.rs", &text);
277
278 assert!(source.too_large);
279 assert!(
280 source
281 .lines
282 .iter()
283 .all(|line| matches!(line, SourceLine::Plain(_)))
284 );
285 }
286
287 #[test]
288 fn a_file_past_the_line_cap_is_plain_and_says_so() {
289 let text = "let x = 1;\n".repeat(MAX_LINES + 1);
290 let source = source_lines("main.rs", &text);
291
292 assert!(source.too_large);
293 assert_eq!(source.lines.len(), MAX_LINES + 1);
294 assert!(matches!(source.lines[0], SourceLine::Plain(_)));
295 }
296
297 #[test]
298 fn a_blank_line_keeps_its_height() {
299 let source = source_lines("main.rs", "fn a() {}\n\nfn b() {}\n");
300
301 assert_eq!(source.lines[1], SourceLine::Plain(" ".to_owned()));
302 }
303
304 #[test]
305 fn plain_text_is_not_dressed_in_spans_that_colour_nothing() {
306 let source = source_lines("notes.txt", "hello\n");
307
308 assert_eq!(source.lines, vec![SourceLine::Plain("hello".to_owned())]);
309 }
310
8331d0cfeat: the languages this repository actually uses19h
311 /// The gap `two_face` was added to close, asserted rather than described: TOML,
312 /// Dockerfile and TypeScript rendered plain until it landed, and the first two are
313 /// in this repository. `.jsx` is still absent — see the handover.
979134bfeat: code reads as code, in classes rather than colours19h
314 #[test]
8331d0cfeat: the languages this repository actually uses19h
315 fn the_languages_this_repository_uses_are_all_highlighted() {
979134bfeat: code reads as code, in classes rather than colours19h
316 for name in [
317 "main.rs",
8331d0cfeat: the languages this repository actually uses19h
318 "install.sh",
979134bfeat: code reads as code, in classes rather than colours19h
319 "Cargo.toml",
320 "Dockerfile",
8331d0cfeat: the languages this repository actually uses19h
321 "app.ts",
322 "App.tsx",
979134bfeat: code reads as code, in classes rather than colours19h
323 "README.md",
8331d0cfeat: the languages this repository actually uses19h
324 "styles.css",
325 "config.yaml",
326 "data.json",
327 "Makefile",
979134bfeat: code reads as code, in classes rather than colours19h
328 "app.py",
329 "index.js",
330 "main.go",
331 "query.sql",
332 "main.c",
333 ] {
8331d0cfeat: the languages this repository actually uses19h
334 assert!(
335 syntax_for(syntaxes(), name, "").is_some(),
336 "{name} has no syntax"
337 );
979134bfeat: code reads as code, in classes rather than colours19h
338 }
339 }
8331d0cfeat: the languages this repository actually uses19h
340
341 #[test]
342 fn toml_is_highlighted_and_not_merely_recognised() {
343 let source = source_lines("Cargo.toml", "[package]\nname = \"steid\"\n");
344
345 assert!(matches!(source.lines[1], SourceLine::Classed(_)));
346 let html = classes(&source.lines[1]);
347 assert!(html.contains("hl-string"), "{html}");
348 }
979134bfeat: code reads as code, in classes rather than colours19h
349}