steid
Sign in

@jamesgill /

steid

steid/src/infrastructure/highlight.rs
12.0 KBCode·Blame·Raw
979134bfeat: code reads as code, in classes rather than colours17h
1//! Syntax highlighting for the blob view.
2//!
3//! Classes, never colours. [`syntect`] can emit inline `style` attributes, and that
4//! would hardcode one palette into the markup — the theme would then follow neither a
5//! palette change nor the colour scheme, which is exactly what `styles.css` forbids
6//! everywhere else. So the output is `hl-`-prefixed classes and the colours live in
7//! `styles.css` beside every other token.
8//!
9//! **One HTML fragment per source line**, because the blob view is a table with a line
10//! number in one cell and the code in the other. `syntect`'s own
11//! `ClassedHTMLGenerator` produces a single blob whose `<span>`s cross line boundaries,
12//! which cannot be split across table rows — so this drives the parser a line at a time
13//! and closes every open span at the end of each line, reopening it at the start of the
14//! next. The DOM shape is unchanged: what was a text node is now the same text wrapped
15//! in spans.
16//!
17//! **Highlighting never turns a viewable file into an error.** Every failure path —
18//! no syntax for the file, a regex that gives up, a file past the cap — falls back to
19//! the plain text the page rendered before.
20
21use std::sync::OnceLock;
22
23use syntect::{
24 html::{ClassStyle, line_tokens_to_classed_spans},
25 parsing::{ParseState, ScopeStack, SyntaxReference, SyntaxSet},
26 util::LinesWithEndings,
27};
28
29/// The class prefix, so a scope atom cannot collide with a Tailwind utility.
30const PREFIX: &str = "hl-";
31
32const CLASS_STYLE: ClassStyle = ClassStyle::SpacedPrefixed { prefix: PREFIX };
33
34/// Past this, a file is shown as plain text.
35///
36/// Highlighting is regex matching over every line, and the fancy-regex engine is pure
37/// Rust rather than fast C — a generated or vendored file can be megabytes of one long
38/// line, and the page should still arrive. Half a megabyte is well above anything
39/// written by hand and far below `MAX_BLOB_BYTES`.
40pub const MAX_BYTES: usize = 512 * 1024;
41
42/// The other half of the cap: many short lines cost per-line work, not per-byte work.
43pub const MAX_LINES: usize = 10_000;
44
45/// A file, ready for the blob view's table: one entry per line, in the same order as
46/// `text.lines()`.
47#[derive(Debug, Clone, PartialEq, Eq)]
48pub struct Source {
49 pub lines: Vec<SourceLine>,
50 /// Whether highlighting was skipped because the file is past the caps. The page
51 /// says so, because otherwise a large Rust file silently looks like an
52 /// unsupported language.
53 pub too_large: bool,
54}
55
56/// One line of a file.
57#[derive(Debug, Clone, PartialEq, Eq)]
58pub enum SourceLine {
59 /// Text, to be escaped by the view like anything else.
60 Plain(String),
61 /// Markup this module wrote, to be emitted verbatim.
62 Classed(String),
63}
64
65/// Splits a file into lines, highlighted where the language is known.
66///
67/// Every fallback lands on [`SourceLine::Plain`]: past the caps, no syntax for the
68/// name, a parser that gave up. Highlighting can make a file prettier; it can never
69/// stop it being readable.
70pub fn source_lines(file_name: &str, text: &str) -> Source {
71 let too_large = text.len() > MAX_BYTES || text.lines().count() > MAX_LINES;
72
73 let classed = (!too_large)
74 .then(|| {
75 let syntaxes = syntaxes();
76 let syntax = syntax_for(syntaxes, file_name, text)?;
77 classed_lines(syntaxes, syntax, text)
78 })
79 .flatten()
80 // A fragment per line is the contract the table relies on; anything else is a
81 // bug in this module, and plain text is the safe reading of it.
82 .filter(|lines| lines.len() == text.lines().count());
83
84 let lines = match classed {
85 Some(classed) => text
86 .lines()
87 .zip(classed)
88 .map(|(line, html)| match line.is_empty() {
89 // An empty cell has no height, so a blank line would close the gap it
90 // is there to make. The plain path has always spent a space on this.
91 true => SourceLine::Plain(" ".to_owned()),
92 false => SourceLine::Classed(html),
93 })
94 .collect(),
95 None => text
96 .lines()
97 .map(|line| match line.is_empty() {
98 true => SourceLine::Plain(" ".to_owned()),
99 false => SourceLine::Plain(line.to_owned()),
100 })
101 .collect(),
102 };
103
104 Source { lines, too_large }
105}
106
107/// The default syntax set, loaded once.
108///
109/// Loading is a few megabytes of deserialization, so it happens on the first blob
110/// viewed and never again. `_newlines` is the variant whose rules expect a trailing
111/// newline on each line, which is what [`LinesWithEndings`] hands it.
112fn syntaxes() -> &'static SyntaxSet {
113 static SYNTAXES: OnceLock<SyntaxSet> = OnceLock::new();
114 SYNTAXES.get_or_init(SyntaxSet::load_defaults_newlines)
115}
116
117/// The language, from the file name and then the first line.
118///
119/// This is `SyntaxSet::find_syntax_for_file` without the filesystem: that method opens
120/// the path to read a shebang, and here the contents are already in memory. The whole
121/// name is tried as an extension first because that is how syntect registers names like
122/// `Makefile`; plain text is treated as no syntax at all, since wrapping every line in
123/// a span that carries no colour is pure markup for nothing.
124fn syntax_for<'a>(
125 syntaxes: &'a SyntaxSet,
126 file_name: &str,
127 text: &str,
128) -> Option<&'a SyntaxReference> {
129 let extension = file_name.rsplit_once('.').map(|(_, ext)| ext);
130
131 let syntax = syntaxes
132 .find_syntax_by_extension(file_name)
133 .or_else(|| extension.and_then(|ext| syntaxes.find_syntax_by_extension(ext)))
134 .or_else(|| syntaxes.find_syntax_by_first_line(text.lines().next().unwrap_or("")))?;
135
136 (syntax.name != syntaxes.find_syntax_plain_text().name).then_some(syntax)
137}
138
139/// One line of HTML per line of source, or `None` if the parser gave up.
140///
141/// The span stack is carried across lines — a block comment opens on one line and
142/// closes on another — but the *markup* is not: the scopes open at the end of a line
143/// are closed there and reopened on the next, so each fragment is balanced and can sit
144/// in its own table cell.
145fn classed_lines(
146 syntaxes: &SyntaxSet,
147 syntax: &SyntaxReference,
148 text: &str,
149) -> Option<Vec<String>> {
150 let mut state = ParseState::new(syntax);
151 let mut stack = ScopeStack::new();
152 let mut lines = Vec::new();
153
154 for line in LinesWithEndings::from(text) {
155 let ops = state.parse_line(line, syntaxes).ok()?;
156
157 let mut html = String::with_capacity(line.len());
158 for scope in &stack.scopes {
159 open(&mut html, &scope.build_string());
160 }
161
162 let (body, _) = line_tokens_to_classed_spans(line, &ops, CLASS_STYLE, &mut stack).ok()?;
163 // The line ending was handed to the parser because the `_newlines` syntaxes
164 // expect it, but it must not reach the cell: `white-space: pre` would render it
165 // as a second, empty line inside every row. It can be escaped anywhere in the
166 // fragment — inside a span, or after one closes — so it is dropped by character
167 // rather than trimmed off the end.
168 html.extend(body.chars().filter(|c| *c != '\n' && *c != '\r'));
169
170 // Whatever the line left open on the stack is open in the markup too, and is
171 // closed here so the fragment stands alone in its own table cell.
172 for _ in 0..stack.scopes.len() {
173 html.push_str("</span>");
174 }
175
176 lines.push(html);
177 }
178
179 Some(lines)
180}
181
182/// Opens one span for a scope, spelling its atoms the way syntect's own writer does.
183fn open(html: &mut String, scope: &str) {
184 html.push_str("<span class=\"");
185 for (index, atom) in scope.split('.').filter(|atom| !atom.is_empty()).enumerate() {
186 if index != 0 {
187 html.push(' ');
188 }
189 html.push_str(PREFIX);
190 html.push_str(atom);
191 }
192 html.push_str("\">");
193}
194
195#[cfg(test)]
196mod tests {
197 use super::*;
198
199 fn classes(line: &SourceLine) -> String {
200 match line {
201 SourceLine::Classed(html) => html.clone(),
202 SourceLine::Plain(text) => text.clone(),
203 }
204 }
205
206 #[test]
207 fn rust_source_is_wrapped_in_prefixed_classes() {
208 let source = source_lines("main.rs", "fn main() {\n // hi\n}\n");
209
210 assert!(!source.too_large);
211 assert_eq!(source.lines.len(), 3);
212 assert!(matches!(source.lines[0], SourceLine::Classed(_)));
213
214 let first = classes(&source.lines[0]);
215 assert!(first.contains("class=\"hl-"), "{first}");
216 // `fn` is `storage.type.function.rust`, so every atom becomes its own class.
217 assert!(first.contains("hl-storage"), "{first}");
218 assert!(first.contains("hl-entity hl-name hl-function"), "{first}");
219 assert!(classes(&source.lines[1]).contains("hl-comment"));
220 }
221
222 #[test]
223 fn a_line_carries_no_line_ending_and_closes_every_span_it_opens() {
224 let source = source_lines("main.rs", "fn main() {\n let x = \"a\";\n}\n");
225
226 for line in &source.lines {
227 let html = classes(line);
228 assert!(!html.contains('\n'), "{html}");
229 assert_eq!(
230 html.matches("<span").count(),
231 html.matches("</span>").count(),
232 "{html}"
233 );
234 }
235 }
236
237 #[test]
238 fn the_source_is_escaped_rather_than_passed_through() {
239 let source = source_lines("main.rs", "// <script>alert(1)</script>\n");
240
241 let html = classes(&source.lines[0]);
242 assert!(!html.contains("<script"), "{html}");
243 assert!(html.contains("&lt;script"), "{html}");
244 }
245
246 #[test]
247 fn an_unknown_extension_stays_plain() {
248 let source = source_lines("notes.wibble", "hello\nworld\n");
249
250 assert_eq!(
251 source.lines,
252 vec![
253 SourceLine::Plain("hello".to_owned()),
254 SourceLine::Plain("world".to_owned()),
255 ]
256 );
257 }
258
259 #[test]
260 fn a_shebang_names_the_language_when_the_name_does_not() {
261 let source = source_lines("run", "#!/bin/sh\necho hi\n");
262
263 assert!(matches!(source.lines[0], SourceLine::Classed(_)));
264 }
265
266 #[test]
267 fn a_file_past_the_byte_cap_is_plain_and_says_so() {
268 let text = format!("fn main() {{}}\n{}\n", "x".repeat(MAX_BYTES));
269 let source = source_lines("main.rs", &text);
270
271 assert!(source.too_large);
272 assert!(
273 source
274 .lines
275 .iter()
276 .all(|line| matches!(line, SourceLine::Plain(_)))
277 );
278 }
279
280 #[test]
281 fn a_file_past_the_line_cap_is_plain_and_says_so() {
282 let text = "let x = 1;\n".repeat(MAX_LINES + 1);
283 let source = source_lines("main.rs", &text);
284
285 assert!(source.too_large);
286 assert_eq!(source.lines.len(), MAX_LINES + 1);
287 assert!(matches!(source.lines[0], SourceLine::Plain(_)));
288 }
289
290 #[test]
291 fn a_blank_line_keeps_its_height() {
292 let source = source_lines("main.rs", "fn a() {}\n\nfn b() {}\n");
293
294 assert_eq!(source.lines[1], SourceLine::Plain(" ".to_owned()));
295 }
296
297 #[test]
298 fn plain_text_is_not_dressed_in_spans_that_colour_nothing() {
299 let source = source_lines("notes.txt", "hello\n");
300
301 assert_eq!(source.lines, vec![SourceLine::Plain("hello".to_owned())]);
302 }
303
304 /// The default syntax set is Sublime's, not a forge's, so the languages it lacks
305 /// are worth knowing rather than discovering on a page.
306 #[test]
307 fn which_common_languages_the_default_set_knows() {
308 for name in [
309 "main.rs",
310 "Cargo.toml",
311 "Dockerfile",
312 "install.sh",
313 "README.md",
314 "app.py",
315 "index.js",
316 "app.ts",
317 "main.go",
318 "config.yaml",
319 "data.json",
320 "styles.css",
321 "query.sql",
322 "Makefile",
323 "main.c",
324 "App.jsx",
325 ] {
326 let known = syntax_for(syntaxes(), name, "").is_some();
327 println!("{name}: {}", if known { "highlighted" } else { "PLAIN" });
328 }
329 }
330}