lumen_script/
text_parse.rs

1//! Text-to-value parsers behind the `parse_json` and `parse_markdown`
2//! builtins in [`crate::builtin_fns::builtin_script_fns`].
3//!
4//! Both hand back a [`ScriptValue`] rather than a string, so a script indexes
5//! the result directly instead of parsing text itself; each host converts the
6//! result into its own value type the same way it converts any other
7//! builtin's return. `parse_json` and `parse_markdown` are each behind their
8//! own Cargo feature (`json`, `markdown`) so a build that links no script
9//! host carries neither `serde_json` nor `pulldown-cmark`.
10
11#[cfg(any(feature = "json", feature = "markdown"))]
12use crate::ScriptValue;
13
14/// Parse `src` as JSON into a [`ScriptValue`]. An object becomes a
15/// [`ScriptValue::Map`], an array a [`ScriptValue::Array`], and a scalar
16/// keeps its JSON type. Malformed input yields [`ScriptValue::Unit`].
17#[cfg(feature = "json")]
18#[must_use]
19pub fn parse_json(src: &str) -> ScriptValue {
20    match serde_json::from_str::<serde_json::Value>(src) {
21        Ok(v) => json_value(&v),
22        Err(_) => ScriptValue::Unit,
23    }
24}
25
26/// Write `value` as JSON text: the reverse of [`parse_json`].
27///
28/// A map becomes an object with its keys in sorted order, so the same value
29/// always writes the same text; an array an array; a scalar keeps its type;
30/// [`ScriptValue::Unit`] is `null`. JSON has no spelling for a float that is
31/// not finite, so one is written as `null` too. An integer past 2^53 is
32/// written exactly, and a JavaScript reader rounds it.
33#[cfg(feature = "json")]
34#[must_use]
35pub fn script_value_to_json(value: &ScriptValue) -> String {
36    to_json_value(value).to_string()
37}
38
39/// Recursive [`ScriptValue`] -> [`serde_json::Value`] projection.
40#[cfg(feature = "json")]
41fn to_json_value(value: &ScriptValue) -> serde_json::Value {
42    match value {
43        ScriptValue::Unit => serde_json::Value::Null,
44        ScriptValue::Bool(b) => serde_json::Value::Bool(*b),
45        ScriptValue::I64(n) => serde_json::Value::from(*n),
46        ScriptValue::F64(n) => serde_json::Number::from_f64(*n)
47            .map_or(serde_json::Value::Null, serde_json::Value::Number),
48        ScriptValue::Str(s) => serde_json::Value::String(s.clone()),
49        ScriptValue::Array(items) => {
50            serde_json::Value::Array(items.iter().map(to_json_value).collect())
51        }
52        ScriptValue::Map(fields) => {
53            let mut keys: Vec<&String> = fields.keys().collect();
54            keys.sort();
55            serde_json::Value::Object(
56                keys.into_iter()
57                    .map(|key| (key.clone(), to_json_value(&fields[key])))
58                    .collect(),
59            )
60        }
61    }
62}
63
64/// Recursive [`serde_json::Value`] -> [`ScriptValue`] projection. A JSON
65/// number lands as [`ScriptValue::I64`] when it is integral and
66/// [`ScriptValue::F64`] otherwise, so `as_int` works on `5` and `as_float` on
67/// `5.5`.
68#[cfg(feature = "json")]
69fn json_value(v: &serde_json::Value) -> ScriptValue {
70    match v {
71        serde_json::Value::Null => ScriptValue::Unit,
72        serde_json::Value::Bool(b) => ScriptValue::Bool(*b),
73        serde_json::Value::Number(n) => n.as_i64().map_or_else(
74            || ScriptValue::F64(n.as_f64().unwrap_or_default()),
75            ScriptValue::I64,
76        ),
77        serde_json::Value::String(s) => ScriptValue::Str(s.clone()),
78        serde_json::Value::Array(items) => {
79            ScriptValue::Array(items.iter().map(json_value).collect())
80        }
81        serde_json::Value::Object(fields) => ScriptValue::Map(
82            fields
83                .iter()
84                .map(|(k, val)| (k.clone(), json_value(val)))
85                .collect(),
86        ),
87    }
88}
89
90/// The block kinds `parse_markdown` recognises.
91#[cfg(feature = "markdown")]
92#[derive(Clone, Copy)]
93enum BlockKind {
94    Heading(u8),
95    Paragraph,
96    CodeBlock,
97    Item,
98}
99
100#[cfg(feature = "markdown")]
101impl BlockKind {
102    /// The `kind` field value a block of this shape carries.
103    const fn tag(self) -> &'static str {
104        match self {
105            Self::Heading(_) => "h",
106            Self::Paragraph => "p",
107            Self::CodeBlock => "code",
108            Self::Item => "li",
109        }
110    }
111
112    /// The `level` field value: the heading depth, `0` for everything else.
113    const fn level(self) -> i64 {
114        match self {
115            Self::Heading(level) => level as i64,
116            _ => 0,
117        }
118    }
119}
120
121/// Build one block record: `{ id, kind, level, text, lang }`.
122#[cfg(feature = "markdown")]
123fn block(counter: &mut usize, kind: &str, level: i64, text: String, lang: String) -> ScriptValue {
124    let id = format!("blk-{counter}");
125    *counter += 1;
126    ScriptValue::Map(std::collections::HashMap::from([
127        ("id".to_owned(), ScriptValue::Str(id)),
128        ("kind".to_owned(), ScriptValue::Str(kind.to_owned())),
129        ("level".to_owned(), ScriptValue::I64(level)),
130        ("text".to_owned(), ScriptValue::Str(text)),
131        ("lang".to_owned(), ScriptValue::Str(lang)),
132    ]))
133}
134
135/// Parse `src` as markdown into a list of block records, one per heading,
136/// paragraph, code block, list item, and horizontal rule, in document order.
137///
138/// Lumen labels render plain text, so inline emphasis keeps its markdown
139/// delimiters in the block text rather than being dropped. Links flatten to
140/// their text.
141#[cfg(feature = "markdown")]
142#[must_use]
143pub fn parse_markdown(src: &str) -> ScriptValue {
144    use pulldown_cmark::{Event, HeadingLevel, Parser, Tag, TagEnd};
145
146    let mut out: Vec<ScriptValue> = Vec::new();
147    let mut counter: usize = 0;
148    let mut cur_kind: Option<BlockKind> = None;
149    let mut cur_text = String::new();
150    let mut cur_lang = String::new();
151
152    for ev in Parser::new(src) {
153        match ev {
154            Event::Start(Tag::Heading { level, .. }) => {
155                cur_kind = Some(BlockKind::Heading(match level {
156                    HeadingLevel::H1 => 1,
157                    HeadingLevel::H2 => 2,
158                    HeadingLevel::H3 => 3,
159                    HeadingLevel::H4 => 4,
160                    HeadingLevel::H5 => 5,
161                    HeadingLevel::H6 => 6,
162                }));
163                cur_text.clear();
164                cur_lang.clear();
165            }
166            Event::Start(Tag::Paragraph) => {
167                cur_kind = Some(BlockKind::Paragraph);
168                cur_text.clear();
169                cur_lang.clear();
170            }
171            Event::Start(Tag::CodeBlock(kind)) => {
172                cur_kind = Some(BlockKind::CodeBlock);
173                cur_text.clear();
174                cur_lang = match kind {
175                    pulldown_cmark::CodeBlockKind::Fenced(lang) => lang.to_string(),
176                    pulldown_cmark::CodeBlockKind::Indented => String::new(),
177                };
178            }
179            Event::Start(Tag::Item) => {
180                cur_kind = Some(BlockKind::Item);
181                cur_text.clear();
182                cur_lang.clear();
183            }
184            Event::Start(Tag::Emphasis) | Event::End(TagEnd::Emphasis) => cur_text.push('*'),
185            Event::Start(Tag::Strong) | Event::End(TagEnd::Strong) => cur_text.push_str("**"),
186            Event::Start(Tag::Strikethrough) | Event::End(TagEnd::Strikethrough) => {
187                cur_text.push('~');
188            }
189            Event::End(
190                TagEnd::Heading(_) | TagEnd::Paragraph | TagEnd::CodeBlock | TagEnd::Item,
191            ) => {
192                if let Some(kind) = cur_kind.take() {
193                    out.push(block(
194                        &mut counter,
195                        kind.tag(),
196                        kind.level(),
197                        std::mem::take(&mut cur_text),
198                        std::mem::take(&mut cur_lang),
199                    ));
200                }
201            }
202            Event::Text(t) => cur_text.push_str(&t),
203            Event::Code(t) => {
204                cur_text.push('`');
205                cur_text.push_str(&t);
206                cur_text.push('`');
207            }
208            Event::SoftBreak => cur_text.push(' '),
209            Event::HardBreak => cur_text.push('\n'),
210            Event::Rule => {
211                out.push(block(&mut counter, "hr", 0, String::new(), String::new()));
212            }
213            _ => {}
214        }
215    }
216    ScriptValue::Array(out)
217}
218
219#[cfg(all(test, any(feature = "json", feature = "markdown")))]
220mod tests {
221    use super::*;
222
223    #[cfg(feature = "json")]
224    fn field<'a>(v: &'a ScriptValue, key: &str) -> Option<&'a ScriptValue> {
225        match v {
226            ScriptValue::Map(m) => m.get(key),
227            _ => None,
228        }
229    }
230
231    #[cfg(feature = "json")]
232    #[test]
233    fn json_keeps_scalar_types() {
234        let v = parse_json(r#"{"n": 5, "f": 2.5, "b": true, "s": "x", "z": null}"#);
235        assert_eq!(field(&v, "n"), Some(&ScriptValue::I64(5)));
236        assert_eq!(field(&v, "f"), Some(&ScriptValue::F64(2.5)));
237        assert_eq!(field(&v, "b"), Some(&ScriptValue::Bool(true)));
238        assert_eq!(field(&v, "s"), Some(&ScriptValue::Str("x".to_owned())));
239        assert_eq!(field(&v, "z"), Some(&ScriptValue::Unit));
240    }
241
242    #[cfg(feature = "json")]
243    #[test]
244    fn json_malformed_is_unit() {
245        assert_eq!(parse_json("{"), ScriptValue::Unit);
246    }
247
248    #[cfg(feature = "json")]
249    #[test]
250    fn json_written_reads_back_as_the_same_value() {
251        let value = ScriptValue::Map(
252            [
253                ("n".to_owned(), ScriptValue::I64(-5)),
254                ("f".to_owned(), ScriptValue::F64(2.5)),
255                ("b".to_owned(), ScriptValue::Bool(false)),
256                ("s".to_owned(), ScriptValue::Str("a \"q\"".to_owned())),
257                ("u".to_owned(), ScriptValue::Unit),
258                (
259                    "a".to_owned(),
260                    ScriptValue::Array(vec![ScriptValue::I64(1), ScriptValue::Str("x".into())]),
261                ),
262            ]
263            .into_iter()
264            .collect(),
265        );
266        let text = script_value_to_json(&value);
267        assert_eq!(parse_json(&text), value);
268        // Keys come out sorted, so one value always writes one text.
269        assert!(text.starts_with(r#"{"a":"#), "{text}");
270    }
271
272    #[cfg(feature = "json")]
273    #[test]
274    fn json_has_no_spelling_for_a_float_that_is_not_finite() {
275        assert_eq!(script_value_to_json(&ScriptValue::F64(f64::NAN)), "null");
276        assert_eq!(script_value_to_json(&ScriptValue::Unit), "null");
277    }
278
279    #[cfg(feature = "markdown")]
280    fn field_md<'a>(v: &'a ScriptValue, key: &str) -> Option<&'a ScriptValue> {
281        match v {
282            ScriptValue::Map(m) => m.get(key),
283            _ => None,
284        }
285    }
286
287    #[cfg(feature = "markdown")]
288    #[test]
289    fn markdown_blocks_carry_kind_level_and_lang() {
290        let v = parse_markdown("# Title\n\nBody text\n\n```rust\nfn f() {}\n```\n\n---\n");
291        let blocks = match &v {
292            ScriptValue::Array(items) => items,
293            _ => panic!("array of blocks"),
294        };
295        assert_eq!(blocks.len(), 4);
296        assert_eq!(
297            field_md(&blocks[0], "kind"),
298            Some(&ScriptValue::Str("h".to_owned()))
299        );
300        assert_eq!(field_md(&blocks[0], "level"), Some(&ScriptValue::I64(1)));
301        assert_eq!(
302            field_md(&blocks[1], "kind"),
303            Some(&ScriptValue::Str("p".to_owned()))
304        );
305        assert_eq!(
306            field_md(&blocks[2], "lang"),
307            Some(&ScriptValue::Str("rust".to_owned()))
308        );
309        assert_eq!(
310            field_md(&blocks[3], "kind"),
311            Some(&ScriptValue::Str("hr".to_owned()))
312        );
313        assert_eq!(
314            field_md(&blocks[0], "id"),
315            Some(&ScriptValue::Str("blk-0".to_owned()))
316        );
317    }
318}
lumen_script/text_parse.rs

lumen_script/
text_parse.rs
