Add DSL playground with tokenizer and AST inspection
This commit is contained in:
Peter Stockings
@@ -0,0 +1,82 @@
|
||||
import { useState } from "react";
|
||||
import { tokenizeExpression, parseExpression, validateAst, extractDependencies, FUNCTION_SIGNATURES, type Expr, type Token } from "../lib/questionnaire/dsl";
|
||||
import { QuestionnaireEngine } from "../lib/questionnaire/questionnaire-engine";
|
||||
import starter from "./editor-questionnaire.json";
|
||||
import { JsonViewer } from "./JsonViewer";
|
||||
|
||||
const examples = [
|
||||
{ name: "Eligibility", expression: "age(answer('dateOfBirth'), today()) >= 18 && answer('consent') === true" },
|
||||
{ name: "Precedence", expression: "1 + 2 * 3" },
|
||||
{ name: "Branching", expression: "derived('inPerson') ? 'Welcome to the venue' : 'We will email your joining link'" },
|
||||
{ name: "Collections", expression: "contains(answer('topics'), 'DESIGN') && count(answer('topics')) <= 2" },
|
||||
{ name: "Validation value", expression: "length(trim(value)) >= 3" },
|
||||
{ name: "Null fallback", expression: "answer('experience') ?? 0" },
|
||||
{ name: "Short circuit", expression: "false && (1 / 0 > 0)" },
|
||||
{ name: "Parser error", expression: "1 +" },
|
||||
{ name: "Validation error", expression: "unknown(1)" },
|
||||
{ name: "Evaluation error", expression: "1 / 0" },
|
||||
];
|
||||
const initialContext = JSON.stringify({
|
||||
answers: { name: "Alex", dateOfBirth: "1992-03-04", attendance: "IN_PERSON", guests: 1, needsSupport: false, topics: ["DESIGN"], experience: 2, consent: true },
|
||||
value: "Alex", today: "2026-09-28",
|
||||
}, null, 2);
|
||||
const engine = new QuestionnaireEngine(starter);
|
||||
type Stage = { name: string; status: "Passed" | "Failed" | "Not run"; detail: string };
|
||||
function inspect(expression: string, context: string) {
|
||||
const stages: Stage[] = [
|
||||
{ name: "Tokenizer", status: "Not run", detail: "Split source into tokens." },
|
||||
{ name: "Parser", status: "Not run", detail: "Build an AST using operator precedence." },
|
||||
{ name: "Validation", status: "Not run", detail: "Check allowed identifiers, function names, arity, and literal IDs." },
|
||||
{ name: "Evaluation", status: "Not run", detail: "Run with the questionnaire session and input context." },
|
||||
];
|
||||
let tokens: Token[] = [], ast: Expr | undefined, result: unknown;
|
||||
let index = 0;
|
||||
try {
|
||||
tokens = tokenizeExpression(expression); stages[0] = { ...stages[0], status: "Passed", detail: `${tokens.length} tokens, including end of input.` }; index++;
|
||||
ast = parseExpression(expression); stages[1] = { ...stages[1], status: "Passed", detail: `Root node: ${ast.type}.` }; index++;
|
||||
validateAst(ast); stages[2] = { ...stages[2], status: "Passed", detail: "Expression passes DSL validation." }; index++;
|
||||
const input = JSON.parse(context);
|
||||
if (!input || typeof input !== "object" || Array.isArray(input) || !input.answers || typeof input.answers !== "object" || Array.isArray(input.answers) || typeof input.today !== "string") throw new Error("Context needs an answers object and a today date string.");
|
||||
result = engine.createSession(input.answers, { today: input.today }).evaluate(expression, input.value);
|
||||
stages[3] = { ...stages[3], status: "Passed", detail: `Result type: ${result === null ? "null" : Array.isArray(result) ? "array" : typeof result}.` };
|
||||
} catch (cause) { stages[index] = { ...stages[index], status: "Failed", detail: cause instanceof Error ? cause.message : String(cause) }; }
|
||||
return { expression, context, stages, tokens, ast, result };
|
||||
}
|
||||
|
||||
function AstTree({ node, edge = "root" }: { node: Expr; edge?: string }) {
|
||||
const title = node.type === "literal" ? `literal ${JSON.stringify(node.value)}` : node.type === "call" ? `call ${node.name}()` : node.type === "identifier" ? `identifier ${node.name}` : node.type === "binary" || node.type === "unary" ? `${node.type} ${node.operator}` : node.type;
|
||||
const children: [string, Expr][] = node.type === "binary" ? [["left", node.left], ["right", node.right]] : node.type === "unary" ? [["argument", node.argument]] : node.type === "conditional" ? [["condition", node.condition], ["true", node.whenTrue], ["false", node.whenFalse]] : node.type === "call" ? node.args.map((arg, i) => [`arg ${i + 1}`, arg]) : node.type === "array" ? node.items.map((item, i) => [`item ${i}`, item]) : [];
|
||||
return <li>{children.length ? <details open><summary><span>{edge}</span><code>{title}</code></summary><ul>{children.map(([key, child]) => <AstTree key={key} edge={key} node={child} />)}</ul></details> : <div className="dsl-leaf"><span>{edge}</span><code>{title}</code></div>}</li>;
|
||||
}
|
||||
|
||||
export function DslExplorerPage() {
|
||||
const [expression, setExpression] = useState(examples[0].expression);
|
||||
const [context, setContext] = useState(initialContext);
|
||||
const [analysis, setAnalysis] = useState(() => inspect(examples[0].expression, initialContext));
|
||||
const [tab, setTab] = useState("AST tree");
|
||||
const stale = expression !== analysis.expression || context !== analysis.context;
|
||||
const tabs = ["AST tree", "AST JSON", "Dependencies", "Functions"];
|
||||
return <main className="dsl-page">
|
||||
<header className="schema-editor-header"><div><p className="eyebrow">Inside the expression engine</p><h1 tabIndex={-1}>DSL playground</h1><p>Follow an expression from source text to tokens, syntax tree, and result.</p></div><button className="primary" onClick={() => setAnalysis(inspect(expression, context))}>▶ Run expression</button></header>
|
||||
<div className="dsl-workbench">
|
||||
<section className="dsl-card"><h2>Expression</h2><label className="schema-input"><span>Load an example</span><select defaultValue="" onChange={event => { const item = examples[Number(event.target.value)]; if (item) { setExpression(item.expression); setAnalysis(inspect(item.expression, context)); } }}><option value="" disabled>Choose an example…</option>{examples.map((item, index) => <option value={index} key={item.name}>{item.name}</option>)}</select></label>
|
||||
<label className="schema-input"><span>DSL expression</span><textarea className="dsl-code-input" rows={5} value={expression} spellCheck={false} onChange={event => setExpression(event.target.value)} onKeyDown={event => { if ((event.ctrlKey || event.metaKey) && event.key === "Enter") { event.preventDefault(); setAnalysis(inspect(expression, context)); } }} /></label>
|
||||
<p className="schema-hint">Run with Ctrl/⌘ + Enter. Strings use quotes; comparisons use <code>===</code> and <code>!==</code>. Arithmetic binds before comparisons, then <code>&&</code>, <code>||</code>, and <code>??</code>.</p>
|
||||
<div className="dsl-result" aria-live="polite"><strong>{stale ? "Inputs changed — run to update" : "Evaluation result"}</strong>{analysis.stages[3].status === "Passed" ? <pre>{JSON.stringify(analysis.result, null, 2)}</pre> : <p>{analysis.stages.find(stage => stage.status === "Failed")?.detail}</p>}</div>
|
||||
</section>
|
||||
<section className="dsl-card"><h2>Evaluation context</h2><p className="schema-hint">Uses the Workshop registration schema. Edit plain answer values, the validation variable <code>value</code>, and the date returned by <code>today()</code>. Hidden or invalid answers resolve to null.</p><label className="schema-input"><span>Context JSON</span><textarea className="dsl-code-input" rows={13} value={context} spellCheck={false} onChange={event => setContext(event.target.value)} /></label><details><summary>Available questionnaire fields and derived values</summary><JsonViewer value={starter} /></details></section>
|
||||
</div>
|
||||
<section className="dsl-pipeline" aria-label="Expression pipeline">{analysis.stages.map((stage, index) => <article key={stage.name} className={`dsl-stage dsl-stage-${stage.status.replace(" ", "-").toLowerCase()}`}><span>{index + 1} · {stage.status}</span><h2>{stage.name}</h2><p>{stage.detail}</p></article>)}</section>
|
||||
<section className="dsl-card"><h2>Tokenizer output</h2><p className="schema-hint">Actual lexer output. Positions are zero-based offsets in the last run’s source; string token values are decoded. The parser turns true, false, and null identifiers into literals.</p><pre className="dsl-source">{analysis.expression}</pre>
|
||||
{analysis.tokens.length ? <div className="dsl-tokens">{analysis.tokens.map((token, index) => <div className={`dsl-token dsl-token-${token.type}`} key={index}><small>{token.type} · @{token.position}</small><code>{token.type === "eof" ? "EOF" : JSON.stringify(token.value)}</code></div>)}</div> : <p>No token stream: tokenization failed.</p>}
|
||||
</section>
|
||||
<section className="dsl-card dsl-inspector"><div className="inspector-tabs" role="tablist" aria-label="DSL inspection">{tabs.map((name, index) => <button key={name} role="tab" aria-selected={tab === name} id={`dsl-tab-${index}`} aria-controls={`dsl-panel-${index}`} tabIndex={tab === name ? 0 : -1} onClick={() => setTab(name)} onKeyDown={event => {
|
||||
const next = event.key === "ArrowRight" ? (index + 1) % tabs.length : event.key === "ArrowLeft" ? (index + tabs.length - 1) % tabs.length : event.key === "Home" ? 0 : event.key === "End" ? tabs.length - 1 : undefined;
|
||||
if (next !== undefined) { event.preventDefault(); setTab(tabs[next]); document.getElementById(`dsl-tab-${next}`)?.focus(); }
|
||||
}}>{name}</button>)}</div>
|
||||
<div className="dsl-inspector-body" role="tabpanel" id={`dsl-panel-${tabs.indexOf(tab)}`} aria-labelledby={`dsl-tab-${tabs.indexOf(tab)}`}>
|
||||
{tab === "Functions" ? <><p>Built-in function signatures from the engine. The result type of <code>answer</code> and <code>derived</code> depends on the referenced field or expression.</p><div className="docs-table-wrap"><table className="docs-table"><thead><tr><th>Function</th><th>Argument types</th><th>Returns</th></tr></thead><tbody>{Object.entries(FUNCTION_SIGNATURES).map(([name, signature]) => <tr key={name}><th>{name}()</th><td>{signature.args.map(arg => arg.join(" | ")).join(", ") || "No arguments"}</td><td>{name === "answer" || name === "derived" ? "Depends on reference" : signature.result}</td></tr>)}</tbody></table></div></> : !analysis.ast ? <p>No AST: the parser did not complete.</p> : tab === "AST tree" ? <><p>The parser groups tokens by precedence. Expand or collapse nodes to explore their children. This is the syntax tree, not an execution trace; short-circuited branches still appear here.</p><ul className="dsl-ast"><AstTree node={analysis.ast} /></ul></> : <JsonViewer value={tab === "AST JSON" ? analysis.ast : extractDependencies(analysis.ast)} />}
|
||||
</div>
|
||||
</section>
|
||||
</main>;
|
||||
}
|
||||
@@ -1,11 +1,13 @@
|
||||
import { useEffect, useState, type ReactNode, type MouseEvent } from "react";
|
||||
import { DocsPage } from "./DocsPage";
|
||||
import { SchemaEditorPage } from "./SchemaEditorPage";
|
||||
import { DslExplorerPage } from "./DslExplorerPage";
|
||||
|
||||
export function SiteLayout({ children }: { children: ReactNode }) {
|
||||
const [path, setPath] = useState(window.location.pathname);
|
||||
const docs = path.replace(/\/$/, "") === "/docs";
|
||||
const editor = path.replace(/\/$/, "") === "/editor";
|
||||
const dsl = path.replace(/\/$/, "") === "/dsl";
|
||||
|
||||
useEffect(() => {
|
||||
const sync = () => setPath(window.location.pathname);
|
||||
@@ -14,11 +16,11 @@ export function SiteLayout({ children }: { children: ReactNode }) {
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
document.title = docs ? "Docs | QuestionGraph" : editor ? "Editor | QuestionGraph" : "QuestionGraph";
|
||||
document.title = docs ? "Docs | QuestionGraph" : editor ? "Editor | QuestionGraph" : dsl ? "DSL playground | QuestionGraph" : "QuestionGraph";
|
||||
if (docs && window.location.hash) {
|
||||
document.getElementById(decodeURIComponent(window.location.hash.slice(1)))?.scrollIntoView();
|
||||
}
|
||||
}, [docs, editor]);
|
||||
}, [docs, editor, dsl]);
|
||||
|
||||
function navigate(event: MouseEvent<HTMLAnchorElement>, target: string) {
|
||||
if (event.button !== 0 || event.metaKey || event.ctrlKey || event.shiftKey || event.altKey) return;
|
||||
@@ -26,20 +28,22 @@ export function SiteLayout({ children }: { children: ReactNode }) {
|
||||
window.history.pushState(null, "", target);
|
||||
setPath(target);
|
||||
window.scrollTo(0, 0);
|
||||
requestAnimationFrame(() => document.querySelector<HTMLElement>(target === "/docs" ? ".docs-page h1" : target === "/editor" ? ".schema-editor-page h1" : ".questionnaire-pane h1")?.focus({ preventScroll: true }));
|
||||
requestAnimationFrame(() => document.querySelector<HTMLElement>(target === "/docs" ? ".docs-page h1" : target === "/dsl" ? ".dsl-page h1" : target === "/editor" ? ".schema-editor-page h1" : ".questionnaire-pane h1")?.focus({ preventScroll: true }));
|
||||
}
|
||||
|
||||
return <>
|
||||
<header className="site-header">
|
||||
<a className="site-brand" href="/" onClick={(event) => navigate(event, "/")}><span aria-hidden="true">Q</span>QuestionGraph</a>
|
||||
<nav aria-label="Main navigation">
|
||||
<a href="/" aria-current={!docs && !editor ? "page" : undefined} onClick={(event) => navigate(event, "/")}>Questionnaire</a>
|
||||
<a href="/" aria-current={!docs && !editor && !dsl ? "page" : undefined} onClick={(event) => navigate(event, "/")}>Questionnaire</a>
|
||||
<a href="/editor" aria-current={editor ? "page" : undefined} onClick={(event) => navigate(event, "/editor")}>Editor</a>
|
||||
<a href="/dsl" aria-current={dsl ? "page" : undefined} onClick={(event) => navigate(event, "/dsl")}>DSL playground</a>
|
||||
<a href="/docs" aria-current={docs ? "page" : undefined} onClick={(event) => navigate(event, "/docs")}>Docs</a>
|
||||
</nav>
|
||||
</header>
|
||||
<div hidden={docs || editor}>{children}</div>
|
||||
<div hidden={docs || editor || dsl}>{children}</div>
|
||||
<div hidden={!editor}><SchemaEditorPage /></div>
|
||||
<div hidden={!dsl}><DslExplorerPage /></div>
|
||||
{docs && <DocsPage />}
|
||||
</>;
|
||||
}
|
||||
|
||||
@@ -47,6 +47,33 @@ body { margin: 0; min-width: 320px; min-height: 100vh; }
|
||||
.schema-result { padding: 28px; }
|
||||
.schema-result .json-code { border-radius: 10px; }
|
||||
.schema-editor-page button:focus-visible { outline: 2px solid #1f5eff; outline-offset: 3px; }
|
||||
.dsl-page { width: min(1320px, calc(100% - 40px)); margin: 40px auto 80px; }
|
||||
.dsl-workbench { display: grid; grid-template-columns: 1fr 1fr; gap: 20px; }
|
||||
.dsl-card { min-width: 0; padding: 24px; margin-bottom: 20px; border: 1px solid #dfe5ee; border-radius: 14px; background: #fff; }
|
||||
.dsl-card h2 { margin: 0 0 16px; font-size: 18px; }
|
||||
.dsl-code-input { width: 100%; resize: vertical; font: 13px/1.65 ui-monospace, Consolas, monospace; background: #111827; color: #e5e7eb; }
|
||||
.dsl-result { padding: 16px; border-radius: 10px; background: #edf5ff; overflow-wrap: anywhere; }
|
||||
.dsl-result strong { font-size: 12px; color: #355a91; }
|
||||
.dsl-result pre { white-space: pre-wrap; font-size: 20px; margin-bottom: 0; }
|
||||
.dsl-pipeline { display: grid; grid-template-columns: repeat(4, minmax(0, 1fr)); gap: 12px; margin-bottom: 20px; }
|
||||
.dsl-stage { border: 1px solid #dfe5ee; border-top: 3px solid #a7b3c6; border-radius: 10px; padding: 18px; background: white; }
|
||||
.dsl-stage span { font-size: 11px; color: #627089; }
|
||||
.dsl-stage h2 { font-size: 16px; margin: 10px 0; }
|
||||
.dsl-stage p, .dsl-inspector-body > p { font-size: 13px; line-height: 1.7; color: #627089; overflow-wrap: anywhere; }
|
||||
.dsl-stage-passed { border-top-color: #279074; }
|
||||
.dsl-stage-failed { border-top-color: #d34464; background: #fff5f7; }
|
||||
.dsl-source { white-space: pre-wrap; overflow-wrap: anywhere; padding: 16px; border-radius: 8px; background: #f2f5fa; }
|
||||
.dsl-tokens { display: flex; flex-wrap: wrap; gap: 8px; max-height: 340px; overflow: auto; }
|
||||
.dsl-token { display: grid; gap: 7px; padding: 10px 12px; border: 1px solid #dfe5ee; border-radius: 8px; background: #f8faff; max-width: 100%; }
|
||||
.dsl-token small { color: #627089; font-size: 10px; }
|
||||
.dsl-token code { white-space: pre-wrap; overflow-wrap: anywhere; font-size: 12px; }
|
||||
.dsl-token-string { background: #eef9f3; }.dsl-token-operator { background: #f5f0ff; }.dsl-token-number { background: #fff5e9; }
|
||||
.dsl-inspector { padding: 0; overflow: hidden; }.dsl-inspector-body { padding: 24px; min-width: 0; overflow: auto; }
|
||||
.dsl-ast, .dsl-ast ul { list-style: none; margin: 12px 0; padding-left: 20px; }
|
||||
.dsl-ast ul { border-left: 1px solid #cad7ea; }.dsl-ast li { margin: 10px 0; }
|
||||
.dsl-ast summary, .dsl-leaf { padding: 10px 12px; border: 1px solid #dfe5ee; border-radius: 8px; background: #f8faff; width: fit-content; }
|
||||
.dsl-ast summary { cursor: pointer; }.dsl-ast span { color: #74839a; font-size: 11px; margin-right: 12px; }.dsl-ast code { font-size: 12px; overflow-wrap: anywhere; }
|
||||
@media (max-width: 750px) { .dsl-workbench { grid-template-columns: 1fr; gap: 0; }.dsl-pipeline { grid-template-columns: 1fr 1fr; }.dsl-card, .dsl-inspector-body { padding: 16px; }.dsl-inspector { padding: 0; }.site-header nav { flex-wrap: wrap; } }
|
||||
.schema-editor-page:has(.schema-visual-diagram) { width: min(1600px, calc(100% - 40px)); }
|
||||
.schema-visual-diagram { display: grid; grid-template-columns: minmax(0, 1fr) 350px; padding: 0; }
|
||||
.schema-node-editor { padding: 20px; border-left: 1px solid #dfe5ee; max-height: 760px; overflow: auto; }
|
||||
|
||||
+2
-1
@@ -1,7 +1,8 @@
|
||||
/** Public entry point for the reusable questionnaire library. */
|
||||
export { QuestionnaireEngine, QuestionnaireSession } from "./questionnaire/questionnaire-engine";
|
||||
export { compileQuestionnaire } from "./questionnaire/questionnaire-compiler";
|
||||
export { compileExpression, evaluateExpression, extractDependencies, FUNCTION_SIGNATURES } from "./questionnaire/dsl";
|
||||
export { compileExpression, parseExpression, tokenizeExpression, validateAst, evaluateExpression, extractDependencies, FUNCTION_SIGNATURES } from "./questionnaire/dsl";
|
||||
export type { Expr, Token } from "./questionnaire/dsl";
|
||||
export { parseResumeDraft } from "./questionnaire/resume-draft";
|
||||
export { QuestionnaireRenderer } from "./react/QuestionnaireRenderer";
|
||||
export type { SaveRequest, SaveResponse, TypedAnswers } from "./persistence/types";
|
||||
|
||||
@@ -9,7 +9,7 @@ export type Expr =
|
||||
|
||||
type TokenType = "number" | "string" | "identifier" | "operator" | "punctuation" | "eof";
|
||||
|
||||
interface Token {
|
||||
export interface Token {
|
||||
type: TokenType;
|
||||
value: string;
|
||||
position: number;
|
||||
@@ -274,10 +274,15 @@ export const FUNCTION_SIGNATURES: Record<string, { args: ExpressionType[][]; res
|
||||
|
||||
const ALLOWED_IDENTIFIERS = new Set(["value"]);
|
||||
|
||||
export function parseExpression(expression: string): Expr {
|
||||
export function tokenizeExpression(expression: string): Token[] {
|
||||
if (expression.length > 4096) throw new Error("Expression exceeds 4096 characters");
|
||||
const tokens = new Lexer(expression).tokenize();
|
||||
if (tokens.length > 512) throw new Error("Expression exceeds 512 tokens");
|
||||
return tokens;
|
||||
}
|
||||
|
||||
export function parseExpression(expression: string): Expr {
|
||||
const tokens = tokenizeExpression(expression);
|
||||
let depth = 0;
|
||||
for (const token of tokens) {
|
||||
if (token.type === "punctuation" && ["(", "["].includes(token.value) && ++depth > 64) throw new Error("Expression nesting exceeds 64 levels");
|
||||
|
||||
@@ -2,13 +2,26 @@ import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { readFileSync } from "node:fs";
|
||||
import { QuestionnaireEngine, type Answers, type QuestionnaireDefinition, type QuestionDefinition } from "../src/lib/questionnaire/questionnaire-engine";
|
||||
import { compileExpression, evaluateExpression } from "../src/lib/questionnaire/dsl";
|
||||
import { compileExpression, evaluateExpression, tokenizeExpression, parseExpression, validateAst } from "../src/lib/questionnaire/dsl";
|
||||
import { parseResumeDraft } from "../src/lib/questionnaire/resume-draft";
|
||||
import { SaveEventRecorder, type SaveEvent } from "../src/lib/persistence/save-events";
|
||||
|
||||
const claim = JSON.parse(readFileSync(new URL("../src/demo/claim-questionnaire.json", import.meta.url), "utf8"));
|
||||
const example: QuestionnaireDefinition = JSON.parse(readFileSync(new URL("../src/demo/example-questionnaire.json", import.meta.url), "utf8"));
|
||||
const today = { today: "2026-09-12" };
|
||||
test("DSL inspection exposes lexer offsets and keeps parsing separate from validation", () => {
|
||||
assert.deepEqual(tokenizeExpression(" 'hi' !== null"), [
|
||||
{ type: "string", value: "hi", position: 2 },
|
||||
{ type: "operator", value: "!==", position: 7 },
|
||||
{ type: "identifier", value: "null", position: 11 },
|
||||
{ type: "eof", value: "", position: 15 },
|
||||
]);
|
||||
const ast = parseExpression("unknown(1)");
|
||||
assert.equal(ast.type, "call");
|
||||
assert.throws(() => validateAst(ast), /not allowed/);
|
||||
assert.throws(() => tokenizeExpression("1 @ 2"), /position 2/);
|
||||
assert.throws(() => tokenizeExpression("1".repeat(4097)), /4096/);
|
||||
});
|
||||
const basic = (fields: QuestionDefinition[]): QuestionnaireDefinition => ({ schemaVersion: 1, id: "test", version: "1", title: "Test", pages: [{ id: "first", title: "First", fields }] });
|
||||
const bool = (id: string): QuestionDefinition => ({ id, type: "boolean", label: id });
|
||||
const text = (id: string): QuestionDefinition => ({ id, type: "text", label: id });
|
||||
|
||||
Reference in New Issue
Block a user