diff --git a/.changeset/rare-rivers-open.md b/.changeset/rare-rivers-open.md new file mode 100644 index 0000000..1bf1d3a --- /dev/null +++ b/.changeset/rare-rivers-open.md @@ -0,0 +1,7 @@ +--- +"@getlang/parser": patch +"@getlang/ast": patch +"@getlang/get": patch +--- + +primitive literals diff --git a/packages/ast/src/ast.ts b/packages/ast/src/ast.ts index 275f685..8964aed 100644 --- a/packages/ast/src/ast.ts +++ b/packages/ast/src/ast.ts @@ -143,6 +143,13 @@ type SliceExpr = { typeInfo: TypeInfo } +type LiteralExpr = { + kind: 'LiteralExpr' + value: boolean | number + raw: Token + typeInfo: TypeInfo +} + export type Stmt = | Program | ExtractStmt @@ -166,6 +173,7 @@ export type Expr = | ObjectLiteralExpr | SliceExpr | DrillExpr + | LiteralExpr export type Node = Stmt | Expr @@ -326,6 +334,13 @@ const sliceExpr = (slice: Token): SliceExpr => ({ slice, }) +const literalExpr = (raw: Token): LiteralExpr => ({ + kind: 'LiteralExpr', + typeInfo: { type: Type.Value }, + raw, + value: JSON.parse(raw.value), +}) + const templateExpr = (elements: (Expr | Token)[]): TemplateExpr => ({ kind: 'TemplateExpr', typeInfo: { type: Type.Value }, @@ -353,4 +368,5 @@ export const t = { objectLiteralExpr, subqueryExpr, drillExpr, + literalExpr, } diff --git a/packages/get/src/execute.ts b/packages/get/src/execute.ts index c77bb5d..53901bc 100644 --- a/packages/get/src/execute.ts +++ b/packages/get/src/execute.ts @@ -78,6 +78,10 @@ export async function execute( return { data, typeInfo: node.typeInfo } }, + LiteralExpr(node) { + return { data: node.value, typeInfo: node.typeInfo } + }, + Program: { enter() { scope.extracted = { data: null, typeInfo: { type: Type.Value } } diff --git a/packages/get/src/modules.ts b/packages/get/src/modules.ts index 724ef9a..04c7372 100644 --- a/packages/get/src/modules.ts +++ b/packages/get/src/modules.ts @@ -93,12 +93,8 @@ export class Modules { macros.push(i) } } - const { - program: simplified, - inputs, - calls, - modifiers, - } = desugar(ast, macros) + const simplified = desugar(ast, macros) + const { inputs, calls, modifiers } = analyze(simplified) const returnTypes: Record = {} for (const call of calls) { diff --git a/packages/parser/src/grammar.ts b/packages/parser/src/grammar.ts index e32373a..99529a0 100644 --- a/packages/parser/src/grammar.ts +++ b/packages/parser/src/grammar.ts @@ -10,11 +10,13 @@ declare var request_block_body: any; declare var request_block_body_end: any; declare var drill_arrow: any; declare var link: any; +declare var identifier_expr: any; declare var call: any; -declare var literal: any; +declare var str: any; declare var interpvar: any; +declare var bool: any; +declare var num: any; declare var slice: any; -declare var identifier_expr: any; declare var ws: any; declare var comment: any; declare var nl: any; @@ -91,7 +93,6 @@ const grammar: Grammar = { {"name": "request_blocks$ebnf$2", "symbols": [], "postprocess": () => null}, {"name": "request_blocks", "symbols": ["request_blocks$ebnf$1", "request_blocks$ebnf$2"], "postprocess": p.requestBlocks}, {"name": "request_block_named", "symbols": [(lexer.has("request_block_name") ? {type: "request_block_name"} : request_block_name), "line_sep", "request_block"], "postprocess": p.requestBlockNamed}, - {"name": "request_block_body", "symbols": [(lexer.has("request_block_body") ? {type: "request_block_body"} : request_block_body), "template", (lexer.has("request_block_body_end") ? {type: "request_block_body_end"} : request_block_body_end)], "postprocess": p.requestBlockBody}, {"name": "request_block$ebnf$1", "symbols": []}, {"name": "request_block$ebnf$1$subexpression$1", "symbols": ["line_sep", "request_entry"]}, {"name": "request_block$ebnf$1", "symbols": ["request_block$ebnf$1", "request_block$ebnf$1$subexpression$1"], "postprocess": (d) => d[0].concat([d[1]])}, @@ -100,6 +101,7 @@ const grammar: Grammar = { {"name": "request_entry$ebnf$1", "symbols": ["request_entry$ebnf$1$subexpression$1"], "postprocess": id}, {"name": "request_entry$ebnf$1", "symbols": [], "postprocess": () => null}, {"name": "request_entry", "symbols": ["template", {"literal":":"}, "request_entry$ebnf$1"], "postprocess": p.requestEntry}, + {"name": "request_block_body", "symbols": [(lexer.has("request_block_body") ? {type: "request_block_body"} : request_block_body), "template", (lexer.has("request_block_body_end") ? {type: "request_block_body_end"} : request_block_body_end)], "postprocess": p.requestBlockBody}, {"name": "expression", "symbols": ["drill"], "postprocess": id}, {"name": "expression$ebnf$1$subexpression$1", "symbols": ["drill", "_", (lexer.has("drill_arrow") ? {type: "drill_arrow"} : drill_arrow), "_"]}, {"name": "expression$ebnf$1", "symbols": ["expression$ebnf$1$subexpression$1"], "postprocess": id}, @@ -112,13 +114,14 @@ const grammar: Grammar = { {"name": "drill$ebnf$2$subexpression$1", "symbols": ["_", (lexer.has("drill_arrow") ? {type: "drill_arrow"} : drill_arrow), "_", "bit"]}, {"name": "drill$ebnf$2", "symbols": ["drill$ebnf$2", "drill$ebnf$2$subexpression$1"], "postprocess": (d) => d[0].concat([d[1]])}, {"name": "drill", "symbols": ["drill$ebnf$1", "bit", "drill$ebnf$2"], "postprocess": p.drill}, - {"name": "bit$subexpression$1", "symbols": ["template"]}, + {"name": "bit$subexpression$1", "symbols": ["literal"]}, {"name": "bit$subexpression$1", "symbols": ["slice"]}, {"name": "bit$subexpression$1", "symbols": ["call"]}, {"name": "bit$subexpression$1", "symbols": ["object"]}, {"name": "bit$subexpression$1", "symbols": ["subquery"]}, {"name": "bit", "symbols": ["bit$subexpression$1"], "postprocess": p.idd}, - {"name": "bit", "symbols": ["id_expr"], "postprocess": p.identifier}, + {"name": "bit", "symbols": ["template"], "postprocess": p.selector}, + {"name": "bit", "symbols": [(lexer.has("identifier_expr") ? {type: "identifier_expr"} : identifier_expr)], "postprocess": p.idbit}, {"name": "subquery", "symbols": [{"literal":"("}, "_", "statements", "_", {"literal":")"}], "postprocess": p.subquery}, {"name": "call$ebnf$1$subexpression$1", "symbols": [{"literal":"("}, "object", {"literal":")"}]}, {"name": "call$ebnf$1", "symbols": ["call$ebnf$1$subexpression$1"], "postprocess": id}, @@ -141,13 +144,13 @@ const grammar: Grammar = { {"name": "object_entry", "symbols": [(lexer.has("identifier") ? {type: "identifier"} : identifier), "object_entry$ebnf$3"], "postprocess": p.objectEntryShorthandSelect}, {"name": "object_entry$ebnf$4", "symbols": [{"literal":"?"}], "postprocess": id}, {"name": "object_entry$ebnf$4", "symbols": [], "postprocess": () => null}, - {"name": "object_entry", "symbols": ["id_expr", "object_entry$ebnf$4"], "postprocess": p.objectEntryShorthandIdent}, - {"name": "template$ebnf$1$subexpression$1", "symbols": [(lexer.has("literal") ? {type: "literal"} : literal)]}, + {"name": "object_entry", "symbols": [(lexer.has("identifier_expr") ? {type: "identifier_expr"} : identifier_expr), "object_entry$ebnf$4"], "postprocess": p.objectEntryShorthandIdent}, + {"name": "template$ebnf$1$subexpression$1", "symbols": [(lexer.has("str") ? {type: "str"} : str)]}, {"name": "template$ebnf$1$subexpression$1", "symbols": [(lexer.has("interpvar") ? {type: "interpvar"} : interpvar)]}, {"name": "template$ebnf$1$subexpression$1", "symbols": ["interp_expr"]}, {"name": "template$ebnf$1$subexpression$1", "symbols": ["interp_tmpl"]}, {"name": "template$ebnf$1", "symbols": ["template$ebnf$1$subexpression$1"]}, - {"name": "template$ebnf$1$subexpression$2", "symbols": [(lexer.has("literal") ? {type: "literal"} : literal)]}, + {"name": "template$ebnf$1$subexpression$2", "symbols": [(lexer.has("str") ? {type: "str"} : str)]}, {"name": "template$ebnf$1$subexpression$2", "symbols": [(lexer.has("interpvar") ? {type: "interpvar"} : interpvar)]}, {"name": "template$ebnf$1$subexpression$2", "symbols": ["interp_expr"]}, {"name": "template$ebnf$1$subexpression$2", "symbols": ["interp_tmpl"]}, @@ -155,8 +158,12 @@ const grammar: Grammar = { {"name": "template", "symbols": ["template$ebnf$1"], "postprocess": p.template}, {"name": "interp_expr", "symbols": [{"literal":"${"}, "_", (lexer.has("identifier") ? {type: "identifier"} : identifier), "_", {"literal":"}"}], "postprocess": p.interpExpr}, {"name": "interp_tmpl", "symbols": [{"literal":"$["}, "_", "template", "_", {"literal":"]"}], "postprocess": p.interpTmpl}, + {"name": "literal$subexpression$1", "symbols": [(lexer.has("bool") ? {type: "bool"} : bool)]}, + {"name": "literal$subexpression$1", "symbols": [(lexer.has("num") ? {type: "num"} : num)]}, + {"name": "literal", "symbols": ["literal$subexpression$1"], "postprocess": p.literal}, + {"name": "literal", "symbols": [{"literal":"'"}, "template", {"literal":"'"}], "postprocess": p.string}, + {"name": "literal", "symbols": [{"literal":"\""}, "template", {"literal":"\""}], "postprocess": p.string}, {"name": "slice", "symbols": [(lexer.has("slice") ? {type: "slice"} : slice)], "postprocess": p.slice}, - {"name": "id_expr", "symbols": [(lexer.has("identifier_expr") ? {type: "identifier_expr"} : identifier_expr)], "postprocess": id}, {"name": "line_sep$ebnf$1", "symbols": []}, {"name": "line_sep$ebnf$1$subexpression$1", "symbols": [(lexer.has("ws") ? {type: "ws"} : ws)]}, {"name": "line_sep$ebnf$1$subexpression$1", "symbols": [(lexer.has("comment") ? {type: "comment"} : comment)]}, diff --git a/packages/parser/src/grammar/getlang.ne b/packages/parser/src/grammar/getlang.ne index 6cbfa54..140d0e1 100644 --- a/packages/parser/src/grammar/getlang.ne +++ b/packages/parser/src/grammar/getlang.ne @@ -36,8 +36,9 @@ expression -> (drill _ %drill_arrow _):? %link _ drill {% p.link %} drill -> (%drill_arrow _):? bit (_ %drill_arrow _ bit):* {% p.drill %} # drill bit -bit -> (template | slice | call | object | subquery) {% p.idd %} -bit -> id_expr {% p.identifier %} +bit -> (literal | slice | call | object | subquery) {% p.idd %} +bit -> template {% p.selector %} +bit -> %identifier_expr {% p.idbit %} # subqueries subquery -> "(" _ statements _ ")" {% p.subquery %} @@ -49,14 +50,18 @@ call -> %call ("(" object ")"):? {% p.call %} object -> "{" _ (object_entry (_ ","):? _):* "}" {% p.object %} object_entry -> "@":? %identifier "?":? ":" _ expression {% p.objectEntry %} object_entry -> %identifier "?":? {% p.objectEntryShorthandSelect %} -object_entry -> id_expr "?":? {% p.objectEntryShorthandIdent %} +object_entry -> %identifier_expr "?":? {% p.objectEntryShorthandIdent %} -# literals -template -> (%literal | %interpvar | interp_expr | interp_tmpl):+ {% p.template %} +# templates +template -> (%str | %interpvar | interp_expr | interp_tmpl):+ {% p.template %} interp_expr -> "${" _ %identifier _ "}" {% p.interpExpr %} interp_tmpl -> "$[" _ template _ "]" {% p.interpTmpl %} + +# literals +literal -> (%bool | %num) {% p.literal %} +literal -> "'" template "'" {% p.string %} +literal -> "\"" template "\"" {% p.string %} slice -> %slice {% p.slice %} -id_expr -> %identifier_expr {% id %} # whitespace line_sep -> (%ws | %comment):* %nl _ {% p.ws %} diff --git a/packages/parser/src/grammar/lex/templates.ts b/packages/parser/src/grammar/lex/templates.ts index efd8fb8..96a2946 100644 --- a/packages/parser/src/grammar/lex/templates.ts +++ b/packages/parser/src/grammar/lex/templates.ts @@ -33,7 +33,7 @@ export const templateUntil = ( return { term: { - defaultType: 'literal', + defaultType: 'str', match: new RegExp(`(?=${term.source})`), lineBreaks: true, ...(next ? { next } : { pop: 1 }), @@ -56,7 +56,7 @@ export const templateUntil = ( ), value: (text: string) => text.slice(1), }, - literal: { + str: { match: until(new RegExp(`[${interpSymbols.join('')}]|${term.source}`)), value: (text: string) => text.replace(/\\(.)/g, '$1').replace(/\s/g, ' '), lineBreaks: true, @@ -91,9 +91,27 @@ const interpTmplParams = { ...templateUntil(/]/, { interpParams: true }), } +const stringS = { + squot: { + match: `'`, + pop: 1, + }, + ...templateUntil(/'/), +} + +const stringD = { + dquot: { + match: '"', + pop: 1, + }, + ...templateUntil(/"/), +} + export const templateStates = { template: templateUntil(/\n|->|=>/, { interpTemplate: false }), interpExpr, interpTmpl, interpTmplParams, + stringS, + stringD, } diff --git a/packages/parser/src/grammar/lexer.ts b/packages/parser/src/grammar/lexer.ts index 3f03283..f8166d6 100644 --- a/packages/parser/src/grammar/lexer.ts +++ b/packages/parser/src/grammar/lexer.ts @@ -22,7 +22,7 @@ const main = { }, drill_arrow: { match: ['->', '=>'], - push: 'expr', + push: 'drillExpr', }, colon: { match: ':', @@ -48,13 +48,12 @@ const main = { symbols: /[{}(),?@]/, } -const expr = { +const exprBase = { ws: patterns.ws, nl: { match: /\n/, lineBreaks: true, }, - drill_arrow: ['->', '=>'], link: { match: patterns.link, value: (text: string) => text.slice(1, -1), @@ -80,6 +79,40 @@ const expr = { value: (text: string) => text.slice(1), pop: 1, }, + squot: { + match: `'`, + next: 'stringS', + }, + dquot: { + match: `"`, + next: 'stringD', + }, +} + +const expr = { + ...exprBase, + drill_arrow: { + match: ['->', '=>'], + next: 'drillExpr', + }, + num: { + match: /\d+(?:\.\d+)?/, + pop: 1, + }, + bool: { + match: ['true', 'false'], + pop: 1, + }, + template: { + defaultType: 'ws', + match: /(?=.)/, + next: 'template', + }, +} + +const drillExpr = { + ...exprBase, + drill_arrow: ['->', '=>'], template: { defaultType: 'ws', match: /(?=.)/, @@ -91,6 +124,7 @@ const lexer: any = moo.states({ $all: { err: moo.error }, main, expr, + drillExpr, ...templateStates, ...requestStates, }) diff --git a/packages/parser/src/grammar/parse.ts b/packages/parser/src/grammar/parse.ts index caa7eea..81e1241 100644 --- a/packages/parser/src/grammar/parse.ts +++ b/packages/parser/src/grammar/parse.ts @@ -124,15 +124,12 @@ export const objectEntryShorthandIdent: PP = ([identifier, optional]) => { function drillBase(bit: Expr, arrow?: string): Expr { const expand = arrow === '=>' - switch (bit.kind) { - case 'TemplateExpr': - return t.selectorExpr(bit, expand) - case 'IdentifierExpr': - return t.drillIdentifierExpr(bit.id, expand) - default: - invariant(!expand, new QuerySyntaxError('Misplaced wide arrow drill')) - return bit + if (bit.kind === 'SelectorExpr' || bit.kind === 'DrillIdentifierExpr') { + bit.expand = expand + } else if (expand) { + throw new QuerySyntaxError('Misplaced wide arrow drill') } + return bit } export const drill: PP = ([arrow, bit, bits]) => { @@ -143,9 +140,8 @@ export const drill: PP = ([arrow, bit, bits]) => { return t.drillExpr([expr, ...exprs]) } -export const identifier: PP = ([id]) => { - return t.identifierExpr(id) -} +export const selector: PP = ([template]) => t.selectorExpr(template, false) +export const idbit: PP = ([id]) => t.drillIdentifierExpr(id, false) export const template: PP = d => { const elements = d[0].reduce((els: any, dd: any) => { @@ -158,10 +154,10 @@ export const template: PP = d => { els.push(el) } else if (el.type === 'interpvar' || el.type === 'identifier') { els.push(t.identifierExpr(el)) - } else if (el.type === 'literal') { + } else if (el.type === 'str') { if (el.value) { const prev = els.at(-1) - if (prev?.type === 'literal') { + if (prev?.type === 'str') { els.pop() els.push({ ...prev, value: prev.value + el.value }) } else { @@ -176,19 +172,22 @@ export const template: PP = d => { }, []) const first = elements.at(0) - if (first.type === 'literal') { + if (first.type === 'str') { elements[0] = { ...first, value: first.value.trimLeft() } } const lastIdx = elements.length - 1 const last = elements[lastIdx] - if (last.type === 'literal') { + if (last.type === 'str') { elements[lastIdx] = { ...last, value: last.value.trimRight() } } return t.templateExpr(elements) } +export const literal: PP = ([[token]]) => t.literalExpr(token) +export const string: PP = ([, template]) => template + export const interpExpr: PP = ([, , token]) => token export const interpTmpl: PP = ([, , template]) => template diff --git a/packages/parser/src/passes/analyze.ts b/packages/parser/src/passes/analyze.ts index fbeae2c..3ed9d6e 100644 --- a/packages/parser/src/passes/analyze.ts +++ b/packages/parser/src/passes/analyze.ts @@ -3,21 +3,29 @@ import { ScopeTracker, transform } from '@getlang/walker' export function analyze(ast: Program) { const scope = new ScopeTracker() + const inputs = new Set() + const calls = new Set() + const modifiers = new Set() const imports = new Set() let isMacro = false transform(ast, { scope, + InputExpr(node) { + inputs.add(node.id.value) + }, ModuleExpr(node) { imports.add(node.module.value) + node.call && calls.add(node.module.value) }, SelectorExpr() { isMacro ||= !scope.context }, - ModifierExpr() { + ModifierExpr(node) { isMacro ||= !scope.context + modifiers.add(node.modifier.value) }, }) - return { imports, isMacro } + return { inputs, imports, calls, modifiers, isMacro } } diff --git a/packages/parser/src/passes/desugar.ts b/packages/parser/src/passes/desugar.ts index 1a6dc56..c6b0810 100644 --- a/packages/parser/src/passes/desugar.ts +++ b/packages/parser/src/passes/desugar.ts @@ -1,5 +1,4 @@ import type { Program } from '@getlang/ast' -import { transform } from '@getlang/walker' import { resolveContext } from './desugar/context.js' import { dropDrills } from './desugar/dropdrill.js' import { settleLinks } from './desugar/links.js' @@ -16,26 +15,6 @@ export type DesugarPass = ( }, ) => Program -function analyze2(ast: Program) { - const inputs = new Set() - const calls = new Set() - const modifiers = new Set() - - transform(ast, { - InputExpr(node) { - inputs.add(node.id.value) - }, - ModuleExpr(node) { - node.call && calls.add(node.module.value) - }, - ModifierExpr(node) { - modifiers.add(node.modifier.value) - }, - }) - - return { inputs, calls, modifiers } -} - const visitors = [ addUrlInputs, resolveContext, @@ -46,14 +25,11 @@ const visitors = [ export function desugar(ast: Program, macros: string[] = []) { const parsers = new RequestParsers() - let program = visitors.reduce((ast, pass) => { + const program = visitors.reduce((ast, pass) => { parsers.reset() return pass(ast, { parsers, macros }) }, ast) - // inference pass `registerCalls` is included in the desugar phase // it produces the list of called modules required for type inference - program = registerCalls(program, macros) - const info = analyze2(program) - return { program, ...info } + return registerCalls(program, macros) } diff --git a/packages/parser/src/print.ts b/packages/parser/src/print.ts index 680dd08..6cad4e6 100644 --- a/packages/parser/src/print.ts +++ b/packages/parser/src/print.ts @@ -65,6 +65,10 @@ const printVisitor: ReduceVisitor = { return group(parts) }, + LiteralExpr(node) { + return String(node.value) + }, + RequestStmt(node) { return node.request }, diff --git a/test/helpers.ts b/test/helpers.ts index b783047..f46e983 100644 --- a/test/helpers.ts +++ b/test/helpers.ts @@ -18,8 +18,8 @@ export type Fetch = (url: string, opts: RequestInit) => MaybePromise export const SELSYN = true function testIdempotency(source: string) { - const print1 = print(desugar(parse(source)).program) - const print2 = print(desugar(parse(print1)).program) + const print1 = print(desugar(parse(source))) + const print2 = print(desugar(parse(print1))) expect(print1).toEqual(print2) } diff --git a/test/values.spec.ts b/test/values.spec.ts index 3fe9204..fab08a6 100644 --- a/test/values.spec.ts +++ b/test/values.spec.ts @@ -460,4 +460,65 @@ describe('values', () => { expect(result).toEqual({}) }) }) + + test('literals', async () => { + const result = await execute(` + extract { + str_s: 'one' + str_d: "two" + int: 12 + float: 12.34 + bool_on: true + bool_off: false + } + `) + + expect(result).toEqual({ + str_s: 'one', + str_d: 'two', + int: 12, + float: 12.34, + bool_on: true, + bool_off: false, + }) + }) + + test('literals drill escape', async () => { + const result = await execute(` + set ctx = |return { + '12': 'pass', + '12.34': 'pass', + 'true': 'pass', + 'false': 'pass', + }| + + extract $ctx -> { + str_s: -> 'one' + str_d: -> "two" + int: -> 12 + float: -> 12.34 + bool_on: -> true + bool_off: -> false + } + `) + + expect(result).toEqual({ + str_s: 'one', + str_d: 'two', + int: 'pass', + float: 'pass', + bool_on: 'pass', + bool_off: 'pass', + }) + }) + + test('literal string interpolation', async () => { + const result = await execute(` + set foo = "foo" + set bar = "bar" + set x = 12.34 + extract "a = $foo, b = \${bar}baz$x" + `) + expect(result).toEqual('a = foo, b = barbaz12.34') + }) })