rixty/SillyTavern
1
0
Fork 0
mirror of https://github.com/SillyTavern/SillyTavern.git synced 2025-06-05 21:59:27 +02:00
Code Issues Packages Projects Releases Wiki Activity

Rewrote lexer modes/tokes to capture errors better

Browse Source
This commit is contained in:
Wolfsblvt
2024-08-10 02:45:50 +02:00
parent b7840eb9cd
commit 2b1e83dc07
2 changed files with 157 additions and 96 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
public/scripts/macros/MacroLexer.js
View File

@@ -6,6 +6,7 @@ import { createToken, Lexer } from '../../lib/chevrotain.js';
const modes = {
plaintext: 'plaintext_mode',
macro_def: 'macro_def_mode',
macro_identifier_end: 'macro_identifier_end',
macro_args: 'macro_args_mode',
};
@@ -19,9 +20,11 @@ const Tokens = {
Start: createToken({ name: 'MacroStart', pattern: /\{\{/ }),
// Separate macro identifier needed, that is similar to the global indentifier, but captures the actual macro "name"
// We need this, because this token is going to switch lexer mode, while the general identifier does not.
Identifier: createToken({ name: 'MacroIdentifier', pattern: /[a-zA-Z][\w-]*/ }),
Flags: createToken({ name: 'MacroFlag', pattern: /[!?#~/.$]/ }),
// CaptureBeforeEnd: createToken({ name: 'MacroCaptureBeforeEnd', pattern: /.*?(?=\}\})/, pop_mode: true/*, group: Lexer.SKIPPED */ }),
Identifier: createToken({ name: 'MacroIdentifier', pattern: /[a-zA-Z][\w-]*/ }),
// At the end of an identifier, there has to be whitspace, or must be directly followed by colon/double-colon separator, output modifier or closing braces
EndOfIdentifier: createToken({ name: 'MacroEndOfIdentifier', pattern: /(?:\s+|(?=:{1,2})|(?=[|}]))/, group: Lexer.SKIPPED }),
BeforeEnd: createToken({ name: 'MacroBeforeEnd', pattern: /(?=\}\})/, group: Lexer.SKIPPED }),
End: createToken({ name: 'MacroEnd', pattern: /\}\}/ }),
},
@@ -46,7 +49,7 @@ const Tokens = {
// DANGER ZONE: Careful with this token. This is used as a way to pop the current mode, if no other token matches.
// Can be used in modes that don't have a "defined" end really, like when capturing a single argument, argument list, etc.
// Has to ALWAYS be the last token.
ModePopper: createToken({ name: 'EndMode', pattern: () => [''], pop_mode: true/*, group: Lexer.SKIPPED */ }),
ModePopper: createToken({ name: 'ModePopper', pattern: () => [''], pop_mode: true, group: Lexer.SKIPPED }),
};
/** @type {Map<string,string>} Saves all token definitions that are marked as entering modes */
@@ -67,8 +70,15 @@ const Def = {
using(Tokens.WhiteSpace),
// Inside a macro, we will match the identifier
// Enter 'macro_args' mode automatically at the end of the identifier, to match any optional arguments
enter(Tokens.Macro.Identifier, modes.macro_args),
// Enter 'macro_identifier_end' mode automatically at the end of the identifier, so we don't match more than one identifier
enter(Tokens.Macro.Identifier, modes.macro_identifier_end),
],
[modes.macro_identifier_end]: [
exits(Tokens.Macro.BeforeEnd, modes.macro_identifier_end),
// After a macro identifier, there are only a few valid options. We check those, before we try to find optional macro args.
// Must either be followed with whitespace or colon/double-colon, which get captured, or must follow-up with macro end braces or an output modifier pipe.
enter(Tokens.Macro.EndOfIdentifier, modes.macro_args, { andExits: modes.macro_identifier_end }),
],
[modes.macro_args]: [
// Macro args allow nested macros
@@ -134,16 +144,22 @@ instance = MacroLexer.instance;
*
* Marks the token to **enter** the following lexer mode.
*
* Optionally, you can specify the modes to exit when entering this mode.
*
* @param {TokenType} token - The token to modify
* @param {string} mode - The mode to set
* @param {object} [options={}] - Additional options
* @param {string?} [options.andExits=null] - The modes to exit when entering this mode
* @returns {TokenType} The token again
*/
function enter(token, mode) {
function enter(token, mode, { andExits = null } = {}) {
if (!token) throw new Error('Token must not be undefined');
if (enterModesMap.has(token.name) && enterModesMap.get(token.name) !== mode) {
throw new Error(`Token ${token.name} already is set to enter mode ${enterModesMap.get(token.name)}. The token definition are global, so they cannot be used to lead to different modes.`);
}
if (andExits) exits(token, andExits);
token.PUSH_MODE = mode;
enterModesMap.set(token.name, mode);
return token;

225
tests/frontend/MacroLexer.test.js
View File

@@ -47,20 +47,6 @@ describe('MacroLexer', () => {
expect(tokens).toEqual(expectedTokens);
});
// {{ some macro }}
it('whitespaces between two valid identifiers will only capture the first as macro identifier', async () => {
const input = '{{ some macro }}';
const tokens = await runLexerGetTokens(input);
const expectedTokens = [
{ type: 'MacroStart', text: '{{' },
{ type: 'MacroIdentifier', text: 'some' },
{ type: 'Identifier', text: 'macro' },
{ type: 'MacroEnd', text: '}}' },
];
expect(tokens).toEqual(expectedTokens);
});
// {{macro1}}{{macro2}}
it('should handle multiple sequential macros', async () => {
const input = '{{macro1}}{{macro2}}';
@@ -75,86 +61,12 @@ describe('MacroLexer', () => {
{ type: 'MacroEnd', text: '}}' },
];
expect(tokens).toEqual(expectedTokens);
});
// {{my2cents}}
it('should allow numerics inside the macro identifier', async () => {
const input = '{{my2cents}}';
const tokens = await runLexerGetTokens(input);
const expectedTokens = [
{ type: 'MacroStart', text: '{{' },
{ type: 'MacroIdentifier', text: 'my2cents' },
{ type: 'MacroEnd', text: '}}' },
];
expect(tokens).toEqual(expectedTokens);
});
// {{SCREAM}}
it('should allow capslock macros', async () => {
const input = '{{SCREAM}}';
const tokens = await runLexerGetTokens(input);
const expectedTokens = [
{ type: 'MacroStart', text: '{{' },
{ type: 'MacroIdentifier', text: 'SCREAM' },
{ type: 'MacroEnd', text: '}}' },
];
expect(tokens).toEqual(expectedTokens);
});
// {{some-longer-macro}}
it('allow dashes in macro identifiers', async () => {
const input = '{{some-longer-macro}}';
const tokens = await runLexerGetTokens(input);
const expectedTokens = [
{ type: 'MacroStart', text: '{{' },
{ type: 'MacroIdentifier', text: 'some-longer-macro' },
{ type: 'MacroEnd', text: '}}' },
];
expect(tokens).toEqual(expectedTokens);
});
// {{macro!@#%}}
it('do not lex special characters as part of the macro identifier', async () => {
const input = '{{macro!@#%}}';
const tokens = await runLexerGetTokens(input);
const expectedTokens = [
{ type: 'MacroStart', text: '{{' },
{ type: 'MacroIdentifier', text: 'macro' },
{ type: 'Unknown', text: '!' },
{ type: 'Unknown', text: '@' },
{ type: 'Unknown', text: '#' },
{ type: 'Unknown', text: '%' },
{ type: 'MacroEnd', text: '}}' },
];
expect(tokens).toEqual(expectedTokens);
});
// {{ma!@#%ro}}
it('invalid chars in macro identifier are not parsed as valid macro identifier', async () => {
const input = '{{ma!@#%ro}}';
const tokens = await runLexerGetTokens(input);
const expectedTokens = [
{ type: 'MacroStart', text: '{{' },
{ type: 'MacroIdentifier', text: 'ma' },
{ type: 'Unknown', text: '!' },
{ type: 'Unknown', text: '@' },
{ type: 'Unknown', text: '#' },
{ type: 'Unknown', text: '%' },
{ type: 'Identifier', text: 'ro' },
{ type: 'MacroEnd', text: '}}' },
];
expect(tokens).toEqual(expectedTokens);
});
});
describe('Macro Nesting', () => {
// {{outerMacro {{innerMacro}}}}
// {{outerMacro {{innerMacro}}}}
it('should handle nested macros', async () => {
const input = '{{outerMacro {{innerMacro}}}}';
const tokens = await runLexerGetTokens(input);
@@ -192,6 +104,115 @@ describe('MacroLexer', () => {
});
});
describe('Macro Identifier', () => {
// {{ a }}
it('allow one-character macro identifiers', async () => {
const input = '{{ a }}';
const tokens = await runLexerGetTokens(input);
const expectedTokens = [
{ type: 'MacroStart', text: '{{' },
{ type: 'MacroIdentifier', text: 'a' },
{ type: 'MacroEnd', text: '}}' },
];
expect(tokens).toEqual(expectedTokens);
});
// {{ some macro }}
it('whitespaces between two valid identifiers will only capture the first as macro identifier', async () => {
const input = '{{ some macro }}';
const tokens = await runLexerGetTokens(input);
const expectedTokens = [
{ type: 'MacroStart', text: '{{' },
{ type: 'MacroIdentifier', text: 'some' },
{ type: 'Identifier', text: 'macro' },
{ type: 'MacroEnd', text: '}}' },
];
expect(tokens).toEqual(expectedTokens);
});
// {{my2cents}}
it('should allow numerics inside the macro identifier', async () => {
const input = '{{my2cents}}';
const tokens = await runLexerGetTokens(input);
const expectedTokens = [
{ type: 'MacroStart', text: '{{' },
{ type: 'MacroIdentifier', text: 'my2cents' },
{ type: 'MacroEnd', text: '}}' },
];
expect(tokens).toEqual(expectedTokens);
});
// {{SCREAM}}
it('should allow capslock macro', async () => {
const input = '{{SCREAM}}';
const tokens = await runLexerGetTokens(input);
const expectedTokens = [
{ type: 'MacroStart', text: '{{' },
{ type: 'MacroIdentifier', text: 'SCREAM' },
{ type: 'MacroEnd', text: '}}' },
];
expect(tokens).toEqual(expectedTokens);
});
// {{some-longer-macro}}
it('allow dashes in macro identifiers', async () => {
const input = '{{some-longer-macro}}';
const tokens = await runLexerGetTokens(input);
const expectedTokens = [
{ type: 'MacroStart', text: '{{' },
{ type: 'MacroIdentifier', text: 'some-longer-macro' },
{ type: 'MacroEnd', text: '}}' },
];
expect(tokens).toEqual(expectedTokens);
});
// {{macro!@#%}}
it('do not lex special characters as part of the macro identifier', async () => {
const input = '{{macro!@#%}}';
const { tokens, errors } = await runLexerGetTokensAndErrors(input);
const expectedErrors = [
{ message: 'unexpected character: ->!<- at offset: 7, skipped 4 characters.' },
];
expect(errors).toMatchObject(expectedErrors);
const expectedTokens = [
{ type: 'MacroStart', text: '{{' },
{ type: 'MacroIdentifier', text: 'macro' },
// Do not lex the wrong characters
{ type: 'MacroEnd', text: '}}' },
];
expect(tokens).toEqual(expectedTokens);
});
// {{ma!@#%ro}}
it('[Error] invalid chars in macro identifier are not parsed as valid macro identifier', async () => {
const input = '{{ma!@#%ro}}';
const { tokens, errors } = await runLexerGetTokensAndErrors(input);
const expectedErrors = [
{ message: 'unexpected character: ->!<- at offset: 4, skipped 6 characters.' },
];
expect(errors).toMatchObject(expectedErrors);
const expectedTokens = [
{ type: 'MacroStart', text: '{{' },
{ type: 'MacroIdentifier', text: 'ma' },
// Do not lex the wrong characters
{ type: 'MacroEnd', text: '}}' },
];
expect(tokens).toEqual(expectedTokens);
});
});
describe('Macro Arguments', () => {
// {{setvar::myVar::This is Sparta!}}
it('should tokenize macros with double colons arguments correctly', async () => {
@@ -710,6 +731,30 @@ describe('MacroLexer', () => {
{ type: 'Plaintext', text: '{ { not a macro } }' },
];
expect(tokens).toEqual(expectedTokens);
});
// invalid {{ 000 }} followed by correct {{ macro }}
it('valid macro still works after an invalid macro', async () => {
const input = 'invalid {{ 000 }} followed by correct {{ macro }}';
const { tokens, errors } = await runLexerGetTokensAndErrors(input);
const expectedErrors = [
{ message: 'unexpected character: ->0<- at offset: 11, skipped 3 characters.' },
];
expect(errors).toMatchObject(expectedErrors);
const expectedTokens = [
{ type: 'Plaintext', text: 'invalid ' },
{ type: 'MacroStart', text: '{{' },
// Do not capture '000' as anything, as it's a lexer error
{ type: 'MacroEnd', text: '}}' },
{ type: 'Plaintext', text: ' followed by correct ' },
{ type: 'MacroStart', text: '{{' },
{ type: 'MacroIdentifier', text: 'macro' },
{ type: 'MacroEnd', text: '}}' },
];
expect(tokens).toEqual(expectedTokens);
});
});
@@ -765,7 +810,7 @@ function getTestableTokens(result) {
const errors = result.errors;
const tokens = result.tokens
// Filter out the mode popper. We don't care aobut that for testing
.filter(token => token.tokenType.name !== 'EndMode')
//.filter(token => !['ModePopper', 'BeforeEnd'].includes(token.tokenType.name))
// Extract relevant properties from tokens for comparison
.map(token => ({
type: token.tokenType.name,