mirror of
https://github.com/SillyTavern/SillyTavern.git
synced 2025-06-05 21:59:27 +02:00
Allow legacy underscores in macro identifiers
This commit is contained in:
@@ -23,7 +23,7 @@ const Tokens = {
|
|||||||
// Separate macro identifier needed, that is similar to the global indentifier, but captures the actual macro "name"
|
// Separate macro identifier needed, that is similar to the global indentifier, but captures the actual macro "name"
|
||||||
// We need this, because this token is going to switch lexer mode, while the general identifier does not.
|
// We need this, because this token is going to switch lexer mode, while the general identifier does not.
|
||||||
Flags: createToken({ name: 'Macro.Flag', pattern: /[!?#~/.$]/ }),
|
Flags: createToken({ name: 'Macro.Flag', pattern: /[!?#~/.$]/ }),
|
||||||
Identifier: createToken({ name: 'Macro.Identifier', pattern: /[a-zA-Z][\w-]*/ }),
|
Identifier: createToken({ name: 'Macro.Identifier', pattern: /[a-zA-Z][\w-_]*/ }),
|
||||||
// At the end of an identifier, there has to be whitspace, or must be directly followed by colon/double-colon separator, output modifier or closing braces
|
// At the end of an identifier, there has to be whitspace, or must be directly followed by colon/double-colon separator, output modifier or closing braces
|
||||||
EndOfIdentifier: createToken({ name: 'Macro.EndOfIdentifier', pattern: /(?:\s+|(?=:{1,2})|(?=[|}]))/, group: Lexer.SKIPPED }),
|
EndOfIdentifier: createToken({ name: 'Macro.EndOfIdentifier', pattern: /(?:\s+|(?=:{1,2})|(?=[|}]))/, group: Lexer.SKIPPED }),
|
||||||
BeforeEnd: createToken({ name: 'Macro.BeforeEnd', pattern: /(?=\}\})/, group: Lexer.SKIPPED }),
|
BeforeEnd: createToken({ name: 'Macro.BeforeEnd', pattern: /(?=\}\})/, group: Lexer.SKIPPED }),
|
||||||
@@ -40,13 +40,13 @@ const Tokens = {
|
|||||||
|
|
||||||
Filter: {
|
Filter: {
|
||||||
Pipe: createToken({ name: 'Filter.Pipe', pattern: /(?<!\\)\|/ }),
|
Pipe: createToken({ name: 'Filter.Pipe', pattern: /(?<!\\)\|/ }),
|
||||||
Identifier: createToken({ name: 'Filter.Identifier', pattern: /[a-zA-Z][\w-]*/ }),
|
Identifier: createToken({ name: 'Filter.Identifier', pattern: /[a-zA-Z][\w-_]*/ }),
|
||||||
// At the end of an identifier, there has to be whitspace, or must be directly followed by colon/double-colon separator, output modifier or closing braces
|
// At the end of an identifier, there has to be whitspace, or must be directly followed by colon/double-colon separator, output modifier or closing braces
|
||||||
EndOfIdentifier: createToken({ name: 'Filter.EndOfIdentifier', pattern: /(?:\s+|(?=:{1,2})|(?=[|}]))/, group: Lexer.SKIPPED }),
|
EndOfIdentifier: createToken({ name: 'Filter.EndOfIdentifier', pattern: /(?:\s+|(?=:{1,2})|(?=[|}]))/, group: Lexer.SKIPPED }),
|
||||||
},
|
},
|
||||||
|
|
||||||
// All tokens that can be captured inside a macro
|
// All tokens that can be captured inside a macro
|
||||||
Identifier: createToken({ name: 'Identifier', pattern: /[a-zA-Z][\w-]*/ }),
|
Identifier: createToken({ name: 'Identifier', pattern: /[a-zA-Z][\w-_]*/ }),
|
||||||
WhiteSpace: createToken({ name: 'WhiteSpace', pattern: /\s+/, group: Lexer.SKIPPED }),
|
WhiteSpace: createToken({ name: 'WhiteSpace', pattern: /\s+/, group: Lexer.SKIPPED }),
|
||||||
|
|
||||||
// Capture unknown characters one by one, to still allow other tokens being matched once they are there
|
// Capture unknown characters one by one, to still allow other tokens being matched once they are there
|
||||||
|
@@ -174,6 +174,19 @@ describe('MacroLexer', () => {
|
|||||||
|
|
||||||
expect(tokens).toEqual(expectedTokens);
|
expect(tokens).toEqual(expectedTokens);
|
||||||
});
|
});
|
||||||
|
// {{legacy_macro}}
|
||||||
|
it('allow underscores as legacy in macro identifiers', async () => {
|
||||||
|
const input = '{{legacy_macro}}';
|
||||||
|
const tokens = await runLexerGetTokens(input);
|
||||||
|
|
||||||
|
const expectedTokens = [
|
||||||
|
{ type: 'Macro.Start', text: '{{' },
|
||||||
|
{ type: 'Macro.Identifier', text: 'legacy_macro' },
|
||||||
|
{ type: 'Macro.End', text: '}}' },
|
||||||
|
];
|
||||||
|
|
||||||
|
expect(tokens).toEqual(expectedTokens);
|
||||||
|
});
|
||||||
|
|
||||||
describe('Error Cases (Macro Identifier)', () => {
|
describe('Error Cases (Macro Identifier)', () => {
|
||||||
// {{macro!@#%}}
|
// {{macro!@#%}}
|
||||||
@@ -310,6 +323,22 @@ describe('MacroLexer', () => {
|
|||||||
|
|
||||||
expect(tokens).toEqual(expectedTokens);
|
expect(tokens).toEqual(expectedTokens);
|
||||||
});
|
});
|
||||||
|
// {{macro legacy_key=blah}}
|
||||||
|
it('should handle legacy argument name identifiers', async () => {
|
||||||
|
const input = '{{macro legacy_key=blah}}';
|
||||||
|
const tokens = await runLexerGetTokens(input);
|
||||||
|
|
||||||
|
const expectedTokens = [
|
||||||
|
{ type: 'Macro.Start', text: '{{' },
|
||||||
|
{ type: 'Macro.Identifier', text: 'macro' },
|
||||||
|
{ type: 'Identifier', text: 'legacy_key' },
|
||||||
|
{ type: 'Args.Equals', text: '=' },
|
||||||
|
{ type: 'Identifier', text: 'blah' },
|
||||||
|
{ type: 'Macro.End', text: '}}' },
|
||||||
|
];
|
||||||
|
|
||||||
|
expect(tokens).toEqual(expectedTokens);
|
||||||
|
});
|
||||||
// {{random "this" "and that" "and some more"}}
|
// {{random "this" "and that" "and some more"}}
|
||||||
it('should handle multiple unnamed arguments in quotation marks', async () => {
|
it('should handle multiple unnamed arguments in quotation marks', async () => {
|
||||||
const input = '{{random "this" "and that" "and some more"}}';
|
const input = '{{random "this" "and that" "and some more"}}';
|
||||||
|
Reference in New Issue
Block a user