From bd0736a11ba2069e1a69e9f2b000969fe6d0708f Mon Sep 17 00:00:00 2001 From: "tao.chen1" Date: Fri, 30 Apr 2021 19:04:59 +0800 Subject: [PATCH] =?UTF-8?q?=E9=87=8D=E6=9E=84=E8=AF=8D=E6=B3=95=E5=88=86?= =?UTF-8?q?=E6=9E=90=E5=99=A8=EF=BC=8C=E4=BD=BF=E7=94=A8=E6=A0=87=E8=AE=B0?= =?UTF-8?q?=E8=A1=A5=E5=85=85=E7=AE=97=E6=B3=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../element/action/DefaultSemanticParser.java | 2 +- .../com/sum/spirit/core/api/CharsHandler.java | 2 +- .../com/sum/spirit/core/api/LexerAction.java | 12 +++ .../core/lexer/AbstractCharsHandler.java | 10 +- ...orAction.java => AbstractCursorLexer.java} | 36 +------ .../sum/spirit/core/lexer/AbstractLexer.java | 79 ++++++++++++++ .../com/sum/spirit/core/lexer/CoreLexer.java | 100 ++---------------- .../core/lexer/action/BorderAction.java | 22 ++-- .../core/lexer/action/RegionAction.java | 68 ++++++------ .../spirit/core/lexer/action/SpaceAction.java | 28 +++++ .../core/lexer/action/SymbolAction.java | 20 ++-- .../core/lexer/entity/LexerContext.java | 18 ++-- .../spirit/core/lexer/entity/LexerResult.java | 18 ++++ .../sum/spirit/core/lexer/entity/Region.java | 4 + .../RegionUtils.java} | 56 +++++++--- .../sum/spirit/core/lexer/test/LexerTest.java | 46 ++++++++ 16 files changed, 304 insertions(+), 217 deletions(-) create mode 100644 spirit-core/spirit-core-lexer/src/main/java/com/sum/spirit/core/api/LexerAction.java rename spirit-core/spirit-core-lexer/src/main/java/com/sum/spirit/core/lexer/{action/CursorAction.java => AbstractCursorLexer.java} (48%) create mode 100644 spirit-core/spirit-core-lexer/src/main/java/com/sum/spirit/core/lexer/AbstractLexer.java create mode 100644 spirit-core/spirit-core-lexer/src/main/java/com/sum/spirit/core/lexer/action/SpaceAction.java create mode 100644 spirit-core/spirit-core-lexer/src/main/java/com/sum/spirit/core/lexer/entity/LexerResult.java rename spirit-core/spirit-core-lexer/src/main/java/com/sum/spirit/core/lexer/{action/AbstractLexerAction.java => utils/RegionUtils.java} (35%) diff --git a/spirit-core/spirit-core-element/src/main/java/com/sum/spirit/core/element/action/DefaultSemanticParser.java b/spirit-core/spirit-core-element/src/main/java/com/sum/spirit/core/element/action/DefaultSemanticParser.java index 1d4ccbbc..08e02204 100644 --- a/spirit-core/spirit-core-element/src/main/java/com/sum/spirit/core/element/action/DefaultSemanticParser.java +++ b/spirit-core/spirit-core-element/src/main/java/com/sum/spirit/core/element/action/DefaultSemanticParser.java @@ -63,7 +63,7 @@ public class DefaultSemanticParser extends AbstractSemanticParser { token.tokenType = getAccessTokenType(word); } - Assert.notNull(token.tokenType, "Token type cannot be null!"); + Assert.notNull(token.tokenType, "Token type cannot be null!word:[" + word + "]"); } public void setTokenValue(String word, Token token) { diff --git a/spirit-core/spirit-core-lexer/src/main/java/com/sum/spirit/core/api/CharsHandler.java b/spirit-core/spirit-core-lexer/src/main/java/com/sum/spirit/core/api/CharsHandler.java index 95fce938..b31e4be6 100644 --- a/spirit-core/spirit-core-lexer/src/main/java/com/sum/spirit/core/api/CharsHandler.java +++ b/spirit-core/spirit-core-lexer/src/main/java/com/sum/spirit/core/api/CharsHandler.java @@ -4,6 +4,6 @@ import com.sum.spirit.core.lexer.entity.CharsContext; public interface CharsHandler { - void handle(CharsContext context, StringBuilder builder, CharAction action); + void handle(CharsContext context, StringBuilder builder); } diff --git a/spirit-core/spirit-core-lexer/src/main/java/com/sum/spirit/core/api/LexerAction.java b/spirit-core/spirit-core-lexer/src/main/java/com/sum/spirit/core/api/LexerAction.java new file mode 100644 index 00000000..59c84cb6 --- /dev/null +++ b/spirit-core/spirit-core-lexer/src/main/java/com/sum/spirit/core/api/LexerAction.java @@ -0,0 +1,12 @@ +package com.sum.spirit.core.api; + +import com.sum.spirit.core.lexer.entity.CharEvent; +import com.sum.spirit.core.lexer.entity.LexerResult; + +public interface LexerAction { + + boolean isTrigger(CharEvent event); + + LexerResult handle(CharEvent event); + +} diff --git a/spirit-core/spirit-core-lexer/src/main/java/com/sum/spirit/core/lexer/AbstractCharsHandler.java b/spirit-core/spirit-core-lexer/src/main/java/com/sum/spirit/core/lexer/AbstractCharsHandler.java index a9993aa0..59bc6c75 100644 --- a/spirit-core/spirit-core-lexer/src/main/java/com/sum/spirit/core/lexer/AbstractCharsHandler.java +++ b/spirit-core/spirit-core-lexer/src/main/java/com/sum/spirit/core/lexer/AbstractCharsHandler.java @@ -8,18 +8,14 @@ import com.sum.spirit.core.lexer.entity.CharsContext; public abstract class AbstractCharsHandler implements CharsHandler, CharAction { @Override - public void handle(CharsContext context, StringBuilder builder, CharAction action) { + public void handle(CharsContext context, StringBuilder builder) { for (; context.index < builder.length(); context.index++) { char ch = builder.charAt(context.index); CharEvent event = new CharEvent(context, ch); - if (action.isTrigger(event)) { - action.handle(event); + if (this.isTrigger(event)) { + this.handle(event); } } } - public void handle(CharsContext context, StringBuilder builder) { - handle(context, builder, this); - } - } diff --git a/spirit-core/spirit-core-lexer/src/main/java/com/sum/spirit/core/lexer/action/CursorAction.java b/spirit-core/spirit-core-lexer/src/main/java/com/sum/spirit/core/lexer/AbstractCursorLexer.java similarity index 48% rename from spirit-core/spirit-core-lexer/src/main/java/com/sum/spirit/core/lexer/action/CursorAction.java rename to spirit-core/spirit-core-lexer/src/main/java/com/sum/spirit/core/lexer/AbstractCursorLexer.java index c3af44dd..b7bb00f4 100644 --- a/spirit-core/spirit-core-lexer/src/main/java/com/sum/spirit/core/lexer/action/CursorAction.java +++ b/spirit-core/spirit-core-lexer/src/main/java/com/sum/spirit/core/lexer/AbstractCursorLexer.java @@ -1,27 +1,11 @@ -package com.sum.spirit.core.lexer.action; +package com.sum.spirit.core.lexer; import com.sum.spirit.common.utils.LineUtils; -import com.sum.spirit.core.api.CharAction; import com.sum.spirit.core.lexer.entity.CharEvent; import com.sum.spirit.core.lexer.entity.LexerContext; -public class CursorAction implements CharAction { - - public CharAction action; - - public CursorAction(CharAction action) { - this.action = action; - } +public abstract class AbstractCursorLexer extends AbstractLexer { - @Override - public boolean isTrigger(CharEvent event) { - return true; - } - - /** - * -接续字符和刷新字符，是游标敏感的，用以标记区域的起始位置 - * -先进行弹栈，再刷新startIndex为-1，例如“[”这种情况 - */ @Override public void handle(CharEvent event) { LexerContext context = (LexerContext) event.context; @@ -29,30 +13,16 @@ public class CursorAction implements CharAction { if ((context.startIndex < 0 && isContinuous(ch)) || isRefreshed(ch)) { context.startIndex = context.index; } - if (action.isTrigger(event)) { - action.handle(event); - } + super.handle(event); if (!isContinuous(ch)) { context.startIndex = -1; } } - /** - * -是否接续字符 - * - * @param ch - * @return - */ public boolean isContinuous(char ch) { return LineUtils.isLetter(ch) || ch == '@' || ch == '.'; } - /** - * -是否刷新字符 - * - * @param ch - * @return - */ public boolean isRefreshed(char ch) { return ch == '.'; } diff --git a/spirit-core/spirit-core-lexer/src/main/java/com/sum/spirit/core/lexer/AbstractLexer.java b/spirit-core/spirit-core-lexer/src/main/java/com/sum/spirit/core/lexer/AbstractLexer.java new file mode 100644 index 00000000..19dff636 --- /dev/null +++ b/spirit-core/spirit-core-lexer/src/main/java/com/sum/spirit/core/lexer/AbstractLexer.java @@ -0,0 +1,79 @@ +package com.sum.spirit.core.lexer; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import org.springframework.beans.factory.annotation.Autowired; + +import com.sum.spirit.common.enums.LiteralEnum; +import com.sum.spirit.common.enums.TypeEnum; +import com.sum.spirit.core.api.Lexer; +import com.sum.spirit.core.api.LexerAction; +import com.sum.spirit.core.lexer.entity.CharEvent; +import com.sum.spirit.core.lexer.entity.CharsContext; +import com.sum.spirit.core.lexer.entity.LexerContext; +import com.sum.spirit.core.lexer.entity.LexerResult; +import com.sum.spirit.core.lexer.entity.Region; +import com.sum.spirit.core.lexer.utils.RegionUtils; +import com.sum.spirit.core.lexer.entity.LexerResult.State; + +public abstract class AbstractLexer extends AbstractCharsHandler implements Lexer { + + @Autowired + public List actions; + + @Override + public void handle(CharsContext context, StringBuilder builder) { + super.handle(context, builder); + completeRegions(context, builder);// 使用标记收集算法后，补全未标记的部分 + } + + public void completeRegions(CharsContext context, StringBuilder builder) { + LexerContext lexerContext = (LexerContext) context; + if (lexerContext.words == null) { + List regions = lexerContext.regions; + Set completedRegions = new HashSet(); + regions = RegionUtils.completeRegions(builder, regions, region -> completedRegions.add(region)); + lexerContext.words = RegionUtils.subRegions(builder, regions, (words, region, text) -> addToWords(words, completedRegions, region, text)); + } + } + + public void addToWords(List words, Set completedRegions, Region region, String text) { + if (completedRegions.contains(region)) { + if (text.indexOf(".") > 0 && !LiteralEnum.isDouble(text) && !TypeEnum.isTypeEnd(text)) { + List subWords = Arrays.asList(text.replaceAll("\\.", " .").split(" ")); + words.addAll(subWords); + } else { + words.add(text); + } + } else { + words.add(text); + } + } + + @Override + public boolean isTrigger(CharEvent event) { + return true; + } + + @Override + public void handle(CharEvent event) { + for (LexerAction action : actions) { + if (action.isTrigger(event)) { + LexerResult result = action.handle(event); + if (result.region != null) { + LexerContext context = (LexerContext) event.context; + context.regions.add(result.region); + } + if (result.state == State.TRY) { + continue; + } else if (result.state == State.SKIP) { + break; + } + } + } + } + +} diff --git a/spirit-core/spirit-core-lexer/src/main/java/com/sum/spirit/core/lexer/CoreLexer.java b/spirit-core/spirit-core-lexer/src/main/java/com/sum/spirit/core/lexer/CoreLexer.java index 4afde66f..75160145 100644 --- a/spirit-core/spirit-core-lexer/src/main/java/com/sum/spirit/core/lexer/CoreLexer.java +++ b/spirit-core/spirit-core-lexer/src/main/java/com/sum/spirit/core/lexer/CoreLexer.java @@ -1,72 +1,34 @@ package com.sum.spirit.core.lexer; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; -import java.util.Map; -import org.springframework.beans.factory.InitializingBean; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.DependsOn; import org.springframework.stereotype.Component; -import com.sum.spirit.common.enums.LiteralEnum; -import com.sum.spirit.common.enums.TypeEnum; -import com.sum.spirit.common.utils.LineUtils; -import com.sum.spirit.common.utils.SpringUtils; -import com.sum.spirit.core.api.CharAction; -import com.sum.spirit.core.api.Lexer; -import com.sum.spirit.core.lexer.action.AbstractLexerAction; import com.sum.spirit.core.lexer.action.BorderAction; -import com.sum.spirit.core.lexer.action.CursorAction; -import com.sum.spirit.core.lexer.entity.CharEvent; import com.sum.spirit.core.lexer.entity.LexerContext; import cn.hutool.core.lang.Assert; @Component -@DependsOn("springUtils") -public class CoreLexer extends AbstractCharsHandler implements Lexer, InitializingBean { +public class CoreLexer extends AbstractCursorLexer { - public List actions; - @Autowired - public BorderAction borderAction; - - @Override - public void afterPropertiesSet() throws Exception { - actions = SpringUtils.getBeansByExcludedTypes(AbstractLexerAction.class, BorderAction.class); - } - - /** - * -1.整体替换语句中的某些区域 - * -2.去掉多余的空格 - * -3.利用空格，进行拆分 - * -4.继续拆分带有“.”的单词 - * -5.还原被替换的单词 - */ @Override public List getWords(String text) { - Assert.notBlank(text, "text cannot be blank!"); + Assert.notBlank(text, "Text cannot be blank!"); StringBuilder builder = new StringBuilder(text.trim()); LexerContext context = new LexerContext(builder); - handle(context, builder, new CursorAction(this)); - text = LineUtils.mergeSpaces(context.builder.toString()); - List words = new ArrayList<>(Arrays.asList(text.split(" "))); - splitWords(words); - restoreWords(words, context.replacedStrs); - return words; + handle(context, builder); + return context.words; } - /** - * -根据弹栈规则，拆分分隔符和单词 - */ @Override public List getSubWords(String text, Character... splitChars) { - Assert.notBlank(text, "text cannot be blank!"); + Assert.notBlank(text, "Text cannot be blank!"); StringBuilder builder = new StringBuilder(text.trim()); - LexerContext context = new LexerContext(builder, splitChars); - handle(context, builder, new CursorAction(borderAction)); - Assert.notNull(context.words, "words of context cannot be null!"); + LexerContext context = new LexerContext(builder, BorderAction.PROFILE, splitChars); + handle(context, builder); + Assert.notNull(context.words, "Words of context cannot be null!"); List words = new ArrayList<>(); for (String word : context.words) { if (word.length() == 1) { @@ -78,50 +40,4 @@ public class CoreLexer extends AbstractCharsHandler implements Lexer, Initializi return words; } - /** - * -继续拆分带有“.”的单词 - * - * @param words - */ - public void splitWords(List words) { - for (int index = 0; index < words.size(); index++) { - String word = words.get(index); - if (word.indexOf(".") > 0 && !LiteralEnum.isDouble(word) && !TypeEnum.isTypeEnd(word)) { - List subWords = Arrays.asList(word.replaceAll("\\.", " .").split(" ")); - words.remove(index); - words.addAll(index, subWords); - } - } - } - - /** - * -替换之前被替换的单词 - * - * @param words - * @param replacedStrs - */ - public void restoreWords(List words, Map replacedStrs) { - for (int index = 0; index < words.size(); index++) { - String str = replacedStrs.get(words.get(index)); - if (str != null) { - words.set(index, str); - } - } - } - - @Override - public boolean isTrigger(CharEvent event) { - return true; - } - - @Override - public void handle(CharEvent event) { - for (CharAction action : actions) { - if (action.isTrigger(event)) { - action.handle(event); - return; - } - } - } - } diff --git a/spirit-core/spirit-core-lexer/src/main/java/com/sum/spirit/core/lexer/action/BorderAction.java b/spirit-core/spirit-core-lexer/src/main/java/com/sum/spirit/core/lexer/action/BorderAction.java index 07ee63bf..c636eafe 100644 --- a/spirit-core/spirit-core-lexer/src/main/java/com/sum/spirit/core/lexer/action/BorderAction.java +++ b/spirit-core/spirit-core-lexer/src/main/java/com/sum/spirit/core/lexer/action/BorderAction.java @@ -3,22 +3,30 @@ package com.sum.spirit.core.lexer.action; import java.util.ArrayList; import java.util.List; +import org.springframework.core.annotation.Order; import org.springframework.stereotype.Component; import com.sum.spirit.common.utils.Splitter; import com.sum.spirit.core.lexer.entity.CharEvent; import com.sum.spirit.core.lexer.entity.LexerContext; +import com.sum.spirit.core.lexer.entity.LexerResult; import com.sum.spirit.core.lexer.entity.Region; +import com.sum.spirit.core.lexer.entity.LexerResult.State; @Component +@Order(-100) public class BorderAction extends RegionAction { - /** - * -该类对弹栈行为进行了一定改造，主要对分隔符进行拆分，并将索引直接置为结束 - */ + public static final String PROFILE = "BORDER_ACTION"; + @Override - public void pushStack(CharEvent event, List regions, String markName) { + public boolean isTrigger(CharEvent event) { + LexerContext context = (LexerContext) event.context; + return PROFILE.equals(context.profile) && super.isTrigger(event); + } + @Override + public LexerResult pushStack(CharEvent event, List regions) { LexerContext context = (LexerContext) event.context; StringBuilder builder = context.builder; List splitChars = context.splitChars; @@ -35,11 +43,7 @@ public class BorderAction extends RegionAction { context.words = Splitter.splitByIndexsTrimRemain(builder.toString(), indexs); context.index = builder.length(); - } - - @Override - public void resetIndex(CharEvent event) { - // ignore + return new LexerResult(State.SKIP, null); } } diff --git a/spirit-core/spirit-core-lexer/src/main/java/com/sum/spirit/core/lexer/action/RegionAction.java b/spirit-core/spirit-core-lexer/src/main/java/com/sum/spirit/core/lexer/action/RegionAction.java index 61531bcf..7087bfa6 100644 --- a/spirit-core/spirit-core-lexer/src/main/java/com/sum/spirit/core/lexer/action/RegionAction.java +++ b/spirit-core/spirit-core-lexer/src/main/java/com/sum/spirit/core/lexer/action/RegionAction.java @@ -7,18 +7,18 @@ import org.springframework.stereotype.Component; import com.sum.spirit.common.enums.TypeEnum; import com.sum.spirit.common.utils.ListUtils; +import com.sum.spirit.core.api.LexerAction; import com.sum.spirit.core.lexer.entity.CharEvent; import com.sum.spirit.core.lexer.entity.LexerContext; +import com.sum.spirit.core.lexer.entity.LexerResult; +import com.sum.spirit.core.lexer.entity.LexerResult.State; import com.sum.spirit.core.lexer.entity.Region; +import com.sum.spirit.core.lexer.utils.RegionUtils; @Component -@Order(-100) -public class RegionAction extends AbstractLexerAction { +@Order(-80) +public class RegionAction implements LexerAction { - /** - * -到达结尾，则直接返回false - * -泛型前缀一般都以大写字母开始 - */ @Override public boolean isTrigger(CharEvent event) { LexerContext context = (LexerContext) event.context; @@ -43,73 +43,65 @@ public class RegionAction extends AbstractLexerAction { return false; } - /** - * str[0] => str [0], String[0] => String[0], String[]{"str", "str"} => String[]{"str", "str"} - */ @Override - public void handle(CharEvent event) { - + public LexerResult handle(CharEvent event) { LexerContext context = (LexerContext) event.context; StringBuilder builder = context.builder; char ch = event.ch; if (ch == '"') { - Region region = findRegion(builder, context.index, '"', '"'); - pushStack(event, ListUtils.toListNonNull(region), "@str"); + Region region = RegionUtils.findRegion(builder, context.index, '"', '"'); + return pushStack(event, ListUtils.toListNonNull(region)); } else if (ch == '\'') { - Region region = findRegion(builder, context.index, '\'', '\''); - pushStack(event, ListUtils.toListNonNull(region), "@char"); + Region region = RegionUtils.findRegion(builder, context.index, '\'', '\''); + return pushStack(event, ListUtils.toListNonNull(region)); } else if (ch == '{') { - Region region = findRegion(builder, context.index, '{', '}'); - pushStack(event, ListUtils.toListNonNull(region), "@map"); + Region region = RegionUtils.findRegion(builder, context.index, '{', '}'); + return pushStack(event, ListUtils.toListNonNull(region)); } else if (ch == '(') { Region region0 = context.startIndex >= 0 ? new Region(context.startIndex, context.index) : null; - Region region1 = findRegion(builder, context.index, '(', ')'); - pushStack(event, ListUtils.toListNonNull(region0, region1), "@invoke_like"); - resetIndex(event); + Region region1 = RegionUtils.findRegion(builder, context.index, '(', ')'); + return pushStack(event, ListUtils.toListNonNull(region0, region1)); } else if (ch == '[') { Region region0 = context.startIndex >= 0 ? new Region(context.startIndex, context.index) : null; - if (region0 != null && !TypeEnum.isTypePrefix(subRegion(builder, region0))) { + if (region0 != null && !TypeEnum.isTypePrefix(RegionUtils.subRegion(builder, region0))) { region0 = null; } - Region region1 = findRegion(builder, context.index, '[', ']'); + Region region1 = RegionUtils.findRegion(builder, context.index, '[', ']'); Region region2 = null; if (region0 != null) { if (isCharAt(builder, region1.endIndex, '{')) { - region2 = findRegion(builder, region1.endIndex, '{', '}'); + region2 = RegionUtils.findRegion(builder, region1.endIndex, '{', '}'); } else if (isCharAt(builder, region1.endIndex, ' ') && isCharAt(builder, region1.endIndex + 1, '{')) { - region2 = findRegion(builder, region1.endIndex + 1, '{', '}'); + region2 = RegionUtils.findRegion(builder, region1.endIndex + 1, '{', '}'); } } - pushStack(event, ListUtils.toListNonNull(region0, region1, region2), "@array_like"); - resetIndex(event); + return pushStack(event, ListUtils.toListNonNull(region0, region1, region2)); } else if (ch == '<') { Region region0 = context.startIndex >= 0 ? new Region(context.startIndex, context.index) : null; - Region region1 = findRegion(builder, context.index, '<', '>'); - Region region2 = isCharAt(builder, region1.endIndex, '(') ? findRegion(builder, region1.endIndex, '(', ')') : null; - pushStack(event, ListUtils.toListNonNull(region0, region1, region2), "@generic"); - resetIndex(event); + Region region1 = RegionUtils.findRegion(builder, context.index, '<', '>'); + Region region2 = isCharAt(builder, region1.endIndex, '(') ? RegionUtils.findRegion(builder, region1.endIndex, '(', ')') : null; + return pushStack(event, ListUtils.toListNonNull(region0, region1, region2)); } - } - public boolean isCharAt(StringBuilder builder, int index, char ch) { - return index < builder.length() && builder.charAt(index) == ch; + throw new RuntimeException("Can't handle the scene!"); } - public void pushStack(CharEvent event, List regions, String markName) { + public LexerResult pushStack(CharEvent event, List regions) { LexerContext context = (LexerContext) event.context; - replaceRegion(context.builder, mergeRegions(regions), markName + context.nameCount++, context.replacedStrs); + Region mergedRegion = RegionUtils.mergeRegions(regions); + context.index = mergedRegion.endIndex - 1; + return new LexerResult(State.SKIP, mergedRegion); } - public void resetIndex(CharEvent event) { - LexerContext context = (LexerContext) event.context; - context.index = context.startIndex >= 0 ? context.startIndex : context.index; + public boolean isCharAt(StringBuilder builder, int index, char ch) { + return index < builder.length() && builder.charAt(index) == ch; } } diff --git a/spirit-core/spirit-core-lexer/src/main/java/com/sum/spirit/core/lexer/action/SpaceAction.java b/spirit-core/spirit-core-lexer/src/main/java/com/sum/spirit/core/lexer/action/SpaceAction.java new file mode 100644 index 00000000..d737680d --- /dev/null +++ b/spirit-core/spirit-core-lexer/src/main/java/com/sum/spirit/core/lexer/action/SpaceAction.java @@ -0,0 +1,28 @@ +package com.sum.spirit.core.lexer.action; + +import org.springframework.core.annotation.Order; +import org.springframework.stereotype.Component; + +import com.sum.spirit.core.api.LexerAction; +import com.sum.spirit.core.lexer.entity.CharEvent; +import com.sum.spirit.core.lexer.entity.LexerContext; +import com.sum.spirit.core.lexer.entity.LexerResult; +import com.sum.spirit.core.lexer.entity.Region; +import com.sum.spirit.core.lexer.entity.LexerResult.State; + +@Component +@Order(-40) +public class SpaceAction implements LexerAction { + + @Override + public boolean isTrigger(CharEvent event) { + return event.ch == ' '; + } + + @Override + public LexerResult handle(CharEvent event) { + LexerContext context = (LexerContext) event.context; + return new LexerResult(State.SKIP, new Region(context.index, context.index + 1)); + } + +} diff --git a/spirit-core/spirit-core-lexer/src/main/java/com/sum/spirit/core/lexer/action/SymbolAction.java b/spirit-core/spirit-core-lexer/src/main/java/com/sum/spirit/core/lexer/action/SymbolAction.java index 33ab6ae0..6b71b27c 100644 --- a/spirit-core/spirit-core-lexer/src/main/java/com/sum/spirit/core/lexer/action/SymbolAction.java +++ b/spirit-core/spirit-core-lexer/src/main/java/com/sum/spirit/core/lexer/action/SymbolAction.java @@ -4,25 +4,24 @@ import org.springframework.core.annotation.Order; import org.springframework.stereotype.Component; import com.sum.spirit.common.enums.SymbolEnum; +import com.sum.spirit.core.api.LexerAction; import com.sum.spirit.core.lexer.entity.CharEvent; import com.sum.spirit.core.lexer.entity.LexerContext; +import com.sum.spirit.core.lexer.entity.LexerResult; import com.sum.spirit.core.lexer.entity.Region; +import com.sum.spirit.core.lexer.entity.LexerResult.State; @Component -@Order(-80) -public class SymbolAction extends AbstractLexerAction { +@Order(-60) +public class SymbolAction implements LexerAction { @Override public boolean isTrigger(CharEvent event) { return SymbolEnum.isSymbolChar(event.ch); } - /** - * -尝试拆分双字符和单字符符号 - */ @Override - public void handle(CharEvent event) { - + public LexerResult handle(CharEvent event) { LexerContext context = (LexerContext) event.context; StringBuilder builder = context.builder; @@ -30,16 +29,15 @@ public class SymbolAction extends AbstractLexerAction { String str = builder.substring(context.index, context.index + 2); if (SymbolEnum.isDoubleSymbol(str)) { Region region = new Region(context.index, context.index + 2); - replaceRegion(builder, region, "@symbol" + context.nameCount++, context.replacedStrs); - return; + context.index++;// 符合条件，则跳过一个单位 + return new LexerResult(State.SKIP, region); } } String str = builder.substring(context.index, context.index + 1); if (SymbolEnum.isSingleSymbol(str)) { Region region = new Region(context.index, context.index + 1); - replaceRegion(builder, region, "@symbol" + context.nameCount++, context.replacedStrs); - return; + return new LexerResult(State.SKIP, region); } throw new RuntimeException("Unable to process symbol!"); diff --git a/spirit-core/spirit-core-lexer/src/main/java/com/sum/spirit/core/lexer/entity/LexerContext.java b/spirit-core/spirit-core-lexer/src/main/java/com/sum/spirit/core/lexer/entity/LexerContext.java index 5731cfa7..77fa55ea 100644 --- a/spirit-core/spirit-core-lexer/src/main/java/com/sum/spirit/core/lexer/entity/LexerContext.java +++ b/spirit-core/spirit-core-lexer/src/main/java/com/sum/spirit/core/lexer/entity/LexerContext.java @@ -1,27 +1,29 @@ package com.sum.spirit.core.lexer.entity; +import java.util.ArrayList; import java.util.Arrays; -import java.util.HashMap; import java.util.List; -import java.util.Map; import lombok.AllArgsConstructor; import lombok.NoArgsConstructor; -import lombok.NonNull; @NoArgsConstructor @AllArgsConstructor public class LexerContext extends CharsContext { - public List splitChars; - public int nameCount; - @NonNull - public Map replacedStrs = new HashMap<>(); public int startIndex = -1; + public List regions = new ArrayList<>(); + public String profile; + public List splitChars; public List words; - public LexerContext(StringBuilder builder, Character... splitChars) { + public LexerContext(StringBuilder builder) { + this.builder = builder; + } + + public LexerContext(StringBuilder builder, String profile, Character... splitChars) { this.builder = builder; + this.profile = profile; this.splitChars = Arrays.asList(splitChars); } diff --git a/spirit-core/spirit-core-lexer/src/main/java/com/sum/spirit/core/lexer/entity/LexerResult.java b/spirit-core/spirit-core-lexer/src/main/java/com/sum/spirit/core/lexer/entity/LexerResult.java new file mode 100644 index 00000000..f738734d --- /dev/null +++ b/spirit-core/spirit-core-lexer/src/main/java/com/sum/spirit/core/lexer/entity/LexerResult.java @@ -0,0 +1,18 @@ +package com.sum.spirit.core.lexer.entity; + +import lombok.AllArgsConstructor; +import lombok.NoArgsConstructor; + +@NoArgsConstructor +@AllArgsConstructor +public class LexerResult { + + public enum State { + TRY, SKIP + } + + public State state; + + public Region region; + +} diff --git a/spirit-core/spirit-core-lexer/src/main/java/com/sum/spirit/core/lexer/entity/Region.java b/spirit-core/spirit-core-lexer/src/main/java/com/sum/spirit/core/lexer/entity/Region.java index 1e63f5f1..8b95018f 100644 --- a/spirit-core/spirit-core-lexer/src/main/java/com/sum/spirit/core/lexer/entity/Region.java +++ b/spirit-core/spirit-core-lexer/src/main/java/com/sum/spirit/core/lexer/entity/Region.java @@ -16,4 +16,8 @@ public class Region { return endIndex - startIndex; } + public boolean contains(int index) { + return startIndex <= index && index < endIndex; + } + } diff --git a/spirit-core/spirit-core-lexer/src/main/java/com/sum/spirit/core/lexer/action/AbstractLexerAction.java b/spirit-core/spirit-core-lexer/src/main/java/com/sum/spirit/core/lexer/utils/RegionUtils.java similarity index 35% rename from spirit-core/spirit-core-lexer/src/main/java/com/sum/spirit/core/lexer/action/AbstractLexerAction.java rename to spirit-core/spirit-core-lexer/src/main/java/com/sum/spirit/core/lexer/utils/RegionUtils.java index 0bc1b948..8ef622e0 100644 --- a/spirit-core/spirit-core-lexer/src/main/java/com/sum/spirit/core/lexer/action/AbstractLexerAction.java +++ b/spirit-core/spirit-core-lexer/src/main/java/com/sum/spirit/core/lexer/utils/RegionUtils.java @@ -1,49 +1,71 @@ -package com.sum.spirit.core.lexer.action; +package com.sum.spirit.core.lexer.utils; +import java.util.ArrayList; import java.util.List; -import java.util.Map; import org.apache.commons.lang3.StringUtils; import com.sum.spirit.common.utils.LineUtils; import com.sum.spirit.common.utils.ListUtils; -import com.sum.spirit.core.api.CharAction; import com.sum.spirit.core.lexer.entity.Region; import cn.hutool.core.lang.Assert; -public abstract class AbstractLexerAction implements CharAction { +public class RegionUtils { - public Region findRegion(StringBuilder builder, int fromIndex, char leftChar, char rightChar) { + public static Region findRegion(StringBuilder builder, int fromIndex, char leftChar, char rightChar) { int endIndex = LineUtils.findEndIndex(builder, fromIndex, leftChar, rightChar); return endIndex != -1 ? new Region(fromIndex, endIndex + 1) : null; } - public String subRegion(StringBuilder builder, Region region) { + public static String subRegion(StringBuilder builder, Region region) { return builder.substring(region.startIndex, region.endIndex); } - public Region mergeRegions(List regions) { + public static Region mergeRegions(List regions) { Assert.notEmpty(regions, "The regions cannot be empty!"); Region startRegion = ListUtils.findOneByScore(regions, region -> 0 - region.startIndex); Region endRegion = ListUtils.findOneByScore(regions, region -> region.endIndex); return new Region(startRegion.startIndex, endRegion.endIndex); } - public String replaceRegion(StringBuilder builder, Region region, String markName) { - if (region == null) { - return null; + public static List completeRegions(StringBuilder builder, List regions, Consumer consumer) { + List newRegions = new ArrayList<>(); + int lastIndex = 0; + for (Region region : regions) { + if (region.startIndex > lastIndex) { + Region newRegion = new Region(lastIndex, region.startIndex); + newRegions.add(newRegion); + consumer.accept(newRegion); + } + newRegions.add(region); + lastIndex = region.endIndex; } - String content = builder.substring(region.startIndex, region.endIndex); - builder.replace(region.startIndex, region.endIndex, " " + markName + " "); - return content; + if (lastIndex < builder.length()) { + Region newRegion = new Region(lastIndex, builder.length()); + newRegions.add(newRegion); + consumer.accept(newRegion); + } + return newRegions; } - public void replaceRegion(StringBuilder builder, Region region, String markName, Map replacedStrs) { - String content = replaceRegion(builder, region, markName); - if (StringUtils.isNotBlank(content) && replacedStrs != null) { - replacedStrs.put(markName, content); + public static List subRegions(StringBuilder builder, List regions, Consumer0 consumer) { + List words = new ArrayList<>(); + for (Region region : regions) { + String text = subRegion(builder, region); + if (StringUtils.isNotBlank(text)) { + consumer.accept(words, region, text.trim()); + } } + return words; + } + + public static interface Consumer { + void accept(T t); + } + + public static interface Consumer0 { + void accept(List words, T t, String text); } } diff --git a/spirit-core/spirit-core-lexer/src/test/java/com/sum/spirit/core/lexer/test/LexerTest.java b/spirit-core/spirit-core-lexer/src/test/java/com/sum/spirit/core/lexer/test/LexerTest.java index bf9ca255..f9e9c5ce 100644 --- a/spirit-core/spirit-core-lexer/src/test/java/com/sum/spirit/core/lexer/test/LexerTest.java +++ b/spirit-core/spirit-core-lexer/src/test/java/com/sum/spirit/core/lexer/test/LexerTest.java @@ -335,4 +335,50 @@ public class LexerTest { assertEquals(words.get(count++), ")"); } + @Test + @DisplayName("MY_TEST") + public void test0022() { + String text = "xxxxG_Alias=\"Clock moved backwards.G_Alias to generate id for %d milliseconds\""; + List words = lexer.getWords(text); + log.info(words.toString()); + assertTrue(words.size() == 3); + int count = 0; + assertEquals(words.get(count++), "xxxxG_Alias"); + assertEquals(words.get(count++), "="); + assertEquals(words.get(count++), "\"Clock moved backwards.G_Alias to generate id for %d milliseconds\""); + } + + @Test + @DisplayName("MY_TEST1") + public void test0023() { + String text = "if s==\"hello\"{"; + List words = lexer.getWords(text); + log.info(words.toString()); + assertTrue(words.size() == 5); + int count = 0; + assertEquals(words.get(count++), "if"); + assertEquals(words.get(count++), "s"); + assertEquals(words.get(count++), "=="); + assertEquals(words.get(count++), "\"hello\""); + assertEquals(words.get(count++), "{"); + } + + @Test + @DisplayName("MY_TEST2") + public void test0024() { + String text = "b=father.child.father.child.father.name"; + List words = lexer.getWords(text); + log.info(words.toString()); + assertTrue(words.size() == 8); + int count = 0; + assertEquals(words.get(count++), "b"); + assertEquals(words.get(count++), "="); + assertEquals(words.get(count++), "father"); + assertEquals(words.get(count++), ".child"); + assertEquals(words.get(count++), ".father"); + assertEquals(words.get(count++), ".child"); + assertEquals(words.get(count++), ".father"); + assertEquals(words.get(count++), ".name"); + } + } -- Gitee