Commit f057df76 by 李传峰

remove ik analyzer

parent fa563b2e
package com.pcloud.book.es.biz.impl;
import org.apache.commons.lang3.StringUtils;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.wltea.analyzer.lucene.IKAnalyzer;
import java.io.IOException;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.List;
/**
* 分词工具
*/
public class IKAnalyzerUtil {
private static final Logger LOGGER = LoggerFactory.getLogger(IKAnalyzerUtil.class);
/**
* 通配符
*/
public static final char WILDCARD_CHAR = '*';
/**
* 前后加通配符
* @param text 内容
* @return
*/
public static String aroundWildcard(String text) {
return WILDCARD_CHAR + text + WILDCARD_CHAR;
}
/**
* 解析成分词后通配字符串
* @param text 内容
* @return String,*通配
*/
public static String parseWildcardString(String text) {
if (StringUtils.isBlank(text)) {
return text;
}
return StringUtils.join(analyzerText(text), WILDCARD_CHAR);
}
/**
* 分词
* @param text 内容
* @return List<分词>
*/
public static List<String> analyzerText(String text) {
List<String> result = new ArrayList<>();
TokenStream ts = null;
try (IKAnalyzer analyzer = new IKAnalyzer(true)) {
ts = analyzer.tokenStream("text", new StringReader(text));
CharTermAttribute term = ts.addAttribute(CharTermAttribute.class);
ts.reset();
while (ts.incrementToken()) {
result.add(term.toString());
}
ts.end();
} catch (IOException ioe) {
result.add(text);
LOGGER.warn("analyzer text exception,text={}", text, ioe);
} finally {
closeTokenStream(text, ts);
}
return result;
}
private static void closeTokenStream(String text, TokenStream ts) {
if (ts != null) {
try {
ts.close();
} catch (IOException ioe) {
LOGGER.warn("analyzer text, close stream exception,text={}", text, ioe);
}
}
}
}
...@@ -3,12 +3,8 @@ package com.pcloud.book.test; ...@@ -3,12 +3,8 @@ package com.pcloud.book.test;
import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSON;
import com.pcloud.book.BookApplication; import com.pcloud.book.BookApplication;
import com.pcloud.book.es.biz.ESBookAndAdviserBiz; import com.pcloud.book.es.biz.ESBookAndAdviserBiz;
import com.pcloud.book.es.biz.impl.IKAnalyzerUtil;
import com.pcloud.book.es.entity.ESBookAndAdviser; import com.pcloud.book.es.entity.ESBookAndAdviser;
import com.pcloud.book.group.dao.BookGroupServeDao;
import com.pcloud.book.group.vo.BookGroupServeCountVO;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.StopWatch; import org.apache.commons.lang3.time.StopWatch;
import org.junit.Test; import org.junit.Test;
import org.junit.runner.RunWith; import org.junit.runner.RunWith;
...@@ -17,9 +13,6 @@ import org.springframework.boot.test.context.SpringBootTest; ...@@ -17,9 +13,6 @@ import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.data.domain.Page; import org.springframework.data.domain.Page;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
import java.util.ArrayList;
import java.util.List;
@Slf4j @Slf4j
@RunWith(SpringJUnit4ClassRunner.class) @RunWith(SpringJUnit4ClassRunner.class)
...@@ -41,7 +34,7 @@ public class ESBookAndAdviserBizTest { ...@@ -41,7 +34,7 @@ public class ESBookAndAdviserBizTest {
0, 100, 0, 100,
null, null, null, null); null, null, null, null);
log.info("key:{},count:{},cost:{},==>{}", keyword, p.getTotalElements(), sw.getTime(), StringUtils.join(IKAnalyzerUtil.analyzerText(keyword), " ")); log.info("key:{},count:{},cost:{}", keyword, p.getTotalElements(), sw.getTime());
p.forEach((baa) -> log.info(JSON.toJSONString(baa))); p.forEach((baa) -> log.info(JSON.toJSONString(baa)));
sw.reset(); sw.reset();
...@@ -52,7 +45,7 @@ public class ESBookAndAdviserBizTest { ...@@ -52,7 +45,7 @@ public class ESBookAndAdviserBizTest {
0, 100, 0, 100,
null, null, null, null); null, null, null, null);
log.info("key:{},count:{},cost:{},==>{}", keyword, p.getTotalElements(), sw.getTime(), StringUtils.join(IKAnalyzerUtil.analyzerText(keyword), " ")); log.info("key:{},count:{},cost:{}", keyword, p.getTotalElements(), sw.getTime());
p.forEach((baa) -> log.info(JSON.toJSONString(baa))); p.forEach((baa) -> log.info(JSON.toJSONString(baa)));
sw.reset(); sw.reset();
...@@ -63,7 +56,7 @@ public class ESBookAndAdviserBizTest { ...@@ -63,7 +56,7 @@ public class ESBookAndAdviserBizTest {
0, 100, 0, 100,
null, null, null, null); null, null, null, null);
log.info("key:{},count:{},cost:{},==>{}", keyword, p.getTotalElements(), sw.getTime(), StringUtils.join(IKAnalyzerUtil.analyzerText(keyword), " ")); log.info("key:{},count:{},cost:{}", keyword, p.getTotalElements(), sw.getTime());
p.forEach((baa) -> log.info(JSON.toJSONString(baa))); p.forEach((baa) -> log.info(JSON.toJSONString(baa)));
sw.reset(); sw.reset();
...@@ -74,7 +67,7 @@ public class ESBookAndAdviserBizTest { ...@@ -74,7 +67,7 @@ public class ESBookAndAdviserBizTest {
0, 100, 0, 100,
null, null, null, null); null, null, null, null);
log.info("key:{},count:{},cost:{},==>{}", keyword, p.getTotalElements(), sw.getTime(), StringUtils.join(IKAnalyzerUtil.analyzerText(keyword), " ")); log.info("key:{},count:{},cost:{}", keyword, p.getTotalElements(), sw.getTime());
p.forEach((baa) -> log.info(JSON.toJSONString(baa))); p.forEach((baa) -> log.info(JSON.toJSONString(baa)));
sw.stop(); sw.stop();
...@@ -90,7 +83,7 @@ public class ESBookAndAdviserBizTest { ...@@ -90,7 +83,7 @@ public class ESBookAndAdviserBizTest {
0, 100, 0, 100,
null, null, null, null); null, null, null, null);
log.info("key:{},count:{},cost:{},==>{}", keyword, p.getTotalElements(), sw.getTime(), StringUtils.join(IKAnalyzerUtil.analyzerText(keyword), " ")); log.info("key:{},count:{},cost:{}", keyword, p.getTotalElements(), sw.getTime());
p.forEach((baa) -> log.info(JSON.toJSONString(baa))); p.forEach((baa) -> log.info(JSON.toJSONString(baa)));
sw.reset(); sw.reset();
...@@ -101,7 +94,7 @@ public class ESBookAndAdviserBizTest { ...@@ -101,7 +94,7 @@ public class ESBookAndAdviserBizTest {
0, 100, 0, 100,
null, null, null); null, null, null);
log.info("key:{},count:{},cost:{},==>{}", keyword, p.getTotalElements(), sw.getTime(), StringUtils.join(IKAnalyzerUtil.analyzerText(keyword), " ")); log.info("key:{},count:{},cost:{}", keyword, p.getTotalElements(), sw.getTime());
p.forEach((baa) -> log.info(JSON.toJSONString(baa))); p.forEach((baa) -> log.info(JSON.toJSONString(baa)));
sw.stop(); sw.stop();
......
package com.pcloud.book.test;
import com.pcloud.book.es.biz.impl.IKAnalyzerUtil;
import lombok.extern.slf4j.Slf4j;
import org.junit.Test;
@Slf4j
public class IKAnalyzerUtilTest {
@Test
public void test() {
log.info(IKAnalyzerUtil.aroundWildcard(IKAnalyzerUtil.parseWildcardString("2020秋百年学典 同步导学与优化训练 英语 三年级 上册 配人教版")));
log.info(IKAnalyzerUtil.aroundWildcard(IKAnalyzerUtil.parseWildcardString("云教金榜·寒假作业")));
log.info(IKAnalyzerUtil.aroundWildcard(IKAnalyzerUtil.parseWildcardString("学基础训练上册")));
log.info(IKAnalyzerUtil.aroundWildcard(IKAnalyzerUtil.parseWildcardString("测试书籍")));
log.info(IKAnalyzerUtil.aroundWildcard(IKAnalyzerUtil.parseWildcardString("计算机互联网")));
}
}
...@@ -4,7 +4,7 @@ import com.alibaba.fastjson.JSON; ...@@ -4,7 +4,7 @@ import com.alibaba.fastjson.JSON;
import com.pcloud.book.BookApplication; import com.pcloud.book.BookApplication;
import com.pcloud.book.elasticsearch7.entity.Es7Book; import com.pcloud.book.elasticsearch7.entity.Es7Book;
import com.pcloud.book.elasticsearch7.search.domain.dto.param.PlatformSearchDto; import com.pcloud.book.elasticsearch7.search.domain.dto.param.PlatformSearchDto;
import com.pcloud.book.elasticsearch7.search.facade.PlatformBookSearchFacade; import com.pcloud.book.elasticsearch7.search.service.PlatformBookSearchService;
import com.pcloud.common.dto.ResponseDto; import com.pcloud.common.dto.ResponseDto;
import com.pcloud.universe.commons.paging.Pagination; import com.pcloud.universe.commons.paging.Pagination;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
...@@ -20,7 +20,7 @@ import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; ...@@ -20,7 +20,7 @@ import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
public class PlatformBookSearchFacadeTest { public class PlatformBookSearchFacadeTest {
@Autowired @Autowired
private PlatformBookSearchFacade platformBookSearchFacade; private PlatformBookSearchService platformBookSearchFacade;
@Test @Test
public void testSearch() { public void testSearch() {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment