Skip to content

M
midjourney-proxy
Commit f057df76 by 李传峰
Options

remove ik analyzer

parent fa563b2e
Showing with 8 additions and 120 deletions
pcloud-service-book/src/main/java/com/pcloud/book/es/biz/impl/IKAnalyzerUtil.java deleted 100644 → 0
package com.pcloud.book.es.biz.impl;
import org.apache.commons.lang3.StringUtils;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.wltea.analyzer.lucene.IKAnalyzer;
import java.io.IOException;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.List;
/**
* 分词工具
*/
public class IKAnalyzerUtil {
private static final Logger LOGGER = LoggerFactory.getLogger(IKAnalyzerUtil.class);
/**
* 通配符
*/
public static final char WILDCARD_CHAR = '*';
/**
* 前后加通配符
* @param text 内容
* @return
*/
public static String aroundWildcard(String text) {
return WILDCARD_CHAR + text + WILDCARD_CHAR;
}
/**
* 解析成分词后通配字符串
* @param text 内容
* @return String，*通配
*/
public static String parseWildcardString(String text) {
if (StringUtils.isBlank(text)) {
return text;
}
return StringUtils.join(analyzerText(text), WILDCARD_CHAR);
}
/**
* 分词
* @param text 内容
* @return List<分词>
*/
public static List<String> analyzerText(String text) {
List<String> result = new ArrayList<>();
TokenStream ts = null;
try (IKAnalyzer analyzer = new IKAnalyzer(true)) {
ts = analyzer.tokenStream("text", new StringReader(text));
CharTermAttribute term = ts.addAttribute(CharTermAttribute.class);
ts.reset();
while (ts.incrementToken()) {
result.add(term.toString());
}
ts.end();
} catch (IOException ioe) {
result.add(text);
LOGGER.warn("analyzer text exception,text={}", text, ioe);
} finally {
closeTokenStream(text, ts);
}
return result;
}
private static void closeTokenStream(String text, TokenStream ts) {
if (ts != null) {
try {
ts.close();
} catch (IOException ioe) {
LOGGER.warn("analyzer text, close stream exception,text={}", text, ioe);
}
}
}
}
pcloud-service-book/src/test/java/com.pcloud.book.test/ESBookAndAdviserBizTest.java
......@@ -3,12 +3,8 @@ package com.pcloud.book.test;
import com.alibaba.fastjson.JSON;
import com.pcloud.book.BookApplication;
import com.pcloud.book.es.biz.ESBookAndAdviserBiz;
import com.pcloud.book.es.biz.impl.IKAnalyzerUtil;
import com.pcloud.book.es.entity.ESBookAndAdviser;
import com.pcloud.book.group.dao.BookGroupServeDao;
import com.pcloud.book.group.vo.BookGroupServeCountVO;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.StopWatch;
import org.junit.Test;
import org.junit.runner.RunWith;
......@@ -17,9 +13,6 @@ import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.data.domain.Page;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
import java.util.ArrayList;
import java.util.List;
@Slf4j
@RunWith(SpringJUnit4ClassRunner.class)
......@@ -41,7 +34,7 @@ public class ESBookAndAdviserBizTest {
0, 100,
null, null, null, null);
log.info("key:{},count:{},cost:{}，==>{}", keyword, p.getTotalElements(), sw.getTime(), StringUtils.join(IKAnalyzerUtil.analyzerText(keyword), " "));
log.info("key:{},count:{},cost:{}", keyword, p.getTotalElements(), sw.getTime());
p.forEach((baa) -> log.info(JSON.toJSONString(baa)));
sw.reset();
......@@ -52,7 +45,7 @@ public class ESBookAndAdviserBizTest {
0, 100,
null, null, null, null);
log.info("key:{},count:{},cost:{}，==>{}", keyword, p.getTotalElements(), sw.getTime(), StringUtils.join(IKAnalyzerUtil.analyzerText(keyword), " "));
log.info("key:{},count:{},cost:{}", keyword, p.getTotalElements(), sw.getTime());
p.forEach((baa) -> log.info(JSON.toJSONString(baa)));
sw.reset();
......@@ -63,7 +56,7 @@ public class ESBookAndAdviserBizTest {
0, 100,
null, null, null, null);
log.info("key:{},count:{},cost:{}，==>{}", keyword, p.getTotalElements(), sw.getTime(), StringUtils.join(IKAnalyzerUtil.analyzerText(keyword), " "));
log.info("key:{},count:{},cost:{}", keyword, p.getTotalElements(), sw.getTime());
p.forEach((baa) -> log.info(JSON.toJSONString(baa)));
sw.reset();
......@@ -74,7 +67,7 @@ public class ESBookAndAdviserBizTest {
0, 100,
null, null, null, null);
log.info("key:{},count:{},cost:{}，==>{}", keyword, p.getTotalElements(), sw.getTime(), StringUtils.join(IKAnalyzerUtil.analyzerText(keyword), " "));
log.info("key:{},count:{},cost:{}", keyword, p.getTotalElements(), sw.getTime());
p.forEach((baa) -> log.info(JSON.toJSONString(baa)));
sw.stop();
......@@ -90,7 +83,7 @@ public class ESBookAndAdviserBizTest {
0, 100,
null, null, null, null);
log.info("key:{},count:{},cost:{}，==>{}", keyword, p.getTotalElements(), sw.getTime(), StringUtils.join(IKAnalyzerUtil.analyzerText(keyword), " "));
log.info("key:{},count:{},cost:{}", keyword, p.getTotalElements(), sw.getTime());
p.forEach((baa) -> log.info(JSON.toJSONString(baa)));
sw.reset();
......@@ -101,7 +94,7 @@ public class ESBookAndAdviserBizTest {
0, 100,
null, null, null);
log.info("key:{},count:{},cost:{}，==>{}", keyword, p.getTotalElements(), sw.getTime(), StringUtils.join(IKAnalyzerUtil.analyzerText(keyword), " "));
log.info("key:{},count:{},cost:{}", keyword, p.getTotalElements(), sw.getTime());
p.forEach((baa) -> log.info(JSON.toJSONString(baa)));
sw.stop();
......
pcloud-service-book/src/test/java/com.pcloud.book.test/IKAnalyzerUtilTest.java deleted 100644 → 0
package com.pcloud.book.test;
import com.pcloud.book.es.biz.impl.IKAnalyzerUtil;
import lombok.extern.slf4j.Slf4j;
import org.junit.Test;
@Slf4j
public class IKAnalyzerUtilTest {
@Test
public void test() {
log.info(IKAnalyzerUtil.aroundWildcard(IKAnalyzerUtil.parseWildcardString("2020秋百年学典 同步导学与优化训练 英语 三年级 上册 配人教版")));
log.info(IKAnalyzerUtil.aroundWildcard(IKAnalyzerUtil.parseWildcardString("云教金榜·寒假作业")));
log.info(IKAnalyzerUtil.aroundWildcard(IKAnalyzerUtil.parseWildcardString("学基础训练上册")));
log.info(IKAnalyzerUtil.aroundWildcard(IKAnalyzerUtil.parseWildcardString("测试书籍")));
log.info(IKAnalyzerUtil.aroundWildcard(IKAnalyzerUtil.parseWildcardString("计算机互联网")));
}
}
pcloud-service-book/src/test/java/com.pcloud.book.test/PlatformBookSearchFacadeTest.java
......@@ -4,7 +4,7 @@ import com.alibaba.fastjson.JSON;
import com.pcloud.book.BookApplication;
import com.pcloud.book.elasticsearch7.entity.Es7Book;
import com.pcloud.book.elasticsearch7.search.domain.dto.param.PlatformSearchDto;
import com.pcloud.book.elasticsearch7.search.facade.PlatformBookSearchFacade;
import com.pcloud.book.elasticsearch7.search.service.PlatformBookSearchService;
import com.pcloud.common.dto.ResponseDto;
import com.pcloud.universe.commons.paging.Pagination;
import lombok.extern.slf4j.Slf4j;
......@@ -20,7 +20,7 @@ import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
public class PlatformBookSearchFacadeTest {
@Autowired
private PlatformBookSearchFacade platformBookSearchFacade;
private PlatformBookSearchService platformBookSearchFacade;
@Test
public void testSearch() {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment