Skip to content

M
midjourney-proxy
Commit 8932cc7c by 李传峰
Options

book_group_qrcode 对接新es

parent df59b9f6
Showing with 52 additions and 278 deletions
pcloud-service-book/pom.xml
......@@ -19,7 +19,7 @@
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<pcloud-book-elasticsearch-api.version>1.0.1-SNAPSHOT</pcloud-book-elasticsearch-api.version>
<pcloud-book-elasticsearch-api.version>2.0.1-SNAPSHOT</pcloud-book-elasticsearch-api.version>
</properties>
<dependencies>
......
pcloud-service-book/src/main/java/com/pcloud/book/consumer/book/elasticsearch/BookElasticSearchConsr.java
......@@ -2,7 +2,7 @@ package com.pcloud.book.consumer.book.elasticsearch;
import com.pcloud.book.elasticsearch7.entity.Es7Book;
import com.pcloud.book.elasticsearch7.search.domain.dto.param.PlatformSearchDto;
import com.pcloud.book.elasticsearch7.search.facade.PlatformBookSearchFacade;
import com.pcloud.book.elasticsearch7.search.service.PlatformBookSearchService;
import com.pcloud.common.dto.ResponseDto;
import com.pcloud.universe.commons.paging.Pagination;
import org.slf4j.Logger;
......@@ -22,11 +22,11 @@ public class BookElasticSearchConsr {
private final Logger logger = LoggerFactory.getLogger(getClass());
@Autowired
private PlatformBookSearchFacade platformBookSearchFacade;
private PlatformBookSearchService platformBookSearchService;
public Pagination<Es7Book> search(PlatformSearchDto dto) {
try {
ResponseDto<Pagination<Es7Book>> r = platformBookSearchFacade.search(dto);
ResponseDto<Pagination<Es7Book>> r = platformBookSearchService.search(dto);
return r.getData();
} catch (Exception e) {
logger.error("platform book es search exception.", e);
......
pcloud-service-book/src/main/java/com/pcloud/book/es/biz/ESBookGroupQrcodeBiz.java
......@@ -9,12 +9,6 @@ import java.util.List;
public interface ESBookGroupQrcodeBiz {
/**
* 初始化数据-将数据从 MySQL 导入到 Elasticsearch 中
*/
void initData();
/**
* 通过微信群ID(xxx@chatroom)获取ES中的群信息
*
* @param wxGroupIds 微信群ID
......
pcloud-service-book/src/main/java/com/pcloud/book/es/biz/impl/ESBookGroupQrcodeBizImpl.java
......@@ -3,53 +3,25 @@ package com.pcloud.book.es.biz.impl;
import cn.hutool.core.bean.BeanUtil;
import cn.hutool.core.collection.CollUtil;
import cn.hutool.core.util.StrUtil;
import com.alibaba.fastjson.JSONObject;
import com.pcloud.book.book.constant.BookConstant;
import com.pcloud.book.consumer.reader.ReaderConsr;
import com.pcloud.book.consumer.trade.TradeConsr;
import com.pcloud.book.consumer.user.AdviserConsr;
import com.pcloud.book.consumer.user.AgentConsr;
import com.pcloud.book.consumer.wechat.WechatConsr;
import com.pcloud.book.elasticsearch7.entity.Es7BookGroupQrcode;
import com.pcloud.book.elasticsearch7.search.domain.dto.param.BookGroupQrcodeSearchDto;
import com.pcloud.book.elasticsearch7.search.service.Es7BookGroupQrcodeService;
import com.pcloud.book.es.biz.ESBookGroupQrcodeBiz;
import com.pcloud.book.es.dto.BookAdviserInfo;
import com.pcloud.book.es.dto.BookTagDTO;
import com.pcloud.book.es.dto.ESBookGroupQrcodeDTO;
import com.pcloud.book.es.dto.GroupQrcodeSearchDTO;
import com.pcloud.book.es.entity.ESBookGroupQrcode;
import com.pcloud.book.es.repository.BookGroupQrcodeRepository;
import com.pcloud.book.group.dao.BookGroupClassifyDao;
import com.pcloud.book.group.dao.GroupQrcodeDao;
import com.pcloud.book.group.entity.GroupQrcode;
import com.pcloud.common.page.PageBeanNew;
import com.pcloud.common.utils.cache.redis.JedisClusterUtils;
import com.pcloud.facade.tradecenter.dto.OrderGroupFormDto;
import com.pcloud.usercenter.party.agent.dto.AgentCity;
import com.pcloud.wechatgroup.group.dto.ESGroupBookQrcodeDTO;
import com.pcloud.universe.commons.paging.Pagination;
import lombok.extern.slf4j.Slf4j;
import org.elasticsearch.action.admin.indices.analyze.AnalyzeAction;
import org.elasticsearch.action.admin.indices.analyze.AnalyzeRequestBuilder;
import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse;
import org.elasticsearch.index.query.MatchQueryBuilder;
import org.elasticsearch.index.query.MultiMatchQueryBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.apache.commons.collections4.CollectionUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.domain.Sort;
import org.springframework.data.elasticsearch.core.ElasticsearchTemplate;
import org.springframework.cglib.beans.BeanCopier;
import org.springframework.stereotype.Component;
import java.math.BigDecimal;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
import static com.pcloud.book.book.constant.BookConstant.BOOK_GROUP_QRCODE_ES_INIT;
/**
* @author guiq
......@@ -61,247 +33,73 @@ import static com.pcloud.book.book.constant.BookConstant.BOOK_GROUP_QRCODE_ES_IN
public class ESBookGroupQrcodeBizImpl implements ESBookGroupQrcodeBiz {
@Autowired
private ElasticsearchTemplate elasticsearchTemplate;
@Autowired
private BookGroupQrcodeRepository bookGroupQrcodeRepository;
private Es7BookGroupQrcodeService es7BookGroupQrcodeService;
@Autowired
private GroupQrcodeDao groupQrcodeDao;
@Autowired
private BookGroupClassifyDao bookGroupClassifyDao;
@Autowired
private ReaderConsr readerConsr;
@Autowired
private WechatConsr wechatConsr;
@Autowired
private TradeConsr tradeConsr;
@Autowired
private AdviserConsr adviserConsr;
@Autowired
private AgentConsr agentConsr;
@Override
public void initData() {
long total = groupQrcodeDao.countAll();
if (JedisClusterUtils.setnx(BOOK_GROUP_QRCODE_ES_INIT, String.valueOf(total))) {
// 每次处理 1000 条数据
final long pageSize = 1000;
// 计算分页数据
long maxPage = (total + pageSize - 1) / pageSize;
long pageNum = 0;
while (pageNum < maxPage) {
// 每 1000 条处理时间大致 60s
JedisClusterUtils.expire(BOOK_GROUP_QRCODE_ES_INIT, 60);
importData(pageNum * pageSize, pageSize);
pageNum++;
}
// 处理完删除
JedisClusterUtils.del(BOOK_GROUP_QRCODE_ES_INIT);
}
}
private void importData(long start, long offset) {
List<ESBookGroupQrcode> qrcodes = groupQrcodeDao.selectPage(start, offset);
if (CollUtil.isEmpty(qrcodes)) {
return;
}
// 将群二维码根据群分类进行分组，获取群对应的书名信息
Map<Long, List<ESBookGroupQrcode>> classifyMap = qrcodes.stream().collect(Collectors.groupingBy(ESBookGroupQrcode::getClassifyId));
Set<Long> classifyIds = classifyMap.keySet();
if (CollUtil.isEmpty(classifyIds)) {
return;
}
List<BookAdviserInfo> infos = bookGroupClassifyDao.getBookAndAdviserByClassifyIds(classifyIds);
Map<Long, BookAdviserInfo> classifyBookMap = new HashMap<>();
if (!CollUtil.isEmpty(infos)) {
classifyBookMap = infos.stream().collect(Collectors.toMap(BookAdviserInfo::getClassifyId, BookAdviserInfo -> BookAdviserInfo));
}
// 获取编辑id集合
List<Long> advisers = qrcodes.stream().map(ESBookGroupQrcode::getCreateUser).distinct().filter(Objects::nonNull).collect(Collectors.toList());
Map<Long, AgentCity> agentCityMap = CollUtil.newHashMap();
if (!CollUtil.isEmpty(advisers)){
agentCityMap = readerConsr.getAgentPositionByAdviserIds(advisers);
}
// 获取群ID集合,查询城市标签及性别比例
List<String> groupIds = qrcodes.stream().map(ESBookGroupQrcode::getWeixinGroupId).distinct().filter(x->!StrUtil.isEmpty(x)).collect(Collectors.toList());
Map<String, ESGroupBookQrcodeDTO> qrcodeDTOMap = wechatConsr.aggGroupMemberUser(groupIds);
// 交易中心查询群内订单数据
List<Long> collect = qrcodes.stream().map(ESBookGroupQrcode::getId).distinct().filter(Objects::nonNull).collect(Collectors.toList());
Map<Long, OrderGroupFormDto> orderGroupForm4ES = tradeConsr.getOrderGroupForm4ES(collect);
// 查询编辑姓名列表
Map<Long, String> adviserNames = CollUtil.newHashMap(0);
if (CollUtil.isNotEmpty(advisers)){
adviserNames = adviserConsr.getNames(advisers);
}
// 查询出版社名称列表
Map<Long, String> agentNames = CollUtil.newHashMap(0);
if (CollUtil.isNotEmpty(agentCityMap)){
List<Long> agentIds = agentCityMap.values().stream().filter(Objects::nonNull).map(AgentCity::getAgentId).distinct().collect(Collectors.toList());
if (CollUtil.isNotEmpty(agentIds)){
agentNames = agentConsr.getNames(agentIds);
}
}
for (ESBookGroupQrcode qrcode : qrcodes) {
if (qrcode == null) {
continue;
}
// 根据群名称分词获取群标签
List<String> groupNameTags = qrcode.getGroupNameTags() == null ? new ArrayList<>() : qrcode.getGroupNameTags();
groupNameTags.addAll(analyzer(qrcode.getGroupName()));
if (!CollUtil.isEmpty(groupNameTags)) {
// 去除重复标签
qrcode.setGroupNameTags(groupNameTags.stream().distinct().collect(Collectors.toList()));
}
if (qrcode.getClassifyId() != null) {
// 根据书名获取分词标签
List<String> bookNameTags = qrcode.getBookNameTags() == null ? new ArrayList<>() : qrcode.getBookNameTags();
List<String> newBookNameTags = new ArrayList<>();
if (!CollUtil.isEmpty(classifyBookMap) && classifyBookMap.get(qrcode.getClassifyId()) != null){
qrcode.setIsbn(classifyBookMap.get(qrcode.getClassifyId()).getIsbn());
}
if (JedisClusterUtils.hexists(BookConstant.BOOK_NAME_TAGS_CACHE, String.valueOf(qrcode.getClassifyId()))) {
BookTagDTO bookTagDTO = JedisClusterUtils.hgetJson2Class(BookConstant.BOOK_NAME_TAGS_CACHE, String.valueOf(qrcode.getClassifyId()), BookTagDTO.class);
newBookNameTags = bookTagDTO.getBookTags();
qrcode.setQrcodeBookName(bookTagDTO.getBookName());
} else if (!CollUtil.isEmpty(classifyBookMap) && classifyBookMap.get(qrcode.getClassifyId()) != null){
String bookName = classifyBookMap.get(qrcode.getClassifyId()).getBookName();
if( !StrUtil.isEmpty(bookName)) {
newBookNameTags = analyzer(bookName);
qrcode.setQrcodeBookName(bookName);
BookTagDTO bookTagDTO = new BookTagDTO(bookName, newBookNameTags);
JedisClusterUtils.hset(BookConstant.BOOK_NAME_TAGS_CACHE, String.valueOf(qrcode.getClassifyId()), JSONObject.toJSONString(bookTagDTO));
}
}
bookNameTags.addAll(newBookNameTags);
qrcode.setBookNameTags(bookNameTags.stream().distinct().collect(Collectors.toList()));
}
// 获取书对应的出版社，并将出版社的地理位置作为群的地理标签,并补充出版社和编辑名称
if (qrcode.getCreateUser() != null) {
qrcode.setAdviserName(adviserNames.get(qrcode.getCreateUser()));
AgentCity agentCity = agentCityMap.get(qrcode.getCreateUser());
if (agentCity != null) {
qrcode.setAgentName(agentNames.get(agentCity.getAgentId()));
List<String> strings = CollUtil.toList(agentCity.getCity());
strings = strings == null ? new ArrayList<>() : strings;
qrcode.setPressTags(CollUtil.removeEmpty(strings));
}
}
// 补充城市及性别信息
if (!CollUtil.isEmpty(qrcodeDTOMap)) {
ESGroupBookQrcodeDTO qrcodeDTO = qrcodeDTOMap.get(qrcode.getWeixinGroupId());
if (qrcodeDTO != null) {
qrcode.setCityTags(qrcodeDTO.getCityTags());
int male = qrcodeDTO.getMaleCount() == null ? 0 : qrcodeDTO.getMaleCount();
int female = qrcodeDTO.getFemaleCount() == null ? 0 : qrcodeDTO.getFemaleCount();
int unknown = qrcodeDTO.getUnknownCount() == null ? 0 : qrcodeDTO.getUnknownCount();
qrcode.setMaleCount(male);
qrcode.setFemaleCount(female);
qrcode.setUnknownCount(unknown);
qrcode.setGroupMemberCount(male + female + unknown);
}
}
// 补充交易信息
if (!CollUtil.isEmpty(orderGroupForm4ES) && qrcode.getId() != null) {
OrderGroupFormDto dto = orderGroupForm4ES.get(qrcode.getId());
if (dto != null) {
qrcode.setAmountActuallyPaid(dto.getAmountActuallyPaid());
qrcode.setPurchaseVolume(dto.getPurchaseVolume());
qrcode.setPurchaseAmount(dto.getPurchaseAmount().toPlainString());
}
}
}
bookGroupQrcodeRepository.save(qrcodes);
}
/**
* 分词方法
*
* @param content 待分词内同
* @return 分词结果
*/
private List<String> analyzer(String content) {
AnalyzeRequestBuilder ikRequest = new AnalyzeRequestBuilder(elasticsearchTemplate.getClient(), AnalyzeAction.INSTANCE, "book", content);
ikRequest.setTokenizer("ik_smart");
final ArrayList<String> list = CollUtil.newArrayList();
final List<AnalyzeResponse.AnalyzeToken> tokens = ikRequest.execute().actionGet().getTokens();
tokens.forEach(x -> list.add(x.getTerm()));
return list;
}
@Override
public List<ESBookGroupQrcodeDTO> searchByWxGroupIds(List<String> wxGroupIds) {
if (CollUtil.isEmpty(wxGroupIds)) {
return new ArrayList<>();
}
List<ESBookGroupQrcodeDTO> result = new ArrayList<>();
Iterable<ESBookGroupQrcode> weixinGroupIds = bookGroupQrcodeRepository.search(QueryBuilders.termsQuery("weixinGroupId", wxGroupIds));
for (ESBookGroupQrcode weixinGroupId : weixinGroupIds) {
if (weixinGroupId != null) {
result.add(BeanUtil.copyProperties(weixinGroupId, ESBookGroupQrcodeDTO.class));
}
List<Es7BookGroupQrcode> list = es7BookGroupQrcodeService.findByWxGroupIds(wxGroupIds);
if (CollectionUtils.isEmpty(list)) {
return result;
}
list.forEach(bgq -> result.add(convert(bgq)));
return result;
}
@Override
public void save(List<ESBookGroupQrcodeDTO> dtos) {
if (!CollUtil.isEmpty(dtos)) {
List<ESBookGroupQrcode> qrcodes = new ArrayList<>();
for (ESBookGroupQrcodeDTO dto : dtos) {
qrcodes.add(BeanUtil.copyProperties(dto, ESBookGroupQrcode.class));
}
bookGroupQrcodeRepository.save(qrcodes);
if (CollectionUtils.isNotEmpty(dtos)) {
List<Es7BookGroupQrcode> qrcodes = new ArrayList<>();
dtos.forEach(dto -> qrcodes.add(convert(dto)));
es7BookGroupQrcodeService.save(qrcodes);
}
}
@Override
public PageBeanNew<GroupQrcodeSearchDTO> search(String text, String methodType, Boolean isOrder, Integer pageSize, Integer pageNum) {
QueryBuilder queryBuilder;
PageRequest pageRequest;
if (Objects.nonNull(isOrder)&&isOrder){
pageRequest = new PageRequest(pageNum,pageSize,new Sort(Sort.Direction.DESC,"groupMemberCount","userNumber"));
}else {
pageRequest = new PageRequest(pageNum, pageSize);
}
BookGroupQrcodeSearchDto dto = new BookGroupQrcodeSearchDto();
dto.setText(text);
dto.setMethodType(methodType);
dto.setOrder(isOrder);
dto.setPageSize(pageSize);
dto.setCurrentPage(pageNum + 1);
Pagination<Es7BookGroupQrcode> data = es7BookGroupQrcodeService.search(dto);
if (StrUtil.isEmpty(text)) {
queryBuilder = QueryBuilders.boolQuery();
} else {
if ("AND".equals(methodType)) {
queryBuilder = QueryBuilders.multiMatchQuery(text, "groupName", "groupNameTags", "qrcodeBookName", "bookNameTags", "cityTags", "pressTags", "pressTags", "adviserName").type(MultiMatchQueryBuilder.Type.CROSS_FIELDS).operator(MatchQueryBuilder.Operator.AND);
} else {
queryBuilder = QueryBuilders.multiMatchQuery(text, "groupName", "groupNameTags", "qrcodeBookName", "bookNameTags", "cityTags", "pressTags", "pressTags", "adviserName");
}
}
Page<ESBookGroupQrcode> page = bookGroupQrcodeRepository.search(queryBuilder, pageRequest);
List<GroupQrcodeSearchDTO> list = convert(data.getList());
List<GroupQrcodeSearchDTO> list = convert(page.getContent());
if (CollUtil.isEmpty(list)) {
return new PageBeanNew<>(pageNum, pageSize, null);
}
return new PageBeanNew<>(pageNum, pageSize, (int) page.getTotalElements(), list);
return new PageBeanNew<>(pageNum, pageSize, (int) data.getTotalCount(), list);
}
private static final BeanCopier BOOK_GROUP_QRCODE_COPIER = BeanCopier.create(Es7BookGroupQrcode.class, ESBookGroupQrcodeDTO.class, false);
private ESBookGroupQrcodeDTO convert(Es7BookGroupQrcode q) {
ESBookGroupQrcodeDTO dto = new ESBookGroupQrcodeDTO();
BOOK_GROUP_QRCODE_COPIER.copy(q, dto, null);
dto.setId(Long.parseLong(q.getId()));
return dto;
}
private static final BeanCopier BOOK_GROUP_QRCODE_COPIER2 = BeanCopier.create(ESBookGroupQrcodeDTO.class, Es7BookGroupQrcode.class, false);
private Es7BookGroupQrcode convert(ESBookGroupQrcodeDTO dto) {
Es7BookGroupQrcode q = new Es7BookGroupQrcode();
BOOK_GROUP_QRCODE_COPIER2.copy(dto, q, null);
q.setId(String.valueOf(dto.getId()));
return q;
}
private List<GroupQrcodeSearchDTO> convert(List<ESBookGroupQrcode> content) {
private List<GroupQrcodeSearchDTO> convert(List<Es7BookGroupQrcode> content) {
if (CollUtil.isEmpty(content)){
return new ArrayList<>();
}
List<GroupQrcodeSearchDTO> list = new ArrayList<>(content.size());
for (ESBookGroupQrcode esBookGroupQrcode : content) {
for (Es7BookGroupQrcode esBookGroupQrcode : content) {
GroupQrcodeSearchDTO groupQrcodeSearchDTO = new GroupQrcodeSearchDTO();
if (CollUtil.isNotEmpty(content)){
BeanUtil.copyProperties(esBookGroupQrcode,groupQrcodeSearchDTO);
......@@ -311,7 +109,7 @@ public class ESBookGroupQrcodeBizImpl implements ESBookGroupQrcodeBiz {
}else {
groupQrcodeSearchDTO.setPurchaseAmount(new BigDecimal(esBookGroupQrcode.getPurchaseAmount()));
}
GroupQrcode qrcode = groupQrcodeDao.getById(esBookGroupQrcode.getId());
GroupQrcode qrcode = groupQrcodeDao.getById(Long.parseLong(esBookGroupQrcode.getId()));
if (qrcode!=null&&qrcode.getQrcodeUrl()!=null){
groupQrcodeSearchDTO.setQrcodeUrl(qrcode.getQrcodeUrl());
}
......
pcloud-service-book/src/main/java/com/pcloud/book/es/facade/impl/ESBookGroupQrcodeFacadeImpl.java
......@@ -24,7 +24,6 @@ public class ESBookGroupQrcodeFacadeImpl {
@ApiOperation("初始化数据")
@RequestMapping(value = "init",method = RequestMethod.GET)
public ResponseDto<?> initData(){
esBookGroupQrcodeBiz.initData();
return new ResponseDto<>();
}
......
pcloud-service-book/src/main/java/com/pcloud/book/es/repository/BookGroupQrcodeRepository.java deleted 100644 → 0
package com.pcloud.book.es.repository;
import com.pcloud.book.es.entity.ESBookGroupQrcode;
import org.springframework.data.elasticsearch.repository.ElasticsearchRepository;
/**
* @author guiq
*/
public interface BookGroupQrcodeRepository extends ElasticsearchRepository<ESBookGroupQrcode,Long> {
}
pcloud-service-book/src/main/java/com/pcloud/book/es/service/impl/ESBookGroupQrcodeServiceImpl.java
......@@ -2,16 +2,11 @@ package com.pcloud.book.es.service.impl;
import com.pcloud.book.es.biz.ESBookGroupQrcodeBiz;
import com.pcloud.book.es.dto.ESBookGroupQrcodeDTO;
import com.pcloud.book.es.repository.BookGroupQrcodeRepository;
import com.pcloud.book.es.service.ESBookGroupQrcodeService;
import io.swagger.annotations.ApiOperation;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.elasticsearch.core.ElasticsearchTemplate;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.bind.annotation.*;
import java.util.List;
......@@ -22,8 +17,6 @@ public class ESBookGroupQrcodeServiceImpl implements ESBookGroupQrcodeService {
@Autowired
private ElasticsearchTemplate elasticsearchTemplate;
@Autowired
private BookGroupQrcodeRepository bookGroupQrcodeRepository;
@Autowired
private ESBookGroupQrcodeBiz esBookGroupQrcodeBiz;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment