bug: [1027381] 筛选ES中存在，但BOOK表不存在的数据

pcloud-service-book/src/main/java/com/pcloud/book/book/dao/BookDao.java

@@ -372,4 +372,6 @@ public interface BookDao extends BaseDao<Book> {
 	Integer checkIsGroupBook(Long bookId);
 
 	BookDto getBaseById4Buy(Map<String, Object> paramMap);
+
+    List<Long> getByIds(List<Long> esBookIds);
 }

pcloud-service-book/src/main/java/com/pcloud/book/book/dao/impl/BookDaoImpl.java

@@ -432,4 +432,9 @@ public class BookDaoImpl extends BaseDaoImpl<Book> implements BookDao {
     public BookDto getBaseById4Buy(Map<String, Object> paramMap) {
         return getSqlSession().selectOne(getStatement("getBaseById4Buy"),paramMap);
     }
+
+    @Override
+    public List<Long> getByIds(List<Long> bookIds) {
+        return getSqlSession().selectList(getStatement("getByIds"), bookIds);
+    }
 }

pcloud-service-book/src/main/java/com/pcloud/book/es/biz/ESBookAndAdviserBiz.java

@@ -5,6 +5,7 @@ import org.springframework.data.domain.Page;
 
 import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 
 /**
  * @Description
@@ -40,7 +41,7 @@ public interface ESBookAndAdviserBiz {
      */
     Page<ESBookAndAdviser> getAdviserBooks4SeriesBook(List<String> keywords, Integer currentPage, Integer numPerPage, List<Long> adviserIds, Long agentId, Long subLabelId, Long graLabelId, String serialNumber);
 
-    HashMap<String, String> getNotFoundBookId();
+    Map<String, Object> getNotFoundBookId(String lastBookId)throws Exception;
 
     void stopNotFoundBookId();
 }

pcloud-service-book/src/main/java/com/pcloud/book/es/biz/impl/ESBookAndAdviserBizImpl.java

 package com.pcloud.book.es.biz.impl;
 
+import cn.hutool.core.map.MapUtil;
+import com.google.common.collect.Lists;
 import com.pcloud.analysisengine.browse.dto.BrowseCacheRecordDto;
 import com.pcloud.book.applet.biz.AppletUserBookcaseBiz;
 import com.pcloud.book.book.biz.BookBiz;
@@ -467,52 +469,85 @@ public class ESBookAndAdviserBizImpl implements ESBookAndAdviserBiz {
     }
 
     @Override
-    public HashMap<String, String> getNotFoundBookId() {
+    public Map<String, Object> getNotFoundBookId(String lastBookId) throws Exception {
         String lock = JedisClusterUtils.get(NOT_FOUND_BOOK_ID_SELECT_KEY);
-        if(StringUtil.isEmpty(lock)){
+        if(!StringUtil.isEmpty(lock)){
             throw new BizException(BizException.DB_DML_FAIL.getCode(), "请不要重复执行");
         }
         JedisClusterUtils.set(NOT_FOUND_BOOK_ID_SELECT_KEY, "lock", 3600);
         // 解除停止状态
         JedisClusterUtils.set(NOT_FOUND_BOOK_ID_STATUS_SELECT_KEY, "running", 3600);
-
+        List<Long> notFoundBookIds = Lists.newArrayList();
         Integer pageNumber = 100;
-        List<String> content = null;
-        try {
+        List<Long> esBookIds = null;
+        List<Long> existsBookIds = null;
+
         String runStatus = "running";
+        try {
+            if(StringUtil.isEmpty(lastBookId)){
+                lastBookId = "0";
+            }
+            Sort sort = new Sort(Sort.Direction.ASC, "bookId");
             while ("running".equalsIgnoreCase(runStatus)) {
                 BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
-                PageRequest pageRequest = new PageRequest(0, 10000);
+                boolQueryBuilder.must(QueryBuilders.rangeQuery("bookId").gt(lastBookId));
+                boolQueryBuilder.queryName("bookId");
+                PageRequest pageRequest = new PageRequest(0, 5000, sort);
                 Page<String> map = bookAndAdviserRepository.search(boolQueryBuilder, pageRequest).map(x -> x.getBookId());
-                if (map.getSize() > 0) {
-                    // 循环查询数据库，每次100条
-                    for (int i = 0; i < (map.getSize() % pageNumber == 0 ? map.getSize()/pageNumber : (map.getSize()/pageNumber+1)); i++) {
-                        content = map.getContent().stream().skip(i * pageNumber).limit(pageNumber).collect(Collectors.toList());
-                        // TODO 查询，并筛选出不存在的数据
-                        throw new BizException("暂未实现");
+                List<String> esContent = map.getContent();
+                if (esContent.size() > 0) {
+                    // 循环每次查询100条
+                    for (int i = 0; i < (esContent.size() % pageNumber == 0 ? esContent.size()/pageNumber : (esContent.size()/pageNumber+1)); i++) {
+                        esBookIds = esContent.stream().skip(i * pageNumber).limit(pageNumber).map(Long::valueOf).collect(Collectors.toList());
+                        if(ListUtils.isEmpty(esBookIds)){
+                            continue;
+                        }
+                        // 查询，并筛选出不存在的数据
+                        existsBookIds = bookDao.getByIds(esBookIds);
+                        Thread.sleep(1);
+                        if(ListUtils.isEmpty(existsBookIds)){
+                            if(ListUtils.isEmpty(esBookIds)){
+                                continue;
+                            }
+                            notFoundBookIds.addAll(esBookIds);
+                        } else {
+                            esBookIds.removeAll(existsBookIds);
+                            if(ListUtils.isEmpty(esBookIds)){
+                                continue;
                             }
+                            notFoundBookIds.addAll(esBookIds);
                         }
-                runStatus = JedisClusterUtils.get(NOT_FOUND_BOOK_ID_STATUS_SELECT_KEY);
                     }
-            if("stop".equalsIgnoreCase(runStatus)){
-                HashMap<String, String> resultMap = new HashMap<>();
-                resultMap.put("runStatus", runStatus);
-                resultMap.put("msg", "执行被终止");
-                return resultMap;
+                    lastBookId = esContent.get(esContent.size()-1);
+                } else {
+                    runStatus = "complete";
+                    break;
+                }
+                // 判断是否强制终止
+                runStatus = JedisClusterUtils.get(NOT_FOUND_BOOK_ID_STATUS_SELECT_KEY);
             }
-            return new HashMap<>();
+            return MapUtil.<String, Object>builder()
+                    .put("runStatus", runStatus)
+                    .put("lastBookId", lastBookId)
+                    .put("notFoundBookIds", notFoundBookIds)
+                    .put("msg", "stop".equalsIgnoreCase(runStatus) ? "执行被终止" : "执行完成")
+                    .build();
+        } catch (Exception e){
+            LOGGER.info("getNotFoundBookId 执行出现异常，notFoundBookIds：" + notFoundBookIds.toString(), e);
+            return MapUtil.<String, Object>builder()
+                    .put("runStatus", runStatus)
+                    .put("lastBookId", lastBookId)
+                    .put("notFoundBookIds", notFoundBookIds)
+                    .put("msg", "执行出现异常")
+                    .build();
         } finally {
-            // 移除执行
+            // 移除执行锁
             JedisClusterUtils.del(NOT_FOUND_BOOK_ID_SELECT_KEY);
         }
     }
 
     @Override
     public void stopNotFoundBookId() {
-        String runStatus = JedisClusterUtils.get(NOT_FOUND_BOOK_ID_STATUS_SELECT_KEY);
-        if(StringUtil.isEmpty(runStatus)){
-            throw new BizException(BizException.DB_SELECT_IS_FAIL.getCode(), "当前没有执行中的任务");
-        }
         JedisClusterUtils.set(NOT_FOUND_BOOK_ID_STATUS_SELECT_KEY, "stop", 3600);
     }
 }

pcloud-service-book/src/main/java/com/pcloud/book/es/facade/ESBookAndAdviserFacade.java

@@ -67,8 +67,8 @@ public class ESBookAndAdviserFacade {
 
     @ApiOperation("查询ES有而BOOK表没有的书")
     @GetMapping("getNotFoundBookId")
-    public ResponseDto<?> getNotFoundBookId() throws BizException {
-        return new ResponseDto<>(esBookAndAdviserBiz.getNotFoundBookId());
+    public ResponseDto<?> getNotFoundBookId(@RequestParam(value = "lastBookId") String lastBookId) throws Exception {
+        return new ResponseDto<>(esBookAndAdviserBiz.getNotFoundBookId(lastBookId));
     }
 
     @ApiOperation("停止查询ES有而BOOK表没有的书")

pcloud-service-book/src/main/resources/mapper/book/Book.Mapper.xml

@@ -2641,4 +2641,11 @@
 		</if>
 		limit 1
 	</select>
+
+	<select id="getByIds" resultType="long">
+		SELECT book_id FROM book WHERE book_id IN
+		<foreach collection="list" item="bookId" open="(" close=")" separator=",">
+			#{bookId}
+		</foreach>
+	</select>
 </mapper>