ZIP打包下载支持分大小打包

pcloud-common/src/main/java/com/pcloud/common/entity/UploadResultInfo.java

@@ -9,6 +9,8 @@ package com.pcloud.common.entity;
 import com.alibaba.fastjson.JSON;
 import com.fasterxml.jackson.annotation.JsonInclude;
 
+import io.swagger.annotations.ApiModelProperty;
+
 /**
  * 
  * @描述：
@@ -107,6 +109,9 @@ public class UploadResultInfo extends BaseEntity {
 	 */
 	private String m3u8Url;
 
+	@ApiModelProperty("子文件数量")
+	private int fileItemCount;
+
 	public UploadResultInfo() {
 		super();
 	}
@@ -152,6 +157,14 @@ public class UploadResultInfo extends BaseEntity {
 		this.url = url;
 	}
 
+	public int getFileItemCount() {
+		return fileItemCount;
+	}
+
+	public void setFileItemCount(int fileItemCount) {
+		this.fileItemCount = fileItemCount;
+	}
+
 	public String getM3u8Url() {
 		return m3u8Url;
 	}

pcloud-common/src/main/java/com/pcloud/common/utils/FileUtils.java

@@ -21,6 +21,7 @@ import java.util.List;
 import java.util.UUID;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
+import java.util.stream.Stream;
 
 import org.apache.commons.codec.binary.Base64;
 import org.apache.commons.io.IOUtils;
@@ -98,58 +99,58 @@ public class FileUtils {
     /**
      * PDF文件格式
      */
-	public static final String[] PDF_GATHER = { "PDF" };
+    public static final String[] PDF_GATHER = {"PDF"};
 
     /**
      * XML文件格式
      */
-	public static final String[] XML_GATHER = { "XML" };
+    public static final String[] XML_GATHER = {"XML"};
 
     /**
      * 图片文件格式
      */
-	public static final String[] IMAGE_GATHER = { "BMP", "JPG", "GIF", "PNG", "PCX", "TIF", "JPEG", "SVG" };
+    public static final String[] IMAGE_GATHER = {"BMP", "JPG", "GIF", "PNG", "PCX", "TIF", "JPEG", "SVG"};
 
     /**
      * 音頻文件格式 【"LRC" 为歌词字幕（这里作为音頻文件上传）】
      */
-	public static final String[] AUDIO_GATHER = { "AMR", "MP3", "WMA", "WAV", "AAC", "APE", "M4A" };
+    public static final String[] AUDIO_GATHER = {"AMR", "MP3", "WMA", "WAV", "AAC", "APE", "M4A"};
 
     /**
      * 視頻文件格式 【"SRT", "SSA", "SMI", "SUB" 为视频“外挂型”字幕（这里作为视频文件上传）】
      */
-	public static final String[] VIDEO_GATHER = { "MPG", "MPEG", "AVI", "MOV", "ASF", "WMV", "NAVI", "3GP", "MKV",
-			"FLV", "F4V", "RMVB", "RM", "MP4", "SRT", "SSA", "SMI", "SUB", "ASX", "M3U8" };
+    public static final String[] VIDEO_GATHER = {"MPG", "MPEG", "AVI", "MOV", "ASF", "WMV", "NAVI", "3GP", "MKV",
+            "FLV", "F4V", "RMVB", "RM", "MP4", "SRT", "SSA", "SMI", "SUB", "ASX", "M3U8"};
 
     /**
      * 压缩包文件格式
      */
-	public static final String[] COMPRESS_GATHER = { "RAR", "ZIP" };
+    public static final String[] COMPRESS_GATHER = {"RAR", "ZIP"};
 
     /**
      * 文件格式
      */
-	public static final String[] TEXT_GATHER = { "TXT", "LRC" };
+    public static final String[] TEXT_GATHER = {"TXT", "LRC"};
 
     /**
      * 编辑器格式
      */
-	public static final String[] EDITOR_GATHER = { "DOC", "DOCX", "XLS", "XLSX", "PPT", "PPTX" };
+    public static final String[] EDITOR_GATHER = {"DOC", "DOCX", "XLS", "XLSX", "PPT", "PPTX"};
 
     /**
      * word格式
      */
-	public static final String[] WORD_GATHER = { "DOC", "DOCX" };
+    public static final String[] WORD_GATHER = {"DOC", "DOCX"};
 
     /**
      * excel格式
      */
-	public static final String[] EXCEL_GATHER = { "XLS", "XLSX", "XLSM", "XLT", "XLTX", "XLTM", "CSV" };
+    public static final String[] EXCEL_GATHER = {"XLS", "XLSX", "XLSM", "XLT", "XLTX", "XLTM", "CSV"};
 
     /**
      * PPT格式
      */
-	public static final String[] PPT_GATHER = { "PPT", "PPTX" };
+    public static final String[] PPT_GATHER = {"PPT", "PPTX"};
 
     /**
      * 根据文件类型获取文件分类
@@ -199,8 +200,7 @@ public class FileUtils {
     /**
      * 传入文件夹路径，该方法能够实现创建整个路径
      *
-	 * @param path
-	 *            文件夹路径，不包含文件名称及后缀名
+     * @param path 文件夹路径，不包含文件名称及后缀名
      */
     public static boolean isDir(String filePath) {
         if (StringUtil.isEmpty(filePath)) {
@@ -390,12 +390,17 @@ public class FileUtils {
         if (StringUtil.isEmpty(fileName)) {
             return null;
         }
+        try {
             String regEx = "[*/\\\\:?\"<|>\\s+%#&=.()]";
             Pattern p = Pattern.compile(regEx);
             Matcher m = p.matcher(fileName);
             String result = m.replaceAll("").trim();
             // 文件名称过长的话,限制40个字
             return result.length() > 40 ? result.substring(0, 40) : result;
+        } catch (Exception e) {
+            LOGGER.error("【文件API】格式化文件名称.[formatName]:" + e.getMessage(), e);
+            return UUIDUitl.taskName();
+        }
     }
 
     /**
@@ -447,8 +452,7 @@ public class FileUtils {
     /**
      * 根据路径删除指定的目录或文件，无论存在与否
      *
-	 * @param sPath
-	 *            要删除的目录或文件
+     * @param sPath 要删除的目录或文件
      * @return 删除成功返回 true，否则返回 false。
      */
     public static boolean deleteFolder(String sPath) {
@@ -473,8 +477,7 @@ public class FileUtils {
     /**
      * 删除单个文件
      *
-	 * @param sPath
-	 *            被删除文件的文件名
+     * @param sPath 被删除文件的文件名
      * @return 单个文件删除成功返回true，否则返回false
      */
     public static boolean deleteFile(String sPath) {
@@ -494,8 +497,7 @@ public class FileUtils {
     /**
      * 删除目录（文件夹）以及目录下的文件
      *
-	 * @param sPath
-	 *            被删除目录的文件路径
+     * @param sPath 被删除目录的文件路径
      * @return 目录删除成功返回true，否则返回false
      */
     public static boolean deleteDirectory(String sPath) {
@@ -540,7 +542,7 @@ public class FileUtils {
     /**
      * 检查文件是否存在
      *
-	 * @param path
+     * @param filePath
      * @return
      */
     public static boolean checkFile(String filePath) {
@@ -669,10 +671,8 @@ public class FileUtils {
     /**
      * 合并两个文件
      *
-	 * @param outFile
-	 *            目标文件
-	 * @param files
-	 *            源文件
+     * @param outFile  目标文件
+     * @param leafFile 源文件
      */
     public static void mergeFiles(File outFile, File leafFile) {
         FileOutputStream fos = null;
@@ -689,7 +689,7 @@ public class FileUtils {
             fos = new FileOutputStream(outFile, true);
             fis = new FileInputStream(leafFile);
             int len = 0;
-			for (byte[] buf = new byte[1024 * 1024]; (len = fis.read(buf)) != -1;) {
+            for (byte[] buf = new byte[1024 * 1024]; (len = fis.read(buf)) != -1; ) {
                 fos.write(buf, 0, len);
             }
         } catch (IOException ioe) {
@@ -708,10 +708,8 @@ public class FileUtils {
     /**
      * 合并多个文件
      *
-	 * @param outFile
-	 *            输出文件,
-	 * @param leafFile
-	 *            文件碎片集
+     * @param outFile   输出文件,
+     * @param leafFiles 文件碎片集
      */
     public static void mergeFileList(File outFile, List<File> leafFiles) {
         FileOutputStream fos = null;
@@ -750,14 +748,11 @@ public class FileUtils {
     /**
      * 音频文件转换
      *
-	 * @author PENG
-	 * @param data
-	 *            音频byte数组
-	 * @param sourExt
-	 *            原始后缀
-	 * @param ext
-	 *            新后缀
+     * @param data    音频byte数组
+     * @param sourExt 原始后缀
+     * @param ext     新后缀
      * @return
+     * @author PENG
      */
     public static String audioFormatConvert(byte[] data, String sourExt, String ext) {
         File tmpdir = new File("tempdir/local");
@@ -859,17 +854,21 @@ public class FileUtils {
         LOGGER.info("【文件API】获取文本的内容8.<START>.[url]=" + url);
         StringBuilder result = new StringBuilder();
         String fileType = getFileType(url);
-		String localFilePath = FilePathConst.TEXT_PATH + UUIDUitl.taskName() + "." + fileType;
+        String localFilePath = url;
+        if (url.startsWith("http")) {
+            localFilePath = FilePathConst.TEXT_PATH + UUIDUitl.taskName() + "." + fileType;
             downloadFileFromUrl(url, localFilePath);
+        }
         try {
-			List<String> lines = Files.readAllLines(Paths.get(localFilePath), StandardCharsets.UTF_8);
-			for (String line : lines) {
+            Stream<String> lines = Files.lines(Paths.get(localFilePath), StandardCharsets.UTF_8);
+            lines.forEach(line -> {
                 result.append(line + System.lineSeparator());
-			}
+            });
         } catch (Exception e) {
             LOGGER.error("【文件工具】读取文件内容8.<ERROR>:" + e.getMessage(), e);
-		}
+        } finally {
             deleteFile(localFilePath);
+        }
         LOGGER.info("【文件API】获取文本的内容8.<END>");
         return result.toString();
     }
@@ -877,9 +876,8 @@ public class FileUtils {
     /**
      * 文件剪切,网络路径、按文件大小的百分比
      *
-	 * @param inputFile
-	 * @param cutPercent
-	 *            1~100的整数
+     * @param url
+     * @param percent 1~100的整数
      * @return 返回剪切后上传后的文件路径
      * @throws BizException
      */
@@ -910,8 +908,7 @@ public class FileUtils {
      * 文件剪切,本地路径、按百分比
      *
      * @param inputFile
-	 * @param cutPercent
-	 *            1~100的整数
+     * @param cutPercent 1~100的整数
      * @return 返回本地路径地址
      * @throws BizException
      */
@@ -929,12 +926,8 @@ public class FileUtils {
     /**
      * 文件剪切
      *
-	 * @param inputFile
-	 *            待剪切的文件
-	 * @param outputFile
-	 *            剪切后的文件
-	 * @param cutLength
-	 *            剪切长度(Byte)
+     * @param inputFile 待剪切的文件
+     * @param cutLength 剪切长度(Byte)
      */
     public static String clipByLength(File inputFile, long cutLength) throws BizException {
         LOGGER.info("【文件API】文件剪切.<START>.[cutLength]=" + cutLength);
@@ -993,7 +986,7 @@ public class FileUtils {
     /**
      * 将文件转成base64 字符串
      *
-	 * @param path文件路径
+     * @param data
      * @return *
      * @throws Exception
      */
@@ -1079,8 +1072,7 @@ public class FileUtils {
     /**
      * 从输入流中获取数据
      *
-	 * @param inStream
-	 *            输入流
+     * @param inStream 输入流
      * @return
      * @throws Exception
      */

pcloud-common/src/main/java/com/pcloud/common/utils/zip/CompressUtils.java

@@ -18,12 +18,16 @@ import org.slf4j.LoggerFactory;
 
 import com.github.junrar.Archive;
 import com.github.junrar.rarfile.FileHeader;
+import com.google.common.collect.Lists;
 import com.pcloud.common.constant.OSConstant;
+import com.pcloud.common.dto.CompressDTO;
+import com.pcloud.common.dto.CompressDTO.CompressFileDTO;
 import com.pcloud.common.entity.UploadResultInfo;
 import com.pcloud.common.exceptions.BizException;
 import com.pcloud.common.exceptions.FileException;
 import com.pcloud.common.utils.FileUtils;
 import com.pcloud.common.utils.ListUtils;
+import com.pcloud.common.utils.LocalDateUtils;
 import com.pcloud.common.utils.UUIDUitl;
 import com.pcloud.common.utils.aliyun.OssUtils;
 import com.pcloud.common.utils.string.StringUtil;
@@ -51,11 +55,63 @@ public class CompressUtils {
     private static String ZIP_FILE_PATH = OSConstant.USERDIR + "/zip/";
 
     /**
+     * 分拆大小值
+     */
+    private static long CUT_SIZE = 1 * 1024 * 1024 * 1024;
+
+    /**
+     * 分批次打包，压缩文件集超过指定大小后，分多个包压缩,目前限定1个压缩包1G。
+     *
+     * @param compressDTO
+     * @return
+     * @throws BizException
+     */
+    public static List<UploadResultInfo> zips(CompressDTO compressDTO) throws BizException {
+        LOGGER.info("【压缩】分批次打包文件.<START>.[compressDTO]=" + compressDTO);
+        if (compressDTO == null) {
+            return null;
+        }
+        List<CompressFileDTO> compressFileDTOs = compressDTO.getCompressFileDTOs();
+        if (ListUtils.isEmpty(compressFileDTOs)) {
+            return null;
+        }
+        List<UploadResultInfo> uploadResultInfos = Lists.newArrayList();
+        List<String[]> fileUrlList = Lists.newArrayList();
+        long totalFileSize = 0;
+        int size = compressFileDTOs.size();
+        int index = 1;
+        for (int i = 0; i < size; i++) {
+            CompressFileDTO compressFileDTO = compressFileDTOs.get(i);
+            totalFileSize += compressFileDTO.getFileSize();
+            fileUrlList.add(new String[]{compressFileDTO.getFileName(), compressFileDTO.getFileUrl()});
+            if (totalFileSize > CUT_SIZE) {
+                String zipName = compressDTO.getCompressName() + "_" + index;
+                UploadResultInfo uploadResultInfo = zip(fileUrlList, zipName);
+                uploadResultInfo.setUrl(OssUtils.urlAddKeyLong2Short(uploadResultInfo.getUrl()));
+                uploadResultInfo.setFileItemCount(fileUrlList.size());
+                uploadResultInfos.add(uploadResultInfo);
+                // 零时变量复位
+                totalFileSize = 0;
+                fileUrlList = Lists.newArrayList();
+                index++;
+            }
+        }
+        // 打包剩余总大小不足1G的文件
+        if (fileUrlList.size() > 0) {
+            UploadResultInfo uploadResultInfo = zip(fileUrlList, compressDTO.getCompressName() + "_" + index);
+            uploadResultInfo.setUrl(OssUtils.urlAddKeyLong2Short(uploadResultInfo.getUrl()));
+            uploadResultInfo.setFileItemCount(fileUrlList.size());
+            uploadResultInfos.add(uploadResultInfo);
+        }
+        LOGGER.info("【压缩】分批次打包文件.<END>");
+        return uploadResultInfos;
+    }
+
+    /**
      * 压缩文件
      *
-	 * @param fileUrlList
-	 *            String[0]=文件名称, String[1]=文件地址
-	 * @param zipName
+     * @param fileUrlList String[0]=文件名称, String[1]=文件地址
+     * @param zipName     压缩包的名称
      * @return
      * @throws BizException
      */
@@ -65,10 +121,13 @@ public class CompressUtils {
             return null;
         }
         zipName = FileUtils.formatName(zipName);
-		String tempZipName = zipName + "_" + UUIDUitl.generateString(12);
+        String tempZipName = zipName + "_" + LocalDateUtils.getYmdhmss();
+        String zipFilePath = ZIP_FILE_PATH + tempZipName + ".zip";
         // 检查临时文件夹是否存在，不存在就创建
         String fileFolderPath = FILE_LOCAL_PATH + tempZipName;
         FileUtils.isDir(fileFolderPath);
+        UploadResultInfo uploadResultInfo = null;
+        try {
             // 下载文件到文件夹中
             int idx = 1;
             for (String[] files : fileUrlList) {
@@ -80,18 +139,18 @@ public class CompressUtils {
             }
             // 检查压缩包临时文件夹是否存在，不存在就创建
             FileUtils.isDir(ZIP_FILE_PATH);
-		String zipFilePath = ZIP_FILE_PATH + tempZipName + ".zip";
-		try {
             CompressUtils.zip(fileFolderPath, zipFilePath);
+            // 上传文件到服务器中
+            uploadResultInfo = OssUtils.uploadLocalFile4CustomName(zipFilePath, zipName);
+            uploadResultInfo.setFileName(zipName);
         } catch (Exception e) {
             LOGGER.error("【压缩】压缩失败,<ERROR>:" + e.getMessage(), e);
             throw new FileException(FileException.ZIP_ERROR, "压缩失败！");
-		}
-		// 上传文件到服务器中
-		UploadResultInfo uploadResultInfo = OssUtils.uploadLocalFile4CustomName(zipFilePath, zipName);
+        } finally {
             // 删除产生的文件
             FileUtils.deleteDirectory(fileFolderPath);
             FileUtils.deleteFile(zipFilePath);
+        }
         LOGGER.info("【压缩】压缩文件.<END>");
         return uploadResultInfo;
     }
@@ -99,8 +158,7 @@ public class CompressUtils {
     /**
      * 压缩文件(带目录)
      *
-	 * @param catalogFiles
-	 *            key : 目录名 String[0]=文件名称, String[1]=文件地址
+     * @param catalogFiles key : 目录名 String[0]=文件名称, String[1]=文件地址
      * @param zipName
      * @return
      * @throws BizException
@@ -155,12 +213,10 @@ public class CompressUtils {
     }
 
     /**
-	 * @Desc ZIP文件解压
-	 * @param zipPath
-	 *            ZIP文件目录
-	 * @param destDir
-	 *            解压目录
+     * @param zipPath ZIP文件目录
+     * @param destDir 解压目录
      * @throws Exception
+     * @Desc ZIP文件解压
      */
     private static void unzip(String zipPath, String destDir) throws Exception {
         try {
@@ -178,12 +234,10 @@ public class CompressUtils {
     }
 
     /**
-	 * @Desc rar文件解压
-	 * @param rarPath
-	 *            rar路径
-	 * @param destDir
-	 *            解压到的文件夹
+     * @param rarPath rar路径
+     * @param destDir 解压到的文件夹
      * @throws Exception
+     * @Desc rar文件解压
      */
     public static void unrar(String rarPath, String destDir) throws Exception {
         if (!rarPath.toLowerCase().endsWith(".rar")) {
@@ -220,10 +274,10 @@ public class CompressUtils {
     }
 
     /**
-	 * @Desc 解压缩文件
-	 * @param zip或者rar文件路径
-	 * @param 解压目录
+     * @param sourceFile zip或者rar文件路径
+     * @param destDir    解压目录
      * @throws Exception
+     * @Desc 解压缩文件
      */
     public static void deCompress(String sourceFile, String destDir) throws Exception {
         // 保证文件夹路径最后是"/"或者"\"
@@ -303,7 +357,6 @@ public class CompressUtils {
      *
      * @param src
      * @param zos
-	 * @param baseDir
      */
     private static void compress(File src, ZipOutputStream zos) {
         compressbyType(src, zos, "", true);