Skip to content

M
midjourney-proxy
Commit 278fbfa1 by songxiang
Options

ZIP打包下载支持分大小打包

parent bec5df1d
Showing with 139 additions and 81 deletions
pcloud-common/src/main/java/com/pcloud/common/entity/UploadResultInfo.java
......@@ -9,6 +9,8 @@ package com.pcloud.common.entity;
import com.alibaba.fastjson.JSON;
import com.fasterxml.jackson.annotation.JsonInclude;
import io.swagger.annotations.ApiModelProperty;
/**
*
* @描述：
......@@ -107,6 +109,9 @@ public class UploadResultInfo extends BaseEntity {
*/
private String m3u8Url;
@ApiModelProperty("子文件数量")
private int fileItemCount;
public UploadResultInfo() {
super();
}
......@@ -152,6 +157,14 @@ public class UploadResultInfo extends BaseEntity {
this.url = url;
}
public int getFileItemCount() {
return fileItemCount;
}
public void setFileItemCount(int fileItemCount) {
this.fileItemCount = fileItemCount;
}
public String getM3u8Url() {
return m3u8Url;
}
......
pcloud-common/src/main/java/com/pcloud/common/utils/FileUtils.java
......@@ -21,6 +21,7 @@ import java.util.List;
import java.util.UUID;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Stream;
import org.apache.commons.codec.binary.Base64;
import org.apache.commons.io.IOUtils;
......@@ -98,58 +99,58 @@ public class FileUtils {
/**
* PDF文件格式
*/
public static final String[] PDF_GATHER = { "PDF" };
public static final String[] PDF_GATHER = {"PDF"};
/**
* XML文件格式
*/
public static final String[] XML_GATHER = { "XML" };
public static final String[] XML_GATHER = {"XML"};
/**
* 图片文件格式
*/
public static final String[] IMAGE_GATHER = { "BMP", "JPG", "GIF", "PNG", "PCX", "TIF", "JPEG", "SVG" };
public static final String[] IMAGE_GATHER = {"BMP", "JPG", "GIF", "PNG", "PCX", "TIF", "JPEG", "SVG"};
/**
* 音頻文件格式 【"LRC" 为歌词字幕（这里作为音頻文件上传）】
*/
public static final String[] AUDIO_GATHER = { "AMR", "MP3", "WMA", "WAV", "AAC", "APE", "M4A" };
public static final String[] AUDIO_GATHER = {"AMR", "MP3", "WMA", "WAV", "AAC", "APE", "M4A"};
/**
* 視頻文件格式 【"SRT", "SSA", "SMI", "SUB" 为视频“外挂型”字幕（这里作为视频文件上传）】
*/
public static final String[] VIDEO_GATHER = { "MPG", "MPEG", "AVI", "MOV", "ASF", "WMV", "NAVI", "3GP", "MKV",
"FLV", "F4V", "RMVB", "RM", "MP4", "SRT", "SSA", "SMI", "SUB", "ASX", "M3U8" };
public static final String[] VIDEO_GATHER = {"MPG", "MPEG", "AVI", "MOV", "ASF", "WMV", "NAVI", "3GP", "MKV",
"FLV", "F4V", "RMVB", "RM", "MP4", "SRT", "SSA", "SMI", "SUB", "ASX", "M3U8"};
/**
* 压缩包文件格式
*/
public static final String[] COMPRESS_GATHER = { "RAR", "ZIP" };
public static final String[] COMPRESS_GATHER = {"RAR", "ZIP"};
/**
* 文件格式
*/
public static final String[] TEXT_GATHER = { "TXT", "LRC" };
public static final String[] TEXT_GATHER = {"TXT", "LRC"};
/**
* 编辑器格式
*/
public static final String[] EDITOR_GATHER = { "DOC", "DOCX", "XLS", "XLSX", "PPT", "PPTX" };
public static final String[] EDITOR_GATHER = {"DOC", "DOCX", "XLS", "XLSX", "PPT", "PPTX"};
/**
* word格式
*/
public static final String[] WORD_GATHER = { "DOC", "DOCX" };
public static final String[] WORD_GATHER = {"DOC", "DOCX"};
/**
* excel格式
*/
public static final String[] EXCEL_GATHER = { "XLS", "XLSX", "XLSM", "XLT", "XLTX", "XLTM", "CSV" };
public static final String[] EXCEL_GATHER = {"XLS", "XLSX", "XLSM", "XLT", "XLTX", "XLTM", "CSV"};
/**
* PPT格式
*/
public static final String[] PPT_GATHER = { "PPT", "PPTX" };
public static final String[] PPT_GATHER = {"PPT", "PPTX"};
/**
* 根据文件类型获取文件分类
......@@ -199,8 +200,7 @@ public class FileUtils {
/**
* 传入文件夹路径，该方法能够实现创建整个路径
*
* @param path
* 文件夹路径，不包含文件名称及后缀名
* @param path 文件夹路径，不包含文件名称及后缀名
*/
public static boolean isDir(String filePath) {
if (StringUtil.isEmpty(filePath)) {
......@@ -390,12 +390,17 @@ public class FileUtils {
if (StringUtil.isEmpty(fileName)) {
return null;
}
try {
String regEx = "[*/\\\\:?\"<|>\\s+%#&=.()]";
Pattern p = Pattern.compile(regEx);
Matcher m = p.matcher(fileName);
String result = m.replaceAll("").trim();
// 文件名称过长的话,限制40个字
return result.length() > 40 ? result.substring(0, 40) : result;
} catch (Exception e) {
LOGGER.error("【文件API】格式化文件名称.[formatName]:" + e.getMessage(), e);
return UUIDUitl.taskName();
}
}
/**
......@@ -447,8 +452,7 @@ public class FileUtils {
/**
* 根据路径删除指定的目录或文件，无论存在与否
*
* @param sPath
* 要删除的目录或文件
* @param sPath 要删除的目录或文件
* @return 删除成功返回 true，否则返回 false。
*/
public static boolean deleteFolder(String sPath) {
......@@ -473,8 +477,7 @@ public class FileUtils {
/**
* 删除单个文件
*
* @param sPath
* 被删除文件的文件名
* @param sPath 被删除文件的文件名
* @return 单个文件删除成功返回true，否则返回false
*/
public static boolean deleteFile(String sPath) {
......@@ -494,8 +497,7 @@ public class FileUtils {
/**
* 删除目录（文件夹）以及目录下的文件
*
* @param sPath
* 被删除目录的文件路径
* @param sPath 被删除目录的文件路径
* @return 目录删除成功返回true，否则返回false
*/
public static boolean deleteDirectory(String sPath) {
......@@ -540,7 +542,7 @@ public class FileUtils {
/**
* 检查文件是否存在
*
* @param path
* @param filePath
* @return
*/
public static boolean checkFile(String filePath) {
......@@ -669,10 +671,8 @@ public class FileUtils {
/**
* 合并两个文件
*
* @param outFile
* 目标文件
* @param files
* 源文件
* @param outFile 目标文件
* @param leafFile 源文件
*/
public static void mergeFiles(File outFile, File leafFile) {
FileOutputStream fos = null;
......@@ -689,7 +689,7 @@ public class FileUtils {
fos = new FileOutputStream(outFile, true);
fis = new FileInputStream(leafFile);
int len = 0;
for (byte[] buf = new byte[1024 * 1024]; (len = fis.read(buf)) != -1;) {
for (byte[] buf = new byte[1024 * 1024]; (len = fis.read(buf)) != -1; ) {
fos.write(buf, 0, len);
}
} catch (IOException ioe) {
......@@ -708,10 +708,8 @@ public class FileUtils {
/**
* 合并多个文件
*
* @param outFile
* 输出文件,
* @param leafFile
* 文件碎片集
* @param outFile 输出文件,
* @param leafFiles 文件碎片集
*/
public static void mergeFileList(File outFile, List<File> leafFiles) {
FileOutputStream fos = null;
......@@ -750,14 +748,11 @@ public class FileUtils {
/**
* 音频文件转换
*
* @author PENG
* @param data
* 音频byte数组
* @param sourExt
* 原始后缀
* @param ext
* 新后缀
* @param data 音频byte数组
* @param sourExt 原始后缀
* @param ext 新后缀
* @return
* @author PENG
*/
public static String audioFormatConvert(byte[] data, String sourExt, String ext) {
File tmpdir = new File("tempdir/local");
......@@ -859,17 +854,21 @@ public class FileUtils {
LOGGER.info("【文件API】获取文本的内容8.<START>.[url]=" + url);
StringBuilder result = new StringBuilder();
String fileType = getFileType(url);
String localFilePath = FilePathConst.TEXT_PATH + UUIDUitl.taskName() + "." + fileType;
String localFilePath = url;
if (url.startsWith("http")) {
localFilePath = FilePathConst.TEXT_PATH + UUIDUitl.taskName() + "." + fileType;
downloadFileFromUrl(url, localFilePath);
}
try {
List<String> lines = Files.readAllLines(Paths.get(localFilePath), StandardCharsets.UTF_8);
for (String line : lines) {
Stream<String> lines = Files.lines(Paths.get(localFilePath), StandardCharsets.UTF_8);
lines.forEach(line -> {
result.append(line + System.lineSeparator());
}
});
} catch (Exception e) {
LOGGER.error("【文件工具】读取文件内容8.<ERROR>:" + e.getMessage(), e);
}
} finally {
deleteFile(localFilePath);
}
LOGGER.info("【文件API】获取文本的内容8.<END>");
return result.toString();
}
......@@ -877,9 +876,8 @@ public class FileUtils {
/**
* 文件剪切,网络路径、按文件大小的百分比
*
* @param inputFile
* @param cutPercent
* 1~100的整数
* @param url
* @param percent 1~100的整数
* @return 返回剪切后上传后的文件路径
* @throws BizException
*/
......@@ -910,8 +908,7 @@ public class FileUtils {
* 文件剪切,本地路径、按百分比
*
* @param inputFile
* @param cutPercent
* 1~100的整数
* @param cutPercent 1~100的整数
* @return 返回本地路径地址
* @throws BizException
*/
......@@ -929,12 +926,8 @@ public class FileUtils {
/**
* 文件剪切
*
* @param inputFile
* 待剪切的文件
* @param outputFile
* 剪切后的文件
* @param cutLength
* 剪切长度(Byte)
* @param inputFile 待剪切的文件
* @param cutLength 剪切长度(Byte)
*/
public static String clipByLength(File inputFile, long cutLength) throws BizException {
LOGGER.info("【文件API】文件剪切.<START>.[cutLength]=" + cutLength);
......@@ -993,7 +986,7 @@ public class FileUtils {
/**
* 将文件转成base64 字符串
*
* @param path文件路径
* @param data
* @return *
* @throws Exception
*/
......@@ -1079,8 +1072,7 @@ public class FileUtils {
/**
* 从输入流中获取数据
*
* @param inStream
* 输入流
* @param inStream 输入流
* @return
* @throws Exception
*/
......
pcloud-common/src/main/java/com/pcloud/common/utils/zip/CompressUtils.java
......@@ -18,12 +18,16 @@ import org.slf4j.LoggerFactory;
import com.github.junrar.Archive;
import com.github.junrar.rarfile.FileHeader;
import com.google.common.collect.Lists;
import com.pcloud.common.constant.OSConstant;
import com.pcloud.common.dto.CompressDTO;
import com.pcloud.common.dto.CompressDTO.CompressFileDTO;
import com.pcloud.common.entity.UploadResultInfo;
import com.pcloud.common.exceptions.BizException;
import com.pcloud.common.exceptions.FileException;
import com.pcloud.common.utils.FileUtils;
import com.pcloud.common.utils.ListUtils;
import com.pcloud.common.utils.LocalDateUtils;
import com.pcloud.common.utils.UUIDUitl;
import com.pcloud.common.utils.aliyun.OssUtils;
import com.pcloud.common.utils.string.StringUtil;
......@@ -51,11 +55,63 @@ public class CompressUtils {
private static String ZIP_FILE_PATH = OSConstant.USERDIR + "/zip/";
/**
* 分拆大小值
*/
private static long CUT_SIZE = 1 * 1024 * 1024 * 1024;
/**
* 分批次打包，压缩文件集超过指定大小后，分多个包压缩,目前限定1个压缩包1G。
*
* @param compressDTO
* @return
* @throws BizException
*/
public static List<UploadResultInfo> zips(CompressDTO compressDTO) throws BizException {
LOGGER.info("【压缩】分批次打包文件.<START>.[compressDTO]=" + compressDTO);
if (compressDTO == null) {
return null;
}
List<CompressFileDTO> compressFileDTOs = compressDTO.getCompressFileDTOs();
if (ListUtils.isEmpty(compressFileDTOs)) {
return null;
}
List<UploadResultInfo> uploadResultInfos = Lists.newArrayList();
List<String[]> fileUrlList = Lists.newArrayList();
long totalFileSize = 0;
int size = compressFileDTOs.size();
int index = 1;
for (int i = 0; i < size; i++) {
CompressFileDTO compressFileDTO = compressFileDTOs.get(i);
totalFileSize += compressFileDTO.getFileSize();
fileUrlList.add(new String[]{compressFileDTO.getFileName(), compressFileDTO.getFileUrl()});
if (totalFileSize > CUT_SIZE) {
String zipName = compressDTO.getCompressName() + "_" + index;
UploadResultInfo uploadResultInfo = zip(fileUrlList, zipName);
uploadResultInfo.setUrl(OssUtils.urlAddKeyLong2Short(uploadResultInfo.getUrl()));
uploadResultInfo.setFileItemCount(fileUrlList.size());
uploadResultInfos.add(uploadResultInfo);
// 零时变量复位
totalFileSize = 0;
fileUrlList = Lists.newArrayList();
index++;
}
}
// 打包剩余总大小不足1G的文件
if (fileUrlList.size() > 0) {
UploadResultInfo uploadResultInfo = zip(fileUrlList, compressDTO.getCompressName() + "_" + index);
uploadResultInfo.setUrl(OssUtils.urlAddKeyLong2Short(uploadResultInfo.getUrl()));
uploadResultInfo.setFileItemCount(fileUrlList.size());
uploadResultInfos.add(uploadResultInfo);
}
LOGGER.info("【压缩】分批次打包文件.<END>");
return uploadResultInfos;
}
/**
* 压缩文件
*
* @param fileUrlList
* String[0]=文件名称, String[1]=文件地址
* @param zipName
* @param fileUrlList String[0]=文件名称, String[1]=文件地址
* @param zipName 压缩包的名称
* @return
* @throws BizException
*/
......@@ -65,10 +121,13 @@ public class CompressUtils {
return null;
}
zipName = FileUtils.formatName(zipName);
String tempZipName = zipName + "_" + UUIDUitl.generateString(12);
String tempZipName = zipName + "_" + LocalDateUtils.getYmdhmss();
String zipFilePath = ZIP_FILE_PATH + tempZipName + ".zip";
// 检查临时文件夹是否存在，不存在就创建
String fileFolderPath = FILE_LOCAL_PATH + tempZipName;
FileUtils.isDir(fileFolderPath);
UploadResultInfo uploadResultInfo = null;
try {
// 下载文件到文件夹中
int idx = 1;
for (String[] files : fileUrlList) {
......@@ -80,18 +139,18 @@ public class CompressUtils {
}
// 检查压缩包临时文件夹是否存在，不存在就创建
FileUtils.isDir(ZIP_FILE_PATH);
String zipFilePath = ZIP_FILE_PATH + tempZipName + ".zip";
try {
CompressUtils.zip(fileFolderPath, zipFilePath);
// 上传文件到服务器中
uploadResultInfo = OssUtils.uploadLocalFile4CustomName(zipFilePath, zipName);
uploadResultInfo.setFileName(zipName);
} catch (Exception e) {
LOGGER.error("【压缩】压缩失败,<ERROR>:" + e.getMessage(), e);
throw new FileException(FileException.ZIP_ERROR, "压缩失败！");
}
// 上传文件到服务器中
UploadResultInfo uploadResultInfo = OssUtils.uploadLocalFile4CustomName(zipFilePath, zipName);
} finally {
// 删除产生的文件
FileUtils.deleteDirectory(fileFolderPath);
FileUtils.deleteFile(zipFilePath);
}
LOGGER.info("【压缩】压缩文件.<END>");
return uploadResultInfo;
}
......@@ -99,8 +158,7 @@ public class CompressUtils {
/**
* 压缩文件(带目录)
*
* @param catalogFiles
* key : 目录名 String[0]=文件名称, String[1]=文件地址
* @param catalogFiles key : 目录名 String[0]=文件名称, String[1]=文件地址
* @param zipName
* @return
* @throws BizException
......@@ -155,12 +213,10 @@ public class CompressUtils {
}
/**
* @Desc ZIP文件解压
* @param zipPath
* ZIP文件目录
* @param destDir
* 解压目录
* @param zipPath ZIP文件目录
* @param destDir 解压目录
* @throws Exception
* @Desc ZIP文件解压
*/
private static void unzip(String zipPath, String destDir) throws Exception {
try {
......@@ -178,12 +234,10 @@ public class CompressUtils {
}
/**
* @Desc rar文件解压
* @param rarPath
* rar路径
* @param destDir
* 解压到的文件夹
* @param rarPath rar路径
* @param destDir 解压到的文件夹
* @throws Exception
* @Desc rar文件解压
*/
public static void unrar(String rarPath, String destDir) throws Exception {
if (!rarPath.toLowerCase().endsWith(".rar")) {
......@@ -220,10 +274,10 @@ public class CompressUtils {
}
/**
* @Desc 解压缩文件
* @param zip或者rar文件路径
* @param 解压目录
* @param sourceFile zip或者rar文件路径
* @param destDir 解压目录
* @throws Exception
* @Desc 解压缩文件
*/
public static void deCompress(String sourceFile, String destDir) throws Exception {
// 保证文件夹路径最后是"/"或者"\"
......@@ -303,7 +357,6 @@ public class CompressUtils {
*
* @param src
* @param zos
* @param baseDir
*/
private static void compress(File src, ZipOutputStream zos) {
compressbyType(src, zos, "", true);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment