|
@@ -0,0 +1,454 @@
|
|
|
+/*
|
|
|
+ * Copyright (c) 2018-2028, Chill Zhuang All rights reserved.
|
|
|
+ *
|
|
|
+ * Redistribution and use in source and binary forms, with or without
|
|
|
+ * modification, are permitted provided that the following conditions are met:
|
|
|
+ *
|
|
|
+ * Redistributions of source code must retain the above copyright notice,
|
|
|
+ * this list of conditions and the following disclaimer.
|
|
|
+ * Redistributions in binary form must reproduce the above copyright
|
|
|
+ * notice, this list of conditions and the following disclaimer in the
|
|
|
+ * documentation and/or other materials provided with the distribution.
|
|
|
+ * Neither the name of the dreamlu.net developer nor the names of its
|
|
|
+ * contributors may be used to endorse or promote products derived from
|
|
|
+ * this software without specific prior written permission.
|
|
|
+ * Author: Chill 庄骞 (smallchill@163.com)
|
|
|
+ */
|
|
|
+package org.springblade.resource.endpoint;
|
|
|
+
|
|
|
+import cn.hutool.core.util.StrUtil;
|
|
|
+import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
|
|
|
+import io.swagger.annotations.Api;
|
|
|
+import lombok.AllArgsConstructor;
|
|
|
+import lombok.SneakyThrows;
|
|
|
+import org.apache.commons.io.FileUtils;
|
|
|
+import org.apache.pdfbox.pdmodel.PDDocument;
|
|
|
+import org.springblade.common.constant.CommonConstant;
|
|
|
+import org.springblade.common.utils.SnowFlakeUtil;
|
|
|
+import org.springblade.core.oss.model.BladeFile;
|
|
|
+import org.springblade.core.oss.model.OssFile;
|
|
|
+import org.springblade.core.secure.annotation.PreAuth;
|
|
|
+import org.springblade.core.tenant.annotation.NonDS;
|
|
|
+import org.springblade.core.tool.api.R;
|
|
|
+import org.springblade.core.tool.constant.RoleConstant;
|
|
|
+import org.springblade.core.tool.utils.FileUtil;
|
|
|
+import org.springblade.core.tool.utils.Func;
|
|
|
+import org.springblade.core.tool.utils.ObjectUtil;
|
|
|
+import org.springblade.resource.builder.oss.OssBuilder;
|
|
|
+import org.springblade.resource.entity.Attach;
|
|
|
+import org.springblade.resource.entity.LargeFile;
|
|
|
+import org.springblade.resource.feign.CommonFileClient;
|
|
|
+import org.springblade.resource.service.IAttachService;
|
|
|
+import org.springblade.resource.service.ILargeFileService;
|
|
|
+import org.springblade.resource.service.IOssService;
|
|
|
+import org.springblade.resource.vo.MultipartFileParam;
|
|
|
+import org.springblade.resource.vo.NewBladeFile;
|
|
|
+import org.springblade.system.cache.ParamCache;
|
|
|
+import org.springframework.beans.BeanUtils;
|
|
|
+import org.springframework.util.DigestUtils;
|
|
|
+import org.springframework.web.bind.annotation.*;
|
|
|
+import org.springframework.web.multipart.MultipartFile;
|
|
|
+
|
|
|
+import javax.imageio.ImageIO;
|
|
|
+import javax.servlet.http.HttpServletRequest;
|
|
|
+import javax.servlet.http.HttpServletResponse;
|
|
|
+import java.awt.*;
|
|
|
+import java.awt.image.BufferedImage;
|
|
|
+import java.io.*;
|
|
|
+import java.lang.reflect.Method;
|
|
|
+import java.nio.MappedByteBuffer;
|
|
|
+import java.nio.channels.FileChannel;
|
|
|
+import java.security.AccessController;
|
|
|
+import java.security.PrivilegedAction;
|
|
|
+import java.util.ArrayList;
|
|
|
+import java.util.List;
|
|
|
+import java.util.Objects;
|
|
|
+import java.util.concurrent.locks.Lock;
|
|
|
+import java.util.concurrent.locks.ReentrantLock;
|
|
|
+
|
|
|
+/**
|
|
|
+ * 大文件存储端点
|
|
|
+ *
|
|
|
+ * @author Chill
|
|
|
+ */
|
|
|
+@NonDS
|
|
|
+@RestController
|
|
|
+@AllArgsConstructor
|
|
|
+@RequestMapping("/largeFile/endpoint")
|
|
|
+@Api(value = "大文件存储端点", tags = "大文件存储端点")
|
|
|
+public class LargeFileEndpoint {
|
|
|
+
|
|
|
+ /**
|
|
|
+ * 对象存储构建类
|
|
|
+ */
|
|
|
+ private final OssBuilder ossBuilder;
|
|
|
+
|
|
|
+ private final ILargeFileService iLargeFileService;
|
|
|
+
|
|
|
+ private final Lock lock = new ReentrantLock();
|
|
|
+ /**
|
|
|
+ * 附件表服务
|
|
|
+ */
|
|
|
+ private final IAttachService attachService;
|
|
|
+
|
|
|
+ private final CommonFileClient commonFileClient;
|
|
|
+
|
|
|
+ /**
|
|
|
+ * 创建存储桶
|
|
|
+ *
|
|
|
+ * @param bucketName 存储桶名称
|
|
|
+ * @return Bucket
|
|
|
+ */
|
|
|
+ @SneakyThrows
|
|
|
+ @PostMapping("/make-bucket")
|
|
|
+ @PreAuth(RoleConstant.HAS_ROLE_ADMIN)
|
|
|
+ public R makeBucket(@RequestParam String bucketName) {
|
|
|
+ ossBuilder.template().makeBucket(bucketName);
|
|
|
+ return R.success("创建成功");
|
|
|
+ }
|
|
|
+
|
|
|
+ /**
|
|
|
+ * 创建存储桶
|
|
|
+ *
|
|
|
+ * @param bucketName 存储桶名称
|
|
|
+ * @return R
|
|
|
+ */
|
|
|
+ @SneakyThrows
|
|
|
+ @PostMapping("/remove-bucket")
|
|
|
+ @PreAuth(RoleConstant.HAS_ROLE_ADMIN)
|
|
|
+ public R removeBucket(@RequestParam String bucketName) {
|
|
|
+ ossBuilder.template().removeBucket(bucketName);
|
|
|
+ return R.success("删除成功");
|
|
|
+ }
|
|
|
+ /**
|
|
|
+ * @return
|
|
|
+ * @throws Exception
|
|
|
+ * **/
|
|
|
+ @SneakyThrows
|
|
|
+ @PostMapping("/upload-file")
|
|
|
+ public R uploadByfile(@RequestParam(value = "file",required=false) MultipartFile file,
|
|
|
+ @RequestParam(value = "identifier",required=false) String identifier,
|
|
|
+ @RequestParam(value = "chunkNumber",required=false) Integer chunkNumber,
|
|
|
+ @RequestParam(value = "chunkSize",required=false) Integer chunkSize,
|
|
|
+ @RequestParam(value = "currentChunkSize",required=false) String currentChunkSize,
|
|
|
+ @RequestParam(value = "filename",required=false) String filename,
|
|
|
+ @RequestParam(value = "relativePath",required=false) String relativePath,
|
|
|
+ @RequestParam(value = "totalChunks",required=false) Integer totalChunks,
|
|
|
+ @RequestParam(value = "totalSize",required=false) String totalSize,
|
|
|
+ @RequestParam(value = "objectType",required=false) String objectType) throws Exception {
|
|
|
+ R result = new R();
|
|
|
+ // 判断是否上传
|
|
|
+ if (file == null) {
|
|
|
+ result.setSuccess(false);
|
|
|
+ result.setMsg("没有文件!");
|
|
|
+ return result;
|
|
|
+ }
|
|
|
+ MultipartFileParam param = new MultipartFileParam();
|
|
|
+ param.setFile(file);
|
|
|
+ param.setIdentifier(identifier);
|
|
|
+ param.setChunkNumber(chunkNumber);
|
|
|
+ param.setChunkSize(chunkSize);
|
|
|
+ param.setCurrentChunkSize(currentChunkSize);
|
|
|
+ param.setFilename(filename);
|
|
|
+ param.setRelativePath(relativePath);
|
|
|
+ param.setTotalChunks(totalChunks);
|
|
|
+ param.setTotalSize(totalSize);
|
|
|
+ param.setObjectType(objectType);
|
|
|
+ return uploadByMappedByteBuffer(param);
|
|
|
+ }
|
|
|
+ /**
|
|
|
+ * 分块上传
|
|
|
+ * 第一步:获取RandomAccessFile,随机访问文件类的对象
|
|
|
+ * 第二步:调用RandomAccessFile的getChannel()方法,打开文件通道 FileChannel
|
|
|
+ * 第三步:获取当前是第几个分块,计算文件的最后偏移量
|
|
|
+ * 第四步:获取当前文件分块的字节数组,用于获取文件字节长度
|
|
|
+ * 第五步:使用文件通道FileChannel类的 map()方法创建直接字节缓冲器 MappedByteBuffer
|
|
|
+ * 第六步:将分块的字节数组放入到当前位置的缓冲区内 mappedByteBuffer.put(byte[] b);
|
|
|
+ * 第七步:释放缓冲区
|
|
|
+ * 第八步:检查文件是否全部完成上传
|
|
|
+ */
|
|
|
+
|
|
|
+ public R uploadByMappedByteBuffer(MultipartFileParam param) throws Exception {
|
|
|
+ R result = new R();
|
|
|
+ if (param.getIdentifier() == null || "".equals(param.getIdentifier())) {
|
|
|
+ return result;
|
|
|
+ }
|
|
|
+ // 判断是否上传
|
|
|
+ if (param.getFile() == null) {
|
|
|
+ result.setSuccess(false);
|
|
|
+ result.setMsg("没有文件!");
|
|
|
+ return result;
|
|
|
+ }
|
|
|
+ if(checkMd5(param.getFile().getInputStream(), param.getIdentifier())){
|
|
|
+ result.setSuccess(false);
|
|
|
+ result.setMsg("文件的Identifier对不上!");
|
|
|
+ return result;
|
|
|
+ }
|
|
|
+ /**
|
|
|
+ * 查询是否已经上传该分片
|
|
|
+ * **/
|
|
|
+ QueryWrapper<LargeFile> wrapper = new QueryWrapper<LargeFile>()
|
|
|
+ .eq("file_key", param.getIdentifier()).eq("is_deleted",0).eq("shard_index",param.getChunkNumber()).orderByDesc("shard_index");
|
|
|
+ LargeFile list = iLargeFileService.getOne(wrapper);
|
|
|
+ if(list != null ){
|
|
|
+ result.setSuccess(true);
|
|
|
+ result.setMsg("该分片已上传!");
|
|
|
+ result.setData(list.getShardIndex());
|
|
|
+ result.setCode(200);
|
|
|
+ return result;
|
|
|
+ }
|
|
|
+ // 文件名称
|
|
|
+ String fileName = getFileName(param);
|
|
|
+ // 临时文件名称
|
|
|
+ String tempFileName = param.getIdentifier() + fileName.substring(fileName.lastIndexOf(".")) + "."+param.getChunkNumber();
|
|
|
+ // 获取文件路径
|
|
|
+ /**Windows文件路径要加在哪个盘**/
|
|
|
+// String filePath = "D:/www/wwwroot/Users/hongchuangyanfa/Desktop/Desktop/ceshi";
|
|
|
+ String filePath = "D:"+ParamCache.getValue(CommonConstant.SYS_LOCAL_URL)+"largeFile/";
|
|
|
+ // 创建文件夹
|
|
|
+// getAbsoluteFile(filePath, fileName);
|
|
|
+// new File(filePath, fileName);
|
|
|
+ // 创建临时文件
|
|
|
+// File tempFile = new File(filePath, tempFileName);
|
|
|
+ File tempFile = buildUploadFile(tempFileName);
|
|
|
+ param.getFile().transferTo(tempFile);
|
|
|
+
|
|
|
+ /**
|
|
|
+ * 以上意思是把每个分片都保存成本地一个文件,如 测试.mp4.1 最后合并
|
|
|
+ * ===================================================
|
|
|
+ * 以下注释是把分片存到一个文件,持续写入,感觉不太保险**/
|
|
|
+// //第一步 获取RandomAccessFile,随机访问文件类的对象
|
|
|
+// RandomAccessFile raf = new RandomAccessFile(tempFile,"rw");
|
|
|
+// //第二步 调用RandomAccessFile的getChannel()方法,打开文件通道 FileChannel
|
|
|
+// FileChannel fileChannel = raf.getChannel();
|
|
|
+// //第三步 获取当前是第几个分块,计算文件的最后偏移量
|
|
|
+// long offset = (param.getChunkNumber() - 1) * param.getChunkSize();
|
|
|
+// //第四步 获取当前文件分块的字节数组,用于获取文件字节长度
|
|
|
+// byte[] fileData = param.getFile().getBytes();
|
|
|
+// //第五步 使用文件通道FileChannel类的 map()方法创建直接字节缓冲器 MappedByteBuffer
|
|
|
+// MappedByteBuffer mappedByteBuffer = fileChannel.map(FileChannel.MapMode.READ_WRITE, offset, fileData.length);
|
|
|
+// //第六步 将分块的字节数组放入到当前位置的缓冲区内 mappedByteBuffer.put(byte[] b)
|
|
|
+// mappedByteBuffer.put(fileData);
|
|
|
+// //第七步 释放缓冲区
|
|
|
+// freeMappedByteBuffer(mappedByteBuffer);
|
|
|
+// fileChannel.close();
|
|
|
+// raf.close();
|
|
|
+ //第八步,保存分片信息
|
|
|
+ LargeFile largeFile = new LargeFile();
|
|
|
+
|
|
|
+ largeFile.setFileKey(param.getIdentifier());
|
|
|
+ largeFile.setName(tempFileName);
|
|
|
+ largeFile.setPath(filePath+fileName);
|
|
|
+ largeFile.setShardIndex(param.getChunkNumber());
|
|
|
+ largeFile.setShardSize(param.getChunkSize());
|
|
|
+ largeFile.setSize(Integer.valueOf(param.getCurrentChunkSize()));
|
|
|
+ largeFile.setShardTotal(param.getTotalChunks());
|
|
|
+ largeFile.setSuffix(param.getObjectType());
|
|
|
+ iLargeFileService.save(largeFile);
|
|
|
+
|
|
|
+ //第八步 检查文件是否全部完成上传
|
|
|
+ lock.lock();
|
|
|
+ try {
|
|
|
+ // 检测是否为最后一块分片
|
|
|
+ QueryWrapper<LargeFile> wrapper1 = new QueryWrapper<LargeFile>()
|
|
|
+ .eq("file_key", param.getIdentifier()).eq("is_deleted",0);
|
|
|
+
|
|
|
+ Integer count = Math.toIntExact(iLargeFileService.count(wrapper1));
|
|
|
+ if (count.equals(param.getTotalChunks())) {
|
|
|
+ /**每个文件保存到本地所使用的合并各个文件**/
|
|
|
+ merge(largeFile,filePath);
|
|
|
+ String path = largeFile.getPath(); //获取到的路径 没有.1 .2 这样的东西
|
|
|
+
|
|
|
+ //截取视频所在的路径
|
|
|
+ path = path.replace(filePath,"");
|
|
|
+ File file = new File(filePath + path);
|
|
|
+ //修改成原来的文件名
|
|
|
+ renameFile(file,param.getFilename());
|
|
|
+ FileInputStream inputStream = new FileInputStream(filePath + param.getFilename());
|
|
|
+// 上传oss
|
|
|
+ BladeFile bladeFile = ossBuilder.template().putFile(param.getFilename(),inputStream);
|
|
|
+
|
|
|
+
|
|
|
+ NewBladeFile newBladeFile = new NewBladeFile();
|
|
|
+// if(param.getFilename().contains("pdf")){
|
|
|
+// PDDocument document = PDDocument.load(inputStream);
|
|
|
+// //获取文件页数
|
|
|
+// newBladeFile.setPage(document.getPages().getCount());
|
|
|
+// //pdf的路径就是文件上传的路径
|
|
|
+// newBladeFile.setPdfUrl(bladeFile.getLink());
|
|
|
+// }
|
|
|
+ BeanUtils.copyProperties(bladeFile, newBladeFile);
|
|
|
+
|
|
|
+ //删除本地文件
|
|
|
+ file.delete();
|
|
|
+ iLargeFileService.updateLargeFileDeleted(param.getIdentifier());
|
|
|
+ result.setSuccess(true);
|
|
|
+ result.setMsg("上传成功!");
|
|
|
+ result.setData(newBladeFile);
|
|
|
+// result.setData(new BladeFile());
|
|
|
+ result.setCode(200);
|
|
|
+ return result;
|
|
|
+ }
|
|
|
+ } catch (FileNotFoundException e) {
|
|
|
+ e.printStackTrace();
|
|
|
+ } catch (InterruptedException e) {
|
|
|
+ e.printStackTrace();
|
|
|
+ } finally {
|
|
|
+ lock.unlock();
|
|
|
+ }
|
|
|
+ result.setCode(200);
|
|
|
+ result.setSuccess(true);
|
|
|
+ result.setMsg("上传成功!");
|
|
|
+ result.setData(largeFile.getShardIndex());
|
|
|
+ return result;
|
|
|
+ }
|
|
|
+
|
|
|
+ /**
|
|
|
+ * 构建上传目录和文件
|
|
|
+ */
|
|
|
+ private File buildUploadFile(String name) {
|
|
|
+ String fileName = name;
|
|
|
+ String fullDir = buildUploadDir();
|
|
|
+ return new File(fullDir, fileName);
|
|
|
+ }
|
|
|
+
|
|
|
+ /**
|
|
|
+ * 构建上传完整目录
|
|
|
+ */
|
|
|
+ private String buildUploadDir() {
|
|
|
+ String fullDir ="D:"+ParamCache.getValue(CommonConstant.SYS_LOCAL_URL)+"largeFile/";
|
|
|
+ File dir = new File(fullDir);
|
|
|
+ if (!dir.exists()) {
|
|
|
+ dir.mkdirs();
|
|
|
+ }
|
|
|
+ return fullDir;
|
|
|
+ }
|
|
|
+
|
|
|
+ /**
|
|
|
+ * md5校验
|
|
|
+ */
|
|
|
+ private boolean checkMd5(InputStream is, String md5) {
|
|
|
+ String check = "";
|
|
|
+ try {
|
|
|
+ check = DigestUtils.md5DigestAsHex(is);
|
|
|
+ } catch (IOException e) {
|
|
|
+ e.printStackTrace();
|
|
|
+ }
|
|
|
+ if (!check.equalsIgnoreCase(md5)) {
|
|
|
+ return false;
|
|
|
+ }
|
|
|
+ return true;
|
|
|
+ }
|
|
|
+ /**
|
|
|
+ * @author fengxinglie
|
|
|
+ * 合并分页
|
|
|
+ */
|
|
|
+ private void merge(LargeFile largeFile,String basePath) throws FileNotFoundException, InterruptedException {
|
|
|
+ //合并分片开始
|
|
|
+ String path = largeFile.getPath(); //获取到的路径 没有.1 .2 这样的东西
|
|
|
+ //截取视频所在的路径
|
|
|
+ path = path.replace(basePath,"");
|
|
|
+ Integer shardTotal= largeFile.getShardTotal();
|
|
|
+ File newFile = new File(basePath + path);
|
|
|
+ FileOutputStream outputStream = new FileOutputStream(newFile,true); // 文件追加写入
|
|
|
+ FileInputStream fileInputStream = null; //分片文件
|
|
|
+ byte[] byt = new byte[10 * 1024 * 1024];
|
|
|
+ int len;
|
|
|
+ try {
|
|
|
+ for (int i = 0; i < shardTotal; i++) {
|
|
|
+ // 读取第i个分片
|
|
|
+ fileInputStream = new FileInputStream(new File(basePath + path + "." + (i + 1))); // course\6sfSqfOwzmik4A4icMYuUe.mp4.1
|
|
|
+ while ((len = fileInputStream.read(byt)) != -1) {
|
|
|
+ outputStream.write(byt, 0, len);
|
|
|
+ }
|
|
|
+ }
|
|
|
+ } catch (IOException e) {
|
|
|
+ } finally {
|
|
|
+ try {
|
|
|
+ if (fileInputStream != null) {
|
|
|
+ fileInputStream.close();
|
|
|
+ }
|
|
|
+ outputStream.close();
|
|
|
+ } catch (Exception e) {
|
|
|
+// log.error("IO流关闭", e);
|
|
|
+ }
|
|
|
+ }
|
|
|
+ //告诉java虚拟机去回收垃圾 至于什么时候回收 这个取决于 虚拟机的决定
|
|
|
+ System.gc();
|
|
|
+ //等待100毫秒 等待垃圾回收去 回收完垃圾
|
|
|
+ Thread.sleep(100);
|
|
|
+ for (int i = 0; i < shardTotal; i++) {
|
|
|
+ String filePath = basePath + path + "." + (i + 1);
|
|
|
+ File file = new File(filePath);
|
|
|
+ boolean result = file.delete();
|
|
|
+// log.info("删除{},{}", filePath, result ? "成功" : "失败");
|
|
|
+ }
|
|
|
+// log.info("删除分片结束");
|
|
|
+ }
|
|
|
+
|
|
|
+ /**
|
|
|
+ * 文件重命名
|
|
|
+ *
|
|
|
+ * @param toBeRenamed 将要修改名字的文件
|
|
|
+ * @param toFileNewName 新的名字
|
|
|
+ * @return
|
|
|
+ */
|
|
|
+ private static boolean renameFile(File toBeRenamed, String toFileNewName) {
|
|
|
+ //检查要重命名的文件是否存在,是否是文件
|
|
|
+ if (!toBeRenamed.exists() || toBeRenamed.isDirectory()) {
|
|
|
+ return false;
|
|
|
+ }
|
|
|
+ String p = toBeRenamed.getParent();
|
|
|
+ File newFile = new File(p + File.separatorChar + toFileNewName);
|
|
|
+ //修改文件名
|
|
|
+ return toBeRenamed.renameTo(newFile);
|
|
|
+ }
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+ /**
|
|
|
+ * 在MappedByteBuffer释放后再对它进行读操作的话就会引发jvm crash,在并发情况下很容易发生
|
|
|
+ * 正在释放时另一个线程正开始读取,于是crash就发生了。所以为了系统稳定性释放前一般需要检 查是否还有线程在读或写
|
|
|
+ *
|
|
|
+ * @param mappedByteBuffer
|
|
|
+ */
|
|
|
+ private static void freeMappedByteBuffer(final MappedByteBuffer mappedByteBuffer) {
|
|
|
+ try {
|
|
|
+ if (mappedByteBuffer == null) {
|
|
|
+ return;
|
|
|
+ }
|
|
|
+ mappedByteBuffer.force();
|
|
|
+ AccessController.doPrivileged(new PrivilegedAction<Object>() {
|
|
|
+ @Override
|
|
|
+ public Object run() {
|
|
|
+ try {
|
|
|
+ Method getCleanerMethod = mappedByteBuffer.getClass().getMethod("cleaner", new Class[0]);
|
|
|
+ //可以访问private的权限
|
|
|
+ getCleanerMethod.setAccessible(true);
|
|
|
+ //在具有指定参数的 方法对象上调用此 方法对象表示的底层方法
|
|
|
+ sun.misc.Cleaner cleaner = (sun.misc.Cleaner) getCleanerMethod.invoke(mappedByteBuffer,
|
|
|
+ new Object[0]);
|
|
|
+ cleaner.clean();
|
|
|
+ } catch (Exception e) {
|
|
|
+// log.error("clean MappedByteBuffer error!!!", e);
|
|
|
+ }
|
|
|
+ return null;
|
|
|
+ }
|
|
|
+ });
|
|
|
+ } catch (Exception e) {
|
|
|
+ e.printStackTrace();
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ private static String getFileName(MultipartFileParam param) {
|
|
|
+ String extension;
|
|
|
+ if (ObjectUtil.isNotEmpty(param.getFile())) {
|
|
|
+ String filename = param.getFile().getOriginalFilename();
|
|
|
+ extension = filename.substring(filename.lastIndexOf("."));
|
|
|
+ } else {
|
|
|
+ extension = param.getFilename().substring(param.getFilename().lastIndexOf("."));
|
|
|
+ }
|
|
|
+ return param.getIdentifier() + extension;
|
|
|
+ }
|
|
|
+
|
|
|
+}
|