-
【Java框架】-- SpringBoot大文件RestTemplate下载解决方案
近期基于项目上使用到的RestTemplate下载文件流,遇到1G以上的大文件,下载需要3-4分钟,因为调用API接口没有做分片与多线程, 文件流全部采用同步方式加载,性能很慢。最近结合网上案例及自己总结,写了一个分片下载tuling/fileServer项目: 1.包含同步下载文件流在浏览器加载输出相关代码; 2.包含分片多线程下载分片文件及合并文件相关代码;
另外在DownloadThread项目中使用代码完成了一个远程RestUrl请求去获取一个远端资源大文件进行多线程分片下载 到本地的一个案例,可以下载一些诸如.mp4/.avi等视频类大文件。相关代码也一并打包上传。
同步下载,支持分片下载Range主要代码:
@Controller public class DownLoadController { private static final String UTF8 = "UTF-8"; @RequestMapping("/download") public void downLoadFile(HttpServletRequest request, HttpServletResponse response) throws IOException { File file = new File("D:\\DevTools\\ideaIU-2021.1.3.exe"); response.setCharacterEncoding(UTF8); InputStream is = null; OutputStream os = null; try { // 分片下载 Range表示方式 bytes=100-1000 100- long fSize = file.length(); response.setContentType("application/x-download"); String fileName = URLEncoder.encode(file.getName(), UTF8); response.addHeader("Content-Disposition", "attachment;filename=" + fileName); // 支持分片下载 response.setHeader("Accept-Range", "bytes"); response.setHeader("fSize", String.valueOf(fSize)); response.setHeader("fName", fileName); long pos = 0, last = fSize - 1, sum = 0; if (null != request.getHeader("Range")) { response.setStatus(HttpServletResponse.SC_PARTIAL_CONTENT); String numberRange = request.getHeader("Range").replaceAll("bytes=", ""); String[] strRange = numberRange.split("-"); if (strRange.length == 2) { pos = Long.parseLong(strRange[0].trim()); last = Long.parseLong(strRange[1].trim()); if (last > fSize-1) { last = fSize - 1; } } else { pos = Long.parseLong(numberRange.replaceAll("-", "").trim()); } } long rangeLength = last - pos + 1; String contentRange = new StringBuffer("bytes").append(pos).append("-").append(last).append("/").append(fSize).toString(); response.setHeader("Content-Range", contentRange); response.setHeader("Content-Length", String.valueOf(rangeLength)); os = new BufferedOutputStream(response.getOutputStream()); is = new BufferedInputStream(new FileInputStream(file)); is.skip(pos); byte[] buffer = new byte[1024]; int length = 0; while (sum < rangeLength) { int readLength = (int) (rangeLength - sum); length = is.read(buffer, 0, (rangeLength - sum) <= buffer.length ? readLength : buffer.length); sum += length; os.write(buffer,0, length); } System.out.println("下载完成"); }finally { if (is != null){ is.close(); } if (os != null){ os.close(); } } } }
多线程分片下载分片文件,下载完成之后合并分片主要代码:
@RestController public class DownloadClient { private static final Logger LOGGER = LoggerFactory.getLogger(DownloadClient.class); private final static long PER_PAGE = 1024L * 1024L * 50L; private final static String DOWN_PATH = "F:\\fileItem"; ExecutorService taskExecutor = Executors.newFixedThreadPool(10); @RequestMapping("/downloadFile") public String downloadFile() { // 探测下载 FileInfo fileInfo = download(0, 10, -1, null); if (fileInfo != null) { long pages = fileInfo.fSize / PER_PAGE; for (long i = 0; i <= pages; i++) { Future<FileInfo> future = taskExecutor.submit(new DownloadThread(i * PER_PAGE, (i + 1) * PER_PAGE - 1, i, fileInfo.fName)); if (!future.isCancelled()) { try { fileInfo = future.get(); } catch (InterruptedException | ExecutionException e) { e.printStackTrace(); } } } return System.getProperty("user.home") + "\\Downloads\\" + fileInfo.fName; } return null; } class FileInfo { long fSize; String fName; public FileInfo(long fSize, String fName) { this.fSize = fSize; this.fName = fName; } } /** * 根据开始位置/结束位置 * 分片下载文件,临时存储文件分片 * 文件大小=结束位置-开始位置 * * @return */ private FileInfo download(long start, long end, long page, String fName) { File dir = new File(DOWN_PATH); if (!dir.exists()) { dir.mkdirs(); } // 断点下载 File file = new File(DOWN_PATH, page + "-" + fName); if (file.exists() && page != -1 && file.length() == PER_PAGE) { return null; } try { HttpClient client = HttpClients.createDefault(); HttpGet httpGet = new HttpGet("http://127.0.0.1:8080/download"); httpGet.setHeader("Range", "bytes=" + start + "-" + end); HttpResponse response = client.execute(httpGet); String fSize = response.getFirstHeader("fSize").getValue(); fName = URLDecoder.decode(response.getFirstHeader("fName").getValue(), "UTF-8"); HttpEntity entity = response.getEntity(); InputStream is = entity.getContent(); FileOutputStream fos = new FileOutputStream(file); byte[] buffer = new byte[1024]; int ch; while ((ch = is.read(buffer)) != -1) { fos.write(buffer, 0, ch); } is.close(); fos.flush(); fos.close(); // 最后一个分片 if (end - Long.parseLong(fSize) > 0) { // 开始合并文件 mergeFile(fName, page); } return new FileInfo(Long.parseLong(fSize), fName); } catch (IOException e) { e.printStackTrace(); } return null; } private void mergeFile(String fName, long page) { File file = new File(DOWN_PATH, fName); try { BufferedOutputStream os = new BufferedOutputStream(new FileOutputStream(file)); for (long i = 0; i <= page; i++) { File tempFile = new File(DOWN_PATH, i + "-" + fName); while (!file.exists() || (i != page && tempFile.length() < PER_PAGE)) { try { Thread.sleep(100); } catch (InterruptedException e) { e.printStackTrace(); } } byte[] bytes = FileUtils.readFileToByteArray(tempFile); os.write(bytes); os.flush(); tempFile.delete(); } File testFile = new File(DOWN_PATH, -1 + "-null"); testFile.delete(); os.flush(); os.close(); } catch (IOException e) { e.printStackTrace(); } } /** * 获取远程文件尺寸 */ private long getRemoteFileSize(String remoteFileUrl) throws IOException { long fileSize = 0; HttpURLConnection httpConnection = (HttpURLConnection) new URL(remoteFileUrl).openConnection(); //使用HEAD方法 httpConnection.setRequestMethod("HEAD"); int responseCode = httpConnection.getResponseCode(); if (responseCode >= 400) { LOGGER.debug("Web服务器响应错误!"); return 0; } String sHeader; for (int i = 1;; i++) { sHeader = httpConnection.getHeaderFieldKey(i); if (sHeader != null && sHeader.equals("Content-Length")) { LOGGER.debug("文件大小ContentLength:" + httpConnection.getContentLength()); fileSize = Long.parseLong(httpConnection.getHeaderField(sHeader)); break; } } return fileSize; } class DownloadThread implements Callable<FileInfo> { long start; long end; long page; String fName; public DownloadThread(long start, long end, long page, String fName) { this.start = start; this.end = end; this.page = page; this.fName = fName; } @Override public FileInfo call() { return download(start, end, page, fName); } } }
代码都在本地亲测(已修复Bug)可用,目前比较欠缺的是没有实现在分片下载时对应浏览器进行下载展示,需要暂存在本地磁盘目录。 目前将代码开源,希望能有更好解决方案的Coder Fork支持!也欢迎Star捧场。
本文代码已上传至GitHub:
BurstDownload
https://github.com/devyf/BurstDownload
来源:https://www.cnblogs.com/yif0118/p/15456666.html
最新更新
带有参数的装饰器
类装饰器
django中的auth模块与admin后台管理
python的日期处理
字符串常用方法
基本数据类型概述
python-map()函数基本用法
python带你实现任意下载AcFun视频数据~
bbs项目之注册功能
变量的定义和使用
三大常用数据库事务详解之三:事务运行
三大常用关系型数据库事务详解之二:基
三大关系型数据库事务详解之一:基本概
MongoDB常用命令(2)
MongoDB基本介绍与安装(1)
SQLServer触发器调用JavaWeb接口
SQL Server索引的原理深入解析
SqlServer2016模糊匹配的三种方式及效率问题
SQL中Truncate的用法
sqlserver 多表关联时在where语句中慎用tri
VB.NET中如何快速访问注册表
ASP.NET中图象处理过程详解
Vue(1)Vue安装与使用
JavaScript 语言入门
js将一段字符串的首字母转成大写
纯原生html编写的h5视频播放器
H5仿原生app短信验证码vue2.0组件附源码地
TypeScript(4)接口
TypeScript(3)基础类型
TypeScript(2)WebStorm自动编译TypeScript配置