java实现多线程续传下载

1. 断点续传的原理

1. 断点续传的原理是通过设置HTTP Content-Range头部来实现的(Content-Range响应的 HTTP 报头指示其中一个完整的身体信息的部分消息所属)。

2. 多线程下载如何实现？

1. 多线程下载必然要每个线程下载文件中的一部分，每个线程下载到的文件内容组装成一个完整的文件，在这个过程中一个 byte 都不能出错,只要计算好每个部分要下载的 bytes 范围即可

3. 语法

Content-Range: <unit> <range-start>-<range-end>/<size>
Content-Range: <unit> <range-start>-<range-end>/*
Content-Range: <unit> */<size> 
<unit> 指定范围的单位，通常这是bytes。
<range-start> 给定单位中的一个整数，表示请求范围的开始
<range-end> 给定单位中的一个整数，表示请求范围的结束
<size> 文件（或如果未知时为'*'）。

4. 举例

单位 bytes，从第 10 个 bytes 开始下载：Content-Range: bytes=10-.
单位 bytes，从第 10 个 bytes 开始下载，下载到第 100 个 bytes：Content-Range: bytes=10-100.
这就是续传实现的原理了，Content-Range 的 start 和 end 已经可以实现分段下载。

参考

5. 网速带宽固定，为什么多线程下载可以提速？

参考

6. 多线程断点续传代码实现

1. 任务等分Content-Range请求指定文件的区间内容

 /*** 切分下载任务到多个线程** @param url* @param futureList* @throws IOException*/public void splitDownload(String url, String fileName, List<Future<Boolean>> futureList) throws IOException {long httpFileContentLength = HttpUtls.getHttpFileContentLength(url);// 任务切分long size = httpFileContentLength / 5;long lastSize = httpFileContentLength - (httpFileContentLength / 5 * (5 - 1));for (int i = 0; i < 5; i++) {long start = i * size;Long downloadWindow = (i == 5 - 1) ? lastSize : size;Long end = start + downloadWindow;if (start != 0) {start++;}DownloadThread downloadThread = new DownloadThread(url, fileName, start, end, i, httpFileContentLength);Future<Boolean> future = executor.submit(downloadThread);futureList.add(future);}}

DownloadThread

public class DownloadThread implements Callable<Boolean> {/*** 下载链接*/private final String url;/*** 文件名*/private final String httpFileName;/*** 下载开始位置*/private Long startPos;/*** 下载结束位置*/private Long endPos;/*** 下载标识*/private final Integer downloadPart;/*** 文件总大小*/private final Long contentLength;public DownloadThread(String url, String httpFileName,long startPos, Long endPos, Integer downloadPart, Long contentLength) {this.url = url;this.startPos = startPos;this.endPos = endPos;this.downloadPart = downloadPart;this.contentLength = contentLength;this.httpFileName = httpFileName;}@Overridepublic Boolean call() throws Exception {if (StringUtils.isEmpty(url)) {throw new RuntimeException("url is null");}String downloadHttpFileName = httpFileName;if (downloadPart != null) {downloadHttpFileName = httpFileName + ".temp" + downloadPart;}// 本地文件大小long localFileContentLength = FileUtils.getFileContentLength(downloadHttpFileName);LogThread.LOCAL_FINISH_SIZE.addAndGet(localFileContentLength);if (localFileContentLength >= endPos - startPos) {//下载完成LogThread.DOWNLOAD_FINISH_THREAD.addAndGet(1);return true;} else {startPos = startPos + localFileContentLength;}if (endPos.equals(contentLength)) {endPos = null;}URL httpUrl = new URL(url);HttpURLConnection httpConnection = (HttpURLConnection) httpUrl.openConnection();httpConnection.setRequestProperty("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36");if (endPos != null) {httpConnection.setRequestProperty("RANGE", "bytes=" + startPos + "-" + endPos);} else {httpConnection.setRequestProperty("RANGE", "bytes=" + startPos + "-");}//获得输入流try (InputStream input = httpConnection.getInputStream();BufferedInputStream bis = new BufferedInputStream(input);RandomAccessFile oSavedFile = new RandomAccessFile(downloadHttpFileName, "rw")) {oSavedFile.seek(localFileContentLength);/*** 每次读取的数据块大小*/byte[] buffer = new byte[1024 * 100];int len = -1;//读到文件末尾则返回-1while ((len = bis.read(buffer)) != -1) {oSavedFile.write(buffer, 0, len);LogThread.DOWNLOAD_SIZE.addAndGet(len);}} catch (Exception e) {log.error("下载出现异常", e);return false;} finally {httpConnection.disconnect();LogThread.DOWNLOAD_FINISH_THREAD.addAndGet(1);}return true;}}

LogThread

public class LogThread implements Callable<Boolean> {/*** 本地下载大小*/public static AtomicLong LOCAL_FINISH_SIZE = new AtomicLong();/*** 下载大小*/public static AtomicLong DOWNLOAD_SIZE = new AtomicLong();/*** 下载完成线程*/public static AtomicLong DOWNLOAD_FINISH_THREAD = new AtomicLong();/*** 文件大小*/private final long httpFileContentLength;public LogThread(long httpFileContentLength) {this.httpFileContentLength = httpFileContentLength;}@Overridepublic Boolean call() throws Exception {double size = 0;double mb = 1024d * 1024d;// 文件总大小String httpFileSize = String.format("%.2f", httpFileContentLength / mb);while (DOWNLOAD_FINISH_THREAD.get() != DownloadMain.DOWNLOAD_THREAD_NUM) {double downloadSize = DOWNLOAD_SIZE.get();// 每秒速度int speed = (int) (Double.valueOf(downloadSize - size).intValue() / 1024d);size = downloadSize;// 剩余时间double surplusSize = httpFileContentLength - downloadSize - LOCAL_FINISH_SIZE.get();String surplusTime = String.format("%.1f", surplusSize / 1024d / speed);if ("Infinity".equals(surplusTime)) {surplusTime = "-";}//已下大小String currentFileSize = String.format("%.2f", downloadSize / mb + LOCAL_FINISH_SIZE.get() / mb);String speedLog = String.format("> 已下载 %smb/%smb,速度 %skb/s,剩余时间 %ss", currentFileSize, httpFileSize, speed, surplusTime);//当单独使用 \r 时，它会将光标移回到当前行的开始位置，但是光标所在行的内容并不会被清除。//这意味着之后输出的内容会覆盖原有的内容。这种特性常用于控制台输出的实时更新，例如进度条或者动态显示某些数值。System.out.print("\r");System.out.print(speedLog);Thread.sleep(1000);}System.out.println();return true;}
}

2. 合并分段临时文件

 public boolean merge(String fileName) {log.info("开始合并文件 {}", fileName);byte[] buffer = new byte[1024 * 10];int len = -1;try (RandomAccessFile oSavedFile = new RandomAccessFile(fileName, "rw")) {for (int i = 0; i < DOWNLOAD_THREAD_NUM; i++) {try (BufferedInputStream bis = new BufferedInputStream(new FileInputStream(fileName + FILE_TEMP_SUFFIX + i))) {while ((len = bis.read(buffer)) != -1) { // 读到文件末尾则返回-1oSavedFile.write(buffer, 0, len);}}}} catch (Exception e) {log.error("exception", e);return false;}return true;}

3. 清理分段临时文件

public boolean clearTemp(String fileName) {for (int i = 0; i < 5; i++) {File file = new File(fileName + ".temp" + i);file.delete();}return true;}

完

感谢您的阅读

如果你发现了错误的地方，可以在留言区提出来，我对其加以修改