Spring-Boot实现HTTP大文件断点续传分片下载

发布时间 2023-10-24 16:41:37作者: Xproer-松鼠

服务端如何将一个大视频文件做切分,分段响应给客户端,让浏览器可以渐进式地播放。

Spring Boot实现HTTP分片下载断点续传,从而实现H5页面的大视频播放问题,实现渐进式播放,每次只播放需要播放的内容就可以了,不需要加载整个文件到内存中。

文件的断点续传、文件多线程并发下载(迅雷就是这么玩的)等。

  <dependencyManagement>
    <dependencies>
      <dependency>
        <groupId>cn.hutool</groupId>
        <artifactId>hutool-bom</artifactId>
        <version>5.8.18</version>
        <type>pom</type>
        <scope>import</scope>
      </dependency>
    </dependencies>
  </dependencyManagement>

  <dependencies>
    <dependency>
      <groupId>cn.hutool</groupId>
      <artifactId>hutool-core</artifactId>
    </dependency>

    <dependency>
      <groupId>org.projectlombok</groupId>
      <artifactId>lombok</artifactId>
      <optional>true</optional>
    </dependency>
  </dependencies>

代码实现

ResourceController

package com.example.insurance.controller;

import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;

import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.List;

import com.example.insurance.common.ContentRange;
import com.example.insurance.common.MediaContentUtil;
import com.example.insurance.common.NioUtils;
import lombok.extern.slf4j.Slf4j;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpRange;
import org.springframework.http.HttpStatus;
import org.springframework.util.CollectionUtils;
import org.springframework.util.StopWatch;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestHeader;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;

/**
 * 内容资源控制器
 */
@SuppressWarnings("unused")
@Slf4j
@RestController("resourceController")
@RequestMapping(path = "/resource")
public class ResourceController {

    /**
     * 获取文件内容
     *
     * @param fileName 内容文件名称
     * @param response 响应对象
     */
    @GetMapping("/media/{fileName}")
    public void getMedia(@PathVariable String fileName, HttpServletRequest request, HttpServletResponse response,
                         @RequestHeader HttpHeaders headers) {
//        printRequestInfo(fileName, request, headers);

        String filePath = MediaContentUtil.filePath();
        try {
            this.download(fileName, filePath, request, response, headers);
        } catch (Exception e) {
            log.error("getMedia error, fileName={}", fileName, e);
        }
    }

    /**
     * 获取封面内容
     *
     * @param fileName 内容封面名称
     * @param response 响应对象
     */
    @GetMapping("/cover/{fileName}")
    public void getCover(@PathVariable String fileName, HttpServletRequest request, HttpServletResponse response,
                         @RequestHeader HttpHeaders headers) {
//        printRequestInfo(fileName, request, headers);

        String filePath = MediaContentUtil.filePath();
        try {
            this.download(fileName, filePath, request, response, headers);
        } catch (Exception e) {
            log.error("getCover error, fileName={}", fileName, e);
        }
    }


    // ======= internal =======

    private static void printRequestInfo(String fileName, HttpServletRequest request, HttpHeaders headers) {
        String requestUri = request.getRequestURI();
        String queryString = request.getQueryString();
        log.debug("file={}, url={}?{}", fileName, requestUri, queryString);
        log.info("headers={}", headers);
    }

    /**
     * 设置请求响应状态、头信息、内容类型与长度 等。
     * <pre>
     * <a href="https://www.rfc-editor.org/rfc/rfc7233">
     *     HTTP/1.1 Range Requests</a>
     * 2. Range Units
     * 4. Responses to a Range Request
     *
     * <a href="https://www.rfc-editor.org/rfc/rfc2616.html">
     *     HTTP/1.1</a>
     * 10.2.7 206 Partial Content
     * 14.5 Accept-Ranges
     * 14.13 Content-Length
     * 14.16 Content-Range
     * 14.17 Content-Type
     * 19.5.1 Content-Disposition
     * 15.5 Content-Disposition Issues
     *
     * <a href="https://www.rfc-editor.org/rfc/rfc2183">
     *     Content-Disposition</a>
     * 2. The Content-Disposition Header Field
     * 2.1 The Inline Disposition Type
     * 2.3 The Filename Parameter
     * </pre>
     *
     * @param response     请求响应对象
     * @param fileName     请求的文件名称
     * @param contentType  内容类型
     * @param contentRange 内容范围对象
     */
    private static void setResponse(
            HttpServletResponse response, String fileName, String contentType,
            ContentRange contentRange) {
        // http状态码要为206:表示获取部分内容
        response.setStatus(HttpStatus.PARTIAL_CONTENT.value());
        // 支持断点续传,获取部分字节内容
        // Accept-Ranges:bytes,表示支持Range请求
        response.setHeader(HttpHeaders.ACCEPT_RANGES, ContentRange.BYTES_STRING);
        // inline表示浏览器直接使用,attachment表示下载,fileName表示下载的文件名
        response.setHeader(HttpHeaders.CONTENT_DISPOSITION,
                "inline;filename=" + MediaContentUtil.encode(fileName));
        // Content-Range,格式为:[要下载的开始位置]-[结束位置]/[文件总大小]
        // Content-Range: bytes 0-10/3103,格式为bytes 开始-结束/全部
        response.setHeader(HttpHeaders.CONTENT_RANGE, contentRange.toContentRange());

        response.setContentType(contentType);
        // Content-Length: 11,本次内容的大小
        response.setContentLengthLong(contentRange.applyAsContentLength());
    }

    /**
     * <a href="https://www.jianshu.com/p/08db5ba3bc95">
     *     Spring Boot 处理 HTTP Headers</a>
     */
    private void download(
            String fileName, String path, HttpServletRequest request, HttpServletResponse response,
            HttpHeaders headers)
            throws IOException {
        Path filePath = Paths.get(path + fileName);
        if (!Files.exists(filePath)) {
            log.warn("file not exist, filePath={}", filePath);
            return;
        }
        long fileLength = Files.size(filePath);
//        long fileLength2 = filePath.toFile().length() - 1;
//        // fileLength=1184856, fileLength2=1184855
//        log.info("fileLength={}, fileLength2={}", fileLength, fileLength2);

        // 内容范围
        ContentRange contentRange = applyAsContentRange(headers, fileLength, request);

        // 要下载的长度
        long contentLength = contentRange.applyAsContentLength();
        log.debug("contentRange={}, contentLength={}", contentRange, contentLength);

        // 文件类型
        String contentType = request.getServletContext().getMimeType(fileName);
        // mimeType=video/mp4, CONTENT_TYPE=null
        log.debug("mimeType={}, CONTENT_TYPE={}", contentType, request.getContentType());

        setResponse(response, fileName, contentType, contentRange);

        // 耗时指标统计
        StopWatch stopWatch = new StopWatch("downloadFile");
        stopWatch.start(fileName);
        try {
            // case-1.参考网上他人的实现
//            if (fileLength >= Integer.MAX_VALUE) {
//                NioUtils.copy(filePath, response, contentRange);
//            } else {
//                NioUtils.copyByChannelAndBuffer(filePath, response, contentRange);
//            }

            // case-2.使用现成API
            NioUtils.copyByBio(filePath, response, contentRange);
//            NioUtils.copyByNio(filePath, response, contentRange);

            // case-3.视频分段渐进式播放
//            if (contentType.startsWith("video")) {
//                NioUtils.copyForBufferSize(filePath, response, contentRange);
//            } else {
//                // 图片、PDF等文件
//                NioUtils.copyByBio(filePath, response, contentRange);
//            }
        } finally {
            stopWatch.stop();
            log.info("download file, fileName={}, time={} ms", fileName, stopWatch.getTotalTimeMillis());
        }
    }

    private static ContentRange applyAsContentRange(
            HttpHeaders headers, long fileLength, HttpServletRequest request) {
        /*
         * 3.1. Range - HTTP/1.1 Range Requests
         * https://www.rfc-editor.org/rfc/rfc7233#section-3.1
         * Range: "bytes" "=" first-byte-pos "-" [ last-byte-pos ]
         *
         * For example:
         * bytes=0-
         * bytes=0-499
         */
        // Range:告知服务端,客户端下载该文件想要从指定的位置开始下载
        List<HttpRange> httpRanges = headers.getRange();

        String range = request.getHeader(HttpHeaders.RANGE);
        // httpRanges=[], range=null
        // httpRanges=[448135688-], range=bytes=448135688-
        log.debug("httpRanges={}, range={}", httpRanges, range);

        // 开始下载位置
        long firstBytePos;
        // 结束下载位置
        long lastBytePos;
        if (CollectionUtils.isEmpty(httpRanges)) {
            firstBytePos = 0;
            lastBytePos = fileLength - 1;
        } else {
            HttpRange httpRange = httpRanges.get(0);
            firstBytePos = httpRange.getRangeStart(fileLength);
            lastBytePos = httpRange.getRangeEnd(fileLength);
        }
        return new ContentRange(firstBytePos, lastBytePos, fileLength);
    }
}

NioUtils

package com.example.insurance.common;

import javax.servlet.http.HttpServletResponse;
import java.io.BufferedOutputStream;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.RandomAccessFile;
import java.nio.MappedByteBuffer;
import java.nio.channels.Channels;
import java.nio.channels.FileChannel;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;

import cn.hutool.core.io.IORuntimeException;
import cn.hutool.core.io.IoUtil;
import cn.hutool.core.io.NioUtil;
import cn.hutool.core.io.StreamProgress;
import cn.hutool.core.io.unit.DataSize;
import lombok.extern.slf4j.Slf4j;
import org.apache.catalina.connector.ClientAbortException;

/**
 * NIO相关工具封装,主要针对Channel读写、拷贝等封装
 */
@Slf4j
public final class NioUtils {

    /**
     * 缓冲区大小 16KB
     *
     * @see NioUtil#DEFAULT_BUFFER_SIZE
     * @see NioUtil#DEFAULT_LARGE_BUFFER_SIZE
     */
//    private static final int BUFFER_SIZE = NioUtil.DEFAULT_MIDDLE_BUFFER_SIZE;
    private static final int BUFFER_SIZE = (int) DataSize.ofKilobytes(16L).toBytes();

    /**
     * <pre>
     * <a href="https://blog.csdn.net/qq_32099833/article/details/109703883">
     *     Java后端实现视频分段渐进式播放</a>
     * 服务端如何将一个大的视频文件做切分,分段响应给客户端,让浏览器可以渐进式地播放。
     * 文件的断点续传、文件多线程并发下载(迅雷就是这么玩的)等。
     *
     * <a href="https://blog.csdn.net/qq_32099833/article/details/109630499">
     *     大文件分片上传前后端实现</a>
     * </pre>
     */
    public static void copyForBufferSize(
            Path filePath, HttpServletResponse response, ContentRange contentRange) {
        String fileName = filePath.getFileName().toString();

        RandomAccessFile randomAccessFile = null;
        OutputStream outputStream = null;
        try {
            // 随机读文件
            randomAccessFile = new RandomAccessFile(filePath.toFile(), "r");
            // 移动访问指针到指定位置
            randomAccessFile.seek(contentRange.getStart());

            // 注意:缓冲区大小 2MB,视频加载正常;1MB时有部分视频加载失败
            int bufferSize = BUFFER_SIZE;

            //获取响应的输出流
            outputStream = new BufferedOutputStream(response.getOutputStream(), bufferSize);

            // 每次请求只返回1MB的视频流
            byte[] buffer = new byte[bufferSize];
            int len = randomAccessFile.read(buffer);
            //设置此次相应返回的数据长度
            response.setContentLength(len);
            // 将这1MB的视频流响应给客户端
            outputStream.write(buffer, 0, len);

            log.info("file download complete, fileName={}, contentRange={}",
                    fileName, contentRange.toContentRange());
        } catch (ClientAbortException | IORuntimeException e) {
            // 捕获此异常表示用户停止下载
            log.warn("client stop file download, fileName={}", fileName);
        } catch (Exception e) {
            log.error("file download error, fileName={}", fileName, e);
        } finally {
            IoUtil.close(outputStream);
            IoUtil.close(randomAccessFile);
        }
    }

    /**
     * 拷贝流,拷贝后关闭流。
     *
     * @param filePath     源文件路径
     * @param response     请求响应
     * @param contentRange 内容范围
     */
    public static void copyByBio(
            Path filePath, HttpServletResponse response, ContentRange contentRange) {
        String fileName = filePath.getFileName().toString();

        InputStream inputStream = null;
        OutputStream outputStream = null;
        try {
            RandomAccessFile randomAccessFile = new RandomAccessFile(filePath.toFile(), "r");
            randomAccessFile.seek(contentRange.getStart());

            inputStream = Channels.newInputStream(randomAccessFile.getChannel());
            outputStream = new BufferedOutputStream(response.getOutputStream(), BUFFER_SIZE);

            StreamProgress streamProgress = new StreamProgressImpl(fileName);

            long transmitted = IoUtil.copy(inputStream, outputStream, BUFFER_SIZE, streamProgress);
            log.info("file download complete, fileName={}, transmitted={}", fileName, transmitted);
        } catch (ClientAbortException | IORuntimeException e) {
            // 捕获此异常表示用户停止下载
            log.warn("client stop file download, fileName={}", fileName);
        } catch (Exception e) {
            log.error("file download error, fileName={}", fileName, e);
        } finally {
            IoUtil.close(outputStream);
            IoUtil.close(inputStream);
        }
    }

    /**
     * 拷贝流,拷贝后关闭流。
     * <pre>
     * <a href="https://www.cnblogs.com/czwbig/p/10035631.html">
     *     Java NIO 学习笔记(一)----概述,Channel/Buffer</a>
     * </pre>
     *
     * @param filePath     源文件路径
     * @param response     请求响应
     * @param contentRange 内容范围
     */
    public static void copyByNio(
            Path filePath, HttpServletResponse response, ContentRange contentRange) {
        String fileName = filePath.getFileName().toString();

        InputStream inputStream = null;
        OutputStream outputStream = null;
        try {
            RandomAccessFile randomAccessFile = new RandomAccessFile(filePath.toFile(), "r");
            randomAccessFile.seek(contentRange.getStart());

            inputStream = Channels.newInputStream(randomAccessFile.getChannel());
            outputStream = new BufferedOutputStream(response.getOutputStream(), BUFFER_SIZE);

            StreamProgress streamProgress = new StreamProgressImpl(fileName);

            long transmitted = NioUtil.copyByNIO(inputStream, outputStream,
                    BUFFER_SIZE, streamProgress);
            log.info("file download complete, fileName={}, transmitted={}", fileName, transmitted);
        } catch (ClientAbortException | IORuntimeException e) {
            // 捕获此异常表示用户停止下载
            log.warn("client stop file download, fileName={}", fileName);
        } catch (Exception e) {
            log.error("file download error, fileName={}", fileName, e);
        } finally {
            IoUtil.close(outputStream);
            IoUtil.close(inputStream);
        }
    }

    /**
     * <pre>
     * <a href="https://blog.csdn.net/lovequanquqn/article/details/104562945">
     *     SpringBoot Java实现Http方式分片下载断点续传+实现H5大视频渐进式播放</a>
     * SpringBoot 实现Http分片下载断点续传,从而实现H5页面的大视频播放问题,实现渐进式播放,每次只播放需要播放的内容就可以了,不需要加载整个文件到内存中。
     * 二、Http分片下载断点续传实现
     * 四、缓存文件定时删除任务
     * </pre>
     */
    public static void copy(Path filePath, HttpServletResponse response, ContentRange contentRange) {
        String fileName = filePath.getFileName().toString();
        // 要下载的长度
        long contentLength = contentRange.applyAsContentLength();

        BufferedOutputStream outputStream = null;
        RandomAccessFile randomAccessFile = null;
        // 已传送数据大小
        long transmitted = 0;
        try {
            randomAccessFile = new RandomAccessFile(filePath.toFile(), "r");
            randomAccessFile.seek(contentRange.getStart());
            outputStream = new BufferedOutputStream(response.getOutputStream(), BUFFER_SIZE);
            // 把数据读取到缓冲区中
            byte[] buffer = new byte[BUFFER_SIZE];

            int len = BUFFER_SIZE;
            //warning:判断是否到了最后不足4096(buffer的length)个byte这个逻辑((transmitted + len) <= contentLength)要放前面
            //不然会会先读取randomAccessFile,造成后面读取位置出错;
            while ((transmitted + len) <= contentLength && (len = randomAccessFile.read(buffer)) != -1) {
                outputStream.write(buffer, 0, len);
                transmitted += len;

                log.info("fileName={}, transmitted={}", fileName, transmitted);
            }
            //处理不足buffer.length部分
            if (transmitted < contentLength) {
                len = randomAccessFile.read(buffer, 0, (int) (contentLength - transmitted));
                outputStream.write(buffer, 0, len);
                transmitted += len;

                log.info("fileName={}, transmitted={}", fileName, transmitted);
            }

            log.info("file download complete, fileName={}, transmitted={}", fileName, transmitted);
        } catch (ClientAbortException e) {
            // 捕获此异常表示用户停止下载
            log.warn("client stop file download, fileName={}, transmitted={}", fileName, transmitted);
        } catch (Exception e) {
            log.error("file download error, fileName={}, transmitted={}", fileName, transmitted, e);
        } finally {
            IoUtil.close(outputStream);
            IoUtil.close(randomAccessFile);
        }
    }

    /**
     * 通过数据传输通道和缓冲区读取文件数据。
     * <pre>
     * 当文件长度超过{@link Integer#MAX_VALUE}时,
     * 使用{@link FileChannel#map(FileChannel.MapMode, long, long)}报如下异常。
     * java.lang.IllegalArgumentException: Size exceeds Integer.MAX_VALUE
     *   at sun.nio.ch.FileChannelImpl.map(FileChannelImpl.java:863)
     *   at com.example.insurance.controller.ResourceController.download(ResourceController.java:200)
     * </pre>
     *
     * @param filePath     源文件路径
     * @param response     请求响应
     * @param contentRange 内容范围
     */
    public static void copyByChannelAndBuffer(
            Path filePath, HttpServletResponse response, ContentRange contentRange) {
        String fileName = filePath.getFileName().toString();
        // 要下载的长度
        long contentLength = contentRange.applyAsContentLength();

        BufferedOutputStream outputStream = null;
        FileChannel inChannel = null;
        // 已传送数据大小
        long transmitted = 0;
        long firstBytePos = contentRange.getStart();
        long fileLength = contentRange.getLength();
        try {
            inChannel = FileChannel.open(filePath, StandardOpenOption.READ, StandardOpenOption.WRITE);
            // 建立直接缓冲区
            MappedByteBuffer inMap = inChannel.map(FileChannel.MapMode.READ_ONLY, firstBytePos, fileLength);
            outputStream = new BufferedOutputStream(response.getOutputStream(), BUFFER_SIZE);
            // 把数据读取到缓冲区中
            byte[] buffer = new byte[BUFFER_SIZE];

            int len = BUFFER_SIZE;
            // warning:判断是否到了最后不足4096(buffer的length)个byte这个逻辑((transmitted + len) <= contentLength)要放前面
            // 不然会会先读取file,造成后面读取位置出错
            while ((transmitted + len) <= contentLength) {
                inMap.get(buffer);
                outputStream.write(buffer, 0, len);
                transmitted += len;

                log.info("fileName={}, transmitted={}", fileName, transmitted);
            }
            // 处理不足buffer.length部分
            if (transmitted < contentLength) {
                len = (int) (contentLength - transmitted);
                buffer = new byte[len];
                inMap.get(buffer);
                outputStream.write(buffer, 0, len);
                transmitted += len;

                log.info("fileName={}, transmitted={}", fileName, transmitted);
            }

            log.info("file download complete, fileName={}, transmitted={}", fileName, transmitted);
        } catch (ClientAbortException e) {
            // 捕获此异常表示用户停止下载
            log.warn("client stop file download, fileName={}, transmitted={}", fileName, transmitted);
        } catch (Exception e) {
            log.error("file download error, fileName={}, transmitted={}", fileName, transmitted, e);
        } finally {
            IoUtil.close(outputStream);
            IoUtil.close(inChannel);
        }
    }

}

ContentRange

package com.example.insurance.common;

import lombok.AllArgsConstructor;
import lombok.Getter;

/**
 * 内容范围对象
 * <pre>
 * <a href="https://www.rfc-editor.org/rfc/rfc7233#section-4.2">
 *     4.2. Content-Range - HTTP/1.1 Range Requests</a>
 * Content-Range: "bytes" first-byte-pos "-" last-byte-pos  "/" complete-length
 *
 * For example:
 * Content-Range: bytes 0-499/1234
 * </pre>
 *
 * @see org.apache.catalina.servlets.DefaultServlet.Range
 */
@Getter
@AllArgsConstructor
public class ContentRange {

    /**
     * 第一个字节的位置
     */
    private final long start;
    /**
     * 最后一个字节的位置
     */
    private long end;
    /**
     * 内容完整的长度/总长度
     */
    private final long length;

    public static final String BYTES_STRING = "bytes";

    /**
     * 组装内容范围的响应头。
     * <pre>
     * <a href="https://www.rfc-editor.org/rfc/rfc7233#section-4.2">
     *     4.2. Content-Range - HTTP/1.1 Range Requests</a>
     * Content-Range: "bytes" first-byte-pos "-" last-byte-pos  "/" complete-length
     *
     * For example:
     * Content-Range: bytes 0-499/1234
     * </pre>
     *
     * @return 内容范围的响应头
     */
    public String toContentRange() {
        return BYTES_STRING + ' ' + start + '-' + end + '/' + length;
//        return "bytes " + start + "-" + end + "/" + length;
    }

    /**
     * 计算内容完整的长度/总长度。
     *
     * @return 内容完整的长度/总长度
     */
    public long applyAsContentLength() {
        return end - start + 1;
    }

    /**
     * Validate range.
     *
     * @return true if the range is valid, otherwise false
     */
    public boolean validate() {
        if (end >= length) {
            end = length - 1;
        }
        return (start >= 0) && (end >= 0) && (start <= end) && (length > 0);
    }

    @Override
    public String toString() {
        return "firstBytePos=" + start +
                ", lastBytePos=" + end +
                ", fileLength=" + length;
    }
}

StreamProgressImpl

package com.example.insurance.common;

import cn.hutool.core.io.StreamProgress;
import lombok.AllArgsConstructor;
import lombok.extern.slf4j.Slf4j;

/**
 * 数据流进度条
 */
@Slf4j
@AllArgsConstructor
public class StreamProgressImpl implements StreamProgress {

    private final String fileName;

    @Override
    public void start() {
        log.info("start progress {}", fileName);
    }

    @Override
    public void progress(long total, long progressSize) {
        log.debug("progress {}, total={}, progressSize={}", fileName, total, progressSize);
    }

    @Override
    public void finish() {
        log.info("finish progress {}", fileName);
    }
}

MediaContentUtil

package com.example.insurance.common;

import java.net.URLDecoder;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;

/**
 * 文件内容辅助方法集
 */
public final class MediaContentUtil {

    public static String filePath() {
        String osName = System.getProperty("os.name");
        String filePath = "/data/files/";
        if (osName.startsWith("Windows")) {
            filePath = "D:\" + filePath;
        }
//        else if (osName.startsWith("Linux")) {
//            filePath = MediaContentConstant.FILE_PATH;
//        }
        else if (osName.startsWith("Mac") || osName.startsWith("Linux")) {
            filePath = "/home/admin" + filePath;
        }
        return filePath;
    }

    public static String encode(String fileName) {
        return URLEncoder.encode(fileName, StandardCharsets.UTF_8);
    }

    public static String decode(String fileName) {
        return URLDecoder.decode(fileName, StandardCharsets.UTF_8);
    }
}