邯郸信息港房屋出售_模板建站有什么不好?_网站seo快速优化技巧_上海建站seo

一、前言
最近在项目中需要实现大文件上传到minio，那么这时候传统的上传方式就不适用了。
这里分析下分片上传有哪些好处：
1.大文件单次传输时，网络波动或中断会导致整个文件需重新上传‌，分片上传允许单独重传失败的分片，避免全量重传‌。
2.单次上传大文件需占用与文件等量的内存资源（如10GB文件需预留10GB内存），而分片上传通过拆分文件降低单次内存占用，提升服务器吞吐量‌。
3.传统上传在大文件上传时效率更低，而分片支持并行上传，例如多个分片同时传输可显著缩短总耗时，尤其适用于跨国传输或高延迟网络环境‌。

二、如何实现文件分片上传至minio？
1.添加maven依赖。

<dependency><groupId>io.minio</groupId><artifactId>minio</artifactId><version>8.4.3</version><exclusions><exclusion><groupId>com.squareup.okhttp3</groupId><artifactId>okhttp</artifactId></exclusion></exclusions></dependency><dependency><groupId>com.squareup.okhttp3</groupId><artifactId>okhttp</artifactId><version>4.10.0</version></dependency>

2.minio连接配置
2.1、配置minio属性

import com.smartcitysz.dp.upload.constants.PlatformEnum;
import lombok.Data;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component;@Data
@Component
@ConfigurationProperties(prefix = "minio")
public class MinioProperties {private String accessKey;private String secretKey;private String endpoint;private String bucket;/*** 访问域名*/private String domain = "";/*** 启用存储*/private Boolean enableStorage = true;/*** 存储平台*/private PlatformEnum platform = PlatformEnum.MINIO;/*** 基础路径*/private String basePath = "";
}

2.2、配置minio连接，代码如下：

import com.smartcitysz.dp.minio.MinioService;
import com.smartcitysz.dp.minio.utils.MinioUtils;
import io.minio.MinioClient;
import org.springframework.boot.autoconfigure.condition.ConditionalOnBean;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;@Configuration
@EnableConfigurationProperties(MinioProperties.class )
public class MinioConfiguration {@Bean@ConditionalOnBean(MinioProperties.class)public MinioClient minioClient(MinioProperties minioProperties) {return MinioClient.builder().endpoint(minioProperties.getEndpoint()).credentials(minioProperties.getAccessKey(), minioProperties.getSecretKey()).build();}
}

2.3、在YAML添加minio配置

minio:endpoint: http://xxxxxx:1222accessKey: xxxxxxxxsecretKey: xxxxxxxxxxxbucket: my-bucket

3.编写实现文件分片上传的Service。

@Service
@Slf4j
@RefreshScope
public class  ShardUploadService {@Autowiredprivate MinioClient minioClient;private ExecutorService executorService;@Value("${minio.bigfile.tmpBucketName:bigfileupload-temp}")private String tmpBucketName;@Value("${minio.bigfile.actualBucketName:bigfileupload-actual}")private String bucketName;@PostConstructpublic void init() throws Exception{if (!minioClient.bucketExists(BucketExistsArgs.builder().bucket(bucketName).build())) {minioClient.makeBucket(MakeBucketArgs.builder().bucket(bucketName).build());log.info("ShardUploadServiceImpl makeBucket bucketName init success！ ");}if (!minioClient.bucketExists(BucketExistsArgs.builder().bucket(tmpBucketName).build())) {minioClient.makeBucket(MakeBucketArgs.builder().bucket(tmpBucketName).build());log.info("ShardUploadServiceImpl makeBucket tmpBucketName init success！ ");}this.executorService =Executors.newFixedThreadPool(Integer.parseInt(mergeThreadCount),new ThreadFactoryBuilder().setNameFormat("uploadfile-merge-service-%d").build());log.info("ShardUploadServiceImpl makeBucket init success！ ");public void uploadFileToMinio(MultipartFileParamDTO param) throws Exception {log.info("文件分片上传，参数=[{}]", param);String md5 = param.getMd5();//总分片数int totalPieces = param.getChunks();//当前分片int sliceIndex = param.getChunk();// 可以最后一个分片传文件名用来保存String fileName = param.getName();LambdaQueryWrapper<FileSplitInfo> queryWrapper = new LambdaQueryWrapper<>();queryWrapper.eq(FileSplitInfo::getFileId, Long.parseLong(param.getFileId()));queryWrapper.eq(FileSplitInfo::getAppId, param.getAppId());queryWrapper.eq(FileSplitInfo::getChunk, param.getChunk());FileSplitInfo splitInfo = splitInfoDao.selectOne(queryWrapper);if(Objects.nonNull(splitInfo)) {log.info("文件id:[{}]，分片序号:[{}]，已经上传成功，不需要再传",param.getFileId(),param.getChunk());return;}//上传文件MultipartFile file = param.getFile();// 上传try {// 设置分片大小为50MBlong partSize = 50 * 1024 * 1024; // 50MBminioClient.putObject(PutObjectArgs.builder().bucket(tmpBucketName).object(md5.concat("/").concat(Integer.toString(sliceIndex))).stream(file.getInputStream(), file.getSize(), partSize).contentType(file.getContentType()).build());} catch (Exception e) {throw e;}//维护分片信息createFileSplitInfo(param);}/*** 这里是将文件分片信息插入数据库*/private void createFileSplitInfo(MultipartFileParamDTO param){FileSplitInfo info = new FileSplitInfo();Long fileId = Long.parseLong(param.getFileId());info.setId(snowflakeGenerator.next());info.setAppId(param.getAppId());info.setFileId(fileId);info.setChunk(param.getChunk());info.setSize(param.getSize());info.setMd5(param.getMd5());info.setCreateTime(new Date());info.setDeleted(false);splitInfoDao.insert(info);}

其中MultipartFileParamDTO定义的请求参数。

@EqualsAndHashCode(callSuper = true)
@Data
@ToString
public class MultipartFileParamDTO extends BaseQuery {@ApiModelProperty(value = "应用code")private String appId;/*** 文件id*/@NotNull(message = "文件id不能为空")private String fileId;/*** 用户id*/private String uid;/*** 任务ID*/private String id;/*** 总分片数量*/private int chunks;/*** 当前为第几块分片*/private int chunk;/*** 当前分片大小*/private long size = 0L;/*** 文件名*/private String name;/*** 分片对象*/private MultipartFile file;/*** MD5*/private String md5;//是否继续上传 true是private boolean isUploadContinued = false;public String getObjectName() {return md5.concat("/").concat(name);}public String getFileAbsolutePath() {return "/".concat(getObjectName());}
}

4.上传完成后，就是合并文件了。

public String uploadFileMerge(MultipartFileParamDTO param) throws Exception {log.info("文件上传完成，合并文件，参数=[{}]", param);String md5 = param.getMd5();//总分片数int totalPieces = param.getChunks();String fileName = param.getName();// 可以最后一个分片传文件名用来保存String objectName = param.getObjectName();try {//判断是否全部上传完成if(isUploadComplete(md5,totalPieces)) {int count = updateFileMergeing(param, FileMergeEnum.MERGEING);if(count < 1) {log.info("文件已合并完成，不用再合并，参数=[{}]", param);return objectName;}CompletableFuture.runAsync(() -> {try {//合并文件mergeFile(totalPieces,md5,objectName);// 删除所有的分片文件deleteTempFile(totalPieces,md5);log.info("uploadFileToMinioMerge完成上传.....");} catch (Exception e) {throw new RuntimeException(e);}}, executorService).exceptionally(throwable->{log.error("异步merge文件异常", throwable);return null;});}return objectName;} catch (Exception e) {throw e;}}/*** 是否上传完成* @param md5* @param totalPieces* @return* @throws Exception*/private boolean isUploadComplete(String md5,int totalPieces) throws Exception{Iterable<Result<Item>> results = minioClient.listObjects(ListObjectsArgs.builder().bucket(tmpBucketName).prefix(md5.concat("/")).build());Set<String> objectNames = Sets.newHashSet();for (Result<Item> item : results) {objectNames.add(item.get().objectName());}return objectNames.size()==totalPieces;}/*** 合并文件* @param totalPieces* @param md5* @param fileName* @throws Exception*/public void mergeFile(int totalPieces,String md5,String fileName) throws Exception{// 完成上传从缓存目录合并迁移到正式目录List<ComposeSource> sourceObjectList = Stream.iterate(0, i -> ++i).limit(totalPieces).map(i -> ComposeSource.builder().bucket(tmpBucketName).object(md5.concat("/").concat(Integer.toString(i))).build()).collect(Collectors.toList());minioClient.composeObject(ComposeObjectArgs.builder().bucket(bucketName).object(fileName).sources(sourceObjectList).build());}/*** 删除临时文件* @param totalPieces* @param md5*/public void deleteTempFile(int totalPieces,String md5) throws Exception{List<DeleteObject> delObjects = Stream.iterate(0, i -> ++i).limit(totalPieces).map(i -> new DeleteObject(md5.concat("/").concat(Integer.toString(i)))).collect(Collectors.toList());Iterable<Result<DeleteError>> results = minioClient.removeObjects(RemoveObjectsArgs.builder().bucket(tmpBucketName).objects(delObjects).build());// 遍历删除结果，处理可能的错误for (Result<DeleteError> result : results) {DeleteError error = result.get();if (error != null) {log.info("Error in deleting object " + error.objectName() + "; " + error.message());throw new BizException("删除临时分片失败，objectName：" + error.objectName() + "; " + error.message());}}log.info("删除临时分片:{}","success....");}

这样就完成了大文件从分片上传minio到合并的整个过程。

总结：
1.分片上传通过‌降低单次传输负载、支持断点续传、提升容错能力‌，成为大文件传输场景的最优解。
2.分片上传解决了传统上传的痛点，提升效率、稳定性和用户体验，所以大文件推荐用分片上传。