之前記錄過一篇minio單機(jī)安裝及Springboot集成minio記錄,但是minioClient里帶的流傳輸?shù)纳蟼魑募椒m然是application/octet-stream的contentType卻不支持大文件上傳,實(shí)測一個(gè)3.5G的文件只能傳上去2G造成文件上傳不全角钩,還不會(huì)報(bào)錯(cuò)。
minio提供了MinioAsyncClient來實(shí)現(xiàn)異步大文件上傳休玩,主要用到這幾個(gè)方法:createMultipartUploadAsync:創(chuàng)建異步分片上傳請(qǐng)求
uploadPartAsync:執(zhí)行異步上傳分片
listPartsAsync:查詢分片數(shù)據(jù)
completeMultipartUploadAsync:完成異步分片上傳合并分片文件
實(shí)現(xiàn)的思路也基本是上面的順序createMultipartUploadAsync->uploadPartAsync->listPartsAsync->completeMultipartUploadAsync
原來的minioClient集成代碼這里省略,只記錄MinioAsyncClient使用步驟:
創(chuàng)建自定義分片多文件上傳工具類 MultipartMinioClient
package com.ly.mp.project.minio;
import java.io.IOException;
import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException;
import com.google.common.collect.Multimap;
import io.minio.CreateMultipartUploadResponse;
import io.minio.ListPartsResponse;
import io.minio.MinioAsyncClient;
import io.minio.ObjectWriteResponse;
import io.minio.UploadPartResponse;
import io.minio.errors.ErrorResponseException;
import io.minio.errors.InsufficientDataException;
import io.minio.errors.InternalException;
import io.minio.errors.InvalidResponseException;
import io.minio.errors.ServerException;
import io.minio.errors.XmlParserException;
import io.minio.messages.Part;
import ly.mp.project.common.minio.MultipartUploadCreateParam;
import ly.mp.project.common.minio.UploadPartCreateParam;
/**
* 自定義minio
* @date 2023-09-15
*/
public class MultipartMinioClient extends MinioAsyncClient {
public MultipartMinioClient(MinioAsyncClient client) {
super(client);
}
/**
* 創(chuàng)建分片上傳請(qǐng)求
*
* @param bucketName 存儲(chǔ)桶
* @param region 區(qū)域
* @param objectName 對(duì)象名
* @param headers 消息頭
* @param extraQueryParams 額外查詢參數(shù)
*/
public String multipartUpload(String bucket, String region, String object, Multimap<String, String> headers, Multimap<String, String> extraQueryParams)
throws IOException, InvalidKeyException, NoSuchAlgorithmException, InsufficientDataException, ServerException, InternalException,
XmlParserException, InvalidResponseException, ErrorResponseException, InterruptedException, ExecutionException {
CompletableFuture<CreateMultipartUploadResponse> response = this.createMultipartUploadAsync(bucket, region, object, headers, extraQueryParams);
return response.get().result().uploadId();
}
public String multipartUpload(MultipartUploadCreateParam param)
throws InvalidKeyException, NoSuchAlgorithmException, InsufficientDataException, ServerException, InternalException, XmlParserException, InvalidResponseException, ErrorResponseException, IOException, InterruptedException, ExecutionException {
return this.multipartUpload(param.getBucketName(), param.getRegion(), param.getObjectName(), param.getHeaders(), param.getExtraQueryParams());
}
/**
* 分片上傳
* @param bucketName Name of the bucket.
* @param region Region of the bucket (Optional).
* @param objectName Object name in the bucket.
* @param data Object data must be InputStream, RandomAccessFile, byte[] or String.
* @param length Length of object data.
* @param uploadId Upload ID.
* @param partNumber Part number.
* @param extraHeaders Extra headers for request (Optional).
* @param extraQueryParams Extra query parameters for request (Optional).
* @throws IOException
* @throws XmlParserException
* @throws NoSuchAlgorithmException
* @throws InternalException
* @throws InsufficientDataException
* @throws InvalidKeyException
*/
public CompletableFuture<UploadPartResponse> uploadPartAsync(String bucketName, String region, String objectName, Object data,
long length, String uploadId, int partNumber, Multimap<String, String> extraHeaders, Multimap<String, String> extraQueryParams)
throws InvalidKeyException, InsufficientDataException, InternalException, NoSuchAlgorithmException, XmlParserException, IOException {
return super.uploadPartAsync(bucketName, region, objectName, data, length, uploadId, partNumber, extraHeaders, extraQueryParams);
}
public CompletableFuture<UploadPartResponse> uploadPartAsync(UploadPartCreateParam param)
throws InvalidKeyException, InsufficientDataException, InternalException, NoSuchAlgorithmException, XmlParserException, IOException{
return this.uploadPartAsync(param.getBucketName(), param.getRegion(), param.getObjectName(), param.getData(), param.getLength(),
param.getUploadId(), param.getPartNumber(),param.getHeaders(),
param.getExtraQueryParams());
}
/**
* 完成分片上傳,執(zhí)行合并文件
*
* @param bucketName 存儲(chǔ)桶
* @param region 區(qū)域
* @param objectName 對(duì)象名
* @param uploadId 上傳ID
* @param parts 分片
* @param extraHeaders 額外消息頭
* @param extraQueryParams 額外查詢參數(shù)
*/
@Override
public CompletableFuture<ObjectWriteResponse> completeMultipartUploadAsync(String bucketName, String region, String objectName, String uploadId, Part[] parts, Multimap<String, String> extraHeaders, Multimap<String, String> extraQueryParams)
throws InsufficientDataException, InternalException, InvalidKeyException, IOException, NoSuchAlgorithmException, XmlParserException {
return super.completeMultipartUploadAsync(bucketName, region, objectName, uploadId, parts, extraHeaders, extraQueryParams);
}
public CompletableFuture<ObjectWriteResponse> completeMultipartUploadAsync(MultipartUploadCreateParam param)
throws InvalidKeyException, InsufficientDataException, InternalException, NoSuchAlgorithmException, XmlParserException, IOException{
return this.completeMultipartUploadAsync(param.getBucketName(), param.getRegion(), param.getObjectName(), param.getUploadId(), param.getParts(),
param.getHeaders(),param.getExtraQueryParams());
}
/**
* 查詢分片數(shù)據(jù)
*
* @param bucketName 存儲(chǔ)桶
* @param region 區(qū)域
* @param objectName 對(duì)象名
* @param uploadId 上傳ID
* @param extraHeaders 額外消息頭
* @param extraQueryParams 額外查詢參數(shù)
*/
@Override
public CompletableFuture<ListPartsResponse> listPartsAsync(String bucketName, String region, String objectName, Integer maxParts, Integer partNumberMarker, String uploadId, Multimap<String, String> extraHeaders, Multimap<String, String> extraQueryParams) throws InsufficientDataException, InternalException, InvalidKeyException, IOException, NoSuchAlgorithmException, XmlParserException {
return super.listPartsAsync(bucketName, region, objectName, maxParts, partNumberMarker, uploadId, extraHeaders, extraQueryParams);
}
public CompletableFuture<ListPartsResponse> listPartsAsync(MultipartUploadCreateParam param)
throws InvalidKeyException, InsufficientDataException, InternalException, NoSuchAlgorithmException, XmlParserException, IOException{
return this.listPartsAsync(param.getBucketName(), param.getRegion(), param.getObjectName(), param.getMaxParts(), param.getPartNumberMarker(),
param.getUploadId(), param.getHeaders(), param.getExtraQueryParams());
}
}
補(bǔ)充參數(shù)類MultipartUploadCreateParam
package ly.mp.project.common.minio;
import com.google.common.collect.Multimap;
import io.minio.messages.Part;
public class MultipartUploadCreateParam {
private String bucketName;
private String region;
private String objectName;
private Multimap<String, String> headers;
private Multimap<String, String> extraQueryParams;
private String uploadId;
private Integer maxParts;
private Part[] parts;
private Integer partNumberMarker;
public String getBucketName() {
return bucketName;
}
public void setBucketName(String bucketName) {
this.bucketName = bucketName;
}
public String getRegion() {
return region;
}
public void setRegion(String region) {
this.region = region;
}
public String getObjectName() {
return objectName;
}
public void setObjectName(String objectName) {
this.objectName = objectName;
}
public Multimap<String, String> getHeaders() {
return headers;
}
public void setHeaders(Multimap<String, String> headers) {
this.headers = headers;
}
public Multimap<String, String> getExtraQueryParams() {
return extraQueryParams;
}
public void setExtraQueryParams(Multimap<String, String> extraQueryParams) {
this.extraQueryParams = extraQueryParams;
}
public String getUploadId() {
return uploadId;
}
public void setUploadId(String uploadId) {
this.uploadId = uploadId;
}
public Integer getMaxParts() {
return maxParts;
}
public void setMaxParts(Integer maxParts) {
this.maxParts = maxParts;
}
public Part[] getParts() {
return parts;
}
public void setParts(Part[] parts) {
this.parts = parts;
}
public Integer getPartNumberMarker() {
return partNumberMarker;
}
public void setPartNumberMarker(Integer partNumberMarker) {
this.partNumberMarker = partNumberMarker;
}
}
補(bǔ)充參數(shù)類UploadPartCreateParam:
package ly.mp.project.common.minio;
import java.io.InputStream;
import com.google.common.collect.Multimap;
public class UploadPartCreateParam {
private String bucketName;
private String region;
private String objectName;
private InputStream data;
private long length;
private String uploadId;
private int partNumber;
private Multimap<String, String> headers;
private Multimap<String, String> extraQueryParams;
public String getBucketName() {
return bucketName;
}
public void setBucketName(String bucketName) {
this.bucketName = bucketName;
}
public String getRegion() {
return region;
}
public void setRegion(String region) {
this.region = region;
}
public String getObjectName() {
return objectName;
}
public void setObjectName(String objectName) {
this.objectName = objectName;
}
public InputStream getData() {
return data;
}
public void setData(InputStream data) {
this.data = data;
}
public long getLength() {
return length;
}
public void setLength(long length) {
this.length = length;
}
public String getUploadId() {
return uploadId;
}
public void setUploadId(String uploadId) {
this.uploadId = uploadId;
}
public int getPartNumber() {
return partNumber;
}
public void setPartNumber(int partNumber) {
this.partNumber = partNumber;
}
public Multimap<String, String> getHeaders() {
return headers;
}
public void setHeaders(Multimap<String, String> headers) {
this.headers = headers;
}
public Multimap<String, String> getExtraQueryParams() {
return extraQueryParams;
}
public void setExtraQueryParams(Multimap<String, String> extraQueryParams) {
this.extraQueryParams = extraQueryParams;
}
}
創(chuàng)建MultipartMinioConfiguration將MultipartMinioClient注入到Spring
package com.ly.mp.project.minio;
import java.lang.reflect.Field;
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import io.minio.MinioAsyncClient;
import io.minio.MinioClient;
@Configuration
public class MultipartMinioConfiguration {
@Bean
@ConditionalOnMissingBean({ MultipartMinioClient.class })
public MultipartMinioClient multipartMinioClient(MinioClient minioClient) throws Throwable {
try {
Field field = minioClient.getClass().getDeclaredField("asyncClient");
field.setAccessible(true);
return new MultipartMinioClient((MinioAsyncClient) field.get(minioClient));
} catch (Throwable ex) {
throw ex;
}
}
}
在MinioHandler里增加大文件上傳方法:
public MinioFile write(InputStream in, String fileName, String minioCustomDir, String contentType, boolean shard) {
MinioFile minioFile = new MinioFile();
//如果文件大于1G 則切分分片上傳
if(shard) {
try {
this.bigFileUpload(in, fileName, minioCustomDir, minioFile, contentType);
} catch (FileNotFoundException e) {
e.printStackTrace();
}
} else {
ObjectWriteResponse res = null;
try {
res = minioTemplate.upload(Paths.get(minioCustomDir + fileName), in, contentType);
} catch (MyMinioException e) {
throw new RuntimeException("minio上傳出錯(cuò):{}", e);
}
minioFile.setObjectName(res.object());
minioFile.setFileUrl(MinioPathUtils.getBaseUrl() + res.object());
minioFile.setInnerUrl(MinioPathUtils.getInnerBaseUrl() + res.object());
}
return minioFile;
}
private void bigFileUpload(InputStream in, String fileName, String minioCustomDir, MinioFile minioFile, String contentType) throws FileNotFoundException {
//文件切分侧纯,50M一個(gè)文件
String localPath = CommonUtils.getLocalRandomPath();
CommonUtils.createDirs(localPath);
FileUtil.cutFile(in, fileName, localPath, 50*1024*1024);
if(null != in) {
try {
in.close();
} catch (IOException e) {
e.printStackTrace();
}
}
/**構(gòu)建分片請(qǐng)求,拿到uploadId**/
MultipartUploadCreateParam param = new MultipartUploadCreateParam();
param.setBucketName(minioProperties.getPublicBucket());
param.setObjectName(MinioOperation.getPathToLinuxStr(Paths.get(minioCustomDir + fileName)));
Multimap<String, String> headers = HashMultimap.create();
headers.put("Content-Type", contentType);
param.setHeaders(headers);
//分片數(shù)量自己計(jì)算
// param.setExtraQueryParams(null);
// param.setMaxParts(null);
// param.setParts(null);
// param.setPartNumberMarker(null);
String uploadId = "";
try {
uploadId = multipartMinioClient.multipartUpload(param);
param.setUploadId(uploadId);
} catch (Exception e) {
e.printStackTrace();
CommonUtils.deleteFileAll(localPath);
throw new RuntimeException("multipartUpload接口報(bào)錯(cuò):" + e.getMessage());
}
File targetDirFile = new File(localPath);
int nameNum = 1;
if(targetDirFile.exists() && targetDirFile.isDirectory()) {
String nameNumDirPath = localPath + nameNum + "/";
String targetFilePath = nameNumDirPath + fileName;
File nameNumDir = new File(nameNumDirPath);
while(null != nameNumDir && nameNumDir.exists()) {
/**分片上傳**/
UploadPartCreateParam createParam = new UploadPartCreateParam();
createParam.setBucketName(param.getBucketName());
createParam.setObjectName(param.getObjectName());
createParam.setData(new FileInputStream(targetFilePath));
createParam.setLength(new File(targetFilePath).length());
createParam.setUploadId(param.getUploadId());
createParam.setPartNumber(nameNum);
createParam.setHeaders(param.getHeaders());
createParam.setExtraQueryParams(param.getExtraQueryParams());
try {
multipartMinioClient.uploadPartAsync(createParam).get();
} catch (Exception e) {
e.printStackTrace();
CommonUtils.deleteFileAll(localPath);
throw new RuntimeException("uploadPartAsync接口報(bào)錯(cuò):" + e.getMessage());
}
nameNum++;
nameNumDirPath = localPath + nameNum + "/";
targetFilePath = nameNumDirPath + fileName;
nameNumDir = new File(nameNumDirPath);
}
}
/**分片合并**/
param.setMaxParts(nameNum + 10);
param.setPartNumberMarker(0);
param.setParts(listParts(param));
try {
ObjectWriteResponse res = multipartMinioClient.completeMultipartUploadAsync(param).get();
minioFile.setObjectName(res.object());
minioFile.setFileUrl(MinioPathUtils.getBaseUrl() + res.object());
minioFile.setInnerUrl(MinioPathUtils.getInnerBaseUrl() + res.object());
} catch (Exception e) {
e.printStackTrace();
throw new RuntimeException("completeMultipartUploadAsync接口報(bào)錯(cuò):" + e.getMessage());
}finally {
//刪不干凈 后面加定時(shí)任務(wù)刪除 tmp文件夾下超一天的文件
CommonUtils.deleteFileAll(localPath);
}
}
public Part[] listParts(MultipartUploadCreateParam param) {
ListPartsResponse listMultipart;
try {
listMultipart = multipartMinioClient.listPartsAsync(param).get();
// LogUtils.info("listMultipart:{}", JsonUtils.writeValue(listMultipart.result().partList()));
} catch (Exception e) {
e.printStackTrace();
throw new RuntimeException("listPartsAsync接口報(bào)錯(cuò):" + e.getMessage());
}
return listMultipart.result().partList().toArray(new Part[]{});
}
主要實(shí)現(xiàn)代碼在上面的bigFileUpload里甲脏,將本地的大文件先切分成小文件眶熬,再一個(gè)一個(gè)按minio的步驟創(chuàng)建分片上傳,執(zhí)行分片上傳块请,合并分片返回最終鏈接聋涨,超過1g的才走大文件上傳,低于1g的還是走原來的upload方法负乡,大文件上傳成功后,不用擔(dān)心大文件下載問題脊凰,原來的minioClient讀取文件流的方法可以下載完整的大文件抖棘,這個(gè)已經(jīng)實(shí)證過。如果有需要http下載大文件的需求狸涌,可參考之前記錄的一篇:java大文件斷點(diǎn)續(xù)傳下載分段下載nginx206問題
下面附上完整的MinioHandler以及FileUtil:
package com.ly.mp.project.minio;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.io.FileUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import com.google.common.collect.HashMultimap;
import com.google.common.collect.Multimap;
import com.lianyou.minioutils.MinioTemplate;
import com.lianyou.minioutils.config.MinioProperties;
import com.lianyou.minioutils.exception.MyMinioException;
import com.lianyou.minioutils.util.MinioOperation;
import com.ly.mp.project.common.CommonUtils;
import com.ly.mp.project.utils.FileUtil;
import io.minio.ListPartsResponse;
import io.minio.ObjectWriteResponse;
import io.minio.StatObjectResponse;
import io.minio.messages.Item;
import io.minio.messages.Part;
import ly.mp.project.common.minio.MinioFile;
import ly.mp.project.common.minio.MinioPathUtils;
import ly.mp.project.common.minio.MultipartUploadCreateParam;
import ly.mp.project.common.minio.UploadPartCreateParam;
import ly.mp.project.common.otautils.DateUtil;
@Service
public class MinioHandler {
@Autowired
MinioTemplate minioTemplate;
@Autowired
MultipartMinioClient multipartMinioClient;
@Autowired
MinioProperties minioProperties;
public MinioFile write(InputStream in, String fileName, String minioCustomDir, String contentType, boolean shard) {
MinioFile minioFile = new MinioFile();
//如果文件大于1G 則切分分片上傳
if(shard) {
try {
this.bigFileUpload(in, fileName, minioCustomDir, minioFile, contentType);
} catch (FileNotFoundException e) {
e.printStackTrace();
}
} else {
ObjectWriteResponse res = null;
try {
res = minioTemplate.upload(Paths.get(minioCustomDir + fileName), in, contentType);
} catch (MyMinioException e) {
throw new RuntimeException("minio上傳出錯(cuò):{}", e);
}
minioFile.setObjectName(res.object());
minioFile.setFileUrl(MinioPathUtils.getBaseUrl() + res.object());
minioFile.setInnerUrl(MinioPathUtils.getInnerBaseUrl() + res.object());
}
return minioFile;
}
private void bigFileUpload(InputStream in, String fileName, String minioCustomDir, MinioFile minioFile, String contentType) throws FileNotFoundException {
//文件切分切省,50M一個(gè)文件
String localPath = CommonUtils.getLocalRandomPath();
CommonUtils.createDirs(localPath);
FileUtil.cutFile(in, fileName, localPath, 50*1024*1024);
if(null != in) {
try {
in.close();
} catch (IOException e) {
e.printStackTrace();
}
}
/**構(gòu)建分片請(qǐng)求,拿到uploadId**/
MultipartUploadCreateParam param = new MultipartUploadCreateParam();
param.setBucketName(minioProperties.getPublicBucket());
param.setObjectName(MinioOperation.getPathToLinuxStr(Paths.get(minioCustomDir + fileName)));
Multimap<String, String> headers = HashMultimap.create();
headers.put("Content-Type", contentType);
param.setHeaders(headers);
//分片數(shù)量自己計(jì)算
// param.setExtraQueryParams(null);
// param.setMaxParts(null);
// param.setParts(null);
// param.setPartNumberMarker(null);
String uploadId = "";
try {
uploadId = multipartMinioClient.multipartUpload(param);
param.setUploadId(uploadId);
} catch (Exception e) {
e.printStackTrace();
CommonUtils.deleteFileAll(localPath);
throw new RuntimeException("multipartUpload接口報(bào)錯(cuò):" + e.getMessage());
}
File targetDirFile = new File(localPath);
int nameNum = 1;
if(targetDirFile.exists() && targetDirFile.isDirectory()) {
String nameNumDirPath = localPath + nameNum + "/";
String targetFilePath = nameNumDirPath + fileName;
File nameNumDir = new File(nameNumDirPath);
while(null != nameNumDir && nameNumDir.exists()) {
/**分片上傳**/
UploadPartCreateParam createParam = new UploadPartCreateParam();
createParam.setBucketName(param.getBucketName());
createParam.setObjectName(param.getObjectName());
createParam.setData(new FileInputStream(targetFilePath));
createParam.setLength(new File(targetFilePath).length());
createParam.setUploadId(param.getUploadId());
createParam.setPartNumber(nameNum);
createParam.setHeaders(param.getHeaders());
createParam.setExtraQueryParams(param.getExtraQueryParams());
try {
multipartMinioClient.uploadPartAsync(createParam).get();
} catch (Exception e) {
e.printStackTrace();
CommonUtils.deleteFileAll(localPath);
throw new RuntimeException("uploadPartAsync接口報(bào)錯(cuò):" + e.getMessage());
}
nameNum++;
nameNumDirPath = localPath + nameNum + "/";
targetFilePath = nameNumDirPath + fileName;
nameNumDir = new File(nameNumDirPath);
}
}
/**分片合并**/
param.setMaxParts(nameNum + 10);
param.setPartNumberMarker(0);
param.setParts(listParts(param));
try {
ObjectWriteResponse res = multipartMinioClient.completeMultipartUploadAsync(param).get();
minioFile.setObjectName(res.object());
minioFile.setFileUrl(MinioPathUtils.getBaseUrl() + res.object());
minioFile.setInnerUrl(MinioPathUtils.getInnerBaseUrl() + res.object());
} catch (Exception e) {
e.printStackTrace();
throw new RuntimeException("completeMultipartUploadAsync接口報(bào)錯(cuò):" + e.getMessage());
}finally {
//刪不干凈 后面加定時(shí)任務(wù)刪除 tmp文件夾下超一天的文件
CommonUtils.deleteFileAll(localPath);
}
}
public Part[] listParts(MultipartUploadCreateParam param) {
ListPartsResponse listMultipart;
try {
listMultipart = multipartMinioClient.listPartsAsync(param).get();
// LogUtils.info("listMultipart:{}", JsonUtils.writeValue(listMultipart.result().partList()));
} catch (Exception e) {
e.printStackTrace();
throw new RuntimeException("listPartsAsync接口報(bào)錯(cuò):" + e.getMessage());
}
return listMultipart.result().partList().toArray(new Part[]{});
}
public MinioFile write(String localFilePath, String fileName, String minioCustomDir) {
return write(localFilePath, fileName, minioCustomDir, "application/octet-stream");
}
public MinioFile write(String localFilePath, String fileName, String minioCustomDir, String contentType) {
File file = new File(localFilePath);
boolean shard = false;
if(file.length() > 1*1024*1024*1024) {
shard = true;
}
try (InputStream in = new FileInputStream(localFilePath)){
return write(in, fileName, minioCustomDir, contentType, shard);
} catch (FileNotFoundException e) {
throw new RuntimeException("minio文件不存在:" + localFilePath);
} catch (IOException e) {
throw new RuntimeException("minio文件關(guān)閉失敗");
}
}
public MinioFile write(InputStream in, String fileName, boolean shard) {
return write(in, fileName, MinioPathUtils.getMinioDefaultDir() + fileName, "application/octet-stream", shard);
}
/**
*
* @param file 本地文件路徑
* @param fileName
* @return
*/
public MinioFile write(String filePath, String fileName) {
File file = new File(filePath);
boolean shard = false;
if(file.length() > 1*1024*1024*1024) {
shard = true;
}
try (InputStream in = new FileInputStream(file)){
return write(in, fileName, shard);
} catch (FileNotFoundException e) {
throw new RuntimeException("minio文件不存在:" + file);
} catch (IOException e) {
throw new RuntimeException("minio文件關(guān)閉失敗");
}
}
public void read(String objectName, File outFile) {
InputStream ins = null;
try {
ins = minioTemplate.getObject(Paths.get(objectName));
} catch (MyMinioException e) {
throw new RuntimeException("minio文件讀取失敗");
}
try {
FileUtils.copyInputStreamToFile(ins, outFile);
} catch (IOException e) {
throw new RuntimeException("minio文件讀取失敗1");
} finally {
if(null != ins) {
try {
ins.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
}
/**
*
* @param fileUrl
* @param outFilePath
* @return
*/
public void read(String fileUrl, String outFilePath) {
read(fileUrl.replace(MinioPathUtils.getBaseUrl(), ""), new File(outFilePath));
}
public void delete(String objectName) {
try {
minioTemplate.remove(Paths.get(objectName));
} catch (MyMinioException e) {
throw new RuntimeException("minio文件刪除失敗:{}", e);
}
}
public void delete(String fileUrl, boolean url) {
if(url) {
fileUrl = fileUrl.replace(MinioPathUtils.getBaseUrl(), "");
}
delete(fileUrl);
}
public boolean exists(String objectName) {
boolean exist = true;
try {
minioTemplate.getMetadata(Paths.get(objectName));
} catch (Exception e) {
exist = false;
}
return exist;
}
public boolean exists(String objectName, boolean url) {
if(url) {
objectName = objectName.replace(MinioPathUtils.getBaseUrl(), "");
}
return exists(objectName);
}
public MinioFile getFileInfo(String objectName) {
StatObjectResponse res = null;
try {
res = minioTemplate.getMetadata(Paths.get(objectName));
} catch (MyMinioException e) {
throw new RuntimeException("minio文件讀取文件詳情失敗:{}", e);
}
MinioFile minioFile = new MinioFile();
minioFile.setObjectName(res.object());
minioFile.setSize(res.size());
minioFile.setFileUrl(MinioPathUtils.getBaseUrl() + objectName);
minioFile.setInnerUrl(MinioPathUtils.getInnerBaseUrl() + objectName);
minioFile.setLastModifiedTimeStamp(DateUtil.datatimeToTimestamp(res.lastModified().toLocalDateTime()));
minioFile.setLastModifiedStr(DateUtil.dateToStr(res.lastModified().toLocalDateTime()));
return minioFile;
}
public MinioFile getFileInfo(String objectName, boolean url) {
if(url) {
objectName = objectName.replace(MinioPathUtils.getBaseUrl(), "");
}
return getFileInfo(objectName);
}
public List<MinioFile> getFileInfosByPath(String dir){
List<MinioFile> resultList = new ArrayList<>();
List<Item> list = minioTemplate.listFullPathObjects(Paths.get(dir));
if(!CollectionUtils.isEmpty(list)) {
for(Item item : list) {
MinioFile minioFile = new MinioFile();
minioFile.setObjectName(item.objectName());
minioFile.setSize(item.size());
minioFile.setFileUrl(MinioPathUtils.getBaseUrl() + item.objectName());
minioFile.setInnerUrl(MinioPathUtils.getInnerBaseUrl() + item.objectName());
minioFile.setLastModifiedTimeStamp(DateUtil.datatimeToTimestamp(item.lastModified().toLocalDateTime()));
minioFile.setLastModifiedStr(DateUtil.dateToStr(item.lastModified().toLocalDateTime()));
resultList.add(minioFile);
}
}
return resultList;
}
public List<MinioFile> getFileInfosByPath(String dir, boolean url){
if(url) {
dir = dir.replace(MinioPathUtils.getBaseUrl(), "");
}
return getFileInfosByPath(dir);
}
public static void main(String[] args) {
String localPath = "D:\\work\\workspace4\\zna-l2-fota\\ly.mp.zna.ota.platform.service\\tmp\\267ec8204c3241d2b3ce7a369a032690\\";
CommonUtils.deleteFileAll(localPath);
}
}
package com.ly.mp.project.utils;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.RandomAccessFile;
import java.util.ArrayList;
import java.util.List;
import ly.mp.project.common.otautils.DownloadClient;
public class FileUtil {
/**
* txt格式轉(zhuǎn)String
*
* @param txtPath
* @return
* @throws IOException
*/
public static String txtToStr(String txtPath) throws IOException {
StringBuilder buffer = new StringBuilder();
BufferedReader bf = null;
try {
bf = new BufferedReader(new InputStreamReader(new FileInputStream(txtPath), "UTF-8"));
String str = null;
while ((str = bf.readLine()) != null) {// 使用readLine方法帕胆,一次讀一行
buffer.append(new String(str.getBytes(), "UTF-8"));
}
} finally {
if(null != bf) bf.close();
}
String xml = buffer.toString();
return xml;
}
public static void downloadNet(String urlPath, String filePath) throws Exception {
DownloadClient.beginDowanload(0, urlPath, filePath);
}
public static void downloadBytes(byte[] bytes, String filePath) throws Exception {
File file = new File(filePath);
if(!file.exists()) {
file.createNewFile();
}
FileOutputStream fs = null;
try {
fs = new FileOutputStream(filePath);
fs.write(bytes);
} finally {
fs.close();
}
}
public static void appendFile(byte[] bytes, String fileName) throws IOException {
// 打開一個(gè)隨機(jī)訪問文件流朝捆,按讀寫方式
RandomAccessFile randomFile = new RandomAccessFile(fileName, "rw");
// 文件長度,字節(jié)數(shù)
long fileLength = randomFile.length();
// 將寫文件指針移到文件尾懒豹。
randomFile.seek(fileLength);
randomFile.write(bytes);
randomFile.close();
}
public static String getTmpDir() {
String tmpDir = "/tmp/";
String os = System.getProperty("os.name");
if (os.toLowerCase().contains("windows")) {
tmpDir = "D://tmp/";
File file = new File(tmpDir);
if (!file.exists())
file.mkdir();
}
return tmpDir;
}
public static void mkdir(String path) {
File file = new File(path);
if (!file.exists()) {
file.mkdir();
} else if (!file.isDirectory()) {
file.mkdir();
}
}
public static void deleteDir(String dirPath) {
File file = new File(dirPath);
File[] fileList = file.listFiles();
for (File f : fileList) {
if (f.exists()) {
if (f.isDirectory()) {
deleteDir(f.getAbsolutePath());
} else {
f.delete();
}
}
}
if (file.exists())
file.delete();
}
/**
*
* @param in 源file 流
* @param fileName 源文件名稱
* @param endDir 目標(biāo)文件目錄
* @param num 分割大小(字節(jié))
*/
public static void cutFile(InputStream in, String fileName, String endDir, int byteNum) {
try {
// 創(chuàng)建規(guī)定大小的byte數(shù)組
byte[] b = new byte[byteNum];
int len = 0;
// name為以后的小文件命名做準(zhǔn)備
int nameNum = 1;
// 遍歷將大文件讀入byte數(shù)組中芙盘,當(dāng)byte數(shù)組讀滿后寫入對(duì)應(yīng)的小文件中
while ((len = in.read(b)) != -1) {
File nameDir = new File(endDir + nameNum + "/");
if(!nameDir.exists()) {
nameDir.mkdirs();
}
// 分別找到原大文件的文件名和文件類型,為下面的小文件命名做準(zhǔn)備
FileOutputStream fos = new FileOutputStream(endDir + nameNum + "/" + fileName);
// 將byte數(shù)組寫入對(duì)應(yīng)的小文件中
fos.write(b, 0, len);
// 結(jié)束資源
fos.close();
nameNum++;
}
} catch (IOException e) {
e.printStackTrace();
} finally {
try {
if (in != null) {
// 結(jié)束資源
in.close();
}
} catch (IOException e) {
e.printStackTrace();
}
}
}
}
===20240610補(bǔ)充MinioFile和CommonUtils兩個(gè)類===
MinioFile:
package ly.mp.project.common.minio;
public class MinioFile {
private String fileUrl;
private String objectName;
private long size;
private long lastModifiedTimeStamp;//13位時(shí)間戳
private String lastModifiedStr;//yyyy-MM-dd HH:mm:ss
private String innerUrl;
public String getFileUrl() {
return fileUrl;
}
public void setFileUrl(String fileUrl) {
this.fileUrl = fileUrl;
}
public String getObjectName() {
return objectName;
}
public void setObjectName(String objectName) {
this.objectName = objectName;
}
public long getSize() {
return size;
}
public void setSize(long size) {
this.size = size;
}
public long getLastModifiedTimeStamp() {
return lastModifiedTimeStamp;
}
public void setLastModifiedTimeStamp(long lastModifiedTimeStamp) {
this.lastModifiedTimeStamp = lastModifiedTimeStamp;
}
public String getLastModifiedStr() {
return lastModifiedStr;
}
public void setLastModifiedStr(String lastModifiedStr) {
this.lastModifiedStr = lastModifiedStr;
}
public String getInnerUrl() {
return innerUrl;
}
public void setInnerUrl(String innerUrl) {
this.innerUrl = innerUrl;
}
}
CommonUtils:
package com.ly.mp.project.common;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.text.DecimalFormat;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.List;
import java.util.UUID;
import org.apache.commons.lang3.StringUtils;
import org.springframework.util.CollectionUtils;
import com.alibaba.fastjson.JSON;
import com.ly.mp.component.helper.StringHelper;
import com.ly.mp.springcloudnacos.nacos.NacosConfigs;
import ly.mp.project.common.otautils.DateUtil;
/**
* @description:
* @author: ly-yanzj
* @date: 2021/3/3 11:21
*/
public class CommonUtils {
public static final String LOCAL_TEMP_PATH = System.getProperty("user.dir") + "/tmp/";
/**
* 對(duì)象轉(zhuǎn)化成json字符串
*/
public static String toJsonStr(Object object) {
try {
return JSON.toJSONString(object);
} catch (Exception e) {
return object.toString();
}
}
/**
* 獲取當(dāng)前項(xiàng)目下的
*
* @return
*/
public static String getLocalRandomPath() {
String uid = UUID.randomUUID().toString().replaceAll("-", "");
return pathSeparateTransfer(LOCAL_TEMP_PATH + uid + "/");
}
/**
* 替換文件間隔路徑符
*
* @param filePath
* @return
*/
public static String pathSeparateTransfer(String filePath) {
if (File.separator.equals("\\")) {
filePath = filePath.replaceAll("/", "\\\\");
} else {
filePath = filePath.replaceAll("\\\\", "/");
}
return filePath;
}
/**
* 刪除文件下所有文件夾和文件
* file:文件對(duì)象
*/
public static void deleteFileAll(File file) {
if (file.exists()) {
File files[] = file.listFiles();
for (File file1: files) {
if (file1.isDirectory()) {
deleteFileAll(file1);
} else {
file1.delete();
}
}
file.delete();
}
}
/**
* 刪除文件下所有文件夾和文件
* path:文件名
*/
public static void deleteFileAll(String path) {
File file = new File(path);
deleteFileAll(file);
}
/**
* 創(chuàng)建多層級(jí)文件夾
*
* @param path
* @return
*/
public static boolean createDirs(String path) {
File fileDir = new File(path);
if (!fileDir.exists()) {
return fileDir.mkdirs();
}
return true;
}
/**
* 字節(jié)合并
* @param byte_1
* @param byte_2
* @return
*/
public static byte[] byteMerger(byte[] byte_1, byte[] byte_2){
byte[] byte_3 = new byte[byte_1.length+byte_2.length];
System.arraycopy(byte_1, 0, byte_3, 0, byte_1.length);
System.arraycopy(byte_2, 0, byte_3, byte_1.length, byte_2.length);
return byte_3;
}
/**
* 16進(jìn)制的字符串轉(zhuǎn)成字節(jié)
* @param hexStr 樣例-3031300d060960864801650304020105000420
* @return
* @throws Exception
*/
public static byte[] hexStringToBytes(String hexStr) throws Exception {
byte[] baKeyword = new byte[hexStr.length() / 2];
for (int i = 0; i < baKeyword.length; i++) {
try {
baKeyword[i] = (byte) (0xff & Integer.parseInt(hexStr.substring(i * 2, i * 2 + 2), 16));
} catch (Exception e) {
throw new Exception("前置固定加密十六進(jìn)制字符串處理異常脸秽!");
}
}
return baKeyword;
}
/**
* 整數(shù)轉(zhuǎn)字節(jié)
*
* @param i
* @return
*/
public static byte[] intToByte4(int i) {
byte[] targets = new byte[4];
targets[3] = (byte) (i & 0xFF);
targets[2] = (byte) (i >> 8 & 0xFF);
targets[1] = (byte) (i >> 16 & 0xFF);
targets[0] = (byte) (i >> 24 & 0xFF);
return targets;
}
public static byte[] unsignedShortToByte2(int s) {
byte[] targets = new byte[2];
targets[0] = (byte) (s >> 8 & 0xFF);
targets[1] = (byte) (s & 0xFF);
return targets;
}
/**
* 字節(jié)打印成二進(jìn)制字符
*
* @param toByteArray
* @return
*/
public static String showBytes(byte[] toByteArray) {
StringBuilder sb = new StringBuilder();
for (byte tByte : toByteArray) {
sb.append(Integer.toBinaryString((tByte & 0xFF) + 0x100).substring(1));
}
return sb.toString();
}
public static String getCurrentDay() {
SimpleDateFormat format = new SimpleDateFormat("yyyyMMdd");
return format.format(new Date());
}
public static String checkSuperUserDomain(String paramDomainId, String headDomainId) {
String domainId = StringUtils.isNotBlank(paramDomainId) ? paramDomainId : headDomainId;
if (StringUtils.isBlank(domainId)) {
return null;
}
return domainId.equals(NacosConfigs.getProp("super.user.domain.id")) ? null : domainId;
}
/**
* 讀取文本文件轉(zhuǎn)為字符
*
* @param filePath 文件全路徑
* @param charset 編碼符
* @param lineSeparator 是否保留原換行符
* @return
* @throws IOException
*/
public static String readTextFile(String filePath, String charset, boolean lineSeparator) throws IOException {
charset = StringUtils.isBlank(charset) ? "UTF-8" : charset;
StringBuilder sb;
try (BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(filePath), charset))) {
sb = new StringBuilder();
String line;
while ((line = br.readLine()) != null) {
sb.append(line);
if (lineSeparator) {
sb.append(System.lineSeparator());
}
}
// 去除末尾換行符
int sbLen = sb.length();
if (lineSeparator && sbLen > 0) {
sb.delete(sbLen - System.lineSeparator().length(), sbLen);
}
} catch (IOException e) {
throw e;
}
return sb.toString();
}
/**
* 將字符串輸出到文件
*
* @param filePath 輸出的文件夾路徑
* @param fileName 輸出的文件名稱
* @param content 輸出的內(nèi)容
* @param charset 字符編碼
*/
public static void writeTextFile(String filePath, String fileName, String content, String charset) throws IOException {
charset = StringUtils.isBlank(charset) ? "UTF-8" : charset;
createDirs(filePath);
try (BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(filePath + fileName), charset))) {
bw.write(content);
} catch (IOException e) {
throw e;
}
}
/**
* 將字符串輸出到文件
* @param filePathName
* @param content
* @param charset
* @throws IOException
*/
public static void writeTextFile(String filePathName, String content, String charset) throws IOException {
charset = StringUtils.isBlank(charset) ? "UTF-8" : charset;
try (BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(filePathName), charset))) {
bw.write(content);
} catch (IOException e) {
throw e;
}
}
public static void main(String[] args) {
//hexStringToBytes
}
public static String getStringByList(String split, List<String> list) {
String result = "";
StringBuilder builder = new StringBuilder();
if(!CollectionUtils.isEmpty(list)) {
for(String str : list) {
builder.append(str + split);
}
if(StringUtils.isNotBlank(builder.toString())) {
result = builder.toString().substring(0, builder.toString().length() - 1);
}
}
return result;
}
public static String getRemoteTmpDir() {
return NacosConfigs.getProp("tk.minio.path") +"/tmp/"+ DateUtil.getSystemDate("yyyyMM") + "/" + StringHelper.GetGUID() + "/";
}
/**
* 判斷是否是字母或數(shù)字字符儒老,是返回true,否則返回false
* @param reference
* @return
*/
public static boolean isLetterOrDigit(String reference) {
for(int i = 0; i < reference.length(); i++) {
if(!Character.isLetterOrDigit(reference.charAt(i))) return false;
}
return true;
}
public static String formatFileSize(Long fileLength) {
String fileSizeString = "";
if (fileLength == null) {
return fileSizeString;
}
DecimalFormat df = new DecimalFormat("#.00");
if (fileLength < 1024) {
fileSizeString = df.format((double) fileLength) + "B";
}
else if (fileLength < 1048576) {
fileSizeString = df.format((double) fileLength / 1024) + "K";
}
else if (fileLength < 1073741824) {
fileSizeString = df.format((double) fileLength / 1048576) + "M";
}
else {
fileSizeString = df.format((double) fileLength / 1073741824) + "G";
}
return fileSizeString;
}
public static String getRemoteTmpDir(String dir) {
return dir + DateUtil.getSystemDate("yyyyMM") + "/" + StringHelper.GetGUID() + "/";
}
}