jhynsoo/worlabel
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Refactor: 이미지 서비스 멀티 스레드 도입하여 최적화

Browse Source
This commit is contained in:
김용수 2024-09-29 01:42:45 +09:00
parent 0d7cf263fc
commit db54b11a5f
5 changed files with 171 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
backend/src/main/java/com/worlabel/domain/image/entity/Image.java
View File

@ -19,7 +19,7 @@ public class Image extends BaseEntity {
*/
@Id
@Column(name = "project_image_id", nullable = false)
@GeneratedValue(strategy = GenerationType.IDENTITY)
@GeneratedValue(strategy = GenerationType.SEQUENCE)
private Long id;
/**

70
backend/src/main/java/com/worlabel/domain/image/service/ImageAsyncService.java Normal file
View File

@ -0,0 +1,70 @@
package com.worlabel.domain.image.service;
import com.worlabel.domain.folder.entity.Folder;
import com.worlabel.domain.image.entity.Image;
import com.worlabel.domain.image.repository.ImageRepository;
import com.worlabel.global.exception.CustomException;
import com.worlabel.global.exception.ErrorCode;
import com.worlabel.global.service.S3UploadService;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.scheduling.annotation.Async;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.web.multipart.MultipartFile;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.CompletableFuture;
@Slf4j
@Service
@RequiredArgsConstructor
public class ImageAsyncService {
private final S3UploadService s3UploadService;
private final ImageRepository imageRepository;
private final ThreadPoolTaskExecutor imageUploadExecutor;
@Transactional
@Async("imageUploadExecutor")
public CompletableFuture<Void> asyncImageUpload(final List<MultipartFile> imageList, final Folder folder, final Integer projectId) {
log.debug("현재 스레드 - {} 업로드 파일 개수 - {}, 현재 작업 큐 용량 - {}",
Thread.currentThread().getName(),
imageList.size(),
imageUploadExecutor.getThreadPoolExecutor().getQueue().size()); // 큐에 쌓인 작업 수 출력);
imageList.forEach(file -> {
try{
String extension = getExtension(file.getOriginalFilename());
String imageKey = s3UploadService.uploadImageFile(file, extension, projectId);
createImage(file, imageKey, folder);
}catch (Exception e){
log.error("이미지 업로드 실패: {}", file.getOriginalFilename(), e);
throw new CustomException(ErrorCode.FAIL_TO_CREATE_FILE);
}
});
log.debug("배치 처리 완료");
return CompletableFuture.completedFuture(null);
}
public void createImage(MultipartFile file, String imageKey, Folder folder) {
try {
String name = file.getOriginalFilename();
String extension = getExtension(name);
Image image = Image.of(name, imageKey, extension, folder);
imageRepository.save(image);
}catch (Exception e){
log.debug("이미지 DB 저장 실패: ", e);
s3UploadService.deleteImageFromS3(imageKey);
throw new CustomException(ErrorCode.FAIL_TO_CREATE_FILE);
}
}
private String getExtension(String fileName) {
return fileName.substring(fileName.lastIndexOf(".") + 1);
}
}

90
backend/src/main/java/com/worlabel/domain/image/service/ImageService.java
View File

@ -26,13 +26,14 @@ import org.springframework.web.multipart.MultipartFile;
import java.io.File;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.CompletableFuture;
@Slf4j
@Service
@ -41,6 +42,7 @@ import java.util.List;
public class ImageService {
private final ProjectRepository projectRepository;
private final ImageAsyncService imageAsyncService;
private final FolderRepository folderRepository;
private final S3UploadService s3UploadService;
private final ImageRepository imageRepository;
@ -51,25 +53,34 @@ public class ImageService {
@CheckPrivilege(value = PrivilegeType.EDITOR)
public void uploadImageList(final List<MultipartFile> imageList, final Integer folderId, final Integer projectId) {
Folder folder = getOrCreateFolder(folderId, projectId);
long prev = System.currentTimeMillis();
// 이미지 리스트를 순차적으로 처리 (향후 비동기/병렬 처리를 위해 분리된 메서드 호출)
imageList.forEach(file -> uploadAndSave(file, folder, projectId));
}
// 동적 배치 크기 계산
int totalImages = imageList.size();
int batchSize;
/**
* 폴더 생성 또는 가져오기
* 폴더 생성 작업이 있음으로 트랜잭션 보호
*/
public Folder getOrCreateFolder(Integer folderId, Integer projectId) {
if (folderId != 0) {
return getFolder(folderId);
if (totalImages <= 100) {
batchSize = 25; // 작은 이미지 수는 작은 배치 크기
} else if (totalImages <= 3000) {
batchSize = 50; // 중간 이미지 수는 중간 배치 크기
} else {
String currentDateTime = LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"));
Project project = getProject(projectId);
Folder folder = Folder.of(currentDateTime, null, project);
folderRepository.save(folder); // 새로운 폴더를 저장
return folder;
batchSize = 100; // 큰 이미지 수는 큰 배치 크기
}
List<CompletableFuture<Void>> futures = new ArrayList<>();
for (int i = 0; i < imageList.size(); i += batchSize) {
List<MultipartFile> batch = imageList.subList(i, Math.min(i + batchSize, imageList.size()));
CompletableFuture<Void> future = imageAsyncService.asyncImageUpload(batch, folder, projectId);
// 모든 비동기 작업이 완료될 때까지 기다림
futures.add(future);
}
// 모든 비동기 작업이 완료될 때까지 기다림
CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])).join();
long after = System.currentTimeMillis();
log.debug("업로드 완료 - 경과시간 {}", ((double) after - prev) / 1000);
}
/**
@ -197,7 +208,7 @@ public class ImageService {
Files.createDirectories(newPath);
} else {
Files.createDirectories(newPath.getParent());
try(OutputStream os = Files.newOutputStream(newPath)){
try (OutputStream os = Files.newOutputStream(newPath)) {
IOUtils.copy(zis, os);
}
}
@ -243,6 +254,22 @@ public class ImageService {
.orElseThrow(() -> new CustomException(ErrorCode.DATA_NOT_FOUND));
}
/**
* 폴더 생성 또는 가져오기
* 폴더 생성 작업이 있음으로 트랜잭션 보호
*/
public Folder getOrCreateFolder(Integer folderId, Integer projectId) {
if (folderId != 0) {
return getFolder(folderId);
} else {
String currentDateTime = LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"));
Project project = getProject(projectId);
Folder folder = Folder.of(currentDateTime, null, project);
folderRepository.save(folder); // 새로운 폴더를 저장
return folder;
}
}
// 이미지 가져오면서 프로젝트 소속 여부를 확인
private Image getImageByIdAndFolderIdAndFolderProjectId(final Integer folderId, final Long imageId, final Integer projectId) {
return imageRepository.findByIdAndFolderIdAndFolderProjectId(imageId, folderId, projectId)
@ -250,13 +277,13 @@ public class ImageService {
}
/*
공통 로직
공통 로직
*/
/**
* MultipartFile 업로드 및 저장
*/
private void uploadAndSave(MultipartFile file, Folder folder, int projectId) {
public void uploadAndSave(MultipartFile file, Folder folder, int projectId) {
try {
String key = uploadToS3(file, projectId);
saveImage(file, key, folder);
@ -274,7 +301,6 @@ public class ImageService {
String key = uploadToS3(file, project.getId());
saveImage(file, key, folder);
} catch (Exception e) {
log.error("파일 업로드 중 오류 발생", e);
throw new CustomException(ErrorCode.FAIL_TO_CREATE_FILE, "이미지 업로드 중 오류 발생");
}
}
@ -284,7 +310,7 @@ public class ImageService {
*/
public void saveImage(final MultipartFile file, final String key, final Folder folder) {
try {
Image image = createImage(file, key, folder);
Image image = createImage(file.getOriginalFilename(), key, folder);
imageRepository.save(image);
} catch (Exception e) {
log.error("이미지 DB 저장 실패 원인: ", e);
@ -298,7 +324,7 @@ public class ImageService {
*/
public void saveImage(final File file, final String key, final Folder folder) {
try {
Image image = createImage(file, key, folder);
Image image = createImage(file.getName(), key, folder);
imageRepository.save(image);
} catch (Exception e) {
s3UploadService.deleteImageFromS3(key);
@ -309,9 +335,10 @@ public class ImageService {
/**
* S3 파일 업로드
*/
private String uploadToS3(final MultipartFile file, final Integer projectId) {
public String uploadToS3(final MultipartFile file, final Integer projectId) {
String extension = getExtension(file.getOriginalFilename());
return s3UploadService.uploadMultipartFile(file, extension, projectId);
// return s3UploadService.uploadImageFile(file, extension, projectId);
return "";
}
/**
@ -319,17 +346,12 @@ public class ImageService {
*/
private String uploadToS3(final File file, final Integer projectId) {
String extension = getExtension(file.getName());
return s3UploadService.uploadFile(file, extension, projectId);
return s3UploadService.uploadImageFile(file, extension, projectId);
}
public Image createImage(MultipartFile file, String key, Folder folder) {
String extension = getExtension(file.getOriginalFilename());
log.debug("이미지 업로드 이름 :{}",file.getOriginalFilename());
return Image.of(file.getOriginalFilename(), key, extension, folder);
}
public Image createImage(File file, String key, Folder folder) {
String extension = getExtension(file.getName());
return Image.of(file.getName(), key, extension, folder);
public Image createImage(String fileName, String key, Folder folder) {
String extension = getExtension(fileName);
log.debug("이미지 업로드 이름 :{}", fileName);
return Image.of(fileName, key, extension, folder);
}
}

16
backend/src/main/java/com/worlabel/global/config/AsyncConfig.java
View File

@ -1,10 +1,26 @@
package com.worlabel.global.config;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.scheduling.annotation.EnableAsync;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
@EnableAsync
@Configuration
public class AsyncConfig {
@Bean(name = "imageUploadExecutor")
public ThreadPoolTaskExecutor imageUploadExecutor(){
ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
// 최적 스레드 수 = CPU 코어 수 * (1 + (I/O 작업 시간 / CPU 작업 시간))
executor.setCorePoolSize(5);
executor.setMaxPoolSize(5);
executor.setQueueCapacity(100);
executor.setThreadNamePrefix("imageUploadExecutor-");
executor.initialize();
return executor;
}
}

50
backend/src/main/java/com/worlabel/global/service/S3UploadService.java
View File

@ -8,20 +8,20 @@ import com.worlabel.global.exception.ErrorCode;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.util.StringUtils;
import org.springframework.web.multipart.MultipartFile;
import javax.imageio.ImageIO;
import java.awt.image.BufferedImage;
import java.io.*;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLDecoder;
import java.nio.charset.StandardCharsets;
import java.util.UUID;
import java.util.concurrent.CompletableFuture;
// TODO: 추후 비동기로 변경해야합니다.
@Slf4j
@Service
@RequiredArgsConstructor
@ -60,26 +60,15 @@ public class S3UploadService {
}
}
/**
* MultipartFile 업로드
*/
public String uploadMultipartFile(final MultipartFile file, final String extension, final Integer projectId) {
try {
return uploadImageToS3(file.getInputStream(), extension, projectId);
} catch (IOException e) {
log.debug("MultipartFile 업로드 에러 발생 ", e);
throw new CustomException(ErrorCode.FAIL_TO_CREATE_FILE);
}
}
/**
* File 업로드
*/
public String uploadFile(final File file, final String extension, final Integer projectId) {
try (InputStream inputStream = new FileInputStream(file)) {
public String uploadImageFile(final MultipartFile file, final String extension, final Integer projectId) {
try (InputStream inputStream = file.getInputStream()) {
return uploadImageToS3(inputStream, extension, projectId);
} catch (IOException e) {
log.debug("이미지 업로드에서 에러 발생 ", e);
log.debug("MultipartFile 업로드 에러 발생 {}",file.getOriginalFilename(), e);
throw new CustomException(ErrorCode.FAIL_TO_CREATE_FILE);
}
}
@ -87,22 +76,39 @@ public class S3UploadService {
/**
* AWS S3 이미지 업로드
*/
private String uploadImageToS3(final InputStream inputStream, final String extension, final Integer projectId) throws IOException {
public String uploadImageToS3(final InputStream inputStream, final String extension, final Integer projectId) throws IOException {
String s3Key = getS3FileName(projectId);
String s3FileName = s3Key + "." + extension;
ObjectMetadata metadata = new ObjectMetadata(); // S3에 업로드할 파일의 메타데이터 설정
metadata.setContentType("image/" + extension); // 콘텐츠 타입 설정
metadata.setContentLength(inputStream.available());
uploadToS3(inputStream, s3FileName, metadata);
PutObjectRequest putRequest = new PutObjectRequest(bucket, s3FileName, inputStream, metadata);
amazonS3.putObject(putRequest);
return url + "/" + s3Key;
}
/**
* InputStream 사용 파일을 S3 업로드
* File 업로드
*/
private void uploadToS3(InputStream inputStream, final String s3Key, final ObjectMetadata metadata) {
public String uploadImageFile(final File file, final String extension, final Integer projectId) {
try (InputStream inputStream = new FileInputStream(file)) {
// return uploadImageToS3(inputStream, extension, projectId);
return "";
} catch (IOException e) {
log.debug("이미지 업로드에서 에러 발생 ", e);
throw new CustomException(ErrorCode.FAIL_TO_CREATE_FILE);
}
}
/**
* InputStream 사용해서 파일을 S3 업로드
*/
public void uploadToS3(InputStream inputStream, final String s3Key, final ObjectMetadata metadata) {
try {
byte[] bytes = IOUtils.toByteArray(inputStream);