From db54b11a5f01c33caa01b61a512597286b4de4ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EC=9A=A9=EC=88=98?= Date: Sun, 29 Sep 2024 01:42:45 +0900 Subject: [PATCH] =?UTF-8?q?Refactor:=20=EC=9D=B4=EB=AF=B8=EC=A7=80=20?= =?UTF-8?q?=EC=84=9C=EB=B9=84=EC=8A=A4=20=EB=A9=80=ED=8B=B0=20=EC=8A=A4?= =?UTF-8?q?=EB=A0=88=EB=93=9C=20=EB=8F=84=EC=9E=85=ED=95=98=EC=97=AC=20?= =?UTF-8?q?=EC=B5=9C=EC=A0=81=ED=99=94?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../worlabel/domain/image/entity/Image.java | 2 +- .../image/service/ImageAsyncService.java | 70 +++++++++++++++ .../domain/image/service/ImageService.java | 90 ++++++++++++------- .../worlabel/global/config/AsyncConfig.java | 16 ++++ .../global/service/S3UploadService.java | 50 ++++++----- 5 files changed, 171 insertions(+), 57 deletions(-) create mode 100644 backend/src/main/java/com/worlabel/domain/image/service/ImageAsyncService.java diff --git a/backend/src/main/java/com/worlabel/domain/image/entity/Image.java b/backend/src/main/java/com/worlabel/domain/image/entity/Image.java index b8ccdb4..ddbb801 100644 --- a/backend/src/main/java/com/worlabel/domain/image/entity/Image.java +++ b/backend/src/main/java/com/worlabel/domain/image/entity/Image.java @@ -19,7 +19,7 @@ public class Image extends BaseEntity { */ @Id @Column(name = "project_image_id", nullable = false) - @GeneratedValue(strategy = GenerationType.IDENTITY) + @GeneratedValue(strategy = GenerationType.SEQUENCE) private Long id; /** diff --git a/backend/src/main/java/com/worlabel/domain/image/service/ImageAsyncService.java b/backend/src/main/java/com/worlabel/domain/image/service/ImageAsyncService.java new file mode 100644 index 0000000..80d2804 --- /dev/null +++ b/backend/src/main/java/com/worlabel/domain/image/service/ImageAsyncService.java @@ -0,0 +1,70 @@ +package com.worlabel.domain.image.service; + +import com.worlabel.domain.folder.entity.Folder; +import com.worlabel.domain.image.entity.Image; +import com.worlabel.domain.image.repository.ImageRepository; +import com.worlabel.global.exception.CustomException; +import com.worlabel.global.exception.ErrorCode; +import com.worlabel.global.service.S3UploadService; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.scheduling.annotation.Async; +import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; +import org.springframework.web.multipart.MultipartFile; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CompletableFuture; + +@Slf4j +@Service +@RequiredArgsConstructor +public class ImageAsyncService { + + private final S3UploadService s3UploadService; + private final ImageRepository imageRepository; + private final ThreadPoolTaskExecutor imageUploadExecutor; + + @Transactional + @Async("imageUploadExecutor") + public CompletableFuture asyncImageUpload(final List imageList, final Folder folder, final Integer projectId) { + log.debug("현재 스레드 - {} 업로드 파일 개수 - {}, 현재 작업 큐 용량 - {}", + Thread.currentThread().getName(), + imageList.size(), + imageUploadExecutor.getThreadPoolExecutor().getQueue().size()); // 큐에 쌓인 작업 수 출력); + + imageList.forEach(file -> { + try{ + String extension = getExtension(file.getOriginalFilename()); + String imageKey = s3UploadService.uploadImageFile(file, extension, projectId); + + createImage(file, imageKey, folder); + }catch (Exception e){ + log.error("이미지 업로드 실패: {}", file.getOriginalFilename(), e); + throw new CustomException(ErrorCode.FAIL_TO_CREATE_FILE); + } + }); + + log.debug("배치 처리 완료"); + return CompletableFuture.completedFuture(null); + } + + public void createImage(MultipartFile file, String imageKey, Folder folder) { + try { + String name = file.getOriginalFilename(); + String extension = getExtension(name); + Image image = Image.of(name, imageKey, extension, folder); + imageRepository.save(image); + }catch (Exception e){ + log.debug("이미지 DB 저장 실패: ", e); + s3UploadService.deleteImageFromS3(imageKey); + throw new CustomException(ErrorCode.FAIL_TO_CREATE_FILE); + } + } + + private String getExtension(String fileName) { + return fileName.substring(fileName.lastIndexOf(".") + 1); + } +} diff --git a/backend/src/main/java/com/worlabel/domain/image/service/ImageService.java b/backend/src/main/java/com/worlabel/domain/image/service/ImageService.java index 0b62e30..bb5ff40 100644 --- a/backend/src/main/java/com/worlabel/domain/image/service/ImageService.java +++ b/backend/src/main/java/com/worlabel/domain/image/service/ImageService.java @@ -26,13 +26,14 @@ import org.springframework.web.multipart.MultipartFile; import java.io.File; import java.io.IOException; import java.io.OutputStream; -import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.time.LocalDateTime; import java.time.format.DateTimeFormatter; +import java.util.ArrayList; import java.util.List; +import java.util.concurrent.CompletableFuture; @Slf4j @Service @@ -41,6 +42,7 @@ import java.util.List; public class ImageService { private final ProjectRepository projectRepository; + private final ImageAsyncService imageAsyncService; private final FolderRepository folderRepository; private final S3UploadService s3UploadService; private final ImageRepository imageRepository; @@ -51,25 +53,34 @@ public class ImageService { @CheckPrivilege(value = PrivilegeType.EDITOR) public void uploadImageList(final List imageList, final Integer folderId, final Integer projectId) { Folder folder = getOrCreateFolder(folderId, projectId); + long prev = System.currentTimeMillis(); - // 이미지 리스트를 순차적으로 처리 (향후 비동기/병렬 처리를 위해 분리된 메서드 호출) - imageList.forEach(file -> uploadAndSave(file, folder, projectId)); - } + // 동적 배치 크기 계산 + int totalImages = imageList.size(); + int batchSize; - /** - * 폴더 생성 또는 가져오기 - * 폴더 생성 작업이 있음으로 트랜잭션 보호 - */ - public Folder getOrCreateFolder(Integer folderId, Integer projectId) { - if (folderId != 0) { - return getFolder(folderId); + if (totalImages <= 100) { + batchSize = 25; // 작은 이미지 수는 작은 배치 크기 + } else if (totalImages <= 3000) { + batchSize = 50; // 중간 이미지 수는 중간 배치 크기 } else { - String currentDateTime = LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")); - Project project = getProject(projectId); - Folder folder = Folder.of(currentDateTime, null, project); - folderRepository.save(folder); // 새로운 폴더를 저장 - return folder; + batchSize = 100; // 큰 이미지 수는 큰 배치 크기 } + + List> futures = new ArrayList<>(); + for (int i = 0; i < imageList.size(); i += batchSize) { + List batch = imageList.subList(i, Math.min(i + batchSize, imageList.size())); + + CompletableFuture future = imageAsyncService.asyncImageUpload(batch, folder, projectId); + // 모든 비동기 작업이 완료될 때까지 기다림 + futures.add(future); + } + + // 모든 비동기 작업이 완료될 때까지 기다림 + CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])).join(); + + long after = System.currentTimeMillis(); + log.debug("업로드 완료 - 경과시간 {}", ((double) after - prev) / 1000); } /** @@ -197,7 +208,7 @@ public class ImageService { Files.createDirectories(newPath); } else { Files.createDirectories(newPath.getParent()); - try(OutputStream os = Files.newOutputStream(newPath)){ + try (OutputStream os = Files.newOutputStream(newPath)) { IOUtils.copy(zis, os); } } @@ -243,6 +254,22 @@ public class ImageService { .orElseThrow(() -> new CustomException(ErrorCode.DATA_NOT_FOUND)); } + /** + * 폴더 생성 또는 가져오기 + * 폴더 생성 작업이 있음으로 트랜잭션 보호 + */ + public Folder getOrCreateFolder(Integer folderId, Integer projectId) { + if (folderId != 0) { + return getFolder(folderId); + } else { + String currentDateTime = LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")); + Project project = getProject(projectId); + Folder folder = Folder.of(currentDateTime, null, project); + folderRepository.save(folder); // 새로운 폴더를 저장 + return folder; + } + } + // 이미지 가져오면서 프로젝트 소속 여부를 확인 private Image getImageByIdAndFolderIdAndFolderProjectId(final Integer folderId, final Long imageId, final Integer projectId) { return imageRepository.findByIdAndFolderIdAndFolderProjectId(imageId, folderId, projectId) @@ -250,13 +277,13 @@ public class ImageService { } /* - 공통 로직 + 공통 로직 */ /** * MultipartFile 업로드 및 저장 */ - private void uploadAndSave(MultipartFile file, Folder folder, int projectId) { + public void uploadAndSave(MultipartFile file, Folder folder, int projectId) { try { String key = uploadToS3(file, projectId); saveImage(file, key, folder); @@ -274,7 +301,6 @@ public class ImageService { String key = uploadToS3(file, project.getId()); saveImage(file, key, folder); } catch (Exception e) { - log.error("파일 업로드 중 오류 발생", e); throw new CustomException(ErrorCode.FAIL_TO_CREATE_FILE, "이미지 업로드 중 오류 발생"); } } @@ -284,7 +310,7 @@ public class ImageService { */ public void saveImage(final MultipartFile file, final String key, final Folder folder) { try { - Image image = createImage(file, key, folder); + Image image = createImage(file.getOriginalFilename(), key, folder); imageRepository.save(image); } catch (Exception e) { log.error("이미지 DB 저장 실패 원인: ", e); @@ -298,7 +324,7 @@ public class ImageService { */ public void saveImage(final File file, final String key, final Folder folder) { try { - Image image = createImage(file, key, folder); + Image image = createImage(file.getName(), key, folder); imageRepository.save(image); } catch (Exception e) { s3UploadService.deleteImageFromS3(key); @@ -309,9 +335,10 @@ public class ImageService { /** * S3 파일 업로드 */ - private String uploadToS3(final MultipartFile file, final Integer projectId) { + public String uploadToS3(final MultipartFile file, final Integer projectId) { String extension = getExtension(file.getOriginalFilename()); - return s3UploadService.uploadMultipartFile(file, extension, projectId); +// return s3UploadService.uploadImageFile(file, extension, projectId); + return ""; } /** @@ -319,17 +346,12 @@ public class ImageService { */ private String uploadToS3(final File file, final Integer projectId) { String extension = getExtension(file.getName()); - return s3UploadService.uploadFile(file, extension, projectId); + return s3UploadService.uploadImageFile(file, extension, projectId); } - public Image createImage(MultipartFile file, String key, Folder folder) { - String extension = getExtension(file.getOriginalFilename()); - log.debug("이미지 업로드 이름 :{}",file.getOriginalFilename()); - return Image.of(file.getOriginalFilename(), key, extension, folder); - } - - public Image createImage(File file, String key, Folder folder) { - String extension = getExtension(file.getName()); - return Image.of(file.getName(), key, extension, folder); + public Image createImage(String fileName, String key, Folder folder) { + String extension = getExtension(fileName); + log.debug("이미지 업로드 이름 :{}", fileName); + return Image.of(fileName, key, extension, folder); } } diff --git a/backend/src/main/java/com/worlabel/global/config/AsyncConfig.java b/backend/src/main/java/com/worlabel/global/config/AsyncConfig.java index 422172f..f8fea1a 100644 --- a/backend/src/main/java/com/worlabel/global/config/AsyncConfig.java +++ b/backend/src/main/java/com/worlabel/global/config/AsyncConfig.java @@ -1,10 +1,26 @@ package com.worlabel.global.config; +import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.scheduling.annotation.EnableAsync; +import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; @EnableAsync @Configuration public class AsyncConfig { + @Bean(name = "imageUploadExecutor") + public ThreadPoolTaskExecutor imageUploadExecutor(){ + ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); + + // 최적 스레드 수 = CPU 코어 수 * (1 + (I/O 작업 시간 / CPU 작업 시간)) + executor.setCorePoolSize(5); + executor.setMaxPoolSize(5); + executor.setQueueCapacity(100); + + executor.setThreadNamePrefix("imageUploadExecutor-"); + executor.initialize(); + + return executor; + } } diff --git a/backend/src/main/java/com/worlabel/global/service/S3UploadService.java b/backend/src/main/java/com/worlabel/global/service/S3UploadService.java index 641d3df..bd9f774 100644 --- a/backend/src/main/java/com/worlabel/global/service/S3UploadService.java +++ b/backend/src/main/java/com/worlabel/global/service/S3UploadService.java @@ -8,20 +8,20 @@ import com.worlabel.global.exception.ErrorCode; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; +import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; import org.springframework.util.StringUtils; import org.springframework.web.multipart.MultipartFile; -import javax.imageio.ImageIO; -import java.awt.image.BufferedImage; import java.io.*; import java.net.MalformedURLException; import java.net.URL; import java.net.URLDecoder; import java.nio.charset.StandardCharsets; import java.util.UUID; +import java.util.concurrent.CompletableFuture; -// TODO: 추후 비동기로 변경해야합니다. @Slf4j @Service @RequiredArgsConstructor @@ -60,26 +60,15 @@ public class S3UploadService { } } + /** * MultipartFile 업로드 */ - public String uploadMultipartFile(final MultipartFile file, final String extension, final Integer projectId) { - try { - return uploadImageToS3(file.getInputStream(), extension, projectId); - } catch (IOException e) { - log.debug("MultipartFile 업로드 에러 발생 ", e); - throw new CustomException(ErrorCode.FAIL_TO_CREATE_FILE); - } - } - - /** - * File 업로드 - */ - public String uploadFile(final File file, final String extension, final Integer projectId) { - try (InputStream inputStream = new FileInputStream(file)) { + public String uploadImageFile(final MultipartFile file, final String extension, final Integer projectId) { + try (InputStream inputStream = file.getInputStream()) { return uploadImageToS3(inputStream, extension, projectId); } catch (IOException e) { - log.debug("이미지 업로드에서 에러 발생 ", e); + log.debug("MultipartFile 업로드 에러 발생 {}",file.getOriginalFilename(), e); throw new CustomException(ErrorCode.FAIL_TO_CREATE_FILE); } } @@ -87,22 +76,39 @@ public class S3UploadService { /** * AWS S3 이미지 업로드 */ - private String uploadImageToS3(final InputStream inputStream, final String extension, final Integer projectId) throws IOException { + + public String uploadImageToS3(final InputStream inputStream, final String extension, final Integer projectId) throws IOException { String s3Key = getS3FileName(projectId); String s3FileName = s3Key + "." + extension; ObjectMetadata metadata = new ObjectMetadata(); // S3에 업로드할 파일의 메타데이터 설정 metadata.setContentType("image/" + extension); // 콘텐츠 타입 설정 + metadata.setContentLength(inputStream.available()); - uploadToS3(inputStream, s3FileName, metadata); + PutObjectRequest putRequest = new PutObjectRequest(bucket, s3FileName, inputStream, metadata); + amazonS3.putObject(putRequest); return url + "/" + s3Key; } /** - * InputStream 사용 파일을 S3 업로드 + * File 업로드 */ - private void uploadToS3(InputStream inputStream, final String s3Key, final ObjectMetadata metadata) { + public String uploadImageFile(final File file, final String extension, final Integer projectId) { + try (InputStream inputStream = new FileInputStream(file)) { +// return uploadImageToS3(inputStream, extension, projectId); + return ""; + } catch (IOException e) { + log.debug("이미지 업로드에서 에러 발생 ", e); + throw new CustomException(ErrorCode.FAIL_TO_CREATE_FILE); + } + } + + + /** + * InputStream 사용해서 파일을 S3 업로드 + */ + public void uploadToS3(InputStream inputStream, final String s3Key, final ObjectMetadata metadata) { try { byte[] bytes = IOUtils.toByteArray(inputStream);