From 248e9c2c468856e3ffff1fb9de4b69c347c5b211 Mon Sep 17 00:00:00 2001 From: Kim JiMyeung Date: Fri, 5 Dec 2025 10:17:08 +0900 Subject: [PATCH 1/9] =?UTF-8?q?/snp-asi=20url=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/main/java/com/snp/batch/global/config/SwaggerConfig.java | 4 ++-- src/main/resources/application.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/snp/batch/global/config/SwaggerConfig.java b/src/main/java/com/snp/batch/global/config/SwaggerConfig.java index 5af2a69..433b980 100644 --- a/src/main/java/com/snp/batch/global/config/SwaggerConfig.java +++ b/src/main/java/com/snp/batch/global/config/SwaggerConfig.java @@ -42,10 +42,10 @@ public class SwaggerConfig { .url("http://10.26.252.39:" + serverPort) .description("로컬 개발 서버"), new Server() - .url("http://211.208.115.83:" + serverPort) + .url("http://211.208.115.83:" + serverPort + "/snp-api") .description("중계 서버"), new Server() - .url("http://10.187.58.58:" + serverPort) + .url("http://10.187.58.58:" + serverPort + "/snp-api") .description("운영 서버") )); } diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 8fc3c41..305b40e 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -55,7 +55,7 @@ spring: # Server Configuration server: - port: 8041 + port: 8081 servlet: context-path: /snp-api From 919b0fc21ad346267d9d570c8df44a711f1b496f Mon Sep 17 00:00:00 2001 From: Kim JiMyeung Date: Mon, 8 Dec 2025 13:00:08 +0900 Subject: [PATCH 2/9] =?UTF-8?q?AnchorageCalls,=20Berthcalls,=20DarkActivit?= =?UTF-8?q?y,=20StsOperations,=20TerminalCalls=20=EC=A6=9D=EB=B6=84Job?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../batch/config/AnchorageCallsJobConfig.java | 115 ++++++++++ .../batch/dto/AnchorageCallsDto.java | 33 +++ .../batch/dto/AnchorageCallsPositionDto.java | 19 ++ .../batch/entity/AnchorageCallsEntity.java | 47 ++++ .../processor/AnchorageCallsProcessor.java | 68 ++++++ .../batch/reader/AnchorageCallsReader.java | 216 ++++++++++++++++++ .../repository/AnchorageCallsRepository.java | 14 ++ .../AnchorageCallsRepositoryImpl.java | 201 ++++++++++++++++ .../batch/writer/AnchorageCallsWriter.java | 38 +++ .../batch/config/BerthCallsJobConfig.java | 107 +++++++++ .../batch/dto/BerthCallsDto.java | 33 +++ .../batch/dto/BerthCallsPositionDto.java | 17 ++ .../batch/entiity/BerthCallsEntity.java | 47 ++++ .../batch/processor/BerthCallsProcessor.java | 68 ++++++ .../batch/reader/BerthCallsReader.java | 213 +++++++++++++++++ .../repository/BerthCallsRepository.java | 14 ++ .../repository/BerthCallsRepositoryImpl.java | 192 ++++++++++++++++ .../batch/writer/BerthCallsWriter.java | 37 +++ .../batch/config/DarkActivityJobConfig.java | 106 +++++++++ .../batch/dto/DarkActivityDto.java | 30 +++ .../batch/entity/DarkActivityEntity.java | 41 ++++ .../processor/DarkActivityProcessor.java | 66 ++++++ .../batch/reader/DarkActivityReader.java | 212 +++++++++++++++++ .../repository/DarkActivityRepository.java | 14 ++ .../DarkActivityRepositoryImpl.java | 186 +++++++++++++++ .../batch/writer/DarkActivityWriter.java | 37 +++ .../batch/config/StsOperationJobConfig.java | 110 +++++++++ .../batch/dto/StsOperationDto.java | 35 +++ .../batch/dto/StsOperationPositionDto.java | 17 ++ .../batch/entity/StsOperationEntity.java | 45 ++++ .../processor/StsOperationProcessor.java | 69 ++++++ .../batch/reader/StsOperationReader.java | 213 +++++++++++++++++ .../repository/StsOperationRepository.java | 12 + .../StsOperationRepositoryImpl.java | 162 +++++++++++++ .../batch/writer/StsOperationWriter.java | 36 +++ .../batch/config/TerminalCallsJobConfig.java | 104 +++++++++ .../batch/dto/TerminalCallsDto.java | 32 +++ .../batch/dto/TerminalCallsPositionDto.java | 17 ++ .../batch/entity/TerminalCallsEntity.java | 43 ++++ .../processor/TerminalCallsProcessor.java | 68 ++++++ .../batch/reader/TerminalCallsReader.java | 213 +++++++++++++++++ .../repository/TerminalCallsRepository.java | 13 ++ .../TerminalCallsRepositoryImpl.java | 152 ++++++++++++ .../batch/writer/TerminalCallsWriter.java | 35 +++ 44 files changed, 3547 insertions(+) create mode 100644 src/main/java/com/snp/batch/jobs/shipMovementAnchorageCalls/batch/config/AnchorageCallsJobConfig.java create mode 100644 src/main/java/com/snp/batch/jobs/shipMovementAnchorageCalls/batch/dto/AnchorageCallsDto.java create mode 100644 src/main/java/com/snp/batch/jobs/shipMovementAnchorageCalls/batch/dto/AnchorageCallsPositionDto.java create mode 100644 src/main/java/com/snp/batch/jobs/shipMovementAnchorageCalls/batch/entity/AnchorageCallsEntity.java create mode 100644 src/main/java/com/snp/batch/jobs/shipMovementAnchorageCalls/batch/processor/AnchorageCallsProcessor.java create mode 100644 src/main/java/com/snp/batch/jobs/shipMovementAnchorageCalls/batch/reader/AnchorageCallsReader.java create mode 100644 src/main/java/com/snp/batch/jobs/shipMovementAnchorageCalls/batch/repository/AnchorageCallsRepository.java create mode 100644 src/main/java/com/snp/batch/jobs/shipMovementAnchorageCalls/batch/repository/AnchorageCallsRepositoryImpl.java create mode 100644 src/main/java/com/snp/batch/jobs/shipMovementAnchorageCalls/batch/writer/AnchorageCallsWriter.java create mode 100644 src/main/java/com/snp/batch/jobs/shipMovementBerthCalls/batch/config/BerthCallsJobConfig.java create mode 100644 src/main/java/com/snp/batch/jobs/shipMovementBerthCalls/batch/dto/BerthCallsDto.java create mode 100644 src/main/java/com/snp/batch/jobs/shipMovementBerthCalls/batch/dto/BerthCallsPositionDto.java create mode 100644 src/main/java/com/snp/batch/jobs/shipMovementBerthCalls/batch/entiity/BerthCallsEntity.java create mode 100644 src/main/java/com/snp/batch/jobs/shipMovementBerthCalls/batch/processor/BerthCallsProcessor.java create mode 100644 src/main/java/com/snp/batch/jobs/shipMovementBerthCalls/batch/reader/BerthCallsReader.java create mode 100644 src/main/java/com/snp/batch/jobs/shipMovementBerthCalls/batch/repository/BerthCallsRepository.java create mode 100644 src/main/java/com/snp/batch/jobs/shipMovementBerthCalls/batch/repository/BerthCallsRepositoryImpl.java create mode 100644 src/main/java/com/snp/batch/jobs/shipMovementBerthCalls/batch/writer/BerthCallsWriter.java create mode 100644 src/main/java/com/snp/batch/jobs/shipMovementDarkActivity/batch/config/DarkActivityJobConfig.java create mode 100644 src/main/java/com/snp/batch/jobs/shipMovementDarkActivity/batch/dto/DarkActivityDto.java create mode 100644 src/main/java/com/snp/batch/jobs/shipMovementDarkActivity/batch/entity/DarkActivityEntity.java create mode 100644 src/main/java/com/snp/batch/jobs/shipMovementDarkActivity/batch/processor/DarkActivityProcessor.java create mode 100644 src/main/java/com/snp/batch/jobs/shipMovementDarkActivity/batch/reader/DarkActivityReader.java create mode 100644 src/main/java/com/snp/batch/jobs/shipMovementDarkActivity/batch/repository/DarkActivityRepository.java create mode 100644 src/main/java/com/snp/batch/jobs/shipMovementDarkActivity/batch/repository/DarkActivityRepositoryImpl.java create mode 100644 src/main/java/com/snp/batch/jobs/shipMovementDarkActivity/batch/writer/DarkActivityWriter.java create mode 100644 src/main/java/com/snp/batch/jobs/shipMovementStsOperations/batch/config/StsOperationJobConfig.java create mode 100644 src/main/java/com/snp/batch/jobs/shipMovementStsOperations/batch/dto/StsOperationDto.java create mode 100644 src/main/java/com/snp/batch/jobs/shipMovementStsOperations/batch/dto/StsOperationPositionDto.java create mode 100644 src/main/java/com/snp/batch/jobs/shipMovementStsOperations/batch/entity/StsOperationEntity.java create mode 100644 src/main/java/com/snp/batch/jobs/shipMovementStsOperations/batch/processor/StsOperationProcessor.java create mode 100644 src/main/java/com/snp/batch/jobs/shipMovementStsOperations/batch/reader/StsOperationReader.java create mode 100644 src/main/java/com/snp/batch/jobs/shipMovementStsOperations/batch/repository/StsOperationRepository.java create mode 100644 src/main/java/com/snp/batch/jobs/shipMovementStsOperations/batch/repository/StsOperationRepositoryImpl.java create mode 100644 src/main/java/com/snp/batch/jobs/shipMovementStsOperations/batch/writer/StsOperationWriter.java create mode 100644 src/main/java/com/snp/batch/jobs/shipMovementTerminalCalls/batch/config/TerminalCallsJobConfig.java create mode 100644 src/main/java/com/snp/batch/jobs/shipMovementTerminalCalls/batch/dto/TerminalCallsDto.java create mode 100644 src/main/java/com/snp/batch/jobs/shipMovementTerminalCalls/batch/dto/TerminalCallsPositionDto.java create mode 100644 src/main/java/com/snp/batch/jobs/shipMovementTerminalCalls/batch/entity/TerminalCallsEntity.java create mode 100644 src/main/java/com/snp/batch/jobs/shipMovementTerminalCalls/batch/processor/TerminalCallsProcessor.java create mode 100644 src/main/java/com/snp/batch/jobs/shipMovementTerminalCalls/batch/reader/TerminalCallsReader.java create mode 100644 src/main/java/com/snp/batch/jobs/shipMovementTerminalCalls/batch/repository/TerminalCallsRepository.java create mode 100644 src/main/java/com/snp/batch/jobs/shipMovementTerminalCalls/batch/repository/TerminalCallsRepositoryImpl.java create mode 100644 src/main/java/com/snp/batch/jobs/shipMovementTerminalCalls/batch/writer/TerminalCallsWriter.java diff --git a/src/main/java/com/snp/batch/jobs/shipMovementAnchorageCalls/batch/config/AnchorageCallsJobConfig.java b/src/main/java/com/snp/batch/jobs/shipMovementAnchorageCalls/batch/config/AnchorageCallsJobConfig.java new file mode 100644 index 0000000..77a35d7 --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/shipMovementAnchorageCalls/batch/config/AnchorageCallsJobConfig.java @@ -0,0 +1,115 @@ +package com.snp.batch.jobs.shipMovementAnchorageCalls.batch.config; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.snp.batch.common.batch.config.BaseJobConfig; +import com.snp.batch.jobs.shipMovement.batch.dto.PortCallDto; +import com.snp.batch.jobs.shipMovement.batch.entity.ShipMovementEntity; +import com.snp.batch.jobs.shipMovement.batch.processor.ShipMovementProcessor; +import com.snp.batch.jobs.shipMovement.batch.reader.ShipMovementReader; +import com.snp.batch.jobs.shipMovement.batch.writer.ShipMovementWriter; +import com.snp.batch.jobs.shipMovementAnchorageCalls.batch.dto.AnchorageCallsDto; +import com.snp.batch.jobs.shipMovementAnchorageCalls.batch.entity.AnchorageCallsEntity; +import com.snp.batch.jobs.shipMovementAnchorageCalls.batch.processor.AnchorageCallsProcessor; +import com.snp.batch.jobs.shipMovementAnchorageCalls.batch.reader.AnchorageCallsReader; +import com.snp.batch.jobs.shipMovementAnchorageCalls.batch.writer.AnchorageCallsWriter; +import lombok.extern.slf4j.Slf4j; +import org.springframework.batch.core.Job; +import org.springframework.batch.core.Step; +import org.springframework.batch.core.configuration.annotation.StepScope; +import org.springframework.batch.core.repository.JobRepository; +import org.springframework.batch.item.ItemProcessor; +import org.springframework.batch.item.ItemReader; +import org.springframework.batch.item.ItemWriter; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.transaction.PlatformTransactionManager; +import org.springframework.web.reactive.function.client.WebClient; + +import java.time.LocalDate; +import java.time.format.DateTimeFormatter; + +/** + * 선박 상세 정보 Import Job Config + * + * 특징: + * - ship_data 테이블에서 IMO 번호 조회 + * - IMO 번호를 100개씩 배치로 분할 + * - Maritime API GetShipsByIHSLRorIMONumbers 호출 + * TODO : GetShipsByIHSLRorIMONumbersAll 호출로 변경 + * - 선박 상세 정보를 ship_detail 테이블에 저장 (UPSERT) + * + * 데이터 흐름: + * ShipMovementReader (ship_data → Maritime API) + * ↓ (PortCallDto) + * ShipMovementProcessor + * ↓ (ShipMovementEntity) + * ShipDetailDataWriter + * ↓ (ship_movement 테이블) + */ + +@Slf4j +@Configuration +public class AnchorageCallsJobConfig extends BaseJobConfig { + + private final AnchorageCallsProcessor anchorageCallsProcessor; + private final AnchorageCallsWriter anchorageCallsWriter; + private final JdbcTemplate jdbcTemplate; + private final WebClient maritimeApiWebClient; + + public AnchorageCallsJobConfig( + JobRepository jobRepository, + PlatformTransactionManager transactionManager, + AnchorageCallsProcessor anchorageCallsProcessor, + AnchorageCallsWriter anchorageCallsWriter, JdbcTemplate jdbcTemplate, + @Qualifier("maritimeServiceApiWebClient") WebClient maritimeApiWebClient + ) { // ObjectMapper 주입 추가 + super(jobRepository, transactionManager); + this.anchorageCallsProcessor = anchorageCallsProcessor; + this.anchorageCallsWriter = anchorageCallsWriter; + this.jdbcTemplate = jdbcTemplate; + this.maritimeApiWebClient = maritimeApiWebClient; + } + + @Override + protected String getJobName() { + return "AnchorageCallsImportJob"; + } + + @Override + protected String getStepName() { + return "AnchorageCallsImportStep"; + } + + @Override + protected ItemReader createReader() { // 타입 변경 + return new AnchorageCallsReader(maritimeApiWebClient, jdbcTemplate); + } + + @Override + protected ItemProcessor createProcessor() { + return anchorageCallsProcessor; + } + + @Override + protected ItemWriter createWriter() { // 타입 변경 + return anchorageCallsWriter; + } + + @Override + protected int getChunkSize() { + return 50; // API에서 100개씩 가져오므로 chunk도 100으로 설정 + } + + @Bean(name = "AnchorageCallsImportJob") + public Job anchorageCallsImportJob() { + return job(); + } + + @Bean(name = "AnchorageCallsImportStep") + public Step anchorageCallsImportStep() { + return step(); + } +} diff --git a/src/main/java/com/snp/batch/jobs/shipMovementAnchorageCalls/batch/dto/AnchorageCallsDto.java b/src/main/java/com/snp/batch/jobs/shipMovementAnchorageCalls/batch/dto/AnchorageCallsDto.java new file mode 100644 index 0000000..cd26678 --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/shipMovementAnchorageCalls/batch/dto/AnchorageCallsDto.java @@ -0,0 +1,33 @@ +package com.snp.batch.jobs.shipMovementAnchorageCalls.batch.dto; + +import lombok.Data; + +import java.time.LocalDateTime; + +@Data +public class AnchorageCallsDto { + private String movementType; + private String imolRorIHSNumber; + private String movementDate; + + private Integer portCallId; + private Integer facilityId; + private String facilityName; + private String facilityType; + + private Integer subFacilityId; + private String subFacilityName; + private String subFacilityType; + + private String countryCode; + private String countryName; + + private Double draught; + private Double latitude; + private Double longitude; + + private AnchorageCallsPositionDto position; + + private String destination; + private String iso2; +} diff --git a/src/main/java/com/snp/batch/jobs/shipMovementAnchorageCalls/batch/dto/AnchorageCallsPositionDto.java b/src/main/java/com/snp/batch/jobs/shipMovementAnchorageCalls/batch/dto/AnchorageCallsPositionDto.java new file mode 100644 index 0000000..23d3613 --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/shipMovementAnchorageCalls/batch/dto/AnchorageCallsPositionDto.java @@ -0,0 +1,19 @@ +package com.snp.batch.jobs.shipMovementAnchorageCalls.batch.dto; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Getter; +import lombok.Setter; + +@Getter +@Setter +public class AnchorageCallsPositionDto { + private Boolean isNull; + private Integer stSrid; + private Double lat; + @JsonProperty("long") + private Double lon; + private Double z; + private Double m; + private Boolean hasZ; + private Boolean hasM; +} diff --git a/src/main/java/com/snp/batch/jobs/shipMovementAnchorageCalls/batch/entity/AnchorageCallsEntity.java b/src/main/java/com/snp/batch/jobs/shipMovementAnchorageCalls/batch/entity/AnchorageCallsEntity.java new file mode 100644 index 0000000..70aaad8 --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/shipMovementAnchorageCalls/batch/entity/AnchorageCallsEntity.java @@ -0,0 +1,47 @@ +package com.snp.batch.jobs.shipMovementAnchorageCalls.batch.entity; + +import com.fasterxml.jackson.databind.JsonNode; +import jakarta.persistence.GeneratedValue; +import jakarta.persistence.GenerationType; +import jakarta.persistence.Id; +import jakarta.persistence.SequenceGenerator; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +import java.time.LocalDateTime; + +@Data +@SuperBuilder +@NoArgsConstructor +@AllArgsConstructor +public class AnchorageCallsEntity { + + private Long id; + + private String movementType; + private String imolRorIHSNumber; + private LocalDateTime movementDate; + + private Integer portCallId; + private Integer facilityId; + private String facilityName; + private String facilityType; + + private Integer subFacilityId; + private String subFacilityName; + private String subFacilityType; + + private String countryCode; + private String countryName; + + private Double draught; + private Double latitude; + private Double longitude; + + private JsonNode position; + + private String destination; + private String iso2; +} diff --git a/src/main/java/com/snp/batch/jobs/shipMovementAnchorageCalls/batch/processor/AnchorageCallsProcessor.java b/src/main/java/com/snp/batch/jobs/shipMovementAnchorageCalls/batch/processor/AnchorageCallsProcessor.java new file mode 100644 index 0000000..ee03a7e --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/shipMovementAnchorageCalls/batch/processor/AnchorageCallsProcessor.java @@ -0,0 +1,68 @@ +package com.snp.batch.jobs.shipMovementAnchorageCalls.batch.processor; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.snp.batch.common.batch.processor.BaseProcessor; +import com.snp.batch.jobs.shipMovementAnchorageCalls.batch.dto.AnchorageCallsDto; +import com.snp.batch.jobs.shipMovementAnchorageCalls.batch.entity.AnchorageCallsEntity; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; + +import java.time.LocalDateTime; + +/** + * 선박 상세 정보 Processor + * ShipDetailDto → ShipDetailEntity 변환 + */ + +/** + * 선박 상세 정보 Processor (해시 비교 및 증분 데이터 추출) + * I: ShipDetailComparisonData (DB 해시 + API Map Data) + * O: ShipDetailUpdate (변경분) + */ +@Slf4j +@Component +public class AnchorageCallsProcessor extends BaseProcessor { + + private final ObjectMapper objectMapper; + + public AnchorageCallsProcessor(ObjectMapper objectMapper) { + this.objectMapper = objectMapper; + } + + @Override + protected AnchorageCallsEntity processItem(AnchorageCallsDto dto) throws Exception { + log.debug("선박 상세 정보 처리 시작: imoNumber={}, facilityName={}", + dto.getImolRorIHSNumber(), dto.getFacilityName()); + + JsonNode positionNode = null; + if (dto.getPosition() != null) { + // Position 객체를 JsonNode로 변환 + positionNode = objectMapper.valueToTree(dto.getPosition()); + } + + AnchorageCallsEntity entity = AnchorageCallsEntity.builder() + .movementType(dto.getMovementType()) + .imolRorIHSNumber(dto.getImolRorIHSNumber()) + .movementDate(LocalDateTime.parse(dto.getMovementDate())) + .portCallId(dto.getPortCallId()) + .facilityId(dto.getFacilityId()) + .facilityName(dto.getFacilityName()) + .facilityType(dto.getFacilityType()) + .subFacilityId(dto.getSubFacilityId()) + .subFacilityName(dto.getSubFacilityName()) + .subFacilityType(dto.getSubFacilityType()) + .countryCode(dto.getCountryCode()) + .countryName(dto.getCountryName()) + .draught(dto.getDraught()) + .latitude(dto.getLatitude()) + .longitude(dto.getLongitude()) + .destination(dto.getDestination()) + .iso2(dto.getIso2()) + .position(positionNode) // JsonNode로 매핑 + .build(); + + return entity; + } + +} diff --git a/src/main/java/com/snp/batch/jobs/shipMovementAnchorageCalls/batch/reader/AnchorageCallsReader.java b/src/main/java/com/snp/batch/jobs/shipMovementAnchorageCalls/batch/reader/AnchorageCallsReader.java new file mode 100644 index 0000000..9c3f782 --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/shipMovementAnchorageCalls/batch/reader/AnchorageCallsReader.java @@ -0,0 +1,216 @@ +package com.snp.batch.jobs.shipMovementAnchorageCalls.batch.reader; + +import com.snp.batch.common.batch.reader.BaseApiReader; +import com.snp.batch.jobs.shipMovementAnchorageCalls.batch.dto.AnchorageCallsDto; +import lombok.extern.slf4j.Slf4j; +import org.springframework.batch.core.configuration.annotation.StepScope; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.web.reactive.function.client.WebClient; + +import java.util.Collections; +import java.util.List; +import java.util.Map; + +/** + * 선박 상세 정보 Reader (v2.0 - Chunk 기반) + * + * 기능: + * 1. ship_data 테이블에서 IMO 번호 전체 조회 (최초 1회) + * 2. IMO 번호를 100개씩 분할하여 배치 단위로 처리 + * 3. fetchNextBatch() 호출 시마다 100개씩 API 호출 + * 4. Spring Batch가 100건씩 Process → Write 수행 + * + * Chunk 처리 흐름: + * - beforeFetch() → IMO 전체 조회 (1회) + * - fetchNextBatch() → 100개 IMO로 API 호출 (1,718회) + * - read() → 1건씩 반환 (100번) + * - Processor/Writer → 100건 처리 + * - 반복... (1,718번의 Chunk) + * + * 기존 방식과의 차이: + * - 기존: 17만건 전체 메모리 로드 → Process → Write + * - 신규: 100건씩 로드 → Process → Write (Chunk 1,718회) + */ +@Slf4j +@StepScope +public class AnchorageCallsReader extends BaseApiReader { + + private final JdbcTemplate jdbcTemplate; + + // 배치 처리 상태 + private List allImoNumbers; + // DB 해시값을 저장할 맵 + private Map dbMasterHashes; + private int currentBatchIndex = 0; + private final int batchSize = 5; + + // @Value("#{jobParameters['startDate']}") +// private String startDate; + private String startDate = "2025-01-01"; + + // @Value("#{jobParameters['stopDate']}") +// private String stopDate; + private String stopDate = "2025-12-31"; + + public AnchorageCallsReader(WebClient webClient, JdbcTemplate jdbcTemplate ) { + super(webClient); + this.jdbcTemplate = jdbcTemplate; + enableChunkMode(); // ✨ Chunk 모드 활성화 + } + + @Override + protected String getReaderName() { + return "AnchorageCallsReader"; + } + + @Override + protected void resetCustomState() { + this.currentBatchIndex = 0; + this.allImoNumbers = null; + this.dbMasterHashes = null; + } + + @Override + protected String getApiPath() { + return "/Movements/AnchorageCalls"; + } + + @Override + protected String getApiBaseUrl() { + return "https://webservices.maritime.spglobal.com"; + } + + private static final String GET_ALL_IMO_QUERY = +// "SELECT imo_number FROM ship_data ORDER BY id"; + "SELECT imo_number FROM snp_data.ship_data where imo_number > (select max(imo) from snp_data.t_anchoragecall) ORDER BY imo_number"; + + private static final String FETCH_ALL_HASHES_QUERY = + "SELECT imo_number, ship_detail_hash FROM ship_detail_hash_json ORDER BY imo_number"; + + /** + * 최초 1회만 실행: ship_data 테이블에서 IMO 번호 전체 조회 + */ + @Override + protected void beforeFetch() { + // 전처리 과정 + // Step 1. IMO 전체 번호 조회 + log.info("[{}] ship_data 테이블에서 IMO 번호 조회 시작...", getReaderName()); + + allImoNumbers = jdbcTemplate.queryForList(GET_ALL_IMO_QUERY, String.class); + int totalBatches = (int) Math.ceil((double) allImoNumbers.size() / batchSize); + + log.info("[{}] 총 {} 개의 IMO 번호 조회 완료", getReaderName(), allImoNumbers.size()); + log.info("[{}] {}개씩 배치로 분할하여 API 호출 예정", getReaderName(), batchSize); + log.info("[{}] 예상 배치 수: {} 개", getReaderName(), totalBatches); + + // API 통계 초기화 + updateApiCallStats(totalBatches, 0); + } + + /** + * ✨ Chunk 기반 핵심 메서드: 다음 100개 배치를 조회하여 반환 + * + * Spring Batch가 100건씩 read() 호출 완료 후 이 메서드 재호출 + * + * @return 다음 배치 100건 (더 이상 없으면 null) + */ + @Override + protected List fetchNextBatch() throws Exception { + + // 모든 배치 처리 완료 확인 + if (allImoNumbers == null || currentBatchIndex >= allImoNumbers.size()) { + return null; // Job 종료 + } + + // 현재 배치의 시작/끝 인덱스 계산 + int startIndex = currentBatchIndex; + int endIndex = Math.min(currentBatchIndex + batchSize, allImoNumbers.size()); + + // 현재 배치의 IMO 번호 추출 (100개) + List currentBatch = allImoNumbers.subList(startIndex, endIndex); + + int currentBatchNumber = (currentBatchIndex / batchSize) + 1; + int totalBatches = (int) Math.ceil((double) allImoNumbers.size() / batchSize); + + log.info("[{}] 배치 {}/{} 처리 중 (IMO {} 개)...", + getReaderName(), currentBatchNumber, totalBatches, currentBatch.size()); + + try { + // IMO 번호를 쉼표로 연결 (예: "1000019,1000021,1000033,...") + String imoParam = String.join(",", currentBatch); + + // API 호출 + List response = callApiWithBatch(imoParam); + + // 다음 배치로 인덱스 이동 + currentBatchIndex = endIndex; + + + // 응답 처리 + if (response != null ) { + List anchorageCalls = response; + log.info("[{}] 배치 {}/{} 완료: {} 건 조회", + getReaderName(), currentBatchNumber, totalBatches, anchorageCalls.size()); + + // API 호출 통계 업데이트 + updateApiCallStats(totalBatches, currentBatchNumber); + + // API 과부하 방지 (다음 배치 전 0.5초 대기) + if (currentBatchIndex < allImoNumbers.size()) { + Thread.sleep(500); + } + + return anchorageCalls; + + } else { + log.warn("[{}] 배치 {}/{} 응답 없음", + getReaderName(), currentBatchNumber, totalBatches); + + // API 호출 통계 업데이트 (실패도 카운트) + updateApiCallStats(totalBatches, currentBatchNumber); + + return Collections.emptyList(); + } + + } catch (Exception e) { + log.error("[{}] 배치 {}/{} 처리 중 오류: {}", + getReaderName(), currentBatchNumber, totalBatches, e.getMessage(), e); + + // 오류 발생 시에도 다음 배치로 이동 (부분 실패 허용) + currentBatchIndex = endIndex; + + // 빈 리스트 반환 (Job 계속 진행) + return Collections.emptyList(); + } + } + + /** + * Query Parameter를 사용한 API 호출 + * + * @param lrno 쉼표로 연결된 IMO 번호 (예: "1000019,1000021,...") + * @return API 응답 + */ + private List callApiWithBatch(String lrno) { + String url = getApiPath() + "?startDate=" + startDate +"&stopDate="+stopDate+"&lrno=" + lrno; + + log.debug("[{}] API 호출: {}", getReaderName(), url); + + return webClient.get() + .uri(url) + .retrieve() + .bodyToFlux(AnchorageCallsDto.class) + .collectList() + .block(); + } + + @Override + protected void afterFetch(List data) { + if (data == null) { + int totalBatches = (int) Math.ceil((double) allImoNumbers.size() / batchSize); + log.info("[{}] 전체 {} 개 배치 처리 완료", getReaderName(), totalBatches); + log.info("[{}] 총 {} 개의 IMO 번호에 대한 API 호출 종료", + getReaderName(), allImoNumbers.size()); + } + } + +} diff --git a/src/main/java/com/snp/batch/jobs/shipMovementAnchorageCalls/batch/repository/AnchorageCallsRepository.java b/src/main/java/com/snp/batch/jobs/shipMovementAnchorageCalls/batch/repository/AnchorageCallsRepository.java new file mode 100644 index 0000000..5bcfa85 --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/shipMovementAnchorageCalls/batch/repository/AnchorageCallsRepository.java @@ -0,0 +1,14 @@ +package com.snp.batch.jobs.shipMovementAnchorageCalls.batch.repository; + +import com.snp.batch.jobs.shipMovement.batch.entity.ShipMovementEntity; +import com.snp.batch.jobs.shipMovementAnchorageCalls.batch.entity.AnchorageCallsEntity; + +import java.util.List; + +/** + * 선박 상세 정보 Repository 인터페이스 + */ + +public interface AnchorageCallsRepository { + void saveAll(List entities); +} diff --git a/src/main/java/com/snp/batch/jobs/shipMovementAnchorageCalls/batch/repository/AnchorageCallsRepositoryImpl.java b/src/main/java/com/snp/batch/jobs/shipMovementAnchorageCalls/batch/repository/AnchorageCallsRepositoryImpl.java new file mode 100644 index 0000000..0e805d0 --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/shipMovementAnchorageCalls/batch/repository/AnchorageCallsRepositoryImpl.java @@ -0,0 +1,201 @@ +package com.snp.batch.jobs.shipMovementAnchorageCalls.batch.repository; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.snp.batch.common.batch.repository.BaseJdbcRepository; +import com.snp.batch.jobs.shipMovement.batch.entity.ShipMovementEntity; +import com.snp.batch.jobs.shipMovement.batch.repository.ShipMovementRepository; +import com.snp.batch.jobs.shipMovementAnchorageCalls.batch.entity.AnchorageCallsEntity; +import lombok.extern.slf4j.Slf4j; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.core.RowMapper; +import org.springframework.stereotype.Repository; + +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Timestamp; +import java.util.List; + +/** + * 선박 상세 정보 Repository 구현체 + * BaseJdbcRepository를 상속하여 JDBC 기반 CRUD 구현 + */ +@Slf4j +@Repository("anchorageCallsRepository") +public class AnchorageCallsRepositoryImpl extends BaseJdbcRepository + implements AnchorageCallsRepository { + + public AnchorageCallsRepositoryImpl(JdbcTemplate jdbcTemplate) { + super(jdbcTemplate); + } + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + @Override + protected String getTableName() { + return "snp_data.t_anchoragecall"; + } + + @Override + protected String getEntityName() { + return "AnchorageCalls"; + } + + @Override + protected String extractId(AnchorageCallsEntity entity) { + return entity.getImolRorIHSNumber(); + } + + @Override + public String getInsertSql() { + return """ + INSERT INTO snp_data.t_anchoragecall( + imo, + mvmn_type, + mvmn_dt, + stpov_id, + fclty_id, + fclty_nm, + fclty_type, + lwrnk_fclty_id, + lwrnk_fclty_nm, + lwrnk_fclty_type, + ntn_cd, + ntn_nm, + draft, + lat, + lon, + dstn, + iso2_ntn_cd, + lcinfo + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT (imo, mvmn_dt) + DO UPDATE SET + mvmn_type = EXCLUDED.mvmn_type, + mvmn_dt = EXCLUDED.mvmn_dt, + stpov_id = EXCLUDED.stpov_id, + fclty_id = EXCLUDED.fclty_id, + fclty_nm = EXCLUDED.fclty_nm, + fclty_type = EXCLUDED.fclty_type, + lwrnk_fclty_id = EXCLUDED.lwrnk_fclty_id, + lwrnk_fclty_nm = EXCLUDED.lwrnk_fclty_nm, + lwrnk_fclty_type = EXCLUDED.lwrnk_fclty_type, + ntn_cd = EXCLUDED.ntn_cd, + ntn_nm = EXCLUDED.ntn_nm, + draft = EXCLUDED.draft, + lat = EXCLUDED.lat, + lon = EXCLUDED.lon, + dstn = EXCLUDED.dstn, + iso2_ntn_cd = EXCLUDED.iso2_ntn_cd, + lcinfo = EXCLUDED.lcinfo + """; + } + + @Override + protected String getUpdateSql() { + return null; + } + + @Override + protected void setInsertParameters(PreparedStatement ps, AnchorageCallsEntity e) throws Exception { + int i = 1; + ps.setString(i++, e.getImolRorIHSNumber()); // imo + ps.setString(i++, e.getMovementType()); // mvmn_type + ps.setTimestamp(i++, e.getMovementDate() != null ? Timestamp.valueOf(e.getMovementDate()) : null); // mvmn_dt + ps.setObject(i++, e.getPortCallId()); // stpov_id + ps.setObject(i++, e.getFacilityId()); // fclty_id + ps.setString(i++, e.getFacilityName()); // fclty_nm + ps.setString(i++, e.getFacilityType()); // fclty_type + ps.setObject(i++, e.getSubFacilityId()); // lwrnk_fclty_id + ps.setString(i++, e.getSubFacilityName()); // lwrnk_fclty_nm + ps.setString(i++, e.getSubFacilityType()); // lwrnk_fclty_type + ps.setString(i++, e.getCountryCode()); // ntn_cd + ps.setString(i++, e.getCountryName()); // ntn_nm + setDoubleOrNull(ps, i++, e.getDraught()); // draft + setDoubleOrNull(ps, i++, e.getLatitude()); // lat + setDoubleOrNull(ps, i++, e.getLongitude());// lon + ps.setString(i++, e.getDestination()); // dstn + ps.setString(i++, e.getIso2()); // iso2_ntn_cd + + if (e.getPosition() != null) { + ps.setObject(i++, OBJECT_MAPPER.writeValueAsString(e.getPosition()), java.sql.Types.OTHER); // lcinfo (jsonb) + } else { + ps.setNull(i++, java.sql.Types.OTHER); + } + +// ps.setString(i++, e.getSchemaType()); + + } + + private void setDoubleOrNull(PreparedStatement ps, int index, Double value) throws Exception { + if (value != null) { + ps.setDouble(index, value); + } else { + // java.sql.Types.DOUBLE을 사용하여 명시적으로 SQL NULL을 설정 + ps.setNull(index, java.sql.Types.DOUBLE); + } + } + + @Override + protected void setUpdateParameters(PreparedStatement ps, AnchorageCallsEntity entity) throws Exception { + + } + + @Override + protected RowMapper getRowMapper() { + return null; + } + + @Override + public void saveAll(List entities) { + if (entities == null || entities.isEmpty()) return; + + log.info("ShipMovement 저장 시작 = {}건", entities.size()); + batchInsert(entities); + + } + + + /** + * ShipDetailEntity RowMapper + */ + private static class AnchorageCallsRowMapper implements RowMapper { + @Override + public AnchorageCallsEntity mapRow(ResultSet rs, int rowNum) throws SQLException { + AnchorageCallsEntity entity = AnchorageCallsEntity.builder() + .id(rs.getLong("id")) + .imolRorIHSNumber(rs.getString("imolRorIHSNumber")) + .portCallId(rs.getObject("portCallId", Integer.class)) + .facilityId(rs.getObject("facilityId", Integer.class)) + .facilityName(rs.getString("facilityName")) + .facilityType(rs.getString("facilityType")) + .subFacilityId(rs.getObject("subFacilityId", Integer.class)) + .subFacilityName(rs.getString("subFacilityName")) + .subFacilityType(rs.getString("subFacilityType")) + .countryCode(rs.getString("countryCode")) + .countryName(rs.getString("countryName")) + .draught(rs.getObject("draught", Double.class)) + .latitude(rs.getObject("latitude", Double.class)) + .longitude(rs.getObject("longitude", Double.class)) + .destination(rs.getString("destination")) + .iso2(rs.getString("iso2")) + .position(parseJson(rs.getString("position"))) + .build(); + + Timestamp movementDate = rs.getTimestamp("movementDate"); + if (movementDate != null) { + entity.setMovementDate(movementDate.toLocalDateTime()); + } + + return entity; + } + + private JsonNode parseJson(String json) { + try { + if (json == null) return null; + return new ObjectMapper().readTree(json); + } catch (Exception e) { + throw new RuntimeException("JSON 파싱 오류: " + json); + } + } + } +} diff --git a/src/main/java/com/snp/batch/jobs/shipMovementAnchorageCalls/batch/writer/AnchorageCallsWriter.java b/src/main/java/com/snp/batch/jobs/shipMovementAnchorageCalls/batch/writer/AnchorageCallsWriter.java new file mode 100644 index 0000000..198d223 --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/shipMovementAnchorageCalls/batch/writer/AnchorageCallsWriter.java @@ -0,0 +1,38 @@ +package com.snp.batch.jobs.shipMovementAnchorageCalls.batch.writer; + +import com.snp.batch.common.batch.writer.BaseWriter; +import com.snp.batch.jobs.shipMovement.batch.repository.ShipMovementRepository; +import com.snp.batch.jobs.shipMovementAnchorageCalls.batch.entity.AnchorageCallsEntity; +import com.snp.batch.jobs.shipMovementAnchorageCalls.batch.repository.AnchorageCallsRepository; +import com.snp.batch.jobs.shipdetail.batch.repository.ShipDetailRepository; +import com.snp.batch.jobs.shipdetail.batch.repository.ShipHashRepository; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; + +import java.util.List; + +/** + * 선박 상세 정보 Writer + */ +@Slf4j +@Component +public class AnchorageCallsWriter extends BaseWriter { + + private final AnchorageCallsRepository anchorageCallsRepository; + + + public AnchorageCallsWriter(AnchorageCallsRepository anchorageCallsRepository) { + super("AnchorageCalls"); + this.anchorageCallsRepository = anchorageCallsRepository; + } + + @Override + protected void writeItems(List items) throws Exception { + + if (items.isEmpty()) { return; } + + anchorageCallsRepository.saveAll(items); + log.info("AnchorageCalls 데이터 저장: {} 건", items.size()); + } + +} diff --git a/src/main/java/com/snp/batch/jobs/shipMovementBerthCalls/batch/config/BerthCallsJobConfig.java b/src/main/java/com/snp/batch/jobs/shipMovementBerthCalls/batch/config/BerthCallsJobConfig.java new file mode 100644 index 0000000..2b43ed5 --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/shipMovementBerthCalls/batch/config/BerthCallsJobConfig.java @@ -0,0 +1,107 @@ +package com.snp.batch.jobs.shipMovementBerthCalls.batch.config; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.snp.batch.common.batch.config.BaseJobConfig; +import com.snp.batch.jobs.shipMovementBerthCalls.batch.dto.BerthCallsDto; +import com.snp.batch.jobs.shipMovementBerthCalls.batch.entiity.BerthCallsEntity; +import com.snp.batch.jobs.shipMovementBerthCalls.batch.processor.BerthCallsProcessor; +import com.snp.batch.jobs.shipMovementBerthCalls.batch.reader.BerthCallsReader; +import com.snp.batch.jobs.shipMovementBerthCalls.batch.writer.BerthCallsWriter; +import lombok.extern.slf4j.Slf4j; +import org.springframework.batch.core.Job; +import org.springframework.batch.core.Step; +import org.springframework.batch.core.repository.JobRepository; +import org.springframework.batch.item.ItemProcessor; +import org.springframework.batch.item.ItemReader; +import org.springframework.batch.item.ItemWriter; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.transaction.PlatformTransactionManager; +import org.springframework.web.reactive.function.client.WebClient; + +/** + * 선박 상세 정보 Import Job Config + * + * 특징: + * - ship_data 테이블에서 IMO 번호 조회 + * - IMO 번호를 100개씩 배치로 분할 + * - Maritime API GetShipsByIHSLRorIMONumbers 호출 + * TODO : GetShipsByIHSLRorIMONumbersAll 호출로 변경 + * - 선박 상세 정보를 ship_detail 테이블에 저장 (UPSERT) + * + * 데이터 흐름: + * ShipMovementReader (ship_data → Maritime API) + * ↓ (PortCallDto) + * ShipMovementProcessor + * ↓ (ShipMovementEntity) + * ShipDetailDataWriter + * ↓ (ship_movement 테이블) + */ + +@Slf4j +@Configuration +public class BerthCallsJobConfig extends BaseJobConfig { + + private final BerthCallsProcessor berthCallsProcessor; + private final BerthCallsWriter berthCallsWriter; + private final JdbcTemplate jdbcTemplate; + private final WebClient maritimeApiWebClient; + private final ObjectMapper objectMapper; // ObjectMapper 주입 추가 + + public BerthCallsJobConfig( + JobRepository jobRepository, + PlatformTransactionManager transactionManager, + BerthCallsProcessor berthCallsProcessor, + BerthCallsWriter berthCallsWriter, JdbcTemplate jdbcTemplate, + @Qualifier("maritimeServiceApiWebClient") WebClient maritimeApiWebClient, + ObjectMapper objectMapper) { // ObjectMapper 주입 추가 + super(jobRepository, transactionManager); + this.berthCallsProcessor = berthCallsProcessor; + this.berthCallsWriter = berthCallsWriter; + this.jdbcTemplate = jdbcTemplate; + this.maritimeApiWebClient = maritimeApiWebClient; + this.objectMapper = objectMapper; // ObjectMapper 초기화 + } + + @Override + protected String getJobName() { + return "BerthCallsImportJob"; + } + + @Override + protected String getStepName() { + return "BerthCallsImportStep"; + } + + @Override + protected ItemReader createReader() { // 타입 변경 + return new BerthCallsReader(maritimeApiWebClient, jdbcTemplate); + } + + @Override + protected ItemProcessor createProcessor() { + return berthCallsProcessor; + } + + @Override + protected ItemWriter createWriter() { // 타입 변경 + return berthCallsWriter; + } + + @Override + protected int getChunkSize() { + return 200; // API에서 100개씩 가져오므로 chunk도 100으로 설정 + } + + @Bean(name = "BerthCallsImportJob") + public Job berthCallsImportJob() { + return job(); + } + + @Bean(name = "BerthCallsImportStep") + public Step berthCallsImportStep() { + return step(); + } +} diff --git a/src/main/java/com/snp/batch/jobs/shipMovementBerthCalls/batch/dto/BerthCallsDto.java b/src/main/java/com/snp/batch/jobs/shipMovementBerthCalls/batch/dto/BerthCallsDto.java new file mode 100644 index 0000000..03b813e --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/shipMovementBerthCalls/batch/dto/BerthCallsDto.java @@ -0,0 +1,33 @@ +package com.snp.batch.jobs.shipMovementBerthCalls.batch.dto; + +import com.snp.batch.jobs.shipMovement.batch.dto.PositionDto; +import lombok.Data; + +@Data +public class BerthCallsDto { + private String movementType; + private String imolRorIHSNumber; + private String movementDate; + + + private Integer facilityId; + private String facilityName; + private String facilityType; + + private Integer parentFacilityId; + private String parentFacilityName; + private String parentFacilityType; + + private String countryCode; + private String countryName; + + private Double draught; + private Double latitude; + private Double longitude; + + private BerthCallsPositionDto position; + + private Integer parentCallId; + private String iso2; + private String eventStartDate; +} diff --git a/src/main/java/com/snp/batch/jobs/shipMovementBerthCalls/batch/dto/BerthCallsPositionDto.java b/src/main/java/com/snp/batch/jobs/shipMovementBerthCalls/batch/dto/BerthCallsPositionDto.java new file mode 100644 index 0000000..ffc652c --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/shipMovementBerthCalls/batch/dto/BerthCallsPositionDto.java @@ -0,0 +1,17 @@ +package com.snp.batch.jobs.shipMovementBerthCalls.batch.dto; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; + +@Data +public class BerthCallsPositionDto { + private boolean isNull; + private int stSrid; + private double lat; + @JsonProperty("long") + private double lon; + private double z; + private double m; + private boolean hasZ; + private boolean hasM; +} diff --git a/src/main/java/com/snp/batch/jobs/shipMovementBerthCalls/batch/entiity/BerthCallsEntity.java b/src/main/java/com/snp/batch/jobs/shipMovementBerthCalls/batch/entiity/BerthCallsEntity.java new file mode 100644 index 0000000..4cc1b8f --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/shipMovementBerthCalls/batch/entiity/BerthCallsEntity.java @@ -0,0 +1,47 @@ +package com.snp.batch.jobs.shipMovementBerthCalls.batch.entiity; + +import com.fasterxml.jackson.databind.JsonNode; +import jakarta.persistence.GeneratedValue; +import jakarta.persistence.GenerationType; +import jakarta.persistence.Id; +import jakarta.persistence.SequenceGenerator; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +import java.time.LocalDateTime; + +@Data +@SuperBuilder +@NoArgsConstructor +@AllArgsConstructor +public class BerthCallsEntity { + + private Long id; + + private String movementType; + private String imolRorIHSNumber; + private LocalDateTime movementDate; + + private Integer facilityId; + private String facilityName; + private String facilityType; + + private Integer parentFacilityId; + private String parentFacilityName; + private String parentFacilityType; + + private String countryCode; + private String countryName; + + private Double draught; + private Double latitude; + private Double longitude; + + private JsonNode position; + + private Integer parentCallId; + private String iso2; + private LocalDateTime eventStartDate; +} diff --git a/src/main/java/com/snp/batch/jobs/shipMovementBerthCalls/batch/processor/BerthCallsProcessor.java b/src/main/java/com/snp/batch/jobs/shipMovementBerthCalls/batch/processor/BerthCallsProcessor.java new file mode 100644 index 0000000..d196256 --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/shipMovementBerthCalls/batch/processor/BerthCallsProcessor.java @@ -0,0 +1,68 @@ +package com.snp.batch.jobs.shipMovementBerthCalls.batch.processor; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.snp.batch.common.batch.processor.BaseProcessor; +import com.snp.batch.jobs.shipMovementBerthCalls.batch.dto.BerthCallsDto; +import com.snp.batch.jobs.shipMovementBerthCalls.batch.entiity.BerthCallsEntity; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; + +import java.time.LocalDateTime; + +/** + * 선박 상세 정보 Processor + * ShipDetailDto → ShipDetailEntity 변환 + */ + +/** + * 선박 상세 정보 Processor (해시 비교 및 증분 데이터 추출) + * I: ShipDetailComparisonData (DB 해시 + API Map Data) + * O: ShipDetailUpdate (변경분) + */ +@Slf4j +@Component +public class BerthCallsProcessor extends BaseProcessor { + + private final ObjectMapper objectMapper; + + public BerthCallsProcessor(ObjectMapper objectMapper) { + this.objectMapper = objectMapper; + } + + @Override + protected BerthCallsEntity processItem(BerthCallsDto dto) throws Exception { + log.debug("선박 상세 정보 처리 시작: imoNumber={}, facilityName={}", + dto.getImolRorIHSNumber(), dto.getFacilityName()); + + JsonNode positionNode = null; + if (dto.getPosition() != null) { + // Position 객체를 JsonNode로 변환 + positionNode = objectMapper.valueToTree(dto.getPosition()); + } + + BerthCallsEntity entity = BerthCallsEntity.builder() + .movementType(dto.getMovementType()) + .imolRorIHSNumber(dto.getImolRorIHSNumber()) + .movementDate(LocalDateTime.parse(dto.getMovementDate())) + .facilityId(dto.getFacilityId()) + .facilityName(dto.getFacilityName()) + .facilityType(dto.getFacilityType()) + .parentFacilityId(dto.getParentFacilityId()) + .parentFacilityName(dto.getParentFacilityName()) + .parentFacilityType(dto.getParentFacilityType()) + .countryCode(dto.getCountryCode()) + .countryName(dto.getCountryName()) + .draught(dto.getDraught()) + .latitude(dto.getLatitude()) + .longitude(dto.getLongitude()) + .position(positionNode) // JsonNode로 매핑 + .parentCallId(dto.getParentCallId()) + .iso2(dto.getIso2()) + .eventStartDate(LocalDateTime.parse(dto.getEventStartDate())) + .build(); + + return entity; + } + +} diff --git a/src/main/java/com/snp/batch/jobs/shipMovementBerthCalls/batch/reader/BerthCallsReader.java b/src/main/java/com/snp/batch/jobs/shipMovementBerthCalls/batch/reader/BerthCallsReader.java new file mode 100644 index 0000000..9f6771d --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/shipMovementBerthCalls/batch/reader/BerthCallsReader.java @@ -0,0 +1,213 @@ +package com.snp.batch.jobs.shipMovementBerthCalls.batch.reader; + +import com.snp.batch.common.batch.reader.BaseApiReader; +import com.snp.batch.jobs.shipMovementBerthCalls.batch.dto.BerthCallsDto; +import lombok.extern.slf4j.Slf4j; +import org.springframework.batch.core.configuration.annotation.StepScope; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.web.reactive.function.client.WebClient; + +import java.util.Collections; +import java.util.List; +import java.util.Map; + +/** + * 선박 상세 정보 Reader (v2.0 - Chunk 기반) + * + * 기능: + * 1. ship_data 테이블에서 IMO 번호 전체 조회 (최초 1회) + * 2. IMO 번호를 100개씩 분할하여 배치 단위로 처리 + * 3. fetchNextBatch() 호출 시마다 100개씩 API 호출 + * 4. Spring Batch가 100건씩 Process → Write 수행 + * + * Chunk 처리 흐름: + * - beforeFetch() → IMO 전체 조회 (1회) + * - fetchNextBatch() → 100개 IMO로 API 호출 (1,718회) + * - read() → 1건씩 반환 (100번) + * - Processor/Writer → 100건 처리 + * - 반복... (1,718번의 Chunk) + * + * 기존 방식과의 차이: + * - 기존: 17만건 전체 메모리 로드 → Process → Write + * - 신규: 100건씩 로드 → Process → Write (Chunk 1,718회) + */ +@Slf4j +@StepScope +public class BerthCallsReader extends BaseApiReader { + + private final JdbcTemplate jdbcTemplate; + + // 배치 처리 상태 + private List allImoNumbers; + // DB 해시값을 저장할 맵 + private Map dbMasterHashes; + private int currentBatchIndex = 0; + private final int batchSize = 5; + + // @Value("#{jobParameters['startDate']}") +// private String startDate; + private String startDate = "2025-01-01"; + + // @Value("#{jobParameters['stopDate']}") +// private String stopDate; + private String stopDate = "2025-12-31"; + + public BerthCallsReader(WebClient webClient, JdbcTemplate jdbcTemplate ) { + super(webClient); + this.jdbcTemplate = jdbcTemplate; + enableChunkMode(); // ✨ Chunk 모드 활성화 + } + + @Override + protected String getReaderName() { + return "BerthCallsReader"; + } + + @Override + protected void resetCustomState() { + this.currentBatchIndex = 0; + this.allImoNumbers = null; + this.dbMasterHashes = null; + } + + @Override + protected String getApiPath() { + return "/Movements/BerthCalls"; + } + + @Override + protected String getApiBaseUrl() { + return "https://webservices.maritime.spglobal.com"; + } + + private static final String GET_ALL_IMO_QUERY = +// "SELECT imo_number FROM ship_data ORDER BY id"; + "SELECT imo_number FROM snp_data.ship_data where imo_number > (select max(imo) from snp_data.t_berthcalls) ORDER BY imo_number"; + + /** + * 최초 1회만 실행: ship_data 테이블에서 IMO 번호 전체 조회 + */ + @Override + protected void beforeFetch() { + // 전처리 과정 + // Step 1. IMO 전체 번호 조회 + log.info("[{}] ship_data 테이블에서 IMO 번호 조회 시작...", getReaderName()); + + allImoNumbers = jdbcTemplate.queryForList(GET_ALL_IMO_QUERY, String.class); + int totalBatches = (int) Math.ceil((double) allImoNumbers.size() / batchSize); + + log.info("[{}] 총 {} 개의 IMO 번호 조회 완료", getReaderName(), allImoNumbers.size()); + log.info("[{}] {}개씩 배치로 분할하여 API 호출 예정", getReaderName(), batchSize); + log.info("[{}] 예상 배치 수: {} 개", getReaderName(), totalBatches); + + // API 통계 초기화 + updateApiCallStats(totalBatches, 0); + } + + /** + * ✨ Chunk 기반 핵심 메서드: 다음 100개 배치를 조회하여 반환 + * + * Spring Batch가 100건씩 read() 호출 완료 후 이 메서드 재호출 + * + * @return 다음 배치 100건 (더 이상 없으면 null) + */ + @Override + protected List fetchNextBatch() throws Exception { + + // 모든 배치 처리 완료 확인 + if (allImoNumbers == null || currentBatchIndex >= allImoNumbers.size()) { + return null; // Job 종료 + } + + // 현재 배치의 시작/끝 인덱스 계산 + int startIndex = currentBatchIndex; + int endIndex = Math.min(currentBatchIndex + batchSize, allImoNumbers.size()); + + // 현재 배치의 IMO 번호 추출 (100개) + List currentBatch = allImoNumbers.subList(startIndex, endIndex); + + int currentBatchNumber = (currentBatchIndex / batchSize) + 1; + int totalBatches = (int) Math.ceil((double) allImoNumbers.size() / batchSize); + + log.info("[{}] 배치 {}/{} 처리 중 (IMO {} 개)...", + getReaderName(), currentBatchNumber, totalBatches, currentBatch.size()); + + try { + // IMO 번호를 쉼표로 연결 (예: "1000019,1000021,1000033,...") + String imoParam = String.join(",", currentBatch); + + // API 호출 + List response = callApiWithBatch(imoParam); + + // 다음 배치로 인덱스 이동 + currentBatchIndex = endIndex; + + + // 응답 처리 + if (response != null ) { + List berthCalls = response; + log.info("[{}] 배치 {}/{} 완료: {} 건 조회", + getReaderName(), currentBatchNumber, totalBatches, berthCalls.size()); + + // API 호출 통계 업데이트 + updateApiCallStats(totalBatches, currentBatchNumber); + + // API 과부하 방지 (다음 배치 전 0.5초 대기) + if (currentBatchIndex < allImoNumbers.size()) { + Thread.sleep(500); + } + + return berthCalls; + + } else { + log.warn("[{}] 배치 {}/{} 응답 없음", + getReaderName(), currentBatchNumber, totalBatches); + + // API 호출 통계 업데이트 (실패도 카운트) + updateApiCallStats(totalBatches, currentBatchNumber); + + return Collections.emptyList(); + } + + } catch (Exception e) { + log.error("[{}] 배치 {}/{} 처리 중 오류: {}", + getReaderName(), currentBatchNumber, totalBatches, e.getMessage(), e); + + // 오류 발생 시에도 다음 배치로 이동 (부분 실패 허용) + currentBatchIndex = endIndex; + + // 빈 리스트 반환 (Job 계속 진행) + return Collections.emptyList(); + } + } + + /** + * Query Parameter를 사용한 API 호출 + * + * @param lrno 쉼표로 연결된 IMO 번호 (예: "1000019,1000021,...") + * @return API 응답 + */ + private List callApiWithBatch(String lrno) { + String url = getApiPath() + "?startDate=" + startDate +"&stopDate="+stopDate+"&lrno=" + lrno; + + log.debug("[{}] API 호출: {}", getReaderName(), url); + + return webClient.get() + .uri(url) + .retrieve() + .bodyToFlux(BerthCallsDto.class) + .collectList() + .block(); + } + + @Override + protected void afterFetch(List data) { + if (data == null) { + int totalBatches = (int) Math.ceil((double) allImoNumbers.size() / batchSize); + log.info("[{}] 전체 {} 개 배치 처리 완료", getReaderName(), totalBatches); + log.info("[{}] 총 {} 개의 IMO 번호에 대한 API 호출 종료", + getReaderName(), allImoNumbers.size()); + } + } + +} diff --git a/src/main/java/com/snp/batch/jobs/shipMovementBerthCalls/batch/repository/BerthCallsRepository.java b/src/main/java/com/snp/batch/jobs/shipMovementBerthCalls/batch/repository/BerthCallsRepository.java new file mode 100644 index 0000000..df2d707 --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/shipMovementBerthCalls/batch/repository/BerthCallsRepository.java @@ -0,0 +1,14 @@ +package com.snp.batch.jobs.shipMovementBerthCalls.batch.repository; + +import com.snp.batch.jobs.shipMovementAnchorageCalls.batch.entity.AnchorageCallsEntity; +import com.snp.batch.jobs.shipMovementBerthCalls.batch.entiity.BerthCallsEntity; + +import java.util.List; + +/** + * 선박 상세 정보 Repository 인터페이스 + */ + +public interface BerthCallsRepository { + void saveAll(List entities); +} diff --git a/src/main/java/com/snp/batch/jobs/shipMovementBerthCalls/batch/repository/BerthCallsRepositoryImpl.java b/src/main/java/com/snp/batch/jobs/shipMovementBerthCalls/batch/repository/BerthCallsRepositoryImpl.java new file mode 100644 index 0000000..757d8a4 --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/shipMovementBerthCalls/batch/repository/BerthCallsRepositoryImpl.java @@ -0,0 +1,192 @@ +package com.snp.batch.jobs.shipMovementBerthCalls.batch.repository; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.snp.batch.common.batch.repository.BaseJdbcRepository; +import com.snp.batch.jobs.shipMovementAnchorageCalls.batch.entity.AnchorageCallsEntity; +import com.snp.batch.jobs.shipMovementAnchorageCalls.batch.repository.AnchorageCallsRepository; +import com.snp.batch.jobs.shipMovementBerthCalls.batch.entiity.BerthCallsEntity; +import lombok.extern.slf4j.Slf4j; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.core.RowMapper; +import org.springframework.stereotype.Repository; + +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Timestamp; +import java.util.List; + +/** + * 선박 상세 정보 Repository 구현체 + * BaseJdbcRepository를 상속하여 JDBC 기반 CRUD 구현 + */ +@Slf4j +@Repository("BerthCallsRepository") +public class BerthCallsRepositoryImpl extends BaseJdbcRepository + implements BerthCallsRepository { + + public BerthCallsRepositoryImpl(JdbcTemplate jdbcTemplate) { + super(jdbcTemplate); + } + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + @Override + protected String getTableName() { + return "snp_data.t_berthcall"; + } + + @Override + protected String getEntityName() { + return "BerthCalls"; + } + + @Override + protected String extractId(BerthCallsEntity entity) { + return entity.getImolRorIHSNumber(); + } + + @Override + public String getInsertSql() { + return """ + INSERT INTO snp_data.t_berthcall( + imo, + mvmn_type, + mvmn_dt, + fclty_id, + fclty_nm, + fclty_type, + up_fclty_id, + up_fclty_nm, + up_fclty_type, + ntn_cd, + ntn_nm, + draft, + lat, + lon, + prnt_call_id, + iso2_ntn_cd, + evt_start_dt, + lcinfo + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT (imo, mvmn_dt) + DO UPDATE SET + mvmn_type = EXCLUDED.mvmn_type, + mvmn_dt = EXCLUDED.mvmn_dt, + fclty_id = EXCLUDED.fclty_id, + fclty_nm = EXCLUDED.fclty_nm, + fclty_type = EXCLUDED.fclty_type, + up_fclty_id = EXCLUDED.up_fclty_id, + up_fclty_nm = EXCLUDED.up_fclty_nm, + up_fclty_type = EXCLUDED.up_fclty_type, + ntn_cd = EXCLUDED.ntn_cd, + ntn_nm = EXCLUDED.ntn_nm, + draft = EXCLUDED.draft, + lat = EXCLUDED.lat, + lon = EXCLUDED.lon, + prnt_call_id = EXCLUDED.prnt_call_id, + iso2_ntn_cd = EXCLUDED.iso2_ntn_cd, + evt_start_dt = EXCLUDED.evt_start_dt, + lcinfo = EXCLUDED.lcinfo + """; + } + + @Override + protected String getUpdateSql() { + return null; + } + + @Override + protected void setInsertParameters(PreparedStatement ps, BerthCallsEntity e) throws Exception { + int i = 1; + ps.setString(i++, e.getImolRorIHSNumber()); // imo + ps.setString(i++, e.getMovementType()); // mvmn_type + ps.setTimestamp(i++, e.getMovementDate() != null ? Timestamp.valueOf(e.getMovementDate()) : null); // mvmn_dt + ps.setObject(i++, e.getFacilityId()); // fclty_id + ps.setString(i++, e.getFacilityName()); // fclty_nm + ps.setString(i++, e.getFacilityType()); // fclty_type + ps.setObject(i++, e.getParentFacilityId()); //up_fclty_id + ps.setString(i++, e.getParentFacilityName()); // up_fclty_nm + ps.setString(i++, e.getParentFacilityType()); //up_fclty_type + ps.setString(i++, e.getCountryCode()); // ntn_cd + ps.setString(i++, e.getCountryName()); // ntn_nm + setDoubleOrNull(ps, i++, e.getDraught()); // draft + setDoubleOrNull(ps, i++, e.getLatitude()); // lat + setDoubleOrNull(ps, i++, e.getLongitude());// lon + ps.setObject(i++, e.getParentCallId()); //prnt_call_id + ps.setString(i++, e.getIso2()); // iso2_ntn_cd + ps.setTimestamp(i++, e.getEventStartDate() != null ? Timestamp.valueOf(e.getEventStartDate()) : null); // evt_start_dt + + if (e.getPosition() != null) { + ps.setObject(i++, OBJECT_MAPPER.writeValueAsString(e.getPosition()), java.sql.Types.OTHER); // lcinfo (jsonb) + } else { + ps.setNull(i++, java.sql.Types.OTHER); + } + } + + private void setDoubleOrNull(PreparedStatement ps, int index, Double value) throws Exception { + if (value != null) { + ps.setDouble(index, value); + } else { + // java.sql.Types.DOUBLE을 사용하여 명시적으로 SQL NULL을 설정 + ps.setNull(index, java.sql.Types.DOUBLE); + } + } + + @Override + protected void setUpdateParameters(PreparedStatement ps, BerthCallsEntity entity) throws Exception { + + } + + @Override + protected RowMapper getRowMapper() { + return null; + } + + @Override + public void saveAll(List entities) { + if (entities == null || entities.isEmpty()) return; + + log.info("BerthCalls 저장 시작 = {}건", entities.size()); + batchInsert(entities); + + } + + + /** + * ShipDetailEntity RowMapper + */ + private static class BerthCallsRowMapper implements RowMapper { + @Override + public BerthCallsEntity mapRow(ResultSet rs, int rowNum) throws SQLException { + BerthCallsEntity entity = BerthCallsEntity.builder() + .id(rs.getLong("id")) + .imolRorIHSNumber(rs.getString("imolRorIHSNumber")) + .facilityId(rs.getObject("facilityId", Integer.class)) + .facilityName(rs.getString("facilityName")) + .facilityType(rs.getString("facilityType")) + .countryCode(rs.getString("countryCode")) + .countryName(rs.getString("countryName")) + .draught(rs.getObject("draught", Double.class)) + .latitude(rs.getObject("latitude", Double.class)) + .longitude(rs.getObject("longitude", Double.class)) + .position(parseJson(rs.getString("position"))) + .build(); + + Timestamp movementDate = rs.getTimestamp("movementDate"); + if (movementDate != null) { + entity.setMovementDate(movementDate.toLocalDateTime()); + } + + return entity; + } + + private JsonNode parseJson(String json) { + try { + if (json == null) return null; + return new ObjectMapper().readTree(json); + } catch (Exception e) { + throw new RuntimeException("JSON 파싱 오류: " + json); + } + } + } +} diff --git a/src/main/java/com/snp/batch/jobs/shipMovementBerthCalls/batch/writer/BerthCallsWriter.java b/src/main/java/com/snp/batch/jobs/shipMovementBerthCalls/batch/writer/BerthCallsWriter.java new file mode 100644 index 0000000..03c1db0 --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/shipMovementBerthCalls/batch/writer/BerthCallsWriter.java @@ -0,0 +1,37 @@ +package com.snp.batch.jobs.shipMovementBerthCalls.batch.writer; + +import com.snp.batch.common.batch.writer.BaseWriter; +import com.snp.batch.jobs.shipMovementAnchorageCalls.batch.entity.AnchorageCallsEntity; +import com.snp.batch.jobs.shipMovementAnchorageCalls.batch.repository.AnchorageCallsRepository; +import com.snp.batch.jobs.shipMovementBerthCalls.batch.entiity.BerthCallsEntity; +import com.snp.batch.jobs.shipMovementBerthCalls.batch.repository.BerthCallsRepository; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; + +import java.util.List; + +/** + * 선박 상세 정보 Writer + */ +@Slf4j +@Component +public class BerthCallsWriter extends BaseWriter { + + private final BerthCallsRepository berthCallsRepository; + + + public BerthCallsWriter(BerthCallsRepository berthCallsRepository) { + super("BerthCalls"); + this.berthCallsRepository = berthCallsRepository; + } + + @Override + protected void writeItems(List items) throws Exception { + + if (items.isEmpty()) { return; } + + berthCallsRepository.saveAll(items); + log.info("BerthCalls 데이터 저장: {} 건", items.size()); + } + +} diff --git a/src/main/java/com/snp/batch/jobs/shipMovementDarkActivity/batch/config/DarkActivityJobConfig.java b/src/main/java/com/snp/batch/jobs/shipMovementDarkActivity/batch/config/DarkActivityJobConfig.java new file mode 100644 index 0000000..0076a18 --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/shipMovementDarkActivity/batch/config/DarkActivityJobConfig.java @@ -0,0 +1,106 @@ +package com.snp.batch.jobs.shipMovementDarkActivity.batch.config; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.snp.batch.common.batch.config.BaseJobConfig; +import com.snp.batch.jobs.shipMovementDarkActivity.batch.dto.DarkActivityDto; +import com.snp.batch.jobs.shipMovementDarkActivity.batch.entity.DarkActivityEntity; +import com.snp.batch.jobs.shipMovementDarkActivity.batch.processor.DarkActivityProcessor; +import com.snp.batch.jobs.shipMovementDarkActivity.batch.reader.DarkActivityReader; +import com.snp.batch.jobs.shipMovementDarkActivity.batch.writer.DarkActivityWriter; +import lombok.extern.slf4j.Slf4j; +import org.springframework.batch.core.Job; +import org.springframework.batch.core.Step; +import org.springframework.batch.core.repository.JobRepository; +import org.springframework.batch.item.ItemProcessor; +import org.springframework.batch.item.ItemReader; +import org.springframework.batch.item.ItemWriter; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.transaction.PlatformTransactionManager; +import org.springframework.web.reactive.function.client.WebClient; + +/** + * 선박 상세 정보 Import Job Config + * + * 특징: + * - ship_data 테이블에서 IMO 번호 조회 + * - IMO 번호를 100개씩 배치로 분할 + * - Maritime API GetShipsByIHSLRorIMONumbers 호출 + * TODO : GetShipsByIHSLRorIMONumbersAll 호출로 변경 + * - 선박 상세 정보를 ship_detail 테이블에 저장 (UPSERT) + * + * 데이터 흐름: + * ShipMovementReader (ship_data → Maritime API) + * ↓ (PortCallDto) + * ShipMovementProcessor + * ↓ (ShipMovementEntity) + * ShipDetailDataWriter + * ↓ (ship_movement 테이블) + */ + +@Slf4j +@Configuration +public class DarkActivityJobConfig extends BaseJobConfig { + + private final DarkActivityProcessor darkActivityProcessor; + private final DarkActivityWriter darkActivityWriter; + private final JdbcTemplate jdbcTemplate; + private final WebClient maritimeApiWebClient; + + public DarkActivityJobConfig( + JobRepository jobRepository, + PlatformTransactionManager transactionManager, + DarkActivityProcessor darkActivityProcessor, + DarkActivityWriter darkActivityWriter, JdbcTemplate jdbcTemplate, + @Qualifier("maritimeServiceApiWebClient") WebClient maritimeApiWebClient, + ObjectMapper objectMapper) { // ObjectMapper 주입 추가 + super(jobRepository, transactionManager); + this.darkActivityProcessor = darkActivityProcessor; + this.darkActivityWriter = darkActivityWriter; + this.jdbcTemplate = jdbcTemplate; + this.maritimeApiWebClient = maritimeApiWebClient; + } + + @Override + protected String getJobName() { + return "DarkActivityImportJob"; + } + + @Override + protected String getStepName() { + return "DarkActivityImportStep"; + } + + @Override + protected ItemReader createReader() { // 타입 변경 + // Reader 생성자 수정: ObjectMapper를 전달합니다. + return new DarkActivityReader(maritimeApiWebClient, jdbcTemplate); + } + + @Override + protected ItemProcessor createProcessor() { + return darkActivityProcessor; + } + + @Override + protected ItemWriter createWriter() { // 타입 변경 + return darkActivityWriter; + } + + @Override + protected int getChunkSize() { + return 5; // API에서 100개씩 가져오므로 chunk도 100으로 설정 + } + + @Bean(name = "DarkActivityImportJob") + public Job darkActivityImportJob() { + return job(); + } + + @Bean(name = "DarkActivityImportStep") + public Step darkActivityImportStep() { + return step(); + } +} diff --git a/src/main/java/com/snp/batch/jobs/shipMovementDarkActivity/batch/dto/DarkActivityDto.java b/src/main/java/com/snp/batch/jobs/shipMovementDarkActivity/batch/dto/DarkActivityDto.java new file mode 100644 index 0000000..9cb7b81 --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/shipMovementDarkActivity/batch/dto/DarkActivityDto.java @@ -0,0 +1,30 @@ +package com.snp.batch.jobs.shipMovementDarkActivity.batch.dto; + +import com.snp.batch.jobs.shipMovementAnchorageCalls.batch.dto.AnchorageCallsPositionDto; +import lombok.Data; + +@Data +public class DarkActivityDto { + private String movementType; + private String imolRorIHSNumber; + private String movementDate; + + private Integer facilityId; + private String facilityName; + private String facilityType; + + private Integer subFacilityId; + private String subFacilityName; + private String subFacilityType; + + private String countryCode; + private String countryName; + + private Double draught; + private Double latitude; + private Double longitude; + + private AnchorageCallsPositionDto position; + + private String eventStartDate; +} diff --git a/src/main/java/com/snp/batch/jobs/shipMovementDarkActivity/batch/entity/DarkActivityEntity.java b/src/main/java/com/snp/batch/jobs/shipMovementDarkActivity/batch/entity/DarkActivityEntity.java new file mode 100644 index 0000000..f05aea5 --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/shipMovementDarkActivity/batch/entity/DarkActivityEntity.java @@ -0,0 +1,41 @@ +package com.snp.batch.jobs.shipMovementDarkActivity.batch.entity; + +import com.fasterxml.jackson.databind.JsonNode; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +import java.time.LocalDateTime; + +@Data +@SuperBuilder +@NoArgsConstructor +@AllArgsConstructor +public class DarkActivityEntity { + + private Long id; + + private String movementType; + private String imolRorIHSNumber; + private LocalDateTime movementDate; + + private Integer facilityId; + private String facilityName; + private String facilityType; + + private Integer subFacilityId; + private String subFacilityName; + private String subFacilityType; + + private String countryCode; + private String countryName; + + private Double draught; + private Double latitude; + private Double longitude; + + private JsonNode position; + + private LocalDateTime eventStartDate; +} diff --git a/src/main/java/com/snp/batch/jobs/shipMovementDarkActivity/batch/processor/DarkActivityProcessor.java b/src/main/java/com/snp/batch/jobs/shipMovementDarkActivity/batch/processor/DarkActivityProcessor.java new file mode 100644 index 0000000..e465f8a --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/shipMovementDarkActivity/batch/processor/DarkActivityProcessor.java @@ -0,0 +1,66 @@ +package com.snp.batch.jobs.shipMovementDarkActivity.batch.processor; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.snp.batch.common.batch.processor.BaseProcessor; +import com.snp.batch.jobs.shipMovementDarkActivity.batch.dto.DarkActivityDto; +import com.snp.batch.jobs.shipMovementDarkActivity.batch.entity.DarkActivityEntity; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; + +import java.time.LocalDateTime; + +/** + * 선박 상세 정보 Processor + * ShipDetailDto → ShipDetailEntity 변환 + */ + +/** + * 선박 상세 정보 Processor (해시 비교 및 증분 데이터 추출) + * I: ShipDetailComparisonData (DB 해시 + API Map Data) + * O: ShipDetailUpdate (변경분) + */ +@Slf4j +@Component +public class DarkActivityProcessor extends BaseProcessor { + + private final ObjectMapper objectMapper; + + public DarkActivityProcessor(ObjectMapper objectMapper) { + this.objectMapper = objectMapper; + } + + @Override + protected DarkActivityEntity processItem(DarkActivityDto dto) throws Exception { + log.debug("선박 상세 정보 처리 시작: imoNumber={}, facilityName={}", + dto.getImolRorIHSNumber(), dto.getFacilityName()); + + JsonNode positionNode = null; + if (dto.getPosition() != null) { + // Position 객체를 JsonNode로 변환 + positionNode = objectMapper.valueToTree(dto.getPosition()); + } + + DarkActivityEntity entity = DarkActivityEntity.builder() + .movementType(dto.getMovementType()) + .imolRorIHSNumber(dto.getImolRorIHSNumber()) + .movementDate(LocalDateTime.parse(dto.getMovementDate())) + .facilityId(dto.getFacilityId()) + .facilityName(dto.getFacilityName()) + .facilityType(dto.getFacilityType()) + .subFacilityId(dto.getSubFacilityId()) + .subFacilityName(dto.getSubFacilityName()) + .subFacilityType(dto.getSubFacilityType()) + .countryCode(dto.getCountryCode()) + .countryName(dto.getCountryName()) + .draught(dto.getDraught()) + .latitude(dto.getLatitude()) + .longitude(dto.getLongitude()) + .position(positionNode) // JsonNode로 매핑 + .eventStartDate(LocalDateTime.parse(dto.getEventStartDate())) + .build(); + + return entity; + } + +} diff --git a/src/main/java/com/snp/batch/jobs/shipMovementDarkActivity/batch/reader/DarkActivityReader.java b/src/main/java/com/snp/batch/jobs/shipMovementDarkActivity/batch/reader/DarkActivityReader.java new file mode 100644 index 0000000..bb74cd0 --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/shipMovementDarkActivity/batch/reader/DarkActivityReader.java @@ -0,0 +1,212 @@ +package com.snp.batch.jobs.shipMovementDarkActivity.batch.reader; + +import com.snp.batch.common.batch.reader.BaseApiReader; +import com.snp.batch.jobs.shipMovementDarkActivity.batch.dto.DarkActivityDto; +import lombok.extern.slf4j.Slf4j; +import org.springframework.batch.core.configuration.annotation.StepScope; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.web.reactive.function.client.WebClient; + +import java.util.Collections; +import java.util.List; +import java.util.Map; + +/** + * 선박 상세 정보 Reader (v2.0 - Chunk 기반) + * + * 기능: + * 1. ship_data 테이블에서 IMO 번호 전체 조회 (최초 1회) + * 2. IMO 번호를 100개씩 분할하여 배치 단위로 처리 + * 3. fetchNextBatch() 호출 시마다 100개씩 API 호출 + * 4. Spring Batch가 100건씩 Process → Write 수행 + * + * Chunk 처리 흐름: + * - beforeFetch() → IMO 전체 조회 (1회) + * - fetchNextBatch() → 100개 IMO로 API 호출 (1,718회) + * - read() → 1건씩 반환 (100번) + * - Processor/Writer → 100건 처리 + * - 반복... (1,718번의 Chunk) + * + * 기존 방식과의 차이: + * - 기존: 17만건 전체 메모리 로드 → Process → Write + * - 신규: 100건씩 로드 → Process → Write (Chunk 1,718회) + */ +@Slf4j +@StepScope +public class DarkActivityReader extends BaseApiReader { + + private final JdbcTemplate jdbcTemplate; + + // 배치 처리 상태 + private List allImoNumbers; + // DB 해시값을 저장할 맵 + private Map dbMasterHashes; + private int currentBatchIndex = 0; + private final int batchSize = 5; + + // @Value("#{jobParameters['startDate']}") +// private String startDate; + private String startDate = "2025-01-01"; + + // @Value("#{jobParameters['stopDate']}") +// private String stopDate; + private String stopDate = "2025-12-31"; + + public DarkActivityReader(WebClient webClient, JdbcTemplate jdbcTemplate ) { + super(webClient); + this.jdbcTemplate = jdbcTemplate; + enableChunkMode(); // ✨ Chunk 모드 활성화 + } + + @Override + protected String getReaderName() { + return "DarkActivityReader"; + } + + @Override + protected void resetCustomState() { + this.currentBatchIndex = 0; + this.allImoNumbers = null; + this.dbMasterHashes = null; + } + + @Override + protected String getApiPath() { + return "/Movements/DarkActivity"; + } + + @Override + protected String getApiBaseUrl() { + return "https://webservices.maritime.spglobal.com"; + } + + private static final String GET_ALL_IMO_QUERY = +// "SELECT imo_number FROM ship_data ORDER BY id"; + "SELECT imo_number FROM snp_data.ship_data where imo_number > (select max(imo) from snp_data.t_darkactivity) ORDER BY imo_number"; + + /** + * 최초 1회만 실행: ship_data 테이블에서 IMO 번호 전체 조회 + */ + @Override + protected void beforeFetch() { + // 전처리 과정 + // Step 1. IMO 전체 번호 조회 + log.info("[{}] ship_data 테이블에서 IMO 번호 조회 시작...", getReaderName()); + + allImoNumbers = jdbcTemplate.queryForList(GET_ALL_IMO_QUERY, String.class); + int totalBatches = (int) Math.ceil((double) allImoNumbers.size() / batchSize); + + log.info("[{}] 총 {} 개의 IMO 번호 조회 완료", getReaderName(), allImoNumbers.size()); + log.info("[{}] {}개씩 배치로 분할하여 API 호출 예정", getReaderName(), batchSize); + log.info("[{}] 예상 배치 수: {} 개", getReaderName(), totalBatches); + + // API 통계 초기화 + updateApiCallStats(totalBatches, 0); + } + + /** + * ✨ Chunk 기반 핵심 메서드: 다음 100개 배치를 조회하여 반환 + * + * Spring Batch가 100건씩 read() 호출 완료 후 이 메서드 재호출 + * + * @return 다음 배치 100건 (더 이상 없으면 null) + */ + @Override + protected List fetchNextBatch() throws Exception { + + // 모든 배치 처리 완료 확인 + if (allImoNumbers == null || currentBatchIndex >= allImoNumbers.size()) { + return null; // Job 종료 + } + + // 현재 배치의 시작/끝 인덱스 계산 + int startIndex = currentBatchIndex; + int endIndex = Math.min(currentBatchIndex + batchSize, allImoNumbers.size()); + + // 현재 배치의 IMO 번호 추출 (100개) + List currentBatch = allImoNumbers.subList(startIndex, endIndex); + + int currentBatchNumber = (currentBatchIndex / batchSize) + 1; + int totalBatches = (int) Math.ceil((double) allImoNumbers.size() / batchSize); + + log.info("[{}] 배치 {}/{} 처리 중 (IMO {} 개)...", + getReaderName(), currentBatchNumber, totalBatches, currentBatch.size()); + + try { + // IMO 번호를 쉼표로 연결 (예: "1000019,1000021,1000033,...") + String imoParam = String.join(",", currentBatch); + // API 호출 + List response = callApiWithBatch(imoParam); + + // 다음 배치로 인덱스 이동 + currentBatchIndex = endIndex; + + + // 응답 처리 + if (response != null ) { + List darkActivityList = response; + log.info("[{}] 배치 {}/{} 완료: {} 건 조회", + getReaderName(), currentBatchNumber, totalBatches, darkActivityList.size()); + + // API 호출 통계 업데이트 + updateApiCallStats(totalBatches, currentBatchNumber); + + // API 과부하 방지 (다음 배치 전 0.5초 대기) + if (currentBatchIndex < allImoNumbers.size()) { + Thread.sleep(500); + } + + return darkActivityList; + + } else { + log.warn("[{}] 배치 {}/{} 응답 없음", + getReaderName(), currentBatchNumber, totalBatches); + + // API 호출 통계 업데이트 (실패도 카운트) + updateApiCallStats(totalBatches, currentBatchNumber); + + return Collections.emptyList(); + } + + } catch (Exception e) { + log.error("[{}] 배치 {}/{} 처리 중 오류: {}", + getReaderName(), currentBatchNumber, totalBatches, e.getMessage(), e); + + // 오류 발생 시에도 다음 배치로 이동 (부분 실패 허용) + currentBatchIndex = endIndex; + + // 빈 리스트 반환 (Job 계속 진행) + return Collections.emptyList(); + } + } + + /** + * Query Parameter를 사용한 API 호출 + * + * @param lrno 쉼표로 연결된 IMO 번호 (예: "1000019,1000021,...") + * @return API 응답 + */ + private List callApiWithBatch(String lrno) { + String url = getApiPath() + "?startDate=" + startDate +"&stopDate="+stopDate+"&lrno=" + lrno; + + log.debug("[{}] API 호출: {}", getReaderName(), url); + + return webClient.get() + .uri(url) + .retrieve() + .bodyToFlux(DarkActivityDto.class) + .collectList() + .block(); + } + + @Override + protected void afterFetch(List data) { + if (data == null) { + int totalBatches = (int) Math.ceil((double) allImoNumbers.size() / batchSize); + log.info("[{}] 전체 {} 개 배치 처리 완료", getReaderName(), totalBatches); + log.info("[{}] 총 {} 개의 IMO 번호에 대한 API 호출 종료", + getReaderName(), allImoNumbers.size()); + } + } + +} diff --git a/src/main/java/com/snp/batch/jobs/shipMovementDarkActivity/batch/repository/DarkActivityRepository.java b/src/main/java/com/snp/batch/jobs/shipMovementDarkActivity/batch/repository/DarkActivityRepository.java new file mode 100644 index 0000000..f18da07 --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/shipMovementDarkActivity/batch/repository/DarkActivityRepository.java @@ -0,0 +1,14 @@ +package com.snp.batch.jobs.shipMovementDarkActivity.batch.repository; + +import com.snp.batch.jobs.shipMovementBerthCalls.batch.entiity.BerthCallsEntity; +import com.snp.batch.jobs.shipMovementDarkActivity.batch.entity.DarkActivityEntity; + +import java.util.List; + +/** + * 선박 상세 정보 Repository 인터페이스 + */ + +public interface DarkActivityRepository { + void saveAll(List entities); +} diff --git a/src/main/java/com/snp/batch/jobs/shipMovementDarkActivity/batch/repository/DarkActivityRepositoryImpl.java b/src/main/java/com/snp/batch/jobs/shipMovementDarkActivity/batch/repository/DarkActivityRepositoryImpl.java new file mode 100644 index 0000000..89eb8fd --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/shipMovementDarkActivity/batch/repository/DarkActivityRepositoryImpl.java @@ -0,0 +1,186 @@ +package com.snp.batch.jobs.shipMovementDarkActivity.batch.repository; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.snp.batch.common.batch.repository.BaseJdbcRepository; +import com.snp.batch.jobs.shipMovementBerthCalls.batch.entiity.BerthCallsEntity; +import com.snp.batch.jobs.shipMovementBerthCalls.batch.repository.BerthCallsRepository; +import com.snp.batch.jobs.shipMovementDarkActivity.batch.entity.DarkActivityEntity; +import lombok.extern.slf4j.Slf4j; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.core.RowMapper; +import org.springframework.stereotype.Repository; + +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Timestamp; +import java.util.List; + +/** + * 선박 상세 정보 Repository 구현체 + * BaseJdbcRepository를 상속하여 JDBC 기반 CRUD 구현 + */ +@Slf4j +@Repository("") +public class DarkActivityRepositoryImpl extends BaseJdbcRepository + implements DarkActivityRepository { + + public DarkActivityRepositoryImpl(JdbcTemplate jdbcTemplate) { + super(jdbcTemplate); + } + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + @Override + protected String getTableName() { + return "snp_data.t_darkactivity"; + } + + @Override + protected String getEntityName() { + return "DarkActivity"; + } + + @Override + protected String extractId(DarkActivityEntity entity) { + return entity.getImolRorIHSNumber(); + } + + @Override + public String getInsertSql() { + return """ + INSERT INTO snp_data.t_darkactivity( + imo, + mvmn_type, + mvmn_dt, + fclty_id, + fclty_nm, + fclty_type, + lwrnk_fclty_id, + lwrnk_fclty_nm, + lwrnk_fclty_type, + ntn_cd, + ntn_nm, + draft, + lat, + lon, + evt_start_dt, + lcinfo + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT (imo, mvmn_dt) + DO UPDATE SET + mvmn_type = EXCLUDED.mvmn_type, + mvmn_dt = EXCLUDED.mvmn_dt, + fclty_id = EXCLUDED.fclty_id, + fclty_nm = EXCLUDED.fclty_nm, + fclty_type = EXCLUDED.fclty_type, + lwrnk_fclty_id = EXCLUDED.lwrnk_fclty_id, + lwrnk_fclty_nm = EXCLUDED.lwrnk_fclty_nm, + lwrnk_fclty_type = EXCLUDED.lwrnk_fclty_type, + ntn_cd = EXCLUDED.ntn_cd, + ntn_nm = EXCLUDED.ntn_nm, + draft = EXCLUDED.draft, + lat = EXCLUDED.lat, + lon = EXCLUDED.lon, + evt_start_dt = EXCLUDED.evt_start_dt, + lcinfo = EXCLUDED.lcinfo + """; + } + + @Override + protected String getUpdateSql() { + return null; + } + + @Override + protected void setInsertParameters(PreparedStatement ps, DarkActivityEntity e) throws Exception { + int i = 1; + ps.setString(i++, e.getImolRorIHSNumber()); // imo + ps.setString(i++, e.getMovementType()); // mvmn_type + ps.setTimestamp(i++, e.getMovementDate() != null ? Timestamp.valueOf(e.getMovementDate()) : null); // mvmn_dt + ps.setObject(i++, e.getFacilityId()); // fclty_id + ps.setString(i++, e.getFacilityName()); // fclty_nm + ps.setString(i++, e.getFacilityType()); // fclty_type + ps.setObject(i++, e.getSubFacilityId()); //lwrnk_fclty_id + ps.setString(i++, e.getSubFacilityName()); // lwrnk_fclty_nm + ps.setString(i++, e.getSubFacilityType()); //lwrnk_fclty_type + ps.setString(i++, e.getCountryCode()); // ntn_cd + ps.setString(i++, e.getCountryName()); // ntn_nm + setDoubleOrNull(ps, i++, e.getDraught()); // draft + setDoubleOrNull(ps, i++, e.getLatitude()); // lat + setDoubleOrNull(ps, i++, e.getLongitude());// lon + ps.setTimestamp(i++, e.getEventStartDate() != null ? Timestamp.valueOf(e.getEventStartDate()) : null); // evt_start_dt + + if (e.getPosition() != null) { + ps.setObject(i++, OBJECT_MAPPER.writeValueAsString(e.getPosition()), java.sql.Types.OTHER); // lcinfo (jsonb) + } else { + ps.setNull(i++, java.sql.Types.OTHER); + } + } + + private void setDoubleOrNull(PreparedStatement ps, int index, Double value) throws Exception { + if (value != null) { + ps.setDouble(index, value); + } else { + // java.sql.Types.DOUBLE을 사용하여 명시적으로 SQL NULL을 설정 + ps.setNull(index, java.sql.Types.DOUBLE); + } + } + + @Override + protected void setUpdateParameters(PreparedStatement ps, DarkActivityEntity entity) throws Exception { + + } + + @Override + protected RowMapper getRowMapper() { + return null; + } + + @Override + public void saveAll(List entities) { + if (entities == null || entities.isEmpty()) return; + + log.info("DarkActivity 저장 시작 = {}건", entities.size()); + batchInsert(entities); + + } + + + /** + * ShipDetailEntity RowMapper + */ + private static class DarkActivityRowMapper implements RowMapper { + @Override + public DarkActivityEntity mapRow(ResultSet rs, int rowNum) throws SQLException { + DarkActivityEntity entity = DarkActivityEntity.builder() + .id(rs.getLong("id")) + .imolRorIHSNumber(rs.getString("imolRorIHSNumber")) + .facilityId(rs.getObject("facilityId", Integer.class)) + .facilityName(rs.getString("facilityName")) + .facilityType(rs.getString("facilityType")) + .countryCode(rs.getString("countryCode")) + .countryName(rs.getString("countryName")) + .draught(rs.getObject("draught", Double.class)) + .latitude(rs.getObject("latitude", Double.class)) + .longitude(rs.getObject("longitude", Double.class)) + .position(parseJson(rs.getString("position"))) + .build(); + + Timestamp movementDate = rs.getTimestamp("movementDate"); + if (movementDate != null) { + entity.setMovementDate(movementDate.toLocalDateTime()); + } + + return entity; + } + + private JsonNode parseJson(String json) { + try { + if (json == null) return null; + return new ObjectMapper().readTree(json); + } catch (Exception e) { + throw new RuntimeException("JSON 파싱 오류: " + json); + } + } + } +} diff --git a/src/main/java/com/snp/batch/jobs/shipMovementDarkActivity/batch/writer/DarkActivityWriter.java b/src/main/java/com/snp/batch/jobs/shipMovementDarkActivity/batch/writer/DarkActivityWriter.java new file mode 100644 index 0000000..901876c --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/shipMovementDarkActivity/batch/writer/DarkActivityWriter.java @@ -0,0 +1,37 @@ +package com.snp.batch.jobs.shipMovementDarkActivity.batch.writer; + +import com.snp.batch.common.batch.writer.BaseWriter; +import com.snp.batch.jobs.shipMovementBerthCalls.batch.entiity.BerthCallsEntity; +import com.snp.batch.jobs.shipMovementBerthCalls.batch.repository.BerthCallsRepository; +import com.snp.batch.jobs.shipMovementDarkActivity.batch.entity.DarkActivityEntity; +import com.snp.batch.jobs.shipMovementDarkActivity.batch.repository.DarkActivityRepository; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; + +import java.util.List; + +/** + * 선박 상세 정보 Writer + */ +@Slf4j +@Component +public class DarkActivityWriter extends BaseWriter { + + private final DarkActivityRepository darkActivityRepository; + + + public DarkActivityWriter(DarkActivityRepository darkActivityRepository) { + super("DarkActivity"); + this.darkActivityRepository = darkActivityRepository; + } + + @Override + protected void writeItems(List items) throws Exception { + + if (items.isEmpty()) { return; } + + darkActivityRepository.saveAll(items); + log.info("DarkActivity 데이터 저장: {} 건", items.size()); + } + +} diff --git a/src/main/java/com/snp/batch/jobs/shipMovementStsOperations/batch/config/StsOperationJobConfig.java b/src/main/java/com/snp/batch/jobs/shipMovementStsOperations/batch/config/StsOperationJobConfig.java new file mode 100644 index 0000000..1d57569 --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/shipMovementStsOperations/batch/config/StsOperationJobConfig.java @@ -0,0 +1,110 @@ +package com.snp.batch.jobs.shipMovementStsOperations.batch.config; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.snp.batch.common.batch.config.BaseJobConfig; +import com.snp.batch.jobs.shipMovementBerthCalls.batch.dto.BerthCallsDto; +import com.snp.batch.jobs.shipMovementBerthCalls.batch.entiity.BerthCallsEntity; +import com.snp.batch.jobs.shipMovementBerthCalls.batch.processor.BerthCallsProcessor; +import com.snp.batch.jobs.shipMovementBerthCalls.batch.reader.BerthCallsReader; +import com.snp.batch.jobs.shipMovementBerthCalls.batch.writer.BerthCallsWriter; +import com.snp.batch.jobs.shipMovementStsOperations.batch.dto.StsOperationDto; +import com.snp.batch.jobs.shipMovementStsOperations.batch.entity.StsOperationEntity; +import com.snp.batch.jobs.shipMovementStsOperations.batch.processor.StsOperationProcessor; +import com.snp.batch.jobs.shipMovementStsOperations.batch.reader.StsOperationReader; +import com.snp.batch.jobs.shipMovementStsOperations.batch.writer.StsOperationWriter; +import lombok.extern.slf4j.Slf4j; +import org.springframework.batch.core.Job; +import org.springframework.batch.core.Step; +import org.springframework.batch.core.repository.JobRepository; +import org.springframework.batch.item.ItemProcessor; +import org.springframework.batch.item.ItemReader; +import org.springframework.batch.item.ItemWriter; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.transaction.PlatformTransactionManager; +import org.springframework.web.reactive.function.client.WebClient; + +/** + * 선박 상세 정보 Import Job Config + * + * 특징: + * - ship_data 테이블에서 IMO 번호 조회 + * - IMO 번호를 100개씩 배치로 분할 + * - Maritime API GetShipsByIHSLRorIMONumbers 호출 + * TODO : GetShipsByIHSLRorIMONumbersAll 호출로 변경 + * - 선박 상세 정보를 ship_detail 테이블에 저장 (UPSERT) + * + * 데이터 흐름: + * ShipMovementReader (ship_data → Maritime API) + * ↓ (PortCallDto) + * ShipMovementProcessor + * ↓ (ShipMovementEntity) + * ShipDetailDataWriter + * ↓ (ship_movement 테이블) + */ + +@Slf4j +@Configuration +public class StsOperationJobConfig extends BaseJobConfig { + + private final StsOperationProcessor stsOperationProcessor; + private final StsOperationWriter stsOperationWriter; + private final JdbcTemplate jdbcTemplate; + private final WebClient maritimeApiWebClient; + + public StsOperationJobConfig( + JobRepository jobRepository, + PlatformTransactionManager transactionManager, + StsOperationProcessor stsOperationProcessor, + StsOperationWriter stsOperationWriter, JdbcTemplate jdbcTemplate, + @Qualifier("maritimeServiceApiWebClient") WebClient maritimeApiWebClient) { // ObjectMapper 주입 추가 + super(jobRepository, transactionManager); + this.stsOperationProcessor = stsOperationProcessor; + this.stsOperationWriter = stsOperationWriter; + this.jdbcTemplate = jdbcTemplate; + this.maritimeApiWebClient = maritimeApiWebClient; + } + + @Override + protected String getJobName() { + return "STSOperationImportJob"; + } + + @Override + protected String getStepName() { + return "STSOperationImportStep"; + } + + @Override + protected ItemReader createReader() { // 타입 변경 + // Reader 생성자 수정: ObjectMapper를 전달합니다. + return new StsOperationReader(maritimeApiWebClient, jdbcTemplate); + } + + @Override + protected ItemProcessor createProcessor() { + return stsOperationProcessor; + } + + @Override + protected ItemWriter createWriter() { // 타입 변경 + return stsOperationWriter; + } + + @Override + protected int getChunkSize() { + return 200; // API에서 100개씩 가져오므로 chunk도 100으로 설정 + } + + @Bean(name = "STSOperationImportJob") + public Job stsOperationImportJob() { + return job(); + } + + @Bean(name = "STSOperationImportStep") + public Step stsOperationImportStep() { + return step(); + } +} diff --git a/src/main/java/com/snp/batch/jobs/shipMovementStsOperations/batch/dto/StsOperationDto.java b/src/main/java/com/snp/batch/jobs/shipMovementStsOperations/batch/dto/StsOperationDto.java new file mode 100644 index 0000000..16c8db8 --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/shipMovementStsOperations/batch/dto/StsOperationDto.java @@ -0,0 +1,35 @@ +package com.snp.batch.jobs.shipMovementStsOperations.batch.dto; + +import com.snp.batch.jobs.shipMovementBerthCalls.batch.dto.BerthCallsPositionDto; +import lombok.Data; + +@Data +public class StsOperationDto { + private String movementType; + private String imolRorIHSNumber; + private String movementDate; + + private Integer facilityId; + private String facilityName; + private String facilityType; + + private Integer parentFacilityId; + private String parentFacilityName; + private String parentFacilityType; + + private Double draught; + private Double latitude; + private Double longitude; + + private StsOperationPositionDto position; + + private Long parentCallId; + + private String countryCode; + private String countryName; + + private String stsLocation; + private String stsType; + + private String eventStartDate; +} diff --git a/src/main/java/com/snp/batch/jobs/shipMovementStsOperations/batch/dto/StsOperationPositionDto.java b/src/main/java/com/snp/batch/jobs/shipMovementStsOperations/batch/dto/StsOperationPositionDto.java new file mode 100644 index 0000000..85496f0 --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/shipMovementStsOperations/batch/dto/StsOperationPositionDto.java @@ -0,0 +1,17 @@ +package com.snp.batch.jobs.shipMovementStsOperations.batch.dto; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; + +@Data +public class StsOperationPositionDto { + private boolean isNull; + private int stSrid; + private double lat; + @JsonProperty("long") + private double lon; + private double z; + private double m; + private boolean hasZ; + private boolean hasM; +} diff --git a/src/main/java/com/snp/batch/jobs/shipMovementStsOperations/batch/entity/StsOperationEntity.java b/src/main/java/com/snp/batch/jobs/shipMovementStsOperations/batch/entity/StsOperationEntity.java new file mode 100644 index 0000000..e47acf0 --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/shipMovementStsOperations/batch/entity/StsOperationEntity.java @@ -0,0 +1,45 @@ +package com.snp.batch.jobs.shipMovementStsOperations.batch.entity; + +import com.fasterxml.jackson.databind.JsonNode; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +import java.time.LocalDateTime; + +@Data +@SuperBuilder +@NoArgsConstructor +@AllArgsConstructor +public class StsOperationEntity { + + private Long id; + + private String movementType; + private String imolRorIHSNumber; + private java.time.LocalDateTime movementDate; + + private Integer facilityId; + private String facilityName; + private String facilityType; + + private Integer parentFacilityId; + private String parentFacilityName; + private String parentFacilityType; + + private Double draught; + private Double latitude; + private Double longitude; + + private JsonNode position; + + private Long parentCallId; + + private String countryCode; + private String countryName; + + private String stsLocation; + private String stsType; + private LocalDateTime eventStartDate; +} diff --git a/src/main/java/com/snp/batch/jobs/shipMovementStsOperations/batch/processor/StsOperationProcessor.java b/src/main/java/com/snp/batch/jobs/shipMovementStsOperations/batch/processor/StsOperationProcessor.java new file mode 100644 index 0000000..fdb73bd --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/shipMovementStsOperations/batch/processor/StsOperationProcessor.java @@ -0,0 +1,69 @@ +package com.snp.batch.jobs.shipMovementStsOperations.batch.processor; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.snp.batch.common.batch.processor.BaseProcessor; +import com.snp.batch.jobs.shipMovementStsOperations.batch.dto.StsOperationDto; +import com.snp.batch.jobs.shipMovementStsOperations.batch.entity.StsOperationEntity; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; + +import java.time.LocalDateTime; + +/** + * 선박 상세 정보 Processor + * ShipDetailDto → ShipDetailEntity 변환 + */ + +/** + * 선박 상세 정보 Processor (해시 비교 및 증분 데이터 추출) + * I: ShipDetailComparisonData (DB 해시 + API Map Data) + * O: ShipDetailUpdate (변경분) + */ +@Slf4j +@Component +public class StsOperationProcessor extends BaseProcessor { + + private final ObjectMapper objectMapper; + + public StsOperationProcessor(ObjectMapper objectMapper) { + this.objectMapper = objectMapper; + } + + @Override + protected StsOperationEntity processItem(StsOperationDto dto) throws Exception { + log.debug("선박 상세 정보 처리 시작: imoNumber={}, facilityName={}", + dto.getImolRorIHSNumber(), dto.getFacilityName()); + + JsonNode positionNode = null; + if (dto.getPosition() != null) { + // Position 객체를 JsonNode로 변환 + positionNode = objectMapper.valueToTree(dto.getPosition()); + } + + StsOperationEntity entity = StsOperationEntity.builder() + .movementType(dto.getMovementType()) + .imolRorIHSNumber(dto.getImolRorIHSNumber()) + .movementDate(LocalDateTime.parse(dto.getMovementDate())) + .facilityId(dto.getFacilityId()) + .facilityName(dto.getFacilityName()) + .facilityType(dto.getFacilityType()) + .parentFacilityId(dto.getParentFacilityId()) + .parentFacilityName(dto.getParentFacilityName()) + .parentFacilityType(dto.getParentFacilityType()) + .draught(dto.getDraught()) + .latitude(dto.getLatitude()) + .longitude(dto.getLongitude()) + .position(positionNode) // JsonNode로 매핑 + .parentCallId(dto.getParentCallId()) + .countryCode(dto.getCountryCode()) + .countryName(dto.getCountryName()) + .stsLocation(dto.getStsLocation()) + .stsType(dto.getStsType()) + .eventStartDate(LocalDateTime.parse(dto.getEventStartDate())) + .build(); + + return entity; + } + +} diff --git a/src/main/java/com/snp/batch/jobs/shipMovementStsOperations/batch/reader/StsOperationReader.java b/src/main/java/com/snp/batch/jobs/shipMovementStsOperations/batch/reader/StsOperationReader.java new file mode 100644 index 0000000..a6c286c --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/shipMovementStsOperations/batch/reader/StsOperationReader.java @@ -0,0 +1,213 @@ +package com.snp.batch.jobs.shipMovementStsOperations.batch.reader; + +import com.snp.batch.common.batch.reader.BaseApiReader; +import com.snp.batch.jobs.shipMovementStsOperations.batch.dto.StsOperationDto; +import lombok.extern.slf4j.Slf4j; +import org.springframework.batch.core.configuration.annotation.StepScope; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.web.reactive.function.client.WebClient; + +import java.util.Collections; +import java.util.List; +import java.util.Map; + +/** + * 선박 상세 정보 Reader (v2.0 - Chunk 기반) + * + * 기능: + * 1. ship_data 테이블에서 IMO 번호 전체 조회 (최초 1회) + * 2. IMO 번호를 100개씩 분할하여 배치 단위로 처리 + * 3. fetchNextBatch() 호출 시마다 100개씩 API 호출 + * 4. Spring Batch가 100건씩 Process → Write 수행 + * + * Chunk 처리 흐름: + * - beforeFetch() → IMO 전체 조회 (1회) + * - fetchNextBatch() → 100개 IMO로 API 호출 (1,718회) + * - read() → 1건씩 반환 (100번) + * - Processor/Writer → 100건 처리 + * - 반복... (1,718번의 Chunk) + * + * 기존 방식과의 차이: + * - 기존: 17만건 전체 메모리 로드 → Process → Write + * - 신규: 100건씩 로드 → Process → Write (Chunk 1,718회) + */ +@Slf4j +@StepScope +public class StsOperationReader extends BaseApiReader { + + private final JdbcTemplate jdbcTemplate; + + // 배치 처리 상태 + private List allImoNumbers; + // DB 해시값을 저장할 맵 + private Map dbMasterHashes; + private int currentBatchIndex = 0; + private final int batchSize = 5; + + // @Value("#{jobParameters['startDate']}") +// private String startDate; + private String startDate = "2025-01-01"; + + // @Value("#{jobParameters['stopDate']}") +// private String stopDate; + private String stopDate = "2025-12-31"; + + public StsOperationReader(WebClient webClient, JdbcTemplate jdbcTemplate ) { + super(webClient); + this.jdbcTemplate = jdbcTemplate; + enableChunkMode(); // ✨ Chunk 모드 활성화 + } + + @Override + protected String getReaderName() { + return "StsOperationReader"; + } + + @Override + protected void resetCustomState() { + this.currentBatchIndex = 0; + this.allImoNumbers = null; + this.dbMasterHashes = null; + } + + @Override + protected String getApiPath() { + return "/Movements/StsOperations"; + } + + @Override + protected String getApiBaseUrl() { + return "https://webservices.maritime.spglobal.com"; + } + + private static final String GET_ALL_IMO_QUERY = +// "SELECT imo_number FROM ship_data ORDER BY id"; + "SELECT imo_number FROM snp_data.ship_data where imo_number > (select max(imo) from snp_data.t_stsoperation) ORDER BY imo_number"; + + /** + * 최초 1회만 실행: ship_data 테이블에서 IMO 번호 전체 조회 + */ + @Override + protected void beforeFetch() { + // 전처리 과정 + // Step 1. IMO 전체 번호 조회 + log.info("[{}] ship_data 테이블에서 IMO 번호 조회 시작...", getReaderName()); + + allImoNumbers = jdbcTemplate.queryForList(GET_ALL_IMO_QUERY, String.class); + int totalBatches = (int) Math.ceil((double) allImoNumbers.size() / batchSize); + + log.info("[{}] 총 {} 개의 IMO 번호 조회 완료", getReaderName(), allImoNumbers.size()); + log.info("[{}] {}개씩 배치로 분할하여 API 호출 예정", getReaderName(), batchSize); + log.info("[{}] 예상 배치 수: {} 개", getReaderName(), totalBatches); + + // API 통계 초기화 + updateApiCallStats(totalBatches, 0); + } + + /** + * ✨ Chunk 기반 핵심 메서드: 다음 100개 배치를 조회하여 반환 + * + * Spring Batch가 100건씩 read() 호출 완료 후 이 메서드 재호출 + * + * @return 다음 배치 100건 (더 이상 없으면 null) + */ + @Override + protected List fetchNextBatch() throws Exception { + + // 모든 배치 처리 완료 확인 + if (allImoNumbers == null || currentBatchIndex >= allImoNumbers.size()) { + return null; // Job 종료 + } + + // 현재 배치의 시작/끝 인덱스 계산 + int startIndex = currentBatchIndex; + int endIndex = Math.min(currentBatchIndex + batchSize, allImoNumbers.size()); + + // 현재 배치의 IMO 번호 추출 (100개) + List currentBatch = allImoNumbers.subList(startIndex, endIndex); + + int currentBatchNumber = (currentBatchIndex / batchSize) + 1; + int totalBatches = (int) Math.ceil((double) allImoNumbers.size() / batchSize); + + log.info("[{}] 배치 {}/{} 처리 중 (IMO {} 개)...", + getReaderName(), currentBatchNumber, totalBatches, currentBatch.size()); + + try { + // IMO 번호를 쉼표로 연결 (예: "1000019,1000021,1000033,...") + String imoParam = String.join(",", currentBatch); + + // API 호출 + List response = callApiWithBatch(imoParam); + + // 다음 배치로 인덱스 이동 + currentBatchIndex = endIndex; + + + // 응답 처리 + if (response != null ) { + List responseList = response; + log.info("[{}] 배치 {}/{} 완료: {} 건 조회", + getReaderName(), currentBatchNumber, totalBatches, responseList.size()); + + // API 호출 통계 업데이트 + updateApiCallStats(totalBatches, currentBatchNumber); + + // API 과부하 방지 (다음 배치 전 0.5초 대기) + if (currentBatchIndex < allImoNumbers.size()) { + Thread.sleep(500); + } + + return responseList; + + } else { + log.warn("[{}] 배치 {}/{} 응답 없음", + getReaderName(), currentBatchNumber, totalBatches); + + // API 호출 통계 업데이트 (실패도 카운트) + updateApiCallStats(totalBatches, currentBatchNumber); + + return Collections.emptyList(); + } + + } catch (Exception e) { + log.error("[{}] 배치 {}/{} 처리 중 오류: {}", + getReaderName(), currentBatchNumber, totalBatches, e.getMessage(), e); + + // 오류 발생 시에도 다음 배치로 이동 (부분 실패 허용) + currentBatchIndex = endIndex; + + // 빈 리스트 반환 (Job 계속 진행) + return Collections.emptyList(); + } + } + + /** + * Query Parameter를 사용한 API 호출 + * + * @param lrno 쉼표로 연결된 IMO 번호 (예: "1000019,1000021,...") + * @return API 응답 + */ + private List callApiWithBatch(String lrno) { + String url = getApiPath() + "?startDate=" + startDate +"&stopDate="+stopDate+"&lrno=" + lrno; + + log.debug("[{}] API 호출: {}", getReaderName(), url); + + return webClient.get() + .uri(url) + .retrieve() + .bodyToFlux(StsOperationDto.class) + .collectList() + .block(); + } + + @Override + protected void afterFetch(List data) { + if (data == null) { + int totalBatches = (int) Math.ceil((double) allImoNumbers.size() / batchSize); + log.info("[{}] 전체 {} 개 배치 처리 완료", getReaderName(), totalBatches); + log.info("[{}] 총 {} 개의 IMO 번호에 대한 API 호출 종료", + getReaderName(), allImoNumbers.size()); + } + } + +} diff --git a/src/main/java/com/snp/batch/jobs/shipMovementStsOperations/batch/repository/StsOperationRepository.java b/src/main/java/com/snp/batch/jobs/shipMovementStsOperations/batch/repository/StsOperationRepository.java new file mode 100644 index 0000000..a081c51 --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/shipMovementStsOperations/batch/repository/StsOperationRepository.java @@ -0,0 +1,12 @@ +package com.snp.batch.jobs.shipMovementStsOperations.batch.repository; + +import com.snp.batch.jobs.shipMovementStsOperations.batch.entity.StsOperationEntity; +import java.util.List; + +/** + * 선박 상세 정보 Repository 인터페이스 + */ + +public interface StsOperationRepository { + void saveAll(List entities); +} diff --git a/src/main/java/com/snp/batch/jobs/shipMovementStsOperations/batch/repository/StsOperationRepositoryImpl.java b/src/main/java/com/snp/batch/jobs/shipMovementStsOperations/batch/repository/StsOperationRepositoryImpl.java new file mode 100644 index 0000000..06cb967 --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/shipMovementStsOperations/batch/repository/StsOperationRepositoryImpl.java @@ -0,0 +1,162 @@ +package com.snp.batch.jobs.shipMovementStsOperations.batch.repository; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.snp.batch.common.batch.repository.BaseJdbcRepository; +import com.snp.batch.jobs.shipMovementStsOperations.batch.entity.StsOperationEntity; +import lombok.extern.slf4j.Slf4j; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.core.RowMapper; +import org.springframework.stereotype.Repository; + +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Timestamp; +import java.util.List; + +/** + * 선박 상세 정보 Repository 구현체 + * BaseJdbcRepository를 상속하여 JDBC 기반 CRUD 구현 + */ +@Slf4j +@Repository("StsOperationRepository") +public class StsOperationRepositoryImpl extends BaseJdbcRepository + implements StsOperationRepository { + + public StsOperationRepositoryImpl(JdbcTemplate jdbcTemplate) { + super(jdbcTemplate); + } + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + @Override + protected String getTableName() { + return "snp_data.t_stsoperation"; + } + + @Override + protected String getEntityName() { + return "StsOperation"; + } + + @Override + protected String extractId(StsOperationEntity entity) { + return entity.getImolRorIHSNumber(); + } + + @Override + public String getInsertSql() { + return """ + INSERT INTO snp_data.t_stsoperation( + imo, + mvmn_type, + mvmn_dt, + fclty_id, + fclty_nm, + fclty_type, + up_fclty_id, + up_fclty_nm, + up_fclty_type, + draft, + lat, + lon, + prnt_call_id, + ntn_cd, + ntn_nm, + sts_location, + sts_type, + evt_start_dt, + lcinfo + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT (imo, mvmn_dt) + DO UPDATE SET + mvmn_type = EXCLUDED.mvmn_type, + mvmn_dt = EXCLUDED.mvmn_dt, + fclty_id = EXCLUDED.fclty_id, + fclty_nm = EXCLUDED.fclty_nm, + fclty_type = EXCLUDED.fclty_type, + up_fclty_id = EXCLUDED.up_fclty_id, + up_fclty_nm = EXCLUDED.up_fclty_nm, + up_fclty_type = EXCLUDED.up_fclty_type, + draft = EXCLUDED.draft, + lat = EXCLUDED.lat, + lon = EXCLUDED.lon, + prnt_call_id = EXCLUDED.prnt_call_id, + ntn_cd = EXCLUDED.ntn_cd, + ntn_nm = EXCLUDED.ntn_nm, + sts_location = EXCLUDED.sts_location, + sts_type = EXCLUDED.sts_type, + evt_start_dt = EXCLUDED.evt_start_dt, + lcinfo = EXCLUDED.lcinfo + """; + } + + @Override + protected String getUpdateSql() { + return null; + } + + @Override + protected void setInsertParameters(PreparedStatement ps, StsOperationEntity e) throws Exception { + int i = 1; + ps.setString(i++, safeString(e.getImolRorIHSNumber())); // imo + ps.setString(i++, safeString(e.getMovementType())); // mvmn_type + ps.setTimestamp(i++, e.getMovementDate() != null ? Timestamp.valueOf(e.getMovementDate()) : null); // mvmn_dt + ps.setObject(i++, e.getFacilityId()); // fclty_id + ps.setString(i++, safeString(e.getFacilityName())); // fclty_nm + ps.setString(i++, safeString(e.getFacilityType())); // fclty_type + ps.setObject(i++, e.getParentFacilityId()); //up_fclty_id + ps.setString(i++, safeString(e.getParentFacilityName())); // up_fclty_nm + ps.setString(i++, safeString(e.getParentFacilityType())); //up_fclty_type + setDoubleOrNull(ps, i++, e.getDraught()); // draft + setDoubleOrNull(ps, i++, e.getLatitude()); // lat + setDoubleOrNull(ps, i++, e.getLongitude());// lon + ps.setObject(i++, e.getParentCallId()); //prnt_call_id + ps.setString(i++, safeString(e.getCountryCode())); // ntn_cd + ps.setString(i++, safeString(e.getCountryName())); // ntn_nm + ps.setString(i++, safeString(e.getStsLocation())); // iso2_ntn_cd + ps.setString(i++, safeString(e.getStsType())); + ps.setTimestamp(i++, e.getEventStartDate() != null ? Timestamp.valueOf(e.getEventStartDate()) : null); // evt_start_dt + + if (e.getPosition() != null) { + ps.setObject(i++, OBJECT_MAPPER.writeValueAsString(e.getPosition()), java.sql.Types.OTHER); // lcinfo (jsonb) + } else { + ps.setNull(i++, java.sql.Types.OTHER); + } + } + + private void setDoubleOrNull(PreparedStatement ps, int index, Double value) throws Exception { + if (value != null) { + ps.setDouble(index, value); + } else { + // java.sql.Types.DOUBLE을 사용하여 명시적으로 SQL NULL을 설정 + ps.setNull(index, java.sql.Types.DOUBLE); + } + } + + @Override + protected void setUpdateParameters(PreparedStatement ps, StsOperationEntity entity) throws Exception { + + } + + @Override + protected RowMapper getRowMapper() { + return null; + } + + @Override + public void saveAll(List entities) { + if (entities == null || entities.isEmpty()) return; + + log.info("StsOperation 저장 시작 = {}건", entities.size()); + batchInsert(entities); + + } + private String safeString(String v) { + if (v == null) return null; + + v = v.trim(); + + return v.isEmpty() ? null : v; + } + +} diff --git a/src/main/java/com/snp/batch/jobs/shipMovementStsOperations/batch/writer/StsOperationWriter.java b/src/main/java/com/snp/batch/jobs/shipMovementStsOperations/batch/writer/StsOperationWriter.java new file mode 100644 index 0000000..44c5536 --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/shipMovementStsOperations/batch/writer/StsOperationWriter.java @@ -0,0 +1,36 @@ +package com.snp.batch.jobs.shipMovementStsOperations.batch.writer; + +import com.snp.batch.common.batch.writer.BaseWriter; +import com.snp.batch.jobs.shipMovementStsOperations.batch.entity.StsOperationEntity; +import com.snp.batch.jobs.shipMovementStsOperations.batch.repository.StsOperationRepository; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; + +import java.util.List; + +/** + * 선박 상세 정보 Writer + */ +@Slf4j +@Component +public class StsOperationWriter extends BaseWriter { + + private final StsOperationRepository stsOperationRepository; + + + public StsOperationWriter(StsOperationRepository stsOperationRepository) { + super("StsOperation"); + this.stsOperationRepository = stsOperationRepository; + } + + @Override + protected void writeItems(List items) throws Exception { + + if (items.isEmpty()) { return; } + + stsOperationRepository.saveAll(items); + log.info("STS OPERATION 데이터 저장: {} 건", items.size()); + + } + +} diff --git a/src/main/java/com/snp/batch/jobs/shipMovementTerminalCalls/batch/config/TerminalCallsJobConfig.java b/src/main/java/com/snp/batch/jobs/shipMovementTerminalCalls/batch/config/TerminalCallsJobConfig.java new file mode 100644 index 0000000..a298f9e --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/shipMovementTerminalCalls/batch/config/TerminalCallsJobConfig.java @@ -0,0 +1,104 @@ +package com.snp.batch.jobs.shipMovementTerminalCalls.batch.config; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.snp.batch.common.batch.config.BaseJobConfig; +import com.snp.batch.jobs.shipMovementTerminalCalls.batch.dto.TerminalCallsDto; +import com.snp.batch.jobs.shipMovementTerminalCalls.batch.entity.TerminalCallsEntity; +import com.snp.batch.jobs.shipMovementTerminalCalls.batch.processor.TerminalCallsProcessor; +import com.snp.batch.jobs.shipMovementTerminalCalls.batch.reader.TerminalCallsReader; +import com.snp.batch.jobs.shipMovementTerminalCalls.batch.writer.TerminalCallsWriter; +import lombok.extern.slf4j.Slf4j; +import org.springframework.batch.core.Job; +import org.springframework.batch.core.Step; +import org.springframework.batch.core.repository.JobRepository; +import org.springframework.batch.item.ItemProcessor; +import org.springframework.batch.item.ItemReader; +import org.springframework.batch.item.ItemWriter; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.transaction.PlatformTransactionManager; +import org.springframework.web.reactive.function.client.WebClient; + +/** + * 선박 상세 정보 Import Job Config + * + * 특징: + * - ship_data 테이블에서 IMO 번호 조회 + * - IMO 번호를 100개씩 배치로 분할 + * - Maritime API GetShipsByIHSLRorIMONumbers 호출 + * TODO : GetShipsByIHSLRorIMONumbersAll 호출로 변경 + * - 선박 상세 정보를 ship_detail 테이블에 저장 (UPSERT) + * + * 데이터 흐름: + * ShipMovementReader (ship_data → Maritime API) + * ↓ (PortCallDto) + * ShipMovementProcessor + * ↓ (ShipMovementEntity) + * ShipDetailDataWriter + * ↓ (ship_movement 테이블) + */ + +@Slf4j +@Configuration +public class TerminalCallsJobConfig extends BaseJobConfig { + + private final TerminalCallsProcessor terminalCallsProcessor; + private final TerminalCallsWriter terminalCallsWriter; + private final JdbcTemplate jdbcTemplate; + private final WebClient maritimeApiWebClient; + + public TerminalCallsJobConfig( + JobRepository jobRepository, + PlatformTransactionManager transactionManager, + TerminalCallsProcessor terminalCallsProcessor, + TerminalCallsWriter terminalCallsWriter, JdbcTemplate jdbcTemplate, + @Qualifier("maritimeServiceApiWebClient") WebClient maritimeApiWebClient) { // ObjectMapper 주입 추가 + super(jobRepository, transactionManager); + this.terminalCallsProcessor = terminalCallsProcessor; + this.terminalCallsWriter = terminalCallsWriter; + this.jdbcTemplate = jdbcTemplate; + this.maritimeApiWebClient = maritimeApiWebClient; + } + + @Override + protected String getJobName() { + return "TerminalCallsImportJob"; + } + + @Override + protected String getStepName() { + return "TerminalCallImportStep"; + } + + @Override + protected ItemReader createReader() { // 타입 변경 + return new TerminalCallsReader(maritimeApiWebClient, jdbcTemplate); + } + + @Override + protected ItemProcessor createProcessor() { + return terminalCallsProcessor; + } + + @Override + protected ItemWriter createWriter() { // 타입 변경 + return terminalCallsWriter; + } + + @Override + protected int getChunkSize() { + return 1000; // API에서 100개씩 가져오므로 chunk도 100으로 설정 + } + + @Bean(name = "TerminalCallsImportJob") + public Job terminalCallsImportJob() { + return job(); + } + + @Bean(name = "TerminalCallImportStep") + public Step terminalCallImportStep() { + return step(); + } +} diff --git a/src/main/java/com/snp/batch/jobs/shipMovementTerminalCalls/batch/dto/TerminalCallsDto.java b/src/main/java/com/snp/batch/jobs/shipMovementTerminalCalls/batch/dto/TerminalCallsDto.java new file mode 100644 index 0000000..d35a7d7 --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/shipMovementTerminalCalls/batch/dto/TerminalCallsDto.java @@ -0,0 +1,32 @@ +package com.snp.batch.jobs.shipMovementTerminalCalls.batch.dto; + +import lombok.Data; + +@Data +public class TerminalCallsDto { + private String movementType; + private String imolRorIHSNumber; + private String movementDate; + + + private Integer facilityId; + private String facilityName; + private String facilityType; + + private Integer parentFacilityId; + private String parentFacilityName; + private String parentFacilityType; + + private String countryCode; + private String countryName; + + private Double draught; + private Double latitude; + private Double longitude; + + private TerminalCallsPositionDto position; + + private Integer parentCallId; + private String iso2; + private String eventStartDate; +} diff --git a/src/main/java/com/snp/batch/jobs/shipMovementTerminalCalls/batch/dto/TerminalCallsPositionDto.java b/src/main/java/com/snp/batch/jobs/shipMovementTerminalCalls/batch/dto/TerminalCallsPositionDto.java new file mode 100644 index 0000000..844f8bb --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/shipMovementTerminalCalls/batch/dto/TerminalCallsPositionDto.java @@ -0,0 +1,17 @@ +package com.snp.batch.jobs.shipMovementTerminalCalls.batch.dto; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; + +@Data +public class TerminalCallsPositionDto { + private boolean isNull; + private int stSrid; + private double lat; + @JsonProperty("long") + private double lon; + private double z; + private double m; + private boolean hasZ; + private boolean hasM; +} diff --git a/src/main/java/com/snp/batch/jobs/shipMovementTerminalCalls/batch/entity/TerminalCallsEntity.java b/src/main/java/com/snp/batch/jobs/shipMovementTerminalCalls/batch/entity/TerminalCallsEntity.java new file mode 100644 index 0000000..a003375 --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/shipMovementTerminalCalls/batch/entity/TerminalCallsEntity.java @@ -0,0 +1,43 @@ +package com.snp.batch.jobs.shipMovementTerminalCalls.batch.entity; + +import com.fasterxml.jackson.databind.JsonNode; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +import java.time.LocalDateTime; + +@Data +@SuperBuilder +@NoArgsConstructor +@AllArgsConstructor +public class TerminalCallsEntity { + + private Long id; + + private String movementType; + private String imolRorIHSNumber; + private LocalDateTime movementDate; + + private Integer facilityId; + private String facilityName; + private String facilityType; + + private Integer parentFacilityId; + private String parentFacilityName; + private String parentFacilityType; + + private String countryCode; + private String countryName; + + private Double draught; + private Double latitude; + private Double longitude; + + private JsonNode position; + + private Integer parentCallId; + private String iso2; + private LocalDateTime eventStartDate; +} diff --git a/src/main/java/com/snp/batch/jobs/shipMovementTerminalCalls/batch/processor/TerminalCallsProcessor.java b/src/main/java/com/snp/batch/jobs/shipMovementTerminalCalls/batch/processor/TerminalCallsProcessor.java new file mode 100644 index 0000000..8438cc4 --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/shipMovementTerminalCalls/batch/processor/TerminalCallsProcessor.java @@ -0,0 +1,68 @@ +package com.snp.batch.jobs.shipMovementTerminalCalls.batch.processor; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.snp.batch.common.batch.processor.BaseProcessor; +import com.snp.batch.jobs.shipMovementTerminalCalls.batch.dto.TerminalCallsDto; +import com.snp.batch.jobs.shipMovementTerminalCalls.batch.entity.TerminalCallsEntity; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; + +import java.time.LocalDateTime; + +/** + * 선박 상세 정보 Processor + * ShipDetailDto → ShipDetailEntity 변환 + */ + +/** + * 선박 상세 정보 Processor (해시 비교 및 증분 데이터 추출) + * I: ShipDetailComparisonData (DB 해시 + API Map Data) + * O: ShipDetailUpdate (변경분) + */ +@Slf4j +@Component +public class TerminalCallsProcessor extends BaseProcessor { + + private final ObjectMapper objectMapper; + + public TerminalCallsProcessor(ObjectMapper objectMapper) { + this.objectMapper = objectMapper; + } + + @Override + protected TerminalCallsEntity processItem(TerminalCallsDto dto) throws Exception { + log.debug("선박 상세 정보 처리 시작: imoNumber={}, facilityName={}", + dto.getImolRorIHSNumber(), dto.getFacilityName()); + + JsonNode positionNode = null; + if (dto.getPosition() != null) { + // Position 객체를 JsonNode로 변환 + positionNode = objectMapper.valueToTree(dto.getPosition()); + } + + TerminalCallsEntity entity = TerminalCallsEntity.builder() + .movementType(dto.getMovementType()) + .imolRorIHSNumber(dto.getImolRorIHSNumber()) + .movementDate(LocalDateTime.parse(dto.getMovementDate())) + .facilityId(dto.getFacilityId()) + .facilityName(dto.getFacilityName()) + .facilityType(dto.getFacilityType()) + .parentFacilityId(dto.getParentFacilityId()) + .parentFacilityName(dto.getParentFacilityName()) + .parentFacilityType(dto.getParentFacilityType()) + .countryCode(dto.getCountryCode()) + .countryName(dto.getCountryName()) + .draught(dto.getDraught()) + .latitude(dto.getLatitude()) + .longitude(dto.getLongitude()) + .position(positionNode) // JsonNode로 매핑 + .parentCallId(dto.getParentCallId()) + .iso2(dto.getIso2()) + .eventStartDate(LocalDateTime.parse(dto.getEventStartDate())) + .build(); + + return entity; + } + +} diff --git a/src/main/java/com/snp/batch/jobs/shipMovementTerminalCalls/batch/reader/TerminalCallsReader.java b/src/main/java/com/snp/batch/jobs/shipMovementTerminalCalls/batch/reader/TerminalCallsReader.java new file mode 100644 index 0000000..9018360 --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/shipMovementTerminalCalls/batch/reader/TerminalCallsReader.java @@ -0,0 +1,213 @@ +package com.snp.batch.jobs.shipMovementTerminalCalls.batch.reader; + +import com.snp.batch.common.batch.reader.BaseApiReader; +import com.snp.batch.jobs.shipMovementTerminalCalls.batch.dto.TerminalCallsDto; +import lombok.extern.slf4j.Slf4j; +import org.springframework.batch.core.configuration.annotation.StepScope; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.web.reactive.function.client.WebClient; + +import java.util.Collections; +import java.util.List; +import java.util.Map; + +/** + * 선박 상세 정보 Reader (v2.0 - Chunk 기반) + * + * 기능: + * 1. ship_data 테이블에서 IMO 번호 전체 조회 (최초 1회) + * 2. IMO 번호를 100개씩 분할하여 배치 단위로 처리 + * 3. fetchNextBatch() 호출 시마다 100개씩 API 호출 + * 4. Spring Batch가 100건씩 Process → Write 수행 + * + * Chunk 처리 흐름: + * - beforeFetch() → IMO 전체 조회 (1회) + * - fetchNextBatch() → 100개 IMO로 API 호출 (1,718회) + * - read() → 1건씩 반환 (100번) + * - Processor/Writer → 100건 처리 + * - 반복... (1,718번의 Chunk) + * + * 기존 방식과의 차이: + * - 기존: 17만건 전체 메모리 로드 → Process → Write + * - 신규: 100건씩 로드 → Process → Write (Chunk 1,718회) + */ +@Slf4j +@StepScope +public class TerminalCallsReader extends BaseApiReader { + + private final JdbcTemplate jdbcTemplate; + + // 배치 처리 상태 + private List allImoNumbers; + // DB 해시값을 저장할 맵 + private Map dbMasterHashes; + private int currentBatchIndex = 0; + private final int batchSize = 5; + + // @Value("#{jobParameters['startDate']}") +// private String startDate; + private String startDate = "2025-01-01"; + + // @Value("#{jobParameters['stopDate']}") +// private String stopDate; + private String stopDate = "2025-12-31"; + + public TerminalCallsReader(WebClient webClient, JdbcTemplate jdbcTemplate ) { + super(webClient); + this.jdbcTemplate = jdbcTemplate; + enableChunkMode(); // ✨ Chunk 모드 활성화 + } + + @Override + protected String getReaderName() { + return "TerminalCalls"; + } + + @Override + protected void resetCustomState() { + this.currentBatchIndex = 0; + this.allImoNumbers = null; + this.dbMasterHashes = null; + } + + @Override + protected String getApiPath() { + return "/Movements/TerminalCalls"; + } + + @Override + protected String getApiBaseUrl() { + return "https://webservices.maritime.spglobal.com"; + } + + private static final String GET_ALL_IMO_QUERY = +// "SELECT imo_number FROM ship_data ORDER BY id"; + "SELECT imo_number FROM snp_data.ship_data where imo_number > (select max(imo) from snp_data.t_terminalcall) ORDER BY imo_number"; + + /** + * 최초 1회만 실행: ship_data 테이블에서 IMO 번호 전체 조회 + */ + @Override + protected void beforeFetch() { + // 전처리 과정 + // Step 1. IMO 전체 번호 조회 + log.info("[{}] ship_data 테이블에서 IMO 번호 조회 시작...", getReaderName()); + + allImoNumbers = jdbcTemplate.queryForList(GET_ALL_IMO_QUERY, String.class); + int totalBatches = (int) Math.ceil((double) allImoNumbers.size() / batchSize); + + log.info("[{}] 총 {} 개의 IMO 번호 조회 완료", getReaderName(), allImoNumbers.size()); + log.info("[{}] {}개씩 배치로 분할하여 API 호출 예정", getReaderName(), batchSize); + log.info("[{}] 예상 배치 수: {} 개", getReaderName(), totalBatches); + + // API 통계 초기화 + updateApiCallStats(totalBatches, 0); + } + + /** + * ✨ Chunk 기반 핵심 메서드: 다음 100개 배치를 조회하여 반환 + * + * Spring Batch가 100건씩 read() 호출 완료 후 이 메서드 재호출 + * + * @return 다음 배치 100건 (더 이상 없으면 null) + */ + @Override + protected List fetchNextBatch() throws Exception { + + // 모든 배치 처리 완료 확인 + if (allImoNumbers == null || currentBatchIndex >= allImoNumbers.size()) { + return null; // Job 종료 + } + + // 현재 배치의 시작/끝 인덱스 계산 + int startIndex = currentBatchIndex; + int endIndex = Math.min(currentBatchIndex + batchSize, allImoNumbers.size()); + + // 현재 배치의 IMO 번호 추출 (100개) + List currentBatch = allImoNumbers.subList(startIndex, endIndex); + + int currentBatchNumber = (currentBatchIndex / batchSize) + 1; + int totalBatches = (int) Math.ceil((double) allImoNumbers.size() / batchSize); + + log.info("[{}] 배치 {}/{} 처리 중 (IMO {} 개)...", + getReaderName(), currentBatchNumber, totalBatches, currentBatch.size()); + + try { + // IMO 번호를 쉼표로 연결 (예: "1000019,1000021,1000033,...") + String imoParam = String.join(",", currentBatch); + + // API 호출 + List response = callApiWithBatch(imoParam); + + // 다음 배치로 인덱스 이동 + currentBatchIndex = endIndex; + + + // 응답 처리 + if (response != null ) { + List terminalCalls = response; + log.info("[{}] 배치 {}/{} 완료: {} 건 조회", + getReaderName(), currentBatchNumber, totalBatches, terminalCalls.size()); + + // API 호출 통계 업데이트 + updateApiCallStats(totalBatches, currentBatchNumber); + + // API 과부하 방지 (다음 배치 전 0.5초 대기) + if (currentBatchIndex < allImoNumbers.size()) { + Thread.sleep(500); + } + + return terminalCalls; + + } else { + log.warn("[{}] 배치 {}/{} 응답 없음", + getReaderName(), currentBatchNumber, totalBatches); + + // API 호출 통계 업데이트 (실패도 카운트) + updateApiCallStats(totalBatches, currentBatchNumber); + + return Collections.emptyList(); + } + + } catch (Exception e) { + log.error("[{}] 배치 {}/{} 처리 중 오류: {}", + getReaderName(), currentBatchNumber, totalBatches, e.getMessage(), e); + + // 오류 발생 시에도 다음 배치로 이동 (부분 실패 허용) + currentBatchIndex = endIndex; + + // 빈 리스트 반환 (Job 계속 진행) + return Collections.emptyList(); + } + } + + /** + * Query Parameter를 사용한 API 호출 + * + * @param lrno 쉼표로 연결된 IMO 번호 (예: "1000019,1000021,...") + * @return API 응답 + */ + private List callApiWithBatch(String lrno) { + String url = getApiPath() + "?startDate=" + startDate +"&stopDate="+stopDate+"&lrno=" + lrno; + + log.debug("[{}] API 호출: {}", getReaderName(), url); + + return webClient.get() + .uri(url) + .retrieve() + .bodyToFlux(TerminalCallsDto.class) + .collectList() + .block(); + } + + @Override + protected void afterFetch(List data) { + if (data == null) { + int totalBatches = (int) Math.ceil((double) allImoNumbers.size() / batchSize); + log.info("[{}] 전체 {} 개 배치 처리 완료", getReaderName(), totalBatches); + log.info("[{}] 총 {} 개의 IMO 번호에 대한 API 호출 종료", + getReaderName(), allImoNumbers.size()); + } + } + +} diff --git a/src/main/java/com/snp/batch/jobs/shipMovementTerminalCalls/batch/repository/TerminalCallsRepository.java b/src/main/java/com/snp/batch/jobs/shipMovementTerminalCalls/batch/repository/TerminalCallsRepository.java new file mode 100644 index 0000000..6b22b39 --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/shipMovementTerminalCalls/batch/repository/TerminalCallsRepository.java @@ -0,0 +1,13 @@ +package com.snp.batch.jobs.shipMovementTerminalCalls.batch.repository; + +import com.snp.batch.jobs.shipMovementTerminalCalls.batch.entity.TerminalCallsEntity; + +import java.util.List; + +/** + * 선박 상세 정보 Repository 인터페이스 + */ + +public interface TerminalCallsRepository { + void saveAll(List entities); +} diff --git a/src/main/java/com/snp/batch/jobs/shipMovementTerminalCalls/batch/repository/TerminalCallsRepositoryImpl.java b/src/main/java/com/snp/batch/jobs/shipMovementTerminalCalls/batch/repository/TerminalCallsRepositoryImpl.java new file mode 100644 index 0000000..53b60ee --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/shipMovementTerminalCalls/batch/repository/TerminalCallsRepositoryImpl.java @@ -0,0 +1,152 @@ +package com.snp.batch.jobs.shipMovementTerminalCalls.batch.repository; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.snp.batch.common.batch.repository.BaseJdbcRepository; +import com.snp.batch.jobs.shipMovementTerminalCalls.batch.entity.TerminalCallsEntity; +import lombok.extern.slf4j.Slf4j; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.core.RowMapper; +import org.springframework.stereotype.Repository; + +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Timestamp; +import java.util.List; + +/** + * 선박 상세 정보 Repository 구현체 + * BaseJdbcRepository를 상속하여 JDBC 기반 CRUD 구현 + */ +@Slf4j +@Repository("TerminalCallsRepository") +public class TerminalCallsRepositoryImpl extends BaseJdbcRepository + implements TerminalCallsRepository { + + public TerminalCallsRepositoryImpl(JdbcTemplate jdbcTemplate) { + super(jdbcTemplate); + } + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + @Override + protected String getTableName() { + return "snp_data.t_terminalcall"; + } + + @Override + protected String getEntityName() { + return "TerminallCalls"; + } + + @Override + protected String extractId(TerminalCallsEntity entity) { + return entity.getImolRorIHSNumber(); + } + + @Override + public String getInsertSql() { + return """ + INSERT INTO snp_data.t_terminalcall( + imo, + mvmn_type, + mvmn_dt, + fclty_id, + fclty_nm, + fclty_type, + up_fclty_id, + up_fclty_nm, + up_fclty_type, + ntn_cd, + ntn_nm, + draft, + lat, + lon, + prnt_call_id, + iso2_ntn_cd, + evt_start_dt, + lcinfo + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT (imo, mvmn_dt) + DO UPDATE SET + mvmn_type = EXCLUDED.mvmn_type, + mvmn_dt = EXCLUDED.mvmn_dt, + fclty_id = EXCLUDED.fclty_id, + fclty_nm = EXCLUDED.fclty_nm, + fclty_type = EXCLUDED.fclty_type, + up_fclty_id = EXCLUDED.up_fclty_id, + up_fclty_nm = EXCLUDED.up_fclty_nm, + up_fclty_type = EXCLUDED.up_fclty_type, + ntn_cd = EXCLUDED.ntn_cd, + ntn_nm = EXCLUDED.ntn_nm, + draft = EXCLUDED.draft, + lat = EXCLUDED.lat, + lon = EXCLUDED.lon, + prnt_call_id = EXCLUDED.prnt_call_id, + iso2_ntn_cd = EXCLUDED.iso2_ntn_cd, + evt_start_dt = EXCLUDED.evt_start_dt, + lcinfo = EXCLUDED.lcinfo + """; + } + + @Override + protected String getUpdateSql() { + return null; + } + + @Override + protected void setInsertParameters(PreparedStatement ps, TerminalCallsEntity e) throws Exception { + int i = 1; + ps.setString(i++, e.getImolRorIHSNumber()); // imo + ps.setString(i++, e.getMovementType()); // mvmn_type + ps.setTimestamp(i++, e.getMovementDate() != null ? Timestamp.valueOf(e.getMovementDate()) : null); // mvmn_dt + ps.setObject(i++, e.getFacilityId()); // fclty_id + ps.setString(i++, e.getFacilityName()); // fclty_nm + ps.setString(i++, e.getFacilityType()); // fclty_type + ps.setObject(i++, e.getParentFacilityId()); //up_fclty_id + ps.setString(i++, e.getParentFacilityName()); // up_fclty_nm + ps.setString(i++, e.getParentFacilityType()); //up_fclty_type + ps.setString(i++, e.getCountryCode()); // ntn_cd + ps.setString(i++, e.getCountryName()); // ntn_nm + setDoubleOrNull(ps, i++, e.getDraught()); // draft + setDoubleOrNull(ps, i++, e.getLatitude()); // lat + setDoubleOrNull(ps, i++, e.getLongitude());// lon + ps.setObject(i++, e.getParentCallId()); //prnt_call_id + ps.setString(i++, e.getIso2()); // iso2_ntn_cd + ps.setTimestamp(i++, e.getEventStartDate() != null ? Timestamp.valueOf(e.getEventStartDate()) : null); // evt_start_dt + + if (e.getPosition() != null) { + ps.setObject(i++, OBJECT_MAPPER.writeValueAsString(e.getPosition()), java.sql.Types.OTHER); // lcinfo (jsonb) + } else { + ps.setNull(i++, java.sql.Types.OTHER); + } + } + + private void setDoubleOrNull(PreparedStatement ps, int index, Double value) throws Exception { + if (value != null) { + ps.setDouble(index, value); + } else { + // java.sql.Types.DOUBLE을 사용하여 명시적으로 SQL NULL을 설정 + ps.setNull(index, java.sql.Types.DOUBLE); + } + } + + @Override + protected void setUpdateParameters(PreparedStatement ps, TerminalCallsEntity entity) throws Exception { + + } + + @Override + protected RowMapper getRowMapper() { + return null; + } + + @Override + public void saveAll(List entities) { + if (entities == null || entities.isEmpty()) return; + + log.info("TerminallCalls 저장 시작 = {}건", entities.size()); + batchInsert(entities); + + } + +} diff --git a/src/main/java/com/snp/batch/jobs/shipMovementTerminalCalls/batch/writer/TerminalCallsWriter.java b/src/main/java/com/snp/batch/jobs/shipMovementTerminalCalls/batch/writer/TerminalCallsWriter.java new file mode 100644 index 0000000..c5d1a2a --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/shipMovementTerminalCalls/batch/writer/TerminalCallsWriter.java @@ -0,0 +1,35 @@ +package com.snp.batch.jobs.shipMovementTerminalCalls.batch.writer; + +import com.snp.batch.common.batch.writer.BaseWriter; +import com.snp.batch.jobs.shipMovementTerminalCalls.batch.entity.TerminalCallsEntity; +import com.snp.batch.jobs.shipMovementTerminalCalls.batch.repository.TerminalCallsRepository; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; + +import java.util.List; + +/** + * 선박 상세 정보 Writer + */ +@Slf4j +@Component +public class TerminalCallsWriter extends BaseWriter { + + private final TerminalCallsRepository terminalCallsRepository; + + + public TerminalCallsWriter(TerminalCallsRepository terminalCallsRepository) { + super("TerminalCalls"); + this.terminalCallsRepository = terminalCallsRepository; + } + + @Override + protected void writeItems(List items) throws Exception { + + if (items.isEmpty()) { return; } + + terminalCallsRepository.saveAll(items); + log.info("TerminalCalls 데이터 저장: {} 건", items.size()); + } + +} From 6c98ebc24f5beea4ef1486c4f92ed687a57b38dc Mon Sep 17 00:00:00 2001 From: Kim JiMyeung Date: Mon, 8 Dec 2025 17:47:30 +0900 Subject: [PATCH 3/9] =?UTF-8?q?Destination,=20Transits,=20CurrentlyAt=20?= =?UTF-8?q?=EC=A6=9D=EB=B6=84Job?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../batch/config/CurrentlyAtJobConfig.java | 103 +++++++++ .../batch/dto/CurrentlyAtDto.java | 37 +++ .../batch/dto/CurrentlyAtPositionDto.java | 17 ++ .../batch/entity/CurrentlyAtEntity.java | 41 ++++ .../batch/processor/CurrentlyAtProcessor.java | 71 ++++++ .../batch/reader/CurrentlyAtReader.java | 211 ++++++++++++++++++ .../repository/CurrentlyAtRepository.java | 13 ++ .../repository/CurrentlyAtRepositoryImpl.java | 211 ++++++++++++++++++ .../batch/writer/CurrentlyAtWriter.java | 36 +++ .../batch/config/ShipMovementJobConfig.java | 8 +- .../{PortCallDto.java => PortCallsDto.java} | 4 +- ...tionDto.java => PortCallsPositionDto.java} | 2 +- .../batch/dto/ShipMovementApiResponse.java | 2 +- .../processor/ShipMovementProcessor.java | 6 +- .../batch/reader/ShipMovementReader.java | 16 +- .../batch/config/AnchorageCallsJobConfig.java | 23 +- .../batch/dto/BerthCallsDto.java | 1 - .../batch/config/DarkActivityJobConfig.java | 12 +- .../batch/config/DestinationsJobConfig.java | 103 +++++++++ .../batch/dto/DestinationDto.java | 24 ++ .../batch/dto/DestinationPositionDto.java | 17 ++ .../batch/entity/DestinationEntity.java | 32 +++ .../batch/processor/DestinationProcessor.java | 61 +++++ .../batch/reader/DestinationReader.java | 211 ++++++++++++++++++ .../repository/DestinationRepository.java | 14 ++ .../repository/DestinationRepositoryImpl.java | 131 +++++++++++ .../batch/writer/DestinationWriter.java | 36 +++ .../batch/config/StsOperationJobConfig.java | 18 +- .../batch/dto/StsOperationDto.java | 1 - .../batch/config/TerminalCallsJobConfig.java | 13 +- .../batch/config/TransitsJobConfig.java | 103 +++++++++ .../batch/dto/TransitsDto.java | 13 ++ .../batch/entity/TransitsEntity.java | 21 ++ .../batch/processor/TransitsProcessor.java | 47 ++++ .../batch/reader/TransitsReader.java | 211 ++++++++++++++++++ .../repository/TransitlsRepositoryImpl.java | 108 +++++++++ .../batch/repository/TransitsRepository.java | 13 ++ .../batch/writer/TransitsWriter.java | 35 +++ 38 files changed, 1960 insertions(+), 66 deletions(-) create mode 100644 src/main/java/com/snp/batch/jobs/shipCurrentlyAt/batch/config/CurrentlyAtJobConfig.java create mode 100644 src/main/java/com/snp/batch/jobs/shipCurrentlyAt/batch/dto/CurrentlyAtDto.java create mode 100644 src/main/java/com/snp/batch/jobs/shipCurrentlyAt/batch/dto/CurrentlyAtPositionDto.java create mode 100644 src/main/java/com/snp/batch/jobs/shipCurrentlyAt/batch/entity/CurrentlyAtEntity.java create mode 100644 src/main/java/com/snp/batch/jobs/shipCurrentlyAt/batch/processor/CurrentlyAtProcessor.java create mode 100644 src/main/java/com/snp/batch/jobs/shipCurrentlyAt/batch/reader/CurrentlyAtReader.java create mode 100644 src/main/java/com/snp/batch/jobs/shipCurrentlyAt/batch/repository/CurrentlyAtRepository.java create mode 100644 src/main/java/com/snp/batch/jobs/shipCurrentlyAt/batch/repository/CurrentlyAtRepositoryImpl.java create mode 100644 src/main/java/com/snp/batch/jobs/shipCurrentlyAt/batch/writer/CurrentlyAtWriter.java rename src/main/java/com/snp/batch/jobs/shipMovement/batch/dto/{PortCallDto.java => PortCallsDto.java} (91%) rename src/main/java/com/snp/batch/jobs/shipMovement/batch/dto/{PositionDto.java => PortCallsPositionDto.java} (90%) create mode 100644 src/main/java/com/snp/batch/jobs/shipMovementDestination/batch/config/DestinationsJobConfig.java create mode 100644 src/main/java/com/snp/batch/jobs/shipMovementDestination/batch/dto/DestinationDto.java create mode 100644 src/main/java/com/snp/batch/jobs/shipMovementDestination/batch/dto/DestinationPositionDto.java create mode 100644 src/main/java/com/snp/batch/jobs/shipMovementDestination/batch/entity/DestinationEntity.java create mode 100644 src/main/java/com/snp/batch/jobs/shipMovementDestination/batch/processor/DestinationProcessor.java create mode 100644 src/main/java/com/snp/batch/jobs/shipMovementDestination/batch/reader/DestinationReader.java create mode 100644 src/main/java/com/snp/batch/jobs/shipMovementDestination/batch/repository/DestinationRepository.java create mode 100644 src/main/java/com/snp/batch/jobs/shipMovementDestination/batch/repository/DestinationRepositoryImpl.java create mode 100644 src/main/java/com/snp/batch/jobs/shipMovementDestination/batch/writer/DestinationWriter.java create mode 100644 src/main/java/com/snp/batch/jobs/shipMovementTransits/batch/config/TransitsJobConfig.java create mode 100644 src/main/java/com/snp/batch/jobs/shipMovementTransits/batch/dto/TransitsDto.java create mode 100644 src/main/java/com/snp/batch/jobs/shipMovementTransits/batch/entity/TransitsEntity.java create mode 100644 src/main/java/com/snp/batch/jobs/shipMovementTransits/batch/processor/TransitsProcessor.java create mode 100644 src/main/java/com/snp/batch/jobs/shipMovementTransits/batch/reader/TransitsReader.java create mode 100644 src/main/java/com/snp/batch/jobs/shipMovementTransits/batch/repository/TransitlsRepositoryImpl.java create mode 100644 src/main/java/com/snp/batch/jobs/shipMovementTransits/batch/repository/TransitsRepository.java create mode 100644 src/main/java/com/snp/batch/jobs/shipMovementTransits/batch/writer/TransitsWriter.java diff --git a/src/main/java/com/snp/batch/jobs/shipCurrentlyAt/batch/config/CurrentlyAtJobConfig.java b/src/main/java/com/snp/batch/jobs/shipCurrentlyAt/batch/config/CurrentlyAtJobConfig.java new file mode 100644 index 0000000..b5a8909 --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/shipCurrentlyAt/batch/config/CurrentlyAtJobConfig.java @@ -0,0 +1,103 @@ +package com.snp.batch.jobs.shipCurrentlyAt.batch.config; + +import com.snp.batch.common.batch.config.BaseJobConfig; +import com.snp.batch.jobs.shipCurrentlyAt.batch.dto.CurrentlyAtDto; +import com.snp.batch.jobs.shipCurrentlyAt.batch.entity.CurrentlyAtEntity; +import com.snp.batch.jobs.shipCurrentlyAt.batch.processor.CurrentlyAtProcessor; +import com.snp.batch.jobs.shipCurrentlyAt.batch.reader.CurrentlyAtReader; +import com.snp.batch.jobs.shipCurrentlyAt.batch.writer.CurrentlyAtWriter; +import lombok.extern.slf4j.Slf4j; +import org.springframework.batch.core.Job; +import org.springframework.batch.core.Step; +import org.springframework.batch.core.repository.JobRepository; +import org.springframework.batch.item.ItemProcessor; +import org.springframework.batch.item.ItemReader; +import org.springframework.batch.item.ItemWriter; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.transaction.PlatformTransactionManager; +import org.springframework.web.reactive.function.client.WebClient; + +/** + * 선박 상세 정보 Import Job Config + * + * 특징: + * - ship_data 테이블에서 IMO 번호 조회 + * - IMO 번호를 100개씩 배치로 분할 + * - Maritime API GetShipsByIHSLRorIMONumbers 호출 + * TODO : GetShipsByIHSLRorIMONumbersAll 호출로 변경 + * - 선박 상세 정보를 ship_detail 테이블에 저장 (UPSERT) + * + * 데이터 흐름: + * CurrentlyAtReader (ship_data → Maritime API) + * ↓ (CurrentlyAtDto) + * CurrentlyAtProcessor + * ↓ (CurrentlyAtEntity) + * CurrentlyAtWriter + * ↓ (currentlyat 테이블) + */ + +@Slf4j +@Configuration +public class CurrentlyAtJobConfig extends BaseJobConfig { + + private final CurrentlyAtProcessor currentlyAtProcessor; + private final CurrentlyAtWriter currentlyAtWriter; + private final JdbcTemplate jdbcTemplate; + private final WebClient maritimeApiWebClient; + + public CurrentlyAtJobConfig( + JobRepository jobRepository, + PlatformTransactionManager transactionManager, + CurrentlyAtProcessor currentlyAtProcessor, + CurrentlyAtWriter currentlyAtWriter, JdbcTemplate jdbcTemplate, + @Qualifier("maritimeServiceApiWebClient") WebClient maritimeApiWebClient) { // ObjectMapper 주입 추가 + super(jobRepository, transactionManager); + this.currentlyAtProcessor = currentlyAtProcessor; + this.currentlyAtWriter = currentlyAtWriter; + this.jdbcTemplate = jdbcTemplate; + this.maritimeApiWebClient = maritimeApiWebClient; + } + + @Override + protected String getJobName() { + return "CurrentlyAtImportJob"; + } + + @Override + protected String getStepName() { + return "CurrentlyAtImportStep"; + } + + @Override + protected ItemReader createReader() { // 타입 변경 + return new CurrentlyAtReader(maritimeApiWebClient, jdbcTemplate); + } + + @Override + protected ItemProcessor createProcessor() { + return currentlyAtProcessor; + } + + @Override + protected ItemWriter createWriter() { // 타입 변경 + return currentlyAtWriter; + } + + @Override + protected int getChunkSize() { + return 50; // API에서 100개씩 가져오므로 chunk도 100으로 설정 + } + + @Bean(name = "CurrentlyAtImportJob") + public Job currentlyAtImportJob() { + return job(); + } + + @Bean(name = "CurrentlyAtImportStep") + public Step currentlyAtImportStep() { + return step(); + } +} diff --git a/src/main/java/com/snp/batch/jobs/shipCurrentlyAt/batch/dto/CurrentlyAtDto.java b/src/main/java/com/snp/batch/jobs/shipCurrentlyAt/batch/dto/CurrentlyAtDto.java new file mode 100644 index 0000000..2f1201e --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/shipCurrentlyAt/batch/dto/CurrentlyAtDto.java @@ -0,0 +1,37 @@ +package com.snp.batch.jobs.shipCurrentlyAt.batch.dto; + +import com.snp.batch.jobs.shipMovement.batch.dto.PortCallsPositionDto; +import lombok.Data; + +@Data +public class CurrentlyAtDto { + private String movementType; + private String imolRorIHSNumber; + private String movementDate; + + private Integer portCallId; + + private Integer facilityId; + private String facilityName; + private String facilityType; + + private Integer subFacilityId; + private String subFacilityName; + private String subFacilityType; + + private Integer parentFacilityId; + private String parentFacilityName; + private String parentFacilityType; + + private String countryCode; + private String countryName; + + private Double draught; + private Double latitude; + private Double longitude; + + private PortCallsPositionDto position; + + private String destination; + private String iso2; +} diff --git a/src/main/java/com/snp/batch/jobs/shipCurrentlyAt/batch/dto/CurrentlyAtPositionDto.java b/src/main/java/com/snp/batch/jobs/shipCurrentlyAt/batch/dto/CurrentlyAtPositionDto.java new file mode 100644 index 0000000..3f0df2a --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/shipCurrentlyAt/batch/dto/CurrentlyAtPositionDto.java @@ -0,0 +1,17 @@ +package com.snp.batch.jobs.shipCurrentlyAt.batch.dto; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; + +@Data +public class CurrentlyAtPositionDto { + private boolean isNull; + private int stSrid; + private double lat; + @JsonProperty("long") + private double lon; + private double z; + private double m; + private boolean hasZ; + private boolean hasM; +} diff --git a/src/main/java/com/snp/batch/jobs/shipCurrentlyAt/batch/entity/CurrentlyAtEntity.java b/src/main/java/com/snp/batch/jobs/shipCurrentlyAt/batch/entity/CurrentlyAtEntity.java new file mode 100644 index 0000000..c935894 --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/shipCurrentlyAt/batch/entity/CurrentlyAtEntity.java @@ -0,0 +1,41 @@ +package com.snp.batch.jobs.shipCurrentlyAt.batch.entity; + +import com.fasterxml.jackson.databind.JsonNode; +import jakarta.persistence.GeneratedValue; +import jakarta.persistence.GenerationType; +import jakarta.persistence.Id; +import jakarta.persistence.SequenceGenerator; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +import java.time.LocalDateTime; + +@Data +@SuperBuilder +@NoArgsConstructor +@AllArgsConstructor +public class CurrentlyAtEntity { + private String movementType; + private String imolRorIHSNumber; + private LocalDateTime movementDate; + private Integer portCallId; + private Integer facilityId; + private String facilityName; + private String facilityType; + private Integer subFacilityId; + private String subFacilityName; + private String subFacilityType; + private Integer parentFacilityId; + private String parentFacilityName; + private String parentFacilityType; + private String countryCode; + private String countryName; + private Double draught; + private Double latitude; + private Double longitude; + private String destination; + private String iso2; + private JsonNode position; +} diff --git a/src/main/java/com/snp/batch/jobs/shipCurrentlyAt/batch/processor/CurrentlyAtProcessor.java b/src/main/java/com/snp/batch/jobs/shipCurrentlyAt/batch/processor/CurrentlyAtProcessor.java new file mode 100644 index 0000000..64256e7 --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/shipCurrentlyAt/batch/processor/CurrentlyAtProcessor.java @@ -0,0 +1,71 @@ +package com.snp.batch.jobs.shipCurrentlyAt.batch.processor; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.snp.batch.common.batch.processor.BaseProcessor; +import com.snp.batch.jobs.shipCurrentlyAt.batch.dto.CurrentlyAtDto; +import com.snp.batch.jobs.shipCurrentlyAt.batch.entity.CurrentlyAtEntity; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; + +import java.time.LocalDateTime; + +/** + * 선박 상세 정보 Processor + * ShipDetailDto → ShipDetailEntity 변환 + */ + +/** + * 선박 상세 정보 Processor (해시 비교 및 증분 데이터 추출) + * I: ShipDetailComparisonData (DB 해시 + API Map Data) + * O: ShipDetailUpdate (변경분) + */ +@Slf4j +@Component +public class CurrentlyAtProcessor extends BaseProcessor { + + private final ObjectMapper objectMapper; + + public CurrentlyAtProcessor(ObjectMapper objectMapper) { + this.objectMapper = objectMapper; + } + + @Override + protected CurrentlyAtEntity processItem(CurrentlyAtDto dto) throws Exception { + log.debug("Currently 정보 처리 시작: imoNumber={}, facilityName={}", + dto.getImolRorIHSNumber(), dto.getFacilityName()); + + JsonNode positionNode = null; + if (dto.getPosition() != null) { + // Position 객체를 JsonNode로 변환 + positionNode = objectMapper.valueToTree(dto.getPosition()); + } + + CurrentlyAtEntity entity = CurrentlyAtEntity.builder() + .movementType(dto.getMovementType()) + .imolRorIHSNumber(dto.getImolRorIHSNumber()) + .movementDate(LocalDateTime.parse(dto.getMovementDate())) + .portCallId(dto.getPortCallId()) + .facilityId(dto.getFacilityId()) + .facilityName(dto.getFacilityName()) + .facilityType(dto.getFacilityType()) + .subFacilityId(dto.getSubFacilityId()) + .subFacilityName(dto.getSubFacilityName()) + .subFacilityType(dto.getSubFacilityType()) + .parentFacilityId(dto.getParentFacilityId()) + .parentFacilityName(dto.getParentFacilityName()) + .parentFacilityType(dto.getParentFacilityType()) + .countryCode(dto.getCountryCode()) + .countryName(dto.getCountryName()) + .draught(dto.getDraught()) + .latitude(dto.getLatitude()) + .longitude(dto.getLongitude()) + .destination(dto.getDestination()) + .iso2(dto.getIso2()) + .position(positionNode) // JsonNode로 매핑 + .build(); + + return entity; + } + +} diff --git a/src/main/java/com/snp/batch/jobs/shipCurrentlyAt/batch/reader/CurrentlyAtReader.java b/src/main/java/com/snp/batch/jobs/shipCurrentlyAt/batch/reader/CurrentlyAtReader.java new file mode 100644 index 0000000..e25bf41 --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/shipCurrentlyAt/batch/reader/CurrentlyAtReader.java @@ -0,0 +1,211 @@ +package com.snp.batch.jobs.shipCurrentlyAt.batch.reader; + +import com.snp.batch.common.batch.reader.BaseApiReader; +import com.snp.batch.jobs.shipCurrentlyAt.batch.dto.CurrentlyAtDto; +import lombok.extern.slf4j.Slf4j; +import org.springframework.batch.core.configuration.annotation.StepScope; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.web.reactive.function.client.WebClient; + +import java.util.Collections; +import java.util.List; + +/** + * 선박 상세 정보 Reader (v2.0 - Chunk 기반) + *

+ * 기능: + * 1. ship_data 테이블에서 IMO 번호 전체 조회 (최초 1회) + * 2. IMO 번호를 100개씩 분할하여 배치 단위로 처리 + * 3. fetchNextBatch() 호출 시마다 100개씩 API 호출 + * 4. Spring Batch가 100건씩 Process → Write 수행 + *

+ * Chunk 처리 흐름: + * - beforeFetch() → IMO 전체 조회 (1회) + * - fetchNextBatch() → 100개 IMO로 API 호출 (1,718회) + * - read() → 1건씩 반환 (100번) + * - Processor/Writer → 100건 처리 + * - 반복... (1,718번의 Chunk) + *

+ * 기존 방식과의 차이: + * - 기존: 17만건 전체 메모리 로드 → Process → Write + * - 신규: 100건씩 로드 → Process → Write (Chunk 1,718회) + */ +@Slf4j +@StepScope +public class CurrentlyAtReader extends BaseApiReader { + + private final JdbcTemplate jdbcTemplate; + + // 배치 처리 상태 + private List allImoNumbers; + // DB 해시값을 저장할 맵 + private int currentBatchIndex = 0; + private final int batchSize = 10; + +// @Value("#{jobParameters['startDate']}") +// private String startDate; +// private String startDate = "2025-01-01"; + +// @Value("#{jobParameters['stopDate']}") +// private String stopDate; +// private String stopDate = "2024-12-31"; + + public CurrentlyAtReader(WebClient webClient, JdbcTemplate jdbcTemplate) { + super(webClient); + this.jdbcTemplate = jdbcTemplate; + enableChunkMode(); // ✨ Chunk 모드 활성화 + } + + @Override + protected String getReaderName() { + return "CurrentlyAtReader"; + } + + @Override + protected void resetCustomState() { + this.currentBatchIndex = 0; + this.allImoNumbers = null; + } + + @Override + protected String getApiPath() { + return "/Movements/CurrentlyAt"; + } + + @Override + protected String getApiBaseUrl() { + return "https://webservices.maritime.spglobal.com"; + } + + private static final String GET_ALL_IMO_QUERY = + "SELECT imo_number FROM ship_data ORDER BY id"; +// "SELECT imo_number FROM snp_data.ship_data where imo_number > (select max(imo) from snp_data.t_currentlyat) ORDER BY imo_number"; + + + /** + * 최초 1회만 실행: ship_data 테이블에서 IMO 번호 전체 조회 + */ + @Override + protected void beforeFetch() { + // 전처리 과정 + // Step 1. IMO 전체 번호 조회 + log.info("[{}] ship_data 테이블에서 IMO 번호 조회 시작...", getReaderName()); + + allImoNumbers = jdbcTemplate.queryForList(GET_ALL_IMO_QUERY, String.class); + int totalBatches = (int) Math.ceil((double) allImoNumbers.size() / batchSize); + + log.info("[{}] 총 {} 개의 IMO 번호 조회 완료", getReaderName(), allImoNumbers.size()); + log.info("[{}] {}개씩 배치로 분할하여 API 호출 예정", getReaderName(), batchSize); + log.info("[{}] 예상 배치 수: {} 개", getReaderName(), totalBatches); + + // API 통계 초기화 + updateApiCallStats(totalBatches, 0); + } + + /** + * ✨ Chunk 기반 핵심 메서드: 다음 100개 배치를 조회하여 반환 + *

+ * Spring Batch가 100건씩 read() 호출 완료 후 이 메서드 재호출 + * + * @return 다음 배치 100건 (더 이상 없으면 null) + */ + @Override + protected List fetchNextBatch() throws Exception { + + // 모든 배치 처리 완료 확인 + if (allImoNumbers == null || currentBatchIndex >= allImoNumbers.size()) { + return null; // Job 종료 + } + + // 현재 배치의 시작/끝 인덱스 계산 + int startIndex = currentBatchIndex; + int endIndex = Math.min(currentBatchIndex + batchSize, allImoNumbers.size()); + + // 현재 배치의 IMO 번호 추출 (100개) + List currentBatch = allImoNumbers.subList(startIndex, endIndex); + + int currentBatchNumber = (currentBatchIndex / batchSize) + 1; + int totalBatches = (int) Math.ceil((double) allImoNumbers.size() / batchSize); + + log.info("[{}] 배치 {}/{} 처리 중 (IMO {} 개)...", + getReaderName(), currentBatchNumber, totalBatches, currentBatch.size()); + + try { + // IMO 번호를 쉼표로 연결 (예: "1000019,1000021,1000033,...") + String imoParam = String.join(",", currentBatch); + + // API 호출 + List response = callApiWithBatch(imoParam); + + // 다음 배치로 인덱스 이동 + currentBatchIndex = endIndex; + + + // 응답 처리 + if (response != null) { + List portCalls = response; + log.info("[{}] 배치 {}/{} 완료: {} 건 조회", + getReaderName(), currentBatchNumber, totalBatches, portCalls.size()); + + // API 호출 통계 업데이트 + updateApiCallStats(totalBatches, currentBatchNumber); + + // API 과부하 방지 (다음 배치 전 0.5초 대기) + if (currentBatchIndex < allImoNumbers.size()) { + Thread.sleep(500); + } + + return portCalls; + + } else { + log.warn("[{}] 배치 {}/{} 응답 없음", + getReaderName(), currentBatchNumber, totalBatches); + + // API 호출 통계 업데이트 (실패도 카운트) + updateApiCallStats(totalBatches, currentBatchNumber); + + return Collections.emptyList(); + } + + } catch (Exception e) { + log.error("[{}] 배치 {}/{} 처리 중 오류: {}", + getReaderName(), currentBatchNumber, totalBatches, e.getMessage(), e); + + // 오류 발생 시에도 다음 배치로 이동 (부분 실패 허용) + currentBatchIndex = endIndex; + + // 빈 리스트 반환 (Job 계속 진행) + return Collections.emptyList(); + } + } + + /** + * Query Parameter를 사용한 API 호출 + * + * @param lrno 쉼표로 연결된 IMO 번호 (예: "1000019,1000021,...") + * @return API 응답 + */ + private List callApiWithBatch(String lrno) { + String url = getApiPath() + "?lrno=" + lrno; + + log.debug("[{}] API 호출: {}", getReaderName(), url); + + return webClient.get() + .uri(url) + .retrieve() + .bodyToFlux(CurrentlyAtDto.class) + .collectList() + .block(); + } + + @Override + protected void afterFetch(List data) { + if (data == null) { + int totalBatches = (int) Math.ceil((double) allImoNumbers.size() / batchSize); + log.info("[{}] 전체 {} 개 배치 처리 완료", getReaderName(), totalBatches); + log.info("[{}] 총 {} 개의 IMO 번호에 대한 API 호출 종료", + getReaderName(), allImoNumbers.size()); + } + } + +} diff --git a/src/main/java/com/snp/batch/jobs/shipCurrentlyAt/batch/repository/CurrentlyAtRepository.java b/src/main/java/com/snp/batch/jobs/shipCurrentlyAt/batch/repository/CurrentlyAtRepository.java new file mode 100644 index 0000000..30f12da --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/shipCurrentlyAt/batch/repository/CurrentlyAtRepository.java @@ -0,0 +1,13 @@ +package com.snp.batch.jobs.shipCurrentlyAt.batch.repository; + +import com.snp.batch.jobs.shipCurrentlyAt.batch.entity.CurrentlyAtEntity; + +import java.util.List; + +/** + * 선박 상세 정보 Repository 인터페이스 + */ + +public interface CurrentlyAtRepository { + void saveAll(List entities); +} diff --git a/src/main/java/com/snp/batch/jobs/shipCurrentlyAt/batch/repository/CurrentlyAtRepositoryImpl.java b/src/main/java/com/snp/batch/jobs/shipCurrentlyAt/batch/repository/CurrentlyAtRepositoryImpl.java new file mode 100644 index 0000000..fc25a22 --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/shipCurrentlyAt/batch/repository/CurrentlyAtRepositoryImpl.java @@ -0,0 +1,211 @@ +package com.snp.batch.jobs.shipCurrentlyAt.batch.repository; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.snp.batch.common.batch.repository.BaseJdbcRepository; +import com.snp.batch.jobs.shipCurrentlyAt.batch.entity.CurrentlyAtEntity; +import com.snp.batch.jobs.shipMovement.batch.entity.ShipMovementEntity; +import com.snp.batch.jobs.shipMovement.batch.repository.ShipMovementRepository; +import lombok.extern.slf4j.Slf4j; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.core.RowMapper; +import org.springframework.stereotype.Repository; + +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Timestamp; +import java.util.List; + +/** + * 선박 상세 정보 Repository 구현체 + * BaseJdbcRepository를 상속하여 JDBC 기반 CRUD 구현 + */ +@Slf4j +@Repository("CurrentlyAtRepository") +public class CurrentlyAtRepositoryImpl extends BaseJdbcRepository + implements CurrentlyAtRepository { + + public CurrentlyAtRepositoryImpl(JdbcTemplate jdbcTemplate) { + super(jdbcTemplate); + } + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + @Override + protected String getTableName() { + return "snp_data.t_currentlyat"; + } + + @Override + protected String getEntityName() { + return "CurrentlyAt"; + } + + @Override + protected String extractId(CurrentlyAtEntity entity) { + return entity.getImolRorIHSNumber(); + } + + @Override + public String getInsertSql() { + return """ + INSERT INTO snp_data.t_currentlyat( + imo, + mvmn_type, + mvmn_dt, + stpov_id, + fclty_id, + fclty_nm, + fclty_type, + lwrnk_fclty_id, + lwrnk_fclty_nm, + lwrnk_fclty_type, + up_fclty_id, + up_fclty_nm, + up_fclty_type, + ntn_cd, + ntn_nm, + draft, + lat, + lon, + dstn, + iso2_ntn_cd, + lcinfo + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT (imo, mvmn_dt) + DO UPDATE SET + mvmn_type = EXCLUDED.mvmn_type, + mvmn_dt = EXCLUDED.mvmn_dt, + stpov_id = EXCLUDED.stpov_id, + fclty_id = EXCLUDED.fclty_id, + fclty_nm = EXCLUDED.fclty_nm, + fclty_type = EXCLUDED.fclty_type, + lwrnk_fclty_id = EXCLUDED.lwrnk_fclty_id, + lwrnk_fclty_nm = EXCLUDED.lwrnk_fclty_nm, + lwrnk_fclty_type = EXCLUDED.lwrnk_fclty_type, + up_fclty_id = EXCLUDED.up_fclty_id, + up_fclty_nm = EXCLUDED.up_fclty_nm, + up_fclty_type = EXCLUDED.up_fclty_type, + ntn_cd = EXCLUDED.ntn_cd, + ntn_nm = EXCLUDED.ntn_nm, + draft = EXCLUDED.draft, + lat = EXCLUDED.lat, + lon = EXCLUDED.lon, + dstn = EXCLUDED.dstn, + iso2_ntn_cd = EXCLUDED.iso2_ntn_cd, + lcinfo = EXCLUDED.lcinfo + """; + } + + @Override + protected String getUpdateSql() { + return null; + } + + @Override + protected void setInsertParameters(PreparedStatement ps, CurrentlyAtEntity e) throws Exception { + int i = 1; + ps.setString(i++, e.getImolRorIHSNumber()); // imo + ps.setString(i++, e.getMovementType()); // mvmn_type + ps.setTimestamp(i++, e.getMovementDate() != null ? Timestamp.valueOf(e.getMovementDate()) : null); // mvmn_dt + ps.setObject(i++, e.getPortCallId()); // stpov_id + ps.setObject(i++, e.getFacilityId()); // fclty_id + ps.setString(i++, e.getFacilityName()); // fclty_nm + ps.setString(i++, e.getFacilityType()); // fclty_type + ps.setObject(i++, e.getSubFacilityId()); // lwrnk_fclty_id + ps.setString(i++, e.getSubFacilityName()); // lwrnk_fclty_nm + ps.setString(i++, e.getSubFacilityType()); // lwrnk_fclty_type + ps.setObject(i++, e.getParentFacilityId()); // up_fclty_id + ps.setString(i++, e.getParentFacilityName()); // up_fclty_nm + ps.setString(i++, e.getParentFacilityType()); // up_fclty_type + ps.setString(i++, e.getCountryCode()); // ntn_cd + ps.setString(i++, e.getCountryName()); // ntn_nm + setDoubleOrNull(ps, i++, e.getDraught()); // draft + setDoubleOrNull(ps, i++, e.getLatitude()); // lat + setDoubleOrNull(ps, i++, e.getLongitude());// lon + ps.setString(i++, e.getDestination()); // dstn + ps.setString(i++, e.getIso2()); // iso2_ntn_cd + + if (e.getPosition() != null) { + ps.setObject(i++, OBJECT_MAPPER.writeValueAsString(e.getPosition()), java.sql.Types.OTHER); // lcinfo (jsonb) + } else { + ps.setNull(i++, java.sql.Types.OTHER); + } + +// ps.setString(i++, e.getSchemaType()); + + } + + private void setDoubleOrNull(PreparedStatement ps, int index, Double value) throws Exception { + if (value != null) { + ps.setDouble(index, value); + } else { + // java.sql.Types.DOUBLE을 사용하여 명시적으로 SQL NULL을 설정 + ps.setNull(index, java.sql.Types.DOUBLE); + } + } + + @Override + protected void setUpdateParameters(PreparedStatement ps, CurrentlyAtEntity entity) throws Exception { + + } + + @Override + protected RowMapper getRowMapper() { + return null; + } + + @Override + public void saveAll(List entities) { + if (entities == null || entities.isEmpty()) return; + + log.info("CurrentltAt 저장 시작 = {}건", entities.size()); + batchInsert(entities); + + } + + + /*private static class ShipMovementRowMapper implements RowMapper { + @Override + public ShipMovementEntity mapRow(ResultSet rs, int rowNum) throws SQLException { + ShipMovementEntity entity = ShipMovementEntity.builder() + .id(rs.getLong("id")) + .imolRorIHSNumber(rs.getString("imolRorIHSNumber")) + .portCallId(rs.getObject("portCallId", Integer.class)) + .facilityId(rs.getObject("facilityId", Integer.class)) + .facilityName(rs.getString("facilityName")) + .facilityType(rs.getString("facilityType")) + .subFacilityId(rs.getObject("subFacilityId", Integer.class)) + .subFacilityName(rs.getString("subFacilityName")) + .subFacilityType(rs.getString("subFacilityType")) + .parentFacilityId(rs.getObject("parentFacilityId", Integer.class)) + .parentFacilityName(rs.getString("parentFacilityName")) + .parentFacilityType(rs.getString("parentFacilityType")) + .countryCode(rs.getString("countryCode")) + .countryName(rs.getString("countryName")) + .draught(rs.getObject("draught", Double.class)) + .latitude(rs.getObject("latitude", Double.class)) + .longitude(rs.getObject("longitude", Double.class)) + .destination(rs.getString("destination")) + .iso2(rs.getString("iso2")) + .position(parseJson(rs.getString("position"))) + .schemaType(rs.getString("schemaType")) + .build(); + + Timestamp movementDate = rs.getTimestamp("movementDate"); + if (movementDate != null) { + entity.setMovementDate(movementDate.toLocalDateTime()); + } + + return entity; + } + + private JsonNode parseJson(String json) { + try { + if (json == null) return null; + return new ObjectMapper().readTree(json); + } catch (Exception e) { + throw new RuntimeException("JSON 파싱 오류: " + json); + } + } + }*/ +} diff --git a/src/main/java/com/snp/batch/jobs/shipCurrentlyAt/batch/writer/CurrentlyAtWriter.java b/src/main/java/com/snp/batch/jobs/shipCurrentlyAt/batch/writer/CurrentlyAtWriter.java new file mode 100644 index 0000000..c84e7f0 --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/shipCurrentlyAt/batch/writer/CurrentlyAtWriter.java @@ -0,0 +1,36 @@ +package com.snp.batch.jobs.shipCurrentlyAt.batch.writer; + +import com.snp.batch.common.batch.writer.BaseWriter; +import com.snp.batch.jobs.shipCurrentlyAt.batch.entity.CurrentlyAtEntity; +import com.snp.batch.jobs.shipCurrentlyAt.batch.repository.CurrentlyAtRepository; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; + +import java.util.List; + +/** + * 선박 상세 정보 Writer + */ +@Slf4j +@Component +public class CurrentlyAtWriter extends BaseWriter { + + private final CurrentlyAtRepository currentlyAtRepository; + + + public CurrentlyAtWriter(CurrentlyAtRepository currentlyAtRepository) { + super("CurrentlyAt"); + this.currentlyAtRepository = currentlyAtRepository; + } + + @Override + protected void writeItems(List items) throws Exception { + + if (items.isEmpty()) { return; } + + currentlyAtRepository.saveAll(items); + log.info("CurrentlyAt 데이터 저장: {} 건", items.size()); + + } + +} diff --git a/src/main/java/com/snp/batch/jobs/shipMovement/batch/config/ShipMovementJobConfig.java b/src/main/java/com/snp/batch/jobs/shipMovement/batch/config/ShipMovementJobConfig.java index 674a579..c840630 100644 --- a/src/main/java/com/snp/batch/jobs/shipMovement/batch/config/ShipMovementJobConfig.java +++ b/src/main/java/com/snp/batch/jobs/shipMovement/batch/config/ShipMovementJobConfig.java @@ -2,7 +2,7 @@ package com.snp.batch.jobs.shipMovement.batch.config; import com.fasterxml.jackson.databind.ObjectMapper; import com.snp.batch.common.batch.config.BaseJobConfig; -import com.snp.batch.jobs.shipMovement.batch.dto.PortCallDto; +import com.snp.batch.jobs.shipMovement.batch.dto.PortCallsDto; import com.snp.batch.jobs.shipMovement.batch.entity.ShipMovementEntity; import com.snp.batch.jobs.shipMovement.batch.processor.ShipMovementProcessor; import com.snp.batch.jobs.shipMovement.batch.reader.ShipMovementReader; @@ -47,7 +47,7 @@ import java.time.format.DateTimeFormatter; @Slf4j @Configuration -public class ShipMovementJobConfig extends BaseJobConfig { +public class ShipMovementJobConfig extends BaseJobConfig { private final ShipMovementProcessor shipMovementProcessor; private final ShipMovementWriter shipMovementWriter; @@ -101,14 +101,14 @@ public class ShipMovementJobConfig extends BaseJobConfig createReader() { // 타입 변경 + protected ItemReader createReader() { // 타입 변경 // Reader 생성자 수정: ObjectMapper를 전달합니다. return shipMovementReader(null, null); //return new ShipMovementReader(maritimeApiWebClient, jdbcTemplate, objectMapper); } @Override - protected ItemProcessor createProcessor() { + protected ItemProcessor createProcessor() { return shipMovementProcessor; } diff --git a/src/main/java/com/snp/batch/jobs/shipMovement/batch/dto/PortCallDto.java b/src/main/java/com/snp/batch/jobs/shipMovement/batch/dto/PortCallsDto.java similarity index 91% rename from src/main/java/com/snp/batch/jobs/shipMovement/batch/dto/PortCallDto.java rename to src/main/java/com/snp/batch/jobs/shipMovement/batch/dto/PortCallsDto.java index 6d02359..c97db50 100644 --- a/src/main/java/com/snp/batch/jobs/shipMovement/batch/dto/PortCallDto.java +++ b/src/main/java/com/snp/batch/jobs/shipMovement/batch/dto/PortCallsDto.java @@ -3,7 +3,7 @@ package com.snp.batch.jobs.shipMovement.batch.dto; import lombok.Data; @Data -public class PortCallDto { +public class PortCallsDto { private String movementType; private String imolRorIHSNumber; private String movementDate; @@ -29,7 +29,7 @@ public class PortCallDto { private Double latitude; private Double longitude; - private PositionDto position; + private PortCallsPositionDto position; private String destination; private String iso2; diff --git a/src/main/java/com/snp/batch/jobs/shipMovement/batch/dto/PositionDto.java b/src/main/java/com/snp/batch/jobs/shipMovement/batch/dto/PortCallsPositionDto.java similarity index 90% rename from src/main/java/com/snp/batch/jobs/shipMovement/batch/dto/PositionDto.java rename to src/main/java/com/snp/batch/jobs/shipMovement/batch/dto/PortCallsPositionDto.java index 9a367ba..8906ba0 100644 --- a/src/main/java/com/snp/batch/jobs/shipMovement/batch/dto/PositionDto.java +++ b/src/main/java/com/snp/batch/jobs/shipMovement/batch/dto/PortCallsPositionDto.java @@ -4,7 +4,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import lombok.Data; @Data -public class PositionDto { +public class PortCallsPositionDto { private boolean isNull; private int stSrid; private double lat; diff --git a/src/main/java/com/snp/batch/jobs/shipMovement/batch/dto/ShipMovementApiResponse.java b/src/main/java/com/snp/batch/jobs/shipMovement/batch/dto/ShipMovementApiResponse.java index effef52..eb8fae8 100644 --- a/src/main/java/com/snp/batch/jobs/shipMovement/batch/dto/ShipMovementApiResponse.java +++ b/src/main/java/com/snp/batch/jobs/shipMovement/batch/dto/ShipMovementApiResponse.java @@ -8,5 +8,5 @@ import java.util.List; @Data public class ShipMovementApiResponse { @JsonProperty("portCalls") - List portCallList; + List portCallList; } diff --git a/src/main/java/com/snp/batch/jobs/shipMovement/batch/processor/ShipMovementProcessor.java b/src/main/java/com/snp/batch/jobs/shipMovement/batch/processor/ShipMovementProcessor.java index a270089..102e404 100644 --- a/src/main/java/com/snp/batch/jobs/shipMovement/batch/processor/ShipMovementProcessor.java +++ b/src/main/java/com/snp/batch/jobs/shipMovement/batch/processor/ShipMovementProcessor.java @@ -3,7 +3,7 @@ package com.snp.batch.jobs.shipMovement.batch.processor; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.snp.batch.common.batch.processor.BaseProcessor; -import com.snp.batch.jobs.shipMovement.batch.dto.PortCallDto; +import com.snp.batch.jobs.shipMovement.batch.dto.PortCallsDto; import com.snp.batch.jobs.shipMovement.batch.entity.ShipMovementEntity; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; @@ -22,7 +22,7 @@ import java.time.LocalDateTime; */ @Slf4j @Component -public class ShipMovementProcessor extends BaseProcessor { +public class ShipMovementProcessor extends BaseProcessor { private final ObjectMapper objectMapper; @@ -31,7 +31,7 @@ public class ShipMovementProcessor extends BaseProcessor { +public class ShipMovementReader extends BaseApiReader { private final JdbcTemplate jdbcTemplate; private final ObjectMapper objectMapper; @@ -125,7 +119,7 @@ public class ShipMovementReader extends BaseApiReader { * @return 다음 배치 100건 (더 이상 없으면 null) */ @Override - protected List fetchNextBatch() throws Exception { + protected List fetchNextBatch() throws Exception { // 모든 배치 처리 완료 확인 if (allImoNumbers == null || currentBatchIndex >= allImoNumbers.size()) { @@ -158,7 +152,7 @@ public class ShipMovementReader extends BaseApiReader { // 응답 처리 if (response != null && response.getPortCallList() != null) { - List portCalls = response.getPortCallList(); + List portCalls = response.getPortCallList(); log.info("[{}] 배치 {}/{} 완료: {} 건 조회", getReaderName(), currentBatchNumber, totalBatches, portCalls.size()); @@ -213,7 +207,7 @@ public class ShipMovementReader extends BaseApiReader { } @Override - protected void afterFetch(List data) { + protected void afterFetch(List data) { if (data == null) { int totalBatches = (int) Math.ceil((double) allImoNumbers.size() / batchSize); log.info("[{}] 전체 {} 개 배치 처리 완료", getReaderName(), totalBatches); diff --git a/src/main/java/com/snp/batch/jobs/shipMovementAnchorageCalls/batch/config/AnchorageCallsJobConfig.java b/src/main/java/com/snp/batch/jobs/shipMovementAnchorageCalls/batch/config/AnchorageCallsJobConfig.java index 77a35d7..dcfd05b 100644 --- a/src/main/java/com/snp/batch/jobs/shipMovementAnchorageCalls/batch/config/AnchorageCallsJobConfig.java +++ b/src/main/java/com/snp/batch/jobs/shipMovementAnchorageCalls/batch/config/AnchorageCallsJobConfig.java @@ -1,12 +1,6 @@ package com.snp.batch.jobs.shipMovementAnchorageCalls.batch.config; -import com.fasterxml.jackson.databind.ObjectMapper; import com.snp.batch.common.batch.config.BaseJobConfig; -import com.snp.batch.jobs.shipMovement.batch.dto.PortCallDto; -import com.snp.batch.jobs.shipMovement.batch.entity.ShipMovementEntity; -import com.snp.batch.jobs.shipMovement.batch.processor.ShipMovementProcessor; -import com.snp.batch.jobs.shipMovement.batch.reader.ShipMovementReader; -import com.snp.batch.jobs.shipMovement.batch.writer.ShipMovementWriter; import com.snp.batch.jobs.shipMovementAnchorageCalls.batch.dto.AnchorageCallsDto; import com.snp.batch.jobs.shipMovementAnchorageCalls.batch.entity.AnchorageCallsEntity; import com.snp.batch.jobs.shipMovementAnchorageCalls.batch.processor.AnchorageCallsProcessor; @@ -15,22 +9,17 @@ import com.snp.batch.jobs.shipMovementAnchorageCalls.batch.writer.AnchorageCalls import lombok.extern.slf4j.Slf4j; import org.springframework.batch.core.Job; import org.springframework.batch.core.Step; -import org.springframework.batch.core.configuration.annotation.StepScope; import org.springframework.batch.core.repository.JobRepository; import org.springframework.batch.item.ItemProcessor; import org.springframework.batch.item.ItemReader; import org.springframework.batch.item.ItemWriter; import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.transaction.PlatformTransactionManager; import org.springframework.web.reactive.function.client.WebClient; -import java.time.LocalDate; -import java.time.format.DateTimeFormatter; - /** * 선박 상세 정보 Import Job Config * @@ -42,12 +31,12 @@ import java.time.format.DateTimeFormatter; * - 선박 상세 정보를 ship_detail 테이블에 저장 (UPSERT) * * 데이터 흐름: - * ShipMovementReader (ship_data → Maritime API) - * ↓ (PortCallDto) - * ShipMovementProcessor - * ↓ (ShipMovementEntity) - * ShipDetailDataWriter - * ↓ (ship_movement 테이블) + * AnchorageCallsReader (ship_data → Maritime API) + * ↓ (AnchorageCallsDto) + * AnchorageCallsProcessor + * ↓ (AnchorageCallsEntity) + * AnchorageCallsWriter + * ↓ (t_anchoragecall 테이블) */ @Slf4j diff --git a/src/main/java/com/snp/batch/jobs/shipMovementBerthCalls/batch/dto/BerthCallsDto.java b/src/main/java/com/snp/batch/jobs/shipMovementBerthCalls/batch/dto/BerthCallsDto.java index 03b813e..9483216 100644 --- a/src/main/java/com/snp/batch/jobs/shipMovementBerthCalls/batch/dto/BerthCallsDto.java +++ b/src/main/java/com/snp/batch/jobs/shipMovementBerthCalls/batch/dto/BerthCallsDto.java @@ -1,6 +1,5 @@ package com.snp.batch.jobs.shipMovementBerthCalls.batch.dto; -import com.snp.batch.jobs.shipMovement.batch.dto.PositionDto; import lombok.Data; @Data diff --git a/src/main/java/com/snp/batch/jobs/shipMovementDarkActivity/batch/config/DarkActivityJobConfig.java b/src/main/java/com/snp/batch/jobs/shipMovementDarkActivity/batch/config/DarkActivityJobConfig.java index 0076a18..a370b0b 100644 --- a/src/main/java/com/snp/batch/jobs/shipMovementDarkActivity/batch/config/DarkActivityJobConfig.java +++ b/src/main/java/com/snp/batch/jobs/shipMovementDarkActivity/batch/config/DarkActivityJobConfig.java @@ -32,12 +32,12 @@ import org.springframework.web.reactive.function.client.WebClient; * - 선박 상세 정보를 ship_detail 테이블에 저장 (UPSERT) * * 데이터 흐름: - * ShipMovementReader (ship_data → Maritime API) - * ↓ (PortCallDto) - * ShipMovementProcessor - * ↓ (ShipMovementEntity) - * ShipDetailDataWriter - * ↓ (ship_movement 테이블) + * DarkActivityReader (ship_data → Maritime API) + * ↓ (DarkActivityDto) + * DarkActivityProcessor + * ↓ (DarkActivityEntity) + * DarkActivityWriter + * ↓ (t_darkactivity 테이블) */ @Slf4j diff --git a/src/main/java/com/snp/batch/jobs/shipMovementDestination/batch/config/DestinationsJobConfig.java b/src/main/java/com/snp/batch/jobs/shipMovementDestination/batch/config/DestinationsJobConfig.java new file mode 100644 index 0000000..807741e --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/shipMovementDestination/batch/config/DestinationsJobConfig.java @@ -0,0 +1,103 @@ +package com.snp.batch.jobs.shipMovementDestination.batch.config; + +import com.snp.batch.common.batch.config.BaseJobConfig; +import com.snp.batch.jobs.shipMovementDestination.batch.dto.DestinationDto; +import com.snp.batch.jobs.shipMovementDestination.batch.entity.DestinationEntity; +import com.snp.batch.jobs.shipMovementDestination.batch.processor.DestinationProcessor; +import com.snp.batch.jobs.shipMovementDestination.batch.reader.DestinationReader; +import com.snp.batch.jobs.shipMovementDestination.batch.writer.DestinationWriter; +import lombok.extern.slf4j.Slf4j; +import org.springframework.batch.core.Job; +import org.springframework.batch.core.Step; +import org.springframework.batch.core.repository.JobRepository; +import org.springframework.batch.item.ItemProcessor; +import org.springframework.batch.item.ItemReader; +import org.springframework.batch.item.ItemWriter; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.transaction.PlatformTransactionManager; +import org.springframework.web.reactive.function.client.WebClient; + +/** + * 선박 상세 정보 Import Job Config + * + * 특징: + * - ship_data 테이블에서 IMO 번호 조회 + * - IMO 번호를 100개씩 배치로 분할 + * - Maritime API GetShipsByIHSLRorIMONumbers 호출 + * TODO : GetShipsByIHSLRorIMONumbersAll 호출로 변경 + * - 선박 상세 정보를 ship_detail 테이블에 저장 (UPSERT) + * + * 데이터 흐름: + * DestinationReader (ship_data → Maritime API) + * ↓ (DestinationDto) + * DestinationProcessor + * ↓ (DestinationEntity) + * DestinationProcessor + * ↓ (t_destination 테이블) + */ + +@Slf4j +@Configuration +public class DestinationsJobConfig extends BaseJobConfig { + + private final DestinationProcessor destinationProcessor; + private final DestinationWriter destinationWriter; + private final JdbcTemplate jdbcTemplate; + private final WebClient maritimeApiWebClient; + + public DestinationsJobConfig( + JobRepository jobRepository, + PlatformTransactionManager transactionManager, + DestinationProcessor destinationProcessor, + DestinationWriter destinationWriter, JdbcTemplate jdbcTemplate, + @Qualifier("maritimeServiceApiWebClient") WebClient maritimeApiWebClient) { // ObjectMapper 주입 추가 + super(jobRepository, transactionManager); + this.destinationProcessor = destinationProcessor; + this.destinationWriter = destinationWriter; + this.jdbcTemplate = jdbcTemplate; + this.maritimeApiWebClient = maritimeApiWebClient; + } + + @Override + protected String getJobName() { + return "DestinationsImportJob"; + } + + @Override + protected String getStepName() { + return "DestinationsImportStep"; + } + + @Override + protected ItemReader createReader() { // 타입 변경 + return new DestinationReader(maritimeApiWebClient, jdbcTemplate); + } + + @Override + protected ItemProcessor createProcessor() { + return destinationProcessor; + } + + @Override + protected ItemWriter createWriter() { // 타입 변경 + return destinationWriter; + } + + @Override + protected int getChunkSize() { + return 1000; // API에서 100개씩 가져오므로 chunk도 100으로 설정 + } + + @Bean(name = "DestinationsImportJob") + public Job destinationsImportJob() { + return job(); + } + + @Bean(name = "DestinationsImportStep") + public Step destinationsImportStep() { + return step(); + } +} diff --git a/src/main/java/com/snp/batch/jobs/shipMovementDestination/batch/dto/DestinationDto.java b/src/main/java/com/snp/batch/jobs/shipMovementDestination/batch/dto/DestinationDto.java new file mode 100644 index 0000000..c150ee4 --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/shipMovementDestination/batch/dto/DestinationDto.java @@ -0,0 +1,24 @@ +package com.snp.batch.jobs.shipMovementDestination.batch.dto; + +import lombok.Data; + +@Data +public class DestinationDto { + private String movementType; + private String imolRorIHSNumber; + private String movementDate; + + private Integer facilityId; + private String facilityName; + private String facilityType; + + private String countryCode; + private String countryName; + + private Double latitude; + private Double longitude; + + private DestinationPositionDto position; + + private String iso2; +} diff --git a/src/main/java/com/snp/batch/jobs/shipMovementDestination/batch/dto/DestinationPositionDto.java b/src/main/java/com/snp/batch/jobs/shipMovementDestination/batch/dto/DestinationPositionDto.java new file mode 100644 index 0000000..f600d28 --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/shipMovementDestination/batch/dto/DestinationPositionDto.java @@ -0,0 +1,17 @@ +package com.snp.batch.jobs.shipMovementDestination.batch.dto; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; + +@Data +public class DestinationPositionDto { + private boolean isNull; + private int stSrid; + private double lat; + @JsonProperty("long") + private double lon; + private double z; + private double m; + private boolean hasZ; + private boolean hasM; +} diff --git a/src/main/java/com/snp/batch/jobs/shipMovementDestination/batch/entity/DestinationEntity.java b/src/main/java/com/snp/batch/jobs/shipMovementDestination/batch/entity/DestinationEntity.java new file mode 100644 index 0000000..fa2a23a --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/shipMovementDestination/batch/entity/DestinationEntity.java @@ -0,0 +1,32 @@ +package com.snp.batch.jobs.shipMovementDestination.batch.entity; + +import com.fasterxml.jackson.databind.JsonNode; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +import java.time.LocalDateTime; + +@Data +@SuperBuilder +@NoArgsConstructor +@AllArgsConstructor +public class DestinationEntity { + private String movementType; + private String imolRorIHSNumber; + private LocalDateTime movementDate; + + private Integer facilityId; + private String facilityName; + private String facilityType; + + private String countryCode; + private String countryName; + + private Double latitude; + private Double longitude; + + private JsonNode position; + private String iso2; +} diff --git a/src/main/java/com/snp/batch/jobs/shipMovementDestination/batch/processor/DestinationProcessor.java b/src/main/java/com/snp/batch/jobs/shipMovementDestination/batch/processor/DestinationProcessor.java new file mode 100644 index 0000000..8379fe5 --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/shipMovementDestination/batch/processor/DestinationProcessor.java @@ -0,0 +1,61 @@ +package com.snp.batch.jobs.shipMovementDestination.batch.processor; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.snp.batch.common.batch.processor.BaseProcessor; +import com.snp.batch.jobs.shipMovementDestination.batch.dto.DestinationDto; +import com.snp.batch.jobs.shipMovementDestination.batch.entity.DestinationEntity; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; + +import java.time.LocalDateTime; + +/** + * 선박 상세 정보 Processor + * ShipDetailDto → ShipDetailEntity 변환 + */ + +/** + * 선박 상세 정보 Processor (해시 비교 및 증분 데이터 추출) + * I: ShipDetailComparisonData (DB 해시 + API Map Data) + * O: ShipDetailUpdate (변경분) + */ +@Slf4j +@Component +public class DestinationProcessor extends BaseProcessor { + + private final ObjectMapper objectMapper; + + public DestinationProcessor(ObjectMapper objectMapper) { + this.objectMapper = objectMapper; + } + + @Override + protected DestinationEntity processItem(DestinationDto dto) throws Exception { + log.debug("선박 상세 정보 처리 시작: imoNumber={}, facilityName={}", + dto.getImolRorIHSNumber(), dto.getFacilityName()); + + JsonNode positionNode = null; + if (dto.getPosition() != null) { + // Position 객체를 JsonNode로 변환 + positionNode = objectMapper.valueToTree(dto.getPosition()); + } + + DestinationEntity entity = DestinationEntity.builder() + .movementType(dto.getMovementType()) + .imolRorIHSNumber(dto.getImolRorIHSNumber()) + .movementDate(LocalDateTime.parse(dto.getMovementDate())) + .facilityId(dto.getFacilityId()) + .facilityName(dto.getFacilityName()) + .facilityType(dto.getFacilityType()) + .countryCode(dto.getCountryCode()) + .countryName(dto.getCountryName()) + .latitude(dto.getLatitude()) + .longitude(dto.getLongitude()) + .position(positionNode) // JsonNode로 매핑 + .iso2(dto.getIso2()) + .build(); + return entity; + } + +} diff --git a/src/main/java/com/snp/batch/jobs/shipMovementDestination/batch/reader/DestinationReader.java b/src/main/java/com/snp/batch/jobs/shipMovementDestination/batch/reader/DestinationReader.java new file mode 100644 index 0000000..3220826 --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/shipMovementDestination/batch/reader/DestinationReader.java @@ -0,0 +1,211 @@ +package com.snp.batch.jobs.shipMovementDestination.batch.reader; + +import com.snp.batch.common.batch.reader.BaseApiReader; +import com.snp.batch.jobs.shipMovementDestination.batch.dto.DestinationDto; +import lombok.extern.slf4j.Slf4j; +import org.springframework.batch.core.configuration.annotation.StepScope; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.web.reactive.function.client.WebClient; + +import java.util.Collections; +import java.util.List; +import java.util.Map; + +/** + * 선박 상세 정보 Reader (v2.0 - Chunk 기반) + * + * 기능: + * 1. ship_data 테이블에서 IMO 번호 전체 조회 (최초 1회) + * 2. IMO 번호를 100개씩 분할하여 배치 단위로 처리 + * 3. fetchNextBatch() 호출 시마다 100개씩 API 호출 + * 4. Spring Batch가 100건씩 Process → Write 수행 + * + * Chunk 처리 흐름: + * - beforeFetch() → IMO 전체 조회 (1회) + * - fetchNextBatch() → 100개 IMO로 API 호출 (1,718회) + * - read() → 1건씩 반환 (100번) + * - Processor/Writer → 100건 처리 + * - 반복... (1,718번의 Chunk) + * + * 기존 방식과의 차이: + * - 기존: 17만건 전체 메모리 로드 → Process → Write + * - 신규: 100건씩 로드 → Process → Write (Chunk 1,718회) + */ +@Slf4j +@StepScope +public class DestinationReader extends BaseApiReader { + + private final JdbcTemplate jdbcTemplate; + + // 배치 처리 상태 + private List allImoNumbers; + // DB 해시값을 저장할 맵 + private int currentBatchIndex = 0; + private final int batchSize = 5; + + // @Value("#{jobParameters['startDate']}") +// private String startDate; + private String startDate = "2025-01-01"; + + // @Value("#{jobParameters['stopDate']}") +// private String stopDate; + private String stopDate = "2025-12-31"; + + public DestinationReader(WebClient webClient, JdbcTemplate jdbcTemplate ) { + super(webClient); + this.jdbcTemplate = jdbcTemplate; + enableChunkMode(); // ✨ Chunk 모드 활성화 + } + + @Override + protected String getReaderName() { + return "Destinations"; + } + + @Override + protected void resetCustomState() { + this.currentBatchIndex = 0; + this.allImoNumbers = null; + } + + @Override + protected String getApiPath() { + return "/Movements/Destinations"; + } + + @Override + protected String getApiBaseUrl() { + return "https://webservices.maritime.spglobal.com"; + } + + private static final String GET_ALL_IMO_QUERY = +// "SELECT imo_number FROM ship_data ORDER BY id"; + "SELECT imo_number FROM snp_data.ship_data where imo_number > (select max(imo) from snp_data.t_destination) ORDER BY imo_number"; + + /** + * 최초 1회만 실행: ship_data 테이블에서 IMO 번호 전체 조회 + */ + @Override + protected void beforeFetch() { + // 전처리 과정 + // Step 1. IMO 전체 번호 조회 + log.info("[{}] ship_data 테이블에서 IMO 번호 조회 시작...", getReaderName()); + + allImoNumbers = jdbcTemplate.queryForList(GET_ALL_IMO_QUERY, String.class); + int totalBatches = (int) Math.ceil((double) allImoNumbers.size() / batchSize); + + log.info("[{}] 총 {} 개의 IMO 번호 조회 완료", getReaderName(), allImoNumbers.size()); + log.info("[{}] {}개씩 배치로 분할하여 API 호출 예정", getReaderName(), batchSize); + log.info("[{}] 예상 배치 수: {} 개", getReaderName(), totalBatches); + + // API 통계 초기화 + updateApiCallStats(totalBatches, 0); + } + + /** + * ✨ Chunk 기반 핵심 메서드: 다음 100개 배치를 조회하여 반환 + * + * Spring Batch가 100건씩 read() 호출 완료 후 이 메서드 재호출 + * + * @return 다음 배치 100건 (더 이상 없으면 null) + */ + @Override + protected List fetchNextBatch() throws Exception { + + // 모든 배치 처리 완료 확인 + if (allImoNumbers == null || currentBatchIndex >= allImoNumbers.size()) { + return null; // Job 종료 + } + + // 현재 배치의 시작/끝 인덱스 계산 + int startIndex = currentBatchIndex; + int endIndex = Math.min(currentBatchIndex + batchSize, allImoNumbers.size()); + + // 현재 배치의 IMO 번호 추출 (100개) + List currentBatch = allImoNumbers.subList(startIndex, endIndex); + + int currentBatchNumber = (currentBatchIndex / batchSize) + 1; + int totalBatches = (int) Math.ceil((double) allImoNumbers.size() / batchSize); + + log.info("[{}] 배치 {}/{} 처리 중 (IMO {} 개)...", + getReaderName(), currentBatchNumber, totalBatches, currentBatch.size()); + + try { + // IMO 번호를 쉼표로 연결 (예: "1000019,1000021,1000033,...") + String imoParam = String.join(",", currentBatch); + + // API 호출 + List response = callApiWithBatch(imoParam); + + // 다음 배치로 인덱스 이동 + currentBatchIndex = endIndex; + + + // 응답 처리 + if (response != null ) { + List destinations = response; + log.info("[{}] 배치 {}/{} 완료: {} 건 조회", + getReaderName(), currentBatchNumber, totalBatches, destinations.size()); + + // API 호출 통계 업데이트 + updateApiCallStats(totalBatches, currentBatchNumber); + + // API 과부하 방지 (다음 배치 전 0.5초 대기) + if (currentBatchIndex < allImoNumbers.size()) { + Thread.sleep(500); + } + + return destinations; + + } else { + log.warn("[{}] 배치 {}/{} 응답 없음", + getReaderName(), currentBatchNumber, totalBatches); + + // API 호출 통계 업데이트 (실패도 카운트) + updateApiCallStats(totalBatches, currentBatchNumber); + + return Collections.emptyList(); + } + + } catch (Exception e) { + log.error("[{}] 배치 {}/{} 처리 중 오류: {}", + getReaderName(), currentBatchNumber, totalBatches, e.getMessage(), e); + + // 오류 발생 시에도 다음 배치로 이동 (부분 실패 허용) + currentBatchIndex = endIndex; + + // 빈 리스트 반환 (Job 계속 진행) + return Collections.emptyList(); + } + } + + /** + * Query Parameter를 사용한 API 호출 + * + * @param lrno 쉼표로 연결된 IMO 번호 (예: "1000019,1000021,...") + * @return API 응답 + */ + private List callApiWithBatch(String lrno) { + String url = getApiPath() + "?startDate=" + startDate +"&stopDate="+stopDate+"&lrno=" + lrno; + + log.debug("[{}] API 호출: {}", getReaderName(), url); + + return webClient.get() + .uri(url) + .retrieve() + .bodyToFlux(DestinationDto.class) + .collectList() + .block(); + } + + @Override + protected void afterFetch(List data) { + if (data == null) { + int totalBatches = (int) Math.ceil((double) allImoNumbers.size() / batchSize); + log.info("[{}] 전체 {} 개 배치 처리 완료", getReaderName(), totalBatches); + log.info("[{}] 총 {} 개의 IMO 번호에 대한 API 호출 종료", + getReaderName(), allImoNumbers.size()); + } + } + +} diff --git a/src/main/java/com/snp/batch/jobs/shipMovementDestination/batch/repository/DestinationRepository.java b/src/main/java/com/snp/batch/jobs/shipMovementDestination/batch/repository/DestinationRepository.java new file mode 100644 index 0000000..4613e37 --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/shipMovementDestination/batch/repository/DestinationRepository.java @@ -0,0 +1,14 @@ +package com.snp.batch.jobs.shipMovementDestination.batch.repository; + +import com.snp.batch.jobs.shipMovementDestination.batch.entity.DestinationEntity; +import com.snp.batch.jobs.shipMovementTerminalCalls.batch.entity.TerminalCallsEntity; + +import java.util.List; + +/** + * 선박 상세 정보 Repository 인터페이스 + */ + +public interface DestinationRepository { + void saveAll(List entities); +} diff --git a/src/main/java/com/snp/batch/jobs/shipMovementDestination/batch/repository/DestinationRepositoryImpl.java b/src/main/java/com/snp/batch/jobs/shipMovementDestination/batch/repository/DestinationRepositoryImpl.java new file mode 100644 index 0000000..d864265 --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/shipMovementDestination/batch/repository/DestinationRepositoryImpl.java @@ -0,0 +1,131 @@ +package com.snp.batch.jobs.shipMovementDestination.batch.repository; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.snp.batch.common.batch.repository.BaseJdbcRepository; +import com.snp.batch.jobs.shipMovementDestination.batch.entity.DestinationEntity; +import lombok.extern.slf4j.Slf4j; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.core.RowMapper; +import org.springframework.stereotype.Repository; + +import java.sql.PreparedStatement; +import java.sql.Timestamp; +import java.util.List; + +/** + * 선박 상세 정보 Repository 구현체 + * BaseJdbcRepository를 상속하여 JDBC 기반 CRUD 구현 + */ +@Slf4j +@Repository("DestinationRepository") +public class DestinationRepositoryImpl extends BaseJdbcRepository + implements DestinationRepository { + + public DestinationRepositoryImpl(JdbcTemplate jdbcTemplate) { + super(jdbcTemplate); + } + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + @Override + protected String getTableName() { + return "snp_data.t_destination"; + } + + @Override + protected String getEntityName() { + return "Destinations"; + } + + @Override + protected String extractId(DestinationEntity entity) { + return entity.getImolRorIHSNumber(); + } + + @Override + public String getInsertSql() { + return """ + INSERT INTO snp_data.t_destination( + imo, + mvmn_type, + mvmn_dt, + fclty_id, + fclty_nm, + fclty_type, + ntn_cd, + ntn_nm, + lat, + lon, + iso2_ntn_cd, + lcinfo + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT (imo, mvmn_dt) + DO UPDATE SET + mvmn_type = EXCLUDED.mvmn_type, + mvmn_dt = EXCLUDED.mvmn_dt, + fclty_id = EXCLUDED.fclty_id, + fclty_nm = EXCLUDED.fclty_nm, + fclty_type = EXCLUDED.fclty_type, + ntn_cd = EXCLUDED.ntn_cd, + ntn_nm = EXCLUDED.ntn_nm, + lat = EXCLUDED.lat, + lon = EXCLUDED.lon, + iso2_ntn_cd = EXCLUDED.iso2_ntn_cd, + lcinfo = EXCLUDED.lcinfo + """; + } + + @Override + protected String getUpdateSql() { + return null; + } + + @Override + protected void setInsertParameters(PreparedStatement ps, DestinationEntity e) throws Exception { + int i = 1; + ps.setString(i++, e.getImolRorIHSNumber()); // imo + ps.setString(i++, e.getMovementType()); // mvmn_type + ps.setTimestamp(i++, e.getMovementDate() != null ? Timestamp.valueOf(e.getMovementDate()) : null); // mvmn_dt + ps.setObject(i++, e.getFacilityId()); // fclty_id + ps.setString(i++, e.getFacilityName()); // fclty_nm + ps.setString(i++, e.getFacilityType()); // fclty_type + ps.setString(i++, e.getCountryCode()); // ntn_cd + ps.setString(i++, e.getCountryName()); // ntn_nm + setDoubleOrNull(ps, i++, e.getLatitude()); // lat + setDoubleOrNull(ps, i++, e.getLongitude());// lon + ps.setString(i++, e.getIso2()); // iso2_ntn_cd + + if (e.getPosition() != null) { + ps.setObject(i++, OBJECT_MAPPER.writeValueAsString(e.getPosition()), java.sql.Types.OTHER); // lcinfo (jsonb) + } else { + ps.setNull(i++, java.sql.Types.OTHER); + } + } + + private void setDoubleOrNull(PreparedStatement ps, int index, Double value) throws Exception { + if (value != null) { + ps.setDouble(index, value); + } else { + // java.sql.Types.DOUBLE을 사용하여 명시적으로 SQL NULL을 설정 + ps.setNull(index, java.sql.Types.DOUBLE); + } + } + + @Override + protected void setUpdateParameters(PreparedStatement ps, DestinationEntity entity) throws Exception { + + } + + @Override + protected RowMapper getRowMapper() { + return null; + } + + @Override + public void saveAll(List entities) { + if (entities == null || entities.isEmpty()) return; + + log.info("Destinations 저장 시작 = {}건", entities.size()); + batchInsert(entities); + + } + +} diff --git a/src/main/java/com/snp/batch/jobs/shipMovementDestination/batch/writer/DestinationWriter.java b/src/main/java/com/snp/batch/jobs/shipMovementDestination/batch/writer/DestinationWriter.java new file mode 100644 index 0000000..be05993 --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/shipMovementDestination/batch/writer/DestinationWriter.java @@ -0,0 +1,36 @@ +package com.snp.batch.jobs.shipMovementDestination.batch.writer; + +import com.snp.batch.common.batch.writer.BaseWriter; +import com.snp.batch.jobs.shipMovementDestination.batch.entity.DestinationEntity; +import com.snp.batch.jobs.shipMovementDestination.batch.repository.DestinationRepository; +import com.snp.batch.jobs.shipMovementTerminalCalls.batch.entity.TerminalCallsEntity; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; + +import java.util.List; + +/** + * 선박 상세 정보 Writer + */ +@Slf4j +@Component +public class DestinationWriter extends BaseWriter { + + private final DestinationRepository destinationRepository; + + + public DestinationWriter(DestinationRepository destinationRepository) { + super("Destinations"); + this.destinationRepository = destinationRepository; + } + + @Override + protected void writeItems(List items) throws Exception { + + if (items.isEmpty()) { return; } + + destinationRepository.saveAll(items); + log.info("Destinations 데이터 저장: {} 건", items.size()); + } + +} diff --git a/src/main/java/com/snp/batch/jobs/shipMovementStsOperations/batch/config/StsOperationJobConfig.java b/src/main/java/com/snp/batch/jobs/shipMovementStsOperations/batch/config/StsOperationJobConfig.java index 1d57569..d2a5cce 100644 --- a/src/main/java/com/snp/batch/jobs/shipMovementStsOperations/batch/config/StsOperationJobConfig.java +++ b/src/main/java/com/snp/batch/jobs/shipMovementStsOperations/batch/config/StsOperationJobConfig.java @@ -1,12 +1,6 @@ package com.snp.batch.jobs.shipMovementStsOperations.batch.config; -import com.fasterxml.jackson.databind.ObjectMapper; import com.snp.batch.common.batch.config.BaseJobConfig; -import com.snp.batch.jobs.shipMovementBerthCalls.batch.dto.BerthCallsDto; -import com.snp.batch.jobs.shipMovementBerthCalls.batch.entiity.BerthCallsEntity; -import com.snp.batch.jobs.shipMovementBerthCalls.batch.processor.BerthCallsProcessor; -import com.snp.batch.jobs.shipMovementBerthCalls.batch.reader.BerthCallsReader; -import com.snp.batch.jobs.shipMovementBerthCalls.batch.writer.BerthCallsWriter; import com.snp.batch.jobs.shipMovementStsOperations.batch.dto.StsOperationDto; import com.snp.batch.jobs.shipMovementStsOperations.batch.entity.StsOperationEntity; import com.snp.batch.jobs.shipMovementStsOperations.batch.processor.StsOperationProcessor; @@ -37,12 +31,12 @@ import org.springframework.web.reactive.function.client.WebClient; * - 선박 상세 정보를 ship_detail 테이블에 저장 (UPSERT) * * 데이터 흐름: - * ShipMovementReader (ship_data → Maritime API) - * ↓ (PortCallDto) - * ShipMovementProcessor - * ↓ (ShipMovementEntity) - * ShipDetailDataWriter - * ↓ (ship_movement 테이블) + * StsOperationReader (ship_data → Maritime API) + * ↓ (StsOperationDto) + * StsOperationProcessor + * ↓ (StsOperationEntity) + * StsOperationWriter + * ↓ (t_stsoperation 테이블) */ @Slf4j diff --git a/src/main/java/com/snp/batch/jobs/shipMovementStsOperations/batch/dto/StsOperationDto.java b/src/main/java/com/snp/batch/jobs/shipMovementStsOperations/batch/dto/StsOperationDto.java index 16c8db8..0a7fca7 100644 --- a/src/main/java/com/snp/batch/jobs/shipMovementStsOperations/batch/dto/StsOperationDto.java +++ b/src/main/java/com/snp/batch/jobs/shipMovementStsOperations/batch/dto/StsOperationDto.java @@ -1,6 +1,5 @@ package com.snp.batch.jobs.shipMovementStsOperations.batch.dto; -import com.snp.batch.jobs.shipMovementBerthCalls.batch.dto.BerthCallsPositionDto; import lombok.Data; @Data diff --git a/src/main/java/com/snp/batch/jobs/shipMovementTerminalCalls/batch/config/TerminalCallsJobConfig.java b/src/main/java/com/snp/batch/jobs/shipMovementTerminalCalls/batch/config/TerminalCallsJobConfig.java index a298f9e..a221b25 100644 --- a/src/main/java/com/snp/batch/jobs/shipMovementTerminalCalls/batch/config/TerminalCallsJobConfig.java +++ b/src/main/java/com/snp/batch/jobs/shipMovementTerminalCalls/batch/config/TerminalCallsJobConfig.java @@ -1,6 +1,5 @@ package com.snp.batch.jobs.shipMovementTerminalCalls.batch.config; -import com.fasterxml.jackson.databind.ObjectMapper; import com.snp.batch.common.batch.config.BaseJobConfig; import com.snp.batch.jobs.shipMovementTerminalCalls.batch.dto.TerminalCallsDto; import com.snp.batch.jobs.shipMovementTerminalCalls.batch.entity.TerminalCallsEntity; @@ -32,12 +31,12 @@ import org.springframework.web.reactive.function.client.WebClient; * - 선박 상세 정보를 ship_detail 테이블에 저장 (UPSERT) * * 데이터 흐름: - * ShipMovementReader (ship_data → Maritime API) - * ↓ (PortCallDto) - * ShipMovementProcessor - * ↓ (ShipMovementEntity) - * ShipDetailDataWriter - * ↓ (ship_movement 테이블) + * TerminalCallsReader (ship_data → Maritime API) + * ↓ (TerminalCallsDto) + * TerminalCallsProcessor + * ↓ (TerminalCallsEntity) + * TerminalCallsWriter + * ↓ (t_terminalcall 테이블) */ @Slf4j diff --git a/src/main/java/com/snp/batch/jobs/shipMovementTransits/batch/config/TransitsJobConfig.java b/src/main/java/com/snp/batch/jobs/shipMovementTransits/batch/config/TransitsJobConfig.java new file mode 100644 index 0000000..f6d65ba --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/shipMovementTransits/batch/config/TransitsJobConfig.java @@ -0,0 +1,103 @@ +package com.snp.batch.jobs.shipMovementTransits.batch.config; + +import com.snp.batch.common.batch.config.BaseJobConfig; +import com.snp.batch.jobs.shipMovementTransits.batch.dto.TransitsDto; +import com.snp.batch.jobs.shipMovementTransits.batch.entity.TransitsEntity; +import com.snp.batch.jobs.shipMovementTransits.batch.processor.TransitsProcessor; +import com.snp.batch.jobs.shipMovementTransits.batch.reader.TransitsReader; +import com.snp.batch.jobs.shipMovementTransits.batch.writer.TransitsWriter; +import lombok.extern.slf4j.Slf4j; +import org.springframework.batch.core.Job; +import org.springframework.batch.core.Step; +import org.springframework.batch.core.repository.JobRepository; +import org.springframework.batch.item.ItemProcessor; +import org.springframework.batch.item.ItemReader; +import org.springframework.batch.item.ItemWriter; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.transaction.PlatformTransactionManager; +import org.springframework.web.reactive.function.client.WebClient; + +/** + * 선박 상세 정보 Import Job Config + * + * 특징: + * - ship_data 테이블에서 IMO 번호 조회 + * - IMO 번호를 100개씩 배치로 분할 + * - Maritime API GetShipsByIHSLRorIMONumbers 호출 + * TODO : GetShipsByIHSLRorIMONumbersAll 호출로 변경 + * - 선박 상세 정보를 ship_detail 테이블에 저장 (UPSERT) + * + * 데이터 흐름: + * TransitsReader (ship_data → Maritime API) + * ↓ (TransitsDto) + * TransitsProcessor + * ↓ (TransitsEntity) + * TransitsWriter + * ↓ (t_transit 테이블) + */ + +@Slf4j +@Configuration +public class TransitsJobConfig extends BaseJobConfig { + + private final TransitsProcessor transitsProcessor; + private final TransitsWriter transitsWriter; + private final JdbcTemplate jdbcTemplate; + private final WebClient maritimeApiWebClient; + + public TransitsJobConfig( + JobRepository jobRepository, + PlatformTransactionManager transactionManager, + TransitsProcessor TransitsProcessor, + TransitsWriter transitsWriter, JdbcTemplate jdbcTemplate, + @Qualifier("maritimeServiceApiWebClient") WebClient maritimeApiWebClient) { // ObjectMapper 주입 추가 + super(jobRepository, transactionManager); + this.transitsProcessor = TransitsProcessor; + this.transitsWriter = transitsWriter; + this.jdbcTemplate = jdbcTemplate; + this.maritimeApiWebClient = maritimeApiWebClient; + } + + @Override + protected String getJobName() { + return "TransitsImportJob"; + } + + @Override + protected String getStepName() { + return "TransitsImportStep"; + } + + @Override + protected ItemReader createReader() { // 타입 변경 + return new TransitsReader(maritimeApiWebClient, jdbcTemplate); + } + + @Override + protected ItemProcessor createProcessor() { + return transitsProcessor; + } + + @Override + protected ItemWriter createWriter() { // 타입 변경 + return transitsWriter; + } + + @Override + protected int getChunkSize() { + return 1000; // API에서 100개씩 가져오므로 chunk도 100으로 설정 + } + + @Bean(name = "TransitsImportJob") + public Job transitsImportJob() { + return job(); + } + + @Bean(name = "TransitsImportStep") + public Step transitsImportStep() { + return step(); + } +} diff --git a/src/main/java/com/snp/batch/jobs/shipMovementTransits/batch/dto/TransitsDto.java b/src/main/java/com/snp/batch/jobs/shipMovementTransits/batch/dto/TransitsDto.java new file mode 100644 index 0000000..7dd2958 --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/shipMovementTransits/batch/dto/TransitsDto.java @@ -0,0 +1,13 @@ +package com.snp.batch.jobs.shipMovementTransits.batch.dto; + +import lombok.Data; + +@Data +public class TransitsDto { + private String movementType; + private String imolRorIHSNumber; + private String movementDate; + private String facilityName; + private String facilityType; + private Double draught; +} diff --git a/src/main/java/com/snp/batch/jobs/shipMovementTransits/batch/entity/TransitsEntity.java b/src/main/java/com/snp/batch/jobs/shipMovementTransits/batch/entity/TransitsEntity.java new file mode 100644 index 0000000..ddfe811 --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/shipMovementTransits/batch/entity/TransitsEntity.java @@ -0,0 +1,21 @@ +package com.snp.batch.jobs.shipMovementTransits.batch.entity; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +import java.time.LocalDateTime; + +@Data +@SuperBuilder +@NoArgsConstructor +@AllArgsConstructor +public class TransitsEntity { + private String movementType; + private String imolRorIHSNumber; + private LocalDateTime movementDate; + private String facilityName; + private String facilityType; + private Double draught; +} diff --git a/src/main/java/com/snp/batch/jobs/shipMovementTransits/batch/processor/TransitsProcessor.java b/src/main/java/com/snp/batch/jobs/shipMovementTransits/batch/processor/TransitsProcessor.java new file mode 100644 index 0000000..8c7df92 --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/shipMovementTransits/batch/processor/TransitsProcessor.java @@ -0,0 +1,47 @@ +package com.snp.batch.jobs.shipMovementTransits.batch.processor; + +import com.snp.batch.common.batch.processor.BaseProcessor; +import com.snp.batch.jobs.shipMovementTransits.batch.dto.TransitsDto; +import com.snp.batch.jobs.shipMovementTransits.batch.entity.TransitsEntity; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; + +import java.time.LocalDateTime; + +/** + * 선박 상세 정보 Processor + * ShipDetailDto → ShipDetailEntity 변환 + */ + +/** + * 선박 상세 정보 Processor (해시 비교 및 증분 데이터 추출) + * I: ShipDetailComparisonData (DB 해시 + API Map Data) + * O: ShipDetailUpdate (변경분) + */ +@Slf4j +@Component +public class TransitsProcessor extends BaseProcessor { + +// private final ObjectMapper objectMapper; + +// public TransitsProcessor(ObjectMapper objectMapper) { +// this.objectMapper = objectMapper; +// } + + @Override + protected TransitsEntity processItem(TransitsDto dto) throws Exception { + log.debug("선박 상세 정보 처리 시작: imoNumber={}, facilityName={}", + dto.getImolRorIHSNumber(), dto.getFacilityName()); + + TransitsEntity entity = TransitsEntity.builder() + .movementType(dto.getMovementType()) + .imolRorIHSNumber(dto.getImolRorIHSNumber()) + .movementDate(LocalDateTime.parse(dto.getMovementDate())) + .facilityName(dto.getFacilityName()) + .facilityType(dto.getFacilityType()) + .draught(dto.getDraught()) + .build(); + return entity; + } + +} diff --git a/src/main/java/com/snp/batch/jobs/shipMovementTransits/batch/reader/TransitsReader.java b/src/main/java/com/snp/batch/jobs/shipMovementTransits/batch/reader/TransitsReader.java new file mode 100644 index 0000000..375b14f --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/shipMovementTransits/batch/reader/TransitsReader.java @@ -0,0 +1,211 @@ +package com.snp.batch.jobs.shipMovementTransits.batch.reader; + +import com.snp.batch.common.batch.reader.BaseApiReader; +import com.snp.batch.jobs.shipMovementTransits.batch.dto.TransitsDto; +import lombok.extern.slf4j.Slf4j; +import org.springframework.batch.core.configuration.annotation.StepScope; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.web.reactive.function.client.WebClient; + +import java.util.Collections; +import java.util.List; +import java.util.Map; + +/** + * 선박 상세 정보 Reader (v2.0 - Chunk 기반) + * + * 기능: + * 1. ship_data 테이블에서 IMO 번호 전체 조회 (최초 1회) + * 2. IMO 번호를 100개씩 분할하여 배치 단위로 처리 + * 3. fetchNextBatch() 호출 시마다 100개씩 API 호출 + * 4. Spring Batch가 100건씩 Process → Write 수행 + * + * Chunk 처리 흐름: + * - beforeFetch() → IMO 전체 조회 (1회) + * - fetchNextBatch() → 100개 IMO로 API 호출 (1,718회) + * - read() → 1건씩 반환 (100번) + * - Processor/Writer → 100건 처리 + * - 반복... (1,718번의 Chunk) + * + * 기존 방식과의 차이: + * - 기존: 17만건 전체 메모리 로드 → Process → Write + * - 신규: 100건씩 로드 → Process → Write (Chunk 1,718회) + */ +@Slf4j +@StepScope +public class TransitsReader extends BaseApiReader { + + private final JdbcTemplate jdbcTemplate; + + // 배치 처리 상태 + private List allImoNumbers; + // DB 해시값을 저장할 맵 + private int currentBatchIndex = 0; + private final int batchSize = 5; + + // @Value("#{jobParameters['startDate']}") +// private String startDate; + private String startDate = "2025-01-01"; + + // @Value("#{jobParameters['stopDate']}") +// private String stopDate; + private String stopDate = "2025-12-31"; + + public TransitsReader(WebClient webClient, JdbcTemplate jdbcTemplate ) { + super(webClient); + this.jdbcTemplate = jdbcTemplate; + enableChunkMode(); // ✨ Chunk 모드 활성화 + } + + @Override + protected String getReaderName() { + return "Transits"; + } + + @Override + protected void resetCustomState() { + this.currentBatchIndex = 0; + this.allImoNumbers = null; + } + + @Override + protected String getApiPath() { + return "/Movements/Transits"; + } + + @Override + protected String getApiBaseUrl() { + return "https://webservices.maritime.spglobal.com"; + } + + private static final String GET_ALL_IMO_QUERY = +// "SELECT imo_number FROM ship_data ORDER BY id"; + "SELECT imo_number FROM snp_data.ship_data where imo_number > (select max(imo) from snp_data.t_transit) ORDER BY imo_number"; + + /** + * 최초 1회만 실행: ship_data 테이블에서 IMO 번호 전체 조회 + */ + @Override + protected void beforeFetch() { + // 전처리 과정 + // Step 1. IMO 전체 번호 조회 + log.info("[{}] ship_data 테이블에서 IMO 번호 조회 시작...", getReaderName()); + + allImoNumbers = jdbcTemplate.queryForList(GET_ALL_IMO_QUERY, String.class); + int totalBatches = (int) Math.ceil((double) allImoNumbers.size() / batchSize); + + log.info("[{}] 총 {} 개의 IMO 번호 조회 완료", getReaderName(), allImoNumbers.size()); + log.info("[{}] {}개씩 배치로 분할하여 API 호출 예정", getReaderName(), batchSize); + log.info("[{}] 예상 배치 수: {} 개", getReaderName(), totalBatches); + + // API 통계 초기화 + updateApiCallStats(totalBatches, 0); + } + + /** + * ✨ Chunk 기반 핵심 메서드: 다음 100개 배치를 조회하여 반환 + * + * Spring Batch가 100건씩 read() 호출 완료 후 이 메서드 재호출 + * + * @return 다음 배치 100건 (더 이상 없으면 null) + */ + @Override + protected List fetchNextBatch() throws Exception { + + // 모든 배치 처리 완료 확인 + if (allImoNumbers == null || currentBatchIndex >= allImoNumbers.size()) { + return null; // Job 종료 + } + + // 현재 배치의 시작/끝 인덱스 계산 + int startIndex = currentBatchIndex; + int endIndex = Math.min(currentBatchIndex + batchSize, allImoNumbers.size()); + + // 현재 배치의 IMO 번호 추출 (100개) + List currentBatch = allImoNumbers.subList(startIndex, endIndex); + + int currentBatchNumber = (currentBatchIndex / batchSize) + 1; + int totalBatches = (int) Math.ceil((double) allImoNumbers.size() / batchSize); + + log.info("[{}] 배치 {}/{} 처리 중 (IMO {} 개)...", + getReaderName(), currentBatchNumber, totalBatches, currentBatch.size()); + + try { + // IMO 번호를 쉼표로 연결 (예: "1000019,1000021,1000033,...") + String imoParam = String.join(",", currentBatch); + + // API 호출 + List response = callApiWithBatch(imoParam); + + // 다음 배치로 인덱스 이동 + currentBatchIndex = endIndex; + + + // 응답 처리 + if (response != null ) { + List transits = response; + log.info("[{}] 배치 {}/{} 완료: {} 건 조회", + getReaderName(), currentBatchNumber, totalBatches, transits.size()); + + // API 호출 통계 업데이트 + updateApiCallStats(totalBatches, currentBatchNumber); + + // API 과부하 방지 (다음 배치 전 0.5초 대기) + if (currentBatchIndex < allImoNumbers.size()) { + Thread.sleep(500); + } + + return transits; + + } else { + log.warn("[{}] 배치 {}/{} 응답 없음", + getReaderName(), currentBatchNumber, totalBatches); + + // API 호출 통계 업데이트 (실패도 카운트) + updateApiCallStats(totalBatches, currentBatchNumber); + + return Collections.emptyList(); + } + + } catch (Exception e) { + log.error("[{}] 배치 {}/{} 처리 중 오류: {}", + getReaderName(), currentBatchNumber, totalBatches, e.getMessage(), e); + + // 오류 발생 시에도 다음 배치로 이동 (부분 실패 허용) + currentBatchIndex = endIndex; + + // 빈 리스트 반환 (Job 계속 진행) + return Collections.emptyList(); + } + } + + /** + * Query Parameter를 사용한 API 호출 + * + * @param lrno 쉼표로 연결된 IMO 번호 (예: "1000019,1000021,...") + * @return API 응답 + */ + private List callApiWithBatch(String lrno) { + String url = getApiPath() + "?startDate=" + startDate +"&stopDate="+stopDate+"&lrno=" + lrno; + + log.debug("[{}] API 호출: {}", getReaderName(), url); + + return webClient.get() + .uri(url) + .retrieve() + .bodyToFlux(TransitsDto.class) + .collectList() + .block(); + } + + @Override + protected void afterFetch(List data) { + if (data == null) { + int totalBatches = (int) Math.ceil((double) allImoNumbers.size() / batchSize); + log.info("[{}] 전체 {} 개 배치 처리 완료", getReaderName(), totalBatches); + log.info("[{}] 총 {} 개의 IMO 번호에 대한 API 호출 종료", + getReaderName(), allImoNumbers.size()); + } + } + +} diff --git a/src/main/java/com/snp/batch/jobs/shipMovementTransits/batch/repository/TransitlsRepositoryImpl.java b/src/main/java/com/snp/batch/jobs/shipMovementTransits/batch/repository/TransitlsRepositoryImpl.java new file mode 100644 index 0000000..f44e00a --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/shipMovementTransits/batch/repository/TransitlsRepositoryImpl.java @@ -0,0 +1,108 @@ +package com.snp.batch.jobs.shipMovementTransits.batch.repository; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.snp.batch.common.batch.repository.BaseJdbcRepository; +import com.snp.batch.jobs.shipMovementTransits.batch.entity.TransitsEntity; +import lombok.extern.slf4j.Slf4j; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.core.RowMapper; +import org.springframework.stereotype.Repository; + +import java.sql.PreparedStatement; +import java.sql.Timestamp; +import java.util.List; + +/** + * 선박 상세 정보 Repository 구현체 + * BaseJdbcRepository를 상속하여 JDBC 기반 CRUD 구현 + */ +@Slf4j +@Repository("TransitsRepository") +public class TransitlsRepositoryImpl extends BaseJdbcRepository + implements TransitsRepository { + + public TransitlsRepositoryImpl(JdbcTemplate jdbcTemplate) { + super(jdbcTemplate); + } + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + @Override + protected String getTableName() { + return "snp_data.t_transit"; + } + + @Override + protected String getEntityName() { + return "Transit"; + } + + @Override + protected String extractId(TransitsEntity entity) { + return entity.getImolRorIHSNumber(); + } + + @Override + public String getInsertSql() { + return """ + INSERT INTO snp_data.t_transit( + imo, + mvmn_type, + mvmn_dt, + fclty_nm, + fclty_type, + draft + ) VALUES (?, ?, ?, ?, ?, ?) + ON CONFLICT (imo, mvmn_dt) + DO UPDATE SET + mvmn_type = EXCLUDED.mvmn_type, + mvmn_dt = EXCLUDED.mvmn_dt, + fclty_nm = EXCLUDED.fclty_nm, + fclty_type = EXCLUDED.fclty_type, + draft = EXCLUDED.draft + """; + } + + @Override + protected String getUpdateSql() { + return null; + } + + @Override + protected void setInsertParameters(PreparedStatement ps, TransitsEntity e) throws Exception { + int i = 1; + ps.setString(i++, e.getImolRorIHSNumber()); // imo + ps.setString(i++, e.getMovementType()); // mvmn_type + ps.setTimestamp(i++, e.getMovementDate() != null ? Timestamp.valueOf(e.getMovementDate()) : null); // mvmn_dt + ps.setString(i++, e.getFacilityName()); // fclty_nm + ps.setString(i++, e.getFacilityType()); // fclty_type + setDoubleOrNull(ps, i++, e.getDraught()); // draft + } + + private void setDoubleOrNull(PreparedStatement ps, int index, Double value) throws Exception { + if (value != null) { + ps.setDouble(index, value); + } else { + // java.sql.Types.DOUBLE을 사용하여 명시적으로 SQL NULL을 설정 + ps.setNull(index, java.sql.Types.DOUBLE); + } + } + + @Override + protected void setUpdateParameters(PreparedStatement ps, TransitsEntity entity) throws Exception { + + } + + @Override + protected RowMapper getRowMapper() { + return null; + } + + @Override + public void saveAll(List entities) { + if (entities == null || entities.isEmpty()) return; + + log.info("Transits 저장 시작 = {}건", entities.size()); + batchInsert(entities); + + } + +} diff --git a/src/main/java/com/snp/batch/jobs/shipMovementTransits/batch/repository/TransitsRepository.java b/src/main/java/com/snp/batch/jobs/shipMovementTransits/batch/repository/TransitsRepository.java new file mode 100644 index 0000000..e134548 --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/shipMovementTransits/batch/repository/TransitsRepository.java @@ -0,0 +1,13 @@ +package com.snp.batch.jobs.shipMovementTransits.batch.repository; + +import com.snp.batch.jobs.shipMovementTransits.batch.entity.TransitsEntity; + +import java.util.List; + +/** + * 선박 상세 정보 Repository 인터페이스 + */ + +public interface TransitsRepository { + void saveAll(List entities); +} diff --git a/src/main/java/com/snp/batch/jobs/shipMovementTransits/batch/writer/TransitsWriter.java b/src/main/java/com/snp/batch/jobs/shipMovementTransits/batch/writer/TransitsWriter.java new file mode 100644 index 0000000..2e72d53 --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/shipMovementTransits/batch/writer/TransitsWriter.java @@ -0,0 +1,35 @@ +package com.snp.batch.jobs.shipMovementTransits.batch.writer; + +import com.snp.batch.common.batch.writer.BaseWriter; +import com.snp.batch.jobs.shipMovementTransits.batch.entity.TransitsEntity; +import com.snp.batch.jobs.shipMovementTransits.batch.repository.TransitsRepository; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; + +import java.util.List; + +/** + * 선박 상세 정보 Writer + */ +@Slf4j +@Component +public class TransitsWriter extends BaseWriter { + + private final TransitsRepository transitsRepository; + + + public TransitsWriter(TransitsRepository transitsRepository) { + super("Transits"); + this.transitsRepository = transitsRepository; + } + + @Override + protected void writeItems(List items) throws Exception { + + if (items.isEmpty()) { return; } + + transitsRepository.saveAll(items); + log.info("Transits 데이터 저장: {} 건", items.size()); + } + +} From 3ee6ae1bf7edf9bfdf6a4a60152d016b0509537e Mon Sep 17 00:00:00 2001 From: Kim JiMyeung Date: Thu, 11 Dec 2025 16:29:28 +0900 Subject: [PATCH 4/9] pscJob --- .../config/MaritimeApiWebClientConfig.java | 2 +- .../batch/config/PscInspectionJobConfig.java | 118 ++++++++ .../batch/dto/PscAllCertificateDto.java | 75 +++++ .../batch/dto/PscApiResponseDto.java | 18 ++ .../batch/dto/PscApsStatusDto.java | 31 ++ .../batch/dto/PscCertificateDto.java | 67 +++++ .../batch/dto/PscDataSetVersionDto.java | 10 + .../pscInspection/batch/dto/PscDefectDto.java | 91 ++++++ .../batch/dto/PscInspectionDto.java | 122 ++++++++ .../batch/entity/PscAllCertificateEntity.java | 48 ++++ .../batch/entity/PscCertificateEntity.java | 45 +++ .../batch/entity/PscDefectEntity.java | 53 ++++ .../batch/entity/PscInspectionEntity.java | 64 +++++ .../processor/PscInspectionProcessor.java | 268 ++++++++++++++++++ .../batch/reader/PscApiReader.java | 173 +++++++++++ .../PscAllCertificateRepository.java | 9 + .../PscAllCertificateRepositoryImpl.java | 146 ++++++++++ .../repository/PscCertificateRepository.java | 10 + .../PscCertificateRepositoryImpl.java | 139 +++++++++ .../batch/repository/PscDefectRepository.java | 10 + .../repository/PscDefectRepositoryImpl.java | 163 +++++++++++ .../repository/PscInspectionRepository.java | 9 + .../PscInspectionRepositoryImpl.java | 186 ++++++++++++ .../batch/writer/PscInspectionWriter.java | 55 ++++ .../batch/reader/BerthCallsReader.java | 2 +- 25 files changed, 1912 insertions(+), 2 deletions(-) create mode 100644 src/main/java/com/snp/batch/jobs/pscInspection/batch/config/PscInspectionJobConfig.java create mode 100644 src/main/java/com/snp/batch/jobs/pscInspection/batch/dto/PscAllCertificateDto.java create mode 100644 src/main/java/com/snp/batch/jobs/pscInspection/batch/dto/PscApiResponseDto.java create mode 100644 src/main/java/com/snp/batch/jobs/pscInspection/batch/dto/PscApsStatusDto.java create mode 100644 src/main/java/com/snp/batch/jobs/pscInspection/batch/dto/PscCertificateDto.java create mode 100644 src/main/java/com/snp/batch/jobs/pscInspection/batch/dto/PscDataSetVersionDto.java create mode 100644 src/main/java/com/snp/batch/jobs/pscInspection/batch/dto/PscDefectDto.java create mode 100644 src/main/java/com/snp/batch/jobs/pscInspection/batch/dto/PscInspectionDto.java create mode 100644 src/main/java/com/snp/batch/jobs/pscInspection/batch/entity/PscAllCertificateEntity.java create mode 100644 src/main/java/com/snp/batch/jobs/pscInspection/batch/entity/PscCertificateEntity.java create mode 100644 src/main/java/com/snp/batch/jobs/pscInspection/batch/entity/PscDefectEntity.java create mode 100644 src/main/java/com/snp/batch/jobs/pscInspection/batch/entity/PscInspectionEntity.java create mode 100644 src/main/java/com/snp/batch/jobs/pscInspection/batch/processor/PscInspectionProcessor.java create mode 100644 src/main/java/com/snp/batch/jobs/pscInspection/batch/reader/PscApiReader.java create mode 100644 src/main/java/com/snp/batch/jobs/pscInspection/batch/repository/PscAllCertificateRepository.java create mode 100644 src/main/java/com/snp/batch/jobs/pscInspection/batch/repository/PscAllCertificateRepositoryImpl.java create mode 100644 src/main/java/com/snp/batch/jobs/pscInspection/batch/repository/PscCertificateRepository.java create mode 100644 src/main/java/com/snp/batch/jobs/pscInspection/batch/repository/PscCertificateRepositoryImpl.java create mode 100644 src/main/java/com/snp/batch/jobs/pscInspection/batch/repository/PscDefectRepository.java create mode 100644 src/main/java/com/snp/batch/jobs/pscInspection/batch/repository/PscDefectRepositoryImpl.java create mode 100644 src/main/java/com/snp/batch/jobs/pscInspection/batch/repository/PscInspectionRepository.java create mode 100644 src/main/java/com/snp/batch/jobs/pscInspection/batch/repository/PscInspectionRepositoryImpl.java create mode 100644 src/main/java/com/snp/batch/jobs/pscInspection/batch/writer/PscInspectionWriter.java diff --git a/src/main/java/com/snp/batch/global/config/MaritimeApiWebClientConfig.java b/src/main/java/com/snp/batch/global/config/MaritimeApiWebClientConfig.java index ddeb443..d05a2f9 100644 --- a/src/main/java/com/snp/batch/global/config/MaritimeApiWebClientConfig.java +++ b/src/main/java/com/snp/batch/global/config/MaritimeApiWebClientConfig.java @@ -64,7 +64,7 @@ public class MaritimeApiWebClientConfig { .defaultHeaders(headers -> headers.setBasicAuth(maritimeApiUsername, maritimeApiPassword)) .codecs(configurer -> configurer .defaultCodecs() - .maxInMemorySize(30 * 1024 * 1024)) // 30MB 버퍼 + .maxInMemorySize(100 * 1024 * 1024)) // 30MB 버퍼 .build(); } diff --git a/src/main/java/com/snp/batch/jobs/pscInspection/batch/config/PscInspectionJobConfig.java b/src/main/java/com/snp/batch/jobs/pscInspection/batch/config/PscInspectionJobConfig.java new file mode 100644 index 0000000..3cf87f0 --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/pscInspection/batch/config/PscInspectionJobConfig.java @@ -0,0 +1,118 @@ +package com.snp.batch.jobs.pscInspection.batch.config; + +import com.snp.batch.common.batch.config.BaseJobConfig; +import com.snp.batch.jobs.pscInspection.batch.dto.PscInspectionDto; +import com.snp.batch.jobs.pscInspection.batch.entity.PscInspectionEntity; +import com.snp.batch.jobs.pscInspection.batch.processor.PscInspectionProcessor; +import com.snp.batch.jobs.pscInspection.batch.reader.PscApiReader; +import com.snp.batch.jobs.pscInspection.batch.writer.PscInspectionWriter; +import lombok.extern.slf4j.Slf4j; +import org.springframework.batch.core.Job; +import org.springframework.batch.core.Step; +import org.springframework.batch.core.configuration.annotation.StepScope; +import org.springframework.batch.core.repository.JobRepository; +import org.springframework.batch.item.ItemProcessor; +import org.springframework.batch.item.ItemReader; +import org.springframework.batch.item.ItemWriter; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.transaction.PlatformTransactionManager; +import org.springframework.web.reactive.function.client.WebClient; +import org.springframework.beans.factory.annotation.Value; + +/** + * 선박 상세 정보 Import Job Config + * + * 특징: + * - ship_data 테이블에서 IMO 번호 조회 + * - IMO 번호를 100개씩 배치로 분할 + * - Maritime API GetShipsByIHSLRorIMONumbers 호출 + * TODO : GetShipsByIHSLRorIMONumbersAll 호출로 변경 + * - 선박 상세 정보를 ship_detail 테이블에 저장 (UPSERT) + * + * 데이터 흐름: + * ShipMovementReader (ship_data → Maritime API) + * ↓ (PortCallDto) + * ShipMovementProcessor + * ↓ (ShipMovementEntity) + * ShipDetailDataWriter + * ↓ (ship_movement 테이블) + */ + +@Slf4j +@Configuration +public class PscInspectionJobConfig extends BaseJobConfig { + + private final PscInspectionProcessor pscInspectionProcessor; + private final PscInspectionWriter pscInspectionWriter; + private final JdbcTemplate jdbcTemplate; + private final WebClient maritimeApiWebClient; + + public PscInspectionJobConfig( + JobRepository jobRepository, + PlatformTransactionManager transactionManager, + PscInspectionProcessor pscInspectionProcessor, + PscInspectionWriter pscInspectionWriter, + JdbcTemplate jdbcTemplate, + @Qualifier("maritimeApiWebClient") WebClient maritimeApiWebClient, PscApiReader pscApiReader) { // ObjectMapper 주입 추가 + super(jobRepository, transactionManager); + this.pscInspectionProcessor = pscInspectionProcessor; + this.pscInspectionWriter = pscInspectionWriter; + this.jdbcTemplate = jdbcTemplate; + this.maritimeApiWebClient = maritimeApiWebClient; + } + + + + @Override + protected String getJobName() { + return "PSCDetailImportJob"; + } + + @Override + protected String getStepName() { + return "PSCDetailImportStep"; + } + + @Bean + @StepScope + public PscApiReader pscApiReader( + @Qualifier("maritimeApiWebClient") WebClient webClient, + @Value("#{jobParameters['fromDate']}") String fromDate, + @Value("#{jobParameters['toDate']}") String toDate + ) { + return new PscApiReader(webClient, fromDate, toDate); + } + + @Override + protected ItemReader createReader() { + return pscApiReader(null, null, null); + } + + @Override + protected ItemProcessor createProcessor() { + return pscInspectionProcessor; + } + + @Override + protected ItemWriter createWriter() { // 타입 변경 + return pscInspectionWriter; + } + + @Override + protected int getChunkSize() { + return 10; // API에서 100개씩 가져오므로 chunk도 100으로 설정 + } + + @Bean(name = "PSCDetailImportJob") + public Job PSCDetailImportJob() { + return job(); + } + + @Bean(name = "PSCDetailImportStep") + public Step PSCDetailImportStep() { + return step(); + } +} diff --git a/src/main/java/com/snp/batch/jobs/pscInspection/batch/dto/PscAllCertificateDto.java b/src/main/java/com/snp/batch/jobs/pscInspection/batch/dto/PscAllCertificateDto.java new file mode 100644 index 0000000..42540d6 --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/pscInspection/batch/dto/PscAllCertificateDto.java @@ -0,0 +1,75 @@ +package com.snp.batch.jobs.pscInspection.batch.dto; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; + +@Data +public class PscAllCertificateDto { + + @JsonProperty("Type_Id") + private String typeId; + + @JsonProperty("DataSetVersion") + private PscDataSetVersionDto dataSetVersion; + + @JsonProperty("Certificate_ID") + private String certificateId; + + @JsonProperty("Inspection_ID") + private String inspectionId; + + @JsonProperty("Lrno") + private String lrno; + + @JsonProperty("Certificate_Title_Code") + private String certificateTitleCode; + + @JsonProperty("Certificate_Title") + private String certificateTitle; + + @JsonProperty("Issuing_Authority_Code") + private String issuingAuthorityCode; + + @JsonProperty("Issuing_Authority") + private String issuingAuthority; + + @JsonProperty("Class_Soc_of_Issuer") + private String classSocOfIssuer; + + @JsonProperty("Other_Issuing_Authority") + private String otherIssuingAuthority; + + @JsonProperty("Issue_Date") + private String issueDate; + + @JsonProperty("Expiry_Date") + private String expiryDate; + + @JsonProperty("Last_Survey_Date") + private String lastSurveyDate; + + @JsonProperty("Survey_Authority_Code") + private String surveyAuthorityCode; + + @JsonProperty("Survey_Authority") + private String surveyAuthority; + + @JsonProperty("Other_Survey_Authority") + private String otherSurveyAuthority; + + @JsonProperty("Latest_Survey_Place") + private String latestSurveyPlace; + + @JsonProperty("Latest_Survey_Place_Code") + private String latestSurveyPlaceCode; + + @JsonProperty("Survey_Authority_Type") + private String surveyAuthorityType; + + @JsonProperty("Inspection_Date") + private String inspectionDate; + + @JsonProperty("Inspected_By") + private String inspectedBy; + +} diff --git a/src/main/java/com/snp/batch/jobs/pscInspection/batch/dto/PscApiResponseDto.java b/src/main/java/com/snp/batch/jobs/pscInspection/batch/dto/PscApiResponseDto.java new file mode 100644 index 0000000..1d3af2a --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/pscInspection/batch/dto/PscApiResponseDto.java @@ -0,0 +1,18 @@ +package com.snp.batch.jobs.pscInspection.batch.dto; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; + +import java.util.List; + +@Data +public class PscApiResponseDto { + + @JsonProperty("Inspections") + private List inspections; + @JsonProperty("inspectionCount") + private Integer inspectionCount; + + @JsonProperty("APSStatus") + private PscApsStatusDto apsStatus; +} diff --git a/src/main/java/com/snp/batch/jobs/pscInspection/batch/dto/PscApsStatusDto.java b/src/main/java/com/snp/batch/jobs/pscInspection/batch/dto/PscApsStatusDto.java new file mode 100644 index 0000000..8e7f20f --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/pscInspection/batch/dto/PscApsStatusDto.java @@ -0,0 +1,31 @@ +package com.snp.batch.jobs.pscInspection.batch.dto; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; + +@Data +public class PscApsStatusDto { + @JsonProperty("SystemVersion") + private String systemVersion; + + @JsonProperty("SystemDate") + private String systemDate; + + @JsonProperty("JobRunDate") + private String jobRunDate; + + @JsonProperty("CompletedOK") + private Boolean completedOK; + + @JsonProperty("ErrorLevel") + private String errorLevel; + + @JsonProperty("ErrorMessage") + private String errorMessage; + + @JsonProperty("RemedialAction") + private String remedialAction; + + @JsonProperty("Guid") + private String guid; +} \ No newline at end of file diff --git a/src/main/java/com/snp/batch/jobs/pscInspection/batch/dto/PscCertificateDto.java b/src/main/java/com/snp/batch/jobs/pscInspection/batch/dto/PscCertificateDto.java new file mode 100644 index 0000000..7a248b4 --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/pscInspection/batch/dto/PscCertificateDto.java @@ -0,0 +1,67 @@ +package com.snp.batch.jobs.pscInspection.batch.dto; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; + +@Data +public class PscCertificateDto { + @JsonProperty("Type_Id") + private String typeId; + + @JsonProperty("DataSetVersion") + private PscDataSetVersionDto dataSetVersion; + + @JsonProperty("Certificate_ID") + private String certificateId; + + @JsonProperty("Certificate_Title") + private String certificateTitle; + + @JsonProperty("Certificate_Title_Code") + private String certificateTitleCode; + + @JsonProperty("Class_SOC_Of_Issuer") + private String classSocOfIssuer; + + @JsonProperty("Expiry_Date") + private String expiryDate; // ISO 날짜 문자열 그대로 받음 + + @JsonProperty("Inspection_ID") + private String inspectionId; + + @JsonProperty("Issue_Date") + private String issueDate; + + @JsonProperty("Issuing_Authority") + private String issuingAuthority; + + @JsonProperty("Issuing_Authority_Code") + private String issuingAuthorityCode; + + @JsonProperty("Last_Survey_Date") + private String lastSurveyDate; + + @JsonProperty("Latest_Survey_Place") + private String latestSurveyPlace; + + @JsonProperty("Latest_Survey_Place_Code") + private String latestSurveyPlaceCode; + + @JsonProperty("Lrno") + private String lrno; + + @JsonProperty("Other_Issuing_Authority") + private String otherIssuingAuthority; + + @JsonProperty("Other_Survey_Authority") + private String otherSurveyAuthority; + + @JsonProperty("Survey_Authority") + private String surveyAuthority; + + @JsonProperty("Survey_Authority_Code") + private String surveyAuthorityCode; + + @JsonProperty("Survey_Authority_Type") + private String surveyAuthorityType; +} diff --git a/src/main/java/com/snp/batch/jobs/pscInspection/batch/dto/PscDataSetVersionDto.java b/src/main/java/com/snp/batch/jobs/pscInspection/batch/dto/PscDataSetVersionDto.java new file mode 100644 index 0000000..41fe574 --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/pscInspection/batch/dto/PscDataSetVersionDto.java @@ -0,0 +1,10 @@ +package com.snp.batch.jobs.pscInspection.batch.dto; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; + +@Data +public class PscDataSetVersionDto { + @JsonProperty("DataSetVersion") + private String dataSetVersion; +} diff --git a/src/main/java/com/snp/batch/jobs/pscInspection/batch/dto/PscDefectDto.java b/src/main/java/com/snp/batch/jobs/pscInspection/batch/dto/PscDefectDto.java new file mode 100644 index 0000000..3557833 --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/pscInspection/batch/dto/PscDefectDto.java @@ -0,0 +1,91 @@ +package com.snp.batch.jobs.pscInspection.batch.dto; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; + +@Data +public class PscDefectDto { + @JsonProperty("Type_Id") + private String typeId; + + @JsonProperty("DataSetVersion") + private PscDataSetVersionDto dataSetVersion; + + @JsonProperty("Action_1") + private String action1; + + @JsonProperty("Action_2") + private String action2; + + @JsonProperty("Action_3") + private String action3; + + @JsonProperty("Action_Code_1") + private String actionCode1; + + @JsonProperty("Action_Code_2") + private String actionCode2; + + @JsonProperty("Action_Code_3") + private String actionCode3; + + @JsonProperty("AmsA_Action_Code_1") + private String amsaActionCode1; + + @JsonProperty("AmsA_Action_Code_2") + private String amsaActionCode2; + + @JsonProperty("AmsA_Action_Code_3") + private String amsaActionCode3; + + @JsonProperty("Class_Is_Responsible") + private String classIsResponsible; + + @JsonProperty("Defect_Code") + private String defectCode; + + @JsonProperty("Defect_ID") + private String defectId; + + @JsonProperty("Defect_Text") + private String defectText; + + @JsonProperty("Defective_Item_Code") + private String defectiveItemCode; + + @JsonProperty("Detention_Reason_Deficiency") + private String detentionReasonDeficiency; + + @JsonProperty("Inspection_ID") + private String inspectionId; + + @JsonProperty("Main_Defect_Code") + private String mainDefectCode; + + @JsonProperty("Main_Defect_Text") + private String mainDefectText; + + @JsonProperty("Nature_Of_Defect_Code") + private String natureOfDefectCode; + + @JsonProperty("Nature_Of_Defect_DeCode") + private String natureOfDefectDecode; + + @JsonProperty("Other_Action") + private String otherAction; + + @JsonProperty("Other_Recognised_Org_Resp") + private String otherRecognisedOrgResp; + + @JsonProperty("Recognised_Org_Resp") + private String recognisedOrgResp; + + @JsonProperty("Recognised_Org_Resp_Code") + private String recognisedOrgRespCode; + + @JsonProperty("Recognised_Org_Resp_YN") + private String recognisedOrgRespYn; + + @JsonProperty("IsAccidentalDamage") + private String isAccidentalDamage; +} diff --git a/src/main/java/com/snp/batch/jobs/pscInspection/batch/dto/PscInspectionDto.java b/src/main/java/com/snp/batch/jobs/pscInspection/batch/dto/PscInspectionDto.java new file mode 100644 index 0000000..6d8ef89 --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/pscInspection/batch/dto/PscInspectionDto.java @@ -0,0 +1,122 @@ +package com.snp.batch.jobs.pscInspection.batch.dto; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; + +import java.math.BigDecimal; +import java.util.List; + +@Data +public class PscInspectionDto { + + @JsonProperty("typeId") + private String typeId; + + @JsonProperty("DataSetVersion") + private PscDataSetVersionDto dataSetVersion; + + @JsonProperty("Authorisation") + private String authorisation; + + @JsonProperty("CallSign") + private String callSign; + + @JsonProperty("Cargo") + private String cargo; + + @JsonProperty("Charterer") + private String charterer; + + @JsonProperty("Class") + private String shipClass; + + @JsonProperty("Country") + private String country; + + @JsonProperty("Inspection_Date") + private String inspectionDate; + + @JsonProperty("Release_Date") + private String releaseDate; + + @JsonProperty("Ship_Detained") + private String shipDetained; + + @JsonProperty("Dead_Weight") + private String deadWeight; + + @JsonProperty("Expanded_Inspection") + private String expandedInspection; + + @JsonProperty("Flag") + private String flag; + + @JsonProperty("Follow_Up_Inspection") + private String followUpInspection; + + @JsonProperty("Gross_Tonnage") + private String grossTonnage; + + @JsonProperty("Inspection_ID") + private String inspectionId; + + @JsonProperty("Inspection_Port_Code") + private String inspectionPortCode; + + @JsonProperty("Inspection_Port_Decode") + private String inspectionPortDecode; + + @JsonProperty("Keel_Laid") + private String keelLaid; + + @JsonProperty("Last_Updated") + private String lastUpdated; + + @JsonProperty("IHSLR_or_IMO_Ship_No") + private String ihslrOrImoShipNo; + + @JsonProperty("Manager") + private String manager; + + @JsonProperty("Number_Of_Days_Detained") + private Integer numberOfDaysDetained; + + @JsonProperty("Number_Of_Defects") + private String numberOfDefects; + + @JsonProperty("Number_Of_Part_Days_Detained") + private BigDecimal numberOfPartDaysDetained; + + @JsonProperty("Other_Inspection_Type") + private String otherInspectionType; + + @JsonProperty("Owner") + private String owner; + + @JsonProperty("Ship_Name") + private String shipName; + + @JsonProperty("Ship_Type_Code") + private String shipTypeCode; + + @JsonProperty("Ship_Type_Decode") + private String shipTypeDecode; + + @JsonProperty("Source") + private String source; + + @JsonProperty("UNLOCODE") + private String unlocode; + + @JsonProperty("Year_Of_Build") + private String yearOfBuild; + + @JsonProperty("PSCDefects") + private List pscDefects; + + @JsonProperty("PSCCertificates") + private List pscCertificates; + + @JsonProperty("PSCAllCertificates") + private List pscAllCertificates; +} diff --git a/src/main/java/com/snp/batch/jobs/pscInspection/batch/entity/PscAllCertificateEntity.java b/src/main/java/com/snp/batch/jobs/pscInspection/batch/entity/PscAllCertificateEntity.java new file mode 100644 index 0000000..f750520 --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/pscInspection/batch/entity/PscAllCertificateEntity.java @@ -0,0 +1,48 @@ +package com.snp.batch.jobs.pscInspection.batch.entity; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +import java.time.LocalDateTime; + +@Data +@SuperBuilder +@NoArgsConstructor +@AllArgsConstructor +public class PscAllCertificateEntity { + + private String certificateId; + + private String typeId; + private String dataSetVersion; + + private String inspectionId; + private String lrno; + + private String certificateTitleCode; + private String certificateTitle; + + private String issuingAuthorityCode; + private String issuingAuthority; + + private String classSocOfIssuer; + private String otherIssuingAuthority; + + private LocalDateTime issueDate; + private LocalDateTime expiryDate; + private LocalDateTime lastSurveyDate; + + private String surveyAuthorityCode; + private String surveyAuthority; + private String otherSurveyAuthority; + + private String latestSurveyPlace; + private String latestSurveyPlaceCode; + + private String surveyAuthorityType; + + private String inspectionDate; + private String inspectedBy; +} diff --git a/src/main/java/com/snp/batch/jobs/pscInspection/batch/entity/PscCertificateEntity.java b/src/main/java/com/snp/batch/jobs/pscInspection/batch/entity/PscCertificateEntity.java new file mode 100644 index 0000000..d360916 --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/pscInspection/batch/entity/PscCertificateEntity.java @@ -0,0 +1,45 @@ +package com.snp.batch.jobs.pscInspection.batch.entity; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +import java.time.LocalDateTime; + +@Data +@SuperBuilder +@NoArgsConstructor +@AllArgsConstructor +public class PscCertificateEntity { + + private String certificateId; + + private String typeId; + private String dataSetVersion; + + private String certificateTitle; + private String certificateTitleCode; + + private String classSocOfIssuer; + + private LocalDateTime expiryDate; + private String inspectionId; + private LocalDateTime issueDate; + + private String issuingAuthority; + private String issuingAuthorityCode; + + private LocalDateTime lastSurveyDate; + private String latestSurveyPlace; + private String latestSurveyPlaceCode; + + private String lrno; + + private String otherIssuingAuthority; + private String otherSurveyAuthority; + + private String surveyAuthority; + private String surveyAuthorityCode; + private String surveyAuthorityType; +} diff --git a/src/main/java/com/snp/batch/jobs/pscInspection/batch/entity/PscDefectEntity.java b/src/main/java/com/snp/batch/jobs/pscInspection/batch/entity/PscDefectEntity.java new file mode 100644 index 0000000..e84278e --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/pscInspection/batch/entity/PscDefectEntity.java @@ -0,0 +1,53 @@ +package com.snp.batch.jobs.pscInspection.batch.entity; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +@Data +@SuperBuilder +@NoArgsConstructor +@AllArgsConstructor +public class PscDefectEntity { + + private String defectId; + + private String typeId; + private String dataSetVersion; + + private String action1; + private String action2; + private String action3; + private String actionCode1; + private String actionCode2; + private String actionCode3; + + private String amsaActionCode1; + private String amsaActionCode2; + private String amsaActionCode3; + + private String classIsResponsible; + + private String defectCode; + private String defectText; + + private String defectiveItemCode; + private String detentionReasonDeficiency; + + private String inspectionId; + + private String mainDefectCode; + private String mainDefectText; + + private String natureOfDefectCode; + private String natureOfDefectDecode; + + private String otherAction; + private String otherRecognisedOrgResp; + private String recognisedOrgResp; + private String recognisedOrgRespCode; + private String recognisedOrgRespYn; + + private String isAccidentalDamage; +} diff --git a/src/main/java/com/snp/batch/jobs/pscInspection/batch/entity/PscInspectionEntity.java b/src/main/java/com/snp/batch/jobs/pscInspection/batch/entity/PscInspectionEntity.java new file mode 100644 index 0000000..040f3ae --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/pscInspection/batch/entity/PscInspectionEntity.java @@ -0,0 +1,64 @@ +package com.snp.batch.jobs.pscInspection.batch.entity; + +import com.snp.batch.jobs.pscInspection.batch.dto.PscAllCertificateDto; +import com.snp.batch.jobs.pscInspection.batch.dto.PscCertificateDto; +import com.snp.batch.jobs.pscInspection.batch.dto.PscDefectDto; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +import java.math.BigDecimal; +import java.time.LocalDateTime; +import java.util.List; + +@Data +@SuperBuilder +@NoArgsConstructor +@AllArgsConstructor +public class PscInspectionEntity { + + private String typeId; + private String dataSetVersion; + private String authorisation; + private String callSign; + private String shipClass; + private String cargo; + private String charterer; + private String country; + private LocalDateTime inspectionDate; + private LocalDateTime releaseDate; + private String shipDetained; + private String deadWeight; + private String expandedInspection; + private String flag; + private String followUpInspection; + private String grossTonnage; + + private String inspectionId; + + private String inspectionPortCode; + private String inspectionPortDecode; + + private String keelLaid; + private LocalDateTime lastUpdated; + private String ihslrOrImoShipNo; + private String manager; + + private Integer numberOfDaysDetained; + private String numberOfDefects; + private BigDecimal numberOfPartDaysDetained; + + private String otherInspectionType; + private String owner; + private String shipName; + private String shipTypeCode; + private String shipTypeDecode; + private String source; + private String unlocode; + private String yearOfBuild; + + private List defects; + private List certificates; + private List allCertificates; +} diff --git a/src/main/java/com/snp/batch/jobs/pscInspection/batch/processor/PscInspectionProcessor.java b/src/main/java/com/snp/batch/jobs/pscInspection/batch/processor/PscInspectionProcessor.java new file mode 100644 index 0000000..a47362d --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/pscInspection/batch/processor/PscInspectionProcessor.java @@ -0,0 +1,268 @@ +package com.snp.batch.jobs.pscInspection.batch.processor; + +import com.snp.batch.common.batch.processor.BaseProcessor; +import com.snp.batch.jobs.pscInspection.batch.dto.*; +import com.snp.batch.jobs.pscInspection.batch.entity.PscAllCertificateEntity; +import com.snp.batch.jobs.pscInspection.batch.entity.PscCertificateEntity; +import com.snp.batch.jobs.pscInspection.batch.entity.PscDefectEntity; +import com.snp.batch.jobs.pscInspection.batch.entity.PscInspectionEntity; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; + +import java.math.BigDecimal; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.OffsetDateTime; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +import static jakarta.xml.bind.DatatypeConverter.parseDateTime; + +@Slf4j +@Component +public class PscInspectionProcessor extends BaseProcessor { + + @Override + public PscInspectionEntity processItem(PscInspectionDto item) throws Exception { + + PscInspectionEntity entity = new PscInspectionEntity(); + + entity.setTypeId(s(item.getTypeId())); + entity.setDataSetVersion(item.getDataSetVersion() != null ? s(item.getDataSetVersion().getDataSetVersion()) : null); + entity.setAuthorisation(s(item.getAuthorisation())); + entity.setCallSign(s(item.getCallSign())); + entity.setShipClass(s(item.getShipClass())); + entity.setCargo(s(item.getCargo())); + entity.setCharterer(s(item.getCharterer())); + entity.setCountry(s(item.getCountry())); + + entity.setInspectionDate(dt(item.getInspectionDate())); + entity.setReleaseDate(dt(item.getReleaseDate())); + entity.setShipDetained(s(item.getShipDetained())); + entity.setDeadWeight(s(item.getDeadWeight())); + + entity.setExpandedInspection(s(item.getExpandedInspection())); + entity.setFlag(s(item.getFlag())); + entity.setFollowUpInspection(s(item.getFollowUpInspection())); + entity.setGrossTonnage(s(item.getGrossTonnage())); + + entity.setInspectionId(s(item.getInspectionId())); + entity.setInspectionPortCode(s(item.getInspectionPortCode())); + entity.setInspectionPortDecode(s(item.getInspectionPortDecode())); + + entity.setKeelLaid(s(item.getKeelLaid())); + entity.setLastUpdated(dt(item.getLastUpdated())); + entity.setIhslrOrImoShipNo(s(item.getIhslrOrImoShipNo())); + entity.setManager(s(item.getManager())); + + entity.setNumberOfDaysDetained(i(item.getNumberOfDaysDetained())); + entity.setNumberOfDefects(s(item.getNumberOfDefects())); + entity.setNumberOfPartDaysDetained(bd(item.getNumberOfPartDaysDetained())); + + entity.setOtherInspectionType(s(item.getOtherInspectionType())); + entity.setOwner(s(item.getOwner())); + entity.setShipName(s(item.getShipName())); + entity.setShipTypeCode(s(item.getShipTypeCode())); + entity.setShipTypeDecode(s(item.getShipTypeDecode())); + entity.setSource(s(item.getSource())); + entity.setUnlocode(s(item.getUnlocode())); + entity.setYearOfBuild(s(item.getYearOfBuild())); + + // 리스트 null-safe + entity.setDefects(item.getPscDefects() == null ? List.of() : convertDefectDtos(item.getPscDefects())); + entity.setCertificates(item.getPscCertificates() == null ? List.of() : convertCertificateDtos(item.getPscCertificates())); + entity.setAllCertificates(item.getPscAllCertificates() == null ? List.of() : convertAllCertificateDtos(item.getPscAllCertificates())); + + + return entity; + } + + + /** ----------------------- 공통 메서드 ----------------------- */ + + private String s(Object v) { + return (v == null) ? null : v.toString().trim(); + } + + private Boolean b(Object v) { + if (v == null) return null; + String s = v.toString().trim().toLowerCase(); + if (s.equals("true") || s.equals("t") || s.equals("1")) return true; + if (s.equals("false") || s.equals("f") || s.equals("0")) return false; + return null; + } + private BigDecimal bd(Object v) { + if (v == null) return null; + try { + return new BigDecimal(v.toString().trim()); + } catch (Exception e) { + return null; + } + } + private Integer i(Object v) { + if (v == null) return null; + try { + return Integer.parseInt(v.toString().trim()); + } catch (Exception e) { + return null; + } + } + + private Double d(Object v) { + if (v == null) return null; + try { + return Double.parseDouble(v.toString().trim()); + } catch (Exception e) { + return null; + } + } + + private LocalDateTime dt(String dateStr) { + if (dateStr == null || dateStr.isBlank()) return null; + + // 가장 흔한 ISO 형태 + try { + return LocalDateTime.parse(dateStr); + } catch (Exception ignored) {} + + // yyyy-MM-dd + try { + return LocalDate.parse(dateStr).atStartOfDay(); + } catch (Exception ignored) {} + + // yyyy-MM-dd HH:mm:ss + try { + return LocalDateTime.parse(dateStr, + DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")); + } catch (Exception ignored) {} + + // yyyy-MM-ddTHH:mm:ssZ 형태 + try { + return OffsetDateTime.parse(dateStr).toLocalDateTime(); + } catch (Exception ignored) {} + + log.warn("⚠️ 날짜 변환 실패 → {}", dateStr); + return null; + } + + public static List convertDefectDtos(List dtos) { + if (dtos == null || dtos.isEmpty()) return List.of(); + + return dtos.stream() + .map(dto -> PscDefectEntity.builder() + .defectId(dto.getDefectId()) + .inspectionId(dto.getInspectionId()) + .typeId(dto.getTypeId()) + .dataSetVersion(dto.getDataSetVersion() != null ? dto.getDataSetVersion().getDataSetVersion() : null) + .action1(dto.getAction1()) + .action2(dto.getAction2()) + .action3(dto.getAction3()) + .actionCode1(dto.getActionCode1()) + .actionCode2(dto.getActionCode2()) + .actionCode3(dto.getActionCode3()) + .amsaActionCode1(dto.getAmsaActionCode1()) + .amsaActionCode2(dto.getAmsaActionCode2()) + .amsaActionCode3(dto.getAmsaActionCode3()) + .classIsResponsible(dto.getClassIsResponsible()) + .defectCode(dto.getDefectCode()) + .defectText(dto.getDefectText()) + .defectiveItemCode(dto.getDefectiveItemCode()) + .detentionReasonDeficiency(dto.getDetentionReasonDeficiency()) + .mainDefectCode(dto.getMainDefectCode()) + .mainDefectText(dto.getMainDefectText()) + .natureOfDefectCode(dto.getNatureOfDefectCode()) + .natureOfDefectDecode(dto.getNatureOfDefectDecode()) + .otherAction(dto.getOtherAction()) + .otherRecognisedOrgResp(dto.getOtherRecognisedOrgResp()) + .recognisedOrgResp(dto.getRecognisedOrgResp()) + .recognisedOrgRespCode(dto.getRecognisedOrgRespCode()) + .recognisedOrgRespYn(dto.getRecognisedOrgRespYn()) + .isAccidentalDamage(dto.getIsAccidentalDamage()) + .build()) + .collect(Collectors.toList()); + } + private List convertCertificateDtos(List dtos) { + if (dtos == null || dtos.isEmpty()) return List.of(); + + return dtos.stream() + .map(dto -> PscCertificateEntity.builder() + .certificateId(dto.getCertificateId()) + .typeId(dto.getTypeId()) + .dataSetVersion(dto.getDataSetVersion() != null ? dto.getDataSetVersion().getDataSetVersion() : null) + .certificateTitle(dto.getCertificateTitle()) + .certificateTitleCode(dto.getCertificateTitleCode()) + .classSocOfIssuer(dto.getClassSocOfIssuer()) + .issueDate(dto.getIssueDate() != null ? parseFlexible(dto.getIssueDate()) : null) + .expiryDate(dto.getExpiryDate() != null ? parseFlexible(dto.getExpiryDate()) : null) + .inspectionId(dto.getInspectionId()) + .issuingAuthority(dto.getIssuingAuthority()) + .issuingAuthorityCode(dto.getIssuingAuthorityCode()) + .lastSurveyDate(dto.getLastSurveyDate() != null ? parseFlexible(dto.getLastSurveyDate()) : null) + .latestSurveyPlace(dto.getLatestSurveyPlace()) + .latestSurveyPlaceCode(dto.getLatestSurveyPlaceCode()) + .lrno(dto.getLrno()) + .otherIssuingAuthority(dto.getOtherIssuingAuthority()) + .otherSurveyAuthority(dto.getOtherSurveyAuthority()) + .surveyAuthority(dto.getSurveyAuthority()) + .surveyAuthorityCode(dto.getSurveyAuthorityCode()) + .surveyAuthorityType(dto.getSurveyAuthorityType()) + .build()) + .collect(Collectors.toList()); + } + + public static List convertAllCertificateDtos(List dtos) { + if (dtos == null || dtos.isEmpty()) return List.of(); + + return dtos.stream() + .map(dto -> PscAllCertificateEntity.builder() + .certificateId(dto.getCertificateId()) + .typeId(dto.getTypeId()) + .dataSetVersion(dto.getDataSetVersion() != null ? dto.getDataSetVersion().getDataSetVersion() : null) + .inspectionId(dto.getInspectionId()) + .lrno(dto.getLrno()) + .certificateTitleCode(dto.getCertificateTitleCode()) + .certificateTitle(dto.getCertificateTitle()) + .issuingAuthorityCode(dto.getIssuingAuthorityCode()) + .issuingAuthority(dto.getIssuingAuthority()) + .classSocOfIssuer(dto.getClassSocOfIssuer()) + .otherIssuingAuthority(dto.getOtherIssuingAuthority()) + .issueDate(dto.getIssueDate() != null ? parseFlexible(dto.getIssueDate()) : null) + .expiryDate(dto.getExpiryDate() != null ? parseFlexible(dto.getExpiryDate()) : null) + .lastSurveyDate(dto.getLastSurveyDate() != null ? parseFlexible(dto.getLastSurveyDate()) : null) + .surveyAuthorityCode(dto.getSurveyAuthorityCode()) + .surveyAuthority(dto.getSurveyAuthority()) + .otherSurveyAuthority(dto.getOtherSurveyAuthority()) + .latestSurveyPlace(dto.getLatestSurveyPlace()) + .latestSurveyPlaceCode(dto.getLatestSurveyPlaceCode()) + .surveyAuthorityType(dto.getSurveyAuthorityType()) + .inspectionDate(dto.getInspectionDate()) + .inspectedBy(dto.getInspectedBy()) + .build()) + .collect(Collectors.toList()); + } + + private static final List FORMATTERS = Arrays.asList( + DateTimeFormatter.ofPattern("dd/MM/yyyy HH:mm:ss"), + DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss"), + DateTimeFormatter.ISO_LOCAL_DATE_TIME + ); + + public static LocalDateTime parseFlexible(String dateStr) { + if (dateStr == null || dateStr.isEmpty()) return null; + + for (DateTimeFormatter formatter : FORMATTERS) { + try { + return LocalDateTime.parse(dateStr, formatter); + } catch (DateTimeParseException ignored) { + // 포맷 실패 시 다음 시도 + } + } + // 모두 실패 시 null 반환 + System.err.println("날짜 파싱 실패: " + dateStr); + return null; + } +} diff --git a/src/main/java/com/snp/batch/jobs/pscInspection/batch/reader/PscApiReader.java b/src/main/java/com/snp/batch/jobs/pscInspection/batch/reader/PscApiReader.java new file mode 100644 index 0000000..70a8f81 --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/pscInspection/batch/reader/PscApiReader.java @@ -0,0 +1,173 @@ +package com.snp.batch.jobs.pscInspection.batch.reader; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.snp.batch.common.batch.reader.BaseApiReader; +import com.snp.batch.jobs.pscInspection.batch.dto.PscApiResponseDto; +import com.snp.batch.jobs.pscInspection.batch.dto.PscInspectionDto; +import lombok.extern.slf4j.Slf4j; +import org.springframework.batch.core.configuration.annotation.StepScope; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.web.reactive.function.client.WebClient; + +import java.util.Collections; +import java.util.List; + +@Slf4j +@StepScope +public class PscApiReader extends BaseApiReader { + + //private final JdbcTemplate jdbcTemplate; + + private final String fromDate; + private final String toDate; +// private List allImoNumbers; + private List allData; + private int currentBatchIndex = 0; + private final int batchSize = 10; + + public PscApiReader(@Qualifier("maritimeApiWebClient") WebClient webClient, + @Value("#{jobParameters['fromDate']}") String fromDate, + @Value("#{jobParameters['toDate']}") String toDate) { + super(webClient); + //this.jdbcTemplate = jdbcTemplate; + this.fromDate = fromDate; + this.toDate = toDate; + enableChunkMode(); + } + + @Override + protected String getReaderName() { + return "PscApiReader"; + } + + @Override + protected void resetCustomState() { + this.currentBatchIndex = 0; +// this.allImoNumbers = null; + } + + @Override + protected String getApiPath() { + return "/MaritimeWCF/PSCService.svc/RESTFul/GetPSCDataByLastUpdateDateRange"; + } + + private static final String GET_ALL_IMO_QUERY = + "SELECT imo_number FROM ship_data ORDER BY id"; +// "SELECT imo_number FROM snp_data.ship_data where imo_number > (select max(imo) from snp_data.t_berthcalls) ORDER BY imo_number"; + + @Override + protected void beforeFetch() { + // 전처리 과정 + // Step 1. IMO 전체 번호 조회 + /*log.info("[{}] ship_data 테이블에서 IMO 번호 조회 시작...", getReaderName()); + + allImoNumbers = jdbcTemplate.queryForList(GET_ALL_IMO_QUERY, String.class); + int totalBatches = (int) Math.ceil((double) allImoNumbers.size() / batchSize); + + log.info("[{}] 총 {} 개의 IMO 번호 조회 완료", getReaderName(), allImoNumbers.size()); + log.info("[{}] {}개씩 배치로 분할하여 API 호출 예정", getReaderName(), batchSize); + log.info("[{}] 예상 배치 수: {} 개", getReaderName(), totalBatches); + + // API 통계 초기화 + updateApiCallStats(totalBatches, 0);*/ + log.info("[PSC] 요청 날짜 범위: {} → {}", fromDate, toDate); + } + + + @Override + protected List fetchNextBatch() { + + + // 1) 처음 호출이면 API 한 번 호출해서 전체 데이터를 가져온다 + if (allData == null) { + log.info("[PSC] 최초 API 조회 실행: {} ~ {}", fromDate, toDate); + allData = callApiWithBatch(fromDate, toDate); + + if (allData == null || allData.isEmpty()) { + log.warn("[PSC] 조회된 데이터 없음 → 종료"); + return null; + } + + log.info("[PSC] 총 {}건 데이터 조회됨. batchSize = {}", allData.size(), batchSize); + } + + // 2) 이미 끝까지 읽었으면 종료 + if (currentBatchIndex >= allData.size()) { + log.info("[PSC] 모든 배치 처리 완료"); + return null; // Step 종료 신호 + } + + // 3) 이번 배치의 end 계산 + int end = Math.min(currentBatchIndex + batchSize, allData.size()); + + // 4) 현재 batch 리스트 잘라서 반환 + List batch = allData.subList(currentBatchIndex, end); + + int batchNum = (currentBatchIndex / batchSize) + 1; + int totalBatches = (int) Math.ceil((double) allData.size() / batchSize); + + log.info("[PSC] 배치 {}/{} 처리 중: {}건", batchNum, totalBatches, batch.size()); + + // 다음 batch 인덱스 이동 + currentBatchIndex = end; + + return batch; + } + + // private List callApiWithBatch(String lrno) { + private List callApiWithBatch(String from, String to) { + + String[] f = from.split("-"); + String[] t = to.split("-"); + + String url = getApiPath() + + "?shipsCategory=0" + + "&fromYear=" + f[0] + + "&fromMonth=" + f[1] + + "&fromDay=" + f[2] + + "&toYear=" + t[0] + + "&toMonth=" + t[1] + + "&toDay=" + t[2]; + + log.info("[PSC] API 호출 URL = {}", url); + + String json = webClient.get() + .uri(url) + .retrieve() + .bodyToMono(String.class) + .block(); + + if (json == null || json.isBlank()) { + log.warn("[PSC] API 응답 없음"); + return Collections.emptyList(); + } + + try { + ObjectMapper mapper = new ObjectMapper(); + PscApiResponseDto resp = mapper.readValue(json, PscApiResponseDto.class); + + if (resp.getInspections() == null) { + log.warn("[PSC] inspections 필드 없음"); + return Collections.emptyList(); + } + + return resp.getInspections(); + + } catch (Exception e) { + log.error("[PSC] JSON 파싱 실패: {}", e.getMessage()); + return Collections.emptyList(); + } + } + + @Override + protected void afterFetch(List data) { + if (data == null) { + int totalBatches = (int) Math.ceil((double) allData.size() / batchSize); + log.info("[{}] 전체 {} 개 배치 처리 완료", getReaderName(), totalBatches); + log.info("[{}] 총 {} 개의 IMO 번호에 대한 API 호출 종료", + getReaderName(), allData.size()); + } + } +} diff --git a/src/main/java/com/snp/batch/jobs/pscInspection/batch/repository/PscAllCertificateRepository.java b/src/main/java/com/snp/batch/jobs/pscInspection/batch/repository/PscAllCertificateRepository.java new file mode 100644 index 0000000..5d4586a --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/pscInspection/batch/repository/PscAllCertificateRepository.java @@ -0,0 +1,9 @@ +package com.snp.batch.jobs.pscInspection.batch.repository; + +import com.snp.batch.jobs.pscInspection.batch.entity.PscAllCertificateEntity; + +import java.util.List; + +public interface PscAllCertificateRepository { + void saveAllCertificates(List certificates); +} diff --git a/src/main/java/com/snp/batch/jobs/pscInspection/batch/repository/PscAllCertificateRepositoryImpl.java b/src/main/java/com/snp/batch/jobs/pscInspection/batch/repository/PscAllCertificateRepositoryImpl.java new file mode 100644 index 0000000..28a8bbb --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/pscInspection/batch/repository/PscAllCertificateRepositoryImpl.java @@ -0,0 +1,146 @@ +package com.snp.batch.jobs.pscInspection.batch.repository; + +import com.snp.batch.common.batch.repository.BaseJdbcRepository; +import com.snp.batch.jobs.pscInspection.batch.entity.PscAllCertificateEntity; +import com.snp.batch.jobs.pscInspection.batch.entity.PscCertificateEntity; +import lombok.extern.slf4j.Slf4j; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.core.RowMapper; +import org.springframework.stereotype.Repository; + +import java.sql.PreparedStatement; +import java.sql.Timestamp; +import java.util.List; + +@Slf4j +@Repository +public class PscAllCertificateRepositoryImpl extends BaseJdbcRepository + implements PscAllCertificateRepository { + public PscAllCertificateRepositoryImpl(JdbcTemplate jdbcTemplate) { + super(jdbcTemplate); + } + + @Override + protected String getTableName() { + return "new_snp.psc_all_certificate"; + } + + @Override + protected RowMapper getRowMapper() { + return null; + } + + @Override + protected String getEntityName() { + return "PscAllCertificate"; + } + + @Override + protected String extractId(PscAllCertificateEntity entity) { + return entity.getCertificateId(); + } + + @Override + public String getInsertSql() { + return """ + INSERT INTO new_snp.psc_all_certificate( + certificate_id, + type_id, + data_set_version, + inspection_id, + lrno, + certificate_title_code, + certificate_title, + issuing_authority_code, + issuing_authority, + class_soc_of_issuer, + other_issuing_authority, + issue_date, + expiry_date, + last_survey_date, + survey_authority_code, + survey_authority, + other_survey_authority, + latest_survey_place, + latest_survey_place_code, + survey_authority_type, + inspection_date, + inspected_by + ) VALUES ( + ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, \s + ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, + ?, ? + ) + ON CONFLICT (certificate_id) + DO UPDATE SET + type_id = EXCLUDED.type_id, + data_set_version = EXCLUDED.data_set_version, + inspection_id = EXCLUDED.inspection_id, + lrno = EXCLUDED.lrno, + certificate_title_code = EXCLUDED.certificate_title_code, + certificate_title = EXCLUDED.certificate_title, + issuing_authority_code = EXCLUDED.issuing_authority_code, + issuing_authority = EXCLUDED.issuing_authority, + class_soc_of_issuer = EXCLUDED.class_soc_of_issuer, + other_issuing_authority = EXCLUDED.other_issuing_authority, + issue_date = EXCLUDED.issue_date, + expiry_date = EXCLUDED.expiry_date, + last_survey_date = EXCLUDED.last_survey_date, + survey_authority_code = EXCLUDED.survey_authority_code, + survey_authority = EXCLUDED.survey_authority, + other_survey_authority = EXCLUDED.other_survey_authority, + latest_survey_place = EXCLUDED.latest_survey_place, + latest_survey_place_code = EXCLUDED.latest_survey_place_code, + survey_authority_type = EXCLUDED.survey_authority_type, + inspection_date = EXCLUDED.inspection_date, + inspected_by = EXCLUDED.inspected_by + """; + + } + + @Override + protected String getUpdateSql() { + return null; + } + + @Override + protected void setInsertParameters(PreparedStatement ps, PscAllCertificateEntity e) throws Exception { + int i = 1; + + ps.setString(i++, e.getCertificateId()); + ps.setString(i++, e.getTypeId()); + ps.setString(i++, e.getDataSetVersion()); + ps.setString(i++, e.getInspectionId()); + ps.setString(i++, e.getLrno()); + ps.setString(i++, e.getCertificateTitleCode()); + ps.setString(i++, e.getCertificateTitle()); + ps.setString(i++, e.getIssuingAuthorityCode()); + ps.setString(i++, e.getIssuingAuthority()); + ps.setString(i++, e.getClassSocOfIssuer()); + ps.setString(i++, e.getOtherIssuingAuthority()); + ps.setTimestamp(i++, e.getIssueDate() != null ? Timestamp.valueOf(e.getIssueDate()) : null); + ps.setTimestamp(i++, e.getExpiryDate() != null ? Timestamp.valueOf(e.getExpiryDate()) : null); + ps.setTimestamp(i++, e.getLastSurveyDate() != null ? Timestamp.valueOf(e.getLastSurveyDate()) : null); + ps.setString(i++, e.getSurveyAuthorityCode()); + ps.setString(i++, e.getSurveyAuthority()); + ps.setString(i++, e.getOtherSurveyAuthority()); + ps.setString(i++, e.getLatestSurveyPlace()); + ps.setString(i++, e.getLatestSurveyPlaceCode()); + ps.setString(i++, e.getSurveyAuthorityType()); + ps.setString(i++, e.getInspectionDate()); + ps.setString(i++, e.getInspectedBy()); + } + + @Override + protected void setUpdateParameters(PreparedStatement ps, PscAllCertificateEntity entity) throws Exception { + + } + + @Override + public void saveAllCertificates(List entities) { + if (entities == null || entities.isEmpty()) return; + log.info("PSC AllCertificates 저장 시작 = {}건", entities.size()); + batchInsert(entities); + } + +} diff --git a/src/main/java/com/snp/batch/jobs/pscInspection/batch/repository/PscCertificateRepository.java b/src/main/java/com/snp/batch/jobs/pscInspection/batch/repository/PscCertificateRepository.java new file mode 100644 index 0000000..97041e1 --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/pscInspection/batch/repository/PscCertificateRepository.java @@ -0,0 +1,10 @@ +package com.snp.batch.jobs.pscInspection.batch.repository; + +import com.snp.batch.jobs.pscInspection.batch.entity.PscCertificateEntity; +import com.snp.batch.jobs.pscInspection.batch.entity.PscDefectEntity; + +import java.util.List; + +public interface PscCertificateRepository { + void saveCertificates(List certificates); +} diff --git a/src/main/java/com/snp/batch/jobs/pscInspection/batch/repository/PscCertificateRepositoryImpl.java b/src/main/java/com/snp/batch/jobs/pscInspection/batch/repository/PscCertificateRepositoryImpl.java new file mode 100644 index 0000000..b7ac013 --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/pscInspection/batch/repository/PscCertificateRepositoryImpl.java @@ -0,0 +1,139 @@ +package com.snp.batch.jobs.pscInspection.batch.repository; + +import com.snp.batch.common.batch.repository.BaseJdbcRepository; +import com.snp.batch.jobs.pscInspection.batch.entity.PscCertificateEntity; +import com.snp.batch.jobs.pscInspection.batch.entity.PscDefectEntity; +import lombok.extern.slf4j.Slf4j; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.core.RowMapper; +import org.springframework.stereotype.Repository; + +import java.sql.PreparedStatement; +import java.sql.Timestamp; +import java.util.List; + +@Slf4j +@Repository +public class PscCertificateRepositoryImpl extends BaseJdbcRepository + implements PscCertificateRepository { + public PscCertificateRepositoryImpl(JdbcTemplate jdbcTemplate) { + super(jdbcTemplate); + } + + @Override + protected String getTableName() { + return "new_snp.psc_certificate"; + } + + @Override + protected RowMapper getRowMapper() { + return null; + } + + @Override + protected String getEntityName() { + return "PscCertificate"; + } + + @Override + protected String extractId(PscCertificateEntity entity) { + return entity.getCertificateId(); + } + + @Override + public String getInsertSql() { + return """ + INSERT INTO new_snp.psc_certificate( + certificate_id, + type_id, + data_set_version, + certificate_title, + certificate_title_code, + class_soc_of_issuer, + expiry_date, + inspection_id, + issue_date, + issuing_authority, + issuing_authority_code, + last_survey_date, + latest_survey_place, + latest_survey_place_code, + lrno, + other_issuing_authority, + other_survey_authority, + survey_authority, + survey_authority_code, + survey_authority_type + ) VALUES ( + ?,?,?,?,?,?,?,?,?,?, + ?,?,?,?,?,?,?,?,?,? + ) + ON CONFLICT (certificate_id) + DO UPDATE SET + type_id = EXCLUDED.type_id, + data_set_version = EXCLUDED.data_set_version, + certificate_title = EXCLUDED.certificate_title, + certificate_title_code = EXCLUDED.certificate_title_code, + class_soc_of_issuer = EXCLUDED.class_soc_of_issuer, + expiry_date = EXCLUDED.expiry_date, + inspection_id = EXCLUDED.inspection_id, + issue_date = EXCLUDED.issue_date, + issuing_authority = EXCLUDED.issuing_authority, + issuing_authority_code = EXCLUDED.issuing_authority_code, + last_survey_date = EXCLUDED.last_survey_date, + latest_survey_place = EXCLUDED.latest_survey_place, + latest_survey_place_code = EXCLUDED.latest_survey_place_code, + lrno = EXCLUDED.lrno, + other_issuing_authority = EXCLUDED.other_issuing_authority, + other_survey_authority = EXCLUDED.other_survey_authority, + survey_authority = EXCLUDED.survey_authority, + survey_authority_code = EXCLUDED.survey_authority_code, + survey_authority_type = EXCLUDED.survey_authority_type + """; + + } + + @Override + protected String getUpdateSql() { + return null; + } + + @Override + protected void setInsertParameters(PreparedStatement ps, PscCertificateEntity e) throws Exception { + int i = 1; + + ps.setString(i++, e.getCertificateId()); + ps.setString(i++, e.getTypeId()); + ps.setString(i++, e.getDataSetVersion()); + ps.setString(i++, e.getCertificateTitle()); + ps.setString(i++, e.getCertificateTitleCode()); + ps.setString(i++, e.getClassSocOfIssuer()); + ps.setTimestamp(i++, e.getExpiryDate() != null ? Timestamp.valueOf(e.getExpiryDate()) : null); + ps.setString(i++, e.getInspectionId()); + ps.setTimestamp(i++, e.getIssueDate() != null ? Timestamp.valueOf(e.getIssueDate()) : null); + ps.setString(i++, e.getIssuingAuthority()); + ps.setString(i++, e.getIssuingAuthorityCode()); + ps.setTimestamp(i++, e.getLastSurveyDate() != null ? Timestamp.valueOf(e.getLastSurveyDate()) : null); + ps.setString(i++, e.getLatestSurveyPlace()); + ps.setString(i++, e.getLatestSurveyPlaceCode()); + ps.setString(i++, e.getLrno()); + ps.setString(i++, e.getOtherIssuingAuthority()); + ps.setString(i++, e.getOtherSurveyAuthority()); + ps.setString(i++, e.getSurveyAuthority()); + ps.setString(i++, e.getSurveyAuthorityCode()); + ps.setString(i++, e.getSurveyAuthorityType()); + } + + @Override + protected void setUpdateParameters(PreparedStatement ps, PscCertificateEntity entity) throws Exception { + + } + + @Override + public void saveCertificates(List entities) { + if (entities == null || entities.isEmpty()) return; + log.info("PSC Certificate 저장 시작 = {}건", entities.size()); + batchInsert(entities); + } + +} diff --git a/src/main/java/com/snp/batch/jobs/pscInspection/batch/repository/PscDefectRepository.java b/src/main/java/com/snp/batch/jobs/pscInspection/batch/repository/PscDefectRepository.java new file mode 100644 index 0000000..35d9029 --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/pscInspection/batch/repository/PscDefectRepository.java @@ -0,0 +1,10 @@ +package com.snp.batch.jobs.pscInspection.batch.repository; + +import com.snp.batch.jobs.pscInspection.batch.entity.PscDefectEntity; +import com.snp.batch.jobs.pscInspection.batch.entity.PscInspectionEntity; + +import java.util.List; + +public interface PscDefectRepository { + void saveDefects(List defects); +} diff --git a/src/main/java/com/snp/batch/jobs/pscInspection/batch/repository/PscDefectRepositoryImpl.java b/src/main/java/com/snp/batch/jobs/pscInspection/batch/repository/PscDefectRepositoryImpl.java new file mode 100644 index 0000000..9fedb79 --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/pscInspection/batch/repository/PscDefectRepositoryImpl.java @@ -0,0 +1,163 @@ +package com.snp.batch.jobs.pscInspection.batch.repository; + +import com.snp.batch.common.batch.repository.BaseJdbcRepository; +import com.snp.batch.jobs.pscInspection.batch.entity.PscDefectEntity; +import com.snp.batch.jobs.pscInspection.batch.entity.PscInspectionEntity; +import lombok.extern.slf4j.Slf4j; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.core.RowMapper; +import org.springframework.stereotype.Repository; + +import java.sql.PreparedStatement; +import java.sql.Timestamp; +import java.sql.Types; +import java.util.List; + +@Slf4j +@Repository +public class PscDefectRepositoryImpl extends BaseJdbcRepository + implements PscDefectRepository { + public PscDefectRepositoryImpl(JdbcTemplate jdbcTemplate) { + super(jdbcTemplate); + } + + @Override + protected String getTableName() { + return "new_snp.psc_detail"; + } + + @Override + protected RowMapper getRowMapper() { + return null; + } + + @Override + protected String getEntityName() { + return "PscInspection"; + } + + @Override + protected String extractId(PscDefectEntity entity) { + return entity.getInspectionId(); + } + + @Override + public String getInsertSql() { + return """ + INSERT INTO new_snp.psc_defect( + defect_id, + inspection_id, + type_id, + data_set_version, + action_1, + action_2, + action_3, + action_code_1, + action_code_2, + action_code_3, + amsa_action_code_1, + amsa_action_code_2, + amsa_action_code_3, + class_is_responsible, + defect_code, + defect_text, + defective_item_code, + detention_reason_deficiency, + main_defect_code, + main_defect_text, + nature_of_defect_code, + nature_of_defect_decode, + other_action, + other_recognised_org_resp, + recognised_org_resp, + recognised_org_resp_code, + recognised_org_resp_yn, + is_accidental_damage + ) VALUES (?,?,?,?,?,?,?,?,?,?, + ?,?,?,?,?,?,?,?,?,?, + ?,?,?,?,?,?,?,?) + ON CONFLICT (defect_id) + DO UPDATE SET + inspection_id = EXCLUDED.inspection_id, + type_id = EXCLUDED.type_id, + data_set_version = EXCLUDED.data_set_version, + action_1 = EXCLUDED.action_1, + action_2 = EXCLUDED.action_2, + action_3 = EXCLUDED.action_3, + action_code_1 = EXCLUDED.action_code_1, + action_code_2 = EXCLUDED.action_code_2, + action_code_3 = EXCLUDED.action_code_3, + amsa_action_code_1 = EXCLUDED.amsa_action_code_1, + amsa_action_code_2 = EXCLUDED.amsa_action_code_2, + amsa_action_code_3 = EXCLUDED.amsa_action_code_3, + class_is_responsible = EXCLUDED.class_is_responsible, + defect_code = EXCLUDED.defect_code, + defect_text = EXCLUDED.defect_text, + defective_item_code = EXCLUDED.defective_item_code, + detention_reason_deficiency = EXCLUDED.detention_reason_deficiency, + main_defect_code = EXCLUDED.main_defect_code, + main_defect_text = EXCLUDED.main_defect_text, + nature_of_defect_code = EXCLUDED.nature_of_defect_code, + nature_of_defect_decode = EXCLUDED.nature_of_defect_decode, + other_action = EXCLUDED.other_action, + other_recognised_org_resp = EXCLUDED.other_recognised_org_resp, + recognised_org_resp = EXCLUDED.recognised_org_resp, + recognised_org_resp_code = EXCLUDED.recognised_org_resp_code, + recognised_org_resp_yn = EXCLUDED.recognised_org_resp_yn, + is_accidental_damage = EXCLUDED.is_accidental_damage + """; + + } + + @Override + protected String getUpdateSql() { + return null; + } + + @Override + protected void setInsertParameters(PreparedStatement ps, PscDefectEntity e) throws Exception { + int i = 1; + + ps.setString(i++, e.getDefectId()); + ps.setString(i++, e.getInspectionId()); + ps.setString(i++, e.getTypeId()); + ps.setString(i++, e.getDataSetVersion()); + ps.setString(i++, e.getAction1()); + ps.setString(i++, e.getAction2()); + ps.setString(i++, e.getAction3()); + ps.setString(i++, e.getActionCode1()); + ps.setString(i++, e.getActionCode2()); + ps.setString(i++, e.getActionCode3()); + ps.setString(i++, e.getAmsaActionCode1()); + ps.setString(i++, e.getAmsaActionCode2()); + ps.setString(i++, e.getAmsaActionCode3()); + ps.setString(i++, e.getClassIsResponsible()); + ps.setString(i++, e.getDefectCode()); + ps.setString(i++, e.getDefectText()); + ps.setString(i++, e.getDefectiveItemCode()); + ps.setString(i++, e.getDetentionReasonDeficiency()); + ps.setString(i++, e.getMainDefectCode()); + ps.setString(i++, e.getMainDefectText()); + ps.setString(i++, e.getNatureOfDefectCode()); + ps.setString(i++, e.getNatureOfDefectDecode()); + ps.setString(i++, e.getOtherAction()); + ps.setString(i++, e.getOtherRecognisedOrgResp()); + ps.setString(i++, e.getRecognisedOrgResp()); + ps.setString(i++, e.getRecognisedOrgRespCode()); + ps.setString(i++, e.getRecognisedOrgRespYn()); + ps.setString(i++, e.getIsAccidentalDamage()); + } + + @Override + protected void setUpdateParameters(PreparedStatement ps, PscDefectEntity entity) throws Exception { + + } + + @Override + public void saveDefects(List entities) { + if (entities == null || entities.isEmpty()) return; + log.info("PSC Defect 저장 시작 = {}건", entities.size()); + batchInsert(entities); + } + +} diff --git a/src/main/java/com/snp/batch/jobs/pscInspection/batch/repository/PscInspectionRepository.java b/src/main/java/com/snp/batch/jobs/pscInspection/batch/repository/PscInspectionRepository.java new file mode 100644 index 0000000..201f6c7 --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/pscInspection/batch/repository/PscInspectionRepository.java @@ -0,0 +1,9 @@ +package com.snp.batch.jobs.pscInspection.batch.repository; + +import com.snp.batch.jobs.pscInspection.batch.entity.PscInspectionEntity; + +import java.util.List; + +public interface PscInspectionRepository { + void saveAll(List entities); +} diff --git a/src/main/java/com/snp/batch/jobs/pscInspection/batch/repository/PscInspectionRepositoryImpl.java b/src/main/java/com/snp/batch/jobs/pscInspection/batch/repository/PscInspectionRepositoryImpl.java new file mode 100644 index 0000000..e558071 --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/pscInspection/batch/repository/PscInspectionRepositoryImpl.java @@ -0,0 +1,186 @@ +package com.snp.batch.jobs.pscInspection.batch.repository; + +import com.snp.batch.common.batch.repository.BaseJdbcRepository; +import com.snp.batch.jobs.pscInspection.batch.entity.PscInspectionEntity; +import lombok.extern.slf4j.Slf4j; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.core.RowMapper; +import org.springframework.stereotype.Repository; + +import java.sql.PreparedStatement; +import java.sql.Timestamp; +import java.sql.Types; +import java.util.List; + +@Slf4j +@Repository +public class PscInspectionRepositoryImpl extends BaseJdbcRepository + implements PscInspectionRepository{ + public PscInspectionRepositoryImpl(JdbcTemplate jdbcTemplate) { + super(jdbcTemplate); + } + + @Override + protected String getTableName() { + return "new_snp.psc_detail"; + } + + @Override + protected String getEntityName() { + return "PscInspection"; + } + + @Override + protected String extractId(PscInspectionEntity entity) { + return entity.getInspectionId(); + } + + @Override + public String getInsertSql() { + return """ + INSERT INTO new_snp.psc_detail( + inspection_id, + type_id, + data_set_version, + authorisation, + call_sign, + class, + cargo, + charterer, + country, + inspection_date, + release_date, + ship_detained, + dead_weight, + expanded_inspection, + flag, + follow_up_inspection, + gross_tonnage, + inspection_port_code, + inspection_port_decode, + keel_laid, + last_updated, + ihslr_or_imo_ship_no, + manager, + number_of_days_detained, + number_of_defects, + number_of_part_days_detained, + other_inspection_type, + owner, + ship_name, + ship_type_code, + ship_type_decode, + source, + unlocode, + year_of_build + ) VALUES ( + ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, + ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, + ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, + ?, ?, ?, ? + ) + ON CONFLICT (inspection_id) + DO UPDATE SET + type_id = EXCLUDED.type_id, + data_set_version = EXCLUDED.data_set_version, + authorisation = EXCLUDED.authorisation, + call_sign = EXCLUDED.call_sign, + class = EXCLUDED.class, + cargo = EXCLUDED.cargo, + charterer = EXCLUDED.charterer, + country = EXCLUDED.country, + inspection_date = EXCLUDED.inspection_date, + release_date = EXCLUDED.release_date, + ship_detained = EXCLUDED.ship_detained, + dead_weight = EXCLUDED.dead_weight, + expanded_inspection = EXCLUDED.expanded_inspection, + flag = EXCLUDED.flag, + follow_up_inspection = EXCLUDED.follow_up_inspection, + gross_tonnage = EXCLUDED.gross_tonnage, + inspection_port_code = EXCLUDED.inspection_port_code, + inspection_port_decode = EXCLUDED.inspection_port_decode, + keel_laid = EXCLUDED.keel_laid, + last_updated = EXCLUDED.last_updated, + ihslr_or_imo_ship_no = EXCLUDED.ihslr_or_imo_ship_no, + manager = EXCLUDED.manager, + number_of_days_detained = EXCLUDED.number_of_days_detained, + number_of_defects = EXCLUDED.number_of_defects, + number_of_part_days_detained = EXCLUDED.number_of_part_days_detained, + other_inspection_type = EXCLUDED.other_inspection_type, + owner = EXCLUDED.owner, + ship_name = EXCLUDED.ship_name, + ship_type_code = EXCLUDED.ship_type_code, + ship_type_decode = EXCLUDED.ship_type_decode, + source = EXCLUDED.source, + unlocode = EXCLUDED.unlocode, + year_of_build = EXCLUDED.year_of_build + """; + + } + + @Override + protected void setInsertParameters(PreparedStatement ps, PscInspectionEntity e) throws Exception { + int i = 1; + + ps.setString(i++, e.getInspectionId()); + ps.setString(i++, e.getTypeId()); + ps.setString(i++, e.getDataSetVersion()); + ps.setString(i++, e.getAuthorisation()); + ps.setString(i++, e.getCallSign()); + ps.setString(i++, e.getShipClass()); + ps.setString(i++, e.getCargo()); + ps.setString(i++, e.getCharterer()); + ps.setString(i++, e.getCountry()); + ps.setTimestamp(i++, e.getInspectionDate() != null ? Timestamp.valueOf(e.getInspectionDate()) : null); + ps.setTimestamp(i++, e.getReleaseDate() != null ? Timestamp.valueOf(e.getReleaseDate()) : null); + ps.setString(i++, e.getShipDetained()); + ps.setString(i++, e.getDeadWeight()); + ps.setString(i++, e.getExpandedInspection()); + ps.setString(i++, e.getFlag()); + ps.setString(i++, e.getFollowUpInspection()); + ps.setString(i++, e.getGrossTonnage()); + ps.setString(i++, e.getInspectionPortCode()); + ps.setString(i++, e.getInspectionPortDecode()); + ps.setString(i++, e.getKeelLaid()); + ps.setTimestamp(i++, e.getLastUpdated() != null ? Timestamp.valueOf(e.getLastUpdated()) : null); + ps.setString(i++, e.getIhslrOrImoShipNo()); + ps.setString(i++, e.getManager()); + if (e.getNumberOfDaysDetained() != null) { + ps.setInt(i++, e.getNumberOfDaysDetained()); + } else { + ps.setNull(i++, Types.INTEGER); + } + ps.setString(i++, e.getNumberOfDefects()); + ps.setBigDecimal(i++, e.getNumberOfPartDaysDetained()); + ps.setString(i++, e.getOtherInspectionType()); + ps.setString(i++, e.getOwner()); + ps.setString(i++, e.getShipName()); + ps.setString(i++, e.getShipTypeCode()); + ps.setString(i++, e.getShipTypeDecode()); + ps.setString(i++, e.getSource()); + ps.setString(i++, e.getUnlocode()); + ps.setString(i++, e.getYearOfBuild()); + } + + @Override + public void saveAll(List entities) { + if (entities == null || entities.isEmpty()) return; + log.info("PSC Inspection 저장 시작 = {}건", entities.size()); + batchInsert(entities); + } + + @Override + protected void setUpdateParameters(PreparedStatement ps, PscInspectionEntity entity) throws Exception { + + } + + @Override + protected String getUpdateSql() { + return null; + } + + @Override + protected RowMapper getRowMapper() { + return null; + } +} diff --git a/src/main/java/com/snp/batch/jobs/pscInspection/batch/writer/PscInspectionWriter.java b/src/main/java/com/snp/batch/jobs/pscInspection/batch/writer/PscInspectionWriter.java new file mode 100644 index 0000000..725d167 --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/pscInspection/batch/writer/PscInspectionWriter.java @@ -0,0 +1,55 @@ +package com.snp.batch.jobs.pscInspection.batch.writer; + +import com.snp.batch.common.batch.writer.BaseWriter; +import com.snp.batch.jobs.pscInspection.batch.entity.PscInspectionEntity; +import com.snp.batch.jobs.pscInspection.batch.repository.PscAllCertificateRepository; +import com.snp.batch.jobs.pscInspection.batch.repository.PscCertificateRepository; +import com.snp.batch.jobs.pscInspection.batch.repository.PscDefectRepository; +import com.snp.batch.jobs.pscInspection.batch.repository.PscInspectionRepository; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; + +import java.util.List; + +@Slf4j +@Component +public class PscInspectionWriter extends BaseWriter { + private final PscInspectionRepository pscInspectionRepository; + private final PscDefectRepository pscDefectRepository; + private final PscCertificateRepository pscCertificateRepository; + private final PscAllCertificateRepository pscAllCertificateRepository; + + public PscInspectionWriter(PscInspectionRepository pscInspectionRepository, + PscDefectRepository pscDefectRepository, + PscCertificateRepository pscCertificateRepository, + PscAllCertificateRepository pscAllCertificateRepository) { + super("PscInspection"); + this.pscInspectionRepository = pscInspectionRepository; + this.pscDefectRepository = pscDefectRepository; + this.pscCertificateRepository = pscCertificateRepository; + this.pscAllCertificateRepository = pscAllCertificateRepository; + } + + @Override + protected void writeItems(List items) throws Exception { + + if (items == null || items.isEmpty()) return; + //pscInspectionRepository.saveAll(items); + log.info("PSC Inspection 저장: {} 건", items.size()); + + for (PscInspectionEntity entity : items) { + pscInspectionRepository.saveAll(List.of(entity)); + pscDefectRepository.saveDefects(entity.getDefects()); + pscCertificateRepository.saveCertificates(entity.getCertificates()); + pscAllCertificateRepository.saveAllCertificates(entity.getAllCertificates()); + + // 효율적으로 로그 + int defectCount = entity.getDefects() != null ? entity.getDefects().size() : 0; + int certificateCount = entity.getCertificates() != null ? entity.getCertificates().size() : 0; + int allCertificateCount = entity.getAllCertificates() != null ? entity.getAllCertificates().size() : 0; + + log.info("Inspection ID: {}, Defects: {}, Certificates: {}, AllCertificates: {}", + entity.getInspectionId(), defectCount, certificateCount, allCertificateCount); + } + } +} diff --git a/src/main/java/com/snp/batch/jobs/shipMovementBerthCalls/batch/reader/BerthCallsReader.java b/src/main/java/com/snp/batch/jobs/shipMovementBerthCalls/batch/reader/BerthCallsReader.java index 9f6771d..38c8d8f 100644 --- a/src/main/java/com/snp/batch/jobs/shipMovementBerthCalls/batch/reader/BerthCallsReader.java +++ b/src/main/java/com/snp/batch/jobs/shipMovementBerthCalls/batch/reader/BerthCallsReader.java @@ -82,7 +82,7 @@ public class BerthCallsReader extends BaseApiReader { private static final String GET_ALL_IMO_QUERY = // "SELECT imo_number FROM ship_data ORDER BY id"; - "SELECT imo_number FROM snp_data.ship_data where imo_number > (select max(imo) from snp_data.t_berthcalls) ORDER BY imo_number"; + "SELECT imo_number FROM snp_data.ship_data where imo_number > (select max(imo) from snp_data.t_berthcall) ORDER BY imo_number"; /** * 최초 1회만 실행: ship_data 테이블에서 IMO 번호 전체 조회 From f2970872fd510661e079be3e3c4e4df881dc9a72 Mon Sep 17 00:00:00 2001 From: Kim JiMyeung Date: Fri, 12 Dec 2025 11:14:10 +0900 Subject: [PATCH 5/9] =?UTF-8?q?mvmn=5Ftype=20on=20conflict=EC=B6=94?= =?UTF-8?q?=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../batch/repository/ShipMovementRepositoryImpl.java | 2 +- .../repository/AnchorageCallsRepositoryImpl.java | 2 +- .../batch/repository/BerthCallsRepositoryImpl.java | 2 +- .../batch/config/CurrentlyAtJobConfig.java | 12 ++++++------ .../batch/dto/CurrentlyAtDto.java | 2 +- .../batch/dto/CurrentlyAtPositionDto.java | 2 +- .../batch/entity/CurrentlyAtEntity.java | 2 +- .../batch/processor/CurrentlyAtProcessor.java | 6 +++--- .../batch/reader/CurrentlyAtReader.java | 4 ++-- .../batch/repository/CurrentlyAtRepository.java | 4 ++-- .../batch/repository/CurrentlyAtRepositoryImpl.java | 11 +++-------- .../batch/writer/CurrentlyAtWriter.java | 6 +++--- .../batch/repository/DarkActivityRepositoryImpl.java | 2 +- .../batch/repository/DestinationRepositoryImpl.java | 2 +- .../batch/repository/StsOperationRepositoryImpl.java | 2 +- .../repository/TerminalCallsRepositoryImpl.java | 2 +- .../batch/repository/TransitlsRepositoryImpl.java | 2 +- 17 files changed, 30 insertions(+), 35 deletions(-) rename src/main/java/com/snp/batch/jobs/{shipCurrentlyAt => shipMovementCurrentlyAt}/batch/config/CurrentlyAtJobConfig.java (87%) rename src/main/java/com/snp/batch/jobs/{shipCurrentlyAt => shipMovementCurrentlyAt}/batch/dto/CurrentlyAtDto.java (93%) rename src/main/java/com/snp/batch/jobs/{shipCurrentlyAt => shipMovementCurrentlyAt}/batch/dto/CurrentlyAtPositionDto.java (84%) rename src/main/java/com/snp/batch/jobs/{shipCurrentlyAt => shipMovementCurrentlyAt}/batch/entity/CurrentlyAtEntity.java (94%) rename src/main/java/com/snp/batch/jobs/{shipCurrentlyAt => shipMovementCurrentlyAt}/batch/processor/CurrentlyAtProcessor.java (91%) rename src/main/java/com/snp/batch/jobs/{shipCurrentlyAt => shipMovementCurrentlyAt}/batch/reader/CurrentlyAtReader.java (98%) rename src/main/java/com/snp/batch/jobs/{shipCurrentlyAt => shipMovementCurrentlyAt}/batch/repository/CurrentlyAtRepository.java (54%) rename src/main/java/com/snp/batch/jobs/{shipCurrentlyAt => shipMovementCurrentlyAt}/batch/repository/CurrentlyAtRepositoryImpl.java (94%) rename src/main/java/com/snp/batch/jobs/{shipCurrentlyAt => shipMovementCurrentlyAt}/batch/writer/CurrentlyAtWriter.java (77%) diff --git a/src/main/java/com/snp/batch/jobs/shipMovement/batch/repository/ShipMovementRepositoryImpl.java b/src/main/java/com/snp/batch/jobs/shipMovement/batch/repository/ShipMovementRepositoryImpl.java index 5c405e0..13a3ac0 100644 --- a/src/main/java/com/snp/batch/jobs/shipMovement/batch/repository/ShipMovementRepositoryImpl.java +++ b/src/main/java/com/snp/batch/jobs/shipMovement/batch/repository/ShipMovementRepositoryImpl.java @@ -69,7 +69,7 @@ public class ShipMovementRepositoryImpl extends BaseJdbcRepository Date: Fri, 12 Dec 2025 11:20:13 +0900 Subject: [PATCH 6/9] =?UTF-8?q?reader=20=EC=88=98=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../jobs/shipMovement/batch/reader/ShipMovementReader.java | 4 ++-- .../batch/reader/AnchorageCallsReader.java | 4 ++-- .../shipMovementBerthCalls/batch/reader/BerthCallsReader.java | 4 ++-- .../batch/reader/DarkActivityReader.java | 4 ++-- .../batch/reader/DestinationReader.java | 4 ++-- .../batch/reader/StsOperationReader.java | 4 ++-- .../batch/reader/TerminalCallsReader.java | 4 ++-- .../shipMovementTransits/batch/reader/TransitsReader.java | 4 ++-- 8 files changed, 16 insertions(+), 16 deletions(-) diff --git a/src/main/java/com/snp/batch/jobs/shipMovement/batch/reader/ShipMovementReader.java b/src/main/java/com/snp/batch/jobs/shipMovement/batch/reader/ShipMovementReader.java index cc1cca3..1277732 100644 --- a/src/main/java/com/snp/batch/jobs/shipMovement/batch/reader/ShipMovementReader.java +++ b/src/main/java/com/snp/batch/jobs/shipMovement/batch/reader/ShipMovementReader.java @@ -85,8 +85,8 @@ public class ShipMovementReader extends BaseApiReader { } private static final String GET_ALL_IMO_QUERY = -// "SELECT imo_number FROM ship_data ORDER BY id"; - "SELECT imo_number FROM snp_data.ship_data where imo_number > (select max(imo) from snp_data.t_ship_stpov_info) ORDER BY imo_number"; + "SELECT imo_number FROM ship_data ORDER BY id"; +// "SELECT imo_number FROM snp_data.ship_data where imo_number > (select max(imo) from snp_data.t_ship_stpov_info) ORDER BY imo_number"; private static final String FETCH_ALL_HASHES_QUERY = "SELECT imo_number, ship_detail_hash FROM ship_detail_hash_json ORDER BY imo_number"; diff --git a/src/main/java/com/snp/batch/jobs/shipMovementAnchorageCalls/batch/reader/AnchorageCallsReader.java b/src/main/java/com/snp/batch/jobs/shipMovementAnchorageCalls/batch/reader/AnchorageCallsReader.java index 9c3f782..60957d2 100644 --- a/src/main/java/com/snp/batch/jobs/shipMovementAnchorageCalls/batch/reader/AnchorageCallsReader.java +++ b/src/main/java/com/snp/batch/jobs/shipMovementAnchorageCalls/batch/reader/AnchorageCallsReader.java @@ -81,8 +81,8 @@ public class AnchorageCallsReader extends BaseApiReader { } private static final String GET_ALL_IMO_QUERY = -// "SELECT imo_number FROM ship_data ORDER BY id"; - "SELECT imo_number FROM snp_data.ship_data where imo_number > (select max(imo) from snp_data.t_anchoragecall) ORDER BY imo_number"; + "SELECT imo_number FROM ship_data ORDER BY id"; +// "SELECT imo_number FROM snp_data.ship_data where imo_number > (select max(imo) from snp_data.t_anchoragecall) ORDER BY imo_number"; private static final String FETCH_ALL_HASHES_QUERY = "SELECT imo_number, ship_detail_hash FROM ship_detail_hash_json ORDER BY imo_number"; diff --git a/src/main/java/com/snp/batch/jobs/shipMovementBerthCalls/batch/reader/BerthCallsReader.java b/src/main/java/com/snp/batch/jobs/shipMovementBerthCalls/batch/reader/BerthCallsReader.java index 38c8d8f..3d7f5c7 100644 --- a/src/main/java/com/snp/batch/jobs/shipMovementBerthCalls/batch/reader/BerthCallsReader.java +++ b/src/main/java/com/snp/batch/jobs/shipMovementBerthCalls/batch/reader/BerthCallsReader.java @@ -81,8 +81,8 @@ public class BerthCallsReader extends BaseApiReader { } private static final String GET_ALL_IMO_QUERY = -// "SELECT imo_number FROM ship_data ORDER BY id"; - "SELECT imo_number FROM snp_data.ship_data where imo_number > (select max(imo) from snp_data.t_berthcall) ORDER BY imo_number"; + "SELECT imo_number FROM ship_data ORDER BY id"; +// "SELECT imo_number FROM snp_data.ship_data where imo_number > (select max(imo) from snp_data.t_berthcall) ORDER BY imo_number"; /** * 최초 1회만 실행: ship_data 테이블에서 IMO 번호 전체 조회 diff --git a/src/main/java/com/snp/batch/jobs/shipMovementDarkActivity/batch/reader/DarkActivityReader.java b/src/main/java/com/snp/batch/jobs/shipMovementDarkActivity/batch/reader/DarkActivityReader.java index bb74cd0..7587fad 100644 --- a/src/main/java/com/snp/batch/jobs/shipMovementDarkActivity/batch/reader/DarkActivityReader.java +++ b/src/main/java/com/snp/batch/jobs/shipMovementDarkActivity/batch/reader/DarkActivityReader.java @@ -81,8 +81,8 @@ public class DarkActivityReader extends BaseApiReader { } private static final String GET_ALL_IMO_QUERY = -// "SELECT imo_number FROM ship_data ORDER BY id"; - "SELECT imo_number FROM snp_data.ship_data where imo_number > (select max(imo) from snp_data.t_darkactivity) ORDER BY imo_number"; + "SELECT imo_number FROM ship_data ORDER BY id"; +// "SELECT imo_number FROM snp_data.ship_data where imo_number > (select max(imo) from snp_data.t_darkactivity) ORDER BY imo_number"; /** * 최초 1회만 실행: ship_data 테이블에서 IMO 번호 전체 조회 diff --git a/src/main/java/com/snp/batch/jobs/shipMovementDestination/batch/reader/DestinationReader.java b/src/main/java/com/snp/batch/jobs/shipMovementDestination/batch/reader/DestinationReader.java index 3220826..a57adbe 100644 --- a/src/main/java/com/snp/batch/jobs/shipMovementDestination/batch/reader/DestinationReader.java +++ b/src/main/java/com/snp/batch/jobs/shipMovementDestination/batch/reader/DestinationReader.java @@ -79,8 +79,8 @@ public class DestinationReader extends BaseApiReader { } private static final String GET_ALL_IMO_QUERY = -// "SELECT imo_number FROM ship_data ORDER BY id"; - "SELECT imo_number FROM snp_data.ship_data where imo_number > (select max(imo) from snp_data.t_destination) ORDER BY imo_number"; + "SELECT imo_number FROM ship_data ORDER BY id"; +// "SELECT imo_number FROM snp_data.ship_data where imo_number > (select max(imo) from snp_data.t_destination) ORDER BY imo_number"; /** * 최초 1회만 실행: ship_data 테이블에서 IMO 번호 전체 조회 diff --git a/src/main/java/com/snp/batch/jobs/shipMovementStsOperations/batch/reader/StsOperationReader.java b/src/main/java/com/snp/batch/jobs/shipMovementStsOperations/batch/reader/StsOperationReader.java index a6c286c..5aff9b9 100644 --- a/src/main/java/com/snp/batch/jobs/shipMovementStsOperations/batch/reader/StsOperationReader.java +++ b/src/main/java/com/snp/batch/jobs/shipMovementStsOperations/batch/reader/StsOperationReader.java @@ -81,8 +81,8 @@ public class StsOperationReader extends BaseApiReader { } private static final String GET_ALL_IMO_QUERY = -// "SELECT imo_number FROM ship_data ORDER BY id"; - "SELECT imo_number FROM snp_data.ship_data where imo_number > (select max(imo) from snp_data.t_stsoperation) ORDER BY imo_number"; + "SELECT imo_number FROM ship_data ORDER BY id"; +// "SELECT imo_number FROM snp_data.ship_data where imo_number > (select max(imo) from snp_data.t_stsoperation) ORDER BY imo_number"; /** * 최초 1회만 실행: ship_data 테이블에서 IMO 번호 전체 조회 diff --git a/src/main/java/com/snp/batch/jobs/shipMovementTerminalCalls/batch/reader/TerminalCallsReader.java b/src/main/java/com/snp/batch/jobs/shipMovementTerminalCalls/batch/reader/TerminalCallsReader.java index 9018360..62a9061 100644 --- a/src/main/java/com/snp/batch/jobs/shipMovementTerminalCalls/batch/reader/TerminalCallsReader.java +++ b/src/main/java/com/snp/batch/jobs/shipMovementTerminalCalls/batch/reader/TerminalCallsReader.java @@ -81,8 +81,8 @@ public class TerminalCallsReader extends BaseApiReader { } private static final String GET_ALL_IMO_QUERY = -// "SELECT imo_number FROM ship_data ORDER BY id"; - "SELECT imo_number FROM snp_data.ship_data where imo_number > (select max(imo) from snp_data.t_terminalcall) ORDER BY imo_number"; + "SELECT imo_number FROM ship_data ORDER BY id"; +// "SELECT imo_number FROM snp_data.ship_data where imo_number > (select max(imo) from snp_data.t_terminalcall) ORDER BY imo_number"; /** * 최초 1회만 실행: ship_data 테이블에서 IMO 번호 전체 조회 diff --git a/src/main/java/com/snp/batch/jobs/shipMovementTransits/batch/reader/TransitsReader.java b/src/main/java/com/snp/batch/jobs/shipMovementTransits/batch/reader/TransitsReader.java index 375b14f..daf2b94 100644 --- a/src/main/java/com/snp/batch/jobs/shipMovementTransits/batch/reader/TransitsReader.java +++ b/src/main/java/com/snp/batch/jobs/shipMovementTransits/batch/reader/TransitsReader.java @@ -79,8 +79,8 @@ public class TransitsReader extends BaseApiReader { } private static final String GET_ALL_IMO_QUERY = -// "SELECT imo_number FROM ship_data ORDER BY id"; - "SELECT imo_number FROM snp_data.ship_data where imo_number > (select max(imo) from snp_data.t_transit) ORDER BY imo_number"; + "SELECT imo_number FROM ship_data ORDER BY id"; +// "SELECT imo_number FROM snp_data.ship_data where imo_number > (select max(imo) from snp_data.t_transit) ORDER BY imo_number"; /** * 최초 1회만 실행: ship_data 테이블에서 IMO 번호 전체 조회 From acd76bd3585d9729c54e87bac0b07a990397b2bb Mon Sep 17 00:00:00 2001 From: hyojin kim Date: Fri, 19 Dec 2025 10:57:40 +0900 Subject: [PATCH 7/9] =?UTF-8?q?:sparkles:=20Event=20Detail=20=EC=A0=81?= =?UTF-8?q?=EC=9E=AC=20=ED=94=84=EB=A1=9C=EC=84=B8=EC=8A=A4=20=EA=B0=9C?= =?UTF-8?q?=EB=B0=9C=20-=20StartDate,=20EndDate=20=EC=B6=94=EC=B6=9C?= =?UTF-8?q?=EC=9E=91=EC=97=85=20=ED=95=84=EC=9A=94?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../batch/config/EventImportJobConfig.java | 14 +- .../batch/jobs/event/batch/dto/CargoDto.java | 50 +++++ .../jobs/event/batch/dto/EventDetailDto.java | 105 ++++++++++ .../event/batch/dto/EventDetailResponse.java | 18 ++ .../event/batch/dto/HumanCasualtyDto.java | 35 ++++ .../jobs/event/batch/dto/RelationshipDto.java | 41 ++++ .../jobs/event/batch/entity/CargoEntity.java | 26 +++ .../event/batch/entity/EventDetailEntity.java | 62 ++++++ .../jobs/event/batch/entity/EventEntity.java | 1 - .../batch/entity/HumanCasualtyEntity.java | 21 ++ .../batch/entity/RelationshipEntity.java | 23 +++ .../batch/processor/EventDataProcessor.java | 63 ++++-- .../event/batch/reader/EventDataReader.java | 164 +++++++++++++-- .../batch/repository/EventRepository.java | 10 +- .../batch/repository/EventRepositoryImpl.java | 187 +++++++++++++++--- .../jobs/event/batch/repository/EventSql.java | 124 ++++++++++++ .../event/batch/writer/EventDataWriter.java | 42 ++-- 17 files changed, 902 insertions(+), 84 deletions(-) create mode 100644 src/main/java/com/snp/batch/jobs/event/batch/dto/CargoDto.java create mode 100644 src/main/java/com/snp/batch/jobs/event/batch/dto/EventDetailDto.java create mode 100644 src/main/java/com/snp/batch/jobs/event/batch/dto/EventDetailResponse.java create mode 100644 src/main/java/com/snp/batch/jobs/event/batch/dto/HumanCasualtyDto.java create mode 100644 src/main/java/com/snp/batch/jobs/event/batch/dto/RelationshipDto.java create mode 100644 src/main/java/com/snp/batch/jobs/event/batch/entity/CargoEntity.java create mode 100644 src/main/java/com/snp/batch/jobs/event/batch/entity/EventDetailEntity.java create mode 100644 src/main/java/com/snp/batch/jobs/event/batch/entity/HumanCasualtyEntity.java create mode 100644 src/main/java/com/snp/batch/jobs/event/batch/entity/RelationshipEntity.java create mode 100644 src/main/java/com/snp/batch/jobs/event/batch/repository/EventSql.java diff --git a/src/main/java/com/snp/batch/jobs/event/batch/config/EventImportJobConfig.java b/src/main/java/com/snp/batch/jobs/event/batch/config/EventImportJobConfig.java index 40c38be..9da156e 100644 --- a/src/main/java/com/snp/batch/jobs/event/batch/config/EventImportJobConfig.java +++ b/src/main/java/com/snp/batch/jobs/event/batch/config/EventImportJobConfig.java @@ -1,8 +1,8 @@ package com.snp.batch.jobs.event.batch.config; import com.snp.batch.common.batch.config.BaseJobConfig; -import com.snp.batch.jobs.event.batch.dto.EventDto; -import com.snp.batch.jobs.event.batch.entity.EventEntity; +import com.snp.batch.jobs.event.batch.dto.EventDetailDto; +import com.snp.batch.jobs.event.batch.entity.EventDetailEntity; import com.snp.batch.jobs.event.batch.processor.EventDataProcessor; import com.snp.batch.jobs.event.batch.reader.EventDataReader; import com.snp.batch.jobs.event.batch.writer.EventDataWriter; @@ -23,7 +23,7 @@ import org.springframework.web.reactive.function.client.WebClient; @Slf4j @Configuration -public class EventImportJobConfig extends BaseJobConfig { +public class EventImportJobConfig extends BaseJobConfig { private final JdbcTemplate jdbcTemplate; private final WebClient maritimeApiWebClient; @@ -34,7 +34,7 @@ public class EventImportJobConfig extends BaseJobConfig { @Override protected int getChunkSize() { - return 5000; // API에서 5000개씩 가져오므로 chunk도 5000으로 설정 + return 10; // API에서 5000개씩 가져오므로 chunk도 5000으로 설정 } public EventImportJobConfig( JobRepository jobRepository, @@ -63,17 +63,17 @@ public class EventImportJobConfig extends BaseJobConfig { } @Override - protected ItemReader createReader() { + protected ItemReader createReader() { return new EventDataReader(maritimeApiWebClient, jdbcTemplate, batchDateService); } @Override - protected ItemProcessor createProcessor() { + protected ItemProcessor createProcessor() { return eventDataProcessor; } @Override - protected ItemWriter createWriter() { return eventDataWriter; } + protected ItemWriter createWriter() { return eventDataWriter; } @Bean(name = "eventImportJob") public Job eventImportJob() { diff --git a/src/main/java/com/snp/batch/jobs/event/batch/dto/CargoDto.java b/src/main/java/com/snp/batch/jobs/event/batch/dto/CargoDto.java new file mode 100644 index 0000000..8a91382 --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/event/batch/dto/CargoDto.java @@ -0,0 +1,50 @@ +package com.snp.batch.jobs.event.batch.dto; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.snp.batch.jobs.event.batch.entity.CargoEntity; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class CargoDto { + @JsonProperty("EventID") + private Integer eventID; + @JsonProperty("Sequence") + private String sequence; + @JsonProperty("IHSLRorIMOShipNo") + private String ihslrOrImoShipNo; + @JsonProperty("Type") + private String type; + @JsonProperty("Quantity") + private Integer quantity; + @JsonProperty("UnitShort") + private String unitShort; + @JsonProperty("Unit") + private String unit; + @JsonProperty("Text") + private String text; + @JsonProperty("CargoDamage") + private String cargoDamage; + @JsonProperty("Dangerous") + private String dangerous; + + public CargoEntity toEntity() { + return CargoEntity.builder() + .eventID(this.eventID) + .sequence(this.sequence) + .ihslrOrImoShipNo(this.ihslrOrImoShipNo) + .type(this.type) + .unit(this.unit) + .quantity(this.quantity) + .unitShort(this.unitShort) + .text(this.text) + .cargoDamage(this.cargoDamage) + .dangerous(this.dangerous) + .build(); + } +} diff --git a/src/main/java/com/snp/batch/jobs/event/batch/dto/EventDetailDto.java b/src/main/java/com/snp/batch/jobs/event/batch/dto/EventDetailDto.java new file mode 100644 index 0000000..15f13ac --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/event/batch/dto/EventDetailDto.java @@ -0,0 +1,105 @@ +package com.snp.batch.jobs.event.batch.dto; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.List; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class EventDetailDto { + @JsonProperty("IncidentID") + private Integer incidentID; + @JsonProperty("EventID") + private Integer eventID; + @JsonProperty("EventTypeID") + private Integer eventTypeID; + @JsonProperty("EventType") + private String eventType; + @JsonProperty("Significance") + private String significance; + @JsonProperty("Headline") + private String headline; + @JsonProperty("IHSLRorIMOShipNo") + private String ihslrOrImoShipNo; + @JsonProperty("VesselName") + private String vesselName; + @JsonProperty("VesselType") + private String vesselType; + @JsonProperty("VesselTypeDecode") + private String vesselTypeDecode; + @JsonProperty("VesselFlag") + private String vesselFlagCode; + @JsonProperty("Flag") + private String vesselFlagDecode; + @JsonProperty("CargoLoadingStatusCode") + private String cargoLoadingStatusCode; + @JsonProperty("VesselDWT") + private Integer vesselDWT; + @JsonProperty("VesselGT") + private Integer vesselGT; + @JsonProperty("LDTAtTime") + private Integer ldtAtTime; + @JsonProperty("DateOfBuild") + private Integer dateOfBuild; + @JsonProperty("RegisteredOwnerCodeAtTime") + private String registeredOwnerCodeAtTime; + @JsonProperty("RegisteredOwnerAtTime") + private String registeredOwnerAtTime; + @JsonProperty("RegisteredOwnerCoDAtTime") + private String registeredOwnerCountryCodeAtTime; + @JsonProperty("RegisteredOwnerCountryAtTime") + private String registeredOwnerCountryAtTime; + @JsonProperty("Weather") + private String weather; + @JsonProperty("EventTypeDetail") + private String eventTypeDetail; + @JsonProperty("EventTypeDetailID") + private Integer eventTypeDetailID; + @JsonProperty("CasualtyAction") + private String casualtyAction; + @JsonProperty("LocationName") + private String locationName; + @JsonProperty("TownName") + private String townName; + @JsonProperty("MarsdenGridReference") + private Integer marsdenGridReference; + @JsonProperty("EnvironmentLocation") + private String environmentLocation; + @JsonProperty("CasualtyZone") + private String casualtyZone; + @JsonProperty("CasualtyZoneCode") + private String casualtyZoneCode; + @JsonProperty("CountryCode") + private String countryCode; + @JsonProperty("AttemptedBoarding") + private String attemptedBoarding; + @JsonProperty("Description") + private String description; + @JsonProperty("Pollutant") + private String pollutant; + @JsonProperty("PollutantUnit") + private String pollutantUnit; + @JsonProperty("PollutantQuantity") + private Double pollutantQuantity; + @JsonProperty("PublishedDate") + private String publishedDate; + @JsonProperty("Component2") + private String component2; + @JsonProperty("FiredUpon") + private String firedUpon; + + @JsonProperty("Cargoes") + private List cargoes; + + @JsonProperty("HumanCasualties") + private List humanCasualties; + + @JsonProperty("Relationships") + private List relationships; +} diff --git a/src/main/java/com/snp/batch/jobs/event/batch/dto/EventDetailResponse.java b/src/main/java/com/snp/batch/jobs/event/batch/dto/EventDetailResponse.java new file mode 100644 index 0000000..fe67261 --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/event/batch/dto/EventDetailResponse.java @@ -0,0 +1,18 @@ +package com.snp.batch.jobs.event.batch.dto; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.List; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class EventDetailResponse { + @JsonProperty("MaritimeEvent") + private EventDetailDto eventDetailDto; +} diff --git a/src/main/java/com/snp/batch/jobs/event/batch/dto/HumanCasualtyDto.java b/src/main/java/com/snp/batch/jobs/event/batch/dto/HumanCasualtyDto.java new file mode 100644 index 0000000..09ed21c --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/event/batch/dto/HumanCasualtyDto.java @@ -0,0 +1,35 @@ +package com.snp.batch.jobs.event.batch.dto; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.snp.batch.jobs.event.batch.entity.HumanCasualtyEntity; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class HumanCasualtyDto { + @JsonProperty("EventID") + private Integer eventID; + @JsonProperty("Scope") + private String scope; + @JsonProperty("Type") + private String type; + @JsonProperty("Qualifier") + private String qualifier; + @JsonProperty("Count") + private Integer count; + + public HumanCasualtyEntity toEntity() { + return HumanCasualtyEntity.builder() + .eventID(this.eventID) + .scope(this.scope) + .type(this.type) + .qualifier(this.qualifier) + .count(this.count) + .build(); + } +} diff --git a/src/main/java/com/snp/batch/jobs/event/batch/dto/RelationshipDto.java b/src/main/java/com/snp/batch/jobs/event/batch/dto/RelationshipDto.java new file mode 100644 index 0000000..1d1beae --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/event/batch/dto/RelationshipDto.java @@ -0,0 +1,41 @@ +package com.snp.batch.jobs.event.batch.dto; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.snp.batch.jobs.event.batch.entity.RelationshipEntity; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class RelationshipDto { + @JsonProperty("IncidentID") + private String incidentID; + @JsonProperty("EventID") + private Integer eventID; + @JsonProperty("RelationshipType") + private String relationshipType; + @JsonProperty("RelationshipTypeCode") + private String relationshipTypeCode; + @JsonProperty("EventID2") + private Integer eventID2; + @JsonProperty("EventType") + private String eventType; + @JsonProperty("EventTypeCode") + private String eventTypeCode; + + public RelationshipEntity toEntity() { + return RelationshipEntity.builder() + .incidentID(this.incidentID) + .eventID(this.eventID) + .relationshipType(this.relationshipType) + .relationshipTypeCode(this.relationshipTypeCode) + .eventID2(this.eventID2) + .eventType(this.eventType) + .eventTypeCode(this.eventTypeCode) + .build(); + } +} diff --git a/src/main/java/com/snp/batch/jobs/event/batch/entity/CargoEntity.java b/src/main/java/com/snp/batch/jobs/event/batch/entity/CargoEntity.java new file mode 100644 index 0000000..8e2e95d --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/event/batch/entity/CargoEntity.java @@ -0,0 +1,26 @@ +package com.snp.batch.jobs.event.batch.entity; + +import com.snp.batch.common.batch.entity.BaseEntity; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +@Data +@SuperBuilder +@NoArgsConstructor +@AllArgsConstructor +@EqualsAndHashCode(callSuper = true) +public class CargoEntity extends BaseEntity { + private Integer eventID; + private String sequence; + private String ihslrOrImoShipNo; + private String type; + private Integer quantity; + private String unitShort; + private String unit; + private String text; + private String cargoDamage; + private String dangerous; +} diff --git a/src/main/java/com/snp/batch/jobs/event/batch/entity/EventDetailEntity.java b/src/main/java/com/snp/batch/jobs/event/batch/entity/EventDetailEntity.java new file mode 100644 index 0000000..b8f04ee --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/event/batch/entity/EventDetailEntity.java @@ -0,0 +1,62 @@ +package com.snp.batch.jobs.event.batch.entity; + +import com.snp.batch.common.batch.entity.BaseEntity; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +import java.util.List; + +@Data +@SuperBuilder +@NoArgsConstructor +@AllArgsConstructor +@EqualsAndHashCode(callSuper = true) +public class EventDetailEntity extends BaseEntity { + private Integer incidentID; + private Integer eventID; + private Integer eventTypeID; + private String eventType; + private String significance; + private String headline; + private String ihslrOrImoShipNo; + private String vesselName; + private String vesselType; + private String vesselTypeDecode; + private String vesselFlagCode; + private String vesselFlagDecode; + private String cargoLoadingStatusCode; + private Integer vesselDWT; + private Integer vesselGT; + private Integer ldtAtTime; + private Integer dateOfBuild; + private String registeredOwnerCodeAtTime; + private String registeredOwnerAtTime; + private String registeredOwnerCountryCodeAtTime; + private String registeredOwnerCountryAtTime; + private String weather; + private String eventTypeDetail; + private Integer eventTypeDetailID; + private String casualtyAction; + private String locationName; + private String townName; + private Integer marsdenGridReference; + private String environmentLocation; + private String casualtyZone; + private String casualtyZoneCode; + private String countryCode; + private String attemptedBoarding; + private String description; + private String pollutant; + private String pollutantUnit; + private Double pollutantQuantity; + private String publishedDate; + private String component2; + private String firedUpon; + + private List cargoes; + private List humanCasualties; + private List relationships; +} diff --git a/src/main/java/com/snp/batch/jobs/event/batch/entity/EventEntity.java b/src/main/java/com/snp/batch/jobs/event/batch/entity/EventEntity.java index 19352c4..6a55161 100644 --- a/src/main/java/com/snp/batch/jobs/event/batch/entity/EventEntity.java +++ b/src/main/java/com/snp/batch/jobs/event/batch/entity/EventEntity.java @@ -1,7 +1,6 @@ package com.snp.batch.jobs.event.batch.entity; import com.snp.batch.common.batch.entity.BaseEntity; -import jakarta.persistence.Embedded; import lombok.AllArgsConstructor; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/src/main/java/com/snp/batch/jobs/event/batch/entity/HumanCasualtyEntity.java b/src/main/java/com/snp/batch/jobs/event/batch/entity/HumanCasualtyEntity.java new file mode 100644 index 0000000..d52bd9e --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/event/batch/entity/HumanCasualtyEntity.java @@ -0,0 +1,21 @@ +package com.snp.batch.jobs.event.batch.entity; + +import com.snp.batch.common.batch.entity.BaseEntity; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +@Data +@SuperBuilder +@NoArgsConstructor +@AllArgsConstructor +@EqualsAndHashCode(callSuper = true) +public class HumanCasualtyEntity extends BaseEntity { + private Integer eventID; + private String scope; + private String type; + private String qualifier; + private Integer count; +} diff --git a/src/main/java/com/snp/batch/jobs/event/batch/entity/RelationshipEntity.java b/src/main/java/com/snp/batch/jobs/event/batch/entity/RelationshipEntity.java new file mode 100644 index 0000000..e3c5ea7 --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/event/batch/entity/RelationshipEntity.java @@ -0,0 +1,23 @@ +package com.snp.batch.jobs.event.batch.entity; + +import com.snp.batch.common.batch.entity.BaseEntity; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +@Data +@SuperBuilder +@NoArgsConstructor +@AllArgsConstructor +@EqualsAndHashCode(callSuper = true) +public class RelationshipEntity extends BaseEntity { + private String incidentID; + private Integer eventID; + private String relationshipType; + private String relationshipTypeCode; + private Integer eventID2; + private String eventType; + private String eventTypeCode; +} diff --git a/src/main/java/com/snp/batch/jobs/event/batch/processor/EventDataProcessor.java b/src/main/java/com/snp/batch/jobs/event/batch/processor/EventDataProcessor.java index 329ab54..9d48d89 100644 --- a/src/main/java/com/snp/batch/jobs/event/batch/processor/EventDataProcessor.java +++ b/src/main/java/com/snp/batch/jobs/event/batch/processor/EventDataProcessor.java @@ -1,34 +1,73 @@ package com.snp.batch.jobs.event.batch.processor; import com.snp.batch.common.batch.processor.BaseProcessor; -import com.snp.batch.jobs.event.batch.dto.EventDto; -import com.snp.batch.jobs.event.batch.entity.EventEntity; +import com.snp.batch.jobs.event.batch.dto.CargoDto; +import com.snp.batch.jobs.event.batch.dto.EventDetailDto; +import com.snp.batch.jobs.event.batch.dto.HumanCasualtyDto; +import com.snp.batch.jobs.event.batch.dto.RelationshipDto; +import com.snp.batch.jobs.event.batch.entity.EventDetailEntity; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; +import java.util.stream.Collectors; + @Slf4j @Component -public class EventDataProcessor extends BaseProcessor { +public class EventDataProcessor extends BaseProcessor { @Override - protected EventEntity processItem(EventDto dto) throws Exception { - log.debug("Event 데이터 처리 시작: Event ID = {}", dto.getEventId()); + protected EventDetailEntity processItem(EventDetailDto dto) throws Exception { + log.debug("Event 데이터 처리 시작: Event ID = {}", dto.getEventID()); - EventEntity entity = EventEntity.builder() - .incidentId(dto.getIncidentId()) - .eventId(dto.getEventId()) - .startDate(dto.getStartDate()) + EventDetailEntity entity = EventDetailEntity.builder() + .incidentID(dto.getIncidentID()) + .eventID(dto.getEventID()) + .eventTypeID(dto.getEventTypeID()) .eventType(dto.getEventType()) .significance(dto.getSignificance()) .headline(dto.getHeadline()) - .endDate(dto.getEndDate()) - .ihslRorImoShipNo(dto.getIhslRorImoShipNo()) + .ihslrOrImoShipNo(dto.getIhslrOrImoShipNo()) .vesselName(dto.getVesselName()) .vesselType(dto.getVesselType()) + .vesselTypeDecode(dto.getVesselTypeDecode()) + .vesselFlagCode(dto.getVesselFlagCode()) + .vesselFlagDecode(dto.getVesselFlagDecode()) + .cargoLoadingStatusCode(dto.getCargoLoadingStatusCode()) + .vesselDWT(dto.getVesselDWT()) + .vesselGT(dto.getVesselGT()) + .ldtAtTime(dto.getLdtAtTime()) + .dateOfBuild(dto.getDateOfBuild()) + .registeredOwnerCodeAtTime(dto.getRegisteredOwnerCodeAtTime()) + .registeredOwnerAtTime(dto.getRegisteredOwnerAtTime()) + .registeredOwnerCountryCodeAtTime(dto.getRegisteredOwnerCountryCodeAtTime()) + .registeredOwnerCountryAtTime(dto.getRegisteredOwnerCountryAtTime()) + .weather(dto.getWeather()) + .eventTypeDetail(dto.getEventTypeDetail()) + .eventTypeDetailID(dto.getEventTypeDetailID()) + .casualtyAction(dto.getCasualtyAction()) .locationName(dto.getLocationName()) + .townName(dto.getTownName()) + .marsdenGridReference(dto.getMarsdenGridReference()) + .environmentLocation(dto.getEnvironmentLocation()) + .casualtyZone(dto.getCasualtyZone()) + .casualtyZoneCode(dto.getCasualtyZoneCode()) + .countryCode(dto.getCountryCode()) + .attemptedBoarding(dto.getAttemptedBoarding()) + .description(dto.getDescription()) + .pollutant(dto.getPollutant()) + .pollutantUnit(dto.getPollutantUnit()) + .pollutantQuantity(dto.getPollutantQuantity()) .publishedDate(dto.getPublishedDate()) + .component2(dto.getComponent2()) + .firedUpon(dto.getFiredUpon()) + .cargoes(dto.getCargoes() != null ? + dto.getCargoes().stream().map(CargoDto::toEntity).collect(Collectors.toList()) : null) + .humanCasualties(dto.getHumanCasualties() != null ? + dto.getHumanCasualties().stream().map(HumanCasualtyDto::toEntity).collect(Collectors.toList()) : null) + .relationships(dto.getRelationships() != null ? + dto.getRelationships().stream().map(RelationshipDto::toEntity).collect(Collectors.toList()) : null) .build(); - log.debug("Event 데이터 처리 완료: Event ID = {}", dto.getEventId()); + log.debug("Event 데이터 처리 완료: Event ID = {}", dto.getEventID()); return entity; } diff --git a/src/main/java/com/snp/batch/jobs/event/batch/reader/EventDataReader.java b/src/main/java/com/snp/batch/jobs/event/batch/reader/EventDataReader.java index e50ef36..096cc46 100644 --- a/src/main/java/com/snp/batch/jobs/event/batch/reader/EventDataReader.java +++ b/src/main/java/com/snp/batch/jobs/event/batch/reader/EventDataReader.java @@ -1,20 +1,24 @@ package com.snp.batch.jobs.event.batch.reader; import com.snp.batch.common.batch.reader.BaseApiReader; -import com.snp.batch.jobs.event.batch.dto.EventDto; -import com.snp.batch.jobs.event.batch.dto.EventResponse; +import com.snp.batch.jobs.event.batch.dto.*; +import com.snp.batch.jobs.event.batch.dto.EventDetailDto; +import com.snp.batch.jobs.event.batch.entity.EventDetailEntity; +import com.snp.batch.jobs.shipdetail.batch.dto.ShipDetailComparisonData; import com.snp.batch.service.BatchDateService; import lombok.extern.slf4j.Slf4j; -import org.springframework.core.ParameterizedTypeReference; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.web.reactive.function.client.WebClient; +import java.time.LocalDate; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; @Slf4j -public class EventDataReader extends BaseApiReader { +public class EventDataReader extends BaseApiReader { private final JdbcTemplate jdbcTemplate; private final BatchDateService batchDateService; // ✨ BatchDateService 필드 추가 @@ -22,6 +26,7 @@ public class EventDataReader extends BaseApiReader { super(webClient); this.jdbcTemplate = jdbcTemplate; this.batchDateService = batchDateService; + enableChunkMode(); // ✨ Chunk 모드 활성화 } @Override @@ -33,27 +38,138 @@ public class EventDataReader extends BaseApiReader { protected String getApiPath() { return "/MaritimeWCF/MaritimeAndTradeEventsService.svc/RESTFul/GetEventListByEventChangeDateRange"; } + + protected String getEventDetailApiPath(){ + return "/MaritimeWCF/MaritimeAndTradeEventsService.svc/RESTFul/GetEventDataByEventID"; + } protected String getApiKey() {return "EVENT_IMPORT_JOB";} - @Override - protected List fetchDataFromApi() { - try { - log.info("Event API 호출 시작"); - EventResponse response = callEventApiWithBatch(); + // 배치 처리 상태 + private List eventIds; + // DB 해시값을 저장할 맵 + private int currentBatchIndex = 0; + private final int batchSize = 1; + + @Override + protected void resetCustomState() { + this.currentBatchIndex = 0; + this.eventIds = null; + } + @Override + protected void beforeFetch(){ + // 1. 기간내 기록된 Event List 조회 (API 요청) + log.info("Event API 호출"); + EventResponse response = callEventApiWithBatch(); + // 2-1. Event List 에서 EventID List 추출 + // TODO: 2-2. Event List 에서 Map> 추출 + eventIds = extractEventIdList(response); + log.info("EvnetId List 추출 완료 : {} 개", eventIds.size()); + + updateApiCallStats(eventIds.size(), 0); + } + @Override + protected List fetchNextBatch() throws Exception { + // 3. EventID List 로 Event Detail 조회 (API요청) : 청크단위 실행 + // 모든 배치 처리 완료 확인 + if (eventIds == null || currentBatchIndex >= eventIds.size()) { + return null; // Job 종료 + } + + // 현재 배치의 시작/끝 인덱스 계산 + int startIndex = currentBatchIndex; + int endIndex = Math.min(currentBatchIndex + batchSize, eventIds.size()); + + // 현재 배치의 IMO 번호 추출 (100개) + List currentBatch = eventIds.subList(startIndex, endIndex); + + int currentBatchNumber = (currentBatchIndex / batchSize) + 1; + int totalBatches = (int) Math.ceil((double) eventIds.size() / batchSize); + + log.info("[{}] 배치 {}/{} 처리 중 (Event ID : {} 개)...", + getReaderName(), currentBatchNumber, totalBatches, currentBatch.size()); + + try { + // API 호출 + EventDetailResponse response = callEventDetailApiWithBatch(currentBatch.get(0)); + // 다음 배치로 인덱스 이동 + currentBatchIndex = endIndex; + + List eventDetailList = new ArrayList<>(); + // TODO: getEventDetailDto에 Map> 데이터 세팅 + eventDetailList.add(response.getEventDetailDto()); + + // 응답 처리 + if (response != null && response.getEventDetailDto() != null) { + + log.info("[{}] 배치 {}/{} 완료: {} 건 조회", + getReaderName(), currentBatchNumber, totalBatches, eventDetailList.size()); + + // API 호출 통계 업데이트 + updateApiCallStats(totalBatches, currentBatchNumber); + + // API 과부하 방지 (다음 배치 전 0.5초 대기) + if (currentBatchIndex < eventIds.size()) { + Thread.sleep(500); + } + + return eventDetailList; - if (response != null && response.getMaritimeEvents() != null) { - log.info("API 응답 성공: 총 {} 개의 Event 데이터 수신", response.getEventCount()); - return response.getMaritimeEvents(); } else { - log.warn("API 응답이 null이거나 Event 데이터가 없습니다"); - return new ArrayList<>(); + log.warn("[{}] 배치 {}/{} 응답 없음", + getReaderName(), currentBatchNumber, totalBatches); + + // API 호출 통계 업데이트 (실패도 카운트) + updateApiCallStats(totalBatches, currentBatchNumber); + + return Collections.emptyList(); } } catch (Exception e) { - log.error("Event API 호출 실패", e); - log.error("에러 메시지: {}", e.getMessage()); - return new ArrayList<>(); + log.error("[{}] 배치 {}/{} 처리 중 오류: {}", + getReaderName(), currentBatchNumber, totalBatches, e.getMessage(), e); + + // 오류 발생 시에도 다음 배치로 이동 (부분 실패 허용) + currentBatchIndex = endIndex; + + // 빈 리스트 반환 (Job 계속 진행) + return Collections.emptyList(); } + + + } + + @Override + protected void afterFetch(List data) { + int totalBatches = (int) Math.ceil((double) eventIds.size() / batchSize); + try{ + if (data == null) { + // 3. ✨ 배치 성공 시 상태 업데이트 (트랜잭션 커밋 직전에 실행) + LocalDate successDate = LocalDate.now(); // 현재 배치 실행 시점의 날짜 (Reader의 toDay와 동일한 값) + batchDateService.updateLastSuccessDate(getApiKey(), successDate); + log.info("batch_last_execution update 완료 : {}", getApiKey()); + + log.info("[{}] 전체 {} 개 배치 처리 완료", getReaderName(), totalBatches); + log.info("[{}] 총 {} 개의 Event ID에 대한 API 호출 종료", + getReaderName(), eventIds.size()); + } + }catch (Exception e){ + log.info("[{}] 전체 {} 개 배치 처리 실패", getReaderName(), totalBatches); + log.info("[{}] 총 {} 개의 Event ID에 대한 API 호출 종료", + getReaderName(), eventIds.size()); + } + } + + private List extractEventIdList(EventResponse response) { + if (response.getMaritimeEvents() == null) { + return Collections.emptyList(); + } + return response.getMaritimeEvents() .stream() + // ShipDto 객체에서 imoNumber 필드 (String 타입)를 추출 + .map(EventDto::getEventId) + // IMO 번호가 null이 아닌 경우만 필터링 (선택 사항이지만 안전성을 위해) + .filter(eventId -> eventId != null) + // 추출된 String imoNumber들을 List으로 수집 + .collect(Collectors.toList()); } private EventResponse callEventApiWithBatch() { @@ -77,4 +193,18 @@ public class EventDataReader extends BaseApiReader { .block(); } + private EventDetailResponse callEventDetailApiWithBatch(Long eventId) { + String url = getEventDetailApiPath(); + log.info("[{}] API 호출: {}", getReaderName(), url); + + return webClient.get() + .uri(url, uriBuilder -> uriBuilder + // 맵에서 파라미터 값을 동적으로 가져와 세팅 + .queryParam("eventID", eventId) + .build()) + .retrieve() + .bodyToMono(EventDetailResponse.class) + .block(); + } + } diff --git a/src/main/java/com/snp/batch/jobs/event/batch/repository/EventRepository.java b/src/main/java/com/snp/batch/jobs/event/batch/repository/EventRepository.java index 2448130..da65ba6 100644 --- a/src/main/java/com/snp/batch/jobs/event/batch/repository/EventRepository.java +++ b/src/main/java/com/snp/batch/jobs/event/batch/repository/EventRepository.java @@ -1,9 +1,15 @@ package com.snp.batch.jobs.event.batch.repository; -import com.snp.batch.jobs.event.batch.entity.EventEntity; +import com.snp.batch.jobs.event.batch.entity.CargoEntity; +import com.snp.batch.jobs.event.batch.entity.EventDetailEntity; +import com.snp.batch.jobs.event.batch.entity.HumanCasualtyEntity; +import com.snp.batch.jobs.event.batch.entity.RelationshipEntity; import java.util.List; public interface EventRepository { - void saveEventAll(List items); + void saveEventAll(List items); + void saveCargoAll(List items); + void saveHumanCasualtyAll(List items); + void saveRelationshipAll(List items); } diff --git a/src/main/java/com/snp/batch/jobs/event/batch/repository/EventRepositoryImpl.java b/src/main/java/com/snp/batch/jobs/event/batch/repository/EventRepositoryImpl.java index a45c451..d955bba 100644 --- a/src/main/java/com/snp/batch/jobs/event/batch/repository/EventRepositoryImpl.java +++ b/src/main/java/com/snp/batch/jobs/event/batch/repository/EventRepositoryImpl.java @@ -1,7 +1,12 @@ package com.snp.batch.jobs.event.batch.repository; import com.snp.batch.common.batch.repository.BaseJdbcRepository; -import com.snp.batch.jobs.event.batch.entity.EventEntity; +import com.snp.batch.jobs.event.batch.entity.CargoEntity; +import com.snp.batch.jobs.event.batch.entity.EventDetailEntity; +import com.snp.batch.jobs.event.batch.entity.HumanCasualtyEntity; +import com.snp.batch.jobs.event.batch.entity.RelationshipEntity; +import com.snp.batch.jobs.shipdetail.batch.entity.GroupBeneficialOwnerHistoryEntity; +import com.snp.batch.jobs.shipdetail.batch.repository.ShipDetailSql; import lombok.extern.slf4j.Slf4j; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.jdbc.core.RowMapper; @@ -13,7 +18,7 @@ import java.util.List; @Slf4j @Repository("EventRepository") -public class EventRepositoryImpl extends BaseJdbcRepository implements EventRepository { +public class EventRepositoryImpl extends BaseJdbcRepository implements EventRepository { public EventRepositoryImpl(JdbcTemplate jdbcTemplate) { super(jdbcTemplate); @@ -25,12 +30,12 @@ public class EventRepositoryImpl extends BaseJdbcRepository i } @Override - protected RowMapper getRowMapper() { + protected RowMapper getRowMapper() { return null; } @Override - protected Long extractId(EventEntity entity) { + protected Long extractId(EventDetailEntity entity) { return null; } @@ -42,7 +47,7 @@ public class EventRepositoryImpl extends BaseJdbcRepository i @Override protected String getUpdateSql() { return """ - INSERT INTO snp_data.event ( + INSERT INTO snp_data.event_detail ( Event_ID, Incident_ID, IHSLRorIMOShipNo, Vessel_Name, Vessel_Type, Event_Type, Significance, Headline, Location_Name, Published_Date, Event_Start_Date, Event_End_Date, batch_flag @@ -69,48 +74,168 @@ public class EventRepositoryImpl extends BaseJdbcRepository i } @Override - protected void setInsertParameters(PreparedStatement ps, EventEntity entity) throws Exception { + protected void setInsertParameters(PreparedStatement ps, EventDetailEntity entity) throws Exception { } - @Override - protected void setUpdateParameters(PreparedStatement ps, EventEntity entity) throws Exception { - int idx = 1; - ps.setLong(idx++, entity.getEventId()); - ps.setLong(idx++, entity.getIncidentId()); - ps.setString(idx++, entity.getIhslRorImoShipNo()); - ps.setString(idx++, entity.getVesselName()); - ps.setString(idx++, entity.getVesselType()); - ps.setString(idx++, entity.getEventType()); - ps.setString(idx++, entity.getSignificance()); - ps.setString(idx++, entity.getHeadline()); - ps.setString(idx++, entity.getLocationName()); - ps.setString(idx++, entity.getPublishedDate()); - ps.setString(idx++, entity.getStartDate()); - ps.setString(idx++, entity.getEndDate()); - } - @Override protected String getEntityName() { - return "EventEntity"; + return "EventDetailEntity"; } @Override - public void saveEventAll(List items) { - if (items == null || items.isEmpty()) { - return; - } - jdbcTemplate.batchUpdate(getUpdateSql(), items, items.size(), + public void saveEventAll(List items) { + String entityName = "EventDetailEntity"; + String sql = EventSql.getEventDetailUpdateSql(); + + jdbcTemplate.batchUpdate(sql, items, items.size(), (ps, entity) -> { try { - setUpdateParameters(ps, entity); + setUpdateParameters(ps, (EventDetailEntity) entity); } catch (Exception e) { log.error("배치 수정 파라미터 설정 실패", e); throw new RuntimeException(e); } }); - log.info("{} 전체 저장 완료: 수정={} 건", getEntityName(), items.size()); + log.info("{} 배치 삽입 완료: {} 건", entityName, items.size()); + } + + @Override + public void saveCargoAll(List items) { + String entityName = "CargoEntity"; + String sql = EventSql.getEventCargoSql(); + + jdbcTemplate.batchUpdate(sql, items, items.size(), + (ps, entity) -> { + try { + setCargoInsertParameters(ps, (CargoEntity) entity); + } catch (Exception e) { + log.error("배치 삽입 파라미터 설정 실패 - " + entityName, e); + throw new RuntimeException(e); + } + }); + + log.info("{} 배치 삽입 완료: {} 건", entityName, items.size()); + } + + @Override + public void saveHumanCasualtyAll(List items) { + String entityName = "HumanCasualtyEntity"; + String sql = EventSql.getEventHumanCasualtySql(); + + jdbcTemplate.batchUpdate(sql, items, items.size(), + (ps, entity) -> { + try { + setHumanCasualtyInsertParameters(ps, (HumanCasualtyEntity) entity); + } catch (Exception e) { + log.error("배치 삽입 파라미터 설정 실패 - " + entityName, e); + throw new RuntimeException(e); + } + }); + + log.info("{} 배치 삽입 완료: {} 건", entityName, items.size()); + } + + @Override + public void saveRelationshipAll(List items) { + String entityName = "RelationshipEntity"; + String sql = EventSql.getEventRelationshipSql(); + + jdbcTemplate.batchUpdate(sql, items, items.size(), + (ps, entity) -> { + try { + setRelationshipInsertParameters(ps, (RelationshipEntity) entity); + } catch (Exception e) { + log.error("배치 삽입 파라미터 설정 실패 - " + entityName, e); + throw new RuntimeException(e); + } + }); + + log.info("{} 배치 삽입 완료: {} 건", entityName, items.size()); + } + + @Override + protected void setUpdateParameters(PreparedStatement ps, EventDetailEntity entity) throws Exception { + int idx = 1; + ps.setObject(idx++, entity.getEventID()); // event_id + ps.setObject(idx++, entity.getIncidentID()); // incident_id (누락됨) + ps.setObject(idx++, entity.getIhslrOrImoShipNo()); // ihslrorimoshipno (누락됨) + ps.setObject(idx++, entity.getPublishedDate()); // published_date (누락됨) + ps.setString(idx++, entity.getAttemptedBoarding()); // attempted_boarding + ps.setString(idx++, entity.getCargoLoadingStatusCode());// cargo_loading_status_code + ps.setString(idx++, entity.getCasualtyAction()); // casualty_action + ps.setString(idx++, entity.getCasualtyZone()); // casualty_zone + ps.setString(idx++, entity.getCasualtyZoneCode()); // casualty_zone_code + ps.setString(idx++, entity.getComponent2()); // component2 + + // 11~20 + ps.setString(idx++, entity.getCountryCode()); // country_code + ps.setObject(idx++, entity.getDateOfBuild()); // date_of_build (Integer) + ps.setString(idx++, entity.getDescription()); // description + ps.setString(idx++, entity.getEnvironmentLocation()); // environment_location + ps.setString(idx++, entity.getLocationName()); // location_name (누락됨) + ps.setObject(idx++, entity.getMarsdenGridReference()); // marsden_grid_reference (Integer) + ps.setString(idx++, entity.getTownName()); // town_name + ps.setString(idx++, entity.getEventType()); // event_type (누락됨) + ps.setString(idx++, entity.getEventTypeDetail()); // event_type_detail + ps.setObject(idx++, entity.getEventTypeDetailID()); // event_type_detail_id (Integer) + + // 21~30 + ps.setObject(idx++, entity.getEventTypeID()); // event_type_id (Integer) + ps.setString(idx++, entity.getFiredUpon()); // fired_upon + ps.setString(idx++, entity.getHeadline()); // headline (누락됨) + ps.setObject(idx++, entity.getLdtAtTime()); // ldt_at_time (Integer) + ps.setString(idx++, entity.getSignificance()); // significance (누락됨) + ps.setString(idx++, entity.getWeather()); // weather + ps.setString(idx++, entity.getPollutant()); // pollutant + ps.setObject(idx++, entity.getPollutantQuantity()); // pollutant_quantity (Double) + ps.setString(idx++, entity.getPollutantUnit()); // pollutant_unit + ps.setString(idx++, entity.getRegisteredOwnerCodeAtTime()); // registered_owner_code_at_time + + // 31~40 + ps.setString(idx++, entity.getRegisteredOwnerAtTime()); // registered_owner_at_time + ps.setString(idx++, entity.getRegisteredOwnerCountryCodeAtTime()); // registered_owner_country_code_at_time + ps.setString(idx++, entity.getRegisteredOwnerCountryAtTime()); // registered_owner_country_at_time + ps.setObject(idx++, entity.getVesselDWT()); // vessel_dwt (Integer) + ps.setString(idx++, entity.getVesselFlagCode()); // vessel_flag_code + ps.setString(idx++, entity.getVesselFlagDecode()); // vessel_flag_decode (누락됨) + ps.setObject(idx++, entity.getVesselGT()); // vessel_gt (Integer) + ps.setString(idx++, entity.getVesselName()); // vessel_name (누락됨) + ps.setString(idx++, entity.getVesselType()); // vessel_type (누락됨) + ps.setString(idx++, entity.getVesselTypeDecode()); // vessel_type_decode + } + private void setCargoInsertParameters(PreparedStatement ps, CargoEntity entity)throws Exception{ + int idx = 1; + // INSERT 필드 + ps.setObject(idx++, entity.getEventID()); + ps.setString(idx++, entity.getSequence()); + ps.setString(idx++, entity.getIhslrOrImoShipNo()); + ps.setString(idx++, entity.getType()); + ps.setObject(idx++, entity.getQuantity()); // quantity 필드 (Entity에 없을 경우 null 처리) + ps.setString(idx++, entity.getUnitShort()); // unit_short 필드 + ps.setString(idx++, entity.getUnit()); + ps.setString(idx++, entity.getCargoDamage()); + ps.setString(idx++, entity.getDangerous()); + ps.setString(idx++, entity.getText()); + } + private void setHumanCasualtyInsertParameters(PreparedStatement ps, HumanCasualtyEntity entity)throws Exception{ + int idx = 1; + ps.setObject(idx++, entity.getEventID()); + ps.setString(idx++, entity.getScope()); + ps.setString(idx++, entity.getType()); + ps.setString(idx++, entity.getQualifier()); + ps.setObject(idx++, entity.getCount()); + } + private void setRelationshipInsertParameters(PreparedStatement ps, RelationshipEntity entity)throws Exception{ + int idx = 1; + ps.setString(idx++, entity.getIncidentID()); + ps.setObject(idx++, entity.getEventID()); + ps.setString(idx++, entity.getRelationshipType()); + ps.setString(idx++, entity.getRelationshipTypeCode()); + ps.setObject(idx++, entity.getEventID2()); + ps.setString(idx++, entity.getEventType()); + ps.setString(idx++, entity.getEventTypeCode()); } private static void setStringOrNull(PreparedStatement ps, int index, String value) throws Exception { diff --git a/src/main/java/com/snp/batch/jobs/event/batch/repository/EventSql.java b/src/main/java/com/snp/batch/jobs/event/batch/repository/EventSql.java new file mode 100644 index 0000000..85a96f9 --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/event/batch/repository/EventSql.java @@ -0,0 +1,124 @@ +package com.snp.batch.jobs.event.batch.repository; + +public class EventSql { + public static String getEventDetailUpdateSql(){ + return """ + INSERT INTO snp_data.event_detail ( + event_id, incident_id, ihslrorimoshipno, published_date, + attempted_boarding, cargo_loading_status_code, casualty_action, + casualty_zone, casualty_zone_code, component2, country_code, + date_of_build, description, environment_location, location_name, + marsden_grid_reference, town_name, event_type, event_type_detail, + event_type_detail_id, event_type_id, fired_upon, headline, + ldt_at_time, significance, weather, pollutant, pollutant_quantity, + pollutant_unit, registered_owner_code_at_time, registered_owner_at_time, + registered_owner_country_code_at_time, registered_owner_country_at_time, + vessel_dwt, vessel_flag_code, vessel_flag_decode, vessel_gt, + vessel_name, vessel_type, vessel_type_decode + ) + VALUES ( + ?, ?, ?, ?::timestamptz, ?, ?, ?, ?, ?, ?, + ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, + ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, + ?, ?, ?, ?, ?, ?, ?, ?, ?, ? + ) + ON CONFLICT (event_id) + DO UPDATE SET + incident_id = EXCLUDED.incident_id, + ihslrorimoshipno = EXCLUDED.ihslrorimoshipno, + published_date = EXCLUDED.published_date, + attempted_boarding = EXCLUDED.attempted_boarding, + cargo_loading_status_code = EXCLUDED.cargo_loading_status_code, + casualty_action = EXCLUDED.casualty_action, + casualty_zone = EXCLUDED.casualty_zone, + casualty_zone_code = EXCLUDED.casualty_zone_code, + component2 = EXCLUDED.component2, + country_code = EXCLUDED.country_code, + date_of_build = EXCLUDED.date_of_build, + description = EXCLUDED.description, + environment_location = EXCLUDED.environment_location, + location_name = EXCLUDED.location_name, + marsden_grid_reference = EXCLUDED.marsden_grid_reference, + town_name = EXCLUDED.town_name, + event_type = EXCLUDED.event_type, + event_type_detail = EXCLUDED.event_type_detail, + event_type_detail_id = EXCLUDED.event_type_detail_id, + event_type_id = EXCLUDED.event_type_id, + fired_upon = EXCLUDED.fired_upon, + headline = EXCLUDED.headline, + ldt_at_time = EXCLUDED.ldt_at_time, + significance = EXCLUDED.significance, + weather = EXCLUDED.weather, + pollutant = EXCLUDED.pollutant, + pollutant_quantity = EXCLUDED.pollutant_quantity, + pollutant_unit = EXCLUDED.pollutant_unit, + registered_owner_code_at_time = EXCLUDED.registered_owner_code_at_time, + registered_owner_at_time = EXCLUDED.registered_owner_at_time, + registered_owner_country_code_at_time = EXCLUDED.registered_owner_country_code_at_time, + registered_owner_country_at_time = EXCLUDED.registered_owner_country_at_time, + vessel_dwt = EXCLUDED.vessel_dwt, + vessel_flag_code = EXCLUDED.vessel_flag_code, + vessel_flag_decode = EXCLUDED.vessel_flag_decode, + vessel_gt = EXCLUDED.vessel_gt, + vessel_name = EXCLUDED.vessel_name, + vessel_type = EXCLUDED.vessel_type, + vessel_type_decode = EXCLUDED.vessel_type_decode, + batch_flag = 'N'; + """; + } + + public static String getEventCargoSql(){ + return """ + INSERT INTO snp_data.event_cargo ( + event_id, "sequence", ihslrorimoshipno, "type", quantity, + unit_short, unit, cargo_damage, dangerous, "text" + ) + VALUES ( + ?, ?, ?, ?, ?, + ?, ?, ?, ?, ? + ) + ON CONFLICT (event_id, ihslrorimoshipno, "type", "sequence") + DO UPDATE SET + quantity = EXCLUDED.quantity, + unit_short = EXCLUDED.unit_short, + unit = EXCLUDED.unit, + cargo_damage = EXCLUDED.cargo_damage, + dangerous = EXCLUDED.dangerous, + "text" = EXCLUDED."text", + batch_flag = 'N'; + """; + } + + public static String getEventRelationshipSql(){ + return """ + INSERT INTO snp_data.event_relationship ( + incident_id, event_id, relationship_type, relationship_type_code, + event_id_2, event_type, event_type_code + ) + VALUES ( + ?, ?, ?, ?, + ?, ?, ? + ) + ON CONFLICT (incident_id, event_id, event_id_2, event_type_code, relationship_type_code) + DO UPDATE SET + relationship_type = EXCLUDED.relationship_type, + event_type = EXCLUDED.event_type, + batch_flag = 'N'; + """; + } + + public static String getEventHumanCasualtySql(){ + return """ + INSERT INTO snp_data.event_humancasualty ( + event_id, "scope", "type", qualifier, "count" + ) + VALUES ( + ?, ?, ?, ?, ? + ) + ON CONFLICT (event_id, "scope", "type", qualifier) + DO UPDATE SET + "count" = EXCLUDED."count", + batch_flag = 'N'; + """; + } +} diff --git a/src/main/java/com/snp/batch/jobs/event/batch/writer/EventDataWriter.java b/src/main/java/com/snp/batch/jobs/event/batch/writer/EventDataWriter.java index 9a61e6a..936ce48 100644 --- a/src/main/java/com/snp/batch/jobs/event/batch/writer/EventDataWriter.java +++ b/src/main/java/com/snp/batch/jobs/event/batch/writer/EventDataWriter.java @@ -1,35 +1,49 @@ package com.snp.batch.jobs.event.batch.writer; import com.snp.batch.common.batch.writer.BaseWriter; -import com.snp.batch.jobs.event.batch.entity.EventEntity; +import com.snp.batch.jobs.event.batch.entity.EventDetailEntity; import com.snp.batch.jobs.event.batch.repository.EventRepository; -import com.snp.batch.service.BatchDateService; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; +import org.springframework.util.CollectionUtils; -import java.time.LocalDate; import java.util.List; @Slf4j @Component -public class EventDataWriter extends BaseWriter { +public class EventDataWriter extends BaseWriter { private final EventRepository eventRepository; - private final BatchDateService batchDateService; // ✨ BatchDateService 필드 추가 protected String getApiKey() {return "EVENT_IMPORT_JOB";} - public EventDataWriter(EventRepository eventRepository, BatchDateService batchDateService) { + public EventDataWriter(EventRepository eventRepository) { super("EventRepository"); this.eventRepository = eventRepository; - this.batchDateService = batchDateService; } @Override - protected void writeItems(List items) throws Exception { - eventRepository.saveEventAll(items); - log.info("Event 저장 완료: 수정={} 건", items.size()); + protected void writeItems(List items) throws Exception { - // ✨ 배치 성공 시 상태 업데이트 (트랜잭션 커밋 직전에 실행) - LocalDate successDate = LocalDate.now(); - batchDateService.updateLastSuccessDate(getApiKey(), successDate); - log.info("batch_last_execution update 완료 : {}", getApiKey()); + if (CollectionUtils.isEmpty(items)) { + return; + } + + // 1. EventDetail 메인 데이터 저장 + eventRepository.saveEventAll(items); + + for (EventDetailEntity event : items) { + // 2. CargoEntityList Save + if (!CollectionUtils.isEmpty(event.getCargoes())) { + eventRepository.saveCargoAll(event.getCargoes()); + } + // 3. HumanCasualtyEntityList Save + if (!CollectionUtils.isEmpty(event.getHumanCasualties())) { + eventRepository.saveHumanCasualtyAll(event.getHumanCasualties()); + } + // 4. RelationshipEntityList Save + if (!CollectionUtils.isEmpty(event.getRelationships())) { + eventRepository.saveRelationshipAll(event.getRelationships()); + } + } + + log.info("Batch Write 완료: {} 건의 Event 처리됨", items.size()); } } From 75531ab5e52c3d9ce5a976dd28391a002401c3de Mon Sep 17 00:00:00 2001 From: Kim JiMyeung Date: Mon, 22 Dec 2025 13:11:25 +0900 Subject: [PATCH 8/9] =?UTF-8?q?startDate,=20endDate=EB=A1=9C=EC=A7=81?= =?UTF-8?q?=EC=B2=98=EB=A6=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../jobs/event/batch/dto/EventDetailDto.java | 6 +- .../jobs/event/batch/dto/EventPeriod.java | 12 ++++ .../event/batch/entity/EventDetailEntity.java | 7 ++- .../batch/processor/EventDataProcessor.java | 4 +- .../event/batch/reader/EventDataReader.java | 58 ++++++++++++++----- .../batch/repository/EventRepositoryImpl.java | 14 +++-- .../jobs/event/batch/repository/EventSql.java | 6 +- 7 files changed, 83 insertions(+), 24 deletions(-) create mode 100644 src/main/java/com/snp/batch/jobs/event/batch/dto/EventPeriod.java diff --git a/src/main/java/com/snp/batch/jobs/event/batch/dto/EventDetailDto.java b/src/main/java/com/snp/batch/jobs/event/batch/dto/EventDetailDto.java index 15f13ac..38a2fd6 100644 --- a/src/main/java/com/snp/batch/jobs/event/batch/dto/EventDetailDto.java +++ b/src/main/java/com/snp/batch/jobs/event/batch/dto/EventDetailDto.java @@ -6,6 +6,8 @@ import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; +import java.time.LocalDate; +import java.time.LocalDateTime; import java.util.List; @Data @@ -16,7 +18,7 @@ public class EventDetailDto { @JsonProperty("IncidentID") private Integer incidentID; @JsonProperty("EventID") - private Integer eventID; + private Long eventID; @JsonProperty("EventTypeID") private Integer eventTypeID; @JsonProperty("EventType") @@ -93,6 +95,8 @@ public class EventDetailDto { private String component2; @JsonProperty("FiredUpon") private String firedUpon; + private String eventStartDate; + private String eventEndDate; @JsonProperty("Cargoes") private List cargoes; diff --git a/src/main/java/com/snp/batch/jobs/event/batch/dto/EventPeriod.java b/src/main/java/com/snp/batch/jobs/event/batch/dto/EventPeriod.java new file mode 100644 index 0000000..05e4703 --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/event/batch/dto/EventPeriod.java @@ -0,0 +1,12 @@ +package com.snp.batch.jobs.event.batch.dto; + +import lombok.AllArgsConstructor; +import lombok.Data; + +import java.time.LocalDateTime; + +@Data +@AllArgsConstructor +public class EventPeriod { + private String eventStartDate; + private String eventEndDate;} diff --git a/src/main/java/com/snp/batch/jobs/event/batch/entity/EventDetailEntity.java b/src/main/java/com/snp/batch/jobs/event/batch/entity/EventDetailEntity.java index b8f04ee..58b4bda 100644 --- a/src/main/java/com/snp/batch/jobs/event/batch/entity/EventDetailEntity.java +++ b/src/main/java/com/snp/batch/jobs/event/batch/entity/EventDetailEntity.java @@ -1,5 +1,6 @@ package com.snp.batch.jobs.event.batch.entity; +import com.fasterxml.jackson.datatype.jsr310.deser.LocalDateDeserializer; import com.snp.batch.common.batch.entity.BaseEntity; import lombok.AllArgsConstructor; import lombok.Data; @@ -7,6 +8,7 @@ import lombok.EqualsAndHashCode; import lombok.NoArgsConstructor; import lombok.experimental.SuperBuilder; +import java.time.LocalDateTime; import java.util.List; @Data @@ -16,7 +18,7 @@ import java.util.List; @EqualsAndHashCode(callSuper = true) public class EventDetailEntity extends BaseEntity { private Integer incidentID; - private Integer eventID; + private Long eventID; private Integer eventTypeID; private String eventType; private String significance; @@ -56,6 +58,9 @@ public class EventDetailEntity extends BaseEntity { private String component2; private String firedUpon; + private String eventStartDate; + private String eventEndDate; + private List cargoes; private List humanCasualties; private List relationships; diff --git a/src/main/java/com/snp/batch/jobs/event/batch/processor/EventDataProcessor.java b/src/main/java/com/snp/batch/jobs/event/batch/processor/EventDataProcessor.java index 9d48d89..74dd115 100644 --- a/src/main/java/com/snp/batch/jobs/event/batch/processor/EventDataProcessor.java +++ b/src/main/java/com/snp/batch/jobs/event/batch/processor/EventDataProcessor.java @@ -19,8 +19,8 @@ public class EventDataProcessor extends BaseProcessor { + + private Map eventPeriodMap; private final JdbcTemplate jdbcTemplate; private final BatchDateService batchDateService; // ✨ BatchDateService 필드 추가 @@ -39,10 +39,13 @@ public class EventDataReader extends BaseApiReader { return "/MaritimeWCF/MaritimeAndTradeEventsService.svc/RESTFul/GetEventListByEventChangeDateRange"; } - protected String getEventDetailApiPath(){ + protected String getEventDetailApiPath() { return "/MaritimeWCF/MaritimeAndTradeEventsService.svc/RESTFul/GetEventDataByEventID"; } - protected String getApiKey() {return "EVENT_IMPORT_JOB";} + + protected String getApiKey() { + return "EVENT_IMPORT_JOB"; + } // 배치 처리 상태 private List eventIds; @@ -54,9 +57,11 @@ public class EventDataReader extends BaseApiReader { protected void resetCustomState() { this.currentBatchIndex = 0; this.eventIds = null; + this.eventPeriodMap = new HashMap<>(); } + @Override - protected void beforeFetch(){ + protected void beforeFetch() { // 1. 기간내 기록된 Event List 조회 (API 요청) log.info("Event API 호출"); EventResponse response = callEventApiWithBatch(); @@ -65,8 +70,19 @@ public class EventDataReader extends BaseApiReader { eventIds = extractEventIdList(response); log.info("EvnetId List 추출 완료 : {} 개", eventIds.size()); + eventPeriodMap = response.getMaritimeEvents().stream() + .filter(e -> e.getEventId() != null) + .collect(Collectors.toMap( + EventDto::getEventId, + e -> new EventPeriod( + e.getStartDate(), + e.getEndDate() + ) + )); + updateApiCallStats(eventIds.size(), 0); } + @Override protected List fetchNextBatch() throws Exception { // 3. EventID List 로 Event Detail 조회 (API요청) : 청크단위 실행 @@ -95,12 +111,21 @@ public class EventDataReader extends BaseApiReader { currentBatchIndex = endIndex; List eventDetailList = new ArrayList<>(); - // TODO: getEventDetailDto에 Map> 데이터 세팅 - eventDetailList.add(response.getEventDetailDto()); // 응답 처리 if (response != null && response.getEventDetailDto() != null) { - + + // TODO: getEventDetailDto에 Map> 데이터 세팅 + EventDetailDto detailDto = response.getEventDetailDto(); + Long eventId = detailDto.getEventID(); + EventPeriod period = eventPeriodMap.get(eventId); + + if (period != null) { + detailDto.setEventStartDate(period.getEventStartDate()); + detailDto.setEventEndDate(period.getEventEndDate()); + } + + eventDetailList.add(response.getEventDetailDto()); log.info("[{}] 배치 {}/{} 완료: {} 건 조회", getReaderName(), currentBatchNumber, totalBatches, eventDetailList.size()); @@ -141,7 +166,7 @@ public class EventDataReader extends BaseApiReader { @Override protected void afterFetch(List data) { int totalBatches = (int) Math.ceil((double) eventIds.size() / batchSize); - try{ + try { if (data == null) { // 3. ✨ 배치 성공 시 상태 업데이트 (트랜잭션 커밋 직전에 실행) LocalDate successDate = LocalDate.now(); // 현재 배치 실행 시점의 날짜 (Reader의 toDay와 동일한 값) @@ -152,7 +177,7 @@ public class EventDataReader extends BaseApiReader { log.info("[{}] 총 {} 개의 Event ID에 대한 API 호출 종료", getReaderName(), eventIds.size()); } - }catch (Exception e){ + } catch (Exception e) { log.info("[{}] 전체 {} 개 배치 처리 실패", getReaderName(), totalBatches); log.info("[{}] 총 {} 개의 Event ID에 대한 API 호출 종료", getReaderName(), eventIds.size()); @@ -163,7 +188,7 @@ public class EventDataReader extends BaseApiReader { if (response.getMaritimeEvents() == null) { return Collections.emptyList(); } - return response.getMaritimeEvents() .stream() + return response.getMaritimeEvents().stream() // ShipDto 객체에서 imoNumber 필드 (String 타입)를 추출 .map(EventDto::getEventId) // IMO 번호가 null이 아닌 경우만 필터링 (선택 사항이지만 안전성을 위해) @@ -207,4 +232,11 @@ public class EventDataReader extends BaseApiReader { .block(); } + private LocalDateTime parseToLocalDate(String value) { + if (value == null || value.isBlank()) { + return null; + } + return LocalDateTime.parse(value); + } + } diff --git a/src/main/java/com/snp/batch/jobs/event/batch/repository/EventRepositoryImpl.java b/src/main/java/com/snp/batch/jobs/event/batch/repository/EventRepositoryImpl.java index d955bba..5cbe1e0 100644 --- a/src/main/java/com/snp/batch/jobs/event/batch/repository/EventRepositoryImpl.java +++ b/src/main/java/com/snp/batch/jobs/event/batch/repository/EventRepositoryImpl.java @@ -162,14 +162,16 @@ public class EventRepositoryImpl extends BaseJdbcRepository Date: Tue, 23 Dec 2025 09:42:50 +0900 Subject: [PATCH 9/9] =?UTF-8?q?risk,=20compliance=EC=9E=A1=20range?= =?UTF-8?q?=ED=98=95=ED=83=9C=EB=A1=9C=20=EC=88=98=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../config/RiskImportRangeJobConfig.java | 94 +++++++++++++ .../batch/reader/RiskDataRangeReader.java | 120 ++++++++++++++++ .../batch/repository/RiskRepositoryImpl.java | 2 +- .../config/SanctionUpdateRangeJobConfig.java | 98 ++++++++++++++ .../reader/ComplianceDataRangeReader.java | 128 ++++++++++++++++++ .../repository/ComplianceRepositoryImpl.java | 2 +- 6 files changed, 442 insertions(+), 2 deletions(-) create mode 100644 src/main/java/com/snp/batch/jobs/risk/batch/config/RiskImportRangeJobConfig.java create mode 100644 src/main/java/com/snp/batch/jobs/risk/batch/reader/RiskDataRangeReader.java create mode 100644 src/main/java/com/snp/batch/jobs/sanction/batch/config/SanctionUpdateRangeJobConfig.java create mode 100644 src/main/java/com/snp/batch/jobs/sanction/batch/reader/ComplianceDataRangeReader.java diff --git a/src/main/java/com/snp/batch/jobs/risk/batch/config/RiskImportRangeJobConfig.java b/src/main/java/com/snp/batch/jobs/risk/batch/config/RiskImportRangeJobConfig.java new file mode 100644 index 0000000..1ff38d5 --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/risk/batch/config/RiskImportRangeJobConfig.java @@ -0,0 +1,94 @@ +package com.snp.batch.jobs.risk.batch.config; + +import com.snp.batch.common.batch.config.BaseJobConfig; +import com.snp.batch.jobs.risk.batch.dto.RiskDto; +import com.snp.batch.jobs.risk.batch.entity.RiskEntity; +import com.snp.batch.jobs.risk.batch.processor.RiskDataProcessor; +import com.snp.batch.jobs.risk.batch.reader.RiskDataRangeReader; +import com.snp.batch.jobs.risk.batch.reader.RiskDataReader; +import com.snp.batch.jobs.risk.batch.writer.RiskDataWriter; +import com.snp.batch.jobs.sanction.batch.reader.ComplianceDataRangeReader; +import lombok.extern.slf4j.Slf4j; +import org.springframework.batch.core.Job; +import org.springframework.batch.core.Step; +import org.springframework.batch.core.configuration.annotation.StepScope; +import org.springframework.batch.core.repository.JobRepository; +import org.springframework.batch.item.ItemProcessor; +import org.springframework.batch.item.ItemReader; +import org.springframework.batch.item.ItemWriter; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.transaction.PlatformTransactionManager; +import org.springframework.web.reactive.function.client.WebClient; + +@Slf4j +@Configuration +public class RiskImportRangeJobConfig extends BaseJobConfig { + private final WebClient maritimeServiceApiWebClient; + private final RiskDataProcessor riskDataProcessor; + private final RiskDataWriter riskDataWriter; + private final RiskDataRangeReader riskDataRangeReader; + + @Override + protected int getChunkSize() { + return 5000; // API에서 5000개씩 가져오므로 chunk도 5000으로 설정 + } + public RiskImportRangeJobConfig( + JobRepository jobRepository, + PlatformTransactionManager transactionManager, + RiskDataProcessor riskDataProcessor, + RiskDataWriter riskDataWriter, + JdbcTemplate jdbcTemplate, + @Qualifier("maritimeServiceApiWebClient")WebClient maritimeServiceApiWebClient, RiskDataRangeReader riskDataRangeReader) { + super(jobRepository, transactionManager); + this.maritimeServiceApiWebClient = maritimeServiceApiWebClient; + this.riskDataProcessor = riskDataProcessor; + this.riskDataWriter = riskDataWriter; + this.riskDataRangeReader = riskDataRangeReader; + } + + @Override + protected String getJobName() { + return "RiskRangeImportJob"; + } + + @Override + protected String getStepName() { + return "RiskRangeImportStep"; + } + + @Override + protected ItemReader createReader() { + return riskDataRangeReader; + } + @Bean + @StepScope + public RiskDataRangeReader riskDataRangeReader( + @Value("#{jobParameters['fromDate']}") String startDate, + @Value("#{jobParameters['toDate']}") String stopDate + ) { + return new RiskDataRangeReader(maritimeServiceApiWebClient, startDate, stopDate); + } + + @Override + protected ItemProcessor createProcessor() { + return riskDataProcessor; + } + + @Override + protected ItemWriter createWriter() { return riskDataWriter; } + + @Bean(name = "RiskRangeImportJob") + public Job riskRangeImportJob() { + return job(); + } + + @Bean(name = "RiskRangeImportStep") + public Step riskRangeImportStep() { + return step(); + } + +} diff --git a/src/main/java/com/snp/batch/jobs/risk/batch/reader/RiskDataRangeReader.java b/src/main/java/com/snp/batch/jobs/risk/batch/reader/RiskDataRangeReader.java new file mode 100644 index 0000000..a29dd09 --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/risk/batch/reader/RiskDataRangeReader.java @@ -0,0 +1,120 @@ +package com.snp.batch.jobs.risk.batch.reader; + +import com.snp.batch.common.batch.reader.BaseApiReader; +import com.snp.batch.jobs.risk.batch.dto.RiskDto; +import com.snp.batch.jobs.sanction.batch.dto.ComplianceDto; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.core.ParameterizedTypeReference; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.web.reactive.function.client.WebClient; + +import java.time.LocalDate; +import java.time.format.DateTimeFormatter; +import java.util.Collections; +import java.util.List; + +@Slf4j +public class RiskDataRangeReader extends BaseApiReader { + + //TODO : + // 1. Core20 IMO_NUMBER 전체 조회 + // 2. IMO번호에 대한 마지막 AIS 신호 요청 (1회 최대 5000개 : Chunk 단위로 반복) + // 3. Response Data -> Core20에 업데이트 (Chunk 단위로 반복) + + private List allData; + private int currentBatchIndex = 0; + private final int batchSize = 100; + private String fromDate; + private String toDate; + public RiskDataRangeReader(WebClient webClient, + @Value("#{jobParameters['fromDate']}") String fromDate, + @Value("#{jobParameters['toDate']}") String toDate) { + super(webClient); + + // 날짜가 없으면 전날 하루 기준 + if (fromDate == null || fromDate.isBlank() || + toDate == null || toDate.isBlank()) { + + LocalDate yesterday = LocalDate.now().minusDays(1); + this.fromDate = yesterday.atStartOfDay().format(DateTimeFormatter.ISO_DATE_TIME) + "Z"; + this.toDate = yesterday.plusDays(1).atStartOfDay().format(DateTimeFormatter.ISO_DATE_TIME) + "Z"; + } else { + this.fromDate = fromDate; + this.toDate = toDate; + } + + enableChunkMode(); + } + + @Override + protected String getReaderName() { + return "riskDataRangeReader"; + } + + @Override + protected void resetCustomState() { + this.currentBatchIndex = 0; + this.allData = null; + } + + @Override + protected String getApiPath() { + return "/RiskAndCompliance/UpdatedRiskList"; + } + + @Override + protected void beforeFetch(){ + log.info("[{}] 요청 날짜 범위: {} → {}", getReaderName(), fromDate, toDate); + } + + @Override + protected List fetchNextBatch() throws Exception { + // 모든 배치 처리 완료 확인 + if (allData == null) { + log.info("[{}] 최초 API 조회 실행: {} ~ {}", getReaderName(), fromDate, toDate); + allData = callApiWithBatch(fromDate, toDate); + + if (allData == null || allData.isEmpty()) { + log.warn("[{}] 조회된 데이터 없음 → 종료", getReaderName()); + return null; + } + + log.info("[{}] 총 {}건 데이터 조회됨. batchSize = {}", getReaderName(), allData.size(), batchSize); + } + + // 2) 이미 끝까지 읽었으면 종료 + if (currentBatchIndex >= allData.size()) { + log.info("[{}] 모든 배치 처리 완료", getReaderName()); + return null; + } + + // 3) 이번 배치의 end 계산 + int end = Math.min(currentBatchIndex + batchSize, allData.size()); + + // 4) 현재 batch 리스트 잘라서 반환 + List batch = allData.subList(currentBatchIndex, end); + + int batchNum = (currentBatchIndex / batchSize) + 1; + int totalBatches = (int) Math.ceil((double) allData.size() / batchSize); + + log.info("[{}] 배치 {}/{} 처리 중: {}건", getReaderName(), batchNum, totalBatches, batch.size()); + + // 다음 batch 인덱스 이동 + currentBatchIndex = end; + updateApiCallStats(totalBatches, batchNum); + + return batch; + } + + private List callApiWithBatch(String fromDate, String stopDate) { + String url = getApiPath() + "?fromDate=" + fromDate +"&stopDate=" + stopDate; + log.debug("[{}] API 호출: {}", getReaderName(), url); + return webClient.get() + .uri(url) + .retrieve() + .bodyToMono(new ParameterizedTypeReference>() {}) + .block(); + } + +} diff --git a/src/main/java/com/snp/batch/jobs/risk/batch/repository/RiskRepositoryImpl.java b/src/main/java/com/snp/batch/jobs/risk/batch/repository/RiskRepositoryImpl.java index a5c8695..a6a07ff 100644 --- a/src/main/java/com/snp/batch/jobs/risk/batch/repository/RiskRepositoryImpl.java +++ b/src/main/java/com/snp/batch/jobs/risk/batch/repository/RiskRepositoryImpl.java @@ -41,7 +41,7 @@ public class RiskRepositoryImpl extends BaseJdbcRepository imp @Override protected String getUpdateSql() { return """ - INSERT INTO snp_data.risk ( + INSERT INTO new_snp.risk ( lrno, lastupdated, riskdatamaintained, dayssincelastseenonais, dayssincelastseenonaisnarrative, daysunderais, daysunderaisnarrative, imocorrectonais, imocorrectonaisnarrative, sailingundername, sailingundernamenarrative, anomalousmessagesfrommmsi, anomalousmessagesfrommmsinarrative, diff --git a/src/main/java/com/snp/batch/jobs/sanction/batch/config/SanctionUpdateRangeJobConfig.java b/src/main/java/com/snp/batch/jobs/sanction/batch/config/SanctionUpdateRangeJobConfig.java new file mode 100644 index 0000000..4da073f --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/sanction/batch/config/SanctionUpdateRangeJobConfig.java @@ -0,0 +1,98 @@ +package com.snp.batch.jobs.sanction.batch.config; + +import com.snp.batch.common.batch.config.BaseJobConfig; +import com.snp.batch.jobs.sanction.batch.dto.ComplianceDto; +import com.snp.batch.jobs.sanction.batch.entity.ComplianceEntity; +import com.snp.batch.jobs.sanction.batch.processor.ComplianceDataProcessor; +import com.snp.batch.jobs.sanction.batch.reader.ComplianceDataRangeReader; +import com.snp.batch.jobs.sanction.batch.reader.ComplianceDataReader; +import com.snp.batch.jobs.sanction.batch.writer.ComplianceDataWriter; +import com.snp.batch.jobs.shipMovementAnchorageCalls.batch.reader.AnchorageCallsRangeReader; +import lombok.extern.slf4j.Slf4j; +import org.springframework.batch.core.Job; +import org.springframework.batch.core.Step; +import org.springframework.batch.core.configuration.annotation.StepScope; +import org.springframework.batch.core.repository.JobRepository; +import org.springframework.batch.item.ItemProcessor; +import org.springframework.batch.item.ItemReader; +import org.springframework.batch.item.ItemWriter; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.transaction.PlatformTransactionManager; +import org.springframework.web.reactive.function.client.WebClient; + +@Slf4j +@Configuration +public class SanctionUpdateRangeJobConfig extends BaseJobConfig { + private final JdbcTemplate jdbcTemplate; + private final WebClient maritimeServiceApiWebClient; + private final ComplianceDataProcessor complianceDataProcessor; + private final ComplianceDataWriter complianceDataWriter; + private final ComplianceDataRangeReader complianceDataRangeReader; + + @Override + protected int getChunkSize() { + return 5000; // API에서 5000개씩 가져오므로 chunk도 5000으로 설정 + } + public SanctionUpdateRangeJobConfig( + JobRepository jobRepository, + PlatformTransactionManager transactionManager, + ComplianceDataProcessor complianceDataProcessor, + ComplianceDataWriter complianceDataWriter, + JdbcTemplate jdbcTemplate, + @Qualifier("maritimeServiceApiWebClient")WebClient maritimeServiceApiWebClient, ComplianceDataRangeReader complianceDataRangeReader) { + super(jobRepository, transactionManager); + this.jdbcTemplate = jdbcTemplate; + this.maritimeServiceApiWebClient = maritimeServiceApiWebClient; + this.complianceDataProcessor = complianceDataProcessor; + this.complianceDataWriter = complianceDataWriter; + this.complianceDataRangeReader = complianceDataRangeReader; + } + + @Override + protected String getJobName() { + return "SanctionRangeUpdateJob"; + } + + @Override + protected String getStepName() { + return "SanctionRangeUpdateStep"; + } + + @Override + protected ItemReader createReader() { + return complianceDataRangeReader; + } + + @Bean + @StepScope + public ComplianceDataRangeReader complianceDataRangeReader( + @Value("#{jobParameters['fromDate']}") String startDate, + @Value("#{jobParameters['toDate']}") String stopDate + ) { + return new ComplianceDataRangeReader(maritimeServiceApiWebClient, startDate, stopDate); + } + @Override + protected ItemProcessor createProcessor() { + return complianceDataProcessor; + } + + @Override + protected ItemWriter createWriter() { + return complianceDataWriter; + } + + @Bean(name = "SanctionRangeUpdateJob") + public Job sanctionRangeUpdateJob() { + return job(); + } + + @Bean(name = "SanctionRangeUpdateStep") + public Step sanctionRangeUpdateStep() { + return step(); + } + +} diff --git a/src/main/java/com/snp/batch/jobs/sanction/batch/reader/ComplianceDataRangeReader.java b/src/main/java/com/snp/batch/jobs/sanction/batch/reader/ComplianceDataRangeReader.java new file mode 100644 index 0000000..60f3677 --- /dev/null +++ b/src/main/java/com/snp/batch/jobs/sanction/batch/reader/ComplianceDataRangeReader.java @@ -0,0 +1,128 @@ +package com.snp.batch.jobs.sanction.batch.reader; + +import com.snp.batch.common.batch.reader.BaseApiReader; +import com.snp.batch.jobs.sanction.batch.dto.ComplianceDto; +import com.snp.batch.jobs.shipMovementAnchorageCalls.batch.dto.AnchorageCallsDto; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.core.ParameterizedTypeReference; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.web.reactive.function.client.WebClient; + +import java.time.LocalDate; +import java.time.format.DateTimeFormatter; +import java.util.Collections; +import java.util.List; + +@Slf4j +public class ComplianceDataRangeReader extends BaseApiReader { + + //TODO : + // 1. Core20 IMO_NUMBER 전체 조회 + // 2. IMO번호에 대한 마지막 AIS 신호 요청 (1회 최대 5000개 : Chunk 단위로 반복) + // 3. Response Data -> Core20에 업데이트 (Chunk 단위로 반복) + + //private final JdbcTemplate jdbcTemplate; + + private List allData; + private int currentBatchIndex = 0; + private final int batchSize = 100; + private String fromDate; + private String toDate; + public ComplianceDataRangeReader(WebClient webClient, + @Value("#{jobParameters['fromDate']}") String fromDate, + @Value("#{jobParameters['toDate']}") String toDate) { + super(webClient); + + // 날짜가 없으면 전날 하루 기준 + if (fromDate == null || fromDate.isBlank() || + toDate == null || toDate.isBlank()) { + + LocalDate yesterday = LocalDate.now().minusDays(1); + this.fromDate = yesterday.atStartOfDay().format(DateTimeFormatter.ISO_DATE_TIME) + "Z"; + this.toDate = yesterday.plusDays(1).atStartOfDay().format(DateTimeFormatter.ISO_DATE_TIME) + "Z"; + } else { + this.fromDate = fromDate; + this.toDate = toDate; + } + + enableChunkMode(); + } + + @Override + protected String getReaderName() { + return "ComplianceDataReader"; + } + + @Override + protected void resetCustomState() { + this.currentBatchIndex = 0; + this.allData = null; + } + + @Override + protected String getApiPath() { + return "/RiskAndCompliance/UpdatedComplianceList"; + } + + private String getTargetTable(){ + return "snp_data.core20"; + } + private String GET_CORE_IMO_LIST = +// "SELECT ihslrorimoshipno FROM " + getTargetTable() + " ORDER BY ihslrorimoshipno"; + "select imo_number as ihslrorimoshipno from snp_data.ship_data order by imo_number"; + @Override + protected void beforeFetch(){ + log.info("[{}] 요청 날짜 범위: {} → {}", getReaderName(), fromDate, toDate); + } + + @Override + protected List fetchNextBatch() throws Exception { + // 모든 배치 처리 완료 확인 + if (allData == null) { + log.info("[{}] 최초 API 조회 실행: {} ~ {}", getReaderName(), fromDate, toDate); + allData = callApiWithBatch(fromDate, toDate); + + if (allData == null || allData.isEmpty()) { + log.warn("[{}] 조회된 데이터 없음 → 종료", getReaderName()); + return null; + } + + log.info("[{}] 총 {}건 데이터 조회됨. batchSize = {}", getReaderName(), allData.size(), batchSize); + } + + // 2) 이미 끝까지 읽었으면 종료 + if (currentBatchIndex >= allData.size()) { + log.info("[{}] 모든 배치 처리 완료", getReaderName()); + return null; + } + + // 3) 이번 배치의 end 계산 + int end = Math.min(currentBatchIndex + batchSize, allData.size()); + + // 4) 현재 batch 리스트 잘라서 반환 + List batch = allData.subList(currentBatchIndex, end); + + int batchNum = (currentBatchIndex / batchSize) + 1; + int totalBatches = (int) Math.ceil((double) allData.size() / batchSize); + + log.info("[{}] 배치 {}/{} 처리 중: {}건", getReaderName(), batchNum, totalBatches, batch.size()); + + // 다음 batch 인덱스 이동 + currentBatchIndex = end; + updateApiCallStats(totalBatches, batchNum); + + return batch; + } + + private List callApiWithBatch(String fromDate, String stopDate) { + String url = getApiPath() + "?fromDate=" + fromDate +"&stopDate=" + stopDate; + log.debug("[{}] API 호출: {}", getReaderName(), url); + return webClient.get() + .uri(url) + .retrieve() + .bodyToMono(new ParameterizedTypeReference>() {}) + .block(); + } + +} diff --git a/src/main/java/com/snp/batch/jobs/sanction/batch/repository/ComplianceRepositoryImpl.java b/src/main/java/com/snp/batch/jobs/sanction/batch/repository/ComplianceRepositoryImpl.java index db90923..e4ace50 100644 --- a/src/main/java/com/snp/batch/jobs/sanction/batch/repository/ComplianceRepositoryImpl.java +++ b/src/main/java/com/snp/batch/jobs/sanction/batch/repository/ComplianceRepositoryImpl.java @@ -42,7 +42,7 @@ public class ComplianceRepositoryImpl extends BaseJdbcRepository