diff --git a/src/main/java/com/snp/batch/common/batch/reader/BaseApiReader.java b/src/main/java/com/snp/batch/common/batch/reader/BaseApiReader.java index 664c8de..6edbc66 100644 --- a/src/main/java/com/snp/batch/common/batch/reader/BaseApiReader.java +++ b/src/main/java/com/snp/batch/common/batch/reader/BaseApiReader.java @@ -254,21 +254,42 @@ public abstract class BaseApiReader implements ItemReader { } // currentBatch가 비어있으면 다음 배치 로드 - if (currentBatch == null || !currentBatch.hasNext()) { + /*if (currentBatch == null || !currentBatch.hasNext()) { List nextBatch = fetchNextBatch(); // 더 이상 데이터가 없으면 종료 - if (nextBatch == null || nextBatch.isEmpty()) { +// if (nextBatch == null || nextBatch.isEmpty()) { + if (nextBatch == null ) { afterFetch(null); log.info("[{}] 모든 배치 처리 완료", getReaderName()); return null; } + // Iterator 갱신 + currentBatch = nextBatch.iterator(); + log.debug("[{}] 배치 로드 완료: {} 건", getReaderName(), nextBatch.size()); + }*/ + // currentBatch가 비어있으면 다음 배치 로드 + while (currentBatch == null || !currentBatch.hasNext()) { + List nextBatch = fetchNextBatch(); + + if (nextBatch == null) { // 진짜 종료 + afterFetch(null); + log.info("[{}] 모든 배치 처리 완료", getReaderName()); + return null; + } + + if (nextBatch.isEmpty()) { // emptyList면 다음 batch를 시도 + log.warn("[{}] 빈 배치 수신 → 다음 배치 재요청", getReaderName()); + continue; // while 반복문으로 다시 fetch + } + currentBatch = nextBatch.iterator(); log.debug("[{}] 배치 로드 완료: {} 건", getReaderName(), nextBatch.size()); } + // Iterator에서 1건씩 반환 return currentBatch.next(); } diff --git a/src/main/java/com/snp/batch/global/config/MaritimeApiWebClientConfig.java b/src/main/java/com/snp/batch/global/config/MaritimeApiWebClientConfig.java index e954a9c..ddeb443 100644 --- a/src/main/java/com/snp/batch/global/config/MaritimeApiWebClientConfig.java +++ b/src/main/java/com/snp/batch/global/config/MaritimeApiWebClientConfig.java @@ -96,7 +96,7 @@ public class MaritimeApiWebClientConfig { .defaultHeaders(headers -> headers.setBasicAuth(maritimeApiUsername, maritimeApiPassword)) .codecs(configurer -> configurer .defaultCodecs() - .maxInMemorySize(30 * 1024 * 1024)) // 30MB 버퍼 + .maxInMemorySize(100 * 1024 * 1024)) // 100MB 버퍼 .build(); } } diff --git a/src/main/java/com/snp/batch/jobs/shipMovement/batch/reader/ShipMovementReader.java b/src/main/java/com/snp/batch/jobs/shipMovement/batch/reader/ShipMovementReader.java index f251206..612a1be 100644 --- a/src/main/java/com/snp/batch/jobs/shipMovement/batch/reader/ShipMovementReader.java +++ b/src/main/java/com/snp/batch/jobs/shipMovement/batch/reader/ShipMovementReader.java @@ -50,7 +50,7 @@ public class ShipMovementReader extends BaseApiReader { // DB 해시값을 저장할 맵 private Map dbMasterHashes; private int currentBatchIndex = 0; - private final int batchSize = 50; + private final int batchSize = 10; @Value("#{jobParameters['startDate']}") private String startDate; @@ -91,7 +91,8 @@ public class ShipMovementReader extends BaseApiReader { } private static final String GET_ALL_IMO_QUERY = - "SELECT imo_number FROM ship_data ORDER BY id"; +// "SELECT imo_number FROM ship_data ORDER BY id"; + "SELECT imo_number FROM snp_data.ship_data where imo_number > (select max(imo) from snp_data.t_ship_stpov_info) ORDER BY imo_number"; private static final String FETCH_ALL_HASHES_QUERY = "SELECT imo_number, ship_detail_hash FROM ship_detail_hash_json ORDER BY imo_number"; @@ -112,20 +113,6 @@ public class ShipMovementReader extends BaseApiReader { log.info("[{}] {}개씩 배치로 분할하여 API 호출 예정", getReaderName(), batchSize); log.info("[{}] 예상 배치 수: {} 개", getReaderName(), totalBatches); - /* // Step 2. 전 배치 결과 imo_number, ship_detail_json, ship_detail_hash 데이터 전체 조회 - log.info("[{}] DB Master Hash 전체 조회 시작...", getReaderName()); - - // 1-1. DB에서 모든 IMO와 Hash 조회 - dbMasterHashes = jdbcTemplate.query(FETCH_ALL_HASHES_QUERY, rs -> { - Map map = new HashMap<>(); - while (rs.next()) { - map.put(rs.getString("imo_number"), rs.getString("ship_detail_hash")); - } - return map; - }); - - log.info("[{}] DB Master Hash 조회 완료. 총 {}건.", getReaderName(), dbMasterHashes.size());*/ - // API 통계 초기화 updateApiCallStats(totalBatches, 0); } diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 305b40e..8fc3c41 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -55,7 +55,7 @@ spring: # Server Configuration server: - port: 8081 + port: 8041 servlet: context-path: /snp-api