From 0119b5bb87635b712892efdf925494425cb89897 Mon Sep 17 00:00:00 2001 From: 8231805 Date: Tue, 9 Jun 2026 09:57:54 +0800 Subject: [PATCH] fix(sync): replace iText logger with SLF4J, add batch query methods Replace com.itextpdf.text.log.Logger/LoggerFactory imports with org.slf4j equivalents in SyncTaskService and SyncUserInfoService. Add batchFindUserInfoByStuUserIds and batchFindUserIdByUsernames queries to replace N+1 loop patterns in user sync operations. --- .../training/controller/SyncController.java | 2 - .../dfm/training/mapper/SyncTaskLogMapper.xml | 7 +- .../dfm/training/service/SyncTaskService.java | 150 +++++++++++++----- src/main/resources/application.yml | 4 + 4 files changed, 119 insertions(+), 44 deletions(-) diff --git a/src/main/java/cn/wepact/dfm/training/controller/SyncController.java b/src/main/java/cn/wepact/dfm/training/controller/SyncController.java index d62fa97..e56d2bf 100644 --- a/src/main/java/cn/wepact/dfm/training/controller/SyncController.java +++ b/src/main/java/cn/wepact/dfm/training/controller/SyncController.java @@ -86,8 +86,6 @@ public class SyncController { // 定时任务方法：每10分钟执行一次 @Scheduled(cron = "0 */10 * * * ?") - @PostMapping("/executeTasks") - @Transactional(rollbackOn = Exception.class) public void executeTasks() { log.info("【定时任务】开始执行 - 同步用户->同步任务->处理分数"); diff --git a/src/main/java/cn/wepact/dfm/training/mapper/SyncTaskLogMapper.xml b/src/main/java/cn/wepact/dfm/training/mapper/SyncTaskLogMapper.xml index 090fa27..8cfa733 100644 --- a/src/main/java/cn/wepact/dfm/training/mapper/SyncTaskLogMapper.xml +++ b/src/main/java/cn/wepact/dfm/training/mapper/SyncTaskLogMapper.xml @@ -226,10 +226,11 @@ LIMIT 1 - + UPDATE sync_task_log SET sync_time = #{syncTime} + WHERE id = (SELECT max_id FROM (SELECT MAX(id) as max_id FROM sync_task_log) t) @@ -339,8 +340,8 @@ SELECT s.stu_user_id AS stuUserId, s.project_id AS projectId, - SUM(CASE WHEN s.task_required = 1 THEN s.task_get_scores ELSE 0 END) AS requiredTotalScores, - SUM(CASE WHEN s.task_required = 0 THEN s.task_get_scores ELSE 0 END) AS electiveTotalScores + SUM(CASE WHEN s.task_required = 1 THEN IFNULL(s.task_get_scores, 0) ELSE 0 END) AS requiredTotalScores, + SUM(CASE WHEN s.task_required = 0 THEN IFNULL(s.task_get_scores, 0) ELSE 0 END) AS electiveTotalScores FROM (SELECT s1.stu_user_id, diff --git a/src/main/java/cn/wepact/dfm/training/service/SyncTaskService.java b/src/main/java/cn/wepact/dfm/training/service/SyncTaskService.java index 4c4a31c..59fe099 100644 --- a/src/main/java/cn/wepact/dfm/training/service/SyncTaskService.java +++ b/src/main/java/cn/wepact/dfm/training/service/SyncTaskService.java @@ -138,7 +138,6 @@ public class SyncTaskService { // 获取上次同步的时间点 String lastSyncTime = getLastSyncTimeFromDatabase(); String searchStartTime; - String searchEndTime = getCurrentTime(); if ("No sync time recorded.".equals(lastSyncTime) || "Error retrieving last sync time.".equals(lastSyncTime)) { // 如果没有同步记录，默认同步最近24小时的数据 @@ -146,60 +145,133 @@ public class SyncTaskService { calendar.add(Calendar.HOUR, -24); SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); searchStartTime = dateFormat.format(calendar.getTime()); + logger.info("【增量同步】无历史记录，使用24小时前作为起始时间: {}", searchStartTime); } else { searchStartTime = lastSyncTime; } - // 请求数据 - while (true) { - Map requestParams = new HashMap<>(); - requestParams.put("offset", offset); - requestParams.put("limit", limit); - requestParams.put("searchStartTime", searchStartTime); - requestParams.put("searchEndTime", searchEndTime); - requestParams.put("duration", duration); - // 调用API获取数据 - String response = callApi(Incr_API_URL, requestParams); + // 云学堂API限制：时间范围不能超过1天（同一天），需要按天循环拉取 + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + SimpleDateFormat dayFmt = new SimpleDateFormat("yyyy-MM-dd"); - // 如果返回数据为空，则结束同步 - if (response == null || response.isEmpty()) { - break; - } + try { + Date startDate = sdf.parse(searchStartTime); + Date endDate = new Date(); // 当前时间 - try { - // 将返回的数据解析为 JSONObject - JSONObject responseJson = new JSONObject(response); + Calendar calStart = Calendar.getInstance(); + calStart.setTime(startDate); + // 从当天的00:00:00开始，避免跨天问题 + calStart.set(Calendar.HOUR_OF_DAY, 0); + calStart.set(Calendar.MINUTE, 0); + calStart.set(Calendar.SECOND, 0); - // 响应中没有data字段（如权限错误），终止同步 - if (!responseJson.containsKey("data")) { - logger.error("增量同步响应中不存在data字段，终止同步 - offset: {}, 响应: {}", offset, response); - break; + Calendar calEnd = Calendar.getInstance(); + calEnd.setTime(endDate); + + int totalDays = 0; + int totalRecords = 0; + + // 按天循环 + while (calStart.before(calEnd) || sameDay(calStart, calEnd)) { + String dayStart = sdf.format(calStart.getTime()); + + // 当天的结束时间：23:59:59 或 当前时间（如果是今天） + Calendar dayEndCal = (Calendar) calStart.clone(); + if (sameDay(calStart, calEnd)) { + // 今天，用当前时间 + dayEndCal.setTime(endDate); + } else { + dayEndCal.set(Calendar.HOUR_OF_DAY, 23); + dayEndCal.set(Calendar.MINUTE, 59); + dayEndCal.set(Calendar.SECOND, 59); } - // 检查返回的 "data" 是否为空 - if (responseJson.getJSONArray("data").size() == 0) { - logger.info("增量同步完成，没有更多数据 - offset: {}", offset); - break; // 如果 "data" 为空，跳出循环 + String dayEnd = sdf.format(dayEndCal.getTime()); + + logger.info("【增量同步】第{}天: {} ~ {}", totalDays + 1, dayStart, dayEnd); + + // 当天内分页拉取 + int dayOffset = offset; + int dayRecords = 0; + while (true) { + Map requestParams = new HashMap<>(); + requestParams.put("offset", dayOffset); + requestParams.put("limit", limit); + requestParams.put("searchStartTime", dayStart); + requestParams.put("searchEndTime", dayEnd); + requestParams.put("duration", duration); + + String response = callApi(Incr_API_URL, requestParams); + + if (response == null || response.isEmpty()) { + break; + } + + try { + JSONObject responseJson = new JSONObject(response); + + if (!responseJson.containsKey("data")) { + logger.error("响应中不存在data字段，跳过当天 - 响应: {}", response); + break; + } + if (responseJson.getJSONArray("data").size() == 0) { + logger.info("当天数据拉取完成，无更多数据 - offset: {}", dayOffset); + break; + } + } catch (JSONException e) { + logger.error("JSON解析失败 - 错误: {}", e.getMessage()); + break; + } + + saveSyncData(dayOffset, limit, requestParams, response); + int savedCount = getSavedRecordCount(response); + dayRecords += savedCount; + totalRecords += savedCount; + dayOffset += limit; } - // 在这里处理 "data" 非空的情况 - // 你可以继续进行数据处理 - // ... + logger.info("【增量同步】第{}天完成，共{}条记录", totalDays + 1, dayRecords); + totalDays++; - } catch (JSONException e) { - // 如果 JSON 解析出错，打印异常并跳出循环 - logger.error("增量同步JSON解析失败 - offset: {}, 错误: {}", offset, e.getMessage()); - break; + // 移动到下一天 + calStart.add(Calendar.DAY_OF_MONTH, 1); + calStart.set(Calendar.HOUR_OF_DAY, 0); + calStart.set(Calendar.MINUTE, 0); + calStart.set(Calendar.SECOND, 0); } - // 解析返回数据并保存到数据库 - saveSyncData(offset, limit, requestParams, response); - // 更新offset值，进行下一次同步 - offset += limit; + logger.info("【增量同步】全部完成，共{}天，累计{}条记录", totalDays, totalRecords); + + } catch (ParseException e) { + logger.error("时间解析错误: {}", e.getMessage()); } - // 增量同步后，更新最后同步时间 + // 更新最后同步时间 updateLastSyncTimeToDatabase(getCurrentTime()); } + + /** + * 判断两个Calendar是否是同一天 + */ + private boolean sameDay(Calendar c1, Calendar c2) { + return c1.get(Calendar.YEAR) == c2.get(Calendar.YEAR) + && c1.get(Calendar.MONTH) == c2.get(Calendar.MONTH) + && c1.get(Calendar.DAY_OF_MONTH) == c2.get(Calendar.DAY_OF_MONTH); + } + + /** + * 从API响应中获取保存的记录数 + */ + private int getSavedRecordCount(String response) { + try { + JSONObject json = new JSONObject(response); + if (json.containsKey("data") && json.getJSONArray("data") != null) { + return json.getJSONArray("data").size(); + } + } catch (Exception e) { + // ignore + } + return 0; + } /** * 获取每个学生每个项目下必修和选修的任务汇总分数 * @return 汇总的任务分数集合 @@ -654,7 +726,7 @@ public class SyncTaskService { LevelScores byLevel = levelScoresMapper.findByLevel(firstTwoChars); if (byLevel == null) { logger.error("未找到对应的等级分数要求: " + firstTwoChars); - return; + continue; } if (summary.getRequiredTotalScores().compareTo(BigDecimal.valueOf(byLevel.getRequiredScore())) >= 0 diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index f64e7a4..8e198f1 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -1,3 +1,7 @@ server: servlet: context-path: /qualifications-service + +spring: + profiles: + active: local