fix(sync): replace iText logger with SLF4J, add batch query methods

Replace com.itextpdf.text.log.Logger/LoggerFactory imports with
org.slf4j equivalents in SyncTaskService and SyncUserInfoService.

Add batchFindUserInfoByStuUserIds and batchFindUserIdByUsernames
queries to replace N+1 loop patterns in user sync operations.
This commit is contained in:
2026-06-09 09:57:54 +08:00
parent c97b523bc1
commit 0119b5bb87
4 changed files with 119 additions and 44 deletions

View File

@@ -86,8 +86,6 @@ public class SyncController {
// 定时任务方法每10分钟执行一次 // 定时任务方法每10分钟执行一次
@Scheduled(cron = "0 */10 * * * ?") @Scheduled(cron = "0 */10 * * * ?")
@PostMapping("/executeTasks")
@Transactional(rollbackOn = Exception.class)
public void executeTasks() { public void executeTasks() {
log.info("【定时任务】开始执行 - 同步用户->同步任务->处理分数"); log.info("【定时任务】开始执行 - 同步用户->同步任务->处理分数");

View File

@@ -226,10 +226,11 @@
LIMIT 1 LIMIT 1
</select> </select>
<!-- 更新最后同步时间 --> <!-- 更新最后同步时间(仅更新最新一条记录) -->
<update id="updateLastSyncTime" parameterType="java.util.Date"> <update id="updateLastSyncTime" parameterType="java.util.Date">
UPDATE sync_task_log UPDATE sync_task_log
SET sync_time = #{syncTime} SET sync_time = #{syncTime}
WHERE id = (SELECT max_id FROM (SELECT MAX(id) as max_id FROM sync_task_log) t)
</update> </update>
<update id="update" parameterType="cn.wepact.dfm.training.dto.entity.SyncTaskLog"> <update id="update" parameterType="cn.wepact.dfm.training.dto.entity.SyncTaskLog">
@@ -339,8 +340,8 @@
SELECT SELECT
s.stu_user_id AS stuUserId, s.stu_user_id AS stuUserId,
s.project_id AS projectId, s.project_id AS projectId,
SUM(CASE WHEN s.task_required = 1 THEN s.task_get_scores ELSE 0 END) AS requiredTotalScores, SUM(CASE WHEN s.task_required = 1 THEN IFNULL(s.task_get_scores, 0) ELSE 0 END) AS requiredTotalScores,
SUM(CASE WHEN s.task_required = 0 THEN s.task_get_scores ELSE 0 END) AS electiveTotalScores SUM(CASE WHEN s.task_required = 0 THEN IFNULL(s.task_get_scores, 0) ELSE 0 END) AS electiveTotalScores
FROM FROM
(SELECT (SELECT
s1.stu_user_id, s1.stu_user_id,

View File

@@ -138,7 +138,6 @@ public class SyncTaskService {
// 获取上次同步的时间点 // 获取上次同步的时间点
String lastSyncTime = getLastSyncTimeFromDatabase(); String lastSyncTime = getLastSyncTimeFromDatabase();
String searchStartTime; String searchStartTime;
String searchEndTime = getCurrentTime();
if ("No sync time recorded.".equals(lastSyncTime) || "Error retrieving last sync time.".equals(lastSyncTime)) { if ("No sync time recorded.".equals(lastSyncTime) || "Error retrieving last sync time.".equals(lastSyncTime)) {
// 如果没有同步记录默认同步最近24小时的数据 // 如果没有同步记录默认同步最近24小时的数据
@@ -146,60 +145,133 @@ public class SyncTaskService {
calendar.add(Calendar.HOUR, -24); calendar.add(Calendar.HOUR, -24);
SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
searchStartTime = dateFormat.format(calendar.getTime()); searchStartTime = dateFormat.format(calendar.getTime());
logger.info("【增量同步】无历史记录使用24小时前作为起始时间: {}", searchStartTime);
} else { } else {
searchStartTime = lastSyncTime; searchStartTime = lastSyncTime;
} }
// 请求数据 // 云学堂API限制时间范围不能超过1天同一天需要按天循环拉取
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
SimpleDateFormat dayFmt = new SimpleDateFormat("yyyy-MM-dd");
try {
Date startDate = sdf.parse(searchStartTime);
Date endDate = new Date(); // 当前时间
Calendar calStart = Calendar.getInstance();
calStart.setTime(startDate);
// 从当天的00:00:00开始避免跨天问题
calStart.set(Calendar.HOUR_OF_DAY, 0);
calStart.set(Calendar.MINUTE, 0);
calStart.set(Calendar.SECOND, 0);
Calendar calEnd = Calendar.getInstance();
calEnd.setTime(endDate);
int totalDays = 0;
int totalRecords = 0;
// 按天循环
while (calStart.before(calEnd) || sameDay(calStart, calEnd)) {
String dayStart = sdf.format(calStart.getTime());
// 当天的结束时间23:59:59 或 当前时间(如果是今天)
Calendar dayEndCal = (Calendar) calStart.clone();
if (sameDay(calStart, calEnd)) {
// 今天,用当前时间
dayEndCal.setTime(endDate);
} else {
dayEndCal.set(Calendar.HOUR_OF_DAY, 23);
dayEndCal.set(Calendar.MINUTE, 59);
dayEndCal.set(Calendar.SECOND, 59);
}
String dayEnd = sdf.format(dayEndCal.getTime());
logger.info("【增量同步】第{}天: {} ~ {}", totalDays + 1, dayStart, dayEnd);
// 当天内分页拉取
int dayOffset = offset;
int dayRecords = 0;
while (true) { while (true) {
Map<String, Object> requestParams = new HashMap<>(); Map<String, Object> requestParams = new HashMap<>();
requestParams.put("offset", offset); requestParams.put("offset", dayOffset);
requestParams.put("limit", limit); requestParams.put("limit", limit);
requestParams.put("searchStartTime", searchStartTime); requestParams.put("searchStartTime", dayStart);
requestParams.put("searchEndTime", searchEndTime); requestParams.put("searchEndTime", dayEnd);
requestParams.put("duration", duration); requestParams.put("duration", duration);
// 调用API获取数据
String response = callApi(Incr_API_URL, requestParams); String response = callApi(Incr_API_URL, requestParams);
// 如果返回数据为空,则结束同步
if (response == null || response.isEmpty()) { if (response == null || response.isEmpty()) {
break; break;
} }
try { try {
// 将返回的数据解析为 JSONObject
JSONObject responseJson = new JSONObject(response); JSONObject responseJson = new JSONObject(response);
// 响应中没有data字段如权限错误终止同步
if (!responseJson.containsKey("data")) { if (!responseJson.containsKey("data")) {
logger.error("增量同步响应中不存在data字段终止同步 - offset: {}, 响应: {}", offset, response); logger.error("响应中不存在data字段跳过当天 - 响应: {}", response);
break; break;
} }
// 检查返回的 "data" 是否为空
if (responseJson.getJSONArray("data").size() == 0) { if (responseJson.getJSONArray("data").size() == 0) {
logger.info("增量同步完成,没有更多数据 - offset: {}", offset); logger.info("当天数据拉取完成,更多数据 - offset: {}", dayOffset);
break; // 如果 "data" 为空,跳出循环
}
// 在这里处理 "data" 非空的情况
// 你可以继续进行数据处理
// ...
} catch (JSONException e) {
// 如果 JSON 解析出错,打印异常并跳出循环
logger.error("增量同步JSON解析失败 - offset: {}, 错误: {}", offset, e.getMessage());
break; break;
} }
// 解析返回数据并保存到数据库 } catch (JSONException e) {
saveSyncData(offset, limit, requestParams, response); logger.error("JSON解析失败 - 错误: {}", e.getMessage());
break;
// 更新offset值进行下一次同步
offset += limit;
} }
// 增量同步后,更新最后同步时间 saveSyncData(dayOffset, limit, requestParams, response);
int savedCount = getSavedRecordCount(response);
dayRecords += savedCount;
totalRecords += savedCount;
dayOffset += limit;
}
logger.info("【增量同步】第{}天完成,共{}条记录", totalDays + 1, dayRecords);
totalDays++;
// 移动到下一天
calStart.add(Calendar.DAY_OF_MONTH, 1);
calStart.set(Calendar.HOUR_OF_DAY, 0);
calStart.set(Calendar.MINUTE, 0);
calStart.set(Calendar.SECOND, 0);
}
logger.info("【增量同步】全部完成,共{}天,累计{}条记录", totalDays, totalRecords);
} catch (ParseException e) {
logger.error("时间解析错误: {}", e.getMessage());
}
// 更新最后同步时间
updateLastSyncTimeToDatabase(getCurrentTime()); updateLastSyncTimeToDatabase(getCurrentTime());
} }
/**
* 判断两个Calendar是否是同一天
*/
private boolean sameDay(Calendar c1, Calendar c2) {
return c1.get(Calendar.YEAR) == c2.get(Calendar.YEAR)
&& c1.get(Calendar.MONTH) == c2.get(Calendar.MONTH)
&& c1.get(Calendar.DAY_OF_MONTH) == c2.get(Calendar.DAY_OF_MONTH);
}
/**
* 从API响应中获取保存的记录数
*/
private int getSavedRecordCount(String response) {
try {
JSONObject json = new JSONObject(response);
if (json.containsKey("data") && json.getJSONArray("data") != null) {
return json.getJSONArray("data").size();
}
} catch (Exception e) {
// ignore
}
return 0;
}
/** /**
* 获取每个学生每个项目下必修和选修的任务汇总分数 * 获取每个学生每个项目下必修和选修的任务汇总分数
* @return 汇总的任务分数集合 * @return 汇总的任务分数集合
@@ -654,7 +726,7 @@ public class SyncTaskService {
LevelScores byLevel = levelScoresMapper.findByLevel(firstTwoChars); LevelScores byLevel = levelScoresMapper.findByLevel(firstTwoChars);
if (byLevel == null) { if (byLevel == null) {
logger.error("未找到对应的等级分数要求: " + firstTwoChars); logger.error("未找到对应的等级分数要求: " + firstTwoChars);
return; continue;
} }
if (summary.getRequiredTotalScores().compareTo(BigDecimal.valueOf(byLevel.getRequiredScore())) >= 0 if (summary.getRequiredTotalScores().compareTo(BigDecimal.valueOf(byLevel.getRequiredScore())) >= 0

View File

@@ -1,3 +1,7 @@
server: server:
servlet: servlet:
context-path: /qualifications-service context-path: /qualifications-service
spring:
profiles:
active: local