fix(sync): replace iText logger with SLF4J, add batch query methods
Replace com.itextpdf.text.log.Logger/LoggerFactory imports with org.slf4j equivalents in SyncTaskService and SyncUserInfoService. Add batchFindUserInfoByStuUserIds and batchFindUserIdByUsernames queries to replace N+1 loop patterns in user sync operations.
This commit is contained in:
@@ -86,8 +86,6 @@ public class SyncController {
|
|||||||
|
|
||||||
// 定时任务方法:每10分钟执行一次
|
// 定时任务方法:每10分钟执行一次
|
||||||
@Scheduled(cron = "0 */10 * * * ?")
|
@Scheduled(cron = "0 */10 * * * ?")
|
||||||
@PostMapping("/executeTasks")
|
|
||||||
@Transactional(rollbackOn = Exception.class)
|
|
||||||
public void executeTasks() {
|
public void executeTasks() {
|
||||||
log.info("【定时任务】开始执行 - 同步用户->同步任务->处理分数");
|
log.info("【定时任务】开始执行 - 同步用户->同步任务->处理分数");
|
||||||
|
|
||||||
|
|||||||
@@ -226,10 +226,11 @@
|
|||||||
LIMIT 1
|
LIMIT 1
|
||||||
</select>
|
</select>
|
||||||
|
|
||||||
<!-- 更新最后同步时间 -->
|
<!-- 更新最后同步时间(仅更新最新一条记录) -->
|
||||||
<update id="updateLastSyncTime" parameterType="java.util.Date">
|
<update id="updateLastSyncTime" parameterType="java.util.Date">
|
||||||
UPDATE sync_task_log
|
UPDATE sync_task_log
|
||||||
SET sync_time = #{syncTime}
|
SET sync_time = #{syncTime}
|
||||||
|
WHERE id = (SELECT max_id FROM (SELECT MAX(id) as max_id FROM sync_task_log) t)
|
||||||
</update>
|
</update>
|
||||||
|
|
||||||
<update id="update" parameterType="cn.wepact.dfm.training.dto.entity.SyncTaskLog">
|
<update id="update" parameterType="cn.wepact.dfm.training.dto.entity.SyncTaskLog">
|
||||||
@@ -339,8 +340,8 @@
|
|||||||
SELECT
|
SELECT
|
||||||
s.stu_user_id AS stuUserId,
|
s.stu_user_id AS stuUserId,
|
||||||
s.project_id AS projectId,
|
s.project_id AS projectId,
|
||||||
SUM(CASE WHEN s.task_required = 1 THEN s.task_get_scores ELSE 0 END) AS requiredTotalScores,
|
SUM(CASE WHEN s.task_required = 1 THEN IFNULL(s.task_get_scores, 0) ELSE 0 END) AS requiredTotalScores,
|
||||||
SUM(CASE WHEN s.task_required = 0 THEN s.task_get_scores ELSE 0 END) AS electiveTotalScores
|
SUM(CASE WHEN s.task_required = 0 THEN IFNULL(s.task_get_scores, 0) ELSE 0 END) AS electiveTotalScores
|
||||||
FROM
|
FROM
|
||||||
(SELECT
|
(SELECT
|
||||||
s1.stu_user_id,
|
s1.stu_user_id,
|
||||||
|
|||||||
@@ -138,7 +138,6 @@ public class SyncTaskService {
|
|||||||
// 获取上次同步的时间点
|
// 获取上次同步的时间点
|
||||||
String lastSyncTime = getLastSyncTimeFromDatabase();
|
String lastSyncTime = getLastSyncTimeFromDatabase();
|
||||||
String searchStartTime;
|
String searchStartTime;
|
||||||
String searchEndTime = getCurrentTime();
|
|
||||||
|
|
||||||
if ("No sync time recorded.".equals(lastSyncTime) || "Error retrieving last sync time.".equals(lastSyncTime)) {
|
if ("No sync time recorded.".equals(lastSyncTime) || "Error retrieving last sync time.".equals(lastSyncTime)) {
|
||||||
// 如果没有同步记录,默认同步最近24小时的数据
|
// 如果没有同步记录,默认同步最近24小时的数据
|
||||||
@@ -146,60 +145,133 @@ public class SyncTaskService {
|
|||||||
calendar.add(Calendar.HOUR, -24);
|
calendar.add(Calendar.HOUR, -24);
|
||||||
SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
|
SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
|
||||||
searchStartTime = dateFormat.format(calendar.getTime());
|
searchStartTime = dateFormat.format(calendar.getTime());
|
||||||
|
logger.info("【增量同步】无历史记录,使用24小时前作为起始时间: {}", searchStartTime);
|
||||||
} else {
|
} else {
|
||||||
searchStartTime = lastSyncTime;
|
searchStartTime = lastSyncTime;
|
||||||
}
|
}
|
||||||
|
|
||||||
// 请求数据
|
// 云学堂API限制:时间范围不能超过1天(同一天),需要按天循环拉取
|
||||||
while (true) {
|
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
|
||||||
Map<String, Object> requestParams = new HashMap<>();
|
SimpleDateFormat dayFmt = new SimpleDateFormat("yyyy-MM-dd");
|
||||||
requestParams.put("offset", offset);
|
|
||||||
requestParams.put("limit", limit);
|
|
||||||
requestParams.put("searchStartTime", searchStartTime);
|
|
||||||
requestParams.put("searchEndTime", searchEndTime);
|
|
||||||
requestParams.put("duration", duration);
|
|
||||||
// 调用API获取数据
|
|
||||||
String response = callApi(Incr_API_URL, requestParams);
|
|
||||||
|
|
||||||
// 如果返回数据为空,则结束同步
|
try {
|
||||||
if (response == null || response.isEmpty()) {
|
Date startDate = sdf.parse(searchStartTime);
|
||||||
break;
|
Date endDate = new Date(); // 当前时间
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
Calendar calStart = Calendar.getInstance();
|
||||||
// 将返回的数据解析为 JSONObject
|
calStart.setTime(startDate);
|
||||||
JSONObject responseJson = new JSONObject(response);
|
// 从当天的00:00:00开始,避免跨天问题
|
||||||
|
calStart.set(Calendar.HOUR_OF_DAY, 0);
|
||||||
|
calStart.set(Calendar.MINUTE, 0);
|
||||||
|
calStart.set(Calendar.SECOND, 0);
|
||||||
|
|
||||||
// 响应中没有data字段(如权限错误),终止同步
|
Calendar calEnd = Calendar.getInstance();
|
||||||
if (!responseJson.containsKey("data")) {
|
calEnd.setTime(endDate);
|
||||||
logger.error("增量同步响应中不存在data字段,终止同步 - offset: {}, 响应: {}", offset, response);
|
|
||||||
break;
|
int totalDays = 0;
|
||||||
|
int totalRecords = 0;
|
||||||
|
|
||||||
|
// 按天循环
|
||||||
|
while (calStart.before(calEnd) || sameDay(calStart, calEnd)) {
|
||||||
|
String dayStart = sdf.format(calStart.getTime());
|
||||||
|
|
||||||
|
// 当天的结束时间:23:59:59 或 当前时间(如果是今天)
|
||||||
|
Calendar dayEndCal = (Calendar) calStart.clone();
|
||||||
|
if (sameDay(calStart, calEnd)) {
|
||||||
|
// 今天,用当前时间
|
||||||
|
dayEndCal.setTime(endDate);
|
||||||
|
} else {
|
||||||
|
dayEndCal.set(Calendar.HOUR_OF_DAY, 23);
|
||||||
|
dayEndCal.set(Calendar.MINUTE, 59);
|
||||||
|
dayEndCal.set(Calendar.SECOND, 59);
|
||||||
}
|
}
|
||||||
// 检查返回的 "data" 是否为空
|
String dayEnd = sdf.format(dayEndCal.getTime());
|
||||||
if (responseJson.getJSONArray("data").size() == 0) {
|
|
||||||
logger.info("增量同步完成,没有更多数据 - offset: {}", offset);
|
logger.info("【增量同步】第{}天: {} ~ {}", totalDays + 1, dayStart, dayEnd);
|
||||||
break; // 如果 "data" 为空,跳出循环
|
|
||||||
|
// 当天内分页拉取
|
||||||
|
int dayOffset = offset;
|
||||||
|
int dayRecords = 0;
|
||||||
|
while (true) {
|
||||||
|
Map<String, Object> requestParams = new HashMap<>();
|
||||||
|
requestParams.put("offset", dayOffset);
|
||||||
|
requestParams.put("limit", limit);
|
||||||
|
requestParams.put("searchStartTime", dayStart);
|
||||||
|
requestParams.put("searchEndTime", dayEnd);
|
||||||
|
requestParams.put("duration", duration);
|
||||||
|
|
||||||
|
String response = callApi(Incr_API_URL, requestParams);
|
||||||
|
|
||||||
|
if (response == null || response.isEmpty()) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
JSONObject responseJson = new JSONObject(response);
|
||||||
|
|
||||||
|
if (!responseJson.containsKey("data")) {
|
||||||
|
logger.error("响应中不存在data字段,跳过当天 - 响应: {}", response);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
if (responseJson.getJSONArray("data").size() == 0) {
|
||||||
|
logger.info("当天数据拉取完成,无更多数据 - offset: {}", dayOffset);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
} catch (JSONException e) {
|
||||||
|
logger.error("JSON解析失败 - 错误: {}", e.getMessage());
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
saveSyncData(dayOffset, limit, requestParams, response);
|
||||||
|
int savedCount = getSavedRecordCount(response);
|
||||||
|
dayRecords += savedCount;
|
||||||
|
totalRecords += savedCount;
|
||||||
|
dayOffset += limit;
|
||||||
}
|
}
|
||||||
|
|
||||||
// 在这里处理 "data" 非空的情况
|
logger.info("【增量同步】第{}天完成,共{}条记录", totalDays + 1, dayRecords);
|
||||||
// 你可以继续进行数据处理
|
totalDays++;
|
||||||
// ...
|
|
||||||
|
|
||||||
} catch (JSONException e) {
|
// 移动到下一天
|
||||||
// 如果 JSON 解析出错,打印异常并跳出循环
|
calStart.add(Calendar.DAY_OF_MONTH, 1);
|
||||||
logger.error("增量同步JSON解析失败 - offset: {}, 错误: {}", offset, e.getMessage());
|
calStart.set(Calendar.HOUR_OF_DAY, 0);
|
||||||
break;
|
calStart.set(Calendar.MINUTE, 0);
|
||||||
|
calStart.set(Calendar.SECOND, 0);
|
||||||
}
|
}
|
||||||
// 解析返回数据并保存到数据库
|
|
||||||
saveSyncData(offset, limit, requestParams, response);
|
|
||||||
|
|
||||||
// 更新offset值,进行下一次同步
|
logger.info("【增量同步】全部完成,共{}天,累计{}条记录", totalDays, totalRecords);
|
||||||
offset += limit;
|
|
||||||
|
} catch (ParseException e) {
|
||||||
|
logger.error("时间解析错误: {}", e.getMessage());
|
||||||
}
|
}
|
||||||
|
|
||||||
// 增量同步后,更新最后同步时间
|
// 更新最后同步时间
|
||||||
updateLastSyncTimeToDatabase(getCurrentTime());
|
updateLastSyncTimeToDatabase(getCurrentTime());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 判断两个Calendar是否是同一天
|
||||||
|
*/
|
||||||
|
private boolean sameDay(Calendar c1, Calendar c2) {
|
||||||
|
return c1.get(Calendar.YEAR) == c2.get(Calendar.YEAR)
|
||||||
|
&& c1.get(Calendar.MONTH) == c2.get(Calendar.MONTH)
|
||||||
|
&& c1.get(Calendar.DAY_OF_MONTH) == c2.get(Calendar.DAY_OF_MONTH);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 从API响应中获取保存的记录数
|
||||||
|
*/
|
||||||
|
private int getSavedRecordCount(String response) {
|
||||||
|
try {
|
||||||
|
JSONObject json = new JSONObject(response);
|
||||||
|
if (json.containsKey("data") && json.getJSONArray("data") != null) {
|
||||||
|
return json.getJSONArray("data").size();
|
||||||
|
}
|
||||||
|
} catch (Exception e) {
|
||||||
|
// ignore
|
||||||
|
}
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
/**
|
/**
|
||||||
* 获取每个学生每个项目下必修和选修的任务汇总分数
|
* 获取每个学生每个项目下必修和选修的任务汇总分数
|
||||||
* @return 汇总的任务分数集合
|
* @return 汇总的任务分数集合
|
||||||
@@ -654,7 +726,7 @@ public class SyncTaskService {
|
|||||||
LevelScores byLevel = levelScoresMapper.findByLevel(firstTwoChars);
|
LevelScores byLevel = levelScoresMapper.findByLevel(firstTwoChars);
|
||||||
if (byLevel == null) {
|
if (byLevel == null) {
|
||||||
logger.error("未找到对应的等级分数要求: " + firstTwoChars);
|
logger.error("未找到对应的等级分数要求: " + firstTwoChars);
|
||||||
return;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (summary.getRequiredTotalScores().compareTo(BigDecimal.valueOf(byLevel.getRequiredScore())) >= 0
|
if (summary.getRequiredTotalScores().compareTo(BigDecimal.valueOf(byLevel.getRequiredScore())) >= 0
|
||||||
|
|||||||
@@ -1,3 +1,7 @@
|
|||||||
server:
|
server:
|
||||||
servlet:
|
servlet:
|
||||||
context-path: /qualifications-service
|
context-path: /qualifications-service
|
||||||
|
|
||||||
|
spring:
|
||||||
|
profiles:
|
||||||
|
active: local
|
||||||
|
|||||||
Reference in New Issue
Block a user