UDI
/
udi-wms-java
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

3/24 数据同步优化 并行处理1.0

Browse Source
dev_2.5_optimize_250306
wangwei 7 months ago
parent d2919b0515
commit dd70503bfc
1 changed files with 104 additions and 60 deletions
Show Stats Download Patch File Download Diff File

164
src/main/java/com/glxp/api/service/sync/HeartService.java
Unescape Escape View File

@ -76,8 +76,13 @@ import java.io.File;
import java.io.FileWriter; import java.io.FileWriter;
import java.io.IOException; import java.io.IOException;
import java.util.*; import java.util.*;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.ForkJoinPool;
import java.util.function.Function; import java.util.function.Function;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import java.util.stream.Stream;
import static com.glxp.api.constant.BasicProcessStatus.NEW_ALL_ORDER; import static com.glxp.api.constant.BasicProcessStatus.NEW_ALL_ORDER;
import static com.glxp.api.constant.BasicProcessStatus.NEW_ALL_UDIS; import static com.glxp.api.constant.BasicProcessStatus.NEW_ALL_UDIS;
@ -1625,74 +1630,113 @@ public class HeartService {
List<SupCompanyEntity> supCompanyList = JSONUtil.toList(jsonObject.getJSONArray(SupCompanyEntity.class.getSimpleName()), SupCompanyEntity.class); List<SupCompanyEntity> supCompanyList = JSONUtil.toList(jsonObject.getJSONArray(SupCompanyEntity.class.getSimpleName()), SupCompanyEntity.class);
List<SupManufacturerEntity> supManufacturerList = JSONUtil.toList(jsonObject.getJSONArray(SupManufacturerEntity.class.getSimpleName()), SupManufacturerEntity.class); List<SupManufacturerEntity> supManufacturerList = JSONUtil.toList(jsonObject.getJSONArray(SupManufacturerEntity.class.getSimpleName()), SupManufacturerEntity.class);
List<SupProductEntity> supProductList = JSONUtil.toList(jsonObject.getJSONArray(SupProductEntity.class.getSimpleName()), SupProductEntity.class); List<SupProductEntity> supProductList = JSONUtil.toList(jsonObject.getJSONArray(SupProductEntity.class.getSimpleName()), SupProductEntity.class);
List<String> syncFiles = JSONUtil.toList(jsonObject.getJSONArray("syncFiles"), String.class); List<String> syncFiles = JSONUtil.toList(jsonObject.getJSONArray("syncFiles"), String.class);
// 使用 ExecutorService 管理并行任务
ExecutorService executorService = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors());
if (CollectionUtil.isNotEmpty(hospTypeList)) { try {
basicHospTypeDao.replaceBatch(hospTypeList); Stream.<Runnable>of(
} () -> processHospType(hospTypeList),
if (CollectionUtil.isNotEmpty(udiRelevanceList)) { () -> processUdiRelevance(udiRelevanceList),
List<List<UdiRelevanceEntity>> splits = CustomUtil.splitList(udiRelevanceList, 200); () -> processProducts(productsList),
splits.forEach(items -> { () -> processRelevance(relevanceList),
udiRelevanceDao.replaceBatch(items); () -> processCorp(corpList),
}); () -> processSupCerts(supCertList, supCertSetList),
} () -> processSupCompanies(supCompanyList),
if (CollectionUtil.isNotEmpty(productsList)) { () -> processSupManufacturers(supManufacturerList),
//过来500 条 数据 过滤掉uuid一样的 100个产品 () -> processSupProducts(supProductList)
List<BasicProductsEntity> uniqueUuidProducts = productsList.stream() ).forEach(task -> executorService.submit(task));
.collect(Collectors.groupingBy(BasicProductsEntity::getUuid)) } finally {
.values() executorService.shutdown();
.stream()
.map(list -> list.get(0))
.collect(Collectors.toList());
for (BasicProductsEntity basicProductsEntity : uniqueUuidProducts){
List<BasicProductsEntity> basicProductsEntities = basicProductsDao.selectList(new LambdaQueryWrapper<BasicProductsEntity>().eq(BasicProductsEntity::getUuid, basicProductsEntity.getUuid()));
if (CollectionUtil.isNotEmpty(basicProductsEntities)){
for (BasicProductsEntity basicProductsEntity1 : basicProductsEntities){
if (basicProductsEntity1.getNameCode() == null){
basicProductsDao.deleteById(basicProductsEntity1);
}
}
}
}
List<List<BasicProductsEntity>> splits = CustomUtil.splitList(productsList, 200);
splits.forEach(items -> {
basicProductsDao.replaceBatch(items);
});
// basicProductsDao.replaceBatchs(productsList, 1000);
}
if (CollectionUtil.isNotEmpty(relevanceList)) {
relevanceDao.replaceBatch(relevanceList);
}
if (CollectionUtil.isNotEmpty(corpList)) {
corpDao.replaceBatch(corpList);
}
if (CollectionUtil.isNotEmpty(supCertList)) {
supCertDao.replaceBatch(supCertList);
}
if (CollectionUtil.isNotEmpty(supCertSetList)) {
supCertSetDao.replaceBatch(supCertSetList);
}
if (CollectionUtil.isNotEmpty(supCompanyList)) {
supCompanyDao.replaceBatch(supCompanyList);
} }
if (CollectionUtil.isNotEmpty(supManufacturerList)) { // 异步处理文件下载
supManufacturerDao.replaceBatch(supManufacturerList); processSyncFilesAsync(syncFiles);
}
// 各处理逻辑封装为独立方法
private void processHospType(List<BasicHospTypeEntity> list) {
if (CollectionUtil.isEmpty(list)) return;
basicHospTypeDao.replaceBatch(list);
}
private void processUdiRelevance(List<UdiRelevanceEntity> list) {
if (CollectionUtil.isEmpty(list)) return;
CustomUtil.splitList(list, 200).forEach(udiRelevanceDao::replaceBatch);
}
private void processProducts(List<BasicProductsEntity> productsList) {
if (CollectionUtil.isEmpty(productsList)) return;
// 去重逻辑优化
Map<String, BasicProductsEntity> uuidToProductMap = productsList.stream()
.collect(Collectors.toMap(BasicProductsEntity::getUuid, product -> product, (existing, replacement) -> existing));
List<BasicProductsEntity> uniqueUuidProducts = new ArrayList<>(uuidToProductMap.values());
// 批量查询数据库中已存在的产品
List<String> uuids = uniqueUuidProducts.stream().map(BasicProductsEntity::getUuid).collect(Collectors.toList());
List<BasicProductsEntity> existingProducts = basicProductsDao.selectBatchIds(uuids);
// 批量删除符合条件的记录
List<String> uuidsToDelete = existingProducts.stream()
.filter(p -> p.getNameCode() == null)
.map(BasicProductsEntity::getUuid)
.collect(Collectors.toList());
if (!uuidsToDelete.isEmpty()) {
basicProductsDao.deleteBatchIds(uuidsToDelete);
} }
if (CollectionUtil.isNotEmpty(supProductList)) {
List<List<SupProductEntity>> splits = CustomUtil.splitList(supProductList, 100); // 分批处理
if (CollUtil.isNotEmpty(splits)) { CustomUtil.splitList(productsList, 500).forEach(basicProductsDao::replaceBatch);
for (List<SupProductEntity> items : splits) { }
supProductDao.insertOrUpdateBatch(items);
} private void processRelevance(List<CompanyProductRelevanceEntity> list) {
} if (CollectionUtil.isEmpty(list)) return;
CustomUtil.splitList(list, 200).forEach(relevanceDao::replaceBatch);
}
private void processCorp(List<BasicCorpEntity> list) {
if (CollectionUtil.isEmpty(list)) return;
CustomUtil.splitList(list, 200).forEach(corpDao::replaceBatch);
}
// 合并证书相关处理
private void processSupCerts(List<SupCertEntity> certList, List<SupCertSetEntity> certSetList) {
if (CollectionUtil.isNotEmpty(certList)) {
CustomUtil.splitList(certList, 200).forEach(supCertDao::replaceBatch);
} }
if (CollUtil.isNotEmpty(syncFiles)) { if (CollectionUtil.isNotEmpty(certSetList)) {
// fileService.download(syncFiles); CustomUtil.splitList(certSetList, 200).forEach(supCertSetDao::replaceBatch);
idcService.batchDownloadFile(spGetHttp.getIpUrl(), syncFiles.toArray(new String[syncFiles.size()]));
} }
} }
private void processSupCompanies(List<SupCompanyEntity> list) {
if (CollectionUtil.isEmpty(list)) return;
CustomUtil.splitList(list, 200).forEach(supCompanyDao::replaceBatch);
}
private void processSupManufacturers(List<SupManufacturerEntity> list) {
if (CollectionUtil.isEmpty(list)) return;
CustomUtil.splitList(list, 200).forEach(supManufacturerDao::replaceBatch);
}
private void processSupProducts(List<SupProductEntity> list) {
if (CollectionUtil.isEmpty(list)) return;
CustomUtil.splitList(list, 100).forEach(supProductDao::insertOrUpdateBatch);
}
// 异步文件处理
private void processSyncFilesAsync(List<String> syncFiles) {
if (CollectionUtil.isEmpty(syncFiles)) return;
CompletableFuture.runAsync(() -> {
idcService.batchDownloadFile(
spGetHttp.getIpUrl(),
syncFiles.toArray(new String[0])
);
}, ForkJoinPool.commonPool());
}
private final IdcService idcService; private final IdcService idcService;
private final IoCodeLostMapper ioCodeLostMapper; private final IoCodeLostMapper ioCodeLostMapper;

Write Preview
Loading…
Cancel
Save