1.适配国家库新接口,使用redis加锁替换线程挂起,提交下载速度

master
x_z 3 years ago
parent 52ca13d288
commit c9e8869907

@ -174,6 +174,11 @@
<scope>provided</scope>
</dependency>
<dependency>
<groupId>redis.clients</groupId>
<artifactId>jedis</artifactId>
<version>4.2.3</version>
</dependency>
</dependencies>

@ -2,6 +2,7 @@ package com.glxp.udidl.admin.controller.device;
import com.glxp.udidl.admin.annotation.AuthRuleAnnotation;
import com.glxp.udidl.admin.entity.udid.DeviceEntity;
import com.glxp.udidl.admin.req.udid.DeviceRequest;
import com.glxp.udidl.admin.res.BaseResponse;
import com.glxp.udidl.admin.res.udid.DataSetSingleHistoryResult;
import com.glxp.udidl.admin.service.dataSync.DeviceSyncService;
@ -16,6 +17,7 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.format.annotation.DateTimeFormat;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RestController;
import java.util.Date;
@ -87,4 +89,20 @@ public class UdidlDeviceController {
public BaseResponse downloadHistory(String key) {
return deviceSyncService.downloadHistory(key);
}
/**
*
*
* @return
*/
@ApiOperation(value = "下载国家库产品数据", response = BaseResponse.class)
@AuthRuleAnnotation("udidl/device/downloadUdiData")
@PostMapping("udidl/device/downloadUdiData")
public BaseResponse downloadUdiData(@RequestBody DeviceRequest deviceRequest) {
if (null == deviceRequest) {
return ResultVOUtils.error(500, "下载参数不能为空");
}
deviceSyncService.downloadUdiData(deviceRequest);
return ResultVOUtils.success("后台已经开始下载");
}
}

@ -16,11 +16,4 @@ public interface JobLogMapper extends BaseMapper<JobLog> {
*/
List<JobLog> list(JobLogFilterRequest jobLogFilterRequest);
/**
*
*
* @param jobLog
* @return
*/
int insert(JobLog jobLog);
}

@ -1,5 +1,8 @@
package com.glxp.udidl.admin.entity.udid;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import com.fasterxml.jackson.annotation.JsonFormat;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
@ -11,11 +14,12 @@ import java.util.Date;
*
*/
@Data
//@Table("job_log")
@TableName("job_log")
@ApiModel("UDI日志信息实体")
public class JobLog {
@ApiModelProperty(value = "日志id")
@TableId(type = IdType.AUTO)
private Integer id;
@ApiModelProperty(value = "类型: error; info; warning")
@ -46,4 +50,11 @@ public class JobLog {
@ApiModelProperty(value = "创建时间", example = "2022-05-09 14:36:00")
@JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss", timezone = "GMT+8")
private Date createTime;//创建时间
@ApiModelProperty(value = "修改时间", example = "2022-05-09 14:36:00")
@JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss", timezone = "GMT+8")
private Date updateTime;//修改时间
@ApiModelProperty(value = "状态 0处理中 1.已完成")
private Integer status;
}

@ -25,7 +25,7 @@ public class GlobalExceptionHandler {
// 拦截API异常
@ExceptionHandler(value = RuntimeException.class)
public BaseResponse handlerRuntimeException(RuntimeException e) {
log.error(e.getMessage());
log.error(e.getMessage(), e);
// 返回对应的错误信息
return ResultVOUtils.error(ResultEnum.NOT_NETWORK);
}

@ -1,5 +1,7 @@
package com.glxp.udidl.admin.req.udid;
import com.fasterxml.jackson.annotation.JsonProperty;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
@Data
@ -13,9 +15,39 @@ public class DeviceRequest {
* currentPageNumber : , 1
*/
@ApiModelProperty(value = "接口调用凭据")
private String accessToken;
@ApiModelProperty(value = "数据类型", example = "1 新发布")
private String dataType;
@ApiModelProperty(value = "请求范围1 按天请求2 按月请求")
private String requestType;
@ApiModelProperty(value = "请求时间范围")
private String rangeValue;
@ApiModelProperty(value = "最小销售单元产品标识;精准查询")
@JsonProperty("ZXXSDYCPBS")
private String ZXXSDYCPBS;
@ApiModelProperty(value = "产品名称/通用名称;模糊查询")
@JsonProperty("CPMCTYMC")
private String CPMCTYMC;
@ApiModelProperty(value = "规格型号;模糊查询")
@JsonProperty("GGXH")
private String GGXH;
@ApiModelProperty(value = "注册/备案人名称;精准查询")
@JsonProperty("YLQXZCRBARMC")
private String YLQXZCRBARMC;
@ApiModelProperty(value = "注册/备案证号;精准查询")
@JsonProperty("ZCZBHHZBAPZBH")
private String ZCZBHHZBAPZBH;
@ApiModelProperty(value = "请求分页数,初始请求时,从 1 开始")
private String currentPageNumber;
}

@ -24,14 +24,16 @@ import com.glxp.udidl.admin.service.inout.ProductInfoService;
import com.glxp.udidl.admin.service.udi.JobLogService;
import com.glxp.udidl.admin.util.DateUtil;
import com.glxp.udidl.admin.util.HttpClient;
import com.glxp.udidl.admin.util.RedisUtil;
import com.glxp.udidl.admin.util.ResultVOUtils;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import javax.annotation.Resource;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
/**
@ -43,28 +45,32 @@ public class DeviceDownloadService {
private String token = "";
int reCount = 0;
private int reCount = 0;
CompanyEntity companyEntity;
private CompanyEntity companyEntity;
String url = "sharing/get";
private String url = "sharing/get";
@Autowired
@Resource
private CompanyService companyService;
@Autowired
JobLogService jobLogService; //日志
@Autowired
DeviceSaveService deviceSaveService;
@Autowired
@Resource
private JobLogService jobLogService; //日志
@Resource
private DeviceSaveService deviceSaveService;
@Resource
private DeviceService deviceService;
@Autowired
@Resource
private ProductInfoService productInfoService;
@Resource
private RedisUtil redisUtil;
private void SetTokenEmpty() {
token = "";
}
private static List<Map<String, String>> tokenList = new ArrayList<>();
/**
* token
*
@ -133,6 +139,35 @@ public class DeviceDownloadService {
return JSONObject.parseObject(response, DataSetResult.class);
}
/**
*
*
* @param rangeValue
* @param requestType
* @param page
* @return
*/
private synchronized DataSetResult getData(DeviceRequest deviceRequest, int page) {
deviceRequest.setAccessToken(getToken());
deviceRequest.setCurrentPageNumber(page + "");
String response = "";
String key = "downloadUdiData";
if (redisUtil.getLock(key)) {
log.info("获取到锁");
redisUtil.setEx("downloadUdiData", "1", 5);
//目前限制为5秒访问一次
log.info("获取到锁的参数:" + page);
response = HttpClient.post(url, deviceRequest);
log.info("获取到结果");
}
log.info("开始解析结果");
DataSetResult dataSetResult = JSONObject.parseObject(response, DataSetResult.class);
log.info("结果解析结束");
return dataSetResult;
}
/**
*
*
@ -177,6 +212,55 @@ public class DeviceDownloadService {
return results;
}
/**
* /
*
* @param date yyyy-MM
* @return
*/
public DataSetTotalResult getData(DeviceRequest deviceRequest) {
DataSetTotalResult results = new DataSetTotalResult();
int page = 1;
int totalRecordCount = 0;
boolean hasData = true; //是否还有数据,如无退出
List<DataSetResult.DataSet> dataSets = new ArrayList<>();
while (hasData) {
DataSetResult dataSetResult = getData(deviceRequest, page);
if (dataSetResult.getReturnCode() != 1 && reCount == 0) {
token = "";
reCount++;
dataSetResult = getData(deviceRequest, page);
}
if (dataSetResult.getReturnCode() != 1) {
results.setCode(-1);
String msg = results.getMsg();
msg = msg + JSON.toJSONString(dataSetResult) + ";";
results.setMsg(msg);
return results;
} else {
if (null != dataSetResult.getDataSet()) {
if (dataSetResult.getDataSet().getDeviceInfo() != null && dataSetResult.getDataSet().getDeviceInfo().size() > 0) {
dataSets.add(dataSetResult.getDataSet());
dataSetResult.getTotalRecordCount();
} else {
hasData = false;
break;
}
} else {
hasData = false;
break;
}
}
page++;
totalRecordCount = dataSetResult.getTotalRecordCount();
}
results.setData(dataSets);
results.setTotalRecordCount(totalRecordCount);
reCount = 0;
return results;
}
public List<String> getKeyList(DataSetResult.DataSet ds) {
return ds.getDeviceInfo().stream().map(item -> item.getDeviceRecordKey()).collect(Collectors.toList());

@ -1,10 +1,13 @@
package com.glxp.udidl.admin.service.dataSync;
import cn.hutool.core.collection.CollUtil;
import cn.hutool.core.date.DateUnit;
import cn.hutool.core.util.StrUtil;
import com.alibaba.fastjson.JSON;
import com.glxp.udidl.admin.entity.udi.ProductInfoEntity;
import com.glxp.udidl.admin.entity.udid.*;
import com.glxp.udidl.admin.req.ProductInfoFilterRequest;
import com.glxp.udidl.admin.req.udid.DeviceRequest;
import com.glxp.udidl.admin.res.BaseResponse;
import com.glxp.udidl.admin.res.udid.DataSetResult;
import com.glxp.udidl.admin.res.udid.DataSetSingleHistoryResult;
@ -13,6 +16,7 @@ import com.glxp.udidl.admin.res.udid.DataSetTotalResult;
import com.glxp.udidl.admin.service.inout.ProductInfoService;
import com.glxp.udidl.admin.service.udi.JobLogService;
import com.glxp.udidl.admin.util.DateUtil;
import com.glxp.udidl.admin.util.RedisUtil;
import com.glxp.udidl.admin.util.ResultVOUtils;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.BeanUtils;
@ -39,6 +43,9 @@ public class DeviceSyncService {
@Resource
private ProductInfoService productInfoService;
@Resource
private RedisUtil redisUtil;
public void downloadAndSave(String day, String downloadType) {
//1.下载唯一标识数据
DataSetTotalResult result;
@ -309,4 +316,113 @@ public class DeviceSyncService {
downloadUdi(day, "auto");
}
}
/**
* UDI
*
* @param deviceRequest
* @return
*/
@Async
public void downloadUdiData(DeviceRequest deviceRequest) {
if (StrUtil.isBlank(deviceRequest.getRangeValue())) {
switch (deviceRequest.getRequestType()) {
case "1":
//按天下载
deviceRequest.setRangeValue(DateUtil.getDate());
break;
case "2":
deviceRequest.setRangeValue(DateUtil.formatDate(new Date(), "yyyy-MM"));
break;
case "3":
//当下载类型为时间段,但值为空时,默认下载当天的
deviceRequest.setRangeValue(DateUtil.getDate());
break;
default:
deviceRequest.setRangeValue(DateUtil.getDate());
break;
}
}
List<String> dates = new ArrayList<>();
if (deviceRequest.getRequestType().equals("3")) {
deviceRequest.setRequestType("1");
//按时间段下载
if (deviceRequest.getRangeValue().contains("--")) {
String[] dateArr = deviceRequest.getRangeValue().split("--");
int compareResult = DateUtil.compareDate(dateArr[0], dateArr[1], "yyyy-MM-dd");
if (compareResult == -1) {
long between = cn.hutool.core.date.DateUtil.between(cn.hutool.core.date.DateUtil.parseDate(dateArr[0]), cn.hutool.core.date.DateUtil.parseDate(dateArr[1]), DateUnit.DAY);
between += 1; //包括自身
for (int i = 0; i < between; i++) {
dates.add(cn.hutool.core.date.DateUtil.offsetDay(DateUtil.parseDate(dateArr[0]), i).toString("yyyy-MM-dd"));
}
} else if (compareResult == 0) {
dates.add(dateArr[0]);
} else {
dates.add(DateUtil.getDate());
}
} else {
dates.add(deviceRequest.getRangeValue());
}
} else {
dates.add(deviceRequest.getRangeValue());
}
if (deviceRequest.getDataType().equals("3")) {
deviceRequest.setDataType(null);
}
log.info("开始下载国家UDI库数据");
redisUtil.set("开始下载时间", DateUtil.getDateTime());
for (String date : dates) {
deviceRequest.setRangeValue(date);
//插入日志
JobLog jobLog = new JobLog();
jobLog.setDownloadType("manual");
jobLog.setDownloadDate(DateUtil.parseDate(date));
jobLog.setType("info");
jobLog.setMsg("下载中");
jobLog.setCreateTime(new Date());
jobLog.setStatus(0);
jobLogService.insertLog(jobLog);
//开始下载数据
DataSetTotalResult data = deviceDownloadService.getData(deviceRequest);
if (data.getCode() != 1) {
log.error(date + " 数据下载失败");
jobLog.setStatus(1);
jobLog.setType("error");
jobLog.setMsg(data.getMsg());
jobLog.setUpdateTime(new Date());
jobLogService.update(jobLog);
continue;
}
jobLog.setRealCount(data.realRecordCount());
jobLog.setTotalCount(data.getTotalRecordCount());
try {
int insertCount = deviceSaveService.DeviceSave(data.getData(), DateUtil.parseDate(date));
jobLog.setStatus(1);
jobLog.setType("success");
jobLog.setMsg("下载成功,总条数:" + data.getTotalRecordCount() + " ;插入数据:" + insertCount + " 条");
jobLog.setUpdateTime(new Date());
jobLog.setInsertCount(insertCount);
jobLogService.update(jobLog);
} catch (Exception e) {
log.error("插入数据失败", e);
jobLog.setStatus(1);
jobLog.setType("error");
jobLog.setMsg("下载成功,数据插入失败");
jobLog.setUpdateTime(new Date());
jobLog.setInsertCount(0);
jobLogService.update(jobLog);
}
}
redisUtil.set("下载结束时间", DateUtil.getDateTime());
}
}

@ -3,6 +3,8 @@ package com.glxp.udidl.admin.service.info;
import com.glxp.udidl.admin.entity.info.CompanyEntity;
import com.glxp.udidl.admin.res.BaseResponse;
import java.util.List;
public interface CompanyService {
CompanyEntity findCompany();
@ -20,4 +22,11 @@ public interface CompanyService {
* @return
*/
BaseResponse testConnection(String appId, String appSecret, String companyName, String creditNum, String dataSource);
/**
*
*
* @return
*/
List<CompanyEntity> findCompanyListForToken();
}

@ -13,6 +13,7 @@ import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import javax.annotation.Resource;
import java.util.List;
@Service
public class CompanyServiceImpl implements CompanyService {
@ -60,4 +61,11 @@ public class CompanyServiceImpl implements CompanyService {
}
return ResultVOUtils.success("连接成功");
}
@Override
public List<CompanyEntity> findCompanyListForToken() {
QueryWrapper<CompanyEntity> queryParams = new QueryWrapper<>();
queryParams.isNotNull("appId").isNotNull("appSecret").isNotNull("tyshxydm");
return companyDao.selectList(queryParams);
}
}

@ -8,4 +8,13 @@ public interface JobLogService {
BaseResponse getList(JobLogFilterRequest param);
int insert(JobLog jobLog);
int update(JobLog jobLog);
/**
*
*
* @param jobLog
*/
int insertLog(JobLog jobLog);
}

@ -11,16 +11,17 @@ import com.glxp.udidl.admin.service.udi.JobLogService;
import com.glxp.udidl.admin.util.DateUtil;
import com.glxp.udidl.admin.util.ResultVOUtils;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import javax.annotation.Resource;
import java.util.List;
@Service
@Slf4j
public class JobLogServiceImpl implements JobLogService {
@Autowired
@Resource
private JobLogMapper jobLogMapper;
public BaseResponse getList(JobLogFilterRequest param) {
@ -39,4 +40,17 @@ public class JobLogServiceImpl implements JobLogService {
public int insert(JobLog jobLog) {
return jobLogMapper.insert(jobLog);
}
@Transactional(rollbackFor = Exception.class)
public int update(JobLog jobLog) {
return jobLogMapper.updateById(jobLog);
}
@Override
@Transactional(rollbackFor = Exception.class)
public int insertLog(JobLog jobLog) {
int insert = jobLogMapper.insert(jobLog);
return insert;
}
}

@ -335,13 +335,13 @@ public class DateUtil extends DateUtils {
} catch (ParseException e) {
log.error(e.getMessage(), e);
}
}
}
}
return new Date[]{beginDate, endDate};
}
public static final int SECONDS_IN_DAY = 60 * 60 * 24;
public static final long MILLIS_IN_DAY = 1000L * SECONDS_IN_DAY;
@ -414,4 +414,29 @@ public class DateUtil extends DateUtils {
}
return 0;
}
/**
*
*
* @param date1 1
* @param date2 2
* @return 1 date1>date2;-1 date1<date2;0 date1=date2
*/
public static int compareDate(String date1, String date2, String format) {
DateFormat df = new SimpleDateFormat(format);
try {
Date dt1 = df.parse(date1);
Date dt2 = df.parse(date2);
if (dt1.getTime() > dt2.getTime()) {
return 1;
} else if (dt1.getTime() < dt2.getTime()) {
return -1;
} else {
return 0;
}
} catch (Exception exception) {
exception.printStackTrace();
}
return 0;
}
}

@ -25,7 +25,7 @@ public class HttpClient {
HttpEntity<MultiValueMap<String, Object>> httpEntity = new HttpEntity<>(postParameters, headers);
logger.info(UdidConfig.apiUrl + url + httpEntity.toString());
String response = restTemplate.postForObject(UdidConfig.apiUrl + url, httpEntity, String.class);
logger.info(response);
//logger.info(response);
return response;
}

@ -0,0 +1,49 @@
package com.glxp.udidl.admin.util;
import cn.hutool.db.nosql.redis.RedisDS;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import redis.clients.jedis.Jedis;
/**
* redis
*/
@Slf4j
@Component
public class RedisUtil {
private final Jedis jedis = RedisDS.create().getJedis();
public String get(String key) {
return jedis.get(key);
}
public String setEx(String key, String value, long expireTime) {
return jedis.setex(key, expireTime, value);
}
public boolean exist(String key) {
return jedis.exists(key);
}
public long getExpireTime(String key) {
return jedis.ttl(key);
}
public String set(String key, String value) {
return jedis.set(key, value);
}
public boolean getLock(String key) {
if (jedis.exists(key)) {
long expireTime = getExpireTime(key);
try {
Thread.sleep(expireTime * 1000);
} catch (Exception e) {
log.error("获取锁异常", e);
}
}
return true;
}
}

@ -4,7 +4,7 @@ spring:
matching-strategy: ant_path_matcher
datasource:
driver-class-name: com.p6spy.engine.spy.P6SpyDriver
url: jdbc:p6spy:mysql://127.0.0.1:3306/udidl_test?serverTimezone=Asia/Shanghai&useUnicode=true&characterEncoding=utf8&zeroDateTimeBehavior=convertToNull&useSSL=false&allowPublicKeyRetrieval=true
url: jdbc:p6spy:mysql://192.168.235.137:33306/udidl?serverTimezone=Asia/Shanghai&useUnicode=true&characterEncoding=utf8&zeroDateTimeBehavior=convertToNull&useSSL=false&allowPublicKeyRetrieval=true
username: root
password: root
servlet:

@ -2,7 +2,7 @@ server:
port: 9994
spring:
profiles:
active: pro
active: dev
jmx:
enabled: false

@ -0,0 +1,59 @@
#-------------------------------------------------------------------------------
# Redis客户端配置样例
# 每一个分组代表一个Redis实例
# 无分组的Pool配置为所有分组的共用配置如果分组自己定义Pool配置则覆盖共用配置
# 池配置来自于https://www.cnblogs.com/jklk/p/7095067.html
#-------------------------------------------------------------------------------
#----- 默认(公有)配置
# 地址默认localhost
host = 192.168.235.137
# 端口默认6379
port = 6379
# 超时默认2000
timeout = 2000
# 连接超时默认timeout
connectionTimeout = 2000
# 读取超时默认timeout
soTimeout = 2000
# 密码,默认无
password = 123456
# 数据库序号默认0
database = 0
# 客户端名,默认"Hutool"
clientName = udidl
# SSL连接默认false
ssl = false;
#----- 自定义分组的连接
[custom]
# 地址默认localhost
host = 192.168.235.137
# 连接耗尽时是否阻塞, false报异常,ture阻塞直到超时, 默认true
BlockWhenExhausted = true;
# 设置的逐出策略类名, 默认DefaultEvictionPolicy(当连接超过最大空闲时间,或连接数超过最大空闲连接数)
evictionPolicyClassName = org.apache.commons.pool2.impl.DefaultEvictionPolicy
# 是否启用pool的jmx管理功能, 默认true
jmxEnabled = true;
# 是否启用后进先出, 默认true
lifo = true;
# 最大空闲连接数, 默认8个
maxIdle = 8
# 最小空闲连接数, 默认0
minIdle = 0
# 最大连接数, 默认8个
maxTotal = 8
# 获取连接时的最大等待毫秒数(如果设置为阻塞时BlockWhenExhausted),如果超时就抛异常, 小于零:阻塞不确定的时间, 默认-1
maxWaitMillis = -1
# 逐出连接的最小空闲时间 默认1800000毫秒(30分钟)
minEvictableIdleTimeMillis = 1800000
# 每次逐出检查时 逐出的最大数目 如果为负数就是 : 1/abs(n), 默认3
numTestsPerEvictionRun = 3;
# 对象空闲多久后逐出, 当空闲时间>该值 且 空闲连接>最大空闲数 时直接逐出,不再根据MinEvictableIdleTimeMillis判断 (默认逐出策略)
SoftMinEvictableIdleTimeMillis = 1800000
# 在获取连接的时候检查有效性, 默认false
testOnBorrow = false
# 在空闲时检查有效性, 默认false
testWhileIdle = false
# 逐出扫描的时间间隔(毫秒) 如果为负数,则不运行逐出线程, 默认-1
timeBetweenEvictionRunsMillis = -1

@ -1,8 +1,10 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="com.glxp.udidl.admin.dao.udid.JobLogMapper">
<select id="list" parameterType="com.glxp.udidl.admin.req.udid.JobLogFilterRequest" resultType="com.glxp.udidl.admin.entity.udid.JobLog">
select * from job_log
<select id="list" parameterType="com.glxp.udidl.admin.req.udid.JobLogFilterRequest"
resultType="com.glxp.udidl.admin.entity.udid.JobLog">
select *
from job_log
<where>
<if test="type != '' and type != null">
and type = #{type}
@ -20,8 +22,4 @@
order by downloadDate desc, createTime desc
</select>
<insert id="insert" parameterType="com.glxp.udidl.admin.entity.udid.JobLog">
insert into job_log (type,downloadType,downloadDate,totalCount,realCount,insertCount, msg, content,createTime)
values (#{type},#{downloadType},#{downloadDate},#{totalCount},#{realCount},#{insertCount},#{msg},#{content},#{createTime})
</insert>
</mapper>
Loading…
Cancel
Save