From 4c9d8599fd25be1c2925d14a5b3abef6c6543bd0 Mon Sep 17 00:00:00 2001 From: ls Date: Mon, 4 Nov 2024 20:09:48 +0800 Subject: [PATCH] update --- docker-compose.yml | 9 + .../src/main/resources/application-prod.yml | 2 +- .../controller/DatabaseRecordController.java | 178 ++++++++++ .../controller/ExperimentController.java | 2 +- .../controller/ImportRecordController.java | 179 ---------- ...{ImportRecord.java => DatabaseRecord.java} | 18 +- .../database/mapper/DatabaseRecordMapper.java | 17 + .../database/mapper/ImportRecordMapper.java | 14 - .../mapper/xml/DatabaseRecordMapper.xml | 5 + .../service/IDatabaseRecordService.java | 14 + .../service/IImportRecordService.java | 16 - .../impl/DatabaseRecordServiceImpl.java | 19 + .../service/impl/ImportRecordServiceImpl.java | 336 ------------------ 13 files changed, 253 insertions(+), 556 deletions(-) create mode 100644 physical-module-system/physical-system-biz/src/main/java/org/jeecg/modules/database/controller/DatabaseRecordController.java delete mode 100755 physical-module-system/physical-system-biz/src/main/java/org/jeecg/modules/database/controller/ImportRecordController.java rename physical-module-system/physical-system-biz/src/main/java/org/jeecg/modules/database/entity/{ImportRecord.java => DatabaseRecord.java} (90%) create mode 100644 physical-module-system/physical-system-biz/src/main/java/org/jeecg/modules/database/mapper/DatabaseRecordMapper.java delete mode 100755 physical-module-system/physical-system-biz/src/main/java/org/jeecg/modules/database/mapper/ImportRecordMapper.java create mode 100644 physical-module-system/physical-system-biz/src/main/java/org/jeecg/modules/database/mapper/xml/DatabaseRecordMapper.xml create mode 100644 physical-module-system/physical-system-biz/src/main/java/org/jeecg/modules/database/service/IDatabaseRecordService.java delete mode 100755 physical-module-system/physical-system-biz/src/main/java/org/jeecg/modules/database/service/IImportRecordService.java create mode 100644 physical-module-system/physical-system-biz/src/main/java/org/jeecg/modules/database/service/impl/DatabaseRecordServiceImpl.java delete mode 100644 physical-module-system/physical-system-biz/src/main/java/org/jeecg/modules/database/service/impl/ImportRecordServiceImpl.java diff --git a/docker-compose.yml b/docker-compose.yml index c247f5e..3f18504 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -65,6 +65,15 @@ services: - 28080:8080 networks: - physical-boot + physical-kkfileview: + restart: on-failure + container_name: physical-kkfileview + image: registry.cn-shanghai.aliyuncs.com/physical/kkfileview + hostname: physical-kkfileview + ports: + - 8099:8012 + networks: + - physical-boot physical-web: # build: diff --git a/physical-launcher/src/main/resources/application-prod.yml b/physical-launcher/src/main/resources/application-prod.yml index 6436097..c76c779 100644 --- a/physical-launcher/src/main/resources/application-prod.yml +++ b/physical-launcher/src/main/resources/application-prod.yml @@ -68,7 +68,7 @@ spring: job-store-type: jdbc initialize-schema: embedded #定时任务开关，true-开 false-关 - auto-startup: true + auto-startup: false #延迟1秒启动定时任务 startup-delay: 1s #启动时更新己存在的Job diff --git a/physical-module-system/physical-system-biz/src/main/java/org/jeecg/modules/database/controller/DatabaseRecordController.java b/physical-module-system/physical-system-biz/src/main/java/org/jeecg/modules/database/controller/DatabaseRecordController.java new file mode 100644 index 0000000..42af2f0 --- /dev/null +++ b/physical-module-system/physical-system-biz/src/main/java/org/jeecg/modules/database/controller/DatabaseRecordController.java @@ -0,0 +1,178 @@ +package org.jeecg.modules.database.controller; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.net.URLDecoder; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import org.jeecg.common.api.vo.Result; +import org.jeecg.common.system.query.QueryGenerator; +import org.jeecg.common.util.oConvertUtils; +import org.jeecg.modules.demo.database.entity.DatabaseRecord; +import org.jeecg.modules.demo.database.service.IDatabaseRecordService; + +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import lombok.extern.slf4j.Slf4j; + +import org.jeecgframework.poi.excel.ExcelImportUtil; +import org.jeecgframework.poi.excel.def.NormalExcelConstants; +import org.jeecgframework.poi.excel.entity.ExportParams; +import org.jeecgframework.poi.excel.entity.ImportParams; +import org.jeecgframework.poi.excel.view.JeecgEntityExcelView; +import org.jeecg.common.system.base.controller.JeecgController; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.*; +import org.springframework.web.multipart.MultipartFile; +import org.springframework.web.multipart.MultipartHttpServletRequest; +import org.springframework.web.servlet.ModelAndView; +import com.alibaba.fastjson.JSON; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.tags.Tag; +import org.jeecg.common.aspect.annotation.AutoLog; +import org.apache.shiro.authz.annotation.RequiresPermissions; + + /** + * @Description: 数据库管理 + * @Author: jeecg-boot + * @Date: 2024-11-04 + * @Version: V1.0 + */ +@Tag(name="数据库管理") +@RestController +@RequestMapping("/database/databaseRecord") +@Slf4j +public class DatabaseRecordController extends JeecgController { + @Autowired + private IDatabaseRecordService databaseRecordService; + + /** + * 分页列表查询 + * + * @param databaseRecord + * @param pageNo + * @param pageSize + * @param req + * @return + */ + //@AutoLog(value = "数据库管理-分页列表查询") + @Operation(summary="数据库管理-分页列表查询") + @GetMapping(value = "/list") + public Result> queryPageList(DatabaseRecord databaseRecord, + @RequestParam(name="pageNo", defaultValue="1") Integer pageNo, + @RequestParam(name="pageSize", defaultValue="10") Integer pageSize, + HttpServletRequest req) { + QueryWrapper queryWrapper = QueryGenerator.initQueryWrapper(databaseRecord, req.getParameterMap()); + Page page = new Page(pageNo, pageSize); + IPage pageList = databaseRecordService.page(page, queryWrapper); + return Result.OK(pageList); + } + + /** + * 添加 + * + * @param databaseRecord + * @return + */ + @AutoLog(value = "数据库管理-添加") + @Operation(summary="数据库管理-添加") + @RequiresPermissions("database:database_record:add") + @PostMapping(value = "/add") + public Result add(@RequestBody DatabaseRecord databaseRecord) { + databaseRecordService.save(databaseRecord); + return Result.OK("添加成功！"); + } + + /** + * 编辑 + * + * @param databaseRecord + * @return + */ + @AutoLog(value = "数据库管理-编辑") + @Operation(summary="数据库管理-编辑") + @RequiresPermissions("database:database_record:edit") + @RequestMapping(value = "/edit", method = {RequestMethod.PUT,RequestMethod.POST}) + public Result edit(@RequestBody DatabaseRecord databaseRecord) { + databaseRecordService.updateById(databaseRecord); + return Result.OK("编辑成功!"); + } + + /** + * 通过id删除 + * + * @param id + * @return + */ + @AutoLog(value = "数据库管理-通过id删除") + @Operation(summary="数据库管理-通过id删除") + @RequiresPermissions("database:database_record:delete") + @DeleteMapping(value = "/delete") + public Result delete(@RequestParam(name="id",required=true) String id) { + databaseRecordService.removeById(id); + return Result.OK("删除成功!"); + } + + /** + * 批量删除 + * + * @param ids + * @return + */ + @AutoLog(value = "数据库管理-批量删除") + @Operation(summary="数据库管理-批量删除") + @RequiresPermissions("database:database_record:deleteBatch") + @DeleteMapping(value = "/deleteBatch") + public Result deleteBatch(@RequestParam(name="ids",required=true) String ids) { + this.databaseRecordService.removeByIds(Arrays.asList(ids.split(","))); + return Result.OK("批量删除成功!"); + } + + /** + * 通过id查询 + * + * @param id + * @return + */ + //@AutoLog(value = "数据库管理-通过id查询") + @Operation(summary="数据库管理-通过id查询") + @GetMapping(value = "/queryById") + public Result queryById(@RequestParam(name="id",required=true) String id) { + DatabaseRecord databaseRecord = databaseRecordService.getById(id); + if(databaseRecord==null) { + return Result.error("未找到对应数据"); + } + return Result.OK(databaseRecord); + } + + /** + * 导出excel + * + * @param request + * @param databaseRecord + */ + @RequiresPermissions("database:database_record:exportXls") + @RequestMapping(value = "/exportXls") + public ModelAndView exportXls(HttpServletRequest request, DatabaseRecord databaseRecord) { + return super.exportXls(request, databaseRecord, DatabaseRecord.class, "数据库管理"); + } + + /** + * 通过excel导入数据 + * + * @param request + * @param response + * @return + */ + @RequiresPermissions("database:database_record:importExcel") + @RequestMapping(value = "/importExcel", method = RequestMethod.POST) + public Result importExcel(HttpServletRequest request, HttpServletResponse response) { + return super.importExcel(request, response, DatabaseRecord.class); + } + +} diff --git a/physical-module-system/physical-system-biz/src/main/java/org/jeecg/modules/database/controller/ExperimentController.java b/physical-module-system/physical-system-biz/src/main/java/org/jeecg/modules/database/controller/ExperimentController.java index 6791fd9..f4af82e 100644 --- a/physical-module-system/physical-system-biz/src/main/java/org/jeecg/modules/database/controller/ExperimentController.java +++ b/physical-module-system/physical-system-biz/src/main/java/org/jeecg/modules/database/controller/ExperimentController.java @@ -58,7 +58,7 @@ public class ExperimentController extends JeecgController> queryPageList(Experiment experiment, @RequestParam(name = "pageNo", defaultValue = "1") Integer pageNo, @RequestParam(name = "pageSize", defaultValue = "10") Integer pageSize, diff --git a/physical-module-system/physical-system-biz/src/main/java/org/jeecg/modules/database/controller/ImportRecordController.java b/physical-module-system/physical-system-biz/src/main/java/org/jeecg/modules/database/controller/ImportRecordController.java deleted file mode 100755 index 874aa5d..0000000 --- a/physical-module-system/physical-system-biz/src/main/java/org/jeecg/modules/database/controller/ImportRecordController.java +++ /dev/null @@ -1,179 +0,0 @@ -package org.jeecg.modules.database.controller; - -import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; -import com.baomidou.mybatisplus.core.metadata.IPage; -import com.baomidou.mybatisplus.extension.plugins.pagination.Page; -import io.swagger.v3.oas.annotations.Operation; -import io.swagger.v3.oas.annotations.tags.Tag; -import jakarta.servlet.http.HttpServletRequest; -import jakarta.servlet.http.HttpServletResponse; -import lombok.extern.slf4j.Slf4j; -import org.apache.shiro.authz.annotation.RequiresPermissions; -import org.jeecg.common.api.vo.Result; -import org.jeecg.common.aspect.annotation.AutoLog; -import org.jeecg.common.system.base.controller.JeecgController; -import org.jeecg.common.system.query.QueryGenerator; -import org.jeecg.config.shiro.IgnoreAuth; -import org.jeecg.modules.database.entity.ImportRecord; -import org.jeecg.modules.database.service.IImportRecordService; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.web.bind.annotation.*; -import org.springframework.web.servlet.ModelAndView; - -import java.util.Arrays; - -/** - * @Description: 三方导入记录 - * @Author: jeecg-boot - * @Date: 2024-09-03 - * @Version: V1.0 - */ -@Tag(name = "三方导入记录") -@RestController -@RequestMapping("/database/crawlerRecord") -@Slf4j -public class ImportRecordController extends JeecgController { - @Autowired - private IImportRecordService crawlerRecordService; - - /** - * 分页列表查询 - * - * @param importRecord - * @param pageNo - * @param pageSize - * @param req - * @return - */ - //@AutoLog(value = "三方导入记录-分页列表查询") - @Operation(summary = "三方导入记录-分页列表查询") - @GetMapping(value = "/list") - public Result> queryPageList(ImportRecord importRecord, - @RequestParam(name = "pageNo", defaultValue = "1") Integer pageNo, - @RequestParam(name = "pageSize", defaultValue = "10") Integer pageSize, - HttpServletRequest req) { - QueryWrapper queryWrapper = QueryGenerator.initQueryWrapper(importRecord, req.getParameterMap()); - Page page = new Page(pageNo, pageSize); - IPage pageList = crawlerRecordService.page(page, queryWrapper); - return Result.OK(pageList); - } - - /** - * 添加 - * - * @param importRecord - * @return - */ - @AutoLog(value = "三方导入记录-添加") - @Operation(summary = "三方导入记录-添加") - @RequiresPermissions("database:crawler_record:add") - @PostMapping(value = "/add") - public Result add(@RequestBody ImportRecord importRecord) { - crawlerRecordService.save(importRecord); - return Result.OK("添加成功！"); - } - - /** - * 编辑 - * - * @param importRecord - * @return - */ - @AutoLog(value = "三方导入记录-编辑") - @Operation(summary = "三方导入记录-编辑") - @RequiresPermissions("database:crawler_record:edit") - @RequestMapping(value = "/edit", method = {RequestMethod.PUT, RequestMethod.POST}) - public Result edit(@RequestBody ImportRecord importRecord) { - crawlerRecordService.updateById(importRecord); - return Result.OK("编辑成功!"); - } - - /** - * 通过id删除 - * - * @param id - * @return - */ - @AutoLog(value = "三方导入记录-通过id删除") - @Operation(summary = "三方导入记录-通过id删除") - @RequiresPermissions("database:crawler_record:delete") - @DeleteMapping(value = "/delete") - public Result delete(@RequestParam(name = "id", required = true) String id) { - crawlerRecordService.removeById(id); - return Result.OK("删除成功!"); - } - - /** - * 批量删除 - * - * @param ids - * @return - */ - @AutoLog(value = "三方导入记录-批量删除") - @Operation(summary = "三方导入记录-批量删除") - @RequiresPermissions("database:crawler_record:deleteBatch") - @DeleteMapping(value = "/deleteBatch") - public Result deleteBatch(@RequestParam(name = "ids", required = true) String ids) { - this.crawlerRecordService.removeByIds(Arrays.asList(ids.split(","))); - return Result.OK("批量删除成功!"); - } - - /** - * 通过id查询 - * - * @param id - * @return - */ - //@AutoLog(value = "三方导入记录-通过id查询") - @Operation(summary = "三方导入记录-通过id查询") - @GetMapping(value = "/queryById") - public Result queryById(@RequestParam(name = "id", required = true) String id) { - ImportRecord importRecord = crawlerRecordService.getById(id); - if (importRecord == null) { - return Result.error("未找到对应数据"); - } - return Result.OK(importRecord); - } - - /** - * 导出excel - * - * @param request - * @param importRecord - */ - @RequiresPermissions("database:crawler_record:exportXls") - @RequestMapping(value = "/exportXls") - public ModelAndView exportXls(HttpServletRequest request, ImportRecord importRecord) { - return super.exportXls(request, importRecord, ImportRecord.class, "三方导入记录"); - } - - /** - * 通过excel导入数据 - * - * @param request - * @param response - * @return - */ - @RequiresPermissions("database:crawler_record:importExcel") - @RequestMapping(value = "/importExcel", method = RequestMethod.POST) - public Result importExcel(HttpServletRequest request, HttpServletResponse response) { - return super.importExcel(request, response, ImportRecord.class); - } - - @IgnoreAuth - @Operation(summary = "执行爬虫-esarad") - @RequestMapping(value = "/crawler/esarad", method = RequestMethod.GET) - public Result esaradCrawler(HttpServletRequest request, HttpServletResponse response) { - crawlerRecordService.esaradCrawler(); - return Result.OK(); - } - - @IgnoreAuth - @Operation(summary = "执行爬虫-radhome") - @RequestMapping(value = "/crawler/radhome", method = RequestMethod.GET) - public Result radhomeCrawler(HttpServletRequest request, HttpServletResponse response) { - crawlerRecordService.radhomeCrawler(); - return Result.OK(); - } - -} diff --git a/physical-module-system/physical-system-biz/src/main/java/org/jeecg/modules/database/entity/ImportRecord.java b/physical-module-system/physical-system-biz/src/main/java/org/jeecg/modules/database/entity/DatabaseRecord.java similarity index 90% rename from physical-module-system/physical-system-biz/src/main/java/org/jeecg/modules/database/entity/ImportRecord.java rename to physical-module-system/physical-system-biz/src/main/java/org/jeecg/modules/database/entity/DatabaseRecord.java index afc7823..cb78f78 100644 --- a/physical-module-system/physical-system-biz/src/main/java/org/jeecg/modules/database/entity/ImportRecord.java +++ b/physical-module-system/physical-system-biz/src/main/java/org/jeecg/modules/database/entity/DatabaseRecord.java @@ -18,17 +18,17 @@ import lombok.EqualsAndHashCode; import lombok.experimental.Accessors; /** - * @Description: 三方导入记录 + * @Description: 数据库管理 * @Author: jeecg-boot - * @Date: 2024-10-21 + * @Date: 2024-11-04 * @Version: V1.0 */ @Data -@TableName("import_record") +@TableName("database_record") @Accessors(chain = true) @EqualsAndHashCode(callSuper = false) -@Schema(description="三方导入记录") -public class ImportRecord implements Serializable { +@Schema(description="数据库管理") +public class DatabaseRecord implements Serializable { private static final long serialVersionUID = 1L; /**主键*/ @@ -79,7 +79,7 @@ public class ImportRecord implements Serializable { @Schema(description = "生产厂家") private String manufacturer; /**试验时间*/ - @Excel(name = "试验时间", width = 20) + @Excel(name = "试验时间", width = 15) @Schema(description = "试验时间") private String experimentDate; /**数据来源*/ @@ -94,8 +94,8 @@ public class ImportRecord implements Serializable { @Excel(name = "条目数统计", width = 15) @Schema(description = "条目数统计") private String totalCount; - /**附件ID*/ - @Excel(name = "附件ID", width = 15) - @Schema(description = "附件ID") + /**附件IDs*/ + @Excel(name = "附件IDs", width = 15) + @Schema(description = "附件IDs") private String fileList; } diff --git a/physical-module-system/physical-system-biz/src/main/java/org/jeecg/modules/database/mapper/DatabaseRecordMapper.java b/physical-module-system/physical-system-biz/src/main/java/org/jeecg/modules/database/mapper/DatabaseRecordMapper.java new file mode 100644 index 0000000..e22fce9 --- /dev/null +++ b/physical-module-system/physical-system-biz/src/main/java/org/jeecg/modules/database/mapper/DatabaseRecordMapper.java @@ -0,0 +1,17 @@ +package org.jeecg.modules.database.mapper; + +import java.util.List; + +import org.apache.ibatis.annotations.Param; +import org.jeecg.modules.demo.database.entity.DatabaseRecord; +import com.baomidou.mybatisplus.core.mapper.BaseMapper; + +/** + * @Description: 数据库管理 + * @Author: jeecg-boot + * @Date: 2024-11-04 + * @Version: V1.0 + */ +public interface DatabaseRecordMapper extends BaseMapper { + +} diff --git a/physical-module-system/physical-system-biz/src/main/java/org/jeecg/modules/database/mapper/ImportRecordMapper.java b/physical-module-system/physical-system-biz/src/main/java/org/jeecg/modules/database/mapper/ImportRecordMapper.java deleted file mode 100755 index 75c37ed..0000000 --- a/physical-module-system/physical-system-biz/src/main/java/org/jeecg/modules/database/mapper/ImportRecordMapper.java +++ /dev/null @@ -1,14 +0,0 @@ -package org.jeecg.modules.database.mapper; - -import org.jeecg.modules.database.entity.ImportRecord; -import com.baomidou.mybatisplus.core.mapper.BaseMapper; - -/** - * @Description: 三方导入记录 - * @Author: jeecg-boot - * @Date: 2024-09-03 - * @Version: V1.0 - */ -public interface ImportRecordMapper extends BaseMapper { - -} diff --git a/physical-module-system/physical-system-biz/src/main/java/org/jeecg/modules/database/mapper/xml/DatabaseRecordMapper.xml b/physical-module-system/physical-system-biz/src/main/java/org/jeecg/modules/database/mapper/xml/DatabaseRecordMapper.xml new file mode 100644 index 0000000..5ce631d --- /dev/null +++ b/physical-module-system/physical-system-biz/src/main/java/org/jeecg/modules/database/mapper/xml/DatabaseRecordMapper.xml @@ -0,0 +1,5 @@ + + + + + \ No newline at end of file diff --git a/physical-module-system/physical-system-biz/src/main/java/org/jeecg/modules/database/service/IDatabaseRecordService.java b/physical-module-system/physical-system-biz/src/main/java/org/jeecg/modules/database/service/IDatabaseRecordService.java new file mode 100644 index 0000000..8835d66 --- /dev/null +++ b/physical-module-system/physical-system-biz/src/main/java/org/jeecg/modules/database/service/IDatabaseRecordService.java @@ -0,0 +1,14 @@ +package org.jeecg.modules.database.service; + +import org.jeecg.modules.demo.database.entity.DatabaseRecord; +import com.baomidou.mybatisplus.extension.service.IService; + +/** + * @Description: 数据库管理 + * @Author: jeecg-boot + * @Date: 2024-11-04 + * @Version: V1.0 + */ +public interface IDatabaseRecordService extends IService { + +} diff --git a/physical-module-system/physical-system-biz/src/main/java/org/jeecg/modules/database/service/IImportRecordService.java b/physical-module-system/physical-system-biz/src/main/java/org/jeecg/modules/database/service/IImportRecordService.java deleted file mode 100755 index 9ced2dc..0000000 --- a/physical-module-system/physical-system-biz/src/main/java/org/jeecg/modules/database/service/IImportRecordService.java +++ /dev/null @@ -1,16 +0,0 @@ -package org.jeecg.modules.database.service; - -import org.jeecg.modules.database.entity.ImportRecord; -import com.baomidou.mybatisplus.extension.service.IService; - -/** - * @Description: 爬虫记录 - * @Author: jeecg-boot - * @Date: 2024-09-03 - * @Version: V1.0 - */ -public interface IImportRecordService extends IService { - void esaradCrawler(); - - void radhomeCrawler(); -} diff --git a/physical-module-system/physical-system-biz/src/main/java/org/jeecg/modules/database/service/impl/DatabaseRecordServiceImpl.java b/physical-module-system/physical-system-biz/src/main/java/org/jeecg/modules/database/service/impl/DatabaseRecordServiceImpl.java new file mode 100644 index 0000000..50f6c3f --- /dev/null +++ b/physical-module-system/physical-system-biz/src/main/java/org/jeecg/modules/database/service/impl/DatabaseRecordServiceImpl.java @@ -0,0 +1,19 @@ +package org.jeecg.modules.database.service.impl; + +import org.jeecg.modules.demo.database.entity.DatabaseRecord; +import org.jeecg.modules.demo.database.mapper.DatabaseRecordMapper; +import org.jeecg.modules.demo.database.service.IDatabaseRecordService; +import org.springframework.stereotype.Service; + +import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; + +/** + * @Description: 数据库管理 + * @Author: jeecg-boot + * @Date: 2024-11-04 + * @Version: V1.0 + */ +@Service +public class DatabaseRecordServiceImpl extends ServiceImpl implements IDatabaseRecordService { + +} diff --git a/physical-module-system/physical-system-biz/src/main/java/org/jeecg/modules/database/service/impl/ImportRecordServiceImpl.java b/physical-module-system/physical-system-biz/src/main/java/org/jeecg/modules/database/service/impl/ImportRecordServiceImpl.java deleted file mode 100644 index 29cfb47..0000000 --- a/physical-module-system/physical-system-biz/src/main/java/org/jeecg/modules/database/service/impl/ImportRecordServiceImpl.java +++ /dev/null @@ -1,336 +0,0 @@ -package org.jeecg.modules.database.service.impl; - -import cn.hutool.core.collection.CollUtil; -import cn.hutool.core.io.FileUtil; -import cn.hutool.core.io.IoUtil; -import cn.hutool.core.thread.ThreadUtil; -import cn.hutool.core.util.ReUtil; -import cn.hutool.core.util.URLUtil; -import cn.hutool.http.Header; -import cn.hutool.http.HttpRequest; -import cn.hutool.http.HttpResponse; -import cn.hutool.http.HttpUtil; -import com.alibaba.fastjson2.JSON; -import com.alibaba.fastjson2.JSONArray; -import com.alibaba.fastjson2.JSONObject; -import com.baomidou.mybatisplus.core.toolkit.Wrappers; -import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; -import lombok.extern.slf4j.Slf4j; -import okhttp3.*; -import org.apache.commons.lang3.StringUtils; -import org.jeecg.common.util.MinioUtil; -import org.jeecg.common.util.RedisUtil; -import org.jeecg.modules.database.entity.ImportRecord; -import org.jeecg.modules.database.mapper.ImportRecordMapper; -import org.jeecg.modules.database.service.IImportRecordService; -import org.jeecg.modules.oss.service.impl.OssFileServiceImpl; -import org.jetbrains.annotations.NotNull; -import org.jsoup.Jsoup; -import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; -import org.jsoup.select.Elements; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - -import java.io.ByteArrayInputStream; -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.nio.charset.Charset; -import java.util.ArrayList; -import java.util.List; -import java.util.Objects; - -@Slf4j -@Service -public class ImportRecordServiceImpl extends ServiceImpl implements IImportRecordService { - @Autowired - private RedisUtil redisUtil; - - @Autowired - private OssFileServiceImpl ossFileService; - - private String esaradKey = "crawler-esarad"; - private String radhomeKey = "crawler-radhome"; - - public static void main(String[] args) { - HttpResponse execute = HttpRequest.get("https://esarad.esa.int/?id=76&handler=DownloadDb").execute(); - final List dispositions = execute.headerList(Header.CONTENT_DISPOSITION.getValue()); - String fileName = null; - if (CollUtil.isNotEmpty(dispositions)) { - for (String disposition : dispositions) { - fileName = ReUtil.getGroup1("filename" + "=([^;]+)", disposition); - } - System.out.println(URLUtil.decode("N2920A%20TID_1009_01.pdf", Charset.defaultCharset())); - System.out.println(fileName); - // filename* 采用了 RFC 5987 中规定的编码方式，优先读取 - - } - - } - - /** - * https://esarad.esa.int/ - */ -// @Override -// public void esaradCrawler() { -// try { -// String crawlerRunning = String.valueOf(redisUtil.get(esaradKey)); -// if (StringUtils.equals(crawlerRunning, "T")) { -// throw new RuntimeException("爬虫任务执行中"); -// } -// redisUtil.set(esaradKey, "T", 24 * 60 * 60); -// -// List tableData = new ArrayList<>(); -// -// Document doc = Jsoup.connect("https://esarad.esa.int").get(); -// Element table = doc.getElementById("dtReports"); -// if (Objects.isNull(table)) { -// redisUtil.del(esaradKey); -// throw new RuntimeException("爬虫获取数据失败"); -// } -// // Get the tbody element within the table -// Element tbody = table.select("tbody").first(); // Select the first tbody element -// -// if (tbody == null) { -// redisUtil.del(esaradKey); -// throw new RuntimeException("爬虫获取数据失败"); -// } -// // Create a list to store the row maps -// -// // Extract headers from the first row -// Elements headers = table.select("thead").first().select("tr").first().select("th"); -// List headerNames = new ArrayList<>(); -// for (Element header : headers) { -// headerNames.add(header.text()); -// } -// -// // Select all rows in the tbody -// Elements rows = tbody.select("tr"); -// -// for (int j = 0; j < rows.size(); j++) { -// if (j > 10) { -// break; -// } -// Element row = rows.get(j); -// -// // Select all cells in the row -// ImportRecord importRecord = new ImportRecord(); -// -// Elements cells = row.select("td"); -// -// if (cells.size() == headerNames.size()) { // Ensure the number of cells matches the number of headers -// -// for (int i = 0; i < cells.size(); i++) { -// -// String header = headerNames.get(i); -// String value = cells.get(i).text(); -// switch (header) { -// case "Radiation Test Method": -// importRecord.setTestMethod(value); -// break; -// case "EPPL Familiy": -// importRecord.setCategory(value); -// break; -// case "EPPL Group": -// importRecord.setSubCategory(value); -// break; -// case "DUT Manufacturer": -// importRecord.setManufacturer(value); -// break; -// case "Function": -// importRecord.setFunctionType(value); -// break; -// case "Report Date": -// importRecord.setReportDate(value); -// break; -// case "Report Source": -// importRecord.setReportSource(value); -// break; -// case "Technology": -// importRecord.setTechnology(value); -// break; -// case "Id": -// importRecord.setReportId(value); -// break; -// case "DUT part type": -// importRecord.setCode(value); -// break; -// case "Radiation Test Type": -// importRecord.setRadiationTestType(value); -// break; -// } -// -// } -// -// // Add the map to the list -// tableData.add(importRecord); -// } -// } -// -// // Print the list of maps -// for (ImportRecord rowMap : tableData) { -// rowMap.setFileUrl("https://esarad.esa.int/?id=" + rowMap.getReportId() + "&handler=DownloadDb"); -// } -// saveEsaradFiles(tableData, esaradKey); -// -// } catch (Exception e) { -// redisUtil.del(esaradKey); -// throw new RuntimeException(e); -// } -// } - - /** - * https://radhome.gsfc.nasa.gov/radhome/RadDataBase/RadDataBase.html - */ -// @Override -// public void radhomeCrawler() { -// try { -// String crawlerRunning = String.valueOf(redisUtil.get(radhomeKey)); -// if (StringUtils.equals(crawlerRunning, "T")) { -// throw new RuntimeException("爬虫任务执行中"); -// } -// redisUtil.set(radhomeKey, "T", 24 * 60 * 60); -// -// OkHttpClient client = new OkHttpClient(); -// String url = "https://radhome.gsfc.nasa.gov/radhome/dev/parts.cfc?method=getParts"; -// FormBody formBody = new FormBody.Builder().add("_search", "false").add("nd", System.currentTimeMillis() + "").add("rows", "10").add("page", "1").add("sidx", "partnumber").add("sord", "asc").build(); -// -// Request request = new Request.Builder().url(url).post(formBody).build(); -// -// client.newCall(request).enqueue(new Callback() { -// @Override -// public void onResponse(@NotNull Call call, @NotNull Response response) throws IOException { -// String jsonResponse = response.body().string(); -// JSONObject jsonObject = JSON.parseObject(jsonResponse); -// Long total = jsonObject.getLong("RECORDS"); -// JSONArray list = jsonObject.getJSONArray("ROWS"); -// System.out.println("total count " + total); -// System.out.println("total list " + list.get(0)); -// List tableData = new ArrayList<>(); -// -// for (int i = 0; i < list.size(); i++) { -// -// if (i > 10) { -// break; -// } -// JSONArray row = (JSONArray) list.get(i); -// String fileNames = String.valueOf(row.get(4)); -// ImportRecord map = new ImportRecord(); -// String fileUrls = fixFileNames(fileNames); -// map.setFileUrl(fileUrls); -// -// map.setCode(String.valueOf(row.get(0))); -// map.setFunctionType(String.valueOf(row.get(1))); -// map.setManufacturer(String.valueOf(row.get(2))); -// map.setReportDate(String.valueOf(row.get(3))); -// map.setTestMethod(String.valueOf(row.get(5))); -// map.setCategory(String.valueOf(row.get(6))); -// map.setReportId(map.getCode().replaceAll(" ", "")); -// tableData.add(map); -// } -// saveRadhomeFiles(tableData, radhomeKey); -// } -// -// @Override -// public void onFailure(@NotNull Call call, @NotNull IOException e) { -// e.printStackTrace(); -// redisUtil.del(radhomeKey); -// } -// }); -// } catch (Exception e) { -// redisUtil.del(radhomeKey); -// throw new RuntimeException(e); -// } -// } - -// private String fixFileNames(String fileNames) { -// String[] split = StringUtils.split(fileNames, ";"); -// List result = new ArrayList<>(); -// for (String s : split) { -// if (!StringUtils.startsWith(s, "http")) { -// result.add("https://radhome.gsfc.nasa.gov/radhome/papers/" + s); -// } else { -// result.add(s); -// } -// } -// return StringUtils.join(result, ";"); -// } -// -// private void saveRadhomeFiles(List fileList, String type) { -// ThreadUtil.execute(() -> { -// try { -// for (ImportRecord record : fileList) { -// String fileUploadResult = ""; -// String fileUrl = record.getFileUrl(); -// if (fileUrl.contains(";")) { -// String[] split = fileUrl.split(";"); -// List result = new ArrayList<>(); -// for (String s : split) { -// byte[] fileBytes = HttpUtil.downloadBytes(s); -// InputStream inputStream = new ByteArrayInputStream(fileBytes); -// result.add(MinioUtil.upload(inputStream, "radhome/" + s.substring(s.lastIndexOf("/") + 1))); -// fileUploadResult = StringUtils.join(result, ";"); -// } -// } else { -// byte[] fileBytes = HttpUtil.downloadBytes(fileUrl); -// InputStream inputStream = new ByteArrayInputStream(fileBytes); -// fileUploadResult = MinioUtil.upload(inputStream, "radhome/" + fileUrl.substring(fileUrl.lastIndexOf("/") + 1)); -// } -// System.out.println(fileUploadResult); -// if (StringUtils.isNotBlank(fileUploadResult)) { -// record.setFileUrl(fileUploadResult); -// save(record); -// } -// } -// redisUtil.del(type); -// } catch (Exception e) { -// e.printStackTrace(); -// } -// }); -// } -// -// private void saveEsaradFiles(List fileList, String type) { -// ThreadUtil.execute(() -> { -// try { -// for (ImportRecord record : fileList) { -// ImportRecord dbData = getOne(Wrappers.lambdaQuery().eq(ImportRecord::getReportSource, record.getReportId())); -// if (Objects.nonNull(dbData)) { -// continue; -// } -// String resultStr = ""; -// -// String dest = FileUtil.getTmpDirPath() + "esarad-" + record.getReportId() + "/"; -// FileUtil.mkdir(dest); -// long fileSize = HttpUtil.downloadFile(record.getFileUrl(), dest); -// -// if (fileSize > 0) { -// List files = FileUtil.loopFiles(dest); -// for (File file : files) { -// resultStr = MinioUtil.upload(IoUtil.toStream(file), "esarad/" + record.getReportId() + "-" + URLUtil.decode(file.getName(), Charset.defaultCharset()) ); -// } -// } -// -// System.out.println(resultStr); -// if (StringUtils.isNotBlank(resultStr)) { -// record.setFileUrl(resultStr); -// save(record); -// } -// } -// redisUtil.del(type); -// } catch (Exception e) { -// e.printStackTrace(); -// } -// }); -// } - - @Override - public void esaradCrawler() { - - } - - @Override - public void radhomeCrawler() { - - } -}