physical3/physical-boot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update

Browse Source
This commit is contained in:
ls
2024-11-04 20:09:48 +08:00
parent 08d878d1dd
commit 4c9d8599fd
13 changed files with 253 additions and 556 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
docker-compose.yml
View File

@@ -65,6 +65,15 @@ services:
- 28080:8080
networks:
- physical-boot
physical-kkfileview:
restart: on-failure
container_name: physical-kkfileview
image: registry.cn-shanghai.aliyuncs.com/physical/kkfileview
hostname: physical-kkfileview
ports:
- 8099:8012
networks:
- physical-boot
physical-web:
# build:

2
physical-launcher/src/main/resources/application-prod.yml
View File

@@ -68,7 +68,7 @@ spring:
job-store-type: jdbc
initialize-schema: embedded
#定时任务开关，true-开 false-关
auto-startup: true
auto-startup: false
#延迟1秒启动定时任务
startup-delay: 1s
#启动时更新己存在的Job

178
physical-module-system/physical-system-biz/src/main/java/org/jeecg/modules/database/controller/DatabaseRecordController.java Normal file
View File

@@ -0,0 +1,178 @@
package org.jeecg.modules.database.controller;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.URLDecoder;
import jakarta.servlet.http.HttpServletRequest;
import jakarta.servlet.http.HttpServletResponse;
import org.jeecg.common.api.vo.Result;
import org.jeecg.common.system.query.QueryGenerator;
import org.jeecg.common.util.oConvertUtils;
import org.jeecg.modules.demo.database.entity.DatabaseRecord;
import org.jeecg.modules.demo.database.service.IDatabaseRecordService;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import lombok.extern.slf4j.Slf4j;
import org.jeecgframework.poi.excel.ExcelImportUtil;
import org.jeecgframework.poi.excel.def.NormalExcelConstants;
import org.jeecgframework.poi.excel.entity.ExportParams;
import org.jeecgframework.poi.excel.entity.ImportParams;
import org.jeecgframework.poi.excel.view.JeecgEntityExcelView;
import org.jeecg.common.system.base.controller.JeecgController;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.multipart.MultipartFile;
import org.springframework.web.multipart.MultipartHttpServletRequest;
import org.springframework.web.servlet.ModelAndView;
import com.alibaba.fastjson.JSON;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.tags.Tag;
import org.jeecg.common.aspect.annotation.AutoLog;
import org.apache.shiro.authz.annotation.RequiresPermissions;
/**
* @Description: 数据库管理
* @Author: jeecg-boot
* @Date: 2024-11-04
* @Version: V1.0
*/
@Tag(name="数据库管理")
@RestController
@RequestMapping("/database/databaseRecord")
@Slf4j
public class DatabaseRecordController extends JeecgController<DatabaseRecord, IDatabaseRecordService> {
@Autowired
private IDatabaseRecordService databaseRecordService;
/**
* 分页列表查询
*
* @param databaseRecord
* @param pageNo
* @param pageSize
* @param req
* @return
*/
//@AutoLog(value = "数据库管理-分页列表查询")
@Operation(summary="数据库管理-分页列表查询")
@GetMapping(value = "/list")
public Result<IPage<DatabaseRecord>> queryPageList(DatabaseRecord databaseRecord,
@RequestParam(name="pageNo", defaultValue="1") Integer pageNo,
@RequestParam(name="pageSize", defaultValue="10") Integer pageSize,
HttpServletRequest req) {
QueryWrapper<DatabaseRecord> queryWrapper = QueryGenerator.initQueryWrapper(databaseRecord, req.getParameterMap());
Page<DatabaseRecord> page = new Page<DatabaseRecord>(pageNo, pageSize);
IPage<DatabaseRecord> pageList = databaseRecordService.page(page, queryWrapper);
return Result.OK(pageList);
}
/**
* 添加
*
* @param databaseRecord
* @return
*/
@AutoLog(value = "数据库管理-添加")
@Operation(summary="数据库管理-添加")
@RequiresPermissions("database:database_record:add")
@PostMapping(value = "/add")
public Result<String> add(@RequestBody DatabaseRecord databaseRecord) {
databaseRecordService.save(databaseRecord);
return Result.OK("添加成功！");
}
/**
* 编辑
*
* @param databaseRecord
* @return
*/
@AutoLog(value = "数据库管理-编辑")
@Operation(summary="数据库管理-编辑")
@RequiresPermissions("database:database_record:edit")
@RequestMapping(value = "/edit", method = {RequestMethod.PUT,RequestMethod.POST})
public Result<String> edit(@RequestBody DatabaseRecord databaseRecord) {
databaseRecordService.updateById(databaseRecord);
return Result.OK("编辑成功!");
}
/**
* 通过id删除
*
* @param id
* @return
*/
@AutoLog(value = "数据库管理-通过id删除")
@Operation(summary="数据库管理-通过id删除")
@RequiresPermissions("database:database_record:delete")
@DeleteMapping(value = "/delete")
public Result<String> delete(@RequestParam(name="id",required=true) String id) {
databaseRecordService.removeById(id);
return Result.OK("删除成功!");
}
/**
* 批量删除
*
* @param ids
* @return
*/
@AutoLog(value = "数据库管理-批量删除")
@Operation(summary="数据库管理-批量删除")
@RequiresPermissions("database:database_record:deleteBatch")
@DeleteMapping(value = "/deleteBatch")
public Result<String> deleteBatch(@RequestParam(name="ids",required=true) String ids) {
this.databaseRecordService.removeByIds(Arrays.asList(ids.split(",")));
return Result.OK("批量删除成功!");
}
/**
* 通过id查询
*
* @param id
* @return
*/
//@AutoLog(value = "数据库管理-通过id查询")
@Operation(summary="数据库管理-通过id查询")
@GetMapping(value = "/queryById")
public Result<DatabaseRecord> queryById(@RequestParam(name="id",required=true) String id) {
DatabaseRecord databaseRecord = databaseRecordService.getById(id);
if(databaseRecord==null) {
return Result.error("未找到对应数据");
}
return Result.OK(databaseRecord);
}
/**
* 导出excel
*
* @param request
* @param databaseRecord
*/
@RequiresPermissions("database:database_record:exportXls")
@RequestMapping(value = "/exportXls")
public ModelAndView exportXls(HttpServletRequest request, DatabaseRecord databaseRecord) {
return super.exportXls(request, databaseRecord, DatabaseRecord.class, "数据库管理");
}
/**
* 通过excel导入数据
*
* @param request
* @param response
* @return
*/
@RequiresPermissions("database:database_record:importExcel")
@RequestMapping(value = "/importExcel", method = RequestMethod.POST)
public Result<?> importExcel(HttpServletRequest request, HttpServletResponse response) {
return super.importExcel(request, response, DatabaseRecord.class);
}
}

2
physical-module-system/physical-system-biz/src/main/java/org/jeecg/modules/database/controller/ExperimentController.java
View File

@@ -58,7 +58,7 @@ public class ExperimentController extends JeecgController<Experiment, IExperimen
//@AutoLog(value = "试验管理-分页列表查询")
@Operation(summary = "试验管理-分页列表查询")
@GetMapping(value = "/list")
@PermissionData(pageComponent="experiment/manage")
// @PermissionData(pageComponent="experiment/manage/ExperimentList")
public Result<IPage<Experiment>> queryPageList(Experiment experiment,
@RequestParam(name = "pageNo", defaultValue = "1") Integer pageNo,
@RequestParam(name = "pageSize", defaultValue = "10") Integer pageSize,

179
physical-module-system/physical-system-biz/src/main/java/org/jeecg/modules/database/controller/ImportRecordController.java
View File

@@ -1,179 +0,0 @@
package org.jeecg.modules.database.controller;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.tags.Tag;
import jakarta.servlet.http.HttpServletRequest;
import jakarta.servlet.http.HttpServletResponse;
import lombok.extern.slf4j.Slf4j;
import org.apache.shiro.authz.annotation.RequiresPermissions;
import org.jeecg.common.api.vo.Result;
import org.jeecg.common.aspect.annotation.AutoLog;
import org.jeecg.common.system.base.controller.JeecgController;
import org.jeecg.common.system.query.QueryGenerator;
import org.jeecg.config.shiro.IgnoreAuth;
import org.jeecg.modules.database.entity.ImportRecord;
import org.jeecg.modules.database.service.IImportRecordService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.servlet.ModelAndView;
import java.util.Arrays;
/**
* @Description: 三方导入记录
* @Author: jeecg-boot
* @Date: 2024-09-03
* @Version: V1.0
*/
@Tag(name = "三方导入记录")
@RestController
@RequestMapping("/database/crawlerRecord")
@Slf4j
public class ImportRecordController extends JeecgController<ImportRecord, IImportRecordService> {
@Autowired
private IImportRecordService crawlerRecordService;
/**
* 分页列表查询
*
* @param importRecord
* @param pageNo
* @param pageSize
* @param req
* @return
*/
//@AutoLog(value = "三方导入记录-分页列表查询")
@Operation(summary = "三方导入记录-分页列表查询")
@GetMapping(value = "/list")
public Result<IPage<ImportRecord>> queryPageList(ImportRecord importRecord,
@RequestParam(name = "pageNo", defaultValue = "1") Integer pageNo,
@RequestParam(name = "pageSize", defaultValue = "10") Integer pageSize,
HttpServletRequest req) {
QueryWrapper<ImportRecord> queryWrapper = QueryGenerator.initQueryWrapper(importRecord, req.getParameterMap());
Page<ImportRecord> page = new Page<ImportRecord>(pageNo, pageSize);
IPage<ImportRecord> pageList = crawlerRecordService.page(page, queryWrapper);
return Result.OK(pageList);
}
/**
* 添加
*
* @param importRecord
* @return
*/
@AutoLog(value = "三方导入记录-添加")
@Operation(summary = "三方导入记录-添加")
@RequiresPermissions("database:crawler_record:add")
@PostMapping(value = "/add")
public Result<String> add(@RequestBody ImportRecord importRecord) {
crawlerRecordService.save(importRecord);
return Result.OK("添加成功！");
}
/**
* 编辑
*
* @param importRecord
* @return
*/
@AutoLog(value = "三方导入记录-编辑")
@Operation(summary = "三方导入记录-编辑")
@RequiresPermissions("database:crawler_record:edit")
@RequestMapping(value = "/edit", method = {RequestMethod.PUT, RequestMethod.POST})
public Result<String> edit(@RequestBody ImportRecord importRecord) {
crawlerRecordService.updateById(importRecord);
return Result.OK("编辑成功!");
}
/**
* 通过id删除
*
* @param id
* @return
*/
@AutoLog(value = "三方导入记录-通过id删除")
@Operation(summary = "三方导入记录-通过id删除")
@RequiresPermissions("database:crawler_record:delete")
@DeleteMapping(value = "/delete")
public Result<String> delete(@RequestParam(name = "id", required = true) String id) {
crawlerRecordService.removeById(id);
return Result.OK("删除成功!");
}
/**
* 批量删除
*
* @param ids
* @return
*/
@AutoLog(value = "三方导入记录-批量删除")
@Operation(summary = "三方导入记录-批量删除")
@RequiresPermissions("database:crawler_record:deleteBatch")
@DeleteMapping(value = "/deleteBatch")
public Result<String> deleteBatch(@RequestParam(name = "ids", required = true) String ids) {
this.crawlerRecordService.removeByIds(Arrays.asList(ids.split(",")));
return Result.OK("批量删除成功!");
}
/**
* 通过id查询
*
* @param id
* @return
*/
//@AutoLog(value = "三方导入记录-通过id查询")
@Operation(summary = "三方导入记录-通过id查询")
@GetMapping(value = "/queryById")
public Result<ImportRecord> queryById(@RequestParam(name = "id", required = true) String id) {
ImportRecord importRecord = crawlerRecordService.getById(id);
if (importRecord == null) {
return Result.error("未找到对应数据");
}
return Result.OK(importRecord);
}
/**
* 导出excel
*
* @param request
* @param importRecord
*/
@RequiresPermissions("database:crawler_record:exportXls")
@RequestMapping(value = "/exportXls")
public ModelAndView exportXls(HttpServletRequest request, ImportRecord importRecord) {
return super.exportXls(request, importRecord, ImportRecord.class, "三方导入记录");
}
/**
* 通过excel导入数据
*
* @param request
* @param response
* @return
*/
@RequiresPermissions("database:crawler_record:importExcel")
@RequestMapping(value = "/importExcel", method = RequestMethod.POST)
public Result<?> importExcel(HttpServletRequest request, HttpServletResponse response) {
return super.importExcel(request, response, ImportRecord.class);
}
@IgnoreAuth
@Operation(summary = "执行爬虫-esarad")
@RequestMapping(value = "/crawler/esarad", method = RequestMethod.GET)
public Result<?> esaradCrawler(HttpServletRequest request, HttpServletResponse response) {
crawlerRecordService.esaradCrawler();
return Result.OK();
}
@IgnoreAuth
@Operation(summary = "执行爬虫-radhome")
@RequestMapping(value = "/crawler/radhome", method = RequestMethod.GET)
public Result<?> radhomeCrawler(HttpServletRequest request, HttpServletResponse response) {
crawlerRecordService.radhomeCrawler();
return Result.OK();
}
}

18
physical-module-system/physical-system-biz/src/main/java/org/jeecg/modules/database/entity/ImportRecord.java → physical-module-system/physical-system-biz/src/main/java/org/jeecg/modules/database/entity/DatabaseRecord.java
View File

@@ -18,17 +18,17 @@ import lombok.EqualsAndHashCode;
import lombok.experimental.Accessors;
/**
* @Description: 三方导入记录
* @Description: 数据库管理
* @Author: jeecg-boot
* @Date: 2024-10-21
* @Date: 2024-11-04
* @Version: V1.0
*/
@Data
@TableName("import_record")
@TableName("database_record")
@Accessors(chain = true)
@EqualsAndHashCode(callSuper = false)
@Schema(description="三方导入记录")
public class ImportRecord implements Serializable {
@Schema(description="数据库管理")
public class DatabaseRecord implements Serializable {
private static final long serialVersionUID = 1L;
/**主键*/
@@ -79,7 +79,7 @@ public class ImportRecord implements Serializable {
@Schema(description = "生产厂家")
private String manufacturer;
/**试验时间*/
@Excel(name = "试验时间", width = 20)
@Excel(name = "试验时间", width = 15)
@Schema(description = "试验时间")
private String experimentDate;
/**数据来源*/
@@ -94,8 +94,8 @@ public class ImportRecord implements Serializable {
@Excel(name = "条目数统计", width = 15)
@Schema(description = "条目数统计")
private String totalCount;
/**附件ID*/
@Excel(name = "附件ID", width = 15)
@Schema(description = "附件ID")
/**附件IDs*/
@Excel(name = "附件IDs", width = 15)
@Schema(description = "附件IDs")
private String fileList;
}

17
physical-module-system/physical-system-biz/src/main/java/org/jeecg/modules/database/mapper/DatabaseRecordMapper.java Normal file
View File

@@ -0,0 +1,17 @@
package org.jeecg.modules.database.mapper;
import java.util.List;
import org.apache.ibatis.annotations.Param;
import org.jeecg.modules.demo.database.entity.DatabaseRecord;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
/**
* @Description: 数据库管理
* @Author: jeecg-boot
* @Date: 2024-11-04
* @Version: V1.0
*/
public interface DatabaseRecordMapper extends BaseMapper<DatabaseRecord> {
}

14
physical-module-system/physical-system-biz/src/main/java/org/jeecg/modules/database/mapper/ImportRecordMapper.java
View File

@@ -1,14 +0,0 @@
package org.jeecg.modules.database.mapper;
import org.jeecg.modules.database.entity.ImportRecord;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
/**
* @Description: 三方导入记录
* @Author: jeecg-boot
* @Date: 2024-09-03
* @Version: V1.0
*/
public interface ImportRecordMapper extends BaseMapper<ImportRecord> {
}

5
physical-module-system/physical-system-biz/src/main/java/org/jeecg/modules/database/mapper/xml/DatabaseRecordMapper.xml Normal file
View File

@@ -0,0 +1,5 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="org.jeecg.modules.demo.database.mapper.DatabaseRecordMapper">
</mapper>

14
physical-module-system/physical-system-biz/src/main/java/org/jeecg/modules/database/service/IDatabaseRecordService.java Normal file
View File

@@ -0,0 +1,14 @@
package org.jeecg.modules.database.service;
import org.jeecg.modules.demo.database.entity.DatabaseRecord;
import com.baomidou.mybatisplus.extension.service.IService;
/**
* @Description: 数据库管理
* @Author: jeecg-boot
* @Date: 2024-11-04
* @Version: V1.0
*/
public interface IDatabaseRecordService extends IService<DatabaseRecord> {
}

16
physical-module-system/physical-system-biz/src/main/java/org/jeecg/modules/database/service/IImportRecordService.java
View File

@@ -1,16 +0,0 @@
package org.jeecg.modules.database.service;
import org.jeecg.modules.database.entity.ImportRecord;
import com.baomidou.mybatisplus.extension.service.IService;
/**
* @Description: 爬虫记录
* @Author: jeecg-boot
* @Date: 2024-09-03
* @Version: V1.0
*/
public interface IImportRecordService extends IService<ImportRecord> {
void esaradCrawler();
void radhomeCrawler();
}

19
physical-module-system/physical-system-biz/src/main/java/org/jeecg/modules/database/service/impl/DatabaseRecordServiceImpl.java Normal file
View File

@@ -0,0 +1,19 @@
package org.jeecg.modules.database.service.impl;
import org.jeecg.modules.demo.database.entity.DatabaseRecord;
import org.jeecg.modules.demo.database.mapper.DatabaseRecordMapper;
import org.jeecg.modules.demo.database.service.IDatabaseRecordService;
import org.springframework.stereotype.Service;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
/**
* @Description: 数据库管理
* @Author: jeecg-boot
* @Date: 2024-11-04
* @Version: V1.0
*/
@Service
public class DatabaseRecordServiceImpl extends ServiceImpl<DatabaseRecordMapper, DatabaseRecord> implements IDatabaseRecordService {
}

336
physical-module-system/physical-system-biz/src/main/java/org/jeecg/modules/database/service/impl/ImportRecordServiceImpl.java
View File

@@ -1,336 +0,0 @@
package org.jeecg.modules.database.service.impl;
import cn.hutool.core.collection.CollUtil;
import cn.hutool.core.io.FileUtil;
import cn.hutool.core.io.IoUtil;
import cn.hutool.core.thread.ThreadUtil;
import cn.hutool.core.util.ReUtil;
import cn.hutool.core.util.URLUtil;
import cn.hutool.http.Header;
import cn.hutool.http.HttpRequest;
import cn.hutool.http.HttpResponse;
import cn.hutool.http.HttpUtil;
import com.alibaba.fastjson2.JSON;
import com.alibaba.fastjson2.JSONArray;
import com.alibaba.fastjson2.JSONObject;
import com.baomidou.mybatisplus.core.toolkit.Wrappers;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import lombok.extern.slf4j.Slf4j;
import okhttp3.*;
import org.apache.commons.lang3.StringUtils;
import org.jeecg.common.util.MinioUtil;
import org.jeecg.common.util.RedisUtil;
import org.jeecg.modules.database.entity.ImportRecord;
import org.jeecg.modules.database.mapper.ImportRecordMapper;
import org.jeecg.modules.database.service.IImportRecordService;
import org.jeecg.modules.oss.service.impl.OssFileServiceImpl;
import org.jetbrains.annotations.NotNull;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
@Slf4j
@Service
public class ImportRecordServiceImpl extends ServiceImpl<ImportRecordMapper, ImportRecord> implements IImportRecordService {
@Autowired
private RedisUtil redisUtil;
@Autowired
private OssFileServiceImpl ossFileService;
private String esaradKey = "crawler-esarad";
private String radhomeKey = "crawler-radhome";
public static void main(String[] args) {
HttpResponse execute = HttpRequest.get("https://esarad.esa.int/?id=76&handler=DownloadDb").execute();
final List<String> dispositions = execute.headerList(Header.CONTENT_DISPOSITION.getValue());
String fileName = null;
if (CollUtil.isNotEmpty(dispositions)) {
for (String disposition : dispositions) {
fileName = ReUtil.getGroup1("filename" + "=([^;]+)", disposition);
}
System.out.println(URLUtil.decode("N2920A%20TID_1009_01.pdf", Charset.defaultCharset()));
System.out.println(fileName);
// filename* 采用了 RFC 5987 中规定的编码方式，优先读取
}
}
/**
* https://esarad.esa.int/
*/
// @Override
// public void esaradCrawler() {
// try {
// String crawlerRunning = String.valueOf(redisUtil.get(esaradKey));
// if (StringUtils.equals(crawlerRunning, "T")) {
// throw new RuntimeException("爬虫任务执行中");
// }
// redisUtil.set(esaradKey, "T", 24 * 60 * 60);
//
// List<ImportRecord> tableData = new ArrayList<>();
//
// Document doc = Jsoup.connect("https://esarad.esa.int").get();
// Element table = doc.getElementById("dtReports");
// if (Objects.isNull(table)) {
// redisUtil.del(esaradKey);
// throw new RuntimeException("爬虫获取数据失败");
// }
// // Get the tbody element within the table
// Element tbody = table.select("tbody").first(); // Select the first tbody element
//
// if (tbody == null) {
// redisUtil.del(esaradKey);
// throw new RuntimeException("爬虫获取数据失败");
// }
// // Create a list to store the row maps
//
// // Extract headers from the first row
// Elements headers = table.select("thead").first().select("tr").first().select("th");
// List<String> headerNames = new ArrayList<>();
// for (Element header : headers) {
// headerNames.add(header.text());
// }
//
// // Select all rows in the tbody
// Elements rows = tbody.select("tr");
//
// for (int j = 0; j < rows.size(); j++) {
// if (j > 10) {
// break;
// }
// Element row = rows.get(j);
//
// // Select all cells in the row
// ImportRecord importRecord = new ImportRecord();
//
// Elements cells = row.select("td");
//
// if (cells.size() == headerNames.size()) { // Ensure the number of cells matches the number of headers
//
// for (int i = 0; i < cells.size(); i++) {
//
// String header = headerNames.get(i);
// String value = cells.get(i).text();
// switch (header) {
// case "Radiation Test Method":
// importRecord.setTestMethod(value);
// break;
// case "EPPL Familiy":
// importRecord.setCategory(value);
// break;
// case "EPPL Group":
// importRecord.setSubCategory(value);
// break;
// case "DUT Manufacturer":
// importRecord.setManufacturer(value);
// break;
// case "Function":
// importRecord.setFunctionType(value);
// break;
// case "Report Date":
// importRecord.setReportDate(value);
// break;
// case "Report Source":
// importRecord.setReportSource(value);
// break;
// case "Technology":
// importRecord.setTechnology(value);
// break;
// case "Id":
// importRecord.setReportId(value);
// break;
// case "DUT part type":
// importRecord.setCode(value);
// break;
// case "Radiation Test Type":
// importRecord.setRadiationTestType(value);
// break;
// }
//
// }
//
// // Add the map to the list
// tableData.add(importRecord);
// }
// }
//
// // Print the list of maps
// for (ImportRecord rowMap : tableData) {
// rowMap.setFileUrl("https://esarad.esa.int/?id=" + rowMap.getReportId() + "&handler=DownloadDb");
// }
// saveEsaradFiles(tableData, esaradKey);
//
// } catch (Exception e) {
// redisUtil.del(esaradKey);
// throw new RuntimeException(e);
// }
// }
/**
* https://radhome.gsfc.nasa.gov/radhome/RadDataBase/RadDataBase.html
*/
// @Override
// public void radhomeCrawler() {
// try {
// String crawlerRunning = String.valueOf(redisUtil.get(radhomeKey));
// if (StringUtils.equals(crawlerRunning, "T")) {
// throw new RuntimeException("爬虫任务执行中");
// }
// redisUtil.set(radhomeKey, "T", 24 * 60 * 60);
//
// OkHttpClient client = new OkHttpClient();
// String url = "https://radhome.gsfc.nasa.gov/radhome/dev/parts.cfc?method=getParts";
// FormBody formBody = new FormBody.Builder().add("_search", "false").add("nd", System.currentTimeMillis() + "").add("rows", "10").add("page", "1").add("sidx", "partnumber").add("sord", "asc").build();
//
// Request request = new Request.Builder().url(url).post(formBody).build();
//
// client.newCall(request).enqueue(new Callback() {
// @Override
// public void onResponse(@NotNull Call call, @NotNull Response response) throws IOException {
// String jsonResponse = response.body().string();
// JSONObject jsonObject = JSON.parseObject(jsonResponse);
// Long total = jsonObject.getLong("RECORDS");
// JSONArray list = jsonObject.getJSONArray("ROWS");
// System.out.println("total count " + total);
// System.out.println("total list " + list.get(0));
// List<ImportRecord> tableData = new ArrayList<>();
//
// for (int i = 0; i < list.size(); i++) {
//
// if (i > 10) {
// break;
// }
// JSONArray row = (JSONArray) list.get(i);
// String fileNames = String.valueOf(row.get(4));
// ImportRecord map = new ImportRecord();
// String fileUrls = fixFileNames(fileNames);
// map.setFileUrl(fileUrls);
//
// map.setCode(String.valueOf(row.get(0)));
// map.setFunctionType(String.valueOf(row.get(1)));
// map.setManufacturer(String.valueOf(row.get(2)));
// map.setReportDate(String.valueOf(row.get(3)));
// map.setTestMethod(String.valueOf(row.get(5)));
// map.setCategory(String.valueOf(row.get(6)));
// map.setReportId(map.getCode().replaceAll(" ", ""));
// tableData.add(map);
// }
// saveRadhomeFiles(tableData, radhomeKey);
// }
//
// @Override
// public void onFailure(@NotNull Call call, @NotNull IOException e) {
// e.printStackTrace();
// redisUtil.del(radhomeKey);
// }
// });
// } catch (Exception e) {
// redisUtil.del(radhomeKey);
// throw new RuntimeException(e);
// }
// }
// private String fixFileNames(String fileNames) {
// String[] split = StringUtils.split(fileNames, ";");
// List<String> result = new ArrayList<>();
// for (String s : split) {
// if (!StringUtils.startsWith(s, "http")) {
// result.add("https://radhome.gsfc.nasa.gov/radhome/papers/" + s);
// } else {
// result.add(s);
// }
// }
// return StringUtils.join(result, ";");
// }
//
// private void saveRadhomeFiles(List<ImportRecord> fileList, String type) {
// ThreadUtil.execute(() -> {
// try {
// for (ImportRecord record : fileList) {
// String fileUploadResult = "";
// String fileUrl = record.getFileUrl();
// if (fileUrl.contains(";")) {
// String[] split = fileUrl.split(";");
// List<String> result = new ArrayList<>();
// for (String s : split) {
// byte[] fileBytes = HttpUtil.downloadBytes(s);
// InputStream inputStream = new ByteArrayInputStream(fileBytes);
// result.add(MinioUtil.upload(inputStream, "radhome/" + s.substring(s.lastIndexOf("/") + 1)));
// fileUploadResult = StringUtils.join(result, ";");
// }
// } else {
// byte[] fileBytes = HttpUtil.downloadBytes(fileUrl);
// InputStream inputStream = new ByteArrayInputStream(fileBytes);
// fileUploadResult = MinioUtil.upload(inputStream, "radhome/" + fileUrl.substring(fileUrl.lastIndexOf("/") + 1));
// }
// System.out.println(fileUploadResult);
// if (StringUtils.isNotBlank(fileUploadResult)) {
// record.setFileUrl(fileUploadResult);
// save(record);
// }
// }
// redisUtil.del(type);
// } catch (Exception e) {
// e.printStackTrace();
// }
// });
// }
//
// private void saveEsaradFiles(List<ImportRecord> fileList, String type) {
// ThreadUtil.execute(() -> {
// try {
// for (ImportRecord record : fileList) {
// ImportRecord dbData = getOne(Wrappers.<ImportRecord>lambdaQuery().eq(ImportRecord::getReportSource, record.getReportId()));
// if (Objects.nonNull(dbData)) {
// continue;
// }
// String resultStr = "";
//
// String dest = FileUtil.getTmpDirPath() + "esarad-" + record.getReportId() + "/";
// FileUtil.mkdir(dest);
// long fileSize = HttpUtil.downloadFile(record.getFileUrl(), dest);
//
// if (fileSize > 0) {
// List<File> files = FileUtil.loopFiles(dest);
// for (File file : files) {
// resultStr = MinioUtil.upload(IoUtil.toStream(file), "esarad/" + record.getReportId() + "-" + URLUtil.decode(file.getName(), Charset.defaultCharset()) );
// }
// }
//
// System.out.println(resultStr);
// if (StringUtils.isNotBlank(resultStr)) {
// record.setFileUrl(resultStr);
// save(record);
// }
// }
// redisUtil.del(type);
// } catch (Exception e) {
// e.printStackTrace();
// }
// });
// }
@Override
public void esaradCrawler() {
}
@Override
public void radhomeCrawler() {
}
}