update
This commit is contained in:
@@ -227,7 +227,7 @@ jeecg:
|
|||||||
#webapp文件路径
|
#webapp文件路径
|
||||||
webapp: /opt/webapp
|
webapp: /opt/webapp
|
||||||
shiro:
|
shiro:
|
||||||
excludeUrls: /sys/user/**,/test/jeecgDemo/demo3,/test/jeecgDemo/redisDemo/**,/bigscreen/category/**,/bigscreen/visual/**,/bigscreen/map/**,/jmreport/bigscreen2/**
|
excludeUrls: /sys/common/upload,/sys/user/**,/test/jeecgDemo/demo3,/test/jeecgDemo/redisDemo/**,/bigscreen/category/**,/bigscreen/visual/**,/bigscreen/map/**,/jmreport/bigscreen2/**
|
||||||
#阿里云oss存储和大鱼短信秘钥配置
|
#阿里云oss存储和大鱼短信秘钥配置
|
||||||
oss:
|
oss:
|
||||||
accessKey: ??
|
accessKey: ??
|
||||||
|
|||||||
@@ -229,7 +229,7 @@ jeecg:
|
|||||||
#webapp文件路径
|
#webapp文件路径
|
||||||
webapp: /opt/jeecg-boot/webapp
|
webapp: /opt/jeecg-boot/webapp
|
||||||
shiro:
|
shiro:
|
||||||
excludeUrls: /test/jeecgDemo/demo3,/test/jeecgDemo/redisDemo/**,/bigscreen/category/**,/bigscreen/visual/**,/bigscreen/map/**,/jmreport/bigscreen2/**,/api/getUserInfo,/api/sys/common/**,/sys/common/**
|
excludeUrls: /sys/common/upload,/test/jeecgDemo/demo3,/test/jeecgDemo/redisDemo/**,/bigscreen/category/**,/bigscreen/visual/**,/bigscreen/map/**,/jmreport/bigscreen2/**,/api/getUserInfo,/api/sys/common/**,/sys/common/**
|
||||||
#阿里云oss存储和大鱼短信秘钥配置
|
#阿里云oss存储和大鱼短信秘钥配置
|
||||||
oss:
|
oss:
|
||||||
accessKey: ??
|
accessKey: ??
|
||||||
|
|||||||
@@ -229,7 +229,7 @@ jeecg:
|
|||||||
#webapp文件路径
|
#webapp文件路径
|
||||||
webapp: /opt/jeecg-boot/webapp
|
webapp: /opt/jeecg-boot/webapp
|
||||||
shiro:
|
shiro:
|
||||||
excludeUrls: /test/jeecgDemo/demo3,/test/jeecgDemo/redisDemo/**,/bigscreen/category/**,/bigscreen/visual/**,/bigscreen/map/**,/jmreport/bigscreen2/**,/api/getUserInfo,/api/sys/common/**,/sys/common/**
|
excludeUrls: /sys/common/upload,/test/jeecgDemo/demo3,/test/jeecgDemo/redisDemo/**,/bigscreen/category/**,/bigscreen/visual/**,/bigscreen/map/**,/jmreport/bigscreen2/**,/api/getUserInfo,/api/sys/common/**,/sys/common/**
|
||||||
#阿里云oss存储和大鱼短信秘钥配置
|
#阿里云oss存储和大鱼短信秘钥配置
|
||||||
oss:
|
oss:
|
||||||
accessKey: ??
|
accessKey: ??
|
||||||
|
|||||||
@@ -72,255 +72,265 @@ public class ImportRecordServiceImpl extends ServiceImpl<ImportRecordMapper, Imp
|
|||||||
/**
|
/**
|
||||||
* https://esarad.esa.int/
|
* https://esarad.esa.int/
|
||||||
*/
|
*/
|
||||||
@Override
|
// @Override
|
||||||
public void esaradCrawler() {
|
// public void esaradCrawler() {
|
||||||
try {
|
// try {
|
||||||
String crawlerRunning = String.valueOf(redisUtil.get(esaradKey));
|
// String crawlerRunning = String.valueOf(redisUtil.get(esaradKey));
|
||||||
if (StringUtils.equals(crawlerRunning, "T")) {
|
// if (StringUtils.equals(crawlerRunning, "T")) {
|
||||||
throw new RuntimeException("爬虫任务执行中");
|
// throw new RuntimeException("爬虫任务执行中");
|
||||||
}
|
// }
|
||||||
redisUtil.set(esaradKey, "T", 24 * 60 * 60);
|
// redisUtil.set(esaradKey, "T", 24 * 60 * 60);
|
||||||
|
//
|
||||||
List<ImportRecord> tableData = new ArrayList<>();
|
// List<ImportRecord> tableData = new ArrayList<>();
|
||||||
|
//
|
||||||
Document doc = Jsoup.connect("https://esarad.esa.int").get();
|
// Document doc = Jsoup.connect("https://esarad.esa.int").get();
|
||||||
Element table = doc.getElementById("dtReports");
|
// Element table = doc.getElementById("dtReports");
|
||||||
if (Objects.isNull(table)) {
|
// if (Objects.isNull(table)) {
|
||||||
redisUtil.del(esaradKey);
|
// redisUtil.del(esaradKey);
|
||||||
throw new RuntimeException("爬虫获取数据失败");
|
// throw new RuntimeException("爬虫获取数据失败");
|
||||||
}
|
// }
|
||||||
// Get the tbody element within the table
|
// // Get the tbody element within the table
|
||||||
Element tbody = table.select("tbody").first(); // Select the first tbody element
|
// Element tbody = table.select("tbody").first(); // Select the first tbody element
|
||||||
|
//
|
||||||
if (tbody == null) {
|
// if (tbody == null) {
|
||||||
redisUtil.del(esaradKey);
|
// redisUtil.del(esaradKey);
|
||||||
throw new RuntimeException("爬虫获取数据失败");
|
// throw new RuntimeException("爬虫获取数据失败");
|
||||||
}
|
// }
|
||||||
// Create a list to store the row maps
|
// // Create a list to store the row maps
|
||||||
|
//
|
||||||
// Extract headers from the first row
|
// // Extract headers from the first row
|
||||||
Elements headers = table.select("thead").first().select("tr").first().select("th");
|
// Elements headers = table.select("thead").first().select("tr").first().select("th");
|
||||||
List<String> headerNames = new ArrayList<>();
|
// List<String> headerNames = new ArrayList<>();
|
||||||
for (Element header : headers) {
|
// for (Element header : headers) {
|
||||||
headerNames.add(header.text());
|
// headerNames.add(header.text());
|
||||||
}
|
// }
|
||||||
|
//
|
||||||
// Select all rows in the tbody
|
// // Select all rows in the tbody
|
||||||
Elements rows = tbody.select("tr");
|
// Elements rows = tbody.select("tr");
|
||||||
|
//
|
||||||
for (int j = 0; j < rows.size(); j++) {
|
// for (int j = 0; j < rows.size(); j++) {
|
||||||
if (j > 10) {
|
// if (j > 10) {
|
||||||
break;
|
// break;
|
||||||
}
|
// }
|
||||||
Element row = rows.get(j);
|
// Element row = rows.get(j);
|
||||||
|
//
|
||||||
// Select all cells in the row
|
// // Select all cells in the row
|
||||||
ImportRecord importRecord = new ImportRecord();
|
// ImportRecord importRecord = new ImportRecord();
|
||||||
|
//
|
||||||
Elements cells = row.select("td");
|
// Elements cells = row.select("td");
|
||||||
|
//
|
||||||
if (cells.size() == headerNames.size()) { // Ensure the number of cells matches the number of headers
|
// if (cells.size() == headerNames.size()) { // Ensure the number of cells matches the number of headers
|
||||||
|
//
|
||||||
for (int i = 0; i < cells.size(); i++) {
|
// for (int i = 0; i < cells.size(); i++) {
|
||||||
|
//
|
||||||
String header = headerNames.get(i);
|
// String header = headerNames.get(i);
|
||||||
String value = cells.get(i).text();
|
// String value = cells.get(i).text();
|
||||||
switch (header) {
|
// switch (header) {
|
||||||
case "Radiation Test Method":
|
// case "Radiation Test Method":
|
||||||
importRecord.setTestMethod(value);
|
// importRecord.setTestMethod(value);
|
||||||
break;
|
// break;
|
||||||
case "EPPL Familiy":
|
// case "EPPL Familiy":
|
||||||
importRecord.setCategory(value);
|
// importRecord.setCategory(value);
|
||||||
break;
|
// break;
|
||||||
case "EPPL Group":
|
// case "EPPL Group":
|
||||||
importRecord.setSubCategory(value);
|
// importRecord.setSubCategory(value);
|
||||||
break;
|
// break;
|
||||||
case "DUT Manufacturer":
|
// case "DUT Manufacturer":
|
||||||
importRecord.setManufacturer(value);
|
// importRecord.setManufacturer(value);
|
||||||
break;
|
// break;
|
||||||
case "Function":
|
// case "Function":
|
||||||
importRecord.setFunctionType(value);
|
// importRecord.setFunctionType(value);
|
||||||
break;
|
// break;
|
||||||
case "Report Date":
|
// case "Report Date":
|
||||||
importRecord.setReportDate(value);
|
// importRecord.setReportDate(value);
|
||||||
break;
|
// break;
|
||||||
case "Report Source":
|
// case "Report Source":
|
||||||
importRecord.setReportSource(value);
|
// importRecord.setReportSource(value);
|
||||||
break;
|
// break;
|
||||||
case "Technology":
|
// case "Technology":
|
||||||
importRecord.setTechnology(value);
|
// importRecord.setTechnology(value);
|
||||||
break;
|
// break;
|
||||||
case "Id":
|
// case "Id":
|
||||||
importRecord.setReportId(value);
|
// importRecord.setReportId(value);
|
||||||
break;
|
// break;
|
||||||
case "DUT part type":
|
// case "DUT part type":
|
||||||
importRecord.setCode(value);
|
// importRecord.setCode(value);
|
||||||
break;
|
// break;
|
||||||
case "Radiation Test Type":
|
// case "Radiation Test Type":
|
||||||
importRecord.setRadiationTestType(value);
|
// importRecord.setRadiationTestType(value);
|
||||||
break;
|
// break;
|
||||||
}
|
// }
|
||||||
|
//
|
||||||
}
|
// }
|
||||||
|
//
|
||||||
// Add the map to the list
|
// // Add the map to the list
|
||||||
tableData.add(importRecord);
|
// tableData.add(importRecord);
|
||||||
}
|
// }
|
||||||
}
|
// }
|
||||||
|
//
|
||||||
// Print the list of maps
|
// // Print the list of maps
|
||||||
for (ImportRecord rowMap : tableData) {
|
// for (ImportRecord rowMap : tableData) {
|
||||||
rowMap.setFileUrl("https://esarad.esa.int/?id=" + rowMap.getReportId() + "&handler=DownloadDb");
|
// rowMap.setFileUrl("https://esarad.esa.int/?id=" + rowMap.getReportId() + "&handler=DownloadDb");
|
||||||
}
|
// }
|
||||||
saveEsaradFiles(tableData, esaradKey);
|
// saveEsaradFiles(tableData, esaradKey);
|
||||||
|
//
|
||||||
} catch (Exception e) {
|
// } catch (Exception e) {
|
||||||
redisUtil.del(esaradKey);
|
// redisUtil.del(esaradKey);
|
||||||
throw new RuntimeException(e);
|
// throw new RuntimeException(e);
|
||||||
}
|
// }
|
||||||
}
|
// }
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* https://radhome.gsfc.nasa.gov/radhome/RadDataBase/RadDataBase.html
|
* https://radhome.gsfc.nasa.gov/radhome/RadDataBase/RadDataBase.html
|
||||||
*/
|
*/
|
||||||
|
// @Override
|
||||||
|
// public void radhomeCrawler() {
|
||||||
|
// try {
|
||||||
|
// String crawlerRunning = String.valueOf(redisUtil.get(radhomeKey));
|
||||||
|
// if (StringUtils.equals(crawlerRunning, "T")) {
|
||||||
|
// throw new RuntimeException("爬虫任务执行中");
|
||||||
|
// }
|
||||||
|
// redisUtil.set(radhomeKey, "T", 24 * 60 * 60);
|
||||||
|
//
|
||||||
|
// OkHttpClient client = new OkHttpClient();
|
||||||
|
// String url = "https://radhome.gsfc.nasa.gov/radhome/dev/parts.cfc?method=getParts";
|
||||||
|
// FormBody formBody = new FormBody.Builder().add("_search", "false").add("nd", System.currentTimeMillis() + "").add("rows", "10").add("page", "1").add("sidx", "partnumber").add("sord", "asc").build();
|
||||||
|
//
|
||||||
|
// Request request = new Request.Builder().url(url).post(formBody).build();
|
||||||
|
//
|
||||||
|
// client.newCall(request).enqueue(new Callback() {
|
||||||
|
// @Override
|
||||||
|
// public void onResponse(@NotNull Call call, @NotNull Response response) throws IOException {
|
||||||
|
// String jsonResponse = response.body().string();
|
||||||
|
// JSONObject jsonObject = JSON.parseObject(jsonResponse);
|
||||||
|
// Long total = jsonObject.getLong("RECORDS");
|
||||||
|
// JSONArray list = jsonObject.getJSONArray("ROWS");
|
||||||
|
// System.out.println("total count " + total);
|
||||||
|
// System.out.println("total list " + list.get(0));
|
||||||
|
// List<ImportRecord> tableData = new ArrayList<>();
|
||||||
|
//
|
||||||
|
// for (int i = 0; i < list.size(); i++) {
|
||||||
|
//
|
||||||
|
// if (i > 10) {
|
||||||
|
// break;
|
||||||
|
// }
|
||||||
|
// JSONArray row = (JSONArray) list.get(i);
|
||||||
|
// String fileNames = String.valueOf(row.get(4));
|
||||||
|
// ImportRecord map = new ImportRecord();
|
||||||
|
// String fileUrls = fixFileNames(fileNames);
|
||||||
|
// map.setFileUrl(fileUrls);
|
||||||
|
//
|
||||||
|
// map.setCode(String.valueOf(row.get(0)));
|
||||||
|
// map.setFunctionType(String.valueOf(row.get(1)));
|
||||||
|
// map.setManufacturer(String.valueOf(row.get(2)));
|
||||||
|
// map.setReportDate(String.valueOf(row.get(3)));
|
||||||
|
// map.setTestMethod(String.valueOf(row.get(5)));
|
||||||
|
// map.setCategory(String.valueOf(row.get(6)));
|
||||||
|
// map.setReportId(map.getCode().replaceAll(" ", ""));
|
||||||
|
// tableData.add(map);
|
||||||
|
// }
|
||||||
|
// saveRadhomeFiles(tableData, radhomeKey);
|
||||||
|
// }
|
||||||
|
//
|
||||||
|
// @Override
|
||||||
|
// public void onFailure(@NotNull Call call, @NotNull IOException e) {
|
||||||
|
// e.printStackTrace();
|
||||||
|
// redisUtil.del(radhomeKey);
|
||||||
|
// }
|
||||||
|
// });
|
||||||
|
// } catch (Exception e) {
|
||||||
|
// redisUtil.del(radhomeKey);
|
||||||
|
// throw new RuntimeException(e);
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
|
||||||
|
// private String fixFileNames(String fileNames) {
|
||||||
|
// String[] split = StringUtils.split(fileNames, ";");
|
||||||
|
// List<String> result = new ArrayList<>();
|
||||||
|
// for (String s : split) {
|
||||||
|
// if (!StringUtils.startsWith(s, "http")) {
|
||||||
|
// result.add("https://radhome.gsfc.nasa.gov/radhome/papers/" + s);
|
||||||
|
// } else {
|
||||||
|
// result.add(s);
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
// return StringUtils.join(result, ";");
|
||||||
|
// }
|
||||||
|
//
|
||||||
|
// private void saveRadhomeFiles(List<ImportRecord> fileList, String type) {
|
||||||
|
// ThreadUtil.execute(() -> {
|
||||||
|
// try {
|
||||||
|
// for (ImportRecord record : fileList) {
|
||||||
|
// String fileUploadResult = "";
|
||||||
|
// String fileUrl = record.getFileUrl();
|
||||||
|
// if (fileUrl.contains(";")) {
|
||||||
|
// String[] split = fileUrl.split(";");
|
||||||
|
// List<String> result = new ArrayList<>();
|
||||||
|
// for (String s : split) {
|
||||||
|
// byte[] fileBytes = HttpUtil.downloadBytes(s);
|
||||||
|
// InputStream inputStream = new ByteArrayInputStream(fileBytes);
|
||||||
|
// result.add(MinioUtil.upload(inputStream, "radhome/" + s.substring(s.lastIndexOf("/") + 1)));
|
||||||
|
// fileUploadResult = StringUtils.join(result, ";");
|
||||||
|
// }
|
||||||
|
// } else {
|
||||||
|
// byte[] fileBytes = HttpUtil.downloadBytes(fileUrl);
|
||||||
|
// InputStream inputStream = new ByteArrayInputStream(fileBytes);
|
||||||
|
// fileUploadResult = MinioUtil.upload(inputStream, "radhome/" + fileUrl.substring(fileUrl.lastIndexOf("/") + 1));
|
||||||
|
// }
|
||||||
|
// System.out.println(fileUploadResult);
|
||||||
|
// if (StringUtils.isNotBlank(fileUploadResult)) {
|
||||||
|
// record.setFileUrl(fileUploadResult);
|
||||||
|
// save(record);
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
// redisUtil.del(type);
|
||||||
|
// } catch (Exception e) {
|
||||||
|
// e.printStackTrace();
|
||||||
|
// }
|
||||||
|
// });
|
||||||
|
// }
|
||||||
|
//
|
||||||
|
// private void saveEsaradFiles(List<ImportRecord> fileList, String type) {
|
||||||
|
// ThreadUtil.execute(() -> {
|
||||||
|
// try {
|
||||||
|
// for (ImportRecord record : fileList) {
|
||||||
|
// ImportRecord dbData = getOne(Wrappers.<ImportRecord>lambdaQuery().eq(ImportRecord::getReportSource, record.getReportId()));
|
||||||
|
// if (Objects.nonNull(dbData)) {
|
||||||
|
// continue;
|
||||||
|
// }
|
||||||
|
// String resultStr = "";
|
||||||
|
//
|
||||||
|
// String dest = FileUtil.getTmpDirPath() + "esarad-" + record.getReportId() + "/";
|
||||||
|
// FileUtil.mkdir(dest);
|
||||||
|
// long fileSize = HttpUtil.downloadFile(record.getFileUrl(), dest);
|
||||||
|
//
|
||||||
|
// if (fileSize > 0) {
|
||||||
|
// List<File> files = FileUtil.loopFiles(dest);
|
||||||
|
// for (File file : files) {
|
||||||
|
// resultStr = MinioUtil.upload(IoUtil.toStream(file), "esarad/" + record.getReportId() + "-" + URLUtil.decode(file.getName(), Charset.defaultCharset()) );
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
//
|
||||||
|
// System.out.println(resultStr);
|
||||||
|
// if (StringUtils.isNotBlank(resultStr)) {
|
||||||
|
// record.setFileUrl(resultStr);
|
||||||
|
// save(record);
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
// redisUtil.del(type);
|
||||||
|
// } catch (Exception e) {
|
||||||
|
// e.printStackTrace();
|
||||||
|
// }
|
||||||
|
// });
|
||||||
|
// }
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void radhomeCrawler() {
|
public void esaradCrawler() {
|
||||||
try {
|
|
||||||
String crawlerRunning = String.valueOf(redisUtil.get(radhomeKey));
|
|
||||||
if (StringUtils.equals(crawlerRunning, "T")) {
|
|
||||||
throw new RuntimeException("爬虫任务执行中");
|
|
||||||
}
|
|
||||||
redisUtil.set(radhomeKey, "T", 24 * 60 * 60);
|
|
||||||
|
|
||||||
OkHttpClient client = new OkHttpClient();
|
|
||||||
String url = "https://radhome.gsfc.nasa.gov/radhome/dev/parts.cfc?method=getParts";
|
|
||||||
FormBody formBody = new FormBody.Builder().add("_search", "false").add("nd", System.currentTimeMillis() + "").add("rows", "10").add("page", "1").add("sidx", "partnumber").add("sord", "asc").build();
|
|
||||||
|
|
||||||
Request request = new Request.Builder().url(url).post(formBody).build();
|
|
||||||
|
|
||||||
client.newCall(request).enqueue(new Callback() {
|
|
||||||
@Override
|
|
||||||
public void onResponse(@NotNull Call call, @NotNull Response response) throws IOException {
|
|
||||||
String jsonResponse = response.body().string();
|
|
||||||
JSONObject jsonObject = JSON.parseObject(jsonResponse);
|
|
||||||
Long total = jsonObject.getLong("RECORDS");
|
|
||||||
JSONArray list = jsonObject.getJSONArray("ROWS");
|
|
||||||
System.out.println("total count " + total);
|
|
||||||
System.out.println("total list " + list.get(0));
|
|
||||||
List<ImportRecord> tableData = new ArrayList<>();
|
|
||||||
|
|
||||||
for (int i = 0; i < list.size(); i++) {
|
|
||||||
|
|
||||||
if (i > 10) {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
JSONArray row = (JSONArray) list.get(i);
|
|
||||||
String fileNames = String.valueOf(row.get(4));
|
|
||||||
ImportRecord map = new ImportRecord();
|
|
||||||
String fileUrls = fixFileNames(fileNames);
|
|
||||||
map.setFileUrl(fileUrls);
|
|
||||||
|
|
||||||
map.setCode(String.valueOf(row.get(0)));
|
|
||||||
map.setFunctionType(String.valueOf(row.get(1)));
|
|
||||||
map.setManufacturer(String.valueOf(row.get(2)));
|
|
||||||
map.setReportDate(String.valueOf(row.get(3)));
|
|
||||||
map.setTestMethod(String.valueOf(row.get(5)));
|
|
||||||
map.setCategory(String.valueOf(row.get(6)));
|
|
||||||
map.setReportId(map.getCode().replaceAll(" ", ""));
|
|
||||||
tableData.add(map);
|
|
||||||
}
|
|
||||||
saveRadhomeFiles(tableData, radhomeKey);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void onFailure(@NotNull Call call, @NotNull IOException e) {
|
public void radhomeCrawler() {
|
||||||
e.printStackTrace();
|
|
||||||
redisUtil.del(radhomeKey);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
} catch (Exception e) {
|
|
||||||
redisUtil.del(radhomeKey);
|
|
||||||
throw new RuntimeException(e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private String fixFileNames(String fileNames) {
|
|
||||||
String[] split = StringUtils.split(fileNames, ";");
|
|
||||||
List<String> result = new ArrayList<>();
|
|
||||||
for (String s : split) {
|
|
||||||
if (!StringUtils.startsWith(s, "http")) {
|
|
||||||
result.add("https://radhome.gsfc.nasa.gov/radhome/papers/" + s);
|
|
||||||
} else {
|
|
||||||
result.add(s);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return StringUtils.join(result, ";");
|
|
||||||
}
|
|
||||||
|
|
||||||
private void saveRadhomeFiles(List<ImportRecord> fileList, String type) {
|
|
||||||
ThreadUtil.execute(() -> {
|
|
||||||
try {
|
|
||||||
for (ImportRecord record : fileList) {
|
|
||||||
String fileUploadResult = "";
|
|
||||||
String fileUrl = record.getFileUrl();
|
|
||||||
if (fileUrl.contains(";")) {
|
|
||||||
String[] split = fileUrl.split(";");
|
|
||||||
List<String> result = new ArrayList<>();
|
|
||||||
for (String s : split) {
|
|
||||||
byte[] fileBytes = HttpUtil.downloadBytes(s);
|
|
||||||
InputStream inputStream = new ByteArrayInputStream(fileBytes);
|
|
||||||
result.add(MinioUtil.upload(inputStream, "radhome/" + s.substring(s.lastIndexOf("/") + 1)));
|
|
||||||
fileUploadResult = StringUtils.join(result, ";");
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
byte[] fileBytes = HttpUtil.downloadBytes(fileUrl);
|
|
||||||
InputStream inputStream = new ByteArrayInputStream(fileBytes);
|
|
||||||
fileUploadResult = MinioUtil.upload(inputStream, "radhome/" + fileUrl.substring(fileUrl.lastIndexOf("/") + 1));
|
|
||||||
}
|
|
||||||
System.out.println(fileUploadResult);
|
|
||||||
if (StringUtils.isNotBlank(fileUploadResult)) {
|
|
||||||
record.setFileUrl(fileUploadResult);
|
|
||||||
save(record);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
redisUtil.del(type);
|
|
||||||
} catch (Exception e) {
|
|
||||||
e.printStackTrace();
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
private void saveEsaradFiles(List<ImportRecord> fileList, String type) {
|
|
||||||
ThreadUtil.execute(() -> {
|
|
||||||
try {
|
|
||||||
for (ImportRecord record : fileList) {
|
|
||||||
ImportRecord dbData = getOne(Wrappers.<ImportRecord>lambdaQuery().eq(ImportRecord::getReportSource, record.getReportId()));
|
|
||||||
if (Objects.nonNull(dbData)) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
String resultStr = "";
|
|
||||||
|
|
||||||
String dest = FileUtil.getTmpDirPath() + "esarad-" + record.getReportId() + "/";
|
|
||||||
FileUtil.mkdir(dest);
|
|
||||||
long fileSize = HttpUtil.downloadFile(record.getFileUrl(), dest);
|
|
||||||
|
|
||||||
if (fileSize > 0) {
|
|
||||||
List<File> files = FileUtil.loopFiles(dest);
|
|
||||||
for (File file : files) {
|
|
||||||
resultStr = MinioUtil.upload(IoUtil.toStream(file), "esarad/" + record.getReportId() + "-" + URLUtil.decode(file.getName(), Charset.defaultCharset()) );
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
System.out.println(resultStr);
|
|
||||||
if (StringUtils.isNotBlank(resultStr)) {
|
|
||||||
record.setFileUrl(resultStr);
|
|
||||||
save(record);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
redisUtil.del(type);
|
|
||||||
} catch (Exception e) {
|
|
||||||
e.printStackTrace();
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,26 +1,30 @@
|
|||||||
package org.jeecg.modules.system.controller;
|
package org.jeecg.modules.system.controller;
|
||||||
|
|
||||||
|
import jakarta.servlet.http.HttpServletRequest;
|
||||||
|
import jakarta.servlet.http.HttpServletResponse;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.jeecg.common.api.vo.Result;
|
import org.jeecg.common.api.vo.Result;
|
||||||
import org.jeecg.common.constant.CommonConstant;
|
import org.jeecg.common.constant.CommonConstant;
|
||||||
import org.jeecg.common.constant.SymbolConstant;
|
import org.jeecg.common.constant.SymbolConstant;
|
||||||
import org.jeecg.common.constant.enums.FileTypeEnum;
|
|
||||||
import org.jeecg.common.exception.JeecgBootException;
|
import org.jeecg.common.exception.JeecgBootException;
|
||||||
import org.jeecg.common.util.CommonUtils;
|
import org.jeecg.common.util.CommonUtils;
|
||||||
import org.jeecg.common.util.filter.SsrfFileTypeFilter;
|
import org.jeecg.common.util.filter.SsrfFileTypeFilter;
|
||||||
import org.jeecg.common.util.oConvertUtils;
|
import org.jeecg.common.util.oConvertUtils;
|
||||||
|
import org.jeecg.modules.oss.entity.OssFile;
|
||||||
|
import org.jeecg.modules.oss.service.IOssFileService;
|
||||||
|
import org.springframework.beans.factory.annotation.Autowired;
|
||||||
import org.springframework.beans.factory.annotation.Value;
|
import org.springframework.beans.factory.annotation.Value;
|
||||||
import org.springframework.context.annotation.Lazy;
|
|
||||||
import org.springframework.util.AntPathMatcher;
|
import org.springframework.util.AntPathMatcher;
|
||||||
import org.springframework.util.FileCopyUtils;
|
import org.springframework.util.FileCopyUtils;
|
||||||
import org.springframework.web.bind.annotation.*;
|
import org.springframework.web.bind.annotation.GetMapping;
|
||||||
|
import org.springframework.web.bind.annotation.PostMapping;
|
||||||
|
import org.springframework.web.bind.annotation.RequestMapping;
|
||||||
|
import org.springframework.web.bind.annotation.RestController;
|
||||||
import org.springframework.web.multipart.MultipartFile;
|
import org.springframework.web.multipart.MultipartFile;
|
||||||
import org.springframework.web.multipart.MultipartHttpServletRequest;
|
import org.springframework.web.multipart.MultipartHttpServletRequest;
|
||||||
import org.springframework.web.servlet.HandlerMapping;
|
import org.springframework.web.servlet.HandlerMapping;
|
||||||
import org.springframework.web.servlet.ModelAndView;
|
import org.springframework.web.servlet.ModelAndView;
|
||||||
|
|
||||||
import jakarta.servlet.http.HttpServletRequest;
|
|
||||||
import jakarta.servlet.http.HttpServletResponse;
|
|
||||||
import java.io.*;
|
import java.io.*;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -36,9 +40,10 @@ import java.io.*;
|
|||||||
@RequestMapping("/sys/common")
|
@RequestMapping("/sys/common")
|
||||||
public class CommonController {
|
public class CommonController {
|
||||||
|
|
||||||
|
@Autowired
|
||||||
|
IOssFileService ossFileService;
|
||||||
@Value(value = "${jeecg.path.upload}")
|
@Value(value = "${jeecg.path.upload}")
|
||||||
private String uploadpath;
|
private String uploadpath;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 本地:local minio:minio 阿里:alioss
|
* 本地:local minio:minio 阿里:alioss
|
||||||
*/
|
*/
|
||||||
@@ -46,9 +51,22 @@ public class CommonController {
|
|||||||
private String uploadType;
|
private String uploadType;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @Author 政辉
|
* 把指定URL后的字符串全部截断当成参数
|
||||||
|
* 这么做是为了防止URL中包含中文或者特殊字符(/等)时,匹配不了的问题
|
||||||
|
*
|
||||||
|
* @param request
|
||||||
* @return
|
* @return
|
||||||
*/
|
*/
|
||||||
|
private static String extractPathFromPattern(final HttpServletRequest request) {
|
||||||
|
String path = (String) request.getAttribute(HandlerMapping.PATH_WITHIN_HANDLER_MAPPING_ATTRIBUTE);
|
||||||
|
String bestMatchPattern = (String) request.getAttribute(HandlerMapping.BEST_MATCHING_PATTERN_ATTRIBUTE);
|
||||||
|
return new AntPathMatcher().extractPathWithinPattern(bestMatchPattern, path);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @return
|
||||||
|
* @Author 政辉
|
||||||
|
*/
|
||||||
@GetMapping("/403")
|
@GetMapping("/403")
|
||||||
public Result<?> noauth() {
|
public Result<?> noauth() {
|
||||||
return Result.error("没有权限,请联系管理员授权,后刷新缓存!");
|
return Result.error("没有权限,请联系管理员授权,后刷新缓存!");
|
||||||
@@ -56,13 +74,14 @@ public class CommonController {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* 文件上传统一方法
|
* 文件上传统一方法
|
||||||
|
*
|
||||||
* @param request
|
* @param request
|
||||||
* @param response
|
* @param response
|
||||||
* @return
|
* @return
|
||||||
*/
|
*/
|
||||||
@PostMapping(value = "/upload")
|
@PostMapping(value = "/upload")
|
||||||
public Result<?> upload(HttpServletRequest request, HttpServletResponse response) throws Exception {
|
public Result<?> upload(HttpServletRequest request, HttpServletResponse response) throws Exception {
|
||||||
Result<?> result = new Result<>();
|
Result<OssFile> result = new Result<>();
|
||||||
String savePath = "";
|
String savePath = "";
|
||||||
String bizPath = request.getParameter("biz");
|
String bizPath = request.getParameter("biz");
|
||||||
|
|
||||||
@@ -108,6 +127,13 @@ public class CommonController {
|
|||||||
} else {
|
} else {
|
||||||
//update-begin-author:taoyan date:20200814 for:文件上传改造
|
//update-begin-author:taoyan date:20200814 for:文件上传改造
|
||||||
savePath = CommonUtils.upload(file, bizPath, uploadType);
|
savePath = CommonUtils.upload(file, bizPath, uploadType);
|
||||||
|
|
||||||
|
OssFile ossFile = new OssFile();
|
||||||
|
ossFile.setFileName(file.getOriginalFilename());
|
||||||
|
ossFile.setUrl(savePath);
|
||||||
|
ossFileService.save(ossFile);
|
||||||
|
result.setResult(ossFile);
|
||||||
|
|
||||||
//update-end-author:taoyan date:20200814 for:文件上传改造
|
//update-end-author:taoyan date:20200814 for:文件上传改造
|
||||||
}
|
}
|
||||||
if (oConvertUtils.isNotEmpty(savePath)) {
|
if (oConvertUtils.isNotEmpty(savePath)) {
|
||||||
@@ -120,8 +146,46 @@ public class CommonController {
|
|||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// @PostMapping(value = "/upload2")
|
||||||
|
// public Result<?> upload2(HttpServletRequest request, HttpServletResponse response) {
|
||||||
|
// Result<?> result = new Result<>();
|
||||||
|
// try {
|
||||||
|
// String ctxPath = uploadpath;
|
||||||
|
// String fileName = null;
|
||||||
|
// String bizPath = "files";
|
||||||
|
// String tempBizPath = request.getParameter("biz");
|
||||||
|
// if(oConvertUtils.isNotEmpty(tempBizPath)){
|
||||||
|
// bizPath = tempBizPath;
|
||||||
|
// }
|
||||||
|
// String nowday = new SimpleDateFormat("yyyyMMdd").format(new Date());
|
||||||
|
// File file = new File(ctxPath + File.separator + bizPath + File.separator + nowday);
|
||||||
|
// if (!file.exists()) {
|
||||||
|
// file.mkdirs();// 创建文件根目录
|
||||||
|
// }
|
||||||
|
// MultipartHttpServletRequest multipartRequest = (MultipartHttpServletRequest) request;
|
||||||
|
// MultipartFile mf = multipartRequest.getFile("file");// 获取上传文件对象
|
||||||
|
// String orgName = mf.getOriginalFilename();// 获取文件名
|
||||||
|
// fileName = orgName.substring(0, orgName.lastIndexOf(".")) + "_" + System.currentTimeMillis() + orgName.substring(orgName.indexOf("."));
|
||||||
|
// String savePath = file.getPath() + File.separator + fileName;
|
||||||
|
// File savefile = new File(savePath);
|
||||||
|
// FileCopyUtils.copy(mf.getBytes(), savefile);
|
||||||
|
// String dbpath = bizPath + File.separator + nowday + File.separator + fileName;
|
||||||
|
// if (dbpath.contains("\\")) {
|
||||||
|
// dbpath = dbpath.replace("\\", "/");
|
||||||
|
// }
|
||||||
|
// result.setMessage(dbpath);
|
||||||
|
// result.setSuccess(true);
|
||||||
|
// } catch (IOException e) {
|
||||||
|
// result.setSuccess(false);
|
||||||
|
// result.setMessage(e.getMessage());
|
||||||
|
// log.error(e.getMessage(), e);
|
||||||
|
// }
|
||||||
|
// return result;
|
||||||
|
// }
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 本地文件上传
|
* 本地文件上传
|
||||||
|
*
|
||||||
* @param mf 文件
|
* @param mf 文件
|
||||||
* @param bizPath 自定义路径
|
* @param bizPath 自定义路径
|
||||||
* @return
|
* @return
|
||||||
@@ -162,41 +226,62 @@ public class CommonController {
|
|||||||
return "";
|
return "";
|
||||||
}
|
}
|
||||||
|
|
||||||
// @PostMapping(value = "/upload2")
|
// /**
|
||||||
// public Result<?> upload2(HttpServletRequest request, HttpServletResponse response) {
|
// * 下载文件
|
||||||
// Result<?> result = new Result<>();
|
// * 请求地址:http://localhost:8080/common/download/{user/20190119/e1fe9925bc315c60addea1b98eb1cb1349547719_1547866868179.jpg}
|
||||||
|
// *
|
||||||
|
// * @param request
|
||||||
|
// * @param response
|
||||||
|
// * @throws Exception
|
||||||
|
// */
|
||||||
|
// @GetMapping(value = "/download/**")
|
||||||
|
// public void download(HttpServletRequest request, HttpServletResponse response) throws Exception {
|
||||||
|
// // ISO-8859-1 ==> UTF-8 进行编码转换
|
||||||
|
// String filePath = extractPathFromPattern(request);
|
||||||
|
// // 其余处理略
|
||||||
|
// InputStream inputStream = null;
|
||||||
|
// OutputStream outputStream = null;
|
||||||
// try {
|
// try {
|
||||||
// String ctxPath = uploadpath;
|
// filePath = filePath.replace("..", "");
|
||||||
// String fileName = null;
|
// if (filePath.endsWith(",")) {
|
||||||
// String bizPath = "files";
|
// filePath = filePath.substring(0, filePath.length() - 1);
|
||||||
// String tempBizPath = request.getParameter("biz");
|
|
||||||
// if(oConvertUtils.isNotEmpty(tempBizPath)){
|
|
||||||
// bizPath = tempBizPath;
|
|
||||||
// }
|
// }
|
||||||
// String nowday = new SimpleDateFormat("yyyyMMdd").format(new Date());
|
// String localPath = uploadpath;
|
||||||
// File file = new File(ctxPath + File.separator + bizPath + File.separator + nowday);
|
// String downloadFilePath = localPath + File.separator + filePath;
|
||||||
// if (!file.exists()) {
|
// File file = new File(downloadFilePath);
|
||||||
// file.mkdirs();// 创建文件根目录
|
// if (file.exists()) {
|
||||||
|
// response.setContentType("application/force-download");// 设置强制下载不打开
|
||||||
|
// response.addHeader("Content-Disposition", "attachment;fileName=" + new String(file.getName().getBytes("UTF-8"),"iso-8859-1"));
|
||||||
|
// inputStream = new BufferedInputStream(new FileInputStream(file));
|
||||||
|
// outputStream = response.getOutputStream();
|
||||||
|
// byte[] buf = new byte[1024];
|
||||||
|
// int len;
|
||||||
|
// while ((len = inputStream.read(buf)) > 0) {
|
||||||
|
// outputStream.write(buf, 0, len);
|
||||||
// }
|
// }
|
||||||
// MultipartHttpServletRequest multipartRequest = (MultipartHttpServletRequest) request;
|
// response.flushBuffer();
|
||||||
// MultipartFile mf = multipartRequest.getFile("file");// 获取上传文件对象
|
|
||||||
// String orgName = mf.getOriginalFilename();// 获取文件名
|
|
||||||
// fileName = orgName.substring(0, orgName.lastIndexOf(".")) + "_" + System.currentTimeMillis() + orgName.substring(orgName.indexOf("."));
|
|
||||||
// String savePath = file.getPath() + File.separator + fileName;
|
|
||||||
// File savefile = new File(savePath);
|
|
||||||
// FileCopyUtils.copy(mf.getBytes(), savefile);
|
|
||||||
// String dbpath = bizPath + File.separator + nowday + File.separator + fileName;
|
|
||||||
// if (dbpath.contains("\\")) {
|
|
||||||
// dbpath = dbpath.replace("\\", "/");
|
|
||||||
// }
|
// }
|
||||||
// result.setMessage(dbpath);
|
//
|
||||||
// result.setSuccess(true);
|
// } catch (Exception e) {
|
||||||
|
// log.info("文件下载失败" + e.getMessage());
|
||||||
|
// // e.printStackTrace();
|
||||||
|
// } finally {
|
||||||
|
// if (inputStream != null) {
|
||||||
|
// try {
|
||||||
|
// inputStream.close();
|
||||||
// } catch (IOException e) {
|
// } catch (IOException e) {
|
||||||
// result.setSuccess(false);
|
// e.printStackTrace();
|
||||||
// result.setMessage(e.getMessage());
|
|
||||||
// log.error(e.getMessage(), e);
|
|
||||||
// }
|
// }
|
||||||
// return result;
|
// }
|
||||||
|
// if (outputStream != null) {
|
||||||
|
// try {
|
||||||
|
// outputStream.close();
|
||||||
|
// } catch (IOException e) {
|
||||||
|
// e.printStackTrace();
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
//
|
||||||
// }
|
// }
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -267,68 +352,10 @@ public class CommonController {
|
|||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// /**
|
|
||||||
// * 下载文件
|
|
||||||
// * 请求地址:http://localhost:8080/common/download/{user/20190119/e1fe9925bc315c60addea1b98eb1cb1349547719_1547866868179.jpg}
|
|
||||||
// *
|
|
||||||
// * @param request
|
|
||||||
// * @param response
|
|
||||||
// * @throws Exception
|
|
||||||
// */
|
|
||||||
// @GetMapping(value = "/download/**")
|
|
||||||
// public void download(HttpServletRequest request, HttpServletResponse response) throws Exception {
|
|
||||||
// // ISO-8859-1 ==> UTF-8 进行编码转换
|
|
||||||
// String filePath = extractPathFromPattern(request);
|
|
||||||
// // 其余处理略
|
|
||||||
// InputStream inputStream = null;
|
|
||||||
// OutputStream outputStream = null;
|
|
||||||
// try {
|
|
||||||
// filePath = filePath.replace("..", "");
|
|
||||||
// if (filePath.endsWith(",")) {
|
|
||||||
// filePath = filePath.substring(0, filePath.length() - 1);
|
|
||||||
// }
|
|
||||||
// String localPath = uploadpath;
|
|
||||||
// String downloadFilePath = localPath + File.separator + filePath;
|
|
||||||
// File file = new File(downloadFilePath);
|
|
||||||
// if (file.exists()) {
|
|
||||||
// response.setContentType("application/force-download");// 设置强制下载不打开
|
|
||||||
// response.addHeader("Content-Disposition", "attachment;fileName=" + new String(file.getName().getBytes("UTF-8"),"iso-8859-1"));
|
|
||||||
// inputStream = new BufferedInputStream(new FileInputStream(file));
|
|
||||||
// outputStream = response.getOutputStream();
|
|
||||||
// byte[] buf = new byte[1024];
|
|
||||||
// int len;
|
|
||||||
// while ((len = inputStream.read(buf)) > 0) {
|
|
||||||
// outputStream.write(buf, 0, len);
|
|
||||||
// }
|
|
||||||
// response.flushBuffer();
|
|
||||||
// }
|
|
||||||
//
|
|
||||||
// } catch (Exception e) {
|
|
||||||
// log.info("文件下载失败" + e.getMessage());
|
|
||||||
// // e.printStackTrace();
|
|
||||||
// } finally {
|
|
||||||
// if (inputStream != null) {
|
|
||||||
// try {
|
|
||||||
// inputStream.close();
|
|
||||||
// } catch (IOException e) {
|
|
||||||
// e.printStackTrace();
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
// if (outputStream != null) {
|
|
||||||
// try {
|
|
||||||
// outputStream.close();
|
|
||||||
// } catch (IOException e) {
|
|
||||||
// e.printStackTrace();
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
//
|
|
||||||
// }
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @功能:pdf预览Iframe
|
|
||||||
* @param modelAndView
|
* @param modelAndView
|
||||||
* @return
|
* @return
|
||||||
|
* @功能:pdf预览Iframe
|
||||||
*/
|
*/
|
||||||
@RequestMapping("/pdf/pdfPreviewIframe")
|
@RequestMapping("/pdf/pdfPreviewIframe")
|
||||||
public ModelAndView pdfPreviewIframe(ModelAndView modelAndView) {
|
public ModelAndView pdfPreviewIframe(ModelAndView modelAndView) {
|
||||||
@@ -336,16 +363,4 @@ public class CommonController {
|
|||||||
return modelAndView;
|
return modelAndView;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* 把指定URL后的字符串全部截断当成参数
|
|
||||||
* 这么做是为了防止URL中包含中文或者特殊字符(/等)时,匹配不了的问题
|
|
||||||
* @param request
|
|
||||||
* @return
|
|
||||||
*/
|
|
||||||
private static String extractPathFromPattern(final HttpServletRequest request) {
|
|
||||||
String path = (String) request.getAttribute(HandlerMapping.PATH_WITHIN_HANDLER_MAPPING_ATTRIBUTE);
|
|
||||||
String bestMatchPattern = (String) request.getAttribute(HandlerMapping.BEST_MATCHING_PATTERN_ATTRIBUTE);
|
|
||||||
return new AntPathMatcher().extractPathWithinPattern(bestMatchPattern, path);
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user