physical3/physical-boot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update

Browse Source
This commit is contained in:
ls
2024-11-12 15:46:28 +08:00
parent d02773609f
commit 835218ffa6
8 changed files with 161 additions and 133 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
physical-launcher/src/main/resources/application-dev.yml
View File

@@ -230,7 +230,7 @@ jeecg:
#webapp文件路径
webapp: /opt/webapp
shiro:
excludeUrls: /database/experimentDoc/**,/sys/common/upload,/sys/user/**,/test/jeecgDemo/demo3,/test/jeecgDemo/redisDemo/**,/bigscreen/category/**,/bigscreen/visual/**,/bigscreen/map/**,/jmreport/bigscreen2/**
excludeUrls: /database/experimentDoc/**,/sys/common/upload,/sys/common/download,/sys/user/**,/test/jeecgDemo/demo3,/test/jeecgDemo/redisDemo/**,/bigscreen/category/**,/bigscreen/visual/**,/bigscreen/map/**,/jmreport/bigscreen2/**
#阿里云oss存储和大鱼短信秘钥配置
oss:
accessKey: ??

2
physical-launcher/src/main/resources/application-prod.yml
View File

@@ -231,7 +231,7 @@ jeecg:
#webapp文件路径
webapp: /opt/jeecg-boot/webapp
shiro:
excludeUrls: /sys/common/upload,/test/jeecgDemo/demo3,/test/jeecgDemo/redisDemo/**,/bigscreen/category/**,/bigscreen/visual/**,/bigscreen/map/**,/jmreport/bigscreen2/**,/api/getUserInfo,/api/sys/common/**,/sys/common/**
excludeUrls: /sys/common/upload,/sys/common/download,/test/jeecgDemo/demo3,/test/jeecgDemo/redisDemo/**,/bigscreen/category/**,/bigscreen/visual/**,/bigscreen/map/**,/jmreport/bigscreen2/**,/api/getUserInfo,/api/sys/common/**,/sys/common/**
#阿里云oss存储和大鱼短信秘钥配置
oss:
accessKey: ??

2
physical-launcher/src/main/resources/application-test.yml
View File

@@ -231,7 +231,7 @@ jeecg:
#webapp文件路径
webapp: /opt/jeecg-boot/webapp
shiro:
excludeUrls: /sys/common/upload,/test/jeecgDemo/demo3,/test/jeecgDemo/redisDemo/**,/bigscreen/category/**,/bigscreen/visual/**,/bigscreen/map/**,/jmreport/bigscreen2/**,/api/getUserInfo,/api/sys/common/**,/sys/common/**
excludeUrls: /sys/common/upload,/sys/common/download,/test/jeecgDemo/demo3,/test/jeecgDemo/redisDemo/**,/bigscreen/category/**,/bigscreen/visual/**,/bigscreen/map/**,/jmreport/bigscreen2/**,/api/getUserInfo,/api/sys/common/**,/sys/common/**
#阿里云oss存储和大鱼短信秘钥配置
oss:
accessKey: ??

215
physical-module-system/physical-system-biz/src/main/java/org/jeecg/modules/system/controller/CommonController.java
View File

@@ -3,29 +3,27 @@ package org.jeecg.modules.system.controller;
import jakarta.servlet.http.HttpServletRequest;
import jakarta.servlet.http.HttpServletResponse;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.jeecg.common.api.vo.Result;
import org.jeecg.common.constant.CommonConstant;
import org.jeecg.common.constant.SymbolConstant;
import org.jeecg.common.exception.JeecgBootException;
import org.jeecg.common.util.CommonUtils;
import org.jeecg.common.util.*;
import org.jeecg.common.util.filter.SsrfFileTypeFilter;
import org.jeecg.common.util.oConvertUtils;
import org.jeecg.modules.oss.entity.OssFile;
import org.jeecg.modules.oss.service.IOssFileService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.util.AntPathMatcher;
import org.springframework.util.FileCopyUtils;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.multipart.MultipartFile;
import org.springframework.web.multipart.MultipartHttpServletRequest;
import org.springframework.web.servlet.HandlerMapping;
import org.springframework.web.servlet.ModelAndView;
import java.io.*;
import java.net.URLDecoder;
/**
* <p>
@@ -72,6 +70,20 @@ public class CommonController {
return Result.error("没有权限，请联系管理员授权，后刷新缓存!");
}
@GetMapping(value = "/download")
public void download(@RequestParam(name = "file", defaultValue = "") String file, HttpServletRequest request,
HttpServletResponse response) throws Exception {
if (StringUtils.isBlank(file)) {
return;
}
String fileName = URLDecoder.decode(file, "UTF-8");
String objectUrl = MinioUtil.getObjectUrl(MinioUtil.getBucketName(), fileName, 2 * 24 * 60 * 60);
if (StringUtils.isBlank(objectUrl)) {
return;
}
response.sendRedirect(objectUrl);
}
/**
* 文件上传统一方法
*
@@ -146,42 +158,43 @@ public class CommonController {
return result;
}
// @PostMapping(value = "/upload2")
// public Result<?> upload2(HttpServletRequest request, HttpServletResponse response) {
// Result<?> result = new Result<>();
// try {
// String ctxPath = uploadpath;
// String fileName = null;
// String bizPath = "files";
// String tempBizPath = request.getParameter("biz");
// if(oConvertUtils.isNotEmpty(tempBizPath)){
// bizPath = tempBizPath;
// }
// String nowday = new SimpleDateFormat("yyyyMMdd").format(new Date());
// File file = new File(ctxPath + File.separator + bizPath + File.separator + nowday);
// if (!file.exists()) {
// file.mkdirs();// 创建文件根目录
// }
// MultipartHttpServletRequest multipartRequest = (MultipartHttpServletRequest) request;
// MultipartFile mf = multipartRequest.getFile("file");// 获取上传文件对象
// String orgName = mf.getOriginalFilename();// 获取文件名
// fileName = orgName.substring(0, orgName.lastIndexOf(".")) + "_" + System.currentTimeMillis() + orgName.substring(orgName.indexOf("."));
// String savePath = file.getPath() + File.separator + fileName;
// File savefile = new File(savePath);
// FileCopyUtils.copy(mf.getBytes(), savefile);
// String dbpath = bizPath + File.separator + nowday + File.separator + fileName;
// if (dbpath.contains("\\")) {
// dbpath = dbpath.replace("\\", "/");
// }
// result.setMessage(dbpath);
// result.setSuccess(true);
// } catch (IOException e) {
// result.setSuccess(false);
// result.setMessage(e.getMessage());
// log.error(e.getMessage(), e);
// }
// return result;
// }
// @PostMapping(value = "/upload2")
// public Result<?> upload2(HttpServletRequest request, HttpServletResponse response) {
// Result<?> result = new Result<>();
// try {
// String ctxPath = uploadpath;
// String fileName = null;
// String bizPath = "files";
// String tempBizPath = request.getParameter("biz");
// if(oConvertUtils.isNotEmpty(tempBizPath)){
// bizPath = tempBizPath;
// }
// String nowday = new SimpleDateFormat("yyyyMMdd").format(new Date());
// File file = new File(ctxPath + File.separator + bizPath + File.separator + nowday);
// if (!file.exists()) {
// file.mkdirs();// 创建文件根目录
// }
// MultipartHttpServletRequest multipartRequest = (MultipartHttpServletRequest) request;
// MultipartFile mf = multipartRequest.getFile("file");// 获取上传文件对象
// String orgName = mf.getOriginalFilename();// 获取文件名
// fileName = orgName.substring(0, orgName.lastIndexOf(".")) + "_" + System.currentTimeMillis() + orgName.substring(orgName
// .indexOf("."));
// String savePath = file.getPath() + File.separator + fileName;
// File savefile = new File(savePath);
// FileCopyUtils.copy(mf.getBytes(), savefile);
// String dbpath = bizPath + File.separator + nowday + File.separator + fileName;
// if (dbpath.contains("\\")) {
// dbpath = dbpath.replace("\\", "/");
// }
// result.setMessage(dbpath);
// result.setSuccess(true);
// } catch (IOException e) {
// result.setSuccess(false);
// result.setMessage(e.getMessage());
// log.error(e.getMessage(), e);
// }
// return result;
// }
/**
* 本地文件上传
@@ -203,7 +216,8 @@ public class CommonController {
String orgName = mf.getOriginalFilename();
orgName = CommonUtils.getFileName(orgName);
if (orgName.indexOf(SymbolConstant.SPOT) != -1) {
fileName = orgName.substring(0, orgName.lastIndexOf(".")) + "_" + System.currentTimeMillis() + orgName.substring(orgName.lastIndexOf("."));
fileName = orgName.substring(0, orgName.lastIndexOf(".")) + "_" + System.currentTimeMillis() + orgName.substring(
orgName.lastIndexOf("."));
} else {
fileName = orgName + "_" + System.currentTimeMillis();
}
@@ -226,63 +240,64 @@ public class CommonController {
return "";
}
// /**
// * 下载文件
// * 请求地址：http://localhost:8080/common/download/{user/20190119/e1fe9925bc315c60addea1b98eb1cb1349547719_1547866868179.jpg}
// *
// * @param request
// * @param response
// * @throws Exception
// */
// @GetMapping(value = "/download/**")
// public void download(HttpServletRequest request, HttpServletResponse response) throws Exception {
// // ISO-8859-1 ==> UTF-8 进行编码转换
// String filePath = extractPathFromPattern(request);
// // 其余处理略
// InputStream inputStream = null;
// OutputStream outputStream = null;
// try {
// filePath = filePath.replace("..", "");
// if (filePath.endsWith(",")) {
// filePath = filePath.substring(0, filePath.length() - 1);
// }
// String localPath = uploadpath;
// String downloadFilePath = localPath + File.separator + filePath;
// File file = new File(downloadFilePath);
// if (file.exists()) {
// response.setContentType("application/force-download");// 设置强制下载不打开            
// response.addHeader("Content-Disposition", "attachment;fileName=" + new String(file.getName().getBytes("UTF-8"),"iso-8859-1"));
// inputStream = new BufferedInputStream(new FileInputStream(file));
// outputStream = response.getOutputStream();
// byte[] buf = new byte[1024];
// int len;
// while ((len = inputStream.read(buf)) > 0) {
// outputStream.write(buf, 0, len);
// }
// response.flushBuffer();
// }
//
// } catch (Exception e) {
// log.info("文件下载失败" + e.getMessage());
// // e.printStackTrace();
// } finally {
// if (inputStream != null) {
// try {
// inputStream.close();
// } catch (IOException e) {
// e.printStackTrace();
// }
// }
// if (outputStream != null) {
// try {
// outputStream.close();
// } catch (IOException e) {
// e.printStackTrace();
// }
// }
// }
//
// }
// /**
// * 下载文件
// * 请求地址：http://localhost:8080/common/download/{user/20190119/e1fe9925bc315c60addea1b98eb1cb1349547719_1547866868179.jpg}
// *
// * @param request
// * @param response
// * @throws Exception
// */
// @GetMapping(value = "/download/**")
// public void download(HttpServletRequest request, HttpServletResponse response) throws Exception {
// // ISO-8859-1 ==> UTF-8 进行编码转换
// String filePath = extractPathFromPattern(request);
// // 其余处理略
// InputStream inputStream = null;
// OutputStream outputStream = null;
// try {
// filePath = filePath.replace("..", "");
// if (filePath.endsWith(",")) {
// filePath = filePath.substring(0, filePath.length() - 1);
// }
// String localPath = uploadpath;
// String downloadFilePath = localPath + File.separator + filePath;
// File file = new File(downloadFilePath);
// if (file.exists()) {
// response.setContentType("application/force-download");// 设置强制下载不打开            
// response.addHeader("Content-Disposition", "attachment;fileName=" + new String(file.getName().getBytes("UTF-8"),
// "iso-8859-1"));
// inputStream = new BufferedInputStream(new FileInputStream(file));
// outputStream = response.getOutputStream();
// byte[] buf = new byte[1024];
// int len;
// while ((len = inputStream.read(buf)) > 0) {
// outputStream.write(buf, 0, len);
// }
// response.flushBuffer();
// }
//
// } catch (Exception e) {
// log.info("文件下载失败" + e.getMessage());
// // e.printStackTrace();
// } finally {
// if (inputStream != null) {
// try {
// inputStream.close();
// } catch (IOException e) {
// e.printStackTrace();
// }
// }
// if (outputStream != null) {
// try {
// outputStream.close();
// } catch (IOException e) {
// e.printStackTrace();
// }
// }
// }
//
// }
/**
* 预览图片&下载文件

33
scripts/common.py
View File

@@ -1,6 +1,7 @@
import hashlib
import os
from datetime import datetime
from datetime import timedelta
import mysql.connector
from minio import Minio
@@ -9,10 +10,10 @@ from minio.error import S3Error
minio_public_url = 'http://58.215.212.230:8005/oss/'
# MySQL 连接配置
db_config = {
'host': 'physical-mysql',
'port': 3306,
# 'host': '192.168.50.100',
# 'port': 23306,
# 'host': 'physical-mysql',
# 'port': 3306,
'host': '192.168.50.100',
'port': 23306,
'user': 'root',
'password': '123456',
'database': 'physical-boot'
@@ -20,8 +21,8 @@ db_config = {
# minio 配置
minio_client = Minio(
"physical-minio:9000", # MinIO服务器地址或IP
# "192.168.50.100:29000", # MinIO服务器地址或IP
# "physical-minio:9000", # MinIO服务器地址或IP
"192.168.50.100:29000", # MinIO服务器地址或IP
access_key="root", # 替换为你的Access Key
secret_key="12345678", # 替换为你的Secret Key
secure=False # 如果使用的是http则为False
@@ -42,8 +43,8 @@ def save_to_db_import_record(connection,data):
cursor = connection.cursor()
try:
"""保存数据到 MySQL 数据库"""
insert_query = """INSERT INTO `import_record` (`id`, `create_by`, `create_time`, `update_by`, `update_time`, `sys_org_code`, `device_type`, `device_name`, `device_mode`, `device_function`, `device_batch`, `manufacturer`, `experiment_date`, `data_source`, `experiment_user`, `total_count`, `file_list`)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s);"""
insert_query = """INSERT INTO `nasa_data_record` (`id`, `create_by`, `create_time`, `update_by`, `update_time`, `sys_org_code`, `device_type`, `device_name`, `device_mode`, `device_function`, `device_batch`, `manufacturer`, `experiment_date`, `data_source`, `experiment_user`, `total_count`, `file_list`,`origin_data`)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s);"""
cursor.execute(insert_query, data)
connection.commit()
except Exception as e:
@@ -80,13 +81,17 @@ def upload_to_minio(connection,folder_path,type):
object_name = f"{type}/{folder_name}/{file_name}"
try:
# 上传文件到 MinIO
minio_client.fput_object(bucket_name, object_name, file_path)
url=minio_client.fput_object(bucket_name, object_name, file_path)
print(f"已上传: {file_path} -> {bucket_name}/{object_name}")
file_id = get_md5(object_name)
file_ids.append(file_id)
db_file = [file_id, file_name,
minio_public_url + bucket_name + '/' + object_name, 'admin', datetime.now()]
save_to_db_oss_file(connection,db_file)
# file_id = get_md5(object_name)
file_ids.append({file_name:os.path.join(minio_public_url,bucket_name,url.object_name)})
# db_file = [file_id, file_name,
# minio_public_url + bucket_name + '/' + object_name, 'admin', datetime.now()]
# save_to_db_oss_file(connection,db_file)
# expires = timedelta(days=1)
# presigned_url = minio_client.presigned_get_object(bucket_name, object_name, expires=expires)
# print(f"Presigned URL for {object_name}: {presigned_url}")
except S3Error as err:
print(f"上传 {file_name} 时出错: {err}")
return file_ids

23
scripts/esa.py
View File

@@ -44,7 +44,7 @@ def scrape():
# 下载文件并获取文件名
file_response = requests.get(download_url)
if file_response.status_code == 200:
if file_response and file_response.status_code == 200:
# 从响应头获取文件名
content_disposition = file_response.headers.get('Content-Disposition')
filename = ''
@@ -202,16 +202,17 @@ def scrape():
else:
print(f'Failed to download: {download_url}')
upload_ids = upload_to_minio(esa_connection, folder_path,'ESA')
data_db = ['ESA-' + cells[0].get_text(strip=True), 'Crawler', datetime.now(), None, None, None,
cells[5].get_text(strip=True),
cells[1].get_text(strip=True), cells[1].get_text(strip=True),
cells[8].get_text(strip=True),
cells[7].get_text(strip=True), cells[2].get_text(strip=True),
cells[11].get_text(strip=True),
'ESA', None, None, ','.join(upload_ids)
]
save_to_db_import_record(esa_connection, data_db)
upload_ids = upload_to_minio(esa_connection, folder_path, 'ESA')
origin_data = ','.join([c.get_text(strip=True) for c in cells])
data_db = ['ESA-' + cells[0].get_text(strip=True), 'Crawler', datetime.now(), None, None, None,
cells[5].get_text(strip=True),
cells[1].get_text(strip=True), cells[1].get_text(strip=True),
cells[8].get_text(strip=True),
cells[7].get_text(strip=True), cells[2].get_text(strip=True),
cells[11].get_text(strip=True),
'ESA', None, None, str(upload_ids), origin_data
]
save_to_db_import_record(esa_connection, data_db)
else:
print(f'Error: {response.status_code}')
finally:

14
scripts/nasa1.py
View File

@@ -73,7 +73,9 @@ def scrape():
# 解析 JSON 数据
json_data = response.json()
# 遍历数据并下载文件
for row in json_data['ROWS']:
print("total:" + str(json_data['RECORDS']))
for index, row in enumerate(json_data['ROWS']):
print("index:"+str(index))
part_number = row[0] # 部件编号
file_links_str = row[4] # 文件链接
@@ -209,7 +211,7 @@ def scrape():
file_response = requests.get(file_url)
file_response.raise_for_status()
# 保存文件
# 保存文件1685+431+314
file_path = os.path.join(folder_path, os.path.basename(file_url))
with open(file_path, 'wb') as file:
file.write(file_response.content)
@@ -226,15 +228,19 @@ def scrape():
device_batch=None
manufacturer=row[2]
experiment_date=row[3]
data_db = [get_md5(row[0]), 'Crawler', datetime.now(), None, None, None,
origin_data = ','.join(row)
data_db = [get_md5(''.join([device_name,device_function,manufacturer])), 'Crawler', datetime.now(), None, None, None,
device_type,
device_name, device_mode,
device_function,
device_batch, manufacturer,
experiment_date,
'NASA', None, None, ','.join(upload_ids)
'NASA', None, None, str(upload_ids),origin_data
]
save_to_db_import_record(nasa1_connection, data_db)
except Exception as e:
print(e)
finally:
# 关闭游标和连接
nasa1_connection.close()

3
scripts/nasa2.py
View File

@@ -198,13 +198,14 @@ def scrape():
device_batch = None
manufacturer = None
experiment_date = columns[8].text_content().strip()
origin_data = ','.join([c.text_content().strip() for c in columns])
data_db = ['NASA-' + columns[0].text_content().strip(), 'Crawler', datetime.now(), None, None, None,
device_type,
device_name, device_mode,
device_function,
device_batch, manufacturer,
experiment_date,
'NASA', None, None, ','.join(upload_ids)
'NASA', None, None, str(upload_ids),origin_data
]
save_to_db_import_record(nasa2_connection, data_db)
except Exception as e: