physical3/physical-boot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update

Browse Source
This commit is contained in:
ls
2024-11-12 15:46:28 +08:00
parent d02773609f
commit 835218ffa6
8 changed files with 161 additions and 133 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
physical-launcher/src/main/resources/application-dev.yml
View File

@@ -230,7 +230,7 @@ jeecg:
#webapp文件路径 #webapp文件路径
webapp: /opt/webapp webapp: /opt/webapp
shiro: shiro:
excludeUrls: /database/experimentDoc/**,/sys/common/upload,/sys/user/**,/test/jeecgDemo/demo3,/test/jeecgDemo/redisDemo/**,/bigscreen/category/**,/bigscreen/visual/**,/bigscreen/map/**,/jmreport/bigscreen2/** excludeUrls: /database/experimentDoc/**,/sys/common/upload,/sys/common/download,/sys/user/**,/test/jeecgDemo/demo3,/test/jeecgDemo/redisDemo/**,/bigscreen/category/**,/bigscreen/visual/**,/bigscreen/map/**,/jmreport/bigscreen2/**
#阿里云oss存储和大鱼短信秘钥配置 #阿里云oss存储和大鱼短信秘钥配置
oss: oss:
accessKey: ?? accessKey: ??

2
physical-launcher/src/main/resources/application-prod.yml
View File

@@ -231,7 +231,7 @@ jeecg:
#webapp文件路径 #webapp文件路径
webapp: /opt/jeecg-boot/webapp webapp: /opt/jeecg-boot/webapp
shiro: shiro:
excludeUrls: /sys/common/upload,/test/jeecgDemo/demo3,/test/jeecgDemo/redisDemo/**,/bigscreen/category/**,/bigscreen/visual/**,/bigscreen/map/**,/jmreport/bigscreen2/**,/api/getUserInfo,/api/sys/common/**,/sys/common/** excludeUrls: /sys/common/upload,/sys/common/download,/test/jeecgDemo/demo3,/test/jeecgDemo/redisDemo/**,/bigscreen/category/**,/bigscreen/visual/**,/bigscreen/map/**,/jmreport/bigscreen2/**,/api/getUserInfo,/api/sys/common/**,/sys/common/**
#阿里云oss存储和大鱼短信秘钥配置 #阿里云oss存储和大鱼短信秘钥配置
oss: oss:
accessKey: ?? accessKey: ??

2
physical-launcher/src/main/resources/application-test.yml
View File

@@ -231,7 +231,7 @@ jeecg:
#webapp文件路径 #webapp文件路径
webapp: /opt/jeecg-boot/webapp webapp: /opt/jeecg-boot/webapp
shiro: shiro:
excludeUrls: /sys/common/upload,/test/jeecgDemo/demo3,/test/jeecgDemo/redisDemo/**,/bigscreen/category/**,/bigscreen/visual/**,/bigscreen/map/**,/jmreport/bigscreen2/**,/api/getUserInfo,/api/sys/common/**,/sys/common/** excludeUrls: /sys/common/upload,/sys/common/download,/test/jeecgDemo/demo3,/test/jeecgDemo/redisDemo/**,/bigscreen/category/**,/bigscreen/visual/**,/bigscreen/map/**,/jmreport/bigscreen2/**,/api/getUserInfo,/api/sys/common/**,/sys/common/**
#阿里云oss存储和大鱼短信秘钥配置 #阿里云oss存储和大鱼短信秘钥配置
oss: oss:
accessKey: ?? accessKey: ??

33
physical-module-system/physical-system-biz/src/main/java/org/jeecg/modules/system/controller/CommonController.java
View File

@@ -3,29 +3,27 @@ package org.jeecg.modules.system.controller;
import jakarta.servlet.http.HttpServletRequest; import jakarta.servlet.http.HttpServletRequest;
import jakarta.servlet.http.HttpServletResponse; import jakarta.servlet.http.HttpServletResponse;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.jeecg.common.api.vo.Result; import org.jeecg.common.api.vo.Result;
import org.jeecg.common.constant.CommonConstant; import org.jeecg.common.constant.CommonConstant;
import org.jeecg.common.constant.SymbolConstant; import org.jeecg.common.constant.SymbolConstant;
import org.jeecg.common.exception.JeecgBootException; import org.jeecg.common.exception.JeecgBootException;
import org.jeecg.common.util.CommonUtils; import org.jeecg.common.util.*;
import org.jeecg.common.util.filter.SsrfFileTypeFilter; import org.jeecg.common.util.filter.SsrfFileTypeFilter;
import org.jeecg.common.util.oConvertUtils;
import org.jeecg.modules.oss.entity.OssFile; import org.jeecg.modules.oss.entity.OssFile;
import org.jeecg.modules.oss.service.IOssFileService; import org.jeecg.modules.oss.service.IOssFileService;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value; import org.springframework.beans.factory.annotation.Value;
import org.springframework.util.AntPathMatcher; import org.springframework.util.AntPathMatcher;
import org.springframework.util.FileCopyUtils; import org.springframework.util.FileCopyUtils;
import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.*;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.multipart.MultipartFile; import org.springframework.web.multipart.MultipartFile;
import org.springframework.web.multipart.MultipartHttpServletRequest; import org.springframework.web.multipart.MultipartHttpServletRequest;
import org.springframework.web.servlet.HandlerMapping; import org.springframework.web.servlet.HandlerMapping;
import org.springframework.web.servlet.ModelAndView; import org.springframework.web.servlet.ModelAndView;
import java.io.*; import java.io.*;
import java.net.URLDecoder;
/** /**
* <p> * <p>
@@ -72,6 +70,20 @@ public class CommonController {
return Result.error("没有权限，请联系管理员授权，后刷新缓存!"); return Result.error("没有权限，请联系管理员授权，后刷新缓存!");
} }
@GetMapping(value = "/download")
public void download(@RequestParam(name = "file", defaultValue = "") String file, HttpServletRequest request,
HttpServletResponse response) throws Exception {
if (StringUtils.isBlank(file)) {
return;
}
String fileName = URLDecoder.decode(file, "UTF-8");
String objectUrl = MinioUtil.getObjectUrl(MinioUtil.getBucketName(), fileName, 2 * 24 * 60 * 60);
if (StringUtils.isBlank(objectUrl)) {
return;
}
response.sendRedirect(objectUrl);
}
/** /**
* 文件上传统一方法 * 文件上传统一方法
* *
@@ -165,7 +177,8 @@ public class CommonController {
// MultipartHttpServletRequest multipartRequest = (MultipartHttpServletRequest) request; // MultipartHttpServletRequest multipartRequest = (MultipartHttpServletRequest) request;
// MultipartFile mf = multipartRequest.getFile("file");// 获取上传文件对象 // MultipartFile mf = multipartRequest.getFile("file");// 获取上传文件对象
// String orgName = mf.getOriginalFilename();// 获取文件名 // String orgName = mf.getOriginalFilename();// 获取文件名
// fileName = orgName.substring(0, orgName.lastIndexOf(".")) + "_" + System.currentTimeMillis() + orgName.substring(orgName.indexOf(".")); // fileName = orgName.substring(0, orgName.lastIndexOf(".")) + "_" + System.currentTimeMillis() + orgName.substring(orgName
// .indexOf("."));
// String savePath = file.getPath() + File.separator + fileName; // String savePath = file.getPath() + File.separator + fileName;
// File savefile = new File(savePath); // File savefile = new File(savePath);
// FileCopyUtils.copy(mf.getBytes(), savefile); // FileCopyUtils.copy(mf.getBytes(), savefile);
@@ -203,7 +216,8 @@ public class CommonController {
String orgName = mf.getOriginalFilename(); String orgName = mf.getOriginalFilename();
orgName = CommonUtils.getFileName(orgName); orgName = CommonUtils.getFileName(orgName);
if (orgName.indexOf(SymbolConstant.SPOT) != -1) { if (orgName.indexOf(SymbolConstant.SPOT) != -1) {
fileName = orgName.substring(0, orgName.lastIndexOf(".")) + "_" + System.currentTimeMillis() + orgName.substring(orgName.lastIndexOf(".")); fileName = orgName.substring(0, orgName.lastIndexOf(".")) + "_" + System.currentTimeMillis() + orgName.substring(
orgName.lastIndexOf("."));
} else { } else {
fileName = orgName + "_" + System.currentTimeMillis(); fileName = orgName + "_" + System.currentTimeMillis();
} }
@@ -251,7 +265,8 @@ public class CommonController {
// File file = new File(downloadFilePath); // File file = new File(downloadFilePath);
// if (file.exists()) { // if (file.exists()) {
// response.setContentType("application/force-download");// 设置强制下载不打开             // response.setContentType("application/force-download");// 设置强制下载不打开            
// response.addHeader("Content-Disposition", "attachment;fileName=" + new String(file.getName().getBytes("UTF-8"),"iso-8859-1")); // response.addHeader("Content-Disposition", "attachment;fileName=" + new String(file.getName().getBytes("UTF-8"),
// "iso-8859-1"));
// inputStream = new BufferedInputStream(new FileInputStream(file)); // inputStream = new BufferedInputStream(new FileInputStream(file));
// outputStream = response.getOutputStream(); // outputStream = response.getOutputStream();
// byte[] buf = new byte[1024]; // byte[] buf = new byte[1024];

33
scripts/common.py
View File

@@ -1,6 +1,7 @@
import hashlib import hashlib
import os import os
from datetime import datetime from datetime import datetime
from datetime import timedelta
import mysql.connector import mysql.connector
from minio import Minio from minio import Minio
@@ -9,10 +10,10 @@ from minio.error import S3Error
minio_public_url = 'http://58.215.212.230:8005/oss/' minio_public_url = 'http://58.215.212.230:8005/oss/'
# MySQL 连接配置 # MySQL 连接配置
db_config = { db_config = {
'host': 'physical-mysql', # 'host': 'physical-mysql',
'port': 3306, # 'port': 3306,
# 'host': '192.168.50.100', 'host': '192.168.50.100',
# 'port': 23306, 'port': 23306,
'user': 'root', 'user': 'root',
'password': '123456', 'password': '123456',
'database': 'physical-boot' 'database': 'physical-boot'
@@ -20,8 +21,8 @@ db_config = {
# minio 配置 # minio 配置
minio_client = Minio( minio_client = Minio(
"physical-minio:9000", # MinIO服务器地址或IP # "physical-minio:9000", # MinIO服务器地址或IP
# "192.168.50.100:29000", # MinIO服务器地址或IP "192.168.50.100:29000", # MinIO服务器地址或IP
access_key="root", # 替换为你的Access Key access_key="root", # 替换为你的Access Key
secret_key="12345678", # 替换为你的Secret Key secret_key="12345678", # 替换为你的Secret Key
secure=False # 如果使用的是http则为False secure=False # 如果使用的是http则为False
@@ -42,8 +43,8 @@ def save_to_db_import_record(connection,data):
cursor = connection.cursor() cursor = connection.cursor()
try: try:
"""保存数据到 MySQL 数据库""" """保存数据到 MySQL 数据库"""
insert_query = """INSERT INTO `import_record` (`id`, `create_by`, `create_time`, `update_by`, `update_time`, `sys_org_code`, `device_type`, `device_name`, `device_mode`, `device_function`, `device_batch`, `manufacturer`, `experiment_date`, `data_source`, `experiment_user`, `total_count`, `file_list`) insert_query = """INSERT INTO `nasa_data_record` (`id`, `create_by`, `create_time`, `update_by`, `update_time`, `sys_org_code`, `device_type`, `device_name`, `device_mode`, `device_function`, `device_batch`, `manufacturer`, `experiment_date`, `data_source`, `experiment_user`, `total_count`, `file_list`,`origin_data`)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s);""" VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s);"""
cursor.execute(insert_query, data) cursor.execute(insert_query, data)
connection.commit() connection.commit()
except Exception as e: except Exception as e:
@@ -80,13 +81,17 @@ def upload_to_minio(connection,folder_path,type):
object_name = f"{type}/{folder_name}/{file_name}" object_name = f"{type}/{folder_name}/{file_name}"
try: try:
# 上传文件到 MinIO # 上传文件到 MinIO
minio_client.fput_object(bucket_name, object_name, file_path) url=minio_client.fput_object(bucket_name, object_name, file_path)
print(f"已上传: {file_path} -> {bucket_name}/{object_name}") print(f"已上传: {file_path} -> {bucket_name}/{object_name}")
file_id = get_md5(object_name) # file_id = get_md5(object_name)
file_ids.append(file_id) file_ids.append({file_name:os.path.join(minio_public_url,bucket_name,url.object_name)})
db_file = [file_id, file_name, # db_file = [file_id, file_name,
minio_public_url + bucket_name + '/' + object_name, 'admin', datetime.now()] # minio_public_url + bucket_name + '/' + object_name, 'admin', datetime.now()]
save_to_db_oss_file(connection,db_file) # save_to_db_oss_file(connection,db_file)
# expires = timedelta(days=1)
# presigned_url = minio_client.presigned_get_object(bucket_name, object_name, expires=expires)
# print(f"Presigned URL for {object_name}: {presigned_url}")
except S3Error as err: except S3Error as err:
print(f"上传 {file_name} 时出错: {err}") print(f"上传 {file_name} 时出错: {err}")
return file_ids return file_ids

5
scripts/esa.py
View File

@@ -44,7 +44,7 @@ def scrape():
# 下载文件并获取文件名 # 下载文件并获取文件名
file_response = requests.get(download_url) file_response = requests.get(download_url)
if file_response.status_code == 200: if file_response and file_response.status_code == 200:
# 从响应头获取文件名 # 从响应头获取文件名
content_disposition = file_response.headers.get('Content-Disposition') content_disposition = file_response.headers.get('Content-Disposition')
filename = '' filename = ''
@@ -203,13 +203,14 @@ def scrape():
print(f'Failed to download: {download_url}') print(f'Failed to download: {download_url}')
upload_ids = upload_to_minio(esa_connection, folder_path, 'ESA') upload_ids = upload_to_minio(esa_connection, folder_path, 'ESA')
origin_data = ','.join([c.get_text(strip=True) for c in cells])
data_db = ['ESA-' + cells[0].get_text(strip=True), 'Crawler', datetime.now(), None, None, None, data_db = ['ESA-' + cells[0].get_text(strip=True), 'Crawler', datetime.now(), None, None, None,
cells[5].get_text(strip=True), cells[5].get_text(strip=True),
cells[1].get_text(strip=True), cells[1].get_text(strip=True), cells[1].get_text(strip=True), cells[1].get_text(strip=True),
cells[8].get_text(strip=True), cells[8].get_text(strip=True),
cells[7].get_text(strip=True), cells[2].get_text(strip=True), cells[7].get_text(strip=True), cells[2].get_text(strip=True),
cells[11].get_text(strip=True), cells[11].get_text(strip=True),
'ESA', None, None, ','.join(upload_ids) 'ESA', None, None, str(upload_ids), origin_data
] ]
save_to_db_import_record(esa_connection, data_db) save_to_db_import_record(esa_connection, data_db)
else: else:

14
scripts/nasa1.py
View File

@@ -73,7 +73,9 @@ def scrape():
# 解析 JSON 数据 # 解析 JSON 数据
json_data = response.json() json_data = response.json()
# 遍历数据并下载文件 # 遍历数据并下载文件
for row in json_data['ROWS']: print("total:" + str(json_data['RECORDS']))
for index, row in enumerate(json_data['ROWS']):
print("index:"+str(index))
part_number = row[0] # 部件编号 part_number = row[0] # 部件编号
file_links_str = row[4] # 文件链接 file_links_str = row[4] # 文件链接
@@ -209,7 +211,7 @@ def scrape():
file_response = requests.get(file_url) file_response = requests.get(file_url)
file_response.raise_for_status() file_response.raise_for_status()
# 保存文件 # 保存文件1685+431+314
file_path = os.path.join(folder_path, os.path.basename(file_url)) file_path = os.path.join(folder_path, os.path.basename(file_url))
with open(file_path, 'wb') as file: with open(file_path, 'wb') as file:
file.write(file_response.content) file.write(file_response.content)
@@ -226,15 +228,19 @@ def scrape():
device_batch=None device_batch=None
manufacturer=row[2] manufacturer=row[2]
experiment_date=row[3] experiment_date=row[3]
data_db = [get_md5(row[0]), 'Crawler', datetime.now(), None, None, None, origin_data = ','.join(row)
data_db = [get_md5(''.join([device_name,device_function,manufacturer])), 'Crawler', datetime.now(), None, None, None,
device_type, device_type,
device_name, device_mode, device_name, device_mode,
device_function, device_function,
device_batch, manufacturer, device_batch, manufacturer,
experiment_date, experiment_date,
'NASA', None, None, ','.join(upload_ids) 'NASA', None, None, str(upload_ids),origin_data
] ]
save_to_db_import_record(nasa1_connection, data_db) save_to_db_import_record(nasa1_connection, data_db)
except Exception as e:
print(e)
finally: finally:
# 关闭游标和连接 # 关闭游标和连接
nasa1_connection.close() nasa1_connection.close()

3
scripts/nasa2.py
View File

@@ -198,13 +198,14 @@ def scrape():
device_batch = None device_batch = None
manufacturer = None manufacturer = None
experiment_date = columns[8].text_content().strip() experiment_date = columns[8].text_content().strip()
origin_data = ','.join([c.text_content().strip() for c in columns])
data_db = ['NASA-' + columns[0].text_content().strip(), 'Crawler', datetime.now(), None, None, None, data_db = ['NASA-' + columns[0].text_content().strip(), 'Crawler', datetime.now(), None, None, None,
device_type, device_type,
device_name, device_mode, device_name, device_mode,
device_function, device_function,
device_batch, manufacturer, device_batch, manufacturer,
experiment_date, experiment_date,
'NASA', None, None, ','.join(upload_ids) 'NASA', None, None, str(upload_ids),origin_data
] ]
save_to_db_import_record(nasa2_connection, data_db) save_to_db_import_record(nasa2_connection, data_db)
except Exception as e: except Exception as e: