diff --git a/physical-launcher/src/main/resources/application-dev.yml b/physical-launcher/src/main/resources/application-dev.yml index ddea611..657158a 100644 --- a/physical-launcher/src/main/resources/application-dev.yml +++ b/physical-launcher/src/main/resources/application-dev.yml @@ -230,7 +230,7 @@ jeecg: #webapp文件路径 webapp: /opt/webapp shiro: - excludeUrls: /database/experimentDoc/**,/sys/common/upload,/sys/user/**,/test/jeecgDemo/demo3,/test/jeecgDemo/redisDemo/**,/bigscreen/category/**,/bigscreen/visual/**,/bigscreen/map/**,/jmreport/bigscreen2/** + excludeUrls: /database/experimentDoc/**,/sys/common/upload,/sys/common/download,/sys/user/**,/test/jeecgDemo/demo3,/test/jeecgDemo/redisDemo/**,/bigscreen/category/**,/bigscreen/visual/**,/bigscreen/map/**,/jmreport/bigscreen2/** #阿里云oss存储和大鱼短信秘钥配置 oss: accessKey: ?? diff --git a/physical-launcher/src/main/resources/application-prod.yml b/physical-launcher/src/main/resources/application-prod.yml index 78b9c41..a5fb27e 100644 --- a/physical-launcher/src/main/resources/application-prod.yml +++ b/physical-launcher/src/main/resources/application-prod.yml @@ -231,7 +231,7 @@ jeecg: #webapp文件路径 webapp: /opt/jeecg-boot/webapp shiro: - excludeUrls: /sys/common/upload,/test/jeecgDemo/demo3,/test/jeecgDemo/redisDemo/**,/bigscreen/category/**,/bigscreen/visual/**,/bigscreen/map/**,/jmreport/bigscreen2/**,/api/getUserInfo,/api/sys/common/**,/sys/common/** + excludeUrls: /sys/common/upload,/sys/common/download,/test/jeecgDemo/demo3,/test/jeecgDemo/redisDemo/**,/bigscreen/category/**,/bigscreen/visual/**,/bigscreen/map/**,/jmreport/bigscreen2/**,/api/getUserInfo,/api/sys/common/**,/sys/common/** #阿里云oss存储和大鱼短信秘钥配置 oss: accessKey: ?? diff --git a/physical-launcher/src/main/resources/application-test.yml b/physical-launcher/src/main/resources/application-test.yml index f302e5b..e57c0b3 100644 --- a/physical-launcher/src/main/resources/application-test.yml +++ b/physical-launcher/src/main/resources/application-test.yml @@ -231,7 +231,7 @@ jeecg: #webapp文件路径 webapp: /opt/jeecg-boot/webapp shiro: - excludeUrls: /sys/common/upload,/test/jeecgDemo/demo3,/test/jeecgDemo/redisDemo/**,/bigscreen/category/**,/bigscreen/visual/**,/bigscreen/map/**,/jmreport/bigscreen2/**,/api/getUserInfo,/api/sys/common/**,/sys/common/** + excludeUrls: /sys/common/upload,/sys/common/download,/test/jeecgDemo/demo3,/test/jeecgDemo/redisDemo/**,/bigscreen/category/**,/bigscreen/visual/**,/bigscreen/map/**,/jmreport/bigscreen2/**,/api/getUserInfo,/api/sys/common/**,/sys/common/** #阿里云oss存储和大鱼短信秘钥配置 oss: accessKey: ?? diff --git a/physical-module-system/physical-system-biz/src/main/java/org/jeecg/modules/system/controller/CommonController.java b/physical-module-system/physical-system-biz/src/main/java/org/jeecg/modules/system/controller/CommonController.java index 6fa39aa..511574b 100644 --- a/physical-module-system/physical-system-biz/src/main/java/org/jeecg/modules/system/controller/CommonController.java +++ b/physical-module-system/physical-system-biz/src/main/java/org/jeecg/modules/system/controller/CommonController.java @@ -3,29 +3,27 @@ package org.jeecg.modules.system.controller; import jakarta.servlet.http.HttpServletRequest; import jakarta.servlet.http.HttpServletResponse; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; import org.jeecg.common.api.vo.Result; import org.jeecg.common.constant.CommonConstant; import org.jeecg.common.constant.SymbolConstant; import org.jeecg.common.exception.JeecgBootException; -import org.jeecg.common.util.CommonUtils; +import org.jeecg.common.util.*; import org.jeecg.common.util.filter.SsrfFileTypeFilter; -import org.jeecg.common.util.oConvertUtils; import org.jeecg.modules.oss.entity.OssFile; import org.jeecg.modules.oss.service.IOssFileService; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.util.AntPathMatcher; import org.springframework.util.FileCopyUtils; -import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.PostMapping; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.bind.annotation.*; import org.springframework.web.multipart.MultipartFile; import org.springframework.web.multipart.MultipartHttpServletRequest; import org.springframework.web.servlet.HandlerMapping; import org.springframework.web.servlet.ModelAndView; import java.io.*; +import java.net.URLDecoder; /** *

@@ -72,6 +70,20 @@ public class CommonController { return Result.error("没有权限，请联系管理员授权，后刷新缓存!"); } + @GetMapping(value = "/download") + public void download(@RequestParam(name = "file", defaultValue = "") String file, HttpServletRequest request, + HttpServletResponse response) throws Exception { + if (StringUtils.isBlank(file)) { + return; + } + String fileName = URLDecoder.decode(file, "UTF-8"); + String objectUrl = MinioUtil.getObjectUrl(MinioUtil.getBucketName(), fileName, 2 * 24 * 60 * 60); + if (StringUtils.isBlank(objectUrl)) { + return; + } + response.sendRedirect(objectUrl); + } + /** * 文件上传统一方法 * @@ -146,42 +158,43 @@ public class CommonController { return result; } -// @PostMapping(value = "/upload2") -// public Result upload2(HttpServletRequest request, HttpServletResponse response) { -// Result result = new Result<>(); -// try { -// String ctxPath = uploadpath; -// String fileName = null; -// String bizPath = "files"; -// String tempBizPath = request.getParameter("biz"); -// if(oConvertUtils.isNotEmpty(tempBizPath)){ -// bizPath = tempBizPath; -// } -// String nowday = new SimpleDateFormat("yyyyMMdd").format(new Date()); -// File file = new File(ctxPath + File.separator + bizPath + File.separator + nowday); -// if (!file.exists()) { -// file.mkdirs();// 创建文件根目录 -// } -// MultipartHttpServletRequest multipartRequest = (MultipartHttpServletRequest) request; -// MultipartFile mf = multipartRequest.getFile("file");// 获取上传文件对象 -// String orgName = mf.getOriginalFilename();// 获取文件名 -// fileName = orgName.substring(0, orgName.lastIndexOf(".")) + "_" + System.currentTimeMillis() + orgName.substring(orgName.indexOf(".")); -// String savePath = file.getPath() + File.separator + fileName; -// File savefile = new File(savePath); -// FileCopyUtils.copy(mf.getBytes(), savefile); -// String dbpath = bizPath + File.separator + nowday + File.separator + fileName; -// if (dbpath.contains("\\")) { -// dbpath = dbpath.replace("\\", "/"); -// } -// result.setMessage(dbpath); -// result.setSuccess(true); -// } catch (IOException e) { -// result.setSuccess(false); -// result.setMessage(e.getMessage()); -// log.error(e.getMessage(), e); -// } -// return result; -// } + // @PostMapping(value = "/upload2") + // public Result upload2(HttpServletRequest request, HttpServletResponse response) { + // Result result = new Result<>(); + // try { + // String ctxPath = uploadpath; + // String fileName = null; + // String bizPath = "files"; + // String tempBizPath = request.getParameter("biz"); + // if(oConvertUtils.isNotEmpty(tempBizPath)){ + // bizPath = tempBizPath; + // } + // String nowday = new SimpleDateFormat("yyyyMMdd").format(new Date()); + // File file = new File(ctxPath + File.separator + bizPath + File.separator + nowday); + // if (!file.exists()) { + // file.mkdirs();// 创建文件根目录 + // } + // MultipartHttpServletRequest multipartRequest = (MultipartHttpServletRequest) request; + // MultipartFile mf = multipartRequest.getFile("file");// 获取上传文件对象 + // String orgName = mf.getOriginalFilename();// 获取文件名 + // fileName = orgName.substring(0, orgName.lastIndexOf(".")) + "_" + System.currentTimeMillis() + orgName.substring(orgName + // .indexOf(".")); + // String savePath = file.getPath() + File.separator + fileName; + // File savefile = new File(savePath); + // FileCopyUtils.copy(mf.getBytes(), savefile); + // String dbpath = bizPath + File.separator + nowday + File.separator + fileName; + // if (dbpath.contains("\\")) { + // dbpath = dbpath.replace("\\", "/"); + // } + // result.setMessage(dbpath); + // result.setSuccess(true); + // } catch (IOException e) { + // result.setSuccess(false); + // result.setMessage(e.getMessage()); + // log.error(e.getMessage(), e); + // } + // return result; + // } /** * 本地文件上传 @@ -203,7 +216,8 @@ public class CommonController { String orgName = mf.getOriginalFilename(); orgName = CommonUtils.getFileName(orgName); if (orgName.indexOf(SymbolConstant.SPOT) != -1) { - fileName = orgName.substring(0, orgName.lastIndexOf(".")) + "_" + System.currentTimeMillis() + orgName.substring(orgName.lastIndexOf(".")); + fileName = orgName.substring(0, orgName.lastIndexOf(".")) + "_" + System.currentTimeMillis() + orgName.substring( + orgName.lastIndexOf(".")); } else { fileName = orgName + "_" + System.currentTimeMillis(); } @@ -226,63 +240,64 @@ public class CommonController { return ""; } -// /** -// * 下载文件 -// * 请求地址：http://localhost:8080/common/download/{user/20190119/e1fe9925bc315c60addea1b98eb1cb1349547719_1547866868179.jpg} -// * -// * @param request -// * @param response -// * @throws Exception -// */ -// @GetMapping(value = "/download/**") -// public void download(HttpServletRequest request, HttpServletResponse response) throws Exception { -// // ISO-8859-1 ==> UTF-8 进行编码转换 -// String filePath = extractPathFromPattern(request); -// // 其余处理略 -// InputStream inputStream = null; -// OutputStream outputStream = null; -// try { -// filePath = filePath.replace("..", ""); -// if (filePath.endsWith(",")) { -// filePath = filePath.substring(0, filePath.length() - 1); -// } -// String localPath = uploadpath; -// String downloadFilePath = localPath + File.separator + filePath; -// File file = new File(downloadFilePath); -// if (file.exists()) { -// response.setContentType("application/force-download");// 设置强制下载不打开             -// response.addHeader("Content-Disposition", "attachment;fileName=" + new String(file.getName().getBytes("UTF-8"),"iso-8859-1")); -// inputStream = new BufferedInputStream(new FileInputStream(file)); -// outputStream = response.getOutputStream(); -// byte[] buf = new byte[1024]; -// int len; -// while ((len = inputStream.read(buf)) > 0) { -// outputStream.write(buf, 0, len); -// } -// response.flushBuffer(); -// } -// -// } catch (Exception e) { -// log.info("文件下载失败" + e.getMessage()); -// // e.printStackTrace(); -// } finally { -// if (inputStream != null) { -// try { -// inputStream.close(); -// } catch (IOException e) { -// e.printStackTrace(); -// } -// } -// if (outputStream != null) { -// try { -// outputStream.close(); -// } catch (IOException e) { -// e.printStackTrace(); -// } -// } -// } -// -// } + // /** + // * 下载文件 + // * 请求地址：http://localhost:8080/common/download/{user/20190119/e1fe9925bc315c60addea1b98eb1cb1349547719_1547866868179.jpg} + // * + // * @param request + // * @param response + // * @throws Exception + // */ + // @GetMapping(value = "/download/**") + // public void download(HttpServletRequest request, HttpServletResponse response) throws Exception { + // // ISO-8859-1 ==> UTF-8 进行编码转换 + // String filePath = extractPathFromPattern(request); + // // 其余处理略 + // InputStream inputStream = null; + // OutputStream outputStream = null; + // try { + // filePath = filePath.replace("..", ""); + // if (filePath.endsWith(",")) { + // filePath = filePath.substring(0, filePath.length() - 1); + // } + // String localPath = uploadpath; + // String downloadFilePath = localPath + File.separator + filePath; + // File file = new File(downloadFilePath); + // if (file.exists()) { + // response.setContentType("application/force-download");// 设置强制下载不打开             + // response.addHeader("Content-Disposition", "attachment;fileName=" + new String(file.getName().getBytes("UTF-8"), + // "iso-8859-1")); + // inputStream = new BufferedInputStream(new FileInputStream(file)); + // outputStream = response.getOutputStream(); + // byte[] buf = new byte[1024]; + // int len; + // while ((len = inputStream.read(buf)) > 0) { + // outputStream.write(buf, 0, len); + // } + // response.flushBuffer(); + // } + // + // } catch (Exception e) { + // log.info("文件下载失败" + e.getMessage()); + // // e.printStackTrace(); + // } finally { + // if (inputStream != null) { + // try { + // inputStream.close(); + // } catch (IOException e) { + // e.printStackTrace(); + // } + // } + // if (outputStream != null) { + // try { + // outputStream.close(); + // } catch (IOException e) { + // e.printStackTrace(); + // } + // } + // } + // + // } /** * 预览图片&下载文件 diff --git a/scripts/common.py b/scripts/common.py index 5da7a8d..2782486 100644 --- a/scripts/common.py +++ b/scripts/common.py @@ -1,6 +1,7 @@ import hashlib import os from datetime import datetime +from datetime import timedelta import mysql.connector from minio import Minio @@ -9,10 +10,10 @@ from minio.error import S3Error minio_public_url = 'http://58.215.212.230:8005/oss/' # MySQL 连接配置 db_config = { - 'host': 'physical-mysql', - 'port': 3306, -# 'host': '192.168.50.100', -# 'port': 23306, + # 'host': 'physical-mysql', + # 'port': 3306, + 'host': '192.168.50.100', + 'port': 23306, 'user': 'root', 'password': '123456', 'database': 'physical-boot' @@ -20,8 +21,8 @@ db_config = { # minio 配置 minio_client = Minio( - "physical-minio:9000", # MinIO服务器地址或IP -# "192.168.50.100:29000", # MinIO服务器地址或IP + # "physical-minio:9000", # MinIO服务器地址或IP + "192.168.50.100:29000", # MinIO服务器地址或IP access_key="root", # 替换为你的Access Key secret_key="12345678", # 替换为你的Secret Key secure=False # 如果使用的是http则为False @@ -42,8 +43,8 @@ def save_to_db_import_record(connection,data): cursor = connection.cursor() try: """保存数据到 MySQL 数据库""" - insert_query = """INSERT INTO `import_record` (`id`, `create_by`, `create_time`, `update_by`, `update_time`, `sys_org_code`, `device_type`, `device_name`, `device_mode`, `device_function`, `device_batch`, `manufacturer`, `experiment_date`, `data_source`, `experiment_user`, `total_count`, `file_list`) - VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s);""" + insert_query = """INSERT INTO `nasa_data_record` (`id`, `create_by`, `create_time`, `update_by`, `update_time`, `sys_org_code`, `device_type`, `device_name`, `device_mode`, `device_function`, `device_batch`, `manufacturer`, `experiment_date`, `data_source`, `experiment_user`, `total_count`, `file_list`,`origin_data`) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s);""" cursor.execute(insert_query, data) connection.commit() except Exception as e: @@ -80,13 +81,17 @@ def upload_to_minio(connection,folder_path,type): object_name = f"{type}/{folder_name}/{file_name}" try: # 上传文件到 MinIO - minio_client.fput_object(bucket_name, object_name, file_path) + url=minio_client.fput_object(bucket_name, object_name, file_path) print(f"已上传: {file_path} -> {bucket_name}/{object_name}") - file_id = get_md5(object_name) - file_ids.append(file_id) - db_file = [file_id, file_name, - minio_public_url + bucket_name + '/' + object_name, 'admin', datetime.now()] - save_to_db_oss_file(connection,db_file) + # file_id = get_md5(object_name) + file_ids.append({file_name:os.path.join(minio_public_url,bucket_name,url.object_name)}) + # db_file = [file_id, file_name, + # minio_public_url + bucket_name + '/' + object_name, 'admin', datetime.now()] + # save_to_db_oss_file(connection,db_file) + # expires = timedelta(days=1) + # presigned_url = minio_client.presigned_get_object(bucket_name, object_name, expires=expires) + # print(f"Presigned URL for {object_name}: {presigned_url}") + except S3Error as err: print(f"上传 {file_name} 时出错: {err}") return file_ids diff --git a/scripts/esa.py b/scripts/esa.py index dbfed67..1445f1f 100644 --- a/scripts/esa.py +++ b/scripts/esa.py @@ -44,7 +44,7 @@ def scrape(): # 下载文件并获取文件名 file_response = requests.get(download_url) - if file_response.status_code == 200: + if file_response and file_response.status_code == 200: # 从响应头获取文件名 content_disposition = file_response.headers.get('Content-Disposition') filename = '' @@ -202,16 +202,17 @@ def scrape(): else: print(f'Failed to download: {download_url}') - upload_ids = upload_to_minio(esa_connection, folder_path,'ESA') - data_db = ['ESA-' + cells[0].get_text(strip=True), 'Crawler', datetime.now(), None, None, None, - cells[5].get_text(strip=True), - cells[1].get_text(strip=True), cells[1].get_text(strip=True), - cells[8].get_text(strip=True), - cells[7].get_text(strip=True), cells[2].get_text(strip=True), - cells[11].get_text(strip=True), - 'ESA', None, None, ','.join(upload_ids) - ] - save_to_db_import_record(esa_connection, data_db) + upload_ids = upload_to_minio(esa_connection, folder_path, 'ESA') + origin_data = ','.join([c.get_text(strip=True) for c in cells]) + data_db = ['ESA-' + cells[0].get_text(strip=True), 'Crawler', datetime.now(), None, None, None, + cells[5].get_text(strip=True), + cells[1].get_text(strip=True), cells[1].get_text(strip=True), + cells[8].get_text(strip=True), + cells[7].get_text(strip=True), cells[2].get_text(strip=True), + cells[11].get_text(strip=True), + 'ESA', None, None, str(upload_ids), origin_data + ] + save_to_db_import_record(esa_connection, data_db) else: print(f'Error: {response.status_code}') finally: diff --git a/scripts/nasa1.py b/scripts/nasa1.py index bf326a6..e0b633d 100644 --- a/scripts/nasa1.py +++ b/scripts/nasa1.py @@ -73,7 +73,9 @@ def scrape(): # 解析 JSON 数据 json_data = response.json() # 遍历数据并下载文件 - for row in json_data['ROWS']: + print("total:" + str(json_data['RECORDS'])) + for index, row in enumerate(json_data['ROWS']): + print("index:"+str(index)) part_number = row[0] # 部件编号 file_links_str = row[4] # 文件链接 @@ -209,7 +211,7 @@ def scrape(): file_response = requests.get(file_url) file_response.raise_for_status() - # 保存文件 + # 保存文件1685+431+314 file_path = os.path.join(folder_path, os.path.basename(file_url)) with open(file_path, 'wb') as file: file.write(file_response.content) @@ -226,15 +228,19 @@ def scrape(): device_batch=None manufacturer=row[2] experiment_date=row[3] - data_db = [get_md5(row[0]), 'Crawler', datetime.now(), None, None, None, + origin_data = ','.join(row) + + data_db = [get_md5(''.join([device_name,device_function,manufacturer])), 'Crawler', datetime.now(), None, None, None, device_type, device_name, device_mode, device_function, device_batch, manufacturer, experiment_date, - 'NASA', None, None, ','.join(upload_ids) + 'NASA', None, None, str(upload_ids),origin_data ] save_to_db_import_record(nasa1_connection, data_db) + except Exception as e: + print(e) finally: # 关闭游标和连接 nasa1_connection.close() diff --git a/scripts/nasa2.py b/scripts/nasa2.py index 932b4be..d3a6061 100644 --- a/scripts/nasa2.py +++ b/scripts/nasa2.py @@ -198,13 +198,14 @@ def scrape(): device_batch = None manufacturer = None experiment_date = columns[8].text_content().strip() + origin_data = ','.join([c.text_content().strip() for c in columns]) data_db = ['NASA-' + columns[0].text_content().strip(), 'Crawler', datetime.now(), None, None, None, device_type, device_name, device_mode, device_function, device_batch, manufacturer, experiment_date, - 'NASA', None, None, ','.join(upload_ids) + 'NASA', None, None, str(upload_ids),origin_data ] save_to_db_import_record(nasa2_connection, data_db) except Exception as e: