diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml index 2ef7ec9..6d06f86 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.yml +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -1,5 +1,5 @@ name: "💞 Feature request" -description: Propose a feature request for RAGFlow. +description: Propose a feature request for RAGFlow-Plus. title: "[Feature Request]: " labels: ["💞 feature"] body: diff --git a/management/server/clean_all_data.py b/management/server/clean_all_data.py new file mode 100644 index 0000000..a493c40 --- /dev/null +++ b/management/server/clean_all_data.py @@ -0,0 +1,141 @@ +import os +from dotenv import load_dotenv +import mysql.connector +from minio import Minio + +# 加载环境变量 +load_dotenv("../../docker/.env") + +# 数据库连接配置 +DB_CONFIG = { + "host": "localhost", + "port": int(os.getenv("MYSQL_PORT", "5455")), + "user": "root", + "password": os.getenv("MYSQL_PASSWORD", "infini_rag_flow"), + "database": "rag_flow" +} + +# MinIO连接配置 +MINIO_CONFIG = { + "endpoint": "localhost:" + os.getenv("MINIO_PORT", "9000"), + "access_key": os.getenv("MINIO_USER", "rag_flow"), + "secret_key": os.getenv("MINIO_PASSWORD", "infini_rag_flow"), + "secure": False +} + +def get_minio_client(): + """创建MinIO客户端""" + return Minio( + endpoint=MINIO_CONFIG["endpoint"], + access_key=MINIO_CONFIG["access_key"], + secret_key=MINIO_CONFIG["secret_key"], + secure=MINIO_CONFIG["secure"] + ) + +def clear_database_tables(): + """清空数据库表""" + try: + conn = mysql.connector.connect(**DB_CONFIG) + cursor = conn.cursor() + + # 禁用外键检查 + cursor.execute("SET FOREIGN_KEY_CHECKS = 0") + + # 清空表数据 + tables = ["document", "file2document", "file"] + for table in tables: + cursor.execute(f"TRUNCATE TABLE {table}") + print(f"已清空表: {table}") + + # 启用外键检查 + cursor.execute("SET FOREIGN_KEY_CHECKS = 1") + + conn.commit() + cursor.close() + conn.close() + + print("数据库表已全部清空") + + except Exception as e: + print(f"清空数据库表失败: {str(e)}") + raise + +def clear_minio_buckets(): + """清空并删除MinIO所有存储桶""" + try: + minio_client = get_minio_client() + buckets = minio_client.list_buckets() + + if not buckets: + print("MinIO中没有存储桶需要清理") + return + + print(f"开始清理 {len(buckets)} 个MinIO存储桶...") + + for bucket in buckets: + bucket_name = bucket.name + + # 跳过系统保留的存储桶 + if bucket_name.startswith('.'): + print(f"跳过系统存储桶: {bucket_name}") + continue + + try: + # 递归删除存储桶中的所有对象(包括版本控制对象) + objects = minio_client.list_objects(bucket_name, recursive=True) + for obj in objects: + try: + # 强制删除对象(包括所有版本) + minio_client.remove_object(bucket_name, obj.object_name, version_id=obj.version_id) + except Exception as e: + print(f"删除对象 {obj.object_name} 失败: {str(e)}") + continue + + # 确保所有对象已删除 + while True: + remaining_objects = list(minio_client.list_objects(bucket_name)) + if not remaining_objects: + break + for obj in remaining_objects: + minio_client.remove_object(bucket_name, obj.object_name) + + # 实际删除存储桶 + try: + minio_client.remove_bucket(bucket_name) + print(f"已删除存储桶: {bucket_name}") + except Exception as e: + print(f"删除存储桶 {bucket_name} 失败: {str(e)}. 尝试强制删除...") + # 强制删除存储桶(即使非空) + try: + # 再次确保删除所有对象 + objects = minio_client.list_objects(bucket_name, recursive=True) + for obj in objects: + minio_client.remove_object(bucket_name, obj.object_name) + minio_client.remove_bucket(bucket_name) + print(f"已强制删除存储桶: {bucket_name}") + except Exception as e: + print(f"强制删除存储桶 {bucket_name} 仍然失败: {str(e)}") + + except Exception as e: + print(f"处理存储桶 {bucket_name} 时发生错误: {str(e)}") + + print("MinIO存储桶清理完成") + + except Exception as e: + print(f"清理MinIO存储桶失败: {str(e)}") + raise + +def confirm_action(): + """确认操作""" + print("警告: 此操作将永久删除所有数据!") + confirmation = input("确认要清空所有数据吗? (输入'y'确认): ") + return confirmation.lower() == 'y' + +if __name__ == "__main__": + if confirm_action(): + print("开始清理数据...") + clear_database_tables() + clear_minio_buckets() + print("数据清理完成") + else: + print("操作已取消") \ No newline at end of file diff --git a/management/server/cleanup_minio_buckets.py b/management/server/cleanup_minio_buckets.py new file mode 100644 index 0000000..715f15a --- /dev/null +++ b/management/server/cleanup_minio_buckets.py @@ -0,0 +1,91 @@ +import os +from dotenv import load_dotenv +import mysql.connector +from minio import Minio + +# 加载环境变量 +load_dotenv("../../docker/.env") + +# 数据库连接配置 +DB_CONFIG = { + "host": "localhost", + "port": int(os.getenv("MYSQL_PORT", "5455")), + "user": "root", + "password": os.getenv("MYSQL_PASSWORD", "infini_rag_flow"), + "database": "rag_flow" +} + +# MinIO连接配置 +MINIO_CONFIG = { + "endpoint": "localhost:" + os.getenv("MINIO_PORT", "9000"), + "access_key": os.getenv("MINIO_USER", "rag_flow"), + "secret_key": os.getenv("MINIO_PASSWORD", "infini_rag_flow"), + "secure": False +} + +def get_used_buckets_from_db(): + """从数据库获取正在使用的存储桶(kb_id)列表""" + try: + conn = mysql.connector.connect(**DB_CONFIG) + cursor = conn.cursor() + + # 查询所有不重复的kb_id + cursor.execute("SELECT DISTINCT kb_id FROM document") + kb_ids = [row[0] for row in cursor.fetchall()] + + cursor.close() + conn.close() + + return kb_ids + + except Exception as e: + print(f"数据库查询失败: {str(e)}") + return [] + +def cleanup_unused_buckets(): + """清理未使用的MinIO存储桶""" + try: + # 获取MinIO客户端 + minio_client = Minio( + endpoint=MINIO_CONFIG["endpoint"], + access_key=MINIO_CONFIG["access_key"], + secret_key=MINIO_CONFIG["secret_key"], + secure=MINIO_CONFIG["secure"] + ) + + # 获取数据库中的有效kb_id列表 + used_buckets = set(get_used_buckets_from_db()) + + # 获取MinIO中的所有存储桶 + all_buckets = minio_client.list_buckets() + minio_bucket_names = {bucket.name for bucket in all_buckets} + + # 计算需要删除的存储桶 + buckets_to_delete = minio_bucket_names - used_buckets + + if not buckets_to_delete: + print("没有需要删除的多余存储桶") + return + + print(f"发现 {len(buckets_to_delete)} 个多余存储桶需要清理:") + + # 删除多余的存储桶 + for bucket_name in buckets_to_delete: + try: + # 先确保存储桶为空 + objects = minio_client.list_objects(bucket_name) + for obj in objects: + minio_client.remove_object(bucket_name, obj.object_name) + + # 删除存储桶 + minio_client.remove_bucket(bucket_name) + print(f"已删除存储桶: {bucket_name}") + + except Exception as e: + print(f"删除存储桶 {bucket_name} 失败: {str(e)}") + + except Exception as e: + print(f"清理存储桶过程中发生错误: {str(e)}") + +if __name__ == "__main__": + cleanup_unused_buckets() \ No newline at end of file diff --git a/management/server/minio_test.py b/management/server/minio_test.py new file mode 100644 index 0000000..086f81c --- /dev/null +++ b/management/server/minio_test.py @@ -0,0 +1,214 @@ +import os +import mysql.connector +from tabulate import tabulate +from dotenv import load_dotenv +from minio import Minio +from io import BytesIO + +# 加载环境变量 +load_dotenv("../../docker/.env") + +# 数据库连接配置 +DB_CONFIG = { + "host": "localhost", # 如果在Docker外运行，使用localhost + "port": int(os.getenv("MYSQL_PORT", "5455")), + "user": "root", + "password": os.getenv("MYSQL_PASSWORD", "infini_rag_flow"), + "database": "rag_flow" +} + +# MinIO连接配置 +MINIO_CONFIG = { + "endpoint": "localhost:" + os.getenv("MINIO_PORT", "9000"), + "access_key": os.getenv("MINIO_USER", "rag_flow"), + "secret_key": os.getenv("MINIO_PASSWORD", "infini_rag_flow"), + "secure": False +} + +def get_all_documents(): + """获取所有文档信息及其在MinIO中的存储位置""" + try: + # 连接到MySQL数据库 + conn = mysql.connector.connect(**DB_CONFIG) + cursor = conn.cursor(dictionary=True) + + # 首先获取document表的列信息 + cursor.execute("SHOW COLUMNS FROM document") + columns = [column['Field'] for column in cursor.fetchall()] + + # 构建动态查询语句，只选择存在的列 + select_fields = [] + for field in ['id', 'name', 'kb_id', 'location', 'size', 'type', 'created_by', 'create_time']: + if field in columns: + select_fields.append(f'd.{field}') + + # 添加可选字段 + optional_fields = ['token_count', 'chunk_count'] + for field in optional_fields: + if field in columns: + select_fields.append(f'd.{field}') + + # 构建并执行查询 + query = f""" + SELECT {', '.join(select_fields)} + FROM document d + ORDER BY d.create_time DESC + """ + cursor.execute(query) + + documents = cursor.fetchall() + + # 获取文档与文件的关联信息 + cursor.execute(""" + SELECT f2d.document_id, f.id as file_id, f.parent_id, f.source_type + FROM file2document f2d + JOIN file f ON f2d.file_id = f.id + """) + + file_mappings = {} + for row in cursor.fetchall(): + file_mappings[row['document_id']] = { + 'file_id': row['file_id'], + 'parent_id': row['parent_id'], + 'source_type': row['source_type'] + } + + # 整合信息 + result = [] + for doc in documents: + doc_id = doc['id'] + kb_id = doc['kb_id'] + location = doc['location'] + + # 确定存储位置 + storage_bucket = kb_id + storage_location = location + + # 如果有文件映射，检查是否需要使用文件的parent_id作为bucket + if doc_id in file_mappings: + file_info = file_mappings[doc_id] + # 模拟File2DocumentService.get_storage_address的逻辑 + if file_info.get('source_type') is None or file_info.get('source_type') == 0: # LOCAL + storage_bucket = file_info['parent_id'] + + # 构建MinIO存储路径 + minio_path = f"{storage_bucket}/{storage_location}" + + # 构建结果字典，只包含存在的字段 + result_item = { + 'id': doc_id, + 'name': doc.get('name', ''), + 'kb_id': kb_id, + 'size': doc.get('size', 0), + 'type': doc.get('type', ''), + 'minio_path': minio_path, + 'storage_bucket': storage_bucket, + 'storage_location': storage_location + } + + # 添加可选字段 + if 'token_count' in doc: + result_item['token_count'] = doc['token_count'] + if 'chunk_count' in doc: + result_item['chunk_count'] = doc['chunk_count'] + + result.append(result_item) + + cursor.close() + conn.close() + + return result + + except Exception as e: + print(f"Error: {e}") + return [] + +def download_document_from_minio(bucket, object_name, output_path): + """从MinIO下载文档""" + try: + # 创建MinIO客户端 + minio_client = Minio( + endpoint=MINIO_CONFIG["endpoint"], + access_key=MINIO_CONFIG["access_key"], + secret_key=MINIO_CONFIG["secret_key"], + secure=MINIO_CONFIG["secure"] + ) + + # 检查bucket是否存在 + if not minio_client.bucket_exists(bucket): + print(f"错误: Bucket '{bucket}' 不存在") + return False + + # 下载文件 + print(f"正在从MinIO下载: {bucket}/{object_name} 到 {output_path}") + minio_client.fget_object(bucket, object_name, output_path) + print(f"文件已成功下载到: {output_path}") + return True + + except Exception as e: + print(f"下载文件时出错: {e}") + return False + +def main(): + """主函数""" + documents = get_all_documents() + + if not documents: + print("未找到任何文档信息") + return + + # 使用tabulate打印表格 + # 动态确定表头 + sample_doc = documents[0] + headers = ['ID', '文档名', '数据集ID', '大小(字节)', '类型', 'MinIO路径'] + if 'token_count' in sample_doc: + headers.insert(-1, 'Token数') + if 'chunk_count' in sample_doc: + headers.insert(-1, '块数') + + # 构建表格数据 + table_data = [] + for doc in documents: + row = [ + doc['id'], + doc['name'], + doc['kb_id'], + doc['size'], + doc['type'] + ] + + if 'token_count' in doc: + row.append(doc['token_count']) + if 'chunk_count' in doc: + row.append(doc['chunk_count']) + + row.append(doc['minio_path']) + table_data.append(row) + + print(tabulate(table_data, headers=headers, tablefmt="grid")) + print(f"总计: {len(documents)}个文档") + + # 下载第一个文档 + if documents: + first_doc = documents[0] + doc_name = first_doc['name'] + bucket = first_doc['storage_bucket'] + object_name = first_doc['storage_location'] + + # 创建下载目录 + download_dir = "downloads" + os.makedirs(download_dir, exist_ok=True) + + # 构建输出文件路径 + output_path = os.path.join(download_dir, doc_name) + + # 下载文件 + # success = download_document_from_minio(bucket, object_name, output_path) + # if success: + # print(f"\n已成功下载第一个文档: {doc_name}") + # print(f"文件保存在: {os.path.abspath(output_path)}") + # else: + # print("\n下载第一个文档失败") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/management/server/routes/__init__.py b/management/server/routes/__init__.py index c86b2b9..b12491c 100644 --- a/management/server/routes/__init__.py +++ b/management/server/routes/__init__.py @@ -5,14 +5,18 @@ from flask import Blueprint users_bp = Blueprint('users', __name__, url_prefix='/api/v1/users') teams_bp = Blueprint('teams', __name__, url_prefix='/api/v1/teams') tenants_bp = Blueprint('tenants', __name__, url_prefix='/api/v1/tenants') +files_bp = Blueprint('files', __name__, url_prefix='/api/v1/files') # 导入路由 from .users.routes import * from .teams.routes import * from .tenants.routes import * +from .files.routes import * + def register_routes(app): """注册所有路由蓝图到应用""" app.register_blueprint(users_bp) app.register_blueprint(teams_bp) - app.register_blueprint(tenants_bp) \ No newline at end of file + app.register_blueprint(tenants_bp) + app.register_blueprint(files_bp) \ No newline at end of file diff --git a/management/server/routes/files/routes.py b/management/server/routes/files/routes.py new file mode 100644 index 0000000..ba4bc54 --- /dev/null +++ b/management/server/routes/files/routes.py @@ -0,0 +1,177 @@ +from flask import jsonify, request, send_file, current_app +from io import BytesIO +from .. import files_bp +from services.files.service import ( + get_files_list, + get_file_info, + download_file_from_minio, + delete_file, + batch_delete_files, + get_minio_client +) + +@files_bp.route('', methods=['GET', 'OPTIONS']) +def get_files(): + """获取文件列表的API端点""" + if request.method == 'OPTIONS': + return '', 200 + + try: + current_page = int(request.args.get('currentPage', 1)) + page_size = int(request.args.get('size', 10)) + name_filter = request.args.get('name', '') + + result, total = get_files_list(current_page, page_size, name_filter) + + return jsonify({ + "code": 0, + "data": { + "list": result, + "total": total + }, + "message": "获取文件列表成功" + }) + + except Exception as e: + return jsonify({ + "code": 500, + "message": f"获取文件列表失败: {str(e)}" + }), 500 + +@files_bp.route('//download', methods=['GET', 'OPTIONS']) +def download_file(file_id): + try: + current_app.logger.info(f"开始处理文件下载请求: {file_id}") + document, _, storage_bucket, storage_location = get_file_info(file_id) + + if not document: + current_app.logger.error(f"文件不存在: {file_id}") + return jsonify({ + "code": 404, + "message": f"文件 {file_id} 不存在", + "details": "文件记录不存在或已被删除" + }), 404 + + current_app.logger.info(f"文件信息获取成功: {file_id}, 存储位置: {storage_bucket}/{storage_location}") + + try: + minio_client = get_minio_client() + current_app.logger.info(f"MinIO客户端创建成功, 准备检查文件: {storage_bucket}/{storage_location}") + + obj = minio_client.stat_object(storage_bucket, storage_location) + if not obj: + current_app.logger.error(f"文件对象为空: {storage_bucket}/{storage_location}") + return jsonify({ + "code": 404, + "message": "文件内容为空", + "details": "MinIO存储桶中存在文件记录但内容为空" + }), 404 + + if obj.size == 0: + current_app.logger.error(f"文件大小为0: {storage_bucket}/{storage_location}") + return jsonify({ + "code": 404, + "message": "文件内容为空", + "details": "MinIO存储桶中文件大小为0" + }), 404 + + current_app.logger.info(f"文件检查成功, 大小: {obj.size} 字节, 准备下载") + + response = minio_client.get_object(storage_bucket, storage_location) + file_data = response.read() + + current_app.logger.info(f"文件读取成功, 大小: {len(file_data)} 字节, 准备发送") + + return send_file( + BytesIO(file_data), + mimetype='application/octet-stream', + as_attachment=True, + download_name=document['name'] + ) + + except Exception as e: + current_app.logger.error(f"MinIO操作异常: {str(e)}", exc_info=True) + # 检查是否是连接错误 + if "connection" in str(e).lower(): + return jsonify({ + "code": 503, + "message": "存储服务连接失败", + "details": f"无法连接到MinIO服务: {str(e)}" + }), 503 + # 检查是否是权限错误 + elif "access denied" in str(e).lower() or "permission" in str(e).lower(): + return jsonify({ + "code": 403, + "message": "存储服务访问被拒绝", + "details": f"MinIO访问权限错误: {str(e)}" + }), 403 + # 其他错误 + else: + return jsonify({ + "code": 500, + "message": "存储服务异常", + "details": str(e) + }), 500 + + except Exception as e: + current_app.logger.error(f"文件下载异常: {str(e)}", exc_info=True) + return jsonify({ + "code": 500, + "message": "文件下载失败", + "details": str(e) + }), 500 + +@files_bp.route('/', methods=['DELETE', 'OPTIONS']) +def delete_file_route(file_id): + """删除文件的API端点""" + if request.method == 'OPTIONS': + return '', 200 + + try: + success = delete_file(file_id) + + if success: + return jsonify({ + "code": 0, + "message": "文件删除成功" + }) + else: + return jsonify({ + "code": 404, + "message": f"文件 {file_id} 不存在" + }), 404 + + except Exception as e: + return jsonify({ + "code": 500, + "message": f"删除文件失败: {str(e)}" + }), 500 + +@files_bp.route('/batch', methods=['DELETE', 'OPTIONS']) +def batch_delete_files_route(): + """批量删除文件的API端点""" + if request.method == 'OPTIONS': + return '', 200 + + try: + data = request.json + file_ids = data.get('ids', []) + + if not file_ids: + return jsonify({ + "code": 400, + "message": "未提供要删除的文件ID" + }), 400 + + success_count = batch_delete_files(file_ids) + + return jsonify({ + "code": 0, + "message": f"成功删除 {success_count}/{len(file_ids)} 个文件" + }) + + except Exception as e: + return jsonify({ + "code": 500, + "message": f"批量删除文件失败: {str(e)}" + }), 500 \ No newline at end of file diff --git a/management/server/services/files/service.py b/management/server/services/files/service.py new file mode 100644 index 0000000..17de3c8 --- /dev/null +++ b/management/server/services/files/service.py @@ -0,0 +1,400 @@ +import os +import mysql.connector +from io import BytesIO +from minio import Minio +from dotenv import load_dotenv + +# 加载环境变量 +load_dotenv("../../docker/.env") + +# 数据库连接配置 +DB_CONFIG = { + "host": "localhost", # 如果在Docker外运行，使用localhost + "port": int(os.getenv("MYSQL_PORT", "5455")), + "user": "root", + "password": os.getenv("MYSQL_PASSWORD", "infini_rag_flow"), + "database": "rag_flow" +} + +# MinIO连接配置 +MINIO_CONFIG = { + "endpoint": "localhost:" + os.getenv("MINIO_PORT", "9000"), + "access_key": os.getenv("MINIO_USER", "rag_flow"), + "secret_key": os.getenv("MINIO_PASSWORD", "infini_rag_flow"), + "secure": False +} + +def get_minio_client(): + """创建MinIO客户端""" + return Minio( + endpoint=MINIO_CONFIG["endpoint"], + access_key=MINIO_CONFIG["access_key"], + secret_key=MINIO_CONFIG["secret_key"], + secure=MINIO_CONFIG["secure"] + ) + +def get_db_connection(): + """创建数据库连接""" + return mysql.connector.connect(**DB_CONFIG) + +def get_files_list(current_page, page_size, name_filter=""): + """ + 获取文件列表 + + Args: + current_page: 当前页码 + page_size: 每页大小 + name_filter: 文件名过滤条件 + + Returns: + tuple: (文件列表, 总数) + """ + try: + # 计算偏移量 + offset = (current_page - 1) * page_size + + # 连接数据库 + conn = get_db_connection() + cursor = conn.cursor(dictionary=True) + + # 构建查询条件 + where_clause = "" + params = [] + + if name_filter: + where_clause = "WHERE d.name LIKE %s" + params.append(f"%{name_filter}%") + + # 查询总数 + count_query = f""" + SELECT COUNT(*) as total + FROM document d + {where_clause} + """ + cursor.execute(count_query, params) + total = cursor.fetchone()['total'] + + # 查询文件列表 + query = f""" + SELECT d.id, d.name, d.kb_id, d.location, d.size, d.type, d.create_time + FROM document d + {where_clause} + ORDER BY d.create_time DESC + LIMIT %s OFFSET %s + """ + cursor.execute(query, params + [page_size, offset]) + documents = cursor.fetchall() + + # 获取文档与文件的关联信息 + doc_ids = [doc['id'] for doc in documents] + file_mappings = {} + + if doc_ids: + placeholders = ', '.join(['%s'] * len(doc_ids)) + cursor.execute(f""" + SELECT f2d.document_id, f.id as file_id, f.parent_id, f.source_type + FROM file2document f2d + JOIN file f ON f2d.file_id = f.id + WHERE f2d.document_id IN ({placeholders}) + """, doc_ids) + + for row in cursor.fetchall(): + file_mappings[row['document_id']] = { + 'file_id': row['file_id'], + 'parent_id': row['parent_id'], + 'source_type': row['source_type'] + } + + # 整合信息 + result = [] + for doc in documents: + doc_id = doc['id'] + kb_id = doc['kb_id'] + location = doc['location'] + + # 确定存储位置 + storage_bucket = kb_id + storage_location = location + + # 如果有文件映射，检查是否需要使用文件的parent_id作为bucket + if doc_ids and doc_id in file_mappings: + file_info = file_mappings[doc_id] + # 模拟File2DocumentService.get_storage_address的逻辑 + if file_info.get('source_type') is None or file_info.get('source_type') == 0: # LOCAL + storage_bucket = file_info['parent_id'] + + # 构建结果字典 + result_item = { + 'id': doc_id, + 'name': doc.get('name', ''), + 'kb_id': kb_id, + 'size': doc.get('size', 0), + 'type': doc.get('type', ''), + 'location': location, + 'create_time': doc.get('create_time', 0) + } + + result.append(result_item) + + cursor.close() + conn.close() + + return result, total + + except Exception as e: + raise e + +def get_file_info(file_id): + """ + 获取文件信息 + + Args: + file_id: 文件ID + + Returns: + tuple: (文档信息, 文件映射信息, 存储桶, 存储位置) + """ + try: + # 连接数据库 + conn = get_db_connection() + cursor = conn.cursor(dictionary=True) + + # 查询文档信息 + cursor.execute(""" + SELECT d.id, d.name, d.kb_id, d.location, d.type + FROM document d + WHERE d.id = %s + """, (file_id,)) + + document = cursor.fetchone() + if not document: + cursor.close() + conn.close() + return None, None, None, None + + # 获取文档与文件的关联信息 + cursor.execute(""" + SELECT f2d.document_id, f.id as file_id, f.parent_id, f.source_type + FROM file2document f2d + JOIN file f ON f2d.file_id = f.id + WHERE f2d.document_id = %s + """, (file_id,)) + + file_mapping = cursor.fetchone() + + # 确定存储位置 + storage_bucket = document['kb_id'] + storage_location = document['location'] + + # 如果有文件映射，检查是否需要使用文件的parent_id作为bucket + if file_mapping: + # 模拟File2DocumentService.get_storage_address的逻辑 + if file_mapping.get('source_type') is None or file_mapping.get('source_type') == 0: # LOCAL + storage_bucket = file_mapping['parent_id'] + + cursor.close() + conn.close() + + return document, file_mapping, storage_bucket, storage_location + + except Exception as e: + raise e + +def download_file_from_minio(storage_bucket, storage_location): + """ + 从MinIO下载文件 + + Args: + storage_bucket: 存储桶 + storage_location: 存储位置 + + Returns: + bytes: 文件数据 + """ + try: + # 从MinIO下载文件 + minio_client = get_minio_client() + + # 检查bucket是否存在 + if not minio_client.bucket_exists(storage_bucket): + raise Exception(f"存储桶 {storage_bucket} 不存在") + + # 下载文件 + response = minio_client.get_object(storage_bucket, storage_location) + file_data = response.read() + + return file_data + + except Exception as e: + raise e + +def delete_file(file_id): + """ + 删除文件 + + Args: + file_id: 文件ID + + Returns: + bool: 是否删除成功 + """ + try: + # 获取文件信息 + document, file_mapping, storage_bucket, storage_location = get_file_info(file_id) + + if not document: + return False + + # 连接数据库 + conn = get_db_connection() + cursor = conn.cursor(dictionary=True) + + # 如果有文件映射，获取文件ID + file_id_to_delete = None + if file_mapping: + file_id_to_delete = file_mapping['file_id'] + + # 开始事务 + conn.start_transaction() + + try: + # 1. 删除document表中的记录 + cursor.execute("DELETE FROM document WHERE id = %s", (file_id,)) + + # 2. 如果有关联的file2document记录，删除它 + if file_mapping: + cursor.execute("DELETE FROM file2document WHERE document_id = %s", (file_id,)) + + # 3. 如果有关联的file记录，删除它 + if file_id_to_delete: + cursor.execute("DELETE FROM file WHERE id = %s", (file_id_to_delete,)) + + # 提交事务 + conn.commit() + + # 从MinIO删除文件 + try: + minio_client = get_minio_client() + + # 检查bucket是否存在 + if minio_client.bucket_exists(storage_bucket): + # 删除文件 + minio_client.remove_object(storage_bucket, storage_location) + except Exception as e: + # 即使MinIO删除失败，也不影响数据库操作的成功 + print(f"从MinIO删除文件失败: {str(e)}") + + return True + + except Exception as e: + # 回滚事务 + conn.rollback() + raise e + + finally: + cursor.close() + conn.close() + + except Exception as e: + raise e + +def batch_delete_files(file_ids): + """ + 批量删除文件 + + Args: + file_ids: 文件ID列表 + + Returns: + int: 成功删除的文件数量 + """ + if not file_ids: + return 0 + + try: + # 连接数据库 + conn = get_db_connection() + cursor = conn.cursor(dictionary=True) + + # 创建MinIO客户端 + minio_client = get_minio_client() + + # 开始事务 + conn.start_transaction() + + try: + success_count = 0 + + for file_id in file_ids: + # 查询文档信息 + cursor.execute(""" + SELECT d.id, d.kb_id, d.location + FROM document d + WHERE d.id = %s + """, (file_id,)) + + document = cursor.fetchone() + if not document: + continue + + # 获取文档与文件的关联信息 + cursor.execute(""" + SELECT f2d.id as f2d_id, f2d.document_id, f2d.file_id, f.parent_id, f.source_type + FROM file2document f2d + JOIN file f ON f2d.file_id = f.id + WHERE f2d.document_id = %s + """, (file_id,)) + + file_mapping = cursor.fetchone() + + # 确定存储位置 + storage_bucket = document['kb_id'] + storage_location = document['location'] + + # 如果有文件映射，检查是否需要使用文件的parent_id作为bucket + file_id_to_delete = None + if file_mapping: + file_id_to_delete = file_mapping['file_id'] + # 模拟File2DocumentService.get_storage_address的逻辑 + if file_mapping.get('source_type') is None or file_mapping.get('source_type') == 0: # LOCAL + storage_bucket = file_mapping['parent_id'] + + # 1. 删除document表中的记录 + cursor.execute("DELETE FROM document WHERE id = %s", (file_id,)) + + # 2. 如果有关联的file2document记录，删除它 + if file_mapping: + cursor.execute("DELETE FROM file2document WHERE id = %s", (file_mapping['f2d_id'],)) + + # 3. 如果有关联的file记录，删除它 + if file_id_to_delete: + cursor.execute("DELETE FROM file WHERE id = %s", (file_id_to_delete,)) + + # 从MinIO删除文件 + try: + # 检查bucket是否存在 + if minio_client.bucket_exists(storage_bucket): + # 删除文件 + minio_client.remove_object(storage_bucket, storage_location) + except Exception as e: + # 即使MinIO删除失败，也不影响数据库操作的成功 + print(f"从MinIO删除文件失败: {str(e)}") + + success_count += 1 + + # 提交事务 + conn.commit() + + return success_count + + except Exception as e: + # 回滚事务 + conn.rollback() + raise e + + finally: + cursor.close() + conn.close() + + except Exception as e: + raise e \ No newline at end of file diff --git a/management/web/src/common/apis/files/index.ts b/management/web/src/common/apis/files/index.ts new file mode 100644 index 0000000..0f9f8e7 --- /dev/null +++ b/management/web/src/common/apis/files/index.ts @@ -0,0 +1,93 @@ +import type { AxiosResponse } from "axios" +import type { FileData, PageQuery, PageResult } from "./type" +import { request } from "@/http/axios" +import axios from "axios" + +/** + * 获取文件列表 + * @param params 查询参数 + */ +export function getFileListApi(params: PageQuery & { name?: string }) { + return request<{ data: PageResult, code: number, message: string }>({ + url: "/api/v1/files", + method: "get", + params + }) +} + +/** + * 下载文件 - 使用流式下载 + * @param fileId 文件ID + * @param onDownloadProgress 下载进度回调 + */ +export function downloadFileApi( + fileId: string, + onDownloadProgress?: (progressEvent: any) => void +): Promise> { + console.log(`发起文件下载请求: ${fileId}`) + const source = axios.CancelToken.source() + + return request({ + url: `/api/v1/files/${fileId}/download`, + method: "get", + responseType: "blob", + timeout: 300000, + onDownloadProgress, + cancelToken: source.token, + validateStatus: (_status) => { + // 允许所有状态码，以便在前端统一处理错误 + return true + } + }).then((response: unknown) => { + const axiosResponse = response as AxiosResponse + console.log(`下载响应: ${axiosResponse.status}`, axiosResponse.data) + // 确保响应对象包含必要的属性 + if (axiosResponse.data instanceof Blob && axiosResponse.data.size > 0) { + // 如果是成功的Blob响应，确保状态码为200 + if (!axiosResponse.status) axiosResponse.status = 200 + return axiosResponse + } + return axiosResponse + }).catch((error: any) => { + console.error("下载请求失败:", error) + // 将错误信息转换为统一格式 + if (error.response) { + error.response.data = { + message: error.response.data?.message || "服务器错误" + } + } + return Promise.reject(error) + }) as Promise> +} + +/** + * 取消下载 + */ +export function cancelDownload() { + if (axios.isCancel(Error)) { + axios.CancelToken.source().cancel("用户取消下载") + } +} + +/** + * 删除文件 + * @param fileId 文件ID + */ +export function deleteFileApi(fileId: string) { + return request<{ code: number, message: string }>({ + url: `/api/v1/files/${fileId}`, + method: "delete" + }) +} + +/** + * 批量删除文件 + * @param fileIds 文件ID数组 + */ +export function batchDeleteFilesApi(fileIds: string[]) { + return request<{ code: number, message: string }>({ + url: "/api/v1/files/batch", + method: "delete", + data: { ids: fileIds } + }) +} diff --git a/management/web/src/common/apis/files/type.ts b/management/web/src/common/apis/files/type.ts new file mode 100644 index 0000000..7556682 --- /dev/null +++ b/management/web/src/common/apis/files/type.ts @@ -0,0 +1,63 @@ +/** + * 文件数据类型 + */ +export interface FileData { + /** 文件ID */ + id: string + /** 文件名称 */ + name: string + /** 文件大小(字节) */ + size: number + /** 文件类型 */ + type: string + /** 知识库ID */ + kb_id: string + /** 存储位置 */ + location: string + /** 创建时间 */ + create_time?: number + /** 更新时间 */ + update_time?: number +} + +/** + * 文件列表结果 + */ +export interface FileListResult { + /** 文件列表 */ + list: FileData[] + /** 总条数 */ + total: number +} + +/** + * 分页查询参数 + */ +export interface PageQuery { + /** 当前页码 */ + currentPage: number + /** 每页条数 */ + size: number +} + +/** + * 分页结果 + */ +export interface PageResult { + /** 数据列表 */ + list: T[] + /** 总条数 */ + total: number +} + +/** + * 通用响应结构 + */ +export interface ApiResponse { + /** 状态码 */ + code: number + /** 响应数据 */ + data: T + /** 响应消息 */ + message: string +} diff --git a/management/web/src/common/assets/icons/file.svg b/management/web/src/common/assets/icons/file.svg new file mode 100644 index 0000000..e42afcd --- /dev/null +++ b/management/web/src/common/assets/icons/file.svg @@ -0,0 +1,25 @@ + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/management/web/src/pages/file/index.vue b/management/web/src/pages/file/index.vue new file mode 100644 index 0000000..f9def3b --- /dev/null +++ b/management/web/src/pages/file/index.vue @@ -0,0 +1,437 @@ + + + + + + + diff --git a/management/web/src/router/index.ts b/management/web/src/router/index.ts index 3a660e5..a19716f 100644 --- a/management/web/src/router/index.ts +++ b/management/web/src/router/index.ts @@ -46,42 +46,6 @@ export const constantRoutes: RouteRecordRaw[] = [ hidden: true } }, - // { - // path: "/", - // component: Layouts, - // redirect: "/dashboard", - // children: [ - // { - // path: "dashboard", - // component: () => import("@/pages/dashboard/index.vue"), - // name: "Dashboard", - // meta: { - // title: "首页", - // svgIcon: "dashboard", - // affix: true - // } - // } - // ] - // }, - - // { - // path: "/", - // component: Layouts, - // redirect: "/dashboard", - // children: [ - // { - // path: "dashboard", - // component: () => import("@/pages/dashboard/index.vue"), - // name: "Dashboard", - // meta: { - // title: "首页", - // svgIcon: "dashboard", - // affix: true - // } - // } - // ] - // }, - { path: "/", component: Layouts, @@ -134,6 +98,24 @@ export const constantRoutes: RouteRecordRaw[] = [ } } ] + }, + { + path: "/file", + component: Layouts, + redirect: "/file/index", + children: [ + { + path: "index", + component: () => import("@/pages/file/index.vue"), + name: "File", + meta: { + title: "文件管理", + svgIcon: "file", + affix: false, + keepAlive: true + } + } + ] } // { // path: "/", diff --git a/management/web/types/auto/svg-component-global.d.ts b/management/web/types/auto/svg-component-global.d.ts index 3ef8ffe..e358aa6 100644 --- a/management/web/types/auto/svg-component-global.d.ts +++ b/management/web/types/auto/svg-component-global.d.ts @@ -9,18 +9,18 @@ declare module 'vue' { export interface GlobalComponents { SvgIcon: import("vue").DefineComponent<{ name: { - type: import("vue").PropType<"dashboard" | "fullscreen-exit" | "fullscreen" | "keyboard-down" | "keyboard-enter" | "keyboard-esc" | "keyboard-up" | "search" | "team-management" | "user-config" | "user-management">; + type: import("vue").PropType<"dashboard" | "file" | "fullscreen-exit" | "fullscreen" | "keyboard-down" | "keyboard-enter" | "keyboard-esc" | "keyboard-up" | "search" | "team-management" | "user-config" | "user-management">; default: string; required: true; }; }, {}, unknown, {}, {}, import("vue").ComponentOptionsMixin, import("vue").ComponentOptionsMixin, {}, string, import("vue").VNodeProps & import("vue").AllowedComponentProps & import("vue").ComponentCustomProps, Readonly; + type: import("vue").PropType<"dashboard" | "file" | "fullscreen-exit" | "fullscreen" | "keyboard-down" | "keyboard-enter" | "keyboard-esc" | "keyboard-up" | "search" | "team-management" | "user-config" | "user-management">; default: string; required: true; }; }>>, { - name: "dashboard" | "fullscreen-exit" | "fullscreen" | "keyboard-down" | "keyboard-enter" | "keyboard-esc" | "keyboard-up" | "search" | "team-management" | "user-config" | "user-management"; + name: "dashboard" | "file" | "fullscreen-exit" | "fullscreen" | "keyboard-down" | "keyboard-enter" | "keyboard-esc" | "keyboard-up" | "search" | "team-management" | "user-config" | "user-management"; }>; } } diff --git a/management/web/types/auto/svg-component.d.ts b/management/web/types/auto/svg-component.d.ts index cc795b0..7c0d45c 100644 --- a/management/web/types/auto/svg-component.d.ts +++ b/management/web/types/auto/svg-component.d.ts @@ -7,20 +7,20 @@ declare module '~virtual/svg-component' { const SvgIcon: import("vue").DefineComponent<{ name: { - type: import("vue").PropType<"dashboard" | "fullscreen-exit" | "fullscreen" | "keyboard-down" | "keyboard-enter" | "keyboard-esc" | "keyboard-up" | "search" | "team-management" | "user-config" | "user-management">; + type: import("vue").PropType<"dashboard" | "file" | "fullscreen-exit" | "fullscreen" | "keyboard-down" | "keyboard-enter" | "keyboard-esc" | "keyboard-up" | "search" | "team-management" | "user-config" | "user-management">; default: string; required: true; }; }, {}, unknown, {}, {}, import("vue").ComponentOptionsMixin, import("vue").ComponentOptionsMixin, {}, string, import("vue").VNodeProps & import("vue").AllowedComponentProps & import("vue").ComponentCustomProps, Readonly; + type: import("vue").PropType<"dashboard" | "file" | "fullscreen-exit" | "fullscreen" | "keyboard-down" | "keyboard-enter" | "keyboard-esc" | "keyboard-up" | "search" | "team-management" | "user-config" | "user-management">; default: string; required: true; }; }>>, { - name: "dashboard" | "fullscreen-exit" | "fullscreen" | "keyboard-down" | "keyboard-enter" | "keyboard-esc" | "keyboard-up" | "search" | "team-management" | "user-config" | "user-management"; + name: "dashboard" | "file" | "fullscreen-exit" | "fullscreen" | "keyboard-down" | "keyboard-enter" | "keyboard-esc" | "keyboard-up" | "search" | "team-management" | "user-config" | "user-management"; }>; - export const svgNames: ["dashboard", "fullscreen-exit", "fullscreen", "keyboard-down", "keyboard-enter", "keyboard-esc", "keyboard-up", "search", "team-management", "user-config", "user-management"]; - export type SvgName = "dashboard" | "fullscreen-exit" | "fullscreen" | "keyboard-down" | "keyboard-enter" | "keyboard-esc" | "keyboard-up" | "search" | "team-management" | "user-config" | "user-management"; + export const svgNames: ["dashboard", "file", "fullscreen-exit", "fullscreen", "keyboard-down", "keyboard-enter", "keyboard-esc", "keyboard-up", "search", "team-management", "user-config", "user-management"]; + export type SvgName = "dashboard" | "file" | "fullscreen-exit" | "fullscreen" | "keyboard-down" | "keyboard-enter" | "keyboard-esc" | "keyboard-up" | "search" | "team-management" | "user-config" | "user-management"; export default SvgIcon; }