126 lines
4.6 KiB
Python
126 lines
4.6 KiB
Python
|
import os
|
|||
|
|
import sys
|
|||
|
|
import asyncio
|
|||
|
|
import pandas as pd
|
|||
|
|
import tempfile
|
|||
|
|
import pickle
|
|||
|
|
from datetime import datetime
|
|||
|
|
from typing import Dict, Any, List
|
|||
|
|
|
|||
|
|
sys.path.insert(0,os.path.join(os.path.dirname(__file__),'..','','backend'))
|
|||
|
|
|
|||
|
|
|
|||
|
|
def execute(df_1,df_2):
|
|||
|
|
# 假设合同日期列是字符串类型，将其转换为日期类型
|
|||
|
|
if '合同日期' in df_1.columns:
|
|||
|
|
df_1['合同日期'] = pd.to_datetime(df_1['合同日期'])
|
|||
|
|
if '合同日期' in df_2.columns:
|
|||
|
|
df_2['合同日期'] = pd.to_datetime(df_2['合同日期'])
|
|||
|
|
|
|||
|
|
# 筛选出2024年和2025年的数据
|
|||
|
|
filtered_df_1 = df_1[
|
|||
|
|
(df_1['合同日期'].dt.year == 2024) | (df_1['合同日期'].dt.year == 2025)]
|
|||
|
|
filtered_df_2 = df_2[
|
|||
|
|
(df_2['合同日期'].dt.year == 2024) | (df_2['合同日期'].dt.year == 2025)]
|
|||
|
|
# 合并两个数据框
|
|||
|
|
combined_df = pd.concat([filtered_df_1[:5], filtered_df_2[:7]], ignore_index=True)
|
|||
|
|
# 在去重前清理空值
|
|||
|
|
# combined_df_clean = combined_df.dropna(subset=['项目号']) # 确保主键不为空
|
|||
|
|
|
|||
|
|
# 填充数值列的空值
|
|||
|
|
combined_df_filled = combined_df.fillna({
|
|||
|
|
'总合同额': 0,
|
|||
|
|
'已确认比例': 0,
|
|||
|
|
'分包合同额': 0
|
|||
|
|
})
|
|||
|
|
# 找出不同的项目
|
|||
|
|
unique_projects = combined_df.drop_duplicates(subset=['项目号'])
|
|||
|
|
return unique_projects
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
def test_load_selected_dataframes():
|
|||
|
|
|
|||
|
|
try:
|
|||
|
|
file1_path = '2025年在手合同数据.xlsx.pkl'
|
|||
|
|
file2_path = '2024年在手合同数据.xlsx.pkl'
|
|||
|
|
target_filenames = [file1_path,file2_path]
|
|||
|
|
dataframes = {}
|
|||
|
|
base_dir = os.path.join("D:\workspace-py\chat-agent\\backend","data","uploads","excel_6")
|
|||
|
|
|
|||
|
|
all_files = os.listdir(base_dir)
|
|||
|
|
for filename in target_filenames:
|
|||
|
|
matching_files = []
|
|||
|
|
for file in all_files:
|
|||
|
|
if file.endswith(f"_{filename}") or file.endswith(f"_{filename}.pkl"):
|
|||
|
|
matching_files.append(file)
|
|||
|
|
if not matching_files:
|
|||
|
|
print(f"未找到匹配的文件: {filename}")
|
|||
|
|
|
|||
|
|
# 如果有多个匹配文件，选择最新的
|
|||
|
|
if len(matching_files) > 1:
|
|||
|
|
matching_files.sort(key=lambda x: os.path.getmtime(os.path.join(base_dir, x)), reverse=True)
|
|||
|
|
print(f"找到多个匹配文件，选择最新的: {matching_files[0]}")
|
|||
|
|
continue
|
|||
|
|
|
|||
|
|
selected_file = matching_files[0]
|
|||
|
|
file_path = os.path.join(base_dir, selected_file)
|
|||
|
|
|
|||
|
|
try:
|
|||
|
|
# 优先加载pickle文件
|
|||
|
|
if selected_file.endswith('.pkl'):
|
|||
|
|
with open(file_path, 'rb') as f:
|
|||
|
|
df = pickle.load(f)
|
|||
|
|
print(f"成功从pickle加载文件: {selected_file}")
|
|||
|
|
else:
|
|||
|
|
# 如果没有pickle文件，尝试加载原始文件
|
|||
|
|
if selected_file.endswith(('.xlsx', '.xls')):
|
|||
|
|
df = pd.read_excel(file_path)
|
|||
|
|
elif selected_file.endswith('.csv'):
|
|||
|
|
df = pd.read_csv(file_path)
|
|||
|
|
else:
|
|||
|
|
print(f"不支持的文件格式: {selected_file}")
|
|||
|
|
continue
|
|||
|
|
print(f"成功从原始文件加载: {selected_file}")
|
|||
|
|
|
|||
|
|
# 使用原始文件名作为key
|
|||
|
|
dataframes[filename] = df
|
|||
|
|
print(f"成功加载DataFrame: {filename}, 形状: {df.shape}")
|
|||
|
|
|
|||
|
|
except Exception as e:
|
|||
|
|
print(f"加载文件失败 {selected_file}: {e}")
|
|||
|
|
continue
|
|||
|
|
|
|||
|
|
return dataframes
|
|||
|
|
except Exception as e:
|
|||
|
|
print(e)
|
|||
|
|
|
|||
|
|
if __name__ == '__main__':
|
|||
|
|
dataframes = test_load_selected_dataframes()
|
|||
|
|
df_names = list(dataframes.keys())
|
|||
|
|
if len(df_names) >= 2:
|
|||
|
|
df_1 = dataframes[df_names[0]]
|
|||
|
|
df_2 = dataframes[df_names[1]]
|
|||
|
|
|
|||
|
|
print(f"DataFrame 1 ({df_names[0]}) 形状: {df_1.shape}")
|
|||
|
|
print(f"DataFrame 1 列名: {list(df_1.columns)}")
|
|||
|
|
print(f"DataFrame 1 前几行:")
|
|||
|
|
print(df_1.head())
|
|||
|
|
print()
|
|||
|
|
|
|||
|
|
print(f"DataFrame 2 ({df_names[1]}) 形状: {df_2.shape}")
|
|||
|
|
print(f"DataFrame 2 列名: {list(df_2.columns)}")
|
|||
|
|
print(f"DataFrame 2 前几行:")
|
|||
|
|
print(df_2.head())
|
|||
|
|
print()
|
|||
|
|
|
|||
|
|
# 执行用户提供的数据处理逻辑
|
|||
|
|
print("执行数据处理逻辑...")
|
|||
|
|
result = execute(df_1, df_2)
|
|||
|
|
|
|||
|
|
print("处理结果:")
|
|||
|
|
print(f"结果形状: {result.shape}")
|
|||
|
|
print(f"结果列名: {list(result.columns)}")
|
|||
|
|
print("结果数据:")
|
|||
|
|
print(result)
|