导入必要的库

import numpy as np import pandas as pd import os from sklearn.model_selection import train_test_split from sklearn import preprocessing

创建MinMaxScaler对象

min_max_scaler = preprocessing.MinMaxScaler()

定义数据转换函数

def data_transform(data): data = np.asarray(data) data = min_max_scaler.fit_transform(data) # 数据归一化 return data

定义数据集划分函数

def split_train_val_test(file_path,cv_n,sheet_name,save_path): # 特征在excel表中的索引始末、特征数量 x_begin_index = 23 x_end_index = 63 x_length = 41 # 判断是0-20cm还是其他深度，确定文件名 if '0-20cm' in sheet_name: file_name_string = 'P_0-20cm' else: file_name_string = 'P_otherdepths' # 读取数据 data = pd.read_excel(file_path, sheet_name=sheet_name) y_data = data['全磷(P g/kg)'].values.tolist() # 以全磷为标签名 Y_data = np.asarray(y_data) x_data = data[data.columns[x_begin_index:x_end_index]] x_data = x_data.values.tolist() X_data = np.asarray(x_data) # X_data = min_max_scaler.fit_transform(X_data) # 数据归一化 random_state = 333333 for i in range(cv_n): random_state = random_state + 2222 # 划分训练集、验证集、测试集 x_train, x_val_test, y_train, y_val_test = train_test_split(X_data, Y_data, test_size=0.2, random_state=random_state) x_validation, x_test, y_validation, y_test = train_test_split(x_val_test, y_val_test, test_size=0.5, random_state=random_state) # 写入训练集文件 w_train = open(save_path+'/'+'train_'+file_name_string+''+str(i)+'.txt','w') for p in range(0,x_length): if p<x_length-1: w_train.write(str(p)) w_train.write(',') else: w_train.write(str(p)) w_train.write(' ') for x_train,y_train in zip(x_train,y_train): x_train=x_train.tolist() w_train.write(str(y_train)) w_train.write(',') m = 0 while m<len(x_train): if m < len(x_train)-1: w_train.write(str(x_train[m])) w_train.write(',') else: w_train.write(str(x_train[m])) m = m + 1 w_train.write(' ') # 写入验证集文件 w_val = open(save_path + '/' + 'val' + file_name_string + '' + str(i) + '.txt', 'w') for p in range(0,x_length): if p<x_length-1: w_val.write(str(p)) w_val.write(',') else: w_val.write(str(p)) w_val.write(' ') for x_val,y_val in zip(x_validation,y_validation): x_val=x_val.tolist() w_val.write(str(y_val)) w_val.write(',') m = 0 while m<len(x_val): if m < len(x_val)-1: w_val.write(str(x_val[m])) w_val.write(',') else: w_val.write(str(x_val[m])) m = m + 1 w_val.write(' ') # 写入测试集文件 w_test = open(save_path + '/test' + file_name_string + '_' + str(i) + '.txt', 'w') for p in range(0,x_length): if p<x_length-1: w_test.write(str(p)) w_test.write(',') else: w_test.write(str(p)) w_test.write(' ') for x_test,y_test in zip(x_test,y_test): x_test=x_test.tolist() w_test.write(str(y_test)) w_test.write(',') m = 0 while m<len(x_test): if m < len(x_test)-1: w_test.write(str(x_test[m])) w_test.write(',') else: w_test.write(str(x_test[m])) m = m + 1 w_test.write(' ') w_train.flush() w_train.close() w_test.flush() w_test.close() w_val.flush() w_val.close()

定义划分数据集的函数

def split_data(): wheat_file_path = '../rawData/小麦.xlsx' paddy_file_path = '../rawData/水稻.xlsx' root_save_path = '../dataSet' wheat_20cm_save_path = root_save_path+'/wheat_20cm' if not os.path.exists(wheat_20cm_save_path): os.makedirs(wheat_20cm_save_path) paddy_20cm_save_path = root_save_path+'/paddy_20cm' if not os.path.exists(paddy_20cm_save_path): os.makedirs(paddy_20cm_save_path) wheat_otherdepths_save_path = root_save_path+'/wheat_otherdepths' if not os.path.exists(wheat_otherdepths_save_path): os.makedirs(wheat_otherdepths_save_path) paddy_otherdepths_save_path = root_save_path+'/paddy_otherdepths' if not os.path.exists(paddy_otherdepths_save_path): os.makedirs(paddy_otherdepths_save_path) sheet_name_20cm = '全磷_0-20cm' sheet_name_otherdepths ='全磷_otherdepths' # 分别划分小麦、水稻的0-20cm和其他深度的数据集 split_train_val_test(wheat_file_path,10,sheet_name_20cm,wheat_20cm_save_path) split_train_val_test(wheat_file_path,10,sheet_name_otherdepths,wheat_otherdepths_save_path) split_train_val_test(paddy_file_path,10,sheet_name_20cm,paddy_20cm_save_path) split_train_val_test(paddy_file_path,10,sheet_name_otherdepths,paddy_otherdepths_save_path)