diff --git a/data_provider/data_loader.py b/data_provider/data_loader.py index dcbea3124..e8cb73f7c 100644 --- a/data_provider/data_loader.py +++ b/data_provider/data_loader.py @@ -1,122 +1,37 @@ import os -import numpy as np -import pandas as pd import glob import re +import warnings + +from typing import override + +import numpy as np +import pandas as pd + +from sklearn.preprocessing import StandardScaler +from sktime.datasets import load_from_tsfile_to_dataframe + import torch from torch.utils.data import Dataset, DataLoader -from sklearn.preprocessing import StandardScaler -from utils.timefeatures import time_features + from data_provider.m4 import M4Dataset, M4Meta from data_provider.uea import subsample, interpolate_missing, Normalizer -from sktime.datasets import load_from_tsfile_to_dataframe -import warnings from utils.augmentation import run_augmentation_single +from utils.timefeatures import time_features warnings.filterwarnings('ignore') -class Dataset_ETT_hour(Dataset): +class Dataset_Custom(Dataset): def __init__(self, args, root_path, flag='train', size=None, features='S', data_path='ETTh1.csv', - target='OT', scale=True, timeenc=0, freq='h', seasonal_patterns=None): + target='OT', scale=True, timeenc=0, freq='h', + seasonal_patterns=None): # size [seq_len, label_len, pred_len] self.args = args - # info - if size == None: - self.seq_len = 24 * 4 * 4 - self.label_len = 24 * 4 - self.pred_len = 24 * 4 - else: - self.seq_len = size[0] - self.label_len = size[1] - self.pred_len = size[2] - # init - assert flag in ['train', 'test', 'val'] - type_map = {'train': 0, 'val': 1, 'test': 2} - self.set_type = type_map[flag] - - self.features = features - self.target = target - self.scale = scale - self.timeenc = timeenc - self.freq = freq - self.root_path = root_path - self.data_path = data_path - self.__read_data__() - - def __read_data__(self): - self.scaler = StandardScaler() - df_raw = pd.read_csv(os.path.join(self.root_path, - self.data_path)) - - border1s = [0, 12 * 30 * 24 - self.seq_len, 12 * 30 * 24 + 4 * 30 * 24 - self.seq_len] - border2s = [12 * 30 * 24, 12 * 30 * 24 + 4 * 30 * 24, 12 * 30 * 24 + 8 * 30 * 24] - border1 = border1s[self.set_type] - border2 = border2s[self.set_type] - - if self.features == 'M' or self.features == 'MS': - cols_data = df_raw.columns[1:] - df_data = df_raw[cols_data] - elif self.features == 'S': - df_data = df_raw[[self.target]] - - if self.scale: - train_data = df_data[border1s[0]:border2s[0]] - self.scaler.fit(train_data.values) - data = self.scaler.transform(df_data.values) - else: - data = df_data.values - - df_stamp = df_raw[['date']][border1:border2] - df_stamp['date'] = pd.to_datetime(df_stamp.date) - if self.timeenc == 0: - df_stamp['month'] = df_stamp.date.apply(lambda row: row.month, 1) - df_stamp['day'] = df_stamp.date.apply(lambda row: row.day, 1) - df_stamp['weekday'] = df_stamp.date.apply(lambda row: row.weekday(), 1) - df_stamp['hour'] = df_stamp.date.apply(lambda row: row.hour, 1) - data_stamp = df_stamp.drop(['date'], 1).values - elif self.timeenc == 1: - data_stamp = time_features(pd.to_datetime(df_stamp['date'].values), freq=self.freq) - data_stamp = data_stamp.transpose(1, 0) - - self.data_x = data[border1:border2] - self.data_y = data[border1:border2] - - if self.set_type == 0 and self.args.augmentation_ratio > 0: - self.data_x, self.data_y, augmentation_tags = run_augmentation_single(self.data_x, self.data_y, self.args) - - self.data_stamp = data_stamp - - def __getitem__(self, index): - s_begin = index - s_end = s_begin + self.seq_len - r_begin = s_end - self.label_len - r_end = r_begin + self.label_len + self.pred_len - - seq_x = self.data_x[s_begin:s_end] - seq_y = self.data_y[r_begin:r_end] - seq_x_mark = self.data_stamp[s_begin:s_end] - seq_y_mark = self.data_stamp[r_begin:r_end] - - return seq_x, seq_y, seq_x_mark, seq_y_mark - - def __len__(self): - return len(self.data_x) - self.seq_len - self.pred_len + 1 - - def inverse_transform(self, data): - return self.scaler.inverse_transform(data) - - -class Dataset_ETT_minute(Dataset): - def __init__(self, args, root_path, flag='train', size=None, - features='S', data_path='ETTm1.csv', - target='OT', scale=True, timeenc=0, freq='t', seasonal_patterns=None): - # size [seq_len, label_len, pred_len] - self.args = args # info - if size == None: + if size is None: self.seq_len = 24 * 4 * 4 self.label_len = 24 * 4 self.pred_len = 24 * 4 @@ -124,6 +39,7 @@ def __init__(self, args, root_path, flag='train', size=None, self.seq_len = size[0] self.label_len = size[1] self.pred_len = size[2] + # init assert flag in ['train', 'test', 'val'] type_map = {'train': 0, 'val': 1, 'test': 2} @@ -139,126 +55,69 @@ def __init__(self, args, root_path, flag='train', size=None, self.data_path = data_path self.__read_data__() - def __read_data__(self): - self.scaler = StandardScaler() - df_raw = pd.read_csv(os.path.join(self.root_path, - self.data_path)) + def _get_borders(self, h: int) -> tuple[list[int], list[int]]: + num_train = int(h * 0.7) + num_test = int(h * 0.2) + num_vali = h - num_train - num_test - border1s = [0, 12 * 30 * 24 * 4 - self.seq_len, 12 * 30 * 24 * 4 + 4 * 30 * 24 * 4 - self.seq_len] - border2s = [12 * 30 * 24 * 4, 12 * 30 * 24 * 4 + 4 * 30 * 24 * 4, 12 * 30 * 24 * 4 + 8 * 30 * 24 * 4] - border1 = border1s[self.set_type] - border2 = border2s[self.set_type] + border1s = [ + 0, + num_train - self.seq_len, + h - num_test - self.seq_len + ] + border2s = [num_train, num_train + num_vali, h] - if self.features == 'M' or self.features == 'MS': - cols_data = df_raw.columns[1:] - df_data = df_raw[cols_data] - elif self.features == 'S': - df_data = df_raw[[self.target]] + return border1s, border2s - if self.scale: - train_data = df_data[border1s[0]:border2s[0]] - self.scaler.fit(train_data.values) - data = self.scaler.transform(df_data.values) - else: - data = df_data.values + def _get_data_stamp( + self, df_raw: pd.DataFrame, + border1: int, border2: int) -> np.ndarray: df_stamp = df_raw[['date']][border1:border2] df_stamp['date'] = pd.to_datetime(df_stamp.date) + if self.timeenc == 0: - df_stamp['month'] = df_stamp.date.apply(lambda row: row.month, 1) - df_stamp['day'] = df_stamp.date.apply(lambda row: row.day, 1) - df_stamp['weekday'] = df_stamp.date.apply(lambda row: row.weekday(), 1) - df_stamp['hour'] = df_stamp.date.apply(lambda row: row.hour, 1) - df_stamp['minute'] = df_stamp.date.apply(lambda row: row.minute, 1) - df_stamp['minute'] = df_stamp.minute.map(lambda x: x // 15) - data_stamp = df_stamp.drop(['date'], 1).values + for label, op in [ + ['month', lambda row: row.month], + ['day', lambda row: row.day], + ['weekday', lambda row: row.weekday()], + ['hour', lambda row: row.hour], + ]: + df_stamp[label] = df_stamp['date'].apply(op, True) + data_stamp = df_stamp.drop(['date'], axis=1).to_numpy() elif self.timeenc == 1: - data_stamp = time_features(pd.to_datetime(df_stamp['date'].values), freq=self.freq) + data_stamp = time_features( + pd.to_datetime( + df_stamp['date'].values), + freq=self.freq) data_stamp = data_stamp.transpose(1, 0) - - self.data_x = data[border1:border2] - self.data_y = data[border1:border2] - - if self.set_type == 0 and self.args.augmentation_ratio > 0: - self.data_x, self.data_y, augmentation_tags = run_augmentation_single(self.data_x, self.data_y, self.args) - - self.data_stamp = data_stamp - - def __getitem__(self, index): - s_begin = index - s_end = s_begin + self.seq_len - r_begin = s_end - self.label_len - r_end = r_begin + self.label_len + self.pred_len - - seq_x = self.data_x[s_begin:s_end] - seq_y = self.data_y[r_begin:r_end] - seq_x_mark = self.data_stamp[s_begin:s_end] - seq_y_mark = self.data_stamp[r_begin:r_end] - - return seq_x, seq_y, seq_x_mark, seq_y_mark - - def __len__(self): - return len(self.data_x) - self.seq_len - self.pred_len + 1 - - def inverse_transform(self, data): - return self.scaler.inverse_transform(data) - - -class Dataset_Custom(Dataset): - def __init__(self, args, root_path, flag='train', size=None, - features='S', data_path='ETTh1.csv', - target='OT', scale=True, timeenc=0, freq='h', seasonal_patterns=None): - # size [seq_len, label_len, pred_len] - self.args = args - # info - if size == None: - self.seq_len = 24 * 4 * 4 - self.label_len = 24 * 4 - self.pred_len = 24 * 4 else: - self.seq_len = size[0] - self.label_len = size[1] - self.pred_len = size[2] - # init - assert flag in ['train', 'test', 'val'] - type_map = {'train': 0, 'val': 1, 'test': 2} - self.set_type = type_map[flag] + raise AttributeError('Invalid option for \'timeenc\'.') - self.features = features - self.target = target - self.scale = scale - self.timeenc = timeenc - self.freq = freq - - self.root_path = root_path - self.data_path = data_path - self.__read_data__() + return data_stamp def __read_data__(self): self.scaler = StandardScaler() - df_raw = pd.read_csv(os.path.join(self.root_path, - self.data_path)) + df_raw = pd.read_csv( + os.path.join(self.root_path, self.data_path)) - ''' - df_raw.columns: ['date', ...(other features), target feature] - ''' + # df_raw.columns: ['date', ...(other features), target feature] cols = list(df_raw.columns) cols.remove(self.target) cols.remove('date') df_raw = df_raw[['date'] + cols + [self.target]] - num_train = int(len(df_raw) * 0.7) - num_test = int(len(df_raw) * 0.2) - num_vali = len(df_raw) - num_train - num_test - border1s = [0, num_train - self.seq_len, len(df_raw) - num_test - self.seq_len] - border2s = [num_train, num_train + num_vali, len(df_raw)] + + border1s, border2s = self._get_borders(len(df_raw)) border1 = border1s[self.set_type] border2 = border2s[self.set_type] - if self.features == 'M' or self.features == 'MS': + if self.features in ['M', 'MS']: cols_data = df_raw.columns[1:] df_data = df_raw[cols_data] elif self.features == 'S': df_data = df_raw[[self.target]] + else: + raise AttributeError('Invalid option for \'features\'.') if self.scale: train_data = df_data[border1s[0]:border2s[0]] @@ -267,31 +126,20 @@ def __read_data__(self): else: data = df_data.values - df_stamp = df_raw[['date']][border1:border2] - df_stamp['date'] = pd.to_datetime(df_stamp.date) - if self.timeenc == 0: - df_stamp['month'] = df_stamp.date.apply(lambda row: row.month, 1) - df_stamp['day'] = df_stamp.date.apply(lambda row: row.day, 1) - df_stamp['weekday'] = df_stamp.date.apply(lambda row: row.weekday(), 1) - df_stamp['hour'] = df_stamp.date.apply(lambda row: row.hour, 1) - data_stamp = df_stamp.drop(['date'], 1).values - elif self.timeenc == 1: - data_stamp = time_features(pd.to_datetime(df_stamp['date'].values), freq=self.freq) - data_stamp = data_stamp.transpose(1, 0) - self.data_x = data[border1:border2] self.data_y = data[border1:border2] if self.set_type == 0 and self.args.augmentation_ratio > 0: - self.data_x, self.data_y, augmentation_tags = run_augmentation_single(self.data_x, self.data_y, self.args) + self.data_x, self.data_y, _ = run_augmentation_single( + self.data_x, self.data_y, self.args) - self.data_stamp = data_stamp + self.data_stamp = self._get_data_stamp(df_raw, border1, border2) def __getitem__(self, index): s_begin = index s_end = s_begin + self.seq_len r_begin = s_end - self.label_len - r_end = r_begin + self.label_len + self.pred_len + r_end = s_end + self.pred_len seq_x = self.data_x[s_begin:s_end] seq_y = self.data_y[r_begin:r_end] @@ -307,10 +155,62 @@ def inverse_transform(self, data): return self.scaler.inverse_transform(data) +class Dataset_ETT_hour(Dataset_Custom): + @override + def _get_borders(self, h: int) -> tuple[list[int], list[int]]: + return [ + 0, + 12 * 30 * 24 - self.seq_len, + 12 * 30 * 24 + 4 * 30 * 24 - self.seq_len + ], [ + 12 * 30 * 24, + 12 * 30 * 24 + 4 * 30 * 24, + 12 * 30 * 24 + 8 * 30 * 24 + ] + + +class Dataset_ETT_minute(Dataset_Custom): + def __init__(self, args, root_path, flag='train', size=None, + features='S', data_path='ETTm1.csv', + target='OT', scale=True, timeenc=0, freq='t', + seasonal_patterns=None): + super(Dataset_ETT_minute, self).__init__( + args, root_path, flag, size, + features, data_path, + target, scale, timeenc, freq, + seasonal_patterns + ) + + @override + def _get_borders(self, h: int) -> tuple[list[int], list[int]]: + return [ + 0, + 12 * 30 * 24 * 4 - self.seq_len, + 12 * 30 * 24 * 4 + 4 * 30 * 24 * 4 - self.seq_len + ], [ + 12 * 30 * 24 * 4, + 12 * 30 * 24 * 4 + 4 * 30 * 24 * 4, + 12 * 30 * 24 * 4 + 8 * 30 * 24 * 4 + ] + + def _get_data_stamp(self, df_raw: pd.DataFrame, + border1: int, border2: int) -> np.ndarray: + data_stamp = super()._get_data_stamp(df_raw, border1, border2) + if self.timeenc == 0: + data_stamp = np.concat([ + data_stamp, + df_raw['date'][border1:border2].apply( + lambda row: row.minute // 15, True + )[:, None] + ], axis=1) + return data_stamp + + class Dataset_M4(Dataset): def __init__(self, args, root_path, flag='pred', size=None, features='S', data_path='ETTh1.csv', - target='OT', scale=False, inverse=False, timeenc=0, freq='15min', + target='OT', scale=False, inverse=False, timeenc=0, + freq='15min', seasonal_patterns='Yearly'): # size [seq_len, label_len, pred_len] # init @@ -337,29 +237,39 @@ def __read_data__(self): if self.flag == 'train': dataset = M4Dataset.load(training=True, dataset_file=self.root_path) else: - dataset = M4Dataset.load(training=False, dataset_file=self.root_path) + dataset = M4Dataset.load( + training=False, dataset_file=self.root_path) training_values = np.array( [v[~np.isnan(v)] for v in - dataset.values[dataset.groups == self.seasonal_patterns]]) # split different frequencies - self.ids = np.array([i for i in dataset.ids[dataset.groups == self.seasonal_patterns]]) - self.timeseries = [ts for ts in training_values] + # split different frequencies + dataset.values[dataset.groups == self.seasonal_patterns]]) + self.ids = np.array( + dataset.ids[dataset.groups == self.seasonal_patterns]) + self.timeseries = list(training_values) def __getitem__(self, index): insample = np.zeros((self.seq_len, 1)) insample_mask = np.zeros((self.seq_len, 1)) outsample = np.zeros((self.pred_len + self.label_len, 1)) - outsample_mask = np.zeros((self.pred_len + self.label_len, 1)) # m4 dataset + outsample_mask = np.zeros( + (self.pred_len + self.label_len, 1)) # m4 dataset sampled_timeseries = self.timeseries[index] - cut_point = np.random.randint(low=max(1, len(sampled_timeseries) - self.window_sampling_limit), - high=len(sampled_timeseries), - size=1)[0] + cut_point = np.random.randint( + low=max(1, len(sampled_timeseries) - self.window_sampling_limit), + high=len(sampled_timeseries), + size=1)[0] - insample_window = sampled_timeseries[max(0, cut_point - self.seq_len):cut_point] + insample_window = sampled_timeseries[max( + 0, cut_point - self.seq_len):cut_point] insample[-len(insample_window):, 0] = insample_window insample_mask[-len(insample_window):, 0] = 1.0 outsample_window = sampled_timeseries[ - max(0, cut_point - self.label_len):min(len(sampled_timeseries), cut_point + self.pred_len)] + max( + 0, cut_point - self.label_len + ):min( + len(sampled_timeseries), cut_point + self.pred_len + )] outsample[:len(outsample_window), 0] = outsample_window outsample_mask[:len(outsample_window), 0] = 1.0 return insample, outsample, insample_mask, outsample_mask @@ -373,9 +283,11 @@ def inverse_transform(self, data): def last_insample_window(self): """ The last window of insample size of all timeseries. - This function does not support batching and does not reshuffle timeseries. + This function does not support batching and does not reshuffle + timeseries. - :return: Last insample window of all timeseries. Shape "timeseries, insample size" + :return: Last insample window of all timeseries. Shape "timeseries, + insample size" """ insample = np.zeros((len(self.timeseries), self.seq_len)) insample_mask = np.zeros((len(self.timeseries), self.seq_len)) @@ -404,33 +316,43 @@ def __init__(self, args, root_path, win_size, step=1, flag="train"): self.train = data data_len = len(self.train) self.val = self.train[(int)(data_len * 0.8):] - self.test_labels = pd.read_csv(os.path.join(root_path, 'test_label.csv')).values[:, 1:] + self.test_labels = pd.read_csv(os.path.join( + root_path, 'test_label.csv')).values[:, 1:] print("test:", self.test.shape) print("train:", self.train.shape) def __len__(self): if self.flag == "train": return (self.train.shape[0] - self.win_size) // self.step + 1 - elif (self.flag == 'val'): + if self.flag == 'val': return (self.val.shape[0] - self.win_size) // self.step + 1 - elif (self.flag == 'test'): + if self.flag == 'test': return (self.test.shape[0] - self.win_size) // self.step + 1 - else: - return (self.test.shape[0] - self.win_size) // self.win_size + 1 + return (self.test.shape[0] - self.win_size) // self.win_size + 1 def __getitem__(self, index): index = index * self.step if self.flag == "train": - return np.float32(self.train[index:index + self.win_size]), np.float32(self.test_labels[0:self.win_size]) - elif (self.flag == 'val'): - return np.float32(self.val[index:index + self.win_size]), np.float32(self.test_labels[0:self.win_size]) - elif (self.flag == 'test'): - return np.float32(self.test[index:index + self.win_size]), np.float32( - self.test_labels[index:index + self.win_size]) - else: - return np.float32(self.test[ - index // self.step * self.win_size:index // self.step * self.win_size + self.win_size]), np.float32( - self.test_labels[index // self.step * self.win_size:index // self.step * self.win_size + self.win_size]) + return np.float32( + self.train[index:index + self.win_size] + ), np.float32(self.test_labels[0:self.win_size]) + if self.flag == 'val': + return np.float32( + self.val[index:index + self.win_size] + ), np.float32(self.test_labels[0:self.win_size]) + if self.flag == 'test': + return np.float32( + self.test[index:index + self.win_size] + ), np.float32( + self.test_labels[index:index + self.win_size] + ) + return np.float32(self.test[ + index // self.step * self.win_size: + index // self.step * self.win_size + self.win_size] + ), np.float32(self.test_labels[ + index // self.step * self.win_size: + index // self.step * self.win_size + self.win_size + ]) class MSLSegLoader(Dataset): @@ -447,33 +369,49 @@ def __init__(self, args, root_path, win_size, step=1, flag="train"): self.train = data data_len = len(self.train) self.val = self.train[(int)(data_len * 0.8):] - self.test_labels = np.load(os.path.join(root_path, "MSL_test_label.npy")) + self.test_labels = np.load( + os.path.join( + root_path, + "MSL_test_label.npy")) print("test:", self.test.shape) print("train:", self.train.shape) def __len__(self): if self.flag == "train": return (self.train.shape[0] - self.win_size) // self.step + 1 - elif (self.flag == 'val'): + if self.flag == 'val': return (self.val.shape[0] - self.win_size) // self.step + 1 - elif (self.flag == 'test'): + if self.flag == 'test': return (self.test.shape[0] - self.win_size) // self.step + 1 - else: - return (self.test.shape[0] - self.win_size) // self.win_size + 1 + return (self.test.shape[0] - self.win_size) // self.win_size + 1 def __getitem__(self, index): index = index * self.step if self.flag == "train": - return np.float32(self.train[index:index + self.win_size]), np.float32(self.test_labels[0:self.win_size]) - elif (self.flag == 'val'): - return np.float32(self.val[index:index + self.win_size]), np.float32(self.test_labels[0:self.win_size]) - elif (self.flag == 'test'): - return np.float32(self.test[index:index + self.win_size]), np.float32( - self.test_labels[index:index + self.win_size]) - else: - return np.float32(self.test[ - index // self.step * self.win_size:index // self.step * self.win_size + self.win_size]), np.float32( - self.test_labels[index // self.step * self.win_size:index // self.step * self.win_size + self.win_size]) + return np.float32( + self.train[index:index + self.win_size] + ), np.float32( + self.test_labels[0:self.win_size] + ) + if self.flag == 'val': + return np.float32( + self.val[index:index + self.win_size] + ), np.float32( + self.test_labels[0:self.win_size] + ) + if self.flag == 'test': + return np.float32( + self.test[index:index + self.win_size] + ), np.float32( + self.test_labels[index:index + self.win_size] + ) + return np.float32(self.test[ + index // self.step * self.win_size: + index // self.step * self.win_size + self.win_size + ]), np.float32(self.test_labels[ + index // self.step * self.win_size: + index // self.step * self.win_size + self.win_size + ]) class SMAPSegLoader(Dataset): @@ -490,34 +428,47 @@ def __init__(self, args, root_path, win_size, step=1, flag="train"): self.train = data data_len = len(self.train) self.val = self.train[(int)(data_len * 0.8):] - self.test_labels = np.load(os.path.join(root_path, "SMAP_test_label.npy")) + self.test_labels = np.load( + os.path.join( + root_path, + "SMAP_test_label.npy")) print("test:", self.test.shape) print("train:", self.train.shape) def __len__(self): - if self.flag == "train": return (self.train.shape[0] - self.win_size) // self.step + 1 - elif (self.flag == 'val'): + if self.flag == 'val': return (self.val.shape[0] - self.win_size) // self.step + 1 - elif (self.flag == 'test'): + if self.flag == 'test': return (self.test.shape[0] - self.win_size) // self.step + 1 - else: - return (self.test.shape[0] - self.win_size) // self.win_size + 1 + return (self.test.shape[0] - self.win_size) // self.win_size + 1 def __getitem__(self, index): index = index * self.step if self.flag == "train": - return np.float32(self.train[index:index + self.win_size]), np.float32(self.test_labels[0:self.win_size]) - elif (self.flag == 'val'): - return np.float32(self.val[index:index + self.win_size]), np.float32(self.test_labels[0:self.win_size]) - elif (self.flag == 'test'): - return np.float32(self.test[index:index + self.win_size]), np.float32( - self.test_labels[index:index + self.win_size]) - else: - return np.float32(self.test[ - index // self.step * self.win_size:index // self.step * self.win_size + self.win_size]), np.float32( - self.test_labels[index // self.step * self.win_size:index // self.step * self.win_size + self.win_size]) + return np.float32( + self.train[index:index + self.win_size] + ), np.float32(self.test_labels[0:self.win_size]) + if self.flag == 'val': + return np.float32( + self.val[index:index + self.win_size] + ), np.float32(self.test_labels[0:self.win_size]) + if self.flag == 'test': + return np.float32( + self.test[index:index + self.win_size] + ), np.float32( + self.test_labels[index:index + self.win_size] + ) + return np.float32( + self.test[ + index // self.step * self.win_size: + index // self.step * self.win_size + self.win_size + ]), np.float32( + self.test_labels[ + index // self.step * self.win_size: + index // self.step * self.win_size + self.win_size + ]) class SMDSegLoader(Dataset): @@ -534,31 +485,46 @@ def __init__(self, args, root_path, win_size, step=100, flag="train"): self.train = data data_len = len(self.train) self.val = self.train[(int)(data_len * 0.8):] - self.test_labels = np.load(os.path.join(root_path, "SMD_test_label.npy")) + self.test_labels = np.load( + os.path.join( + root_path, + "SMD_test_label.npy")) def __len__(self): if self.flag == "train": return (self.train.shape[0] - self.win_size) // self.step + 1 - elif (self.flag == 'val'): + if self.flag == 'val': return (self.val.shape[0] - self.win_size) // self.step + 1 - elif (self.flag == 'test'): + if self.flag == 'test': return (self.test.shape[0] - self.win_size) // self.step + 1 - else: - return (self.test.shape[0] - self.win_size) // self.win_size + 1 + return (self.test.shape[0] - self.win_size) // self.win_size + 1 def __getitem__(self, index): index = index * self.step if self.flag == "train": - return np.float32(self.train[index:index + self.win_size]), np.float32(self.test_labels[0:self.win_size]) - elif (self.flag == 'val'): - return np.float32(self.val[index:index + self.win_size]), np.float32(self.test_labels[0:self.win_size]) - elif (self.flag == 'test'): - return np.float32(self.test[index:index + self.win_size]), np.float32( - self.test_labels[index:index + self.win_size]) - else: - return np.float32(self.test[ - index // self.step * self.win_size:index // self.step * self.win_size + self.win_size]), np.float32( - self.test_labels[index // self.step * self.win_size:index // self.step * self.win_size + self.win_size]) + return np.float32( + self.train[index:index + self.win_size] + ), np.float32( + self.test_labels[0:self.win_size] + ) + if self.flag == 'val': + return np.float32( + self.val[index:index + self.win_size] + ), np.float32(self.test_labels[0:self.win_size]) + if self.flag == 'test': + return np.float32( + self.test[index:index + self.win_size] + ), np.float32( + self.test_labels[index:index + self.win_size] + ) + return np.float32( + self.test[ + index // self.step * self.win_size: + index // self.step * self.win_size + self.win_size] + ), np.float32(self.test_labels[ + index // self.step * self.win_size: + index // self.step * self.win_size + self.win_size + ]) class SWATSegLoader(Dataset): @@ -591,26 +557,33 @@ def __len__(self): """ if self.flag == "train": return (self.train.shape[0] - self.win_size) // self.step + 1 - elif (self.flag == 'val'): + if self.flag == 'val': return (self.val.shape[0] - self.win_size) // self.step + 1 - elif (self.flag == 'test'): + if self.flag == 'test': return (self.test.shape[0] - self.win_size) // self.step + 1 - else: - return (self.test.shape[0] - self.win_size) // self.win_size + 1 + return (self.test.shape[0] - self.win_size) // self.win_size + 1 def __getitem__(self, index): index = index * self.step if self.flag == "train": - return np.float32(self.train[index:index + self.win_size]), np.float32(self.test_labels[0:self.win_size]) - elif (self.flag == 'val'): - return np.float32(self.val[index:index + self.win_size]), np.float32(self.test_labels[0:self.win_size]) - elif (self.flag == 'test'): - return np.float32(self.test[index:index + self.win_size]), np.float32( - self.test_labels[index:index + self.win_size]) - else: - return np.float32(self.test[ - index // self.step * self.win_size:index // self.step * self.win_size + self.win_size]), np.float32( - self.test_labels[index // self.step * self.win_size:index // self.step * self.win_size + self.win_size]) + return np.float32( + self.train[index:index + self.win_size] + ), np.float32(self.test_labels[0:self.win_size]) + if self.flag == 'val': + return np.float32( + self.val[index:index + self.win_size] + ), np.float32(self.test_labels[0:self.win_size]) + if self.flag == 'test': + return np.float32( + self.test[index:index + self.win_size] + ), np.float32(self.test_labels[index:index + self.win_size]) + return np.float32(self.test[ + index // self.step * self.win_size: + index // self.step * self.win_size + self.win_size + ]), np.float32(self.test_labels[ + index // self.step * self.win_size: + index // self.step * self.win_size + self.win_size + ]) class UEAloader(Dataset): @@ -620,22 +593,33 @@ class UEAloader(Dataset): Argument: limit_size: float in (0, 1) for debug Attributes: - all_df: (num_samples * seq_len, num_columns) dataframe indexed by integer indices, with multiple rows corresponding to the same index (sample). - Each row is a time step; Each column contains either metadata (e.g. timestamp) or a feature. - feature_df: (num_samples * seq_len, feat_dim) dataframe; contains the subset of columns of `all_df` which correspond to selected features - feature_names: names of columns contained in `feature_df` (same as feature_df.columns) - all_IDs: (num_samples,) series of IDs contained in `all_df`/`feature_df` (same as all_df.index.unique() ) - labels_df: (num_samples, num_labels) pd.DataFrame of label(s) for each sample - max_seq_len: maximum sequence (time series) length. If None, script argument `max_seq_len` will be used. + all_df: (num_samples * seq_len, num_columns) dataframe indexed by + integer indices, with multiple rows corresponding to the same index + (sample). + Each row is a time step; Each column contains either metadata (e.g. + timestamp) or a feature. + feature_df: (num_samples * seq_len, feat_dim) dataframe; contains the + subset of columns of `all_df` which correspond to selected features + feature_names: names of columns contained in `feature_df` (same as + feature_df.columns) + all_IDs: (num_samples,) series of IDs contained in `all_df` + `feature_df` (same as all_df.index.unique() ) + labels_df: (num_samples, num_labels) pd.DataFrame of label(s) for each + sample + max_seq_len: maximum sequence (time series) length. If None, script + argument `max_seq_len` will be used. (Moreover, script argument overrides this attribute) """ - def __init__(self, args, root_path, file_list=None, limit_size=None, flag=None): + def __init__(self, args, root_path, file_list=None, + limit_size=None, flag=None): self.args = args self.root_path = root_path self.flag = flag - self.all_df, self.labels_df = self.load_all(root_path, file_list=file_list, flag=flag) - self.all_IDs = self.all_df.index.unique() # all sample IDs (integer indices 0 ... num_samples-1) + self.all_df, self.labels_df = self.load_all( + root_path, file_list=file_list, flag=flag) + # all sample IDs (integer indices 0 ... num_samples-1) + self.all_IDs = self.all_df.index.unique() if limit_size is not None: if limit_size > 1: @@ -656,62 +640,94 @@ def __init__(self, args, root_path, file_list=None, limit_size=None, flag=None): def load_all(self, root_path, file_list=None, flag=None): """ - Loads datasets from ts files contained in `root_path` into a dataframe, optionally choosing from `pattern` + Loads datasets from ts files contained in `root_path` into a dataframe, + optionally choosing from `pattern` Args: root_path: directory containing all individual .ts files - file_list: optionally, provide a list of file paths within `root_path` to consider. + file_list: optionally, provide a list of file paths within + `root_path` to consider. Otherwise, entire `root_path` contents will be used. Returns: - all_df: a single (possibly concatenated) dataframe with all data corresponding to specified files + all_df: a single (possibly concatenated) dataframe with all data + corresponding to specified files labels_df: dataframe containing label(s) for each sample """ # Select paths for training and evaluation if file_list is None: - data_paths = glob.glob(os.path.join(root_path, '*')) # list of all paths + data_paths = glob.glob( + os.path.join( + root_path, + '*')) # list of all paths else: data_paths = [os.path.join(root_path, p) for p in file_list] if len(data_paths) == 0: - raise Exception('No files found using: {}'.format(os.path.join(root_path, '*'))) + raise Exception( + 'No files found using: {}'.format( + os.path.join( + root_path, '*'))) if flag is not None: data_paths = list(filter(lambda x: re.search(flag, x), data_paths)) - input_paths = [p for p in data_paths if os.path.isfile(p) and p.endswith('.ts')] + input_paths = [ + p for p in data_paths if os.path.isfile(p) and p.endswith('.ts')] if len(input_paths) == 0: - pattern='*.ts' - raise Exception("No .ts files found using pattern: '{}'".format(pattern)) + pattern = '*.ts' + raise Exception( + "No .ts files found using pattern: '{}'".format(pattern)) - all_df, labels_df = self.load_single(input_paths[0]) # a single file contains dataset + all_df, labels_df = self.load_single( + input_paths[0]) # a single file contains dataset return all_df, labels_df def load_single(self, filepath): - df, labels = load_from_tsfile_to_dataframe(filepath, return_separate_X_and_y=True, - replace_missing_vals_with='NaN') + df, labels = load_from_tsfile_to_dataframe( + filepath, + return_separate_X_and_y=True, + replace_missing_vals_with='NaN' + ) labels = pd.Series(labels, dtype="category") self.class_names = labels.cat.categories - labels_df = pd.DataFrame(labels.cat.codes, - dtype=np.int8) # int8-32 gives an error when using nn.CrossEntropyLoss + labels_df = pd.DataFrame( + labels.cat.codes, + # int8-32 gives an error when using nn.CrossEntropyLoss + dtype=np.int8 + ) lengths = df.applymap( - lambda x: len(x)).values # (num_samples, num_dimensions) array containing the length of each series + # (num_samples, num_dimensions) array containing the length of each + # series + len + ).values horiz_diffs = np.abs(lengths - np.expand_dims(lengths[:, 0], -1)) - if np.sum(horiz_diffs) > 0: # if any row (sample) has varying length across dimensions + if np.sum( + # if any row (sample) has varying length across dimensions + horiz_diffs) > 0: df = df.applymap(subsample) - lengths = df.applymap(lambda x: len(x)).values + lengths = df.applymap(len).values vert_diffs = np.abs(lengths - np.expand_dims(lengths[0, :], 0)) - if np.sum(vert_diffs) > 0: # if any column (dimension) has varying length across samples + if np.sum( + # if any column (dimension) has varying length across samples + vert_diffs) > 0: self.max_seq_len = int(np.max(lengths[:, 0])) else: self.max_seq_len = lengths[0, 0] - # First create a (seq_len, feat_dim) dataframe for each sample, indexed by a single integer ("ID" of the sample) - # Then concatenate into a (num_samples * seq_len, feat_dim) dataframe, with multiple rows corresponding to the - # sample index (i.e. the same scheme as all datasets in this project) + # First create a (seq_len, feat_dim) dataframe for each sample, indexed + # by a single integer ("ID" of the sample) + # Then concatenate into a (num_samples * seq_len, feat_dim) dataframe, + # with multiple rows corresponding to the sample index (i.e. the same + # scheme as all datasets in this project) - df = pd.concat((pd.DataFrame({col: df.loc[row, col] for col in df.columns}).reset_index(drop=True).set_index( - pd.Series(lengths[row, 0] * [row])) for row in range(df.shape[0])), axis=0) + df = pd.concat(( + pd.DataFrame( + {col: df.loc[row, col] for col in df.columns} + ).reset_index(drop=True).set_index( + pd.Series(lengths[row, 0] * [row]) + ) for row in range(df.shape[0]) + ), axis=0) # Replace NaN values grp = df.groupby(by=df.index) @@ -720,10 +736,17 @@ def load_single(self, filepath): return df, labels_df def instance_norm(self, case): - if self.root_path.count('EthanolConcentration') > 0: # special process for numerical stability + if self.root_path.count( + 'EthanolConcentration') > 0: # special process for numerical stability mean = case.mean(0, keepdim=True) case = case - mean - stdev = torch.sqrt(torch.var(case, dim=1, keepdim=True, unbiased=False) + 1e-5) + stdev = torch.sqrt( + torch.var( + case, + dim=1, + keepdim=True, + unbiased=False) + + 1e-5) case /= stdev return case else: @@ -737,12 +760,13 @@ def __getitem__(self, ind): num_columns = self.feature_df.shape[1] seq_len = int(self.feature_df.shape[0] / num_samples) batch_x = batch_x.reshape((1, seq_len, num_columns)) - batch_x, labels, augmentation_tags = run_augmentation_single(batch_x, labels, self.args) + batch_x, labels, augmentation_tags = run_augmentation_single( + batch_x, labels, self.args) batch_x = batch_x.reshape((1 * seq_len, num_columns)) return self.instance_norm(torch.from_numpy(batch_x)), \ - torch.from_numpy(labels) + torch.from_numpy(labels) def __len__(self): return len(self.all_IDs)