Column-Design-Optimization/normalization.py at main · IS2AI/Column-Design-Optimization · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
import csv
import os
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import math
pd.options.display.max_columns = None
pd.options.display.max_rows = None

# Load the pre-processsed dataset
df = pd.read_hdf('pre-processed.h5')

# Shuffling the dataset
df = df.sample(frac=1).reset_index(drop=True)

# Splitting the first 85% of the dataset for train set
train = df[:round(len(df)*0.85)]
# Min-max normalization of the train set
train_norm = (train - train.min())/(train.max() - train.min())
train_min = train.min()
train_max = train.max()
# Save the train set
train_norm.to_hdf("train_norm.h5", key='w')

# Save train set min and max values for back normalization of the inference cases
train_min.to_hdf("train_min.h5", key='w')
train_max.to_hdf("train_max.h5", key='w')

# Splitting and normalizing the validation set
val = df[round(len(df)*0.85):round(len(df)*0.95)]
val.to_hdf("val.h5", key='w')
val_norm = (val - train.min()) / (train.max() - train.min())
# Save the validation set
val_norm.to_hdf("val_norm.h5", key='w')

# Splitting and normalizing the validation set
test = df[round(len(df)*0.95):]
test.to_hdf("test.h5", key='w')
test_norm = (test - train.min()) / (train.max() - train.min())
# Save the test set
test_norm.to_hdf("test_norm.h5", key='w')