0
\$\begingroup\$

I am working on a Pytorch LSTM model that is able to detect patterns in sequence of N variables that leads to a good outcome vs bad outcome.

The code is below. I tested it with a test case and it was able to detect a pattern that increasing relationship was 0 and decreasing was 1. But a review by experts to see if I am doing it correctly would help.

  • I would appreciate a review of the code to tell me if I am creating the model correctly
  • I am seeing that model runs very slow even if I use GPU. Do you see any optimization opportunities?
  • On the test case at a threshold of 0.6, I am able to 100% precision and recall. But when I use the Pytorch's PrecisionRecallCurve() utility to get the PR curve, it looks like attached figure which seems to suggest Recall is small. Any suggestions on what I am doing wrong?enter image description here

Code

Model

class LSTMModel(nn.Module):
    def __init__(self, input_dim, hidden_dim):
        super(LSTMModel, self).__init__()
        self.input_dim = input_dim
        self.hidden_dim = hidden_dim
        
        # LSTM
        self.lstm = nn.LSTM(input_dim, hidden_dim, batch_first=True)
        
        # Readout layer
        self.fc = nn.Linear(hidden_dim, 1)
        self.sigmoid = nn.Sigmoid()

    
    def forward(self, inputs):
        mlp_out, (hidden, _) = self.lstm(inputs)
        output = self.fc(hidden)
        output = self.sigmoid(output)
        return output

Dataset code

class LSTMDataset(Dataset):
  def __init__(self, x, y):
    self.x = x
    self.y = y

  def __len__(self):
    return len(self.y)
  
  def __getitem__(self,idx):
    inputs = [torch.from_numpy(self.x[idx][ts]).unsqueeze(0) for ts in range(len(self.x[idx]))]
    inputs = torch.cat(inputs)
    target = torch.Tensor([self.y[idx]])
    return inputs, target

Training code

# This function is used to train a single model
def train_lstm(
    X_training,
    y_training,
    X_testing,
    y_testing,
    hidden_dim=10,
    lr=1e-4,
    f1_thresh = 0.5,
    use_gpu=False,
    batch_size=100,
    num_epochs=4000,
    assym_wt=0, #Equal weights for 0 and 1
):
    start_time = time.time()
    clf = LSTMModel(len(X_training[0][0]), hidden_dim)
    # Move to GPU if available
    use_gpu = use_gpu and torch.cuda.is_available()
    device = torch.device("cuda" if use_gpu else "cpu")

    # Define the loss function and optimizer
    optimizer = torch.optim.Adam(clf.parameters(), lr=lr)
    clf = clf.to(device)
    loss_function = nn.BCELoss()
    loss_function = loss_function.to(device)

    dataset = LSTMDataset(X_training, y_training)
    trainloader = torch.utils.data.DataLoader(dataset, batch_size=batch_size, shuffle=True)

    # Run the training loop
    # per_epoch_precision = []
    # per_epoch_recall = []
    train_preds = []
    train_targets = []
    for epoch in range(0, num_epochs):
        # Set current loss value
        current_loss = 0.0
        
        # Iterate over the DataLoader for training data
        clf.train()  # set to train mode
        for i, data in enumerate(trainloader):
            # Get inputs
            inputs, targets = data                

            # Zero the gradients
            optimizer.zero_grad()

            # Perform forward pass
            outputs = clf(inputs)
            # Store predictions/targets in last epoch to compute accuracy stats 
            if epoch == num_epochs - 1:
                train_targets += targets
                train_preds += outputs.view(-1).tolist()
                

            # Compute loss
            targets = torch.FloatTensor(targets).unsqueeze(0)
            # Apply assymetric weights to handle unbalanced datasets
            if assym_wt > 0:
                loss_function = nn.BCELoss(weight=assym_wt * targets + 1)
                loss_function = loss_function.to(device)
            loss = loss_function(outputs, targets)
            
            # Perform backward pass
            loss.backward()

            # Perform optimization
            optimizer.step()

            # Print statistics
            current_loss += loss.item()

        if (epoch % 250) == 249:
            print("Loss after epoch %5d: %.3f" % (epoch + 1, current_loss / 500))
            current_loss = 0.0

    # Process is complete.
    print("Training process has finished.")
    train_preds = torch.FloatTensor(train_preds)
    train_targets = torch.FloatTensor(train_targets)

    # Calculate test set performance
    clf.eval()  # set to eval mode
    dataset = LSTMDataset(X_testing, y_testing)
    testloader = torch.utils.data.DataLoader(dataset, batch_size=batch_size, shuffle=True)
    test_preds = []
    test_targets = []
    with torch.no_grad():
        for i, data in enumerate(testloader):
            # Get inputs
            inputs, targets = data
            # Perform forward pass
            preds = clf(inputs)
            # Store predictions and targets so that we can compute stats
            test_preds += preds.view(-1).tolist()
            test_targets += targets
            
        
        test_preds = torch.FloatTensor(test_preds)
        test_targets = torch.FloatTensor(test_targets)
        pr_fig = go.Figure()
        pr_fig.update_xaxes(title_text="Recall")
        pr_fig.update_yaxes(title_text="Precision")
        plot_pr_withtorch(test_targets, test_preds, pr_fig, "Test PR ")
        plot_pr_withtorch(train_targets, train_preds, pr_fig, "Train PR ")
        pr_fig.show()
   
        # Get accuracy scores on train and test set
        train_F1 = get_scores(train_targets, train_preds > f1_thresh, "Train set scores")
        test_F1 = get_scores(test_targets, test_preds > f1_thresh, "Test set scores")
        print(f"Train time is {time.time() - start_time}")

    return train_F1, test_F1

Test case that shows the pre-processing steps that will be done on data and how the training is invoked

# Training Data:  Both variables decreasing classified as 0, else classified as 1
# FOrmat of data is [f1_ts0, f1_ts2, f2_ts0, f2_ts2, label]
train = np.array([[10, 20, 5, 10, 1], [10, 5, 8, 3, 0], [10, 5, 5, 10, 1]])
#create more examples with same pattern but different absolute values
for ind in range(0, 3):
    for spread in [5, 10, 50]:
        newrow = spread + train[ind]
        newrow[-1] -= spread
        train = np.vstack([train, newrow])

test = np.array([[0, 8, 3, 7, 1], [19, 8, 12, 3, 0], [1000, 450, 75, 135, 1]])
train_df = pd.DataFrame(train, columns = ['f1_1','f1_2','f2_1','f2_2', 'op'])
test_df = pd.DataFrame(test, columns = ['f1_1','f1_2','f2_1','f2_2', 'op'])
tc_Xtrain = train_df[train_df.columns[~train_df.columns.isin(['op'])]]
tc_ytrain = train_df['op'].astype(np.float32)
tc_Xtest = test_df[test_df.columns[~test_df.columns.isin(['op'])]]
tc_ytest = test_df['op'].astype(np.float32)

#normalize values
scaler = StandardScaler()
tc_Xtrain = scaler.fit_transform(tc_Xtrain)
tc_Xtest = scaler.transform(tc_Xtest)
X_train = np.apply_along_axis(create_ts_features, 1, tc_Xtrain, num_features=2).astype(np.float32)
X_test = np.apply_along_axis(create_ts_features, 1, tc_Xtest, num_features=2).astype(np.float32)


train_lstm(X_train, tc_ytrain, X_test, tc_ytest, num_epochs=4000, f1_thresh=0.6, batch_size=100)

Metric computation

def plot_pr_withtorch(target, pred, fig, title):
    pr_curve = PrecisionRecallCurve(pos_label=1)
    precision, recall, thresholds = pr_curve(pred, target)
    print_key_prs(precision, recall, thresholds, title)
    N = len(recall)
    fig.add_trace(go.Scatter(x=recall[0 : N - 1], y=precision[0 : N - 1], name=title))


def get_scores(y, y_preds, print_label):
    print(f"{print_label} summary:")
    confusion = confusion_matrix(y, y_preds)
    print(f"Confusion matrix: {confusion}")
    print("Accuracy: {:.2f}".format(accuracy_score(y, y_preds)))
    print("Precision: {:.2f}".format(precision_score(y, y_preds)))
    print("Recall: {:.2f}".format(recall_score(y, y_preds)))
    F1_score = f1_score(y, y_preds)
    print("F1: {:.2f}".format(f1_score(y, y_preds)))
    return F1_score
\$\endgroup\$

0

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Browse other questions tagged or ask your own question.