Training loss is increasing while testing loss is decreasing and Accuracy stands still

I am following the Udacity Intro to Pytorch on Exercise 7 which is to make a model which can recognize dogs or cats.

Here is my code.

import matplotlib.pyplot as plt
import torch
from torchvision import datasets, transforms

# %%
base_path = './assets/'
data_dir = 'Cat_Dog_data'

train_folder = base_path + data_dir

# TODO: Define transforms for the training data and testing data
train_transforms = transforms.Compose([
    transforms.RandomRotation(30),
    transforms.RandomResizedCrop(224),
    transforms.RandomHorizontalFlip(),
    transforms.ToTensor(),
    transforms.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5])
])

test_transforms = transforms.Compose([
    transforms.Resize(255),
    transforms.CenterCrop(224),
    transforms.ToTensor(),
    transforms.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5])
])


# Pass transforms in here, then run the next cell to see how the transforms look
train_data = datasets.ImageFolder(train_folder + '/train', transform=train_transforms)
test_data = datasets.ImageFolder(train_folder + '/test', transform=test_transforms)

trainloader = torch.utils.data.DataLoader(train_data, batch_size=32)
testloader = torch.utils.data.DataLoader(test_data, batch_size=32)

# %%
# Optional TODO: Attempt to build a network to classify cats vs dogs from this dataset
from torch import nn, optim
import torch.nn.functional as F

class Classifier(nn.Module):
    def __init__(self):
        super().__init__()
        self.fc1 = nn.Linear(150528, 512)  # use a closer number to 150525 as output features
        self.fc2 = nn.Linear(512, 256)
        self.fc3 = nn.Linear(256, 128)
        self.fc4 = nn.Linear(128, 64)
        self.fc5 = nn.Linear(64, 32)
        self.fc6 = nn.Linear(32, 2)

        self.dropOut = nn.Dropout(p=0.2)

    def forward(self, x):
        # x.shape = [64, 3, 224, 224]
        x = x.view(x.shape[0], -1) # reshape to [63, 3 * 224 * 224]
        output = self.dropOut(F.relu(self.fc1(x)))
        output = self.dropOut(F.relu(self.fc2(output)))
        output = self.dropOut(F.relu(self.fc3(output)))
        output = self.dropOut(F.relu(self.fc4(output)))
        output = self.dropOut(F.relu(self.fc5(output)))

        output = F.log_softmax(self.fc6(output), dim=1)

        return output


# %%
epochs = 5
model = Classifier()
# model.cuda()

criterion = nn.NLLLoss()
optimizer = optim.Adam(model.parameters(), lr = 0.0001)

train_losses, test_losses = [], []

printed = False

for e in range(epochs):
    running_loss = 0
    for images, labels in trainloader:
        
        optimizer.zero_grad()
        log_ps = model(images) 

        # print(f'labels shape: {labels.shape}')# [32]
        # print(f'log_ps shape: {log_ps.shape}')# [32,1]
        # log_ps = log_ps.view(-1)
    
        loss = criterion(log_ps, labels)
        loss.backward()
        optimizer.step()

        running_loss += loss.item()

    else:
        test_loss = 0
        accuracy = 0

        with torch.no_grad():
            model.eval()
            for images, labels in testloader:
                log_ps = model(images)

                if printed == False:
                    print(f'images.shape:{images.shape}')
                    printed = True
                    
                test_loss += criterion(log_ps, labels)
                ps = torch.exp(log_ps)

                top_p, top_class = ps.topk(1, dim=1)
                equals = top_class == labels.view(*top_class.shape)
                accuracy += torch.mean(equals.type(torch.FloatTensor))

        model.train()

        train_losses.append(running_loss/len(trainloader))
        test_losses.append(test_loss/len(testloader))

        print("Epoch: {}/{}.. ".format(e+1, epochs),
              "Training Loss: {:.3f}.. ".format(train_losses[-1]),
              "Test Loss: {:.3f}.. ".format(test_losses[-1]),
              "Test Accuracy: {:.3f}".format(accuracy/len(testloader)))

Here is the output:

images.shape:torch.Size([32, 3, 224, 224]) 
Epoch: 1/5..  Training Loss: 0.152..  Test Loss: 15.427..  Test Accuracy: 0.506 
Epoch: 2/5..  Training Loss: 0.435..  Test Loss: 13.295..  Test Accuracy: 0.506 
Epoch: 3/5..  Training Loss: 0.484..  Test Loss: 14.875..  Test Accuracy: 0.506 
Epoch: 4/5..  Training Loss: 0.542..  Test Loss: 15.111..  Test Accuracy: 0.506 
Epoch: 5/5..  Training Loss: 0.617..  Test Loss: 10.860..  Test Accuracy: 0.506

I am confused that why the test accuracy isn’t improving and training loss is increasing.