Player jitters when Jumping at object edges

So I have a bit of an issue with my Player controller script. Everything works fine except with very specific jump cases. If I jump at an object and I don’t reach the top edge I will just fall normally with no issue. However, when the object is short enough for me to possible jump on top of it, if I collide with its side when jumping my player will halt in that spot, jitter for a second and then fall normally.

Here is my code:

using System;
using System.Collections;
using System.Collections.Generic;
using UnityEngine;

public class ThirdPersonPlayerController : MonoBehaviour
{
    public CharacterController controller;
    public Transform camera;
    public Transform groundCheck;

    Vector3 direction;
    Vector3 velocity;

    public float moveSpeed = 10f;
    public float turnSmoothTime = 0.1f;
    private float turnSmoothVelocity;
    public float gravity = -39.24f;
    public float groundDistance = 0.4f;
    public float jumpHeight = 3f;

    public LayerMask groundMask;
    private bool isGrounded;

    // Update is called once per frame
    // Unity is currently working on a new input system***
    private void Update()
    {
        IsPlayerGrounded();

        float horizontal = Input.GetAxisRaw("Horizontal");
        float vertical = Input.GetAxisRaw("Vertical");
        direction = new Vector3(horizontal, 0f, vertical).normalized; // direction player is moving

        if (Input.GetButtonDown("Jump") && isGrounded)
        {
            Jump();
        }

        MoveAndRotate();
        Gravity();
    }
    

    private void MoveAndRotate()
    {
        if (direction.magnitude >= 0.1f)
        {
            float targetAngle = Mathf.Atan2(direction.x, direction.z) * Mathf.Rad2Deg + camera.eulerAngles.y;
            float angle = Mathf.SmoothDampAngle(transform.eulerAngles.y, targetAngle, ref turnSmoothVelocity, turnSmoothTime);
            transform.rotation = Quaternion.Euler(0f, angle, 0f);

            Vector3 moveDirection = Quaternion.Euler(0f, targetAngle, 0f) * Vector3.forward;
            controller.Move(moveDirection.normalized * moveSpeed * Time.deltaTime);
        }
    }

    // Implemented proper gravity from /_ y = 1/2g * t^2
    private void Gravity()
    {
        velocity.y += gravity * Time.deltaTime;

        controller.Move(velocity * Time.deltaTime);
    }

    // Check if player is grounded and reset velocity 
    private void IsPlayerGrounded()
    {
        isGrounded = Physics.CheckSphere(groundCheck.position, groundDistance, groundMask);

        if (isGrounded && velocity.y < 0)
        {
            velocity.y = -2f;
        }
    }

    private void Jump()
    {
        velocity.y = Mathf.Sqrt(jumpHeight * -2 * gravity);
    }

    // Current issue (Not yet fixed): When jumping against a wall the player will stick/jitter (starts to jitter at the top edge). 
}

Here is a visual representation of what I mean by colliding near its top edge

Any help at all is greatly appreciated. Note: I thought it had something to do with objects having the Ground layer attached to them, but the bug is still present if they don’t have a layer.