Player freezes mid air when not also moving with WASD

Current issue: Player freezes mid air unless you are also using WASD to move around. Also, if you are not moving and you press space it wont jump until you start to move. Here is my script:

using System;
using System.Collections;
using System.Collections.Generic;
using TMPro;
using UnityEngine;

public class ThirdPersonPlayerController : MonoBehaviour
{
    public CharacterController controller;
    public Transform camera;
    public Transform groundCheck;

    Vector3 direction;
    Vector3 velocity;

    public float moveSpeed = 10f;
    public float turnSmoothTime = 0.1f;
    private float turnSmoothVelocity;
    public float gravity = -39.24f;
    public float groundDistance = 0.4f;
    public float jumpHeight = 3f;

    public LayerMask groundMask;
    private bool isGrounded;

    // Update is called once per frame
    // Unity is currently working on a new input system***
    private void Update()
    {
        IsPlayerGrounded();

        float horizontal = Input.GetAxisRaw("Horizontal");
        float vertical = Input.GetAxisRaw("Vertical");
        direction = new Vector3(horizontal, 0f, vertical).normalized; // direction player is moving

        if (Input.GetButtonDown("Jump") && isGrounded)
        {
            Jump();
        }
    }

    private void FixedUpdate()
    {
        MoveAndRotate();
    }


    private void MoveAndRotate()
    {
        if (direction.magnitude >= 0.1f)
        {
            float targetAngle = Mathf.Atan2(direction.x, direction.z) * Mathf.Rad2Deg + camera.eulerAngles.y;
            float angle = Mathf.SmoothDampAngle(transform.eulerAngles.y, targetAngle, ref turnSmoothVelocity, turnSmoothTime);
            transform.rotation = Quaternion.Euler(0f, angle, 0f);
            //  Testing out gravity and this function as one

            Gravity();

            // end test
            Vector3 moveDirection = Quaternion.Euler(0f, targetAngle, 0f) * Vector3.forward;

            

            controller.Move(moveDirection.normalized * moveSpeed * Time.fixedDeltaTime + velocity * Time.fixedDeltaTime);
        }
    }

    // Implemented proper gravity from /_ y = 1/2g * t^2
    private void Gravity()
    {
        velocity.y += gravity * Time.fixedDeltaTime;

        /*controller.Move(velocity * Time.fixedDeltaTime);*/
    }

    // Check if player is grounded and reset velocity 
    private void IsPlayerGrounded()
    {
        isGrounded = Physics.CheckSphere(groundCheck.position, groundDistance, groundMask);

        if (isGrounded && velocity.y < 0)
        {
            velocity.y = -2f;
        }
    }

    private void Jump()
    {
        velocity.y = Mathf.Sqrt(jumpHeight * -2f * gravity);
    }
}

Any help would be much appreciated. I cannot seem to find any logical reason as to why this is happening!