Extract Raster Values to Matching Points in a FeatureCollection

I am trying to extract time series data to a set of points that I have uploaded to GEE as an asset. The catch is that I only want to sample the raster layer that aligns temporally with a given point. I do not want the entire time series extracted to every point.

For example, if I have two points dated 1/10/2020 and 3/16/2020, I am hoping to end up with a shapefile or table containing the original points and a singular raster value for each point, with raster values extracted from the image that matches the date of the indivdual points. Is it possible to do something like that?

// REFORMAT DATES FOR ALL POINTS IN THE SHAPEFILE
var BurnPermits = BurnPermits.map(function (feature) {
  return feature.set('system:time_start', ee.Date(feature.get('DATE')))
})

// PRINT ATTRIBUTES TO SEE IF DATE IS FORMATTED CORRECTLY
Map.addLayer(BurnPermits)

var Permits2020 = BurnPermits.filterDate('2020-01-01','2020-12-30')
print(Permits2020)
print(TestPermit)

var GRIDMET_2020 = GRIDMET.filterBounds(AOI).filterDate("2020-01-01","2020-12-31")
//print(GRIDMET_2020)

// This function adds a band representing the image timestamp.
var addTime = function(image) {
  return image.addBands(image.metadata('system:time_start'));
};

var GRIDMET_w_TIME = GRIDMET_2020.map(addTime)
print(GRIDMET_w_TIME)

// FUNCTION TO EXTRACT GRIDMET VALUES TO POINTS THAT ALIGN TEMPORALLY
var ExtractToPoint_ByDate = function(ft){
  // CREATE GEOMETRY FROM FEATURE
  var ft_geometry = ft.geometry()
  // DEFINE START AND END DATES
  var start_date = ft.get('DATE')
  // USE START AND END DATES TO FILTER GRIDMET COLLECTION
  var image_for_permit = GRIDMET_w_TIME.filterDate(start_date,'2020-12-31').first()
  // CREATE A NEW POINT FILE THAT HAS THE GRIDMET VALUES EXTRACTED
  var permits_w_GRIDMET = image_for_permit.sample({
  region: ft_geometry,
  scale: 4000,
  geometries:true
});
  return permits_w_GRIDMET
}

var Permits_w_GRIDMET = Permits2020.map(ExtractToPoint_ByDate).flatten()

// REFORMAT DATES FOR ALL POINTS IN THE SHAPEFILE
var Permits_w_GRIDMET = Permits_w_GRIDMET.map(function (feature) {
  return feature.set('DATE', ee.Date(feature.get('system:time_start')))
})

print(Permits_w_GRIDMET)

Full Script Here