Google Earth Engine memory capacity exceeded

Question

I am trying to export to a csv table with total precipitation per month-year per polygon (~5571 polygons) from 2001-2019 (228 time steps). I am using the CHIRPS dataset (daily data) so had to reduce the data to mean month-year.
I can download about two months of data but when I try to download 2+ years of data I get the message: Error: User memory limit exceeded.
I have tried every suggestion I have found searching online.
Is there a way to rewrite my code (linked here) to not get this error message?
// Set years and month
var startYear = 2001;
var endYear = 2019; //need to run in subsets?
var years = ee.List.sequence(startYear, endYear);
var months = ee.List.sequence(1,12);

// load the image collection
var Daily = ee.ImageCollection("UCSB-CHG/CHIRPS/DAILY")

//~5571 features (polygons)  
var municipalities = ee.FeatureCollection("shapefiles")

//rescale features to help with memory error
var municipScaled = municipalities.map(function(feature) {
  return feature.simplify(100);
});

// make monthly summed mosaics
// loop over the years and months to get summed monthly images
var byMonth = ee.ImageCollection(ee.FeatureCollection(years.map(function(y){
  var yearCollection = Daily.filter(ee.Filter.calendarRange(y, y, 'year'));
  var byYear = ee.ImageCollection.fromImages(
    months.map(function(m) {
      var summedImage = yearCollection.filter(ee.Filter.calendarRange(m, m, 'month'))
                  .reduce(ee.Reducer.sum()); 
      var date = ee.Date.fromYMD(y, m, 1).format("MM_dd_YYYY");
      return summedImage.set('system:time_start', ee.Date.fromYMD(y, m, 1)).rename(ee.String("summed_precip")
                                       .cat(date));
      //.set('month', m).set('year', y); // eventually set year and month 
  }));
  return byYear;
})).flatten());
print(byMonth)

// filter the empty one out
var outputMonthly = byMonth.filter(ee.Filter.listContains('system:band_names', 'constant').not())
                    .sort('system:time_start').toBands();
print(outputMonthly);

//test to make sure range makes sense
//Map.addLayer(outputMonthly.select("0_0_summed_precip01_01_2001"), {min:0, max:50});

//determine scale
var scale = Daily.first().projection().nominalScale();
print(scale)

//reduce to total precipitation per municipality 
var muncip_monthly_precip = outputMonthly.reduceRegions(municipScaled, ee.Reducer.sum(), 5000, 'EPSG:4326')
                                .map(function(feature){
                      return(ee.Feature(feature).setGeometry(null)); // map over the feature collection and drop the geometry for memory saving
                    }).copyProperties(municipScaled, ee.List(["CD_MUN"]));

// save the table to google drive                    
Export.table.toDrive({
  collection: muncip_monthly_precip,
  description: "total_monthly_precip",
  folder: 'VL_GEE',
  fileFormat: 'CSV'})

JonasV · Accepted Answer

Your issue is most likely this line:
var outputMonthly = byMonth.filter(ee.Filter.listContains('system:band_names', 'constant').not())
                .sort('system:time_start').toBands();

I assume you are using toBands() to be able to easily call .reduceRegions(). The big downside is, that Earth Engine has to allocate a ridiculously giant ee.Image with over 200 bands spanning the entire earth. Instead you should keep it as an Image Collection and map over it, like this:
var outputMonthly = byMonth.filter(ee.Filter.listContains('system:band_names', 'constant').not())
                .sort('system:time_start')

var muncip_monthly_precip = outputMonthly.map(function(image){
  return image.reduceRegions({
    collection: municipScaled, 
    reducer: ee.Reducer.sum(), 
    scale: 5000
  })
})
var flat = muncip_monthly_precip.flatten()

In the end you have to flatten it, to go from a collection of collections to a flat collection of features.
If this still fails, I suggest to get rid of all print statements and if it still fails it's likely that municipScaled simply has too many, too big features.
Anyway, I hope this helps.

margie · Answer

Using this code (increasing tileScale) also worked for exporting to table without getting the user memory error.
// Set years and month
var startYear = 2001;
var endYear = 2019; 
var years = ee.List.sequence(startYear, endYear);
var months = ee.List.sequence(1,12);

// make monthly summed mosaics
// loop over the years and months to get summed monthly images
var byMonthYear = ee.ImageCollection.fromImages(
  years.map(function(y) {
    return months.map(function (m) {
      var date = ee.Date.fromYMD(y, m, 5).format("MM_dd_YYYY");
      return Daily
        .filter(ee.Filter.calendarRange(y, y, 'year'))
        .filter(ee.Filter.calendarRange(m, m, 'month'))
        .reduce(ee.Reducer.sum())
        .set('month', m).set('year', y)
        .rename(ee.String("summed_precip")
                                       .cat(date));
  });
}).flatten());

//print(byMonthYear)

var singleImage = byMonthYear.toBands();

var muncip_monthly_precip = singleImage.reduceRegions({
  collection: municipalities, 
  reducer: ee.Reducer.sum(), 
  scale : 5000,
  tileScale : 16
})
                                .map(function(feature){
                      return(ee.Feature(feature).setGeometry(null)); // map over the feature collection and drop the geometry for memory saving
                    }).copyProperties(municipalities, ee.List(["CD_MUN"]));

Google Earth Engine memory capacity exceeded

2 Answers

Add your own answers!

Ask a Question