Import CSV File into Google Cloud SQL from Cloud Storage

December 03, 2017

You can upload one or more CSV files to a specific bucket in Google Cloud Storage and then use Google Apps Script to import the CSV files from Cloud Storage into your Google Cloud SQL database.

In the method here, the CSV file is deleted from Cloud Storage after the import operation is complete. You can however call the /copyTo/ endpoint to move the CSV files into another Cloud Storage folder after processing.

It is important to add a wait (sleep) function because the API will throw an error if you begin uploading another file while the previous import operation is pending. The file names must be encoded as well.

function uploadtoCloudSQL() {
  // Written by Amit Agarwal amit@labnol.org
  // Web: www.ctrlq.org

  var service = getService();

  if (!service.hasAccess()) {
    Logger.log(service.getAuthorizationUrl());
    return;
  }

  var token = service.getAccessToken();

  // Getting list of files to be processed
  var result = JSON.parse(
    UrlFetchApp.fetch('https://www.googleapis.com/storage/v1/b/BUCKET_NAME/o', {
      method: 'GET',
      headers: {
        Authorization: 'Bearer ' + token,
      },
    }).getContentText()
  );

  for (var i = 0; i < result.items.length; i++) {
    if (result.items[i].name.indexOf('.') !== -1) {
      files.push(result.items[i].name);
    }
  }

  for (var f = 0; f < files.length; f++) {
    var path = files[f].split('/');

    var payload =
      '{"importContext" : { "csvImportOptions": {"table":"MY_TABLE"}, "fileType": "CSV", "database": "MY_DATABASE", "uri": "gs://BUCKET_NAME/FOLDER/CSVFILE"}}'
        .replace('FOLDER', path[0])
        .replace('CSVFILE', path[1]);

    UrlFetchApp.fetch('https://www.googleapis.com/sql/v1beta4/projects/PROJECT/instances/INSTANCE/import', {
      method: 'POST',
      contentType: 'application/json',
      headers: {
        Authorization: 'Bearer ' + token,
      },
      payload: payload,
      muteHttpExceptions: true,
    });

    UrlFetchApp.fetch('https://www.googleapis.com/storage/v1/b/BUCKET_NAME/o/' + encodeURIComponent(files[f]), {
      method: 'DELETE',
      headers: {
        Authorization: 'Bearer ' + token,
      },
    });

    // Wait for the previous import job to end
    Utilities.sleep(5000);
  }
}

Google Cloud Google Apps Script

ShareWhatsAppTwitterFacebook

Amit Agarwal

Google Developer Expert, Google Cloud Champion

Amit Agarwal is a Google Developer Expert in Google Workspace and Google Apps Script. He holds an engineering degree in Computer Science (I.I.T.) and is the first professional blogger in India.

Amit has developed several popular Google add-ons including Mail Merge for Gmail and Document Studio. Read more on Lifehacker and YourStory