I have written a Google Apps Script that will automatically upload data from one or more files in your Google Drive to your BigQuery table. This script looks for CSV file in a particular Drive Folder, uploads them to BigQuery tablet and then moves the file to another folder in Drive to indicate that it has been processed.
You need to have a table read in your BigQuery dataset and the BigQuery API needs to be enabled inside the Google Developer Console. If you are getting errors, you probably need to enable the Billing option under the BigQuery API settings. You won’t be charged but the credit cards needs to be added for you to create datasets and tables inside BigQuery.
function importCSVtoBigQuery() {
try {
var CSVFolder = '\\Data\\BigQuery';
var ProcessedFolder = '\\Data\\BigQuery\\Processed';
var projectId = '1234';
var datasetId = 'bigquery_dataset_1';
var tableId = 'bigquery_table_1';
CSVFolder = getDriveFolder(CSVFolder);
ProcessedFolder = getDriveFolder(ProcessedFolder);
if (CSVFolder && ProcessedFolder) {
var data,
job,
file,
files = CSVFolder.getFiles();
while (files.hasNext()) {
file = files.next();
if (file.getMimeType() === 'text/csv') {
data = file.getBlob().setContentType('application/octet-stream');
job = {
configuration: {
load: {
destinationTable: {
projectId: projectId,
datasetId: datasetId,
tableId: tableId
},
skipLeadingRows: 1
}
}
};
job = BigQuery.Jobs.insert(job, projectId, data);
file.makeCopy(file.getName(), ProcessedFolder);
file.setTrashed(true);
Logger.log('Job status for %s https://bigquery.cloud.google.com/jobs/%s', file.getName(), projectId);
}
}
}
} catch (e) {
Logger.log(e.toString());
}
}
// Return the ID of the Google Drive nested folder
function getDriveFolder(name) {
var results,
folders = name.split('\\');
var folder = DriveApp.getRootFolder();
for (var i = 0; i < folders.length; i++) {
if (folders[i] === '') continue;
results = folder.getFoldersByName(folders[i]);
if (results.hasNext()) {
folder = results.next();
} else {
folder = folder.createFolder(folders[i]);
}
}
return folder;
}