Skip to content

GoogleAPI startRow/paging support and Data capture loop to import multiple dates #54

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
pfauenauge wants to merge 1 commit into
base: master
Choose a base branch
from
Open

GoogleAPI startRow/paging support and Data capture loop to import multiple dates #54

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions organic-search-analytics/apis/Google/Service/Webmasters.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -738,6 +738,7 @@ class Google_Service_Webmasters_SearchAnalyticsQueryRequest extends Google_Colle
public $dimensions;
public $endDate;
public $rowLimit;
public $startRow;
public $searchType;
public $startDate;

Expand Down Expand Up @@ -782,6 +783,14 @@ public function getRowLimit()
{
return $this->rowLimit;
}
public function setStartRow($startRow)
{
$this->startRow = $startRow;
}
public function getStartRow()
{
return $this->startRow;
}
public function setSearchType($searchType)
{
$this->searchType = $searchType;
Expand Down
65 changes: 51 additions & 14 deletions organic-search-analytics/data-capture-run.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@

if( isset( $_GET ) && is_array( $_GET ) && count( $_GET ) > 0 ) {
$params = $_GET;
} elseif( isset( $argv ) && is_array( $argv ) && count( $argv ) >= 4 ) {
} elseif( isset( $argv ) && is_array( $argv ) && count( $argv ) >= 3 ) {
$params = array();
$params['type'] = $argv[1];
$params['domain'] = $argv[2];
Expand All @@ -28,17 +28,20 @@
if( isset( $argv[8] ) ) {
$params['search_type'] = $argv[8];
}
if( isset( $argv[9] ) ) {
$params['start_row'] = $argv[9];
}
}


if( isset($params) && isset($params['type']) && isset($params['domain']) && isset($params['date']) && preg_match("/^[0-9]{4}-(0[1-9]|1[0-2])-(0[1-9]|[1-2][0-9]|3[0-1])$/",$params['date']) ) {
if( isset($params) && isset($params['type']) && isset($params['domain']) /*&& isset($params['date'])*/ && (preg_match("/^[0-9]{4}-(0[1-9]|1[0-2])-(0[1-9]|[1-2][0-9]|3[0-1])$/",$params['date']) || $params['date'] == null) ) {
/* Set the max allowed execution time for the page to allow for longer procesing times. */
ini_set('max_execution_time', 600); //300 seconds = 5 minutes

/* Set overrides from URL paramters */
$overrideSettings = array();
if( isset( $params['mode'] ) ) { $overrideSettings['mode'] = $params['mode']; }
if( isset( $params['row_limit'] ) ) { $overrideSettings['row_limit'] = $params['row_limit']; }
if( isset( $params['start_row'] ) ) { $overrideSettings['start_row'] = $params['start_row']; }
if( isset( $params['dimensions'] ) ) { $overrideSettings['dimensions'] = explode( ',', $params['dimensions'] ); }
if( isset( $params['search_type'] ) ) { $overrideSettings['search_type'] = explode( ',', $params['search_type'] ); }
if( isset( $params['filters'] ) ) {
Expand All @@ -56,16 +59,50 @@

switch( $params['type'] ) {
case 'googleSearchAnalytics':
$recordsImported = $dataCapture->downloadGoogleSearchAnalytics( $params['domain'],$params['date'], $overrideSettings );
if( !isset( $params['mode'] ) || $params['mode'] != 'return' ) {
switch( $recordsImported ) {
case -1:
echo "There was an error in authorizing your API connection.";
break;
default:
echo number_format( $recordsImported ) . " records succesfully imported to the database for " . $params['domain'] . " for date: " . $params['date'] . ".";
}
}
if( $params['date'] == '' ) {
$maxDaysBatchImport = 1;
if( defined( 'config::maxDaysBatchImport' ) ) {
$maxDaysBatchImport = min(max(config::maxDaysBatchImport, 1), 999);
}
$googleSearchAnalyticsDates = $dataCapture->checkNeededDataGoogleSearchAnalytics($params['domain']);
if( count( $googleSearchAnalyticsDates['datesWithNoData'] ) > 0 ) {
$i=0;
foreach( $googleSearchAnalyticsDates['datesWithNoData'] as $date ) {
if( $i < $maxDaysBatchImport ) {
echo "\n".$date.': ';
$recordsImported = $dataCapture->downloadGoogleSearchAnalyticsPaged( $params['domain'], $date, $overrideSettings );
if( !isset( $params['mode'] ) || $params['mode'] != 'return' ) {
switch( $recordsImported ) {
case -1:
echo "There was an error in authorizing your API connection.";
break;
default:
echo number_format( $recordsImported ) . " records succesfully imported to the database for " . $params['domain'] . " for date: " . $params['date'] . ".";
}
}
else {
break;
}
sleep(1); //make sure Google limits are not hit
}
$i++;
}
}
}
else {
$recordsImported = $dataCapture->downloadGoogleSearchAnalyticsPaged( $params['domain'],$params['date'], $overrideSettings );
if( !isset( $params['mode'] ) || $params['mode'] != 'return' ) {
switch( $recordsImported ) {
case -1:
echo "There was an error in authorizing your API connection.";
break;
default:
echo number_format( $recordsImported ) . " records succesfully imported to the database for " . $params['domain'] . " for date: " . $params['date'] . ".";
}
}
}


break;
case 'bingSearchKeywords':
$recordsImported = $dataCapture->downloadBingSearchKeywords($params['domain'],$params['date'], $overrideSettings);
Expand Down
39 changes: 36 additions & 3 deletions organic-search-analytics/inc/code/dataCapture.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,8 @@ private function defaultGoogleSearchAnalyticsSettings() {
return array(
'mode' => 'import', /* What to do with the data. Valid options: import, return */
'dimensions' => $this->getDimensions(),
'row_limit' => 5000 /* Number of rows to capture from Google. Valid options: 1-5000 */
'row_limit' => 5000, /* Number of rows to capture from Google. Valid options: 1-5000 */
'start_row' => 0 /* Start rows to capture from Google. Valid options: >=0 */
);
}

Expand Down Expand Up @@ -204,6 +205,36 @@ public function checkNeededDataGoogleSearchAnalytics($website) {
return $returnArray;
}

/**
* Request Google Search Analytics API and loop through (max. 20) pages
*
* @param $website String Website URL that is enabled in Google Search Console
* @param $date Date (YYYY-MM-DD) Date for which to request data
* @param $overrides Array Values to override default settings for request
*
* @returns Integer,array Number of records found or var_dump of returned data from Google depending on mode
*/
public function downloadGoogleSearchAnalyticsPaged( $website, $date, $overrides = array() ) {
$params = array_merge( $this->defaultGoogleSearchAnalyticsSettings(), $overrides );

$recordsImported = null;
$recordsImportedTotal = 0;
$maxPages = 1;
if( defined( 'config::downloadGoogleMaxPages' ) ) {
$maxPages = max(config::downloadGoogleMaxPages, 1);
}
for ($page = 0; (is_null($recordsImported) || $recordsImported > 0) && $page < $maxPages; $page++) {
$recordsImported = $this->downloadGoogleSearchAnalytics( $website, $date, $overrides );
if( $recordsImported > 0 ) {
$recordsImportedTotal = $recordsImportedTotal + $recordsImported;
$overrides['start_row'] = $params['row_limit']*($page+1);
}
else {
break;
}
}
return $recordsImportedTotal;
}

/**
* Request Google Search Analytics API
Expand Down Expand Up @@ -239,11 +270,13 @@ public function downloadGoogleSearchAnalytics( $website, $date, $overrides = arr
/* Build Search Analytics Request */
$searchAnalyticsRequest->setDimensions( $params['dimensions'] );
$searchAnalyticsRequest->setRowLimit( $params['row_limit'] ); /* Valid options: 1-5000 */
$searchAnalyticsRequest->setStartRow( $params['start_row'] ); /* Valid options: >=0 */


/* Set date for Search Analytics Request */
$searchAnalyticsRequest->setStartDate( $date );
$searchAnalyticsRequest->setEndDate( $date );

if( isset( $params['filters'] ) || isset( $params['groups'] ) ) {
$searchAnalyticsDimensionFilterGroup = new Google_Service_Webmasters_ApiDimensionFilterGroup;
}
Expand All @@ -260,7 +293,7 @@ public function downloadGoogleSearchAnalytics( $website, $date, $overrides = arr
$searchAnalyticsDimensionFilterGroup->setFilters( $filters );
$searchAnalyticsRequest->setDimensionFilterGroups( array( $searchAnalyticsDimensionFilterGroup ) );
}

if( isset( $params['groups'] ) ) {
/* TODO */
// $dimensionFilterGroups['groups'] = $params['groups'];
Expand Down