From 2ef60cc0165d99776d41b91c7119a2da2f9158ff Mon Sep 17 00:00:00 2001 From: Phil Hopper Date: Thu, 9 Jan 2020 11:30:22 -0500 Subject: [PATCH 1/7] Keep below the API rate limit --- src/CurlClient.php | 61 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 60 insertions(+), 1 deletion(-) diff --git a/src/CurlClient.php b/src/CurlClient.php index 364b34d..0be7678 100644 --- a/src/CurlClient.php +++ b/src/CurlClient.php @@ -20,6 +20,9 @@ class CurlClient const HTTP_ERROR = 500; const RATE_LIMIT_RESET = "x-rate-limit-reset"; + const RATE_LIMIT = "x-rate-limit"; + const RATE_LIMIT_REMAINING = "x-rate-limit-remaining"; + const DEFAULT_RATE_LIMIT = 180; /** * @var array of headers sent with HTTP requests @@ -27,6 +30,15 @@ class CurlClient private $_requestHeaders = array(); /** + * @var array of headers received with HTTP responses + */ + private $_responseHeaders = array(); + + /** + * @var bool keep the request rate below the API rate limit + */ + private $_keepBelowRateLimit; + * @var array of extra CURL options */ private $_curlOptions = array(); @@ -36,9 +48,10 @@ class CurlClient */ private $_lastStatusCode; - public function __construct($apiKey = "", $siteID = "") + public function __construct($apiKey = "", $siteID = "", $keepBelowRateLimit = true) { $this->setCredentials($apiKey, $siteID); + $this->_keepBelowRateLimit = $keepBelowRateLimit; } /** @@ -196,6 +209,11 @@ public function httpRequest($requestParams, $url, $method, $requiredParams, $opt return false; } + if ($this->_keepBelowRateLimit) + { + $this->_checkRateLimit(); + } + if ($options && is_array($options) && array_key_exists("headers", $options) && @@ -278,6 +296,8 @@ function($curl, $header) use (&$headers) $result = curl_exec($curlHandle); + $this->_responseHeaders = $headers; + $this->_lastStatusCode = curl_getinfo($curlHandle, CURLINFO_HTTP_CODE); if ($this->_lastStatusCode == self::HTTP_RATE_LIMIT) { @@ -325,4 +345,43 @@ public function retry($wait, $curlHandle) $this->_lastStatusCode = curl_getinfo($curlHandle, CURLINFO_HTTP_CODE); return $result; } + + private function _checkRateLimit() + { + // If no API calls have been made, no need to delay. + if (empty($this->_responseHeaders)) + { + return; + } + + // Default the $limit and $remaining values if not set in the last response header. + /** @var int $limit */ + $limit = isset($this->_responseHeaders[self::RATE_LIMIT]) + ? (int)$this->_responseHeaders[self::RATE_LIMIT] + : self::DEFAULT_RATE_LIMIT; + + /** @var int $remaining */ + $remaining = isset($this->_responseHeaders[self::RATE_LIMIT_REMAINING]) + ? (int)$this->_responseHeaders[self::RATE_LIMIT_REMAINING] + : self::DEFAULT_RATE_LIMIT; + + // If no API calls have been made, no need to delay. + if ($limit == $remaining) + { + return; + } + + // If we are below 5% remaining, sleep for 0.50 seconds. + if ($remaining / $limit < 0.05) + { + usleep(500000); + return; + } + + // If we are below 10% remaining, sleep for 0.25 seconds. + if ($remaining / $limit < 0.1) + { + usleep(250000); + } + } } From acb746110c126851a8aeec13a1fa2b0fdca44e2e Mon Sep 17 00:00:00 2001 From: Phil Hopper Date: Wed, 15 Jan 2020 09:00:00 -0500 Subject: [PATCH 2/7] Add curl debug logging --- src/CurlClient.php | 48 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 47 insertions(+), 1 deletion(-) diff --git a/src/CurlClient.php b/src/CurlClient.php index 0be7678..89d19a9 100644 --- a/src/CurlClient.php +++ b/src/CurlClient.php @@ -43,15 +43,26 @@ class CurlClient */ private $_curlOptions = array(); + /** + * @var bool log curl requests and responses + */ + private $_debug; + + /** + * @var string file name for curl debugging log + */ + private $_logFile = '/tmp/curl_data.log'; + /** * @var int the last HTTP status code received */ private $_lastStatusCode; - public function __construct($apiKey = "", $siteID = "", $keepBelowRateLimit = true) + public function __construct($apiKey = "", $siteID = "", $keepBelowRateLimit = true, $debug = false) { $this->setCredentials($apiKey, $siteID); $this->_keepBelowRateLimit = $keepBelowRateLimit; + $this->_debug = $debug; } /** @@ -294,8 +305,38 @@ function($curl, $header) use (&$headers) ); curl_setopt($curlHandle, CURLOPT_TIMEOUT, 60); + /** ********** DEBUGGER ********** */ + if ($this->_debug) { + + // capture the PHP output + ob_start(); + $out = fopen('php://output', 'w'); + + curl_setopt($curlHandle, CURLOPT_VERBOSE, true); + curl_setopt($curlHandle, CURLOPT_STDERR, $out); + } + $result = curl_exec($curlHandle); + /** ********** DEBUGGER ********** */ + if ($this->_debug) { + + /** @noinspection PhpUndefinedVariableInspection */ + fclose($out); + + // get the curl output + $data = ob_get_clean(); + + // insert the request parameters sent + $data = preg_replace('/(\r?\n){2}/', "\n\n$requestParams\n\n", $data, 1); + + // append the response received + $data .= "\n\n" . $result . "\n\n"; + + // write to a log file + file_put_contents($this->_logFile, $data, FILE_APPEND); + } + $this->_responseHeaders = $headers; $this->_lastStatusCode = curl_getinfo($curlHandle, CURLINFO_HTTP_CODE); @@ -384,4 +425,9 @@ private function _checkRateLimit() usleep(250000); } } + + public function setLogFile($fileName) + { + $this->_logFile = $fileName; + } } From 147f5bc3e00bc2fe5457d45d8cd9941e3a903ba3 Mon Sep 17 00:00:00 2001 From: Phil Hopper Date: Mon, 20 Jan 2020 10:46:19 -0500 Subject: [PATCH 3/7] Allow using new client object for each call --- src/CurlClient.php | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/CurlClient.php b/src/CurlClient.php index 89d19a9..6c3ea87 100644 --- a/src/CurlClient.php +++ b/src/CurlClient.php @@ -32,7 +32,7 @@ class CurlClient /** * @var array of headers received with HTTP responses */ - private $_responseHeaders = array(); + private static $_responseHeaders = array(); /** * @var bool keep the request rate below the API rate limit @@ -337,7 +337,7 @@ function($curl, $header) use (&$headers) file_put_contents($this->_logFile, $data, FILE_APPEND); } - $this->_responseHeaders = $headers; + self::$_responseHeaders = $headers; $this->_lastStatusCode = curl_getinfo($curlHandle, CURLINFO_HTTP_CODE); if ($this->_lastStatusCode == self::HTTP_RATE_LIMIT) @@ -390,20 +390,20 @@ public function retry($wait, $curlHandle) private function _checkRateLimit() { // If no API calls have been made, no need to delay. - if (empty($this->_responseHeaders)) + if (empty(self::$_responseHeaders)) { return; } // Default the $limit and $remaining values if not set in the last response header. /** @var int $limit */ - $limit = isset($this->_responseHeaders[self::RATE_LIMIT]) - ? (int)$this->_responseHeaders[self::RATE_LIMIT] + $limit = isset(self::$_responseHeaders[self::RATE_LIMIT]) + ? (int)self::$_responseHeaders[self::RATE_LIMIT] : self::DEFAULT_RATE_LIMIT; /** @var int $remaining */ - $remaining = isset($this->_responseHeaders[self::RATE_LIMIT_REMAINING]) - ? (int)$this->_responseHeaders[self::RATE_LIMIT_REMAINING] + $remaining = isset(self::$_responseHeaders[self::RATE_LIMIT_REMAINING]) + ? (int)self::$_responseHeaders[self::RATE_LIMIT_REMAINING] : self::DEFAULT_RATE_LIMIT; // If no API calls have been made, no need to delay. From b5d093ffa23437e285254f431b7a1e8419763a8a Mon Sep 17 00:00:00 2001 From: Phil Hopper Date: Tue, 4 Feb 2020 14:59:24 -0500 Subject: [PATCH 4/7] Increase curl timeout to 2 minutes Having it set to 1 minute prevented it from receiving the 504 timeout from the server --- src/CurlClient.php | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/CurlClient.php b/src/CurlClient.php index 6c3ea87..aa5d0d7 100644 --- a/src/CurlClient.php +++ b/src/CurlClient.php @@ -38,7 +38,8 @@ class CurlClient * @var bool keep the request rate below the API rate limit */ private $_keepBelowRateLimit; - + + /** * @var array of extra CURL options */ private $_curlOptions = array(); @@ -303,7 +304,8 @@ function($curl, $header) use (&$headers) return $len; } ); - curl_setopt($curlHandle, CURLOPT_TIMEOUT, 60); + curl_setopt($curlHandle, CURLOPT_CONNECTTIMEOUT, 30); + curl_setopt($curlHandle, CURLOPT_TIMEOUT, 60 * 2); /** ********** DEBUGGER ********** */ if ($this->_debug) { From 6c342410b81d0ce087d30f1ed5c2caef282dc799 Mon Sep 17 00:00:00 2001 From: Phillip Hopper Date: Sat, 26 Apr 2025 12:33:43 -0400 Subject: [PATCH 5/7] Implement curl mocking --- src/CurlClient.php | 79 ++++++++++++++++++++++++++++++++++++++-- src/ICurlResponse.php | 13 +++++++ src/MockCurlResponse.php | 21 +++++++++++ 3 files changed, 110 insertions(+), 3 deletions(-) create mode 100644 src/ICurlResponse.php create mode 100644 src/MockCurlResponse.php diff --git a/src/CurlClient.php b/src/CurlClient.php index aa5d0d7..5dc3fc2 100644 --- a/src/CurlClient.php +++ b/src/CurlClient.php @@ -59,6 +59,16 @@ class CurlClient */ private $_lastStatusCode; + public $MockCurl = false; + + /** + * Array key should be part of the URL to which the response belongs + * @var MockCurlResponse[] + */ + public $UrlMockResponse = []; + + private $custom_request = ''; + public function __construct($apiKey = "", $siteID = "", $keepBelowRateLimit = true, $debug = false) { $this->setCredentials($apiKey, $siteID); @@ -268,6 +278,7 @@ public function httpRequest($requestParams, $url, $method, $requiredParams, $opt case "put": curl_setopt($curlHandle, CURLOPT_CUSTOMREQUEST, "PUT"); curl_setopt($curlHandle, CURLOPT_POSTFIELDS, $requestParams); + $this->custom_request = 'PUT'; break; case "delete": @@ -281,6 +292,7 @@ public function httpRequest($requestParams, $url, $method, $requiredParams, $opt { $url = $url."?".$requestParams; } + $this->custom_request = 'DELETE'; break; } @@ -307,6 +319,17 @@ function($curl, $header) use (&$headers) curl_setopt($curlHandle, CURLOPT_CONNECTTIMEOUT, 30); curl_setopt($curlHandle, CURLOPT_TIMEOUT, 60 * 2); + if ($this->MockCurl) + $result = $this->DoMockCurl($url, $headers); + else + $result = $this->DoCurl($curlHandle, $requestParams, $headers); + + unset($this->_requestHeaders["Content-Type"]); + return $result; + } + + private function DoCurl($curlHandle, $requestParams, $headers) + { /** ********** DEBUGGER ********** */ if ($this->_debug) { @@ -343,16 +366,66 @@ function($curl, $header) use (&$headers) $this->_lastStatusCode = curl_getinfo($curlHandle, CURLINFO_HTTP_CODE); if ($this->_lastStatusCode == self::HTTP_RATE_LIMIT) - { $result = $this->retry($headers[self::RATE_LIMIT_RESET], $curlHandle); - } curl_close($curlHandle); - unset($this->_requestHeaders["Content-Type"]); return $result; } + private function DoMockCurl($url, $headers) + { + $custom_request = $this->custom_request; + $has_post_fields = array_key_exists(CURLOPT_POSTFIELDS, $params ?? []); + + /** @var ICurlResponse $response */ + $response = null; + $return_key = ''; + + // look for a mock response that matches the URL + foreach ($this->UrlMockResponse as $key => $value) { + + // does this response match the requested URL? + if (!str_contains($url, $key)) + continue; + + if (!empty($custom_request) && $custom_request == $value->Method) { + + // this is a PUT or DELETE (or possibly POST) + $response = $value->Response; + $return_key = $key; + break; + } + elseif ($has_post_fields && empty($custom_request) && $value->Method == 'POST') { + + // this is a POST + $response = $value->Response; + $return_key = $key; + break; + } + elseif (!$has_post_fields && empty($custom_request) && $value->Method == 'GET') { + + // this is a GET + $response = $value->Response; + $return_key = $key; + break; + } + } + + self::$_responseHeaders = $headers; + + if (empty($return_key)) + return ''; + + $this->_lastStatusCode = $response->HttpCode; + + // remove this response from the stack + unset($this->UrlMockResponse[$return_key]); + + // return the value + return $response->Content; + } + /** * @brief get the last HTTP status code received * diff --git a/src/ICurlResponse.php b/src/ICurlResponse.php new file mode 100644 index 0000000..4987dc8 --- /dev/null +++ b/src/ICurlResponse.php @@ -0,0 +1,13 @@ +Method = $method; + $this->Response = $response; + } +} From 74bdc7c17a2247e0844ddc0f59aeba8052fd271d Mon Sep 17 00:00:00 2001 From: Phillip Hopper Date: Mon, 28 Apr 2025 09:48:57 -0400 Subject: [PATCH 6/7] Reset custom_request value for new request --- src/CurlClient.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/CurlClient.php b/src/CurlClient.php index 5dc3fc2..5edca0b 100644 --- a/src/CurlClient.php +++ b/src/CurlClient.php @@ -268,11 +268,13 @@ public function httpRequest($requestParams, $url, $method, $requiredParams, $opt case "post": curl_setopt($curlHandle, CURLOPT_POST, 1); curl_setopt($curlHandle, CURLOPT_POSTFIELDS, $requestParams); + $this->custom_request = ''; break; case "get": curl_setopt($curlHandle, CURLOPT_HTTPGET, 1); $url = $url."?".$requestParams; + $this->custom_request = ''; break; case "put": From 0f74e785404883695379341c06959d7858b31642 Mon Sep 17 00:00:00 2001 From: Phillip Hopper Date: Fri, 27 Jun 2025 11:42:58 -0400 Subject: [PATCH 7/7] Debug curl mocking --- src/CurlClient.php | 31 +++---------------------------- 1 file changed, 3 insertions(+), 28 deletions(-) diff --git a/src/CurlClient.php b/src/CurlClient.php index 5edca0b..cc6d5f4 100644 --- a/src/CurlClient.php +++ b/src/CurlClient.php @@ -67,8 +67,6 @@ class CurlClient */ public $UrlMockResponse = []; - private $custom_request = ''; - public function __construct($apiKey = "", $siteID = "", $keepBelowRateLimit = true, $debug = false) { $this->setCredentials($apiKey, $siteID); @@ -268,19 +266,16 @@ public function httpRequest($requestParams, $url, $method, $requiredParams, $opt case "post": curl_setopt($curlHandle, CURLOPT_POST, 1); curl_setopt($curlHandle, CURLOPT_POSTFIELDS, $requestParams); - $this->custom_request = ''; break; case "get": curl_setopt($curlHandle, CURLOPT_HTTPGET, 1); $url = $url."?".$requestParams; - $this->custom_request = ''; break; case "put": curl_setopt($curlHandle, CURLOPT_CUSTOMREQUEST, "PUT"); curl_setopt($curlHandle, CURLOPT_POSTFIELDS, $requestParams); - $this->custom_request = 'PUT'; break; case "delete": @@ -294,7 +289,6 @@ public function httpRequest($requestParams, $url, $method, $requiredParams, $opt { $url = $url."?".$requestParams; } - $this->custom_request = 'DELETE'; break; } @@ -322,7 +316,7 @@ function($curl, $header) use (&$headers) curl_setopt($curlHandle, CURLOPT_TIMEOUT, 60 * 2); if ($this->MockCurl) - $result = $this->DoMockCurl($url, $headers); + $result = $this->DoMockCurl($url, $method, $headers); else $result = $this->DoCurl($curlHandle, $requestParams, $headers); @@ -375,11 +369,8 @@ private function DoCurl($curlHandle, $requestParams, $headers) return $result; } - private function DoMockCurl($url, $headers) + private function DoMockCurl($url, $method, $headers) { - $custom_request = $this->custom_request; - $has_post_fields = array_key_exists(CURLOPT_POSTFIELDS, $params ?? []); - /** @var ICurlResponse $response */ $response = null; $return_key = ''; @@ -391,23 +382,7 @@ private function DoMockCurl($url, $headers) if (!str_contains($url, $key)) continue; - if (!empty($custom_request) && $custom_request == $value->Method) { - - // this is a PUT or DELETE (or possibly POST) - $response = $value->Response; - $return_key = $key; - break; - } - elseif ($has_post_fields && empty($custom_request) && $value->Method == 'POST') { - - // this is a POST - $response = $value->Response; - $return_key = $key; - break; - } - elseif (!$has_post_fields && empty($custom_request) && $value->Method == 'GET') { - - // this is a GET + if (strtoupper($value->Method) == strtoupper($method)) { $response = $value->Response; $return_key = $key; break;