diff --git a/BigQuery/composer.json b/BigQuery/composer.json index 71d5fe5d2a6c..478c231f6d52 100644 --- a/BigQuery/composer.json +++ b/BigQuery/composer.json @@ -5,7 +5,7 @@ "minimum-stability": "stable", "require": { "php": "^8.1", - "google/cloud-core": "^1.64", + "google/cloud-core": "^1.65", "ramsey/uuid": "^3.0|^4.0" }, "require-dev": { diff --git a/BigQuery/src/BigQueryClient.php b/BigQuery/src/BigQueryClient.php index e9df0095d7c8..64adada5c277 100644 --- a/BigQuery/src/BigQueryClient.php +++ b/BigQuery/src/BigQueryClient.php @@ -17,6 +17,8 @@ namespace Google\Cloud\BigQuery; +use Google\ApiCore\ValidationException; +use Google\Auth\ApplicationDefaultCredentials; use Google\Auth\FetchAuthTokenInterface; use Google\Cloud\BigQuery\Connection\ConnectionInterface; use Google\Cloud\BigQuery\Connection\Rest; @@ -29,6 +31,8 @@ use Google\Cloud\Core\RetryDeciderTrait; use Psr\Cache\CacheItemPoolInterface; use Psr\Http\Message\StreamInterface; +use Psr\Log\LoggerInterface; +use Psr\Log\LogLevel; /** * Google Cloud BigQuery allows you to create, manage, share and query data. @@ -51,9 +55,8 @@ class BigQueryClient } const VERSION = '1.34.6'; - const MAX_DELAY_MICROSECONDS = 32000000; - + const SERVICE_NAME = 'bigquery'; const SCOPE = 'https://www.googleapis.com/auth/bigquery'; const INSERT_SCOPE = 'https://www.googleapis.com/auth/bigquery.insertdata'; @@ -154,6 +157,9 @@ class BigQueryClient * fetched over the network it will take precedent over this * setting (by calling * {@see Table::reload()}, for example). + * @type false|LoggerInterface $logger + * A PSR-3 compliant logger. If set to false, logging is disabled, ignoring the + * 'GOOGLE_SDK_PHP_LOGGING' environment flag * } */ public function __construct(array $config = []) @@ -175,10 +181,14 @@ public function __construct(array $config = []) mt_rand(0, 1000000) + (pow(2, $attempt) * 1000000), self::MAX_DELAY_MICROSECONDS ); - } + }, //@codeCoverageIgnoreEnd + 'logger' => null ]; + $config['logger'] = $this->getLogger($config); + $this->logConfiguration($config['logger'], $config); + $this->connection = new Rest($this->configureAuthentication($config)); $this->mapper = new ValueMapper($config['returnInt64AsObject']); } @@ -1073,4 +1083,57 @@ public function load(array $options = []) $this->location ); } + + /** + * Gets the appropriate logger depending on the configuration passed by the user. + * + * @param array $config The client configuration + * @return LoggerInterface|false|null + * @throws ValidationException + */ + private function getLogger(array $config): LoggerInterface|false|null + { + $configuration = $config['logger']; + + if (is_null($configuration)) { + return ApplicationDefaultCredentials::getDefaultLogger(); + } + + if ($configuration !== null + && $configuration !== false + && !$configuration instanceof LoggerInterface + ) { + throw new ValidationException( + 'The "logger" option in the options array should be PSR-3 LoggerInterface compatible.' + ); + } + + return $configuration; + } + + /** + * Log the current configuration for the client + * + * @param LoggerInterface|false|null $logger The logger to be used. + * @param array $config The client configuration + * @return void + */ + private function logConfiguration(LoggerInterface|false|null $logger, array $config): void + { + if (!$logger) { + return; + } + + $configurationLog = [ + 'timestamp' => date(DATE_RFC3339), + 'severity' => strtoupper(LogLevel::DEBUG), + 'processId' => getmypid(), + 'jsonPayload' => [ + 'serviceName' => self::SERVICE_NAME, + 'clientConfiguration' => $config, + ] + ]; + + $logger->debug(json_encode($configurationLog)); + } } diff --git a/BigQuery/src/Connection/Rest.php b/BigQuery/src/Connection/Rest.php index 46a7854a68b6..c6c4b8890dd9 100644 --- a/BigQuery/src/Connection/Rest.php +++ b/BigQuery/src/Connection/Rest.php @@ -17,7 +17,9 @@ namespace Google\Cloud\BigQuery\Connection; +use Exception; use Google\Auth\GetUniverseDomainInterface; +use Google\Auth\HttpHandler\HttpHandlerFactory; use Google\Cloud\BigQuery\BigQueryClient; use Google\Cloud\Core\RequestBuilder; use Google\Cloud\Core\RequestWrapper; @@ -71,6 +73,7 @@ public function __construct(array $config = []) 'serviceDefinitionPath' => __DIR__ . '/ServiceDefinition/bigquery-v2.json', 'componentVersion' => BigQueryClient::VERSION, 'apiEndpoint' => null, + 'logger' => null, // If the user has not supplied a universe domain, use the environment variable if set. // Otherwise, use the default ("googleapis.com"). 'universeDomain' => getenv('GOOGLE_CLOUD_UNIVERSE_DOMAIN') @@ -79,7 +82,7 @@ public function __construct(array $config = []) $apiEndpoint = $this->getApiEndpoint(null, $config, self::DEFAULT_API_ENDPOINT_TEMPLATE); - $this->setRequestWrapper(new RequestWrapper($config)); + $this->setRequestWrapper($this->getRequestWrapper($config)); $this->setRequestBuilder(new RequestBuilder( $config['serviceDefinitionPath'], $apiEndpoint @@ -419,4 +422,26 @@ public function testTableIamPermissions(array $args = []) { return $this->send('tables', 'testIamPermissions', $args); } + + /** + * Creates a request wrapper and sets the HTTP Handler logger to the configured one. + * + * @param array $config + * @return RequestWrapper + */ + private function getRequestWrapper(array $config): RequestWrapper + { + // Because we are setting a logger, we build a handler here instead of using the default + $config['httpHandler'] = $config['httpHandler'] ?? HttpHandlerFactory::build(logger: $config['logger']); + $config['restRetryListener'] = $this->getRetryListener(); + return new RequestWrapper($config); + } + + private function getRetryListener(): callable + { + return function (Exception $ex, int $retryAttempt, array &$arguments) { + // The REST calls are [$request, $options]. We need to modify the options. + $arguments[1]['retryAttempt'] = $retryAttempt; + }; + } } diff --git a/BigQuery/tests/Unit/BigQueryClientTest.php b/BigQuery/tests/Unit/BigQueryClientTest.php index 5a7a9564d60c..d7b8cc7f2d52 100644 --- a/BigQuery/tests/Unit/BigQueryClientTest.php +++ b/BigQuery/tests/Unit/BigQueryClientTest.php @@ -17,6 +17,9 @@ namespace Google\Cloud\BigQuery\Tests\Unit; +use Google\ApiCore\ValidationException; +use Google\Auth\FetchAuthTokenInterface; +use Google\Auth\HttpHandler\Guzzle6HttpHandler; use Google\Cloud\BigQuery\BigNumeric; use Google\Cloud\BigQuery\BigQueryClient; use Google\Cloud\BigQuery\Bytes; @@ -36,11 +39,20 @@ use Google\Cloud\BigQuery\Time; use Google\Cloud\BigQuery\Timestamp; use Google\Cloud\Core\Int64; +use Google\Cloud\Core\Testing\Snippet\Fixtures; use Google\Cloud\Core\Testing\TestHelpers; use Google\Cloud\Core\Upload\AbstractUploader; +use GuzzleHttp\Client; +use GuzzleHttp\Handler\MockHandler; +use GuzzleHttp\HandlerStack; +use GuzzleHttp\Psr7\Request; +use GuzzleHttp\Psr7\Response; +use GuzzleHttp\Exception\RequestException; use PHPUnit\Framework\TestCase; use Prophecy\Argument; use Prophecy\PhpUnit\ProphecyTrait; +use Psr\Log\LoggerInterface; +use stdClass; /** * @group bigquery @@ -698,4 +710,105 @@ public function testGetsJson() $this->assertInstanceOf(Json::class, $json); } + + /** + * @runInSeparateProcess + */ + public function testEnablingLoggerWithFlagLogsToStdOut() + { + putenv('GOOGLE_SDK_PHP_LOGGING=true'); + $client = new BigQueryClient([ + 'projectId' => self::PROJECT_ID, + 'keyFilePath' => Fixtures::KEYFILE_STUB_FIXTURE() + ]); + + $output = $this->getActualOutput(); + $parsed = json_decode($output, true); + $this->assertNotFalse($parsed); + $this->assertArrayHasKey('severity', $parsed); + } + + /** + * @runInSeparateProcess + */ + public function testDisableLoggerWithOptionsDoesNotLogToStdOut() + { + putenv('GOOGLE_SDK_PHP_LOGGING=true'); + $client = new BigQueryClient([ + 'projectId' => self::PROJECT_ID, + 'keyFilePath' => Fixtures::KEYFILE_STUB_FIXTURE(), + 'logger' => false + ]); + + $output = $this->getActualOutput(); + $this->assertEmpty($output); + } + + public function testPassingTheWrongLoggerRaisesAnException() + { + $this->expectException(ValidationException::class); + + $client = new BigQueryClient([ + 'projectId' => self::PROJECT_ID, + 'keyFilePath' => Fixtures::KEYFILE_STUB_FIXTURE(), + 'logger' => new stdClass() + ]); + } + + public function testRetryLogging() + { + $doneJobResponse = $this->jobResponse; + $doneJobResponse['status']['state'] = 'DONE'; + + $apiMockHandler = new MockHandler([ + new RequestException( + 'Transient error', + new Request('POST', ''), + new Response(502) + ), + new Response(200, [], json_encode($this->jobResponse)), + new Response(200, [], json_encode($doneJobResponse)) + ]); + + $authMockHandler = new MockHandler([ + new Response(200, [], json_encode(['access_token' => 'token'])) + ]); + + $retryHeaderAppeared = false; + $logger = $this->prophesize(LoggerInterface::class); + $logger->debug( + Argument::that(function (string $jsonString) use (&$retryHeaderAppeared) { + $jsonParsed = json_decode($jsonString, true); + if (isset($jsonParsed['jsonPayload']['retryAttempt'])) { + $retryHeaderAppeared = true; + } + return true; + }) + )->shouldBeCalled(); + + $credentials = $this->prophesize(FetchAuthTokenInterface::class); + $credentials->fetchAuthToken(Argument::any()) + ->willReturn(['access_token' => 'foo']); + + $apiHandlerStack = HandlerStack::create($apiMockHandler); + $apiHttpClient = new Client(['handler' => $apiHandlerStack]); + + $authHandlerStack = HandlerStack::create($authMockHandler); + $authHttpClient = new Client(['handler' => $authHandlerStack]); + + $client = new BigQueryClient([ + 'credentials' => $credentials->reveal(), + 'projectId' => self::PROJECT_ID, + 'logger' => $logger->reveal(), + 'httpHandler' => new Guzzle6HttpHandler($apiHttpClient, $logger->reveal()), + 'authHttpHandler' => new Guzzle6HttpHandler($authHttpClient) + ]); + + $queryConfig = $client->query( + 'SELECT * FROM `random_project.random_dataset.random_table`' + ); + + $client->startQuery($queryConfig); + $this->assertTrue($retryHeaderAppeared); + } } diff --git a/Core/src/RequestWrapper.php b/Core/src/RequestWrapper.php index 849d15f2bec9..8170a282e824 100644 --- a/Core/src/RequestWrapper.php +++ b/Core/src/RequestWrapper.php @@ -83,6 +83,12 @@ class RequestWrapper */ private $retryFunction; + /** + * @var callable|null Lets the user listen for retries and + * modify the next retry arguments + */ + private $retryListener; + /** * @var callable Executes a delay. */ @@ -136,6 +142,8 @@ class RequestWrapper * determining how long to wait between attempts to retry. Function * signature should match: `function (int $attempt) : int`. * @type string $universeDomain The expected universe of the credentials. Defaults to "googleapis.com". + * @type callable $restRetryListener A function to run custom logic between retries. This function can modify + * the next server call arguments for the next retry. * } */ public function __construct(array $config = []) @@ -151,6 +159,7 @@ public function __construct(array $config = []) 'componentVersion' => null, 'restRetryFunction' => null, 'restDelayFunction' => null, + 'restRetryListener' => null, 'restCalcDelayFunction' => null, 'universeDomain' => GetUniverseDomainInterface::DEFAULT_UNIVERSE_DOMAIN, ]; @@ -160,6 +169,7 @@ public function __construct(array $config = []) $this->restOptions = $config['restOptions']; $this->shouldSignRequest = $config['shouldSignRequest']; $this->retryFunction = $config['restRetryFunction'] ?: $this->getRetryFunction(); + $this->retryListener = $config['restRetryListener']; $this->delayFunction = $config['restDelayFunction'] ?: function ($delay) { usleep($delay); }; @@ -362,7 +372,7 @@ private function applyHeaders(RequestInterface $request, array $options = []) */ private function addAuthHeaders(RequestInterface $request, FetchAuthTokenInterface $fetcher) { - $backoff = new ExponentialBackoff($this->retries, $this->getRetryFunction()); + $backoff = new ExponentialBackoff($this->retries, $this->getRetryFunction(), $this->retryListener); try { return $backoff->execute( @@ -485,7 +495,7 @@ private function getRetryOptions(array $options) : $this->retryFunction, 'retryListener' => isset($options['restRetryListener']) ? $options['restRetryListener'] - : null, + : $this->retryListener, 'delayFunction' => isset($options['restDelayFunction']) ? $options['restDelayFunction'] : $this->delayFunction, diff --git a/Core/tests/Unit/RequestWrapperTest.php b/Core/tests/Unit/RequestWrapperTest.php index 84c8aaac1bc6..83aab55d39d2 100644 --- a/Core/tests/Unit/RequestWrapperTest.php +++ b/Core/tests/Unit/RequestWrapperTest.php @@ -40,6 +40,7 @@ use PHPUnit\Framework\TestCase; use Prophecy\Argument; use Prophecy\PhpUnit\ProphecyTrait; +use ReflectionClass; /** * @group core @@ -859,6 +860,27 @@ public function testCheckUniverseDomainPasses(?string $universeDomain, ?string $ $this->assertTrue($called); } + public function testRetryListenerOnConstructor() + { + $listener = function ($_, int $__, array &$___) { + return; + }; + $wrapper = new RequestWrapper([ + 'restRetryListener' => $listener + ]); + + $reflectionClass = new ReflectionClass($wrapper); + $property = $reflectionClass->getProperty('retryListener'); + $property->setAccessible(true); + + $this->assertNotEmpty($property->getValue($wrapper), 'The retryListener property should be set.'); + $this->assertEquals( + $listener, + $property->getValue($wrapper), + 'The retryListener should be the same as the one passed via options.' + ); + } + public function provideCheckUniverseDomainPasses() { return [