Merge pull request 4x99#40 from 4x99/develop

1.2.0

Author: yoonper

app/Console/Commands/JobRunCommand.php

@@ -3,7 +3,6 @@
 namespace App\Console\Commands;
 
 use Exception;
-use Github\Client;
 use App\Models\CodeFragment;
 use App\Models\CodeLeak;
 use App\Models\ConfigJob;
@@ -48,7 +47,7 @@ class JobRunCommand extends Command
     /**
      * GitHub Service
      *
-     * @var Client
+     * @var GitHubService
      */
     protected $service;
 
@@ -72,11 +71,6 @@ public function handle()
         $this->log = Log::channel($this->signature);
         $this->log->info('Start job');
 
-        if (!QueueJob::count()) {
-            $this->log->info('The queue is empty');
-            exit;
-        }
-
         $this->createGitHubService();
         $this->whitelist = ConfigWhitelist::all()->keyBy('value');
 

app/Console/Commands/TokenCheckCommand.php

@@ -24,7 +24,6 @@ class Kernel extends ConsoleKernel
      */
     protected function schedule(Schedule $schedule)
     {
-        $schedule->command('code6:token-check')->everyMinute()->withoutOverlapping()->runInBackground();
         $schedule->command('code6:job-add')->everyMinute();
         $schedule->command('code6:job-run')->everyMinute()->withoutOverlapping();
     }

app/Console/Kernel.php

@@ -30,8 +30,8 @@ public function metric()
     {
         $data = [
             'codeLeakCount' => CodeLeak::count(),
-            'codeLeakPending' => CodeLeak::where('status', 0)->count(),
-            'codeLeakSolved' => CodeLeak::where('status', 3)->count(),
+            'codeLeakPending' => CodeLeak::where('status', CodeLeak::STATUS_PENDING)->count(),
+            'codeLeakSolved' => CodeLeak::where('status', CodeLeak::STATUS_SOLVED)->count(),
             'queueJobCount' => QueueJob::count(),
         ];
         return ['success' => true, 'data' => $data];
@@ -88,8 +88,8 @@ public function memory()
     public function tokenQuota()
     {
         try {
-            $total = ConfigToken::where('status', 1)->sum('api_limit');
-            $remaining = ConfigToken::where('status', 1)->sum('api_remaining');
+            $total = ConfigToken::where('status', ConfigToken::STATUS_NORMAL)->sum('api_limit');
+            $remaining = ConfigToken::where('status', ConfigToken::STATUS_NORMAL)->sum('api_remaining');
             $used = $total - $remaining;
             $data = [
                 [

app/Http/Controllers/HomeController.php

@@ -4,6 +4,10 @@
 
 class CodeLeak extends ModelBase
 {
+    const STATUS_PENDING = 0;
+    const STATUS_FALSE = 1;
+    const STATUS_ABNORMAL = 2;
+    const STATUS_SOLVED = 3;
     protected $table = 'code_leak';
     protected $fillable = [
         'uuid',

app/Models/CodeLeak.php

@@ -4,6 +4,9 @@
 
 class ConfigToken extends ModelBase
 {
+    const STATUS_UNKNOWN = 0;
+    const STATUS_NORMAL = 1;
+    const STATUS_ABNORMAL = 2;
     protected $table = 'config_token';
     protected $fillable = ['token', 'description'];
 }

app/Models/ConfigToken.php

@@ -2,21 +2,37 @@
 
 namespace App\Services;
 
+use Closure;
 use Exception;
-use App\Models\ConfigToken;
 use Github\Client;
+use App\Models\ConfigToken;
+use Illuminate\Support\Arr;
 use Github\HttpClient\Builder;
 use Illuminate\Support\Facades\Log;
-use Http\Adapter\Guzzle6\Client AS GuzzleClient;
+use Http\Adapter\Guzzle6\Client as GuzzleClient;
+use GuzzleHttp\Handler\CurlHandler;
+use GuzzleHttp\HandlerStack;
+use GuzzleHttp\Middleware;
+use GuzzleHttp\Psr7\Request;
+use GuzzleHttp\Psr7\Response;
+use GuzzleHttp\Exception\RequestException;
 
 class GitHubService
 {
+    const HTTP_TIMEOUT = 30;
+    const HTTP_DELAY = 2000;
+    const HTTP_MAX_RETRIES = 5;
     const GET_CLIENT_TIMEOUT = 1800;
+    const HTTP_CODE_UNAUTHORIZED = 401;
+    const RATE_LIMIT_UNAUTHENTICATED = 10; // 未授权限制请求频率：10 次 / 分钟
+
     public $clients = [];
+    private $userAgent = 'Code6';
     private $currentKey = -1;
 
     public function __construct()
     {
+        $this->userAgent = config('app.name');
         $this->createClients();
     }
 
@@ -40,18 +56,19 @@ public function getClient()
             $client = &$this->clients[$this->currentKey];
 
             // 更新接口配额
-            if (time() >= $client['reset']) {
+            if (time() >= $client['api_reset_at']) {
                 $this->updateClient($client);
             }
 
             // 当前客户端配额是否耗尽
-            if ($client['remaining'] <= 0) {
+            if ($client['api_remaining'] <= 0) {
                 sleep(1);
                 continue;
             }
 
             // 返回客户端
-            $client['remaining']--;
+            $client['api_remaining']--;
+            $this->updateConfigToken($client);
             return $client['client'];
         }
     }
@@ -63,8 +80,15 @@ private function createClients()
     {
         $tokens = ConfigToken::inRandomOrder()->get()->pluck('token');
         foreach ($tokens as $token) {
+            $handlerStack = HandlerStack::create(new CurlHandler());
+            $handlerStack->push(Middleware::retry($this->retryDecider()));
+            $builder = new Builder(GuzzleClient::createWithConfig([
+                'timeout' => self::HTTP_TIMEOUT,
+                'delay' => self::HTTP_DELAY,
+                'headers' => ['User-Agent' => $this->userAgent],
+                'handler' => $handlerStack,
+            ]));
             $client = ['token' => $token];
-            $builder = new Builder(GuzzleClient::createWithConfig(['timeout' => 30]));
             $client['client'] = new Client($builder, 'v3.text-match');
             $client['client']->authenticate($token, null, Client::AUTH_HTTP_TOKEN);
             if ($this->updateClient($client)) {
@@ -81,17 +105,60 @@ private function createClients()
      */
     private function updateClient(&$client)
     {
+        $code = $resource = null;
         try {
-            if (!$resource = $client['client']->api('rate_limit')->getResource('search')) {
+            $resource = $client['client']->api('rate_limit')->getResource('search');
+        } catch (Exception $e) {
+            $code = $e->getCode();
+            Log::debug($e->getMessage(), ['token' => $client['token']]);
+        }
+
+        $client['api_limit'] = $resource ? $resource->getLimit() : 0;
+        $client['api_reset_at'] = $resource ? $resource->getReset() : null;
+        $client['api_remaining'] = $resource ? $resource->getRemaining() : 0;
+        if ($code == self::HTTP_CODE_UNAUTHORIZED || $client['api_limit'] == self::RATE_LIMIT_UNAUTHENTICATED) {
+            $client['status'] = ConfigToken::STATUS_ABNORMAL;
+        } else {
+            $client['status'] = $resource ? ConfigToken::STATUS_NORMAL : ConfigToken::STATUS_UNKNOWN;
+        }
+
+        $this->updateConfigToken($client);
+        return $client['status'] == ConfigToken::STATUS_NORMAL;
+    }
+
+    /**
+     * 更新数据库
+     *
+     * @param $client
+     * @return mixed
+     */
+    private function updateConfigToken($client)
+    {
+        $data = Arr::only($client, ['status', 'api_limit', 'api_remaining']);
+        $data['api_reset_at'] = $client['api_reset_at'] ? date('Y-m-d H:i:s', $client['api_reset_at']) : null;
+        return ConfigToken::where('token', $client['token'])->update($data);
+    }
+
+    /**
+     * 决定是否重试请求
+     *
+     * @return Closure （true：重试 false：不重试）
+     */
+    protected function retryDecider()
+    {
+        return function ($retries, Request $request, Response $response = null, RequestException $exception = null) {
+            // 最大次数
+            if ($retries >= self::HTTP_MAX_RETRIES) {
                 return false;
             }
-            $client['limit'] = $resource->getLimit();
-            $client['reset'] = $resource->getReset();
-            $client['remaining'] = $resource->getRemaining();
-        } catch (Exception $e) {
-            Log::warning($e->getMessage(), ['token' => $client['token']]);
+
+            // 请求失败
+            if (!is_null($exception)) {
+                Log::debug('Retry request', ['exception' => $exception->getMessage()]);
+                return true;
+            }
+
             return false;
-        }
-        return true;
+        };
     }
 }

app/Services/GitHubService.php

@@ -51,13 +51,6 @@
             'days' => 7,
         ],
 
-        'code6:token-check' => [
-            'driver' => 'daily',
-            'path' => storage_path('logs/token-check.log'),
-            'level' => 'info',
-            'days' => 7,
-        ],
-
         'code6:job-run' => [
             'driver' => 'daily',
             'path' => storage_path('logs/job-run.log'),

config/logging.php

@@ -0,0 +1,33 @@
+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\Schema;
+use Illuminate\Support\Facades\DB;
+
+class ChangeStatusToConfigTokenTable extends Migration
+{
+    /**
+     * Run the migrations.
+     *
+     * @return void
+     */
+    public function up()
+    {
+        Schema::table('config_token', function (Blueprint $table) {
+            DB::statement("ALTER TABLE `config_token` CHANGE `status` `status` tinyint(3) unsigned NOT NULL DEFAULT '0' COMMENT '0:Unknown 1:Normal 2:Abnormal'");
+        });
+    }
+
+    /**
+     * Reverse the migrations.
+     *
+     * @return void
+     */
+    public function down()
+    {
+        Schema::table('config_token', function (Blueprint $table) {
+            DB::statement("ALTER TABLE `config_token` CHANGE `status` `status` tinyint(3) unsigned NOT NULL DEFAULT '1' COMMENT '0:Abnormal 1:Normal'");
+        });
+    }
+}

database/migrations/2020_07_03_091934_change_status_to_config_token_table.php

@@ -351,7 +351,7 @@
                                 },
                                 {
                                     text: '更多',
-                                    iconCls: 'icon-bullet-orange',
+                                    iconCls: 'icon-bullet-blue',
                                     margin: '0 0 0 10',
                                     menu: {
                                         items: [