Merge pull request 4x99#34 from 4x99/develop

v 1.1.0

Author: yoonper

Dockerfile

@@ -0,0 +1,38 @@
+FROM php:7.4-apache
+
+EXPOSE 80
+
+ENV MYSQL_HOST="172.17.0.1"
+ENV MYSQL_PORT="3306"
+ENV MYSQL_DATABASE="code6"
+ENV MYSQL_USERNAME=""
+ENV MYSQL_PASSWORD=""
+
+# 复制代码
+COPY . /var/www/html
+WORKDIR /var/www/html
+
+# 安装 PHP 扩展
+RUN docker-php-ext-install pdo_mysql
+
+# 配置 Web 路径
+ENV APACHE_DOCUMENT_ROOT=/var/www/html/public
+RUN sed -ri -e "s!/var/www/html!${APACHE_DOCUMENT_ROOT}!g" /etc/apache2/sites-available/*.conf
+RUN sed -ri -e "s!/var/www/!${APACHE_DOCUMENT_ROOT}!g" /etc/apache2/apache2.conf /etc/apache2/conf-available/*.conf
+
+# 使用阿里镜像并安装包
+RUN mv /etc/apt/sources.list /etc/apt/sources.list.bak
+RUN echo 'deb http://mirrors.aliyun.com/debian buster main' >> /etc/apt/sources.list
+RUN echo 'deb http://mirrors.aliyun.com/debian buster-updates main' >> /etc/apt/sources.list
+RUN apt-get update && apt-get install -y zip cron
+RUN rm -rf /var/lib/apt/lists/* && apt-get clean
+
+# 安装 Composer 及项目依赖包
+RUN curl -O https://mirrors.aliyun.com/composer/composer.phar
+RUN chmod +x composer.phar
+RUN mv composer.phar /usr/local/bin/composer
+RUN composer config repo.packagist composer https://mirrors.aliyun.com/composer/
+RUN composer install --no-dev --optimize-autoloader
+
+RUN chmod +x docker-entrypoint.sh
+ENTRYPOINT /bin/bash docker-entrypoint.sh

README.md

@@ -32,6 +32,11 @@
 
 ---
 
+## 安装部署
+码小六支持 [Docker 部署](doc/deploy-docker.md) 与 [源码部署](doc/deploy-source.md)，请根据情况选择！
+
+---
+
 ## 系统界面
 请点击浏览大图 :P
 
@@ -40,97 +45,3 @@
 ![应用概况](https://raw.githubusercontent.com/4x99/snapshot/master/code6/home.png)
 
 ![扫描结果](https://raw.githubusercontent.com/4x99/snapshot/master/code6/code-leak.png)
-
----
-
-## 安装指南
-
-### 克隆代码
-```
-git clone https://github.com/4x99/code6.git <项目目录>
-```
-
-### 配置 Apache
-请确保已开启 `mod_rewrite` 模块
-```
-<VirtualHost *:80>
-    ServerName <ServerName>
-    DocumentRoot "<项目目录>/public"
-</VirtualHost>
-```
-
-### 下载依赖包
-安装 Composer：
-```
-curl -O https://mirrors.aliyun.com/composer/composer.phar
-chmod +x composer.phar
-mv composer.phar /usr/local/bin/composer
-```
-
-配置阿里云镜像：
-```
-composer config repo.packagist composer https://mirrors.aliyun.com/composer/
-```
-
-下载项目依赖包：
-```
-cd <项目目录> && composer install --no-dev --prefer-dist --optimize-autoloader
-```
-
-### 项目配置
-设置目录权限：
-```
-chmod -R 755 storage
-chown -R <用户名>:<组名> storage
-```
-
-创建配置文件：
-```
-cp .env.example .env
-```
-
-生成应用密钥：
-```
-php artisan key:generate
-```
-
-创建并配置数据库：
-```
-vim .env
-```
-```
-DB_CONNECTION=mysql
-DB_HOST=127.0.0.1
-DB_PORT=3306
-DB_DATABASE=
-DB_USERNAME=
-DB_PASSWORD=
-```
-
-自动生成表结构：
-```
-php artisan migrate
-```
-
-### 配置任务调度
-```
-crontab -e -u <用户>
-```
-
-```
-* * * * * cd <项目目录> && php artisan schedule:run >> /dev/null 2>&1
-```
-
-### 创建用户
-```
-php artisan code6:user-add <邮箱> <密码>
-```
-
-如需查看用户列表或删除用户请执行：
-```
-php artisan code6:user-list
-php artisan code6:user-delete <邮箱>
-```
-
-### 配置令牌与任务
-进入系统后请前往 `[ 令牌配置 ]` 和 `[ 任务配置 ]` 模块进行配置，配置完毕即可使用！

app/Console/Commands/JobRunCommand.php

@@ -72,24 +72,26 @@ public function handle()
         $this->log = Log::channel($this->signature);
         $this->log->info('Start job');
 
-        while ($job = $this->takeJob()) {
-            if (!$this->service) {
-                $this->createGitHubService();
-                $this->whitelist = ConfigWhitelist::all()->keyBy('value');
-            }
+        if (!QueueJob::count()) {
+            $this->log->info('The queue is empty');
+            exit;
+        }
+
+        $this->createGitHubService();
+        $this->whitelist = ConfigWhitelist::all()->keyBy('value');
 
+        while ($job = $this->takeJob()) {
             $page = 1;
             $keyword = $job->keyword;
             $configJob = ConfigJob::where('keyword', $keyword)->first();
             $configJob->last_scan_at = date('Y-m-d H:i:s');
             do {
                 $client = $this->service->getClient();
-                $data = $this->searchCode($client, $keyword, $nextPage ?? null);
-                $count = $this->store($data, $keyword);
+                $data = $this->searchCode($client, $keyword, $page);
+                $count = $this->store($data, $configJob);
                 $this->log->info('Stored', ['keyword' => $keyword, 'page' => $page, 'count' => $count]);
                 $lastResponse = ResponseMediator::getPagination($client->getLastResponse());
-                $nextPage = $lastResponse['next'] ?? false;
-            } while ($nextPage && (++$page <= $configJob->scan_page));
+            } while ($lastResponse['next'] && (++$page <= $configJob->scan_page));
             $configJob->save();
         }
 
@@ -117,7 +119,6 @@ private function createGitHubService()
     private function takeJob()
     {
         if (!$job = QueueJob::orderBy('created_at')->first()) {
-            $this->log->info('The queue is empty');
             return false;
         }
         $job->delete();
@@ -129,17 +130,14 @@ private function takeJob()
      *
      * @param $client
      * @param $keyword
-     * @param  null  $url
+     * @param  int  $page
      * @return array|bool|string
      */
-    private function searchCode($client, $keyword, $url = null)
+    private function searchCode($client, $keyword, $page = 1)
     {
         try {
-            if ($url) { // 非首页
-                return ResponseMediator::getContent($client->getHttpClient()->get($url));
-            }
             $keyword = sprintf('"%s"', $keyword); // 精确匹配
-            return $client->api('search')->code($keyword, 'indexed'); // 首页
+            return $client->api('search')->setPage($page)->code($keyword, 'indexed');
         } catch (Exception $e) {
             $this->log->warning($e->getMessage());
             return false;
@@ -150,10 +148,10 @@ private function searchCode($client, $keyword, $url = null)
      * 保存数据
      *
      * @param $data
-     * @param $keyword
-     * @return array
+     * @param $configJob
+     * @return int[]
      */
-    private function store($data, $keyword)
+    private function store($data, $configJob)
     {
         $count = ['leak' => 0, 'fragment' => 0];
 
@@ -162,8 +160,8 @@ private function store($data, $keyword)
         }
 
         foreach ($data['items'] as $item) {
-            $item['keyword'] = $keyword;
-            if (!$uuid = $this->storeLeak($item)) {
+            $item['keyword'] = $configJob->keyword;
+            if (!$uuid = $this->storeLeak($item, $configJob->store_type)) {
                 continue;
             }
             $count['leak']++;
@@ -181,9 +179,10 @@ private function store($data, $keyword)
      * 保存代码泄露数据
      *
      * @param $item
+     * @param $storeType
      * @return bool|string
      */
-    private function storeLeak($item)
+    private function storeLeak($item, $storeType)
     {
         $repoOwner = $item['repository']['owner']['login'];
         $repoName = $item['repository']['name'];
@@ -200,10 +199,22 @@ private function storeLeak($item)
         }
 
         // 数据入库
+        $where = [];
         $uuid = md5("$repoOwner/$repoName/$blob/{$item['path']}");
-        $leak = CodeLeak::firstOrCreate(
-            ['uuid' => $uuid],
-            [
+        switch ($storeType) {
+            case configJob::STORE_TYPE_ALL:
+                $where = ['uuid' => $uuid];
+                break;
+            case configJob::STORE_TYPE_FILE_STORE_ONCE:
+                $where = ['repo_owner' => $repoOwner, 'repo_name' => $repoName, 'path' => $item['path']];
+                break;
+            case configJob::STORE_TYPE_REPO_STORE_ONCE:
+                $where = ['repo_owner' => $repoOwner, 'repo_name' => $repoName];
+                break;
+        }
+
+        $leak = CodeLeak::firstOrCreate($where, [
+                'uuid' => $uuid,
                 'keyword' => $item['keyword'],
                 'repo_owner' => $repoOwner,
                 'repo_name' => $repoName,

app/Http/Controllers/ConfigJobController.php

@@ -26,7 +26,7 @@ public function index(Request $request)
     }
 
     /**
-     * 保存令牌
+     * 保存任务
      *
      * @param  Request  $request
      * @return array
@@ -40,6 +40,7 @@ public function store(Request $request)
                 [
                     'scan_page' => $request->input('scan_page', 100),
                     'scan_interval_min' => $request->input('scan_interval_min', 60),
+                    'store_type' => $request->input('store_type', 0),
                     'description' => $request->input('description') ?? ''
                 ]
             );
@@ -63,7 +64,7 @@ public function update(Request $request, $id)
     {
         try {
             $request->validate(['keyword' => ['required', 'string', 'max:255']]);
-            $fields = ['keyword', 'scan_page', 'scan_interval_min', 'description'];
+            $fields = ['keyword', 'scan_page', 'scan_interval_min', 'store_type', 'description'];
             $configJob = ConfigJob::find($id);
             $success = $configJob->update($request->all($fields));
             return ['success' => $success, 'data' => $configJob];

app/Models/ConfigJob.php

@@ -4,6 +4,9 @@
 
 class ConfigJob extends ModelBase
 {
+    const STORE_TYPE_ALL = 0;
+    const STORE_TYPE_FILE_STORE_ONCE = 1;
+    const STORE_TYPE_REPO_STORE_ONCE = 2;
     protected $table = 'config_job';
-    protected $fillable = ['keyword', 'scan_page', 'scan_interval_min', 'description'];
+    protected $fillable = ['keyword', 'scan_page', 'scan_interval_min', 'store_type', 'description'];
 }

config/logging.php

@@ -39,7 +39,7 @@
 
         'daily' => [
             'driver' => 'daily',
-            'path' => storage_path(sprintf('logs/%s.log', config('app.name'))),
+            'path' => storage_path('logs/code6.log'),
             'level' => 'info',
             'days' => 7,
         ],

database/migrations/2020_06_13_143710_add_store_type_to_config_job_table.php

@@ -0,0 +1,32 @@
+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\Schema;
+
+class AddStoreTypeToConfigJobTable extends Migration
+{
+    /**
+     * Run the migrations.
+     *
+     * @return void
+     */
+    public function up()
+    {
+        Schema::table('config_job', function (Blueprint $table) {
+            $table->tinyInteger('store_type')->unsigned()->after('scan_interval_min')->comment('0:All 1:File Store Once 2:Repo Store Once');
+        });
+    }
+
+    /**
+     * Reverse the migrations.
+     *
+     * @return void
+     */
+    public function down()
+    {
+        Schema::table('config_job', function (Blueprint $table) {
+            $table->dropColumn('storage_type');
+        });
+    }
+}

database/migrations/2020_06_13_150004_add_index_repo_owner_and_repo_name_to_code_leak_table.php

@@ -0,0 +1,34 @@
+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\Schema;
+
+class AddIndexRepoOwnerAndRepoNameToCodeLeakTable extends Migration
+{
+    /**
+     * Run the migrations.
+     *
+     * @return void
+     */
+    public function up()
+    {
+        Schema::table('code_leak', function (Blueprint $table) {
+            $table->index('repo_owner');
+            $table->index('repo_name');
+        });
+    }
+
+    /**
+     * Reverse the migrations.
+     *
+     * @return void
+     */
+    public function down()
+    {
+        Schema::table('code_leak', function (Blueprint $table) {
+            $table->dropIndex('code_leak_repo_owner_index');
+            $table->dropIndex('code_leak_repo_name_index');
+        });
+    }
+}