init

Author: yufeizhao

.gitignore

@@ -5,3 +5,4 @@ __pycache__
 node_modules
 dist
 app/config/config.local.yaml
+*.sqlite

Pipfile

@@ -24,6 +24,16 @@ orjson = "*"
 aiohttp = "*"
 # aioredis = "*"
 # asyncpg = "*"
+sslyze = "*"
+dnspython = "*"
+tldextract = "*"
+wafw00f = "*"
+requests-cache = "*"
+requests = "*"
+lxml = "*"
+beautifulsoup4 = "*"
+ipwhois = "*"
+python-whois = "*"
 
 [requires]
 python_version = "3.7"

Pipfile.lock

@@ -1,5 +1,5 @@
 #!/bin/bash
-PROJECT_NAME="template-tornado"
+PROJECT_NAME="website-scanner"
 LOG_DIR=/app/log/${PROJECT_NAME}
 
 NOTICE_ENV="Can not load ${ENV} from system, only (dev, test, pre, prod) is available"

app.sh

@@ -5,6 +5,7 @@ server:
   port: 8000
   allow_origin: "*"
   thread_worker: 60
+  requests_cache: 60
 
 http:
   timeout: 3

app/config/config.dev.yaml

@@ -5,6 +5,7 @@ server:
   port: 8000
   allow_origin: "*"
   thread_worker: 60
+  requests_cache: 60
 
 http:
   timeout: 3

app/config/config.pre.yaml

@@ -5,6 +5,7 @@ server:
   port: 8000
   allow_origin: "*"
   thread_worker: 60
+  requests_cache: 60
 
 http:
   timeout: 3

app/config/config.prod.yaml

@@ -5,6 +5,7 @@ server:
   port: 8000
   allow_origin: "*"
   thread_worker: 60
+  requests_cache: 60
 
 http:
   timeout: 3

app/config/config.test.yaml

@@ -18,5 +18,5 @@ class HTTPCode(IntEnum):
 
 
 class PluginType(StrEnum):
-    SCANNER = "scanner_plugin"
-    WAF = "waf_plugin"
+    ASYNC_SCANNER = "async_scanner"
+    SYNC_SCANNER = "sync_scanner"

app/config/const_config.py

@@ -7,6 +7,7 @@
 from tornado.platform.asyncio import AsyncIOMainLoop
 from tornado.process import fork_processes
 from tornado.netutil import bind_sockets
+import requests_cache
 
 from app import application, router
 from app.hook import before_hook, after_hook
@@ -41,6 +42,7 @@
 
 
 def main():
+    requests_cache.install_cache("requests_cache", expire_after=config.server["requests_cache"])
     settings = config.server
     settings["static_path"] = config.static_path
     settings["template_path"] = config.template_path