fofa/tools/dump_db_http_header.rb at master · BugLike/fofa · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
#!/usr/bin/env ruby
require 'yaml'
require 'json'
require 'erb'
require 'active_record'
require 'thinking_sphinx'

@root_path = File.expand_path(File.dirname(__FILE__))
require @root_path+"/../app/workers/module/httpmodule.rb"
require @root_path+"/../app/workers/module/webdb2_class.rb"

require @root_path+"/../app/helpers/search_helper.rb"
include SearchHelper
require @root_path+"/../app/models/subdomain.rb"
require @root_path+"/../app/models/rule.rb"
require @root_path+"/../app/models/charts.rb"

Dir.chdir @root_path+"/../"
puts "working dir: #{Dir.pwd}"

rails_env = ENV['RAILS_ENV'] || 'development'
thinking_config = YAML::load(File.open(@root_path+"/../config/thinking_sphinx.yml"))[rails_env]

config = YAML::load(File.open(@root_path+"/../config/database.yml"))[rails_env]
ActiveRecord::Base.establish_connection (config)

USE_THINKING_SPHINX=true
if USE_THINKING_SPHINX
  ThinkingSphinx::SphinxQL.functions!
  #ThinkingSphinx::Middlewares::DEFAULT.delete ThinkingSphinx::Middlewares::UTF8
  ThinkingSphinx::Configuration.instance.searchd.address = thinking_config['address']
  ThinkingSphinx::Configuration.instance.searchd.port = thinking_config['port']
else
  @mysql ||= Mysql2::Client.new(:host => thinking_config['address'],
                                :username => thinking_config['connection_options']['username'],
                                :password => thinking_config['connection_options']['password'],
                                :database => thinking_config['connection_options']['database'],
                                :port => thinking_config['mysql41'],
                                :encoding => 'utf8', :reconnect => true)
end

def query(query_info, max_id)
  headers = []
  match_query =  SphinxProcessor.parse(query_info)

  options = {:match_mode => :extended, :index => 'subdomain_core',
             :with => {:id => max_id..9999999999},
             :sql => { :select => 'id,header,host'}, :per_page => 1000,
             :page => 1, :order => "id asc"}
  ThinkingSphinx.search(match_query, options).each{|r|
    headers << r.header
    max_id = r.id
  }
  [headers, max_id.to_i]
end

$maxid = 0
$keys = []
$known_keys = %w|Last-Modified Date Content-Length Connection Cache-Control Content-Location Last-Modified Etag Transfer-Encoding Content-Encoding Vary Expires Pragma Content-Language Location|
while 1
  headers,$maxid = query('host=".gov.cn"', $maxid)
  unless headers.size>0
    break
  end

  headers.each{|header|
    header.split(/[\r\n]/).each{|l|
      if l.include? ':'
        k,v = l.split(':')
        unless $keys.include?(k) || $known_keys.include?(k)
          $keys << k
          puts l
        end
      end
    }
  }

  puts "-----", $maxid
  $maxid += 1
end