#!/usr/bin/ruby

## httpdumper v0.1 : A tool to display and dump HTTP conversations
## Copyright (C) 2010  Franck GUENICHOT
## franck {dot} guenichot {at} orange {dot} fr
##
## This program is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License
## as published by the Free Software Foundation; either version 3
## of the License, or any later version.
##
## This program is distributed in the hope that it will be useful,
## but WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
## GNU General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with this program.  If not, see <http://www.gnu.org/licenses/>.


## Written for the Network Forensics Puzzle #3
## http://forensicscontest.com

require 'rubygems'
require 'packetfu'
require 'zlib'
require 'terminal-table/import'
require 'optparse'

###########################################################
#
# Class def.
#
###########################################################


class HttpConversationArray < Array
		
	def find(host1,port1,host2,port2)
		index = nil		
		self.each { |conv|
				index = self.index(conv) if conv.is_member?(host1,port1) && conv.is_member?(host2,port2)
		}
		index
	end

end #class


class HttpConversation
	attr_reader :host_1,:host_2,:cumulative_len
	attr_accessor :flows

	def initialize(flow)
		@host_1 = {'ip' => flow.ipsrc, 'port' => flow.srcport	}
		@host_2 = {'ip' => flow.ipdst, 'port' => flow.dstport	}
		@flows = []
		@cumulative_len = 0
		add_flow(flow)
	end

	def add_flow(httpflow)
		@flows << httpflow
		@cumulative_len += httpflow.flow_data.length
	end

	def is_member?(host,port)
		found = false
		[@host_1,@host_2].each { |h|
			if h['ip'] == host && h['port'] == port
				found = true
			end
		}
		found
	end

	def flow_count(type=nil)
		res = 0
		if type
			res = count(type)  	
		else 
			res = flows.length
		end
		res
	end

	#

	private

	def count(type)
		n = 0
		flows.each { |flow|
			n += 1 if flow.flow_data.message_type == type
		}
		n
	end

end #class


class HttpFlow
	attr_reader :ipsrc,:ipdst,:srcport,:dstport
	attr_accessor :flow_data
	
	def initialize(ipsrc,srcport,ipdst,dstport)
		@ipsrc = ipsrc
		@ipdst = ipdst
		@srcport = srcport
		@dstport = dstport
		@flow_data = nil
	end

end #class

class HttpMessage
	attr_reader :start_line,:message_type,:message_headers
	attr_accessor :message_body 
	
	alias :headers :message_headers
		
	def initialize()
		@message_type = "UNKNOW"
		@start_line = ""
		@message_headers = ""
		@message_body = ""
	end

	def parse(message)
		
		end_of_headers = false

		message.each { |line|
			
			if @start_line == ""
				@start_line << line				    # first line is HTTP message start_line
			elsif !end_of_headers				
				@message_headers << line			# Headers are after the start_line till end_of_headers
			  if line.match(/^\r\n$/)
					end_of_headers = true
				end
			else
				@message_body << line				  # rest is message body
			end
		}
		
	end
	
		
	def header(hd)
		return	@message_headers.match(/#{hd}:\s+(\S+)/).to_s
	end

	

	def header_length()
		return @message_headers.length
	end

	def length()
		return @message_body.length
	end

	def content_length()
		return self.header('Content-Length').split[1].to_i
	end

	def content_encoding()
		return self.header('Content-Encoding').split[1]
	end

	def content_type()
		return self.header('Content-Type').split[1]
	end

	
end #class


class HttpRequest < HttpMessage
	
	def initialize()
		super		
		@message_type = "REQUEST"
	end

	def requested_uri()
		return @start_line.split[1]
	end

end #class


class HttpResponse < HttpMessage
		
	def initialize()
		super
		@message_type = "RESPONSE"
	end
end #class


class HttpHost

	attr_reader :hostname
	attr_accessor :ip,:uri_list

	def initialize(host,ip)
		@hostname = host
		@ip = ip
		@uri_list = []
	end

	def request_count()
		return @uri_list.length
	end

end #class

class HttpStats

	def initialize(httpconv_array)
		@httpconv_array = httpconv_array
	end
	

	def uri_list(requester_ip,host)
		result = []
		st = generate_stats("REQUEST",requester_ip)
		st.each { |hostname,httphost|
			filter = host.nil? ? true : hostname == host			
			if filter 	
				result << httphost
			end
		}
		result
	end

	def request_stats(requester_ip)
		result = []
		st = generate_stats("REQUEST",requester_ip)
		st.each { |hostname,httphost|
			result << [hostname,httphost.ip,httphost.uri_list.length]
		}
		result
	end
	
	def generate_stats(type,requester_ip)
		result = {}		
		

		if type == "REQUEST"
			
			@httpconv_array.each { |conv|
				conv.flows.each { |flow|
					filter = (requester_ip.nil? && flow.flow_data.message_type == "REQUEST") || (flow.flow_data.message_type == "REQUEST" && flow.ipsrc == requester_ip)
					
					if filter
						
						host = flow.flow_data.header('Host').split[1]
						uri = flow.flow_data.requested_uri
						host_ip = flow.ipdst   # Will be a false value if using a proxy, will store only one IP for the same hostname

						if result.has_key?(host)
							result[host].uri_list << [@httpconv_array.index(conv),conv.flows.index(flow),uri]
						else
							new_host = HttpHost.new(host,host_ip)
							new_host.uri_list << [@httpconv_array.index(conv),conv.flows.index(flow),uri]
							result[host] = new_host	
						end
					end
				}
			}
		end
		result
	end

end #class
				


$VERSION = "0.1"
$PROGNAME = "httpdumper"

#################################################################################################

# handle command line args
$options = {}
opts = OptionParser.new do|opts|
  opts.banner = "
    #{$PROGNAME} version #{$VERSION} 
    Copyright (C) 2010 Franck GUENICHOT
    #{$PROGNAME} comes with ABSOLUTELY NO WARRANTY;
    This is free software, and you are welcome
    to redistribute it under certain conditions.
    (GPL v3)	
    
    Usage: #{$PROGNAME} [options] -r <pcap_file>"
  

  $options[:infile] = nil
  opts.on( '-r', '--read <pcap_file>', 'Read the given pcap file [REQUIRED]' ) do|infile|
    $options[:infile] = infile
  end

	$options[:conv_filter] = nil
  opts.on( '-c', '--conversation #', 'List only flows for conversation #' ) do|conv_num|
		if conv_num =~ /\b\d+\b/
    	$options[:conv_filter] = conv_num
		else
			puts "Error: -c needs a number !"
			exit(0)
		end
  end

	$options[:flow_filter] = nil
  opts.on( '-f', '--flow #', 'List only flow #' ) do|flow_num|
		if flow_num =~ /\b\d+\b/
    	$options[:flow_filter] = flow_num
		else
			puts "Error: -c needs a number !"
			exit(0)
		end
  end

	$options[:headers] = false
  opts.on( '--with-headers', 'For Display ONLY' ) do 
		$options[:headers] = true
	end

	$options[:dump2disk] = nil
  opts.on( '-d', '--dump', 'Dump the selected conversation or flow' ) do 
		$options[:dump2disk] = true
	end
  
	$options[:port] = nil
  opts.on( '-p', '--port <number>', 'Define custom HTTP port' ) do|port|
    $options[:port] = port
  end

	$options[:stats] = nil
	opts.on( '-s', '--stats type,[val1],[val2]', Array, 'Displays statistics',
		"	Valid options:",	
	 "	Request stats: request,[requester_ip],[requested_host]",
		"	URI list: uri,[requester_ip],[target_hostname]") do|stats|
		$options[:stats] = stats
	end

	opts.on( '-v', '--version', 'Display version information' ) do
    puts $VERSION
    exit
  end

  opts.on( '-h', '--help', 'Display this screen' ) do
    puts opts
    exit
  end
end

#parse command line args
opts.parse!


# if no pcap file in input => display help and exit
if $options[:infile] == nil
  puts opts
  exit(0)
end

#File not found error.
unless File.exist?($options[:infile])
  puts "File: #{$options[:infile]} does not exist."
  exit(0)
end





def dump_conversation_flows(httpconv_array,conv_index=nil,flow_index=nil,filename=nil)
	if conv_index
		if flow_index
			flow = httpconv_array[conv_index].flows[flow_index]
			dump_content_to_file(flow,filename)
		else
			httpconv_array[conv_index].flows.each { |flow|
				dump_content_to_file(flow,filename)
			}
		end
	else
		httpconv_array.each { |conv|
			conv.flows.each {	|flow|
				dump_content_to_file(flow)
			}
		}
	end
end
	



def dump_content_to_file(httpflow_object,filename=nil)
	
	if httpflow_object.flow_data.message_type == "RESPONSE"		# By now, dump only http responses to disk
		c_type = httpflow_object.flow_data.content_type
		c_length = httpflow_object.flow_data.content_length
		c_encoding = httpflow_object.flow_data.content_encoding
		c_data = httpflow_object.flow_data.message_body

		if !filename
			t = Time.now.usec  	#dummy value to avoid same filenames
			filename = "#{httpflow_object.ipsrc}_#{httpflow_object.srcport}-#{httpflow_object.ipdst}_#{httpflow_object.dstport}-#{t}.#{c_type.match(/\/\w+/).to_s.gsub(/\//,"")}"
		end
		puts " "		
		puts "Dumping data to disk: #{filename}"
		#puts "Content type: #{c_type}"
		#puts "Length: #{c_length}"
		#puts "Encoding: #{c_encoding ? c_encoding : "none"}"
		

		#If content is compressed: decompress it
		if c_encoding == "gzip"
			puts "Inflating gzipped content"
			File.open('tempfile.gz','w+') { |gz| gz.write(httpflow_object.flow_data.message_body)}    ## Create tempfile to decompress: Is it really needed ?
			Zlib::GzipReader.open('tempfile.gz') { |gzip| 
				c_data = gzip.read 
			}
			File.delete('tempfile.gz')    # delete tempfile after use
		end	


		# Write File (Overwrite if exists)
		File.open(filename,'w+') { |f| f.write(c_data)
		f.close
		}
	end
end

def display_conversation_flows(httpconv_array,conv_index,flow_index=nil,headers=nil)

	if headers
		puts " "
		puts "Listing flows for conversation #{conv_index} with full http headers"
		puts "----------------------------------------------------------------------"
		puts " "

		if flow_index
			flow = httpconv_array[conv_index].flows[flow_index]
			content = flow.flow_data.message_type == "REQUEST" ? flow.flow_data.requested_uri : flow.flow_data.content_type
			puts " "
			puts "Flow Index: #{flow_index}	#{flow.ipsrc}:#{flow.srcport} -> #{flow.ipdst}:#{flow.dstport}	#{flow.flow_data.message_type}	#{content}	Length: #{flow.flow_data.content_length}"
			puts " "
			puts " -----------"
			puts " HTTP HEADER"
			puts " -----------"
			puts " "
			puts flow.flow_data.headers
		else
			httpconv_array[conv_index].flows.each { |flow|
				content = flow.flow_data.message_type == "REQUEST" ? flow.flow_data.requested_uri : flow.flow_data.content_type
				puts " "
				puts "Flow Index: #{httpconv_array[conv_index].flows.index(flow)}	#{flow.ipsrc}:#{flow.srcport} -> #{flow.ipdst}:#{flow.dstport}	#{flow.flow_data.message_type}	#{content}	#{flow.flow_data.content_length}"
				puts " "
				puts " -----------"
				puts " HTTP HEADER"
				puts " -----------"
				puts " "
				puts flow.flow_data.headers
				puts "----------------------------------------------------------------------"
			}
		end
	else

		puts "FLOWS TABLE"
		flows_table = table do |t|
			t.headings = ["Flow Index","Hosts","HTTP message type","HTTP Request or Content type", "HTTP Content Length"]
			if !flow_index
				httpconv_array[conv_index].flows.each { |flow|
					content = flow.flow_data.message_type == "REQUEST" ? flow.flow_data.requested_uri : flow.flow_data.content_type
					t << ["#{httpconv_array[conv_index].flows.index(flow)}","#{flow.ipsrc}:#{flow.srcport} -> #{flow.ipdst}:#{flow.dstport}", "#{flow.flow_data.message_type}","#{content}","#{flow.flow_data.content_length}"]
				}
			else
				flow = httpconv_array[conv_index].flows[flow_index]
				content = flow.flow_data.message_type == "REQUEST" ? flow.flow_data.requested_uri : flow.flow_data.content_type
				t << ["#{flow_index}","#{flow.ipsrc}:#{flow.srcport} -> #{flow.ipdst}:#{flow.dstport}", "#{flow.flow_data.message_type}","#{content}","#{flow.flow_data.content_length}"]
			end
		end
		puts flows_table
	end
end
	














if !$options[:port]
	$http_port = 80 
else
	$http_port = $options[:port]
end

$httpconv_a = HttpConversationArray.new
httpflow_a = []
fragments = nil
total_len = 0


if $options[:host]
	$host_filter = $options[:host].match(/\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b/).to_s
	$port_filter = $options[:host].match(/:\d{1,5}$/).to_s.gsub(/:/,"")
	
end

puts "Reading file #{$options[:infile]}"
pkt_array = PacketFu::Read.f2a(:file => $options[:infile])
puts "Parsing packets..."
ps_start = Time.now
pkt_array.each { |pkt|
  packet = PacketFu::Packet.parse(pkt)																									# packetfu Initial packet parsing
	
																																					
	if packet.is_tcp? and (packet.tcp_sport==$http_port or packet.tcp_dport==$http_port)		# only tcp packets matching http_port (src or dst)
		
		if packet.payload.length != 0																												# Only tcp packet with payload are interesting
		
			if fragments
				fragments += packet.payload
				if fragments.length == total_len
					reassembled = HttpResponse.new
					reassembled.parse(fragments)
					httpflow = HttpFlow.new(packet.ip_saddr,packet.tcp_sport,packet.ip_daddr,packet.tcp_dport)
					httpflow.flow_data = reassembled
					httpflow_a << httpflow

					conv_index = $httpconv_a.find(packet.ip_saddr,packet.tcp_sport,packet.ip_daddr,packet.tcp_dport)
					if conv_index
						$httpconv_a[conv_index].add_flow(httpflow)
					else
						conv = HttpConversation.new(httpflow)
						$httpconv_a << conv
					end					
						
					
					fragments = nil
					total_len = 0
				end
																																	
			elsif packet.payload =~ /OPTIONS|GET|HEAD|POST|PUT|DELETE|TRACE|CONNECT\s+(\S+)/ 	# Http request message	
				http_request = HttpRequest.new							
				http_request.parse(packet.payload)
				httpflow = HttpFlow.new(packet.ip_saddr,packet.tcp_sport,packet.ip_daddr,packet.tcp_dport)
				httpflow.flow_data = http_request
				httpflow_a << httpflow					
				
				conv_index = $httpconv_a.find(packet.ip_saddr,packet.tcp_sport,packet.ip_daddr,packet.tcp_dport)
					if conv_index
						$httpconv_a[conv_index].add_flow(httpflow)
					else
						conv = HttpConversation.new(httpflow)
						$httpconv_a << conv
					end			
			
		
			elsif packet.payload =~ /^(HTTP\/.*)$/																										
				http_response = HttpResponse.new							
				http_response.parse(packet.payload)
				if http_response.message_body.length == http_response.content_length						# Single segment content
					httpflow = HttpFlow.new(packet.ip_saddr,packet.tcp_sport,packet.ip_daddr,packet.tcp_dport)
					httpflow.flow_data = http_response
					httpflow_a << httpflow	

					conv_index = $httpconv_a.find(packet.ip_saddr,packet.tcp_sport,packet.ip_daddr,packet.tcp_dport)
					if conv_index
						$httpconv_a[conv_index].add_flow(httpflow)
					else
						conv = HttpConversation.new(httpflow)
						$httpconv_a << conv
					end							
				
				else																																						# multi-segment fragmented content
					total_len = http_response.start_line.length + http_response.header_length + http_response.content_length
					fragments = packet.payload
				end
			end
		end
	end
}
ps_stop = Time.now
ps_time = ps_stop - ps_start
puts "#{pkt_array.length} packets read in %.3f sec." % [ps_time]
puts " "

if $options[:stats]
	# stats arguments minimal verification
	stats_type, requester_ip, hostname = $options[:stats]
	if stats_type.match(/request/)  
		
		# Create the HttpStats object
		httpstats = HttpStats.new($httpconv_a)
		# Generate statistics
		stats_result = httpstats.request_stats(requester_ip)
		stats_table = table do |t|
			t.headings = ["Requested Host", "Requested Host IP", "Request Count"]
			stats_result.each { |stat|
				t << stat
			}
		end
		#Displays it
		puts "HTTP REQUEST STATISTICS"
		puts "-----------------------"
		begin
			puts stats_table
			exit(0)
		rescue
			puts "No stats found for #{host_ip}"
			exit(0)
		end
	elsif stats_type.match(/uri/) 
		# Create the HttpStats object
		httpstats = HttpStats.new($httpconv_a)
		# Generate statistics
		stats_result = httpstats.uri_list(requester_ip,hostname)
		puts "----------------------------------"
		puts "Listing URI requested %s" % [ requester_ip.nil? ? "ALL clients": "by #{requester_ip}"]
		puts "----------------------------------"
		stats_result.each {|httphost|
			puts "----------------------------------"
			puts "Requested to #{httphost.hostname}"
			puts "----------------------------------"
			httphost.uri_list.each { |uri|
				puts "[conv: %d] [flow: %d] %s" % [uri[0],uri[1],uri[2]]
			}
		}
		exit(0)
	end
end	



flow_filter = $options[:flow_filter].to_i if $options[:flow_filter]
display_headers = $options[:headers]

if $options[:conv_filter]
	conversation_filter = $options[:conv_filter].to_i
	if $options[:dump2disk]
		# Want to dump conversation data
		dump_conversation_flows($httpconv_a,conversation_filter,flow_filter)
	else
		# Want to display only
		display_conversation_flows($httpconv_a,conversation_filter,flow_filter,display_headers)
	end
elsif $options[:dump2disk]
	#Want to dump all conversation datas
	puts "Dumping ALL conversations data to disk !"
	dump_conversation_flows($httpconv_a)


else
	# Default output : Displays all conversation summary.
	puts "Found #{$httpconv_a.length} HTTP conversation(s)"
	conv_table = table do |t| 
			t.headings = ['Conversation Index','Hosts', 'HTTP Flow count',"Request","Response", "Cumulative length"]
			$httpconv_a.each { |conv|
				t <<  ["#{$httpconv_a.index(conv)}","#{conv.host_1['ip']}:#{conv.host_1['port']} < - > #{conv.host_2['ip']}:#{conv.host_2['port']}","#{conv.flows.length}","#{conv.flow_count("REQUEST")}","#{conv.flow_count("RESPONSE")}","#{conv.cumulative_len}"]
			}
	end
	puts conv_table
end