20a21,33
> # (http://www.brendangregg.com/chaosreader.html seems to be outdated.
> # E-mail to the address brendan@sun.com mentioned on that web site returned
> # with an error.)
> #
> # 11-Sep-2011, ver 0.95
> # 24-Sep 2011, ver 0.95b
> # 04-Jan 2012, ver 0.95c
> # 10-Jan 2012, ver 0.95d
> # 15-Mar 2013, ver 0.95e
> # 15-Apr 2013, ver 0.95f
> # 18-Apr 2013, ver 0.95g
> # 12-Apr 2014, ver 0.95h
> #
32c45
< # USAGE: chaosreader [-aehikqrvxAHIRTUXY] [-D dir] 
---
> # USAGE: chaosreader [-adehiknqrvxAHIRTUXY] [-D dir]
45a59
> #    -d, --preferdns       # Show DNS names instead of IP addresses
63a78
> #    -n, --names           # Include hostnames in hyperlinked HTTPlog (HTML)
199c214
< # SEE ALSO: ethereal (GUI packet viewer), dsniff (sniffing toolkit)
---
> # SEE ALSO: wireshark (GUI packet viewer), dsniff (sniffing toolkit)
202,206c217,224
< #
< #  This program is free software; you can redistribute it and/or
< #  modify it under the terms of the GNU General Public License
< #  as published by the Free Software Foundation; either version 2
< #  of the License, or (at your option) any later version. 
---
> #            Copyright (c) 2008 Indian Larry.
> #            Copyright (c) 2011, 2012, 2013 Jens Lechtenbörger.
> #            Copyright (c) 2014 Pavel Hančar.
> #
> # This program is free software: you can redistribute it and/or modify
> # it under the terms of the GNU General Public License as published by
> # the Free Software Foundation, either version 3 of the License, or
> # (at your option) any later version.
214,215c232
< #  along with this program; if not, write to the Free Software Foundation, 
< #  Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
---
> # along with this program.  If not, see <http://www.gnu.org/licenses/>.
217,219c234,237
< #  (http://www.gnu.org/copyleft/gpl.html)
< #
< # Author: Brendan Gregg  [Sydney, Australia]
---
> # Authors: Brendan Gregg      [Sydney, Australia]
> #          Indian Larry       [http://refrequelate.blogspot.com/]
> #          Jens Lechtenbörger [Münster, Germany]
> #          Pavel Hančar       [Hořice, Czech Republic]
240a259,314
> #
> # 11-Sep-2011, Jens Lechtenbörger:
> # - Switch from GPLv2 to GPLv3
> # - Integrate diff from
> #   http://refrequelate.blogspot.com/2008/07/more-de-chunking-chaosreader-patch.html
> #   to reassemble chunked HTTP transfers.
> # - Parse linux cooked captures, which result from listening on "any"
> #   interface.  (Chaosreader0.94 does not produce any output for such
> #   pcap files.)
> # - Use HTTP content-type to identify file types such as HTML, XML,
> #   Javascript, CSS; use those types for better file extensions than
> #   "data".
> # - Uncompress gzip'ed data.
> # - Add new command line switch to show host names in HTTPlog and to
> #   create href-links from HTTPlog rows to the corresponding rows in
> #   the table on index.html.
> # - Several minor improvements (see comments with "JL:").
> #
> # 24-Sep-2011, Jens Lechtenbörger:
> # - More systematic Content-Type handling based on MIME types.
> # - More image types included in Image Report based on MIME types.
> #
> # 4-Jan-2012, Jens Lechtenbörger:
> # - Parsing of DNS replies to show names instead of IP addresses (new
> #   command line switch -d) and to save DNS replies as text files.
> #
> # 10-Feb-2012, Jens Lechtenbörger:
> # - Use file magic (again) to detect MIME type if HTTP's Content-Type is
> #   application/octet-stream.  (Some Web servers report images incorrectly.)
> #
> # 15-Mar-2013, Jens Lechtenbörger:
> # - Create additional HTTP log file in text format.  That file contains one
> #   line per GET request, which shows the referrer (if present) and indicates
> #   whether cookies have been sent in the request or received in the reply.
> #
> # 15-Apr-2013, Jens Lechtenbörger:
> # - Link additional HTTP log file from index.html.
> # - Also look for images in plain/text Content-Types (seen in the wild).
> # - Extend GET/POST report to include all GETs; not only those including a
> #   question mark (with parameters).
> #
> # 18-Apr-2013, Jens Lechtenbörger:
> # - Build new "External Image Report" (linked from index.html), where images
> #   are embedded from their origin servers.
> #   In contrast, the "Image Report" points to images on the local hard disk.
> #   The new report may be more suitable for publication on Web pages as it
> #   does not require to publish (potentially copyright protected) images.
> # - Parse CNAME DNS replies to show original host names (which are hopefully
> #   more familiar than aliases).
> # - Show also empty parts on index.html that result from cache hits.
> # - Create directory passed after switch "-D".
> #
> # 11-Apr-2014, Pavel Hančar:
> # - Optimized hexadecimal dumps to use less memory.
> # - Modified "IP Count" to "IP and MAC Count".
> # - Fixed a few bugs concerning output.
241a316
> # Jens frequently calls this program with options "-vden -D <somedir>".
244a320,321
> use IO::Uncompress::Gunzip qw(gunzip $GunzipError);
> use Net::DNS::Packet;
316c393,394
< 
---
> &Set_MIME_Types(); #JL
> &Set_DNS(); #JL
362c440
< 	%Index = (); %Image = (); %GETPOST = (); 
---
> 	%Index = (); %Image = (); %ExtImage = (); %GETPOST = ();
500c578
< 	%Index = (); %Image = (); %GETPOST = (); 
---
> 	%Index = (); %Image = (); %ExtImage = (); %GETPOST = ();
577c655
< 	%Index = (); %Image = (); %GETPOST = (); 
---
> 	%Index = (); %Image = (); %ExtImage = (); %GETPOST = ();
861a940,944
> 	    # JL: Try linux cooked capture
> 	    ($lptype,$lladdr_type,$lladdr_len,
> 	     $ether_src,$ll_dummy,$ether_type,$ether_data) =
> 		unpack('nnnH12nH4a*',$packet_data) unless $decoded;
> 	    if ($ether_type ne "0800") {
863a947
> 	}
963c1047
< 	$Count{IP}{$ip_src}++;
---
> 	$Count{IP}{"$ip_src,$ether_src"}++;
1250,1257d1333
< 		#
< 		#  Populate %Hex{TCP}{} with coloured HTML 2-way
< 		#  traffic, if needed.
< 		#
< 		if ($Arg{output_hex}) {
< 			&Process_Hex("TCP",$session_id,$tcp_data,"blue");
< 		}
< 
1282c1358
< 
---
> 	}
1284c1360
< 		#  Populate %Hex{TCP}{} with coloured HTML 2-way
---
> 	#  Populate %Hex{TCP}{} with data necessary to generate coloured HTML 2-way
1288,1290c1364
< 			&Process_Hex("TCP",$session_id,$tcp_data,"red");
< 		}
< 
---
> 		push(@{$Hex{"TCP"}{$session_id}}, [$from_server, $tcp_data]);
1292d1365
< 
1342a1416,1420
> 	# JL: DNS parsing
> 	if (($udp_src_port == 53) || ($udp_dest_port == 53)) {
> 	    &Process_DNS($udp_data, $session_id);
> 	}
> 
1361,1369d1438
< 
< 		#
< 		#  Populate %Hex{UDP}{} with coloured HTML 2-way
< 		#  traffic, if needed.
< 		#
< 		if ($Arg{output_hex}) {
< 			&Process_Hex("UDP",$session_id,$udp_data,"blue");
< 		}
< 
1383a1453
> 	}
1385c1455
< 		#  Populate %Hex{UDP}{} with coloured HTML 2-way
---
> 	#  Populate %Hex{UDP}{} with data necessary to generate coloured HTML 2-way
1389c1459
< 			&Process_Hex("UDP",$session_id,$udp_data,"red");
---
> 		push(@{$Hex{"UDP"}{$session_id}}, [$from_server, $udp_data]);
1392d1461
< 	}
1494,1495c1563,1564
< 	#  Populate %Hex{ICMP}{} with coloured HTML 
< 	#  traffic, if needed.
---
> 	#  Populate %Hex{ICMP}{} with data necessary to generate coloured HTML
> 	#  traffic, if needed. $from_server is always 1. session_id is time
1498c1567
< 		&Process_Hex("ICMP",$time,$icmp_data,"blue");
---
> 		push(@{$Hex{"ICMP"}{$time}}, [1, $icmp_data]);
1580a1650
> 
1630c1700,1702
< 	$Index{HTML}[$number] = "<tr><td><i>$number.</i></td>" .
---
> 	# JL: Added id attribute as link target
> 	$Index{HTML}[$number] = "<tr id=\"$number\">" .
> 	 "<td><i>$number.</i></td>" .
1645a1718,1722
> 	### JL: Construct external image info line (in case it is needed)
> 	$ExtImage{HTML}[$number]{info} = "<tr><td><i>$number.</i>" .
> 	 "</td><td><b>$starttime</b></td><td> " .
> 	 "<font color=\"blue\">$id_html </font></td><td><td>\n";
> 
1729,1731c1806
< 		&Process_Hex_Finish("TCP",$session_id);
< 		&Save_Hex_HTML("TCP",$session_id,$number,$service_name,
< 		 $id_html);
---
> 		my ($text, $html) = &Process_Hex("TCP", $session_id);
1733c1808,1810
< 		 $id_text);
---
> 			$id_text, $text);
> 		&Save_Hex_HTML("TCP", $session_id, $number, $service_name,
> 			$id_html, $html);
1758c1835
< 		&Process_HTTP($session_id);
---
> 		&Process_HTTP($session_id,$number);
1902a1980,1983
> 	if ($Arg{prefer_dns}) {
> 	    $ip_src = &Get_Name_For_IP($ip_src);
> 	    $ip_dest = &Get_Name_For_IP($ip_dest);
> 	}
1982,1984c2063
< 		&Process_Hex_Finish("UDP",$session_id);
< 		&Save_Hex_HTML("UDP",$session_id,$number,$service_name,
< 		 $id_html);
---
> 		my ($text, $html) = &Process_Hex("UDP", $session_id);
1986c2065,2067
< 		 $id_text);
---
> 			$id_text, $text);
> 		&Save_Hex_HTML("UDP", $session_id, $number, $service_name,
> 			$id_text, $html);
1996a2078,2081
> 	if ($service == 53) {
> 	        &Save_DNS_File($session_id,$number);
> 	}
> 
2150,2152c2235,2239
< 		&Process_Hex_Finish("ICMP",$time);
< 		&Save_Hex_HTML("ICMP",$time,$number,$service_name,$id_html);
< 		&Save_Hex_Text("ICMP",$time,$number,$service_name,$id_text);
---
> 		my ($text, $html) = &Process_Hex("ICMP", $time);
> 		&Save_Hex_Text("ICMP", $session_id, $number, $service_name,
> 			$id_text, $text);
> 		&Save_Hex_HTML("ICMP", $session_id, $number, $service_name,
> 			$id_html, $html);
2160a2248,2275
> # JL: Process_DNS - DNS processing.  Look for DNS replies and store
> #     names for IP addresses into %DNS.
> #     Also store CNAME aliases so that the "original" name can be retrieved.
> #
> sub Process_DNS {
> 	my $data = shift;
> 	my $session_id = shift;
> 
> 	my $dns = Net::DNS::Packet->new(\$data);
> 
> 	unless ($dns) {
> 	    #print "Failed to create Net::DNS::Packet!\n";
> 	    return;
> 	}
> 
> 	$UDP{id}{$session_id}{DNS} = $dns->string;
> 	foreach my $rr ($dns->answer) {
> 	    if ($rr->type eq "A") {
> 		$DNS{$rr->address} = $rr->name;
> 	    }
> 	    if ($rr->type eq "CNAME") {
> 		$DNS{$rr->cname} = $rr->name;
> 	    }
> 	}
> }
> 
> 
> 
2162a2278
> # JL: Added host parameter
2165c2281
< 	my ($junk,$var,$value,$term,$data,$request,$site,$post,$get,$reply);
---
> 	my ($junk,$var,$value,$term,$data,$request,$host,$site,$post,$get,$reply);
2173a2290,2291
> 	my $number = shift;
> 	my $partnum = 0;
2218c2336
< 
---
> 	   $partnum++;
2220a2339,2344
> 		### JL: Get the host string, referer, and cookies.
> 		($host) = $request =~ /\sHost:\s(\S*)\s/is;
> 		($referer) = $request =~ /\sReferer:\s(\S*)/is;
> 		($cookie) = $request =~ /\sCookie:\s(\S*)/is;
> 		($setcookie) = $reply =~ /\sSet-Cookie:\s(\S*)/is;
> 
2224a2349,2352
> 		        # JL: Prefer hostname over IP address
> 		        if ($Arg{httplog_html}) {
> 			    $site = "http://${host}$site";
> 			} else {
2226a2355
> 		}
2230,2231c2359,2361
< 		($type) = $reply =~ /Content-Type:\s(\S*)/s;
< 		($size) = $reply =~ /Content-Length:\s(\S*)/s;
---
> 		# JL: Be careful to use case insensitive matching
> 		($type) = $reply =~ /Content-Type:\s(\S*)/is;
> 		($size) = $reply =~ /Content-Length:\s(\S*)/is;
2233a2364
> 
2238,2241c2369,2375
< 		 sprintf("%9d.%03d %6d %s %s/%03d %d %s %s %s %s%s/%s %s\n",
< 		 int($time),(($time - int($time))*1000),($duration*1000),
< 		 $src,$result,$status,$size,"GET",$site,"-","NONE","",
< 		 "-",$type);
---
> 			Print_Log_Line($number,$time,$duration,
> 				$src,$dest,$result,$status,$size,
> 				"GET",$site,"-","NONE","","-",$type);
> 		$HTTPtxtlog{time}{$time} =
> 			Print_TxtLog_Line($number,$time,
> 				$referer,$cookie,$setcookie,
> 				"GET",$site);
2243a2378,2381
> 		### JL: External image data.
> 		if ( defined $ExtImage{HTML}[$number]{parts}[$partnum] ) {
> 		    $ExtImage{HTML}[$number]{links} .= "<img src=\"$site\"> ";
> 		}
2250a2389,2393
> 		### JL: Get the host string, referer, and cookies.
> 		($host) = $request =~ /\sHost:\s(\S*)\s/is;
> 		($referer) = $request =~ /\sReferer:\s(\S*)/is;
> 		($cookie) = $request =~ /\sCookie:\s(\S*)/is;
> 		($setcookie) = $reply =~ /\sSet-Cookie:\s(\S*)/is;
2254,2255c2397,2398
< 		($type) = $reply =~ /Content-Type:\s(\S*)/s;
< 		($size) = $reply =~ /Content-Length:\s(\S*)/s;
---
> 		($type) = $reply =~ /Content-Type:\s(\S*)/is;
> 		($size) = $reply =~ /Content-Length:\s(\S*)/is;
2262,2265c2405,2411
< 		 sprintf("%9d.%03d %6d %s %s/%03d %d %s %s %s %s%s/%s %s\n",
< 		 int($time),(($time - int($time))*1000),($duration*1000),
< 		 $src,$result,$status,$size,"POST",$site,"-","NONE","",
< 		 "-",$type);
---
> 		    Print_Log_Line($number,$time,$duration,
> 				   $src,$dest,$result,$status,$size,
> 				   "POST",$site,"-","NONE","","-",$type);
> 		$HTTPtxtlog{time}{$time} =
> 			Print_TxtLog_Line($number,$time,
> 				$referer,$cookie,$setcookie,
> 				"POST",$site);
2273c2419,2422
< 	   if ($request =~ /^GET \S*\?\S* HTTP/) {
---
> 	   # JL: chaosreader 0.94 includes only URIs containing a question
> 	   # mark.  Why?  Go for all instead.
> 	   #if ($request =~ /^GET \S*\?\S* HTTP/) {
> 	   if ($request =~ /^GET \S* HTTP/) {
2277c2426,2428
< 
---
> 	     if ($site eq "") {
> 		 ($site) = $request =~ /^GET (\S*)\s/;
> 	     }
2279c2430,2431
< 	     if ($get =~ /=/) {
---
> 	     # JL: Why only those with parameters?
> 	     #if ($get =~ /=/) {
2309c2461
< 	     }
---
> 	     #}
2569c2721
< 		print "Chaosreader ver 0.94\n\n";
---
> 		print "Chaosreader ver 0.95h\n\n";
2645a2798,2799
> <a href="extimage.html"><font color="blue"><b>External Image Report</b></font></a>
>  $image_empty - Click here for a report embedding external images.<br>
2648,2649c2802,2805
< <a href="httplog.text"><font color="blue"><b>HTTP Proxy Log</b></font></a> 
<  $httplog_empty - Click here for a generated proxy style HTTP log.<p>
---
> <a href="$Arg{httplog_name}"><font color="blue"><b>HTTP Proxy Log</b></font></a>
>  $httplog_empty - Click here for a generated proxy style HTTP log.<br>
> <a href="$Arg{httplog_txt}"><font color="blue"><b>New HTTP Proxy Log</b></font></a>
>  $httplog_empty - Click here for HTTP log with referers and Cookie indicators.<p>
2660c2816
< <font size=+2>IP Count</font><br>
---
> <font size=+2>IP and MAC Count</font><br>
2661a2818
> <tr><th>IP</th><th>MAC</th><th>Count</th></tr>
2663c2820
< 	foreach $IP (sort {$Count{IP}{$b} <=> $Count{IP}{$a}} 
---
> 	foreach $IP_MAC (sort {$Count{IP}{$b} <=> $Count{IP}{$a}}
2665c2822,2824
< 		print FILE "<tr><td>$IP</td><td>$Count{IP}{$IP}</td></tr>\n";
---
> 		#print FILE "<tr><td>$IP</td><td>$Count{IP}{$IP}</td></tr>\n";
> 		($ip, $mac) = split(',', $IP_MAC);
> 		print FILE "<tr><td>$ip</td><td>$mac</td><td>$Count{IP}{$IP_MAC}</td></tr>\n";
2759a2919,2947
> 	# --- extimage.html ---
> 
> 	#
> 	#  Create HTML External Image Index file to display images
> 	#
> 	open(FILE,">extimage.html") || die "ERROR24: creating index: $!\n";
> 	print FILE <<END_HTML;
> <html>
> <head><title>Chaosreader External Image Report</title></head>
> <body bgcolor="white" textcolor="black">
> <font size=+3>Chaosreader External Image Report</font><br>
> <font size=+1>Created at: $the_date, Type: $TYPE</font><p>
> <font size=+2>Images</font><br>
> <table border=2>
> END_HTML
> 	for ($html_index=0; $html_index <= $#{$Index{HTML}}; $html_index++) {
> 		$html_line = $ExtImage{HTML}[$html_index]{info};
> 		$html_links = $ExtImage{HTML}[$html_index]{links};
> 		next unless defined $html_links;
> 		print FILE "$html_line $html_links </td></tr>\n";
> 	}
> 	print FILE <<END_HTML;
> </table><p>
> </body>
> </html>
> END_HTML
> 
> 
> 	######################
2934a3123,3174
> # JL: Print a line for the HTTPlog
> #
> sub Print_Log_Line {
> 	my $number = shift;
>         my $time = shift;
> 	my $duration = shift;
> 	my $src = shift;
> 	my $dest = shift;
> 	my $result = shift;
> 	my $status = shift;
> 	my $size = shift;
> 	my $method = shift;
> 	my $site = shift;
> 	my $type = shift;
> 
> 	if ($Arg{httplog_html}) {
> 	    sprintf("<pre><a href=\"index.html#%d\">%d</a>" .
> 		    " %9d.%03d %6d " .
> 		    "%-15s %-15s %s/%03d %d %s %s %s %s%s/%s %s</pre><br/>\n",
> 		    $number,$number,
> 		    int($time),(($time - int($time))*1000),($duration*1000),
> 		    $src,$dest,$result,$status,$size,
> 		    $method,$site,"-","NONE","","-",$type);
> 	} else {
> 	    sprintf("%9d.%03d %6d %s %s/%03d %d %s %s %s %s%s/%s %s\n",
> 		    int($time),(($time - int($time))*1000),($duration*1000),
> 		    $src,$result,$status,$size,
> 		    $method,$site,"-","NONE","","-",$type);
> 	}
> }
> 
> # JL: Print a line for the new text HTTPlog
> #
> sub Print_TxtLog_Line {
> 	my $number = shift;
>         my $time = shift;
> 	my $referer = shift;
> 	my $cookie = shift;
> 	my $setcookie = shift;
> 	my $method = shift;
> 	my $site = shift;
> 
> 	($second, $minute, $hour, $dayOfMonth, $month, $yearOffset, $dayOfWeek, $dayOfYear, $daylightSavings) = localtime($time);
> 	$referer = "Referer: " . $referer if $referer ne "";
> 	$cookie = "Cookie sent." if $cookie ne "";
> 	$setcookie = "Sets cookie." if $setcookie ne "";
> 	sprintf("%-4s %02d:%02d:%02d %s %s %s %s %s\n",
> 		$number,$hour,$minute,$second,
> 		$method,$site,$referer,$cookie,$setcookie);
> }
> 
> 
2942c3182,3183
< 	#  Create HTTPlog.text
---
> 	#  Create httplog file
>         # JL: Don't use hardcoded filename
2944,2945c3185
< 	open(FILE,">httplog.text") || die "ERROR29: creating HTTP log: $!\n";
< 
---
>         open(FILE,">$Arg{httplog_name}") || die "ERROR29: creating HTTP log: $!\n";
2950a3191,3198
> 
>         open(FILE,">$Arg{httplog_txt}") || die "ERROR29: creating HTTP text log: $!\n";
> 
> 	foreach $time (sort { $a <=> $b }(keys (%{$HTTPtxtlog{time}}))) {
> 		print FILE $HTTPtxtlog{time}{$time};
> 	}
> 
> 	close FILE;
2960a3209,3215
> 	if ( $http_data eq "" ) {
> 	    return "empty";
> 	}
> 	if ( length($http_data) < 8 ) {
> 	    return "small";
> 	}
> 
2962a3218
> 	elsif ($data =~ /^.PNG/) 	        { $type = "png"; } # JL
2969a3226
> 	elsif ($data =~ /<?xml/i) 		{ $type = "xml"; } # JL
2981,2984c3238,3239
< 	return 1 if ($ext eq "jpeg");
< 	return 1 if ($ext eq "gif");
< 
< 	return 0;
---
> 	# JL: Use MIME types.
> 	return ($ext_types{$ext} eq "image");
3079,3081c3334
< #		For code reuse it uses it's own data structure %Hex.
< #		(Originally used %TCP{id}{$session_id}{hex}).
< #
---
> #		Uses data stored to data structure %Hex.
3086,3094c3339,3340
< 	my $data = shift;
< 	my $colour = shift;
< 	my $pos = $Hex{$type}{$session_id}{pos};
< 	my $offset = $Hex{$type}{$session_id}{offset};
< 	my $hexhtml = $Hex{$type}{$session_id}{hexhtml};
< 	my $viewhtml = $Hex{$type}{$session_id}{viewhtml};
< 	my $hextext = $Hex{$type}{$session_id}{hextext};
< 	my $viewtext = $Hex{$type}{$session_id}{viewtext};
< 	my (@Bytes,$byte,$view,$view2);
---
> 	my $offset = 0;
> 	my (@Bytes,$byte,$colour,$from_server,$hexhtml,$hextext,$html,$pos,$text,$view,$view2,$viewhtml,$viewtext);
3096a3343,3346
> 	### Process
> 	foreach $from_server_AND_data (@{$Hex{$type}{$session_id}}) {
> 		($from_server, $data) = @{$from_server_AND_data};
> 		$colour = $from_server ? "blue" : "red";
3098d3347
< 	$offset = 0 unless defined $offset;
3101,3102d3349
< 
< 	### Process
3117,3118c3364
< 			$Hex{$type}{$session_id}{text} .= 
< 			 sprintf("%6.08x",$offset) . "  $hextext  $viewtext\n";
---
> 				$text .= sprintf("%6.08x",$offset) . "  $hextext  $viewtext\n";
3121,3123c3367,3369
< 			$Hex{$type}{$session_id}{HTML} .= 
< 			 '<font color="green">' . sprintf("%6.08x",$offset) . 
< 			 "</font>  $hexhtml  $viewhtml\n";
---
> 				$hexhtml .= "</font>";
> 				$viewhtml .= "</font>";
> 				$html .= '<font color="green">' . sprintf("%6.08x",$offset) .  "</font>  $hexhtml  $viewhtml\n";
3127,3128c3373,3374
< 			$hexhtml = "</font><font color=\"$colour\">";
< 			$viewhtml = "</font><font color=\"$colour\">";
---
> 				$hexhtml = "<font color=\"$colour\">";
> 				$viewhtml = "<font color=\"$colour\">";
3138,3144d3383
< 
< 	$Hex{$type}{$session_id}{pos} = $pos;
< 	$Hex{$type}{$session_id}{offset} = $offset;
< 	$Hex{$type}{$session_id}{hexhtml} = $hexhtml;
< 	$Hex{$type}{$session_id}{viewhtml} = $viewhtml;
< 	$Hex{$type}{$session_id}{hextext} = $hextext;
< 	$Hex{$type}{$session_id}{viewtext} = $viewtext;
3147,3162d3385
< 
< 
< # Process_Hex_Finish - Finish the hex dumps.
< #
< sub Process_Hex_Finish {
< 	### Input
< 	my $type = shift;
< 	my $session_id = shift;
< 	my $pos = $Hex{$type}{$session_id}{pos};
< 	my $offset = $Hex{$type}{$session_id}{offset};
< 	my $hexhtml = $Hex{$type}{$session_id}{hexhtml};
< 	my $viewhtml = $Hex{$type}{$session_id}{viewhtml};
< 	my $hextext = $Hex{$type}{$session_id}{hextext};
< 	my $viewtext = $Hex{$type}{$session_id}{viewtext};
< 	my ($short);
< 
3164c3387
< 	return if $pos == 1;
---
> 	return ($text, $html) if $pos == 1;
3171,3172c3394
< 	$Hex{$type}{$session_id}{text} .= 
< 	 sprintf("%6.08x",$offset) . "  $hextext  $viewtext\n";
---
> 	$text .= sprintf("%6.08x",$offset) . "  $hextext  $viewtext\n";
3175,3177c3397
< 	$Hex{$type}{$session_id}{HTML} .= 
< 	 '<font color="green">' . sprintf("%6.08x",$offset) . 
< 	 "</font>  $hexhtml  $viewhtml\n";
---
> 	$html .= '<font color="green">' . sprintf("%6.08x",$offset) .  "</font>  $hexhtml  $viewhtml\n";
3178a3399
> 	return ($text, $html)
3430d3650
< 
3442c3662
< 	my $session_data = $Hex{$type}{$session_id}{HTML};
---
> 	my $session_data = shift;
3487d3706
< 
3496a3716
> 	my $service_name = shift;
3498c3718
< 	my $session_data = $Hex{$type}{$session_id}{text};
---
> 	my $session_data = shift;
5347a5568,5572
> 	        # JL.  I want to see all parts, in partcular empty ones
> 		# resulting from 304 (Not Modified).  Thus, the original
> 		# check below on $http_data is too strong.
> 		next if $http_part eq "";
> 
5350,5351c5575,5576
< 		next if $http_data eq "";
< 		next if length($http_data) < 8;
---
> 		# next if $http_data eq "";
> 		# next if length($http_data) < 8;
5354a5580,5596
> 		### JL: Chunk Check, patch from http://refrequelate.blogspot.com/2008/07/more-de-chunking-chaosreader-patch.html
> 		if ( $http_header =~ /Transfer-Encoding: chunked/ ) {
> 		    my $new_http_data="";
> 		    my $chunksize=-1;
> 		    my $pos=0;
> 		    until ($chunksize==0) {
> 			my $eolpos=index($http_data,"\r\n",$pos);
> 			$chunksize=hex(substr($http_data,$pos,$eolpos - $pos));
> 			$pos=($eolpos+2);
> 			if ($chunksize > 0) {
> 			    $new_http_data.=substr($http_data,$pos,$chunksize);
> 			}
> 			$pos+=($chunksize+2);
> 		    }
> 		    $http_data=$new_http_data;
> 		}
> 
5356c5598
< 		$http_type = &File_Type($http_data);
---
> 		my $http_type = &File_Type($http_data);
5359a5602,5628
> 		### JL: Check for known MIME type in Content-Type
> 		my ($content_type) = $http_header =~ /Content-Type:\s+(\S*)/is;
> 		my $file_extension = "";
> 		if ($content_type ne "") {
> 		    for my $pattern ( keys %mime_types ) {
> 			my $value = $mime_types{$pattern};
> 			if ( $content_type =~ /$pattern/i ) {
> 			    $file_extension = $value;
> 			    last;
> 			}
> 		    }
> 		    if (($file_extension eq "bin")
> 			or ($file_extension eq "asc")) {
> 			# Not too specific.  Some HTTP servers return images
> 			# with Content-Type: application/octet-stream, others
> 			# with Content-Type: text/plain
> 			# Prefer http_type then...
> 			if ($http_type ne "data") {
> 			    $file_extension = $http_type;
> 			}
> 		    }
> 		    elsif ($file_extension eq "") {
> 			print "Unkown Content-Type $content_type.";
> 			print "  May want to extend MIME types.\n";
> 		    }
> 		}
> 
5361,5362c5630,5637
< 	        $filename = "session_${numtext}.part_$parttext${ext}." .
< 		 "$http_type";
---
> 		# JL: Create filename based on Content-Type
> 	        my $filename = "session_${numtext}.part_$parttext${ext}";
> 		if ($file_extension ne "") {
> 		    $filename .= ".$file_extension";
> 		}
> 		if ( ($file_extension eq "") or ($http_type eq "gz") ) {
> 		    $filename .= ".$http_type";
> 		}
5368a5644,5654
> 		### JL: gz decompressing
> 		if ( $http_type eq "gz" ) {
> 		    my $gunzipped = substr($filename, 0, length($filename) - 3);
> 		    my $gunzip_failed = 0;
> 		    gunzip $filename => $gunzipped
> 			or $gunzip_failed = 1;
> 		    if ( $gunzip_failed == 0 ) {
> 			$filename = $gunzipped;
> 		    }
> 		}
> 
5375,5377c5661,5663
< 		if (&Is_Image($http_type)) {
< 			$Image{HTML}[$number]{links} .= 
< 			 "<img src=\"$filename\"> ";
---
> 		if (&Is_Image($http_type) or
> 		    &Is_Image($file_extension)) { # JL: Also check file ext.
> 		    $Image{HTML}[$number]{links} .= "<img src=\"$filename\"> ";
5378a5665,5666
> 		    # JL: Remember this part as image.
> 		    $ExtImage{HTML}[$number]{parts}[$partnum] = 1;
5431a5720,5743
> # Save_DNS_File - Save DNS file.
> #
> sub Save_DNS_File {
> 	my ($filename);
> 	my $session_id = shift;
> 	my $number = shift;
> 	my $numtext = sprintf("%04d",$number);
> 	my $text = $UDP{id}{$session_id}{DNS};
> 
> 	### Output
>         $filename = "session_${numtext}.domain.txt";
> 	open (OUT,">$filename") || die "ERROR52: file create, $filename: $!\n";
> 	print OUT $text;
> 	close OUT;
> 
> 	### Global Vars
> 	my $length = length($text);
> 	$Index{HTML}[$number] .= "<li><a href=\"$filename\">$filename</a>" .
> 	 " $length bytes</li>\n";
> 	$Index{Text}[$number] .= sprintf("%-4s %-45s %-10s %8s bytes\n",
> 	 '"' , "   $filename","",$length);
> }
> 
> 
5509a5822,5835
> # Retrieve DNS name for IP address based on DNS traffic of this capture.
> # If possible, retrieve the original name for a CNAME of the IP address.
> #
> sub Get_Name_For_IP {
>     my $ip_addr = shift;
>     my $result = $ip_addr;
>     if (defined $DNS{$ip_addr}) {
> 	$result = $DNS{$ip_addr};
> 	while (defined $DNS{$result}) {
> 	    $result = $DNS{$result};
> 	}
>     }
>     return $result;
> }
5578a5905,5908
> 		if ($Arg{prefer_dns}) {
> 		    $ip_src = &Get_Name_For_IP($ip_src);
> 		    $ip_dest = &Get_Name_For_IP($ip_dest);
> 		}
5581a5912,5915
> 	        if ($Arg{prefer_dns}) {
> 		    $ip_src = &Get_Name_For_IP($ip_src);
> 		    $ip_dest = &Get_Name_For_IP($ip_dest);
> 		}
6177a6512,6872
> # JL: Initialize Hostnames for IP addresses
> sub Set_DNS {
>     %DNS = (
> 	"192.168.178.1" => "fritz.box"
> 	)
> }
> 
> # JL: Set_MIME_Types - create hash of MIME types and file extensions.
> sub Set_MIME_Types {
>     # Initialize with types seen in the wild but not covered below.
>     %mime_types = (
> 	"application/binary" => "binary",
> 	"application/ocsp-response" => "ocsp",
> 	"application/pln" => "pln", # bahn.de schedule
> 	"application/vnd.google.safebrowsing-update" => "safebrowsing-update",
> 	"application/vnd.google.safebrowsing-chunk" => "safebrowsing-chunk",
> 	"application/vnd.ms-sync.wbxml" => "mssync", # MS ActiveSync
> 	"application/x-protobuffer" => "protobuffer", # google
> 	"application/x-smd" => "smd", # MapDroyd boundary
> 	"(application|text)/((x-)?javascript|(x-)?js)" => "js",
> 	"(application|text)/json" => "json",
> 	"image/bmp" => "bmp",
> 	"image/vnd.microsoft.icon" => "ico",
> 	"image/x-gif" => "gif",
> 	"(image/x-)?jp(e)?g" => "jpeg",
> 	"image/x-png" => "png",
> 	"text/xml" => "xml");
>     # Following created with:
>     # grep -o -P "^(text|application|audio|image|video).*\t+([a-z0-9]+)" /etc/mime.types > mime.types
>     $raw_mime_types = <<END;
> application/andrew-inset			ez
> application/annodex				anx
> application/atom+xml				atom
> application/atomcat+xml				atomcat
> application/atomserv+xml			atomsrv
> application/bbolin				lin
> application/cap					cap
> application/cu-seeme				cu
> application/davmount+xml			davmount
> application/dsptype				tsp
> application/ecmascript				es
> application/futuresplash			spl
> application/hta					hta
> application/java-archive			jar
> application/java-serialized-object		ser
> application/java-vm				class
> application/javascript				js
> application/m3g					m3g
> application/mac-binhex40			hqx
> application/mac-compactpro			cpt
> application/mathematica				nb
> application/msaccess				mdb
> application/msword				doc
> application/mxf					mxf
> application/octet-stream			bin
> application/oda					oda
> application/ogg					ogx
> application/pdf					pdf
> application/pgp-keys				key
> application/pgp-signature			pgp
> application/pics-rules				prf
> application/postscript				ps
> application/rar					rar
> application/rdf+xml				rdf
> application/rss+xml				rss
> application/rtf					rtf
> application/smil				smi
> application/xhtml+xml				xhtml
> application/xml					xml
> application/xspf+xml				xspf
> application/zip					zip
> application/vnd.android.package-archive		apk
> application/vnd.cinderella			cdy
> application/vnd.google-earth.kml+xml		kml
> application/vnd.google-earth.kmz		kmz
> application/vnd.mozilla.xul+xml			xul
> application/vnd.ms-excel			xls
> application/vnd.ms-pki.seccat			cat
> application/vnd.ms-pki.stl			stl
> application/vnd.ms-powerpoint			ppt
> application/vnd.oasis.opendocument.chart			odc
> application/vnd.oasis.opendocument.database			odb
> application/vnd.oasis.opendocument.formula			odf
> application/vnd.oasis.opendocument.graphics			odg
> application/vnd.oasis.opendocument.graphics-template		otg
> application/vnd.oasis.opendocument.image			odi
> application/vnd.oasis.opendocument.presentation			odp
> application/vnd.oasis.opendocument.presentation-template	otp
> application/vnd.oasis.opendocument.spreadsheet			ods
> application/vnd.oasis.opendocument.spreadsheet-template		ots
> application/vnd.oasis.opendocument.text				odt
> application/vnd.oasis.opendocument.text-master			odm
> application/vnd.oasis.opendocument.text-template		ott
> application/vnd.oasis.opendocument.text-web			oth
> application/vnd.openxmlformats-officedocument.spreadsheetml.sheet		xlsx
> application/vnd.openxmlformats-officedocument.spreadsheetml.template		xltx
> application/vnd.openxmlformats-officedocument.presentationml.presentation	pptx
> application/vnd.openxmlformats-officedocument.presentationml.slideshow		ppsx
> application/vnd.openxmlformats-officedocument.presentationml.template		potx
> application/vnd.openxmlformats-officedocument.wordprocessingml.document		docx
> application/vnd.openxmlformats-officedocument.wordprocessingml.template		dotx
> application/vnd.rim.cod				cod
> application/vnd.smaf				mmf
> application/vnd.stardivision.calc		sdc
> application/vnd.stardivision.chart		sds
> application/vnd.stardivision.draw		sda
> application/vnd.stardivision.impress		sdd
> application/vnd.stardivision.math		sdf
> application/vnd.stardivision.writer		sdw
> application/vnd.stardivision.writer-global	sgl
> application/vnd.sun.xml.calc			sxc
> application/vnd.sun.xml.calc.template		stc
> application/vnd.sun.xml.draw			sxd
> application/vnd.sun.xml.draw.template		std
> application/vnd.sun.xml.impress			sxi
> application/vnd.sun.xml.impress.template	sti
> application/vnd.sun.xml.math			sxm
> application/vnd.sun.xml.writer			sxw
> application/vnd.sun.xml.writer.global		sxg
> application/vnd.sun.xml.writer.template		stw
> application/vnd.symbian.install			sis
> application/vnd.visio				vsd
> application/vnd.wap.wbxml			wbxml
> application/vnd.wap.wmlc			wmlc
> application/vnd.wap.wmlscriptc			wmlsc
> application/vnd.wordperfect			wpd
> application/vnd.wordperfect5.1			wp5
> application/x-123				wk
> application/x-7z-compressed			7z
> application/x-abiword				abw
> application/x-apple-diskimage			dmg
> application/x-bcpio				bcpio
> application/x-bittorrent			torrent
> application/x-cab				cab
> application/x-cbr				cbr
> application/x-cbz				cbz
> application/x-cdf				cdf
> application/x-cdlink				vcd
> application/x-chess-pgn				pgn
> application/x-cpio				cpio
> application/x-csh				csh
> application/x-debian-package			deb
> application/x-director				dcr
> application/x-dms				dms
> application/x-doom				wad
> application/x-dvi				dvi
> application/x-httpd-eruby			rhtml
> application/x-font				pfa
> application/x-freemind				mm
> application/x-futuresplash			spl
> application/x-gnumeric				gnumeric
> application/x-go-sgf				sgf
> application/x-graphing-calculator		gcf
> application/x-gtar				gtar
> application/x-hdf				hdf
> application/x-httpd-php				phtml
> application/x-httpd-php-source			phps
> application/x-httpd-php3			php3
> application/x-httpd-php3-preprocessed		php3p
> application/x-httpd-php4			php4
> application/x-httpd-php5			php5
> application/x-ica				ica
> application/x-info				info
> application/x-internet-signup			ins
> application/x-iphone				iii
> application/x-iso9660-image			iso
> application/x-jam				jam
> application/x-java-jnlp-file			jnlp
> application/x-jmol				jmz
> application/x-kchart				chrt
> application/x-killustrator			kil
> application/x-koan				skp
> application/x-kpresenter			kpr
> application/x-kspread				ksp
> application/x-kword				kwd
> application/x-latex				latex
> application/x-lha				lha
> application/x-lyx				lyx
> application/x-lzh				lzh
> application/x-lzx				lzx
> application/x-maker				frm
> application/x-mif				mif
> application/x-ms-wmd				wmd
> application/x-ms-wmz				wmz
> application/x-msdos-program			com
> application/x-msi				msi
> application/x-netcdf				nc
> application/x-ns-proxy-autoconfig		pac
> application/x-nwc				nwc
> application/x-object				o
> application/x-oz-application			oza
> application/x-pkcs7-certreqresp			p7r
> application/x-pkcs7-crl				crl
> application/x-python-code			pyc
> application/x-qgis				qgs
> application/x-quicktimeplayer			qtl
> application/x-redhat-package-manager		rpm
> application/x-ruby				rb
> application/x-sh				sh
> application/x-shar				shar
> application/x-shockwave-flash			swf
> application/x-silverlight			scr
> application/x-stuffit				sit
> application/x-sv4cpio				sv4cpio
> application/x-sv4crc				sv4crc
> application/x-tar				tar
> application/x-tcl				tcl
> application/x-tex-gf				gf
> application/x-tex-pk				pk
> application/x-texinfo				texinfo
> application/x-troff				t
> application/x-troff-man				man
> application/x-troff-me				me
> application/x-troff-ms				ms
> application/x-ustar				ustar
> application/x-wais-source			src
> application/x-wingz				wz
> application/x-x509-ca-cert			crt
> application/x-xcf				xcf
> application/x-xfig				fig
> application/x-xpinstall				xpi
> audio/amr					amr
> audio/amr-wb					awb
> audio/amr					amr
> audio/amr-wb					awb
> audio/annodex					axa
> audio/basic					au
> audio/flac					flac
> audio/midi					mid
> audio/mpeg					mpga
> audio/mpegurl					m3u
> audio/ogg					oga
> audio/prs.sid					sid
> audio/x-aiff					aif
> audio/x-gsm					gsm
> audio/x-mpegurl					m3u
> audio/x-ms-wma					wma
> audio/x-ms-wax					wax
> audio/x-pn-realaudio				ra
> audio/x-realaudio				ra
> audio/x-scpls					pls
> audio/x-sd2					sd2
> audio/x-wav					wav
> image/gif					gif
> image/ief					ief
> image/jpeg					jpeg
> image/pcx					pcx
> image/png					png
> image/svg+xml					svg
> image/tiff					tiff
> image/vnd.djvu					djvu
> image/vnd.wap.wbmp				wbmp
> image/x-canon-cr2				cr2
> image/x-canon-crw				crw
> image/x-cmu-raster				ras
> image/x-coreldraw				cdr
> image/x-coreldrawpattern			pat
> image/x-coreldrawtemplate			cdt
> image/x-corelphotopaint				cpt
> image/x-epson-erf				erf
> image/x-icon					ico
> image/x-jg					art
> image/x-jng					jng
> image/x-ms-bmp					bmp
> image/x-nikon-nef				nef
> image/x-olympus-orf				orf
> image/x-photoshop				psd
> image/x-portable-anymap				pnm
> image/x-portable-bitmap				pbm
> image/x-portable-graymap			pgm
> image/x-portable-pixmap				ppm
> image/x-rgb					rgb
> image/x-xbitmap					xbm
> image/x-xpixmap					xpm
> image/x-xwindowdump				xwd
> text/cache-manifest				manifest
> text/calendar					ics
> text/css					css
> text/csv					csv
> text/h323					323
> text/html					html
> text/iuls					uls
> text/mathml					mml
> text/plain					asc
> text/richtext					rtx
> text/scriptlet					sct
> text/texmacs					tm
> text/tab-separated-values			tsv
> text/vnd.sun.j2me.app-descriptor		jad
> text/vnd.wap.wml				wml
> text/vnd.wap.wmlscript				wmls
> text/x-bibtex					bib
> text/x-boo					boo
> text/x-c++hdr					h
> text/x-c++src					c
> text/x-chdr					h
> text/x-component				htc
> text/x-csh					csh
> text/x-csrc					c
> text/x-dsrc					d
> text/x-diff					diff
> text/x-haskell					hs
> text/x-java					java
> text/x-literate-haskell				lhs
> text/x-moc					moc
> text/x-pascal					p
> text/x-pcs-gcd					gcd
> text/x-perl					pl
> text/x-python					py
> text/x-scala					scala
> text/x-setext					etx
> text/x-sh					sh
> text/x-tcl					tcl
> text/x-tex					tex
> text/x-vcalendar				vcs
> text/x-vcard					vcf
> video/3gpp					3gp
> video/annodex					axv
> video/dl					dl
> video/dv					dif
> video/fli					fli
> video/gl					gl
> video/mpeg					mpeg
> video/mp4					mp4
> video/quicktime					qt
> video/ogg					ogv
> video/vnd.mpegurl				mxu
> video/x-flv					flv
> video/x-la-asf					lsf
> video/x-mng					mng
> video/x-ms-asf					asf
> video/x-ms-wm					wm
> video/x-ms-wmv					wmv
> video/x-ms-wmx					wmx
> video/x-ms-wvx					wvx
> video/x-msvideo					avi
> video/x-sgi-movie				movie
> video/x-matroska				mpv
> END
>     %ext_types = ();
>     foreach $line (split(/\n/, $raw_mime_types)) {
> 	my ($mime_type, $extension) = split(/\t+/, $line);
> 	# Beware.  "+" needs to be escaped in patterns.
> 	$mime_type =~ s/\+/\\\+/g;
> 	$mime_types{$mime_type} = $extension;
> 	my ($type, $sub_type) = split(/\//, $mime_type);
> 	# We check whether we already have a value for this extension.
> 	# We don't care if application is overwritten.
> 	if ((defined $ext_types{$extension}) &&
> 	    ($ext_types{$extension} ne $type) &&
> 	    ($ext_types{$extension} ne "application")) {
> 	    print "$extension already has $ext_types{$extension}.";
> 	    print "Should now become $type.  Fix mime.types?\n"
> 	}
> 	else {
> 	    $ext_types{$extension} = $type;
> 	}
>     }
> }
> 
> 
6302a6998,7001
> 	$Arg{prefer_dns} = 0; # JL: Prefer DNS names over IP addresses?
> 	$Arg{httplog_html} = 0; # JL: Should we create HTTPlog in HTML?
> 	$Arg{httplog_name} = "httplog.text"; # JL: Old default as variable
> 	$Arg{httplog_txt} = "httplog.txt";   # JL: New text format
6312a7012
> 				"d|preferdns" => \$opt_d, # JL: new option
6316a7017
> 				"n|names" => \$opt_n, # JL: new option
6365a7067,7069
> 	$Arg{prefer_dns} = 1 if $opt_d;
> 	$Arg{httplog_html} = 1 if $opt_n;
> 	$Arg{httplog_name} = "httplog.html" if $opt_n;
6382a7087,7088
> 	mkdir $Arg{output_dir};
> 
6521c7227
<         print "USAGE: chaosreader [-aehikqrvxAHIRTUXY] [-D dir] 
---
>         print "USAGE: chaosreader [-adehiknqrvxAEHIRTUXY] [-D dir]
6545c7251
<         print "Version 0.94, 01-May-2004
---
>         print "Version 0.95h, 11-Apr-2014
6547c7253
< USAGE: chaosreader [-aehikqrvxAHIRTUXY] [-D dir] 
---
> USAGE: chaosreader [-adehiknqrvxAHIRTUXY] [-D dir]
6560a7267
>    -d, --preferdns       # Show DNS names instead of IP addresses
6578a7286
>    -n, --names           # Include hostnames in hyperlinked HTTPlog (HTML)
6810,6816c7518
< 				-> offset
< 				-> pos
< 				-> hextext
< 				-> hexhtml
< 				-> viewtext
< 				-> viewhtml
< 
---
> 				-> [ $from_server, data ]
6830,6833d7531
< 
< 
< 
< 
