Swap from amazon to hetzner ip

bin/flush_reject

@@ -121,8 +121,7 @@ sub opnsense_api_delete {
 	$req->content('{"address":"'.$ip.'"}');
 	my $res		= $ua->request($req);
 	unless ($res->is_success) {
-		print $req->status_line."\n";
-		print $req->as_string."\n";
+		&log(2, 'Failed to request to delete '.$ip.' from opnsense.');
 	}
 }
 

bin/parserfilter-tester

@@ -4,16 +4,16 @@ use warnings;
 use Getopt::Long;
 use My::parser::stats;
 my $module;
-my @modules	= ('ssh','dovecot','exim','apache','gitea_ssh');
+my @modules	= ('ssh','dovecot','exim','apache','gitea_ssh', 'haproxy');
 my $program	= $0;
 GetOptions	("module=s"	=> \$module);	#Only test one module
 unless($module) {
-	print 'No module specified, use argument --module=(ssh|dovecot|exim|apache|gitea_ssh)'."\n";
+	print 'No module specified, use argument --module=(ssh|dovecot|exim|apache|gitea_ssh|haproxy)'."\n";
 	exit;
 }
 my @matches = grep { /$module/ } @modules;
 unless(@matches) {
-	print 'Unsupported module '.$module.' specified, use argument --module=(ssh|dovecot|exim|apache|gitea_ssh)'."\n";
+	print 'Unsupported module '.$module.' specified, use argument --module=(ssh|dovecot|exim|apache|gitea_ssh|haproxy)'."\n";
 	exit;
 }
 print "Please paste a line to parse here:\n";

conf/parserfilter.conf.sample

@@ -13,7 +13,7 @@
 # this is based on the normal syslog-in-db format
 
 logfile = '/var/log/parser_filter.log' #Our logfile
-modules = 'dovecot','exim','ssh','apache','gitea' #List of modules available
+modules = 'dovecot','exim','ssh','apache','gitea','haproxy' #List of modules available
 
 #One entry per module, beware, no stray spaces allowed:)
 dovecot = 'file','/usr/local/jails/thinjails/dovecot/var/log/maillog'
@@ -21,6 +21,7 @@ exim = 'file','/var/log/exim/mainlog'
 apache = 'file','/var/log/apache-error.log'
 ssh = 'db','syslog.logs','sshd'
 gitea = 'file','/usr/local/jails/thinjails/gitea/var/log/gitea/gitea.log'
+haproxy = 'file','/var/log/haproxy.log'
 #why we have the first entry here, I'm not too sure about, since the module itself also 
 #needs to know what type of fetcher it wants. Ancient code is ancient. Undocumented ideas are bad:)
 

lib/addtolist.pm

@@ -18,21 +18,22 @@ sub add {
 	my $self	= shift;
 	my $params	= shift;
 	die "No params" unless $params;
-	$self->{'dbh'}		= $self->{'config'}->get_dbh unless($self->{'dbh'});
+	my $dbh			= $self->{'config'}->db_connect();
 	my $return		= {};
-	$return			= $self->addtoblacklist($params)	if($params->{'list'} eq 'black');
-	$return			= $self->addtorejectlist($params)	if($params->{'list'} eq 'reject');
+	$return			= $self->addtoblacklist($dbh, $params)	if($params->{'list'} eq 'black');
+	$return			= $self->addtorejectlist($dbh, $params)	if($params->{'list'} eq 'reject');
 	return $return;
 }
 
 sub addtoblacklist {
 	my $self	= shift;
+	my $dbh		= shift;
 	my $work	= shift;
 	my $host	= $work->{'host'};
 	my $asn		= $work->{'asn'} || 0;
 	my $iso		= $work->{'iso'} || '';
 	return { retval => 0, retmsg => "No host specified" } unless($host);
-	my $sth = $self->{'dbh'}->prepare("
+	my $sth = $dbh->prepare("
 		SELECT COUNT(*)
 		FROM reject_iptables
 		WHERE ip = ? AND
@@ -40,7 +41,7 @@ sub addtoblacklist {
 	$sth->execute($host);
 	my $rows = $sth->fetchrow_arrayref->[0];
 	unless($rows) {
-		my $sth = $self->{'dbh'}->prepare("INSERT INTO reject_iptables(ip,asn,iso) VALUES(?,?,?)") || return { retval => 0, retmsg => 'Failed to prepare statement in addtoblacklist '.DBI::errstr };
+		my $sth = $dbh->prepare("INSERT INTO reject_iptables(ip,asn,iso) VALUES(?,?,?)") || return { retval => 0, retmsg => 'Failed to prepare statement in addtoblacklist '.DBI::errstr };
 		$sth->execute($host,$asn,$iso) || return { retval => 0, retmsg => 'Failed to execute statement in addtoblacklist '.DBI::errstr };
 		system "/sbin/pfctl -q -t $self->{'block_table'} -T add $host";
 		system "/sbin/pfctl -q -k $host";
@@ -53,13 +54,14 @@ sub addtoblacklist {
 
 sub addtorejectlist {
 	my $self	= shift;
+	my $dbh		= shift;
 	my $work	= shift;
 	my $host	= $work->{'host'};
 	my $service	= $work->{'service'} || '';
 	my $asn		= $work->{'asn'} || 0;
 	my $iso		= $work->{'iso'} || '';
 	return { retval => 0, retmsg => "No host specified" } unless($host);
-	my $sth = $self->{'dbh'}->prepare("INSERT INTO reject(ip,service,asn,iso) VALUES(?,?,?,?)") || return { retval => 0, retmsg => 'Failed to prepare statement in addtorejectlist '.DBI::errstr };
+	my $sth = $dbh->prepare("INSERT INTO reject(ip,service,asn,iso) VALUES(?,?,?,?)") || return { retval => 0, retmsg => 'Failed to prepare statement in addtorejectlist '.DBI::errstr };
 	$sth->execute($host,$service,$asn,$iso) || return { retval => 0, retmsg => 'Failed to executute statement in addtorejectlist '.DBI::errstr };
 	return { retval => 1, retmsg => "Added to reject list" };
 }
@@ -100,8 +102,7 @@ sub opnsense_api_add {
 
 	my $res		= $ua->request($req);
 	unless ($res->is_success) {
-		print $res->status_line."\n";
-		print $req->content."\n";
+		print 'Failed to request for '.$host.' to be added to opnsense.'."\n";
 	}
 }
 

lib/config.pm

@@ -22,6 +22,7 @@ sub new {
 	$self->parse if($self->load);
 	$self->{'config'}->{'test'}	= $test;
 	$self->{'config'}->{'parse'}	= $parse;
+	$self->{'db'}			= undef;
 	return $self;
 }
 
@@ -120,16 +121,38 @@ sub get_fetcher_module {
 	}	
 }
 
+sub db_connect {
+	my $self	= shift;
+	my $dbh		= $self->get_dbh();
+	if ($dbh) {
+		if ($dbh->ping()) {
+			return $self->get_dbh();
+			#We can ping, all is good
+		} else {
+			#No can ping, time to reconnect
+			return $self->{'db'}->connect();
+		}
+	} else {
+		#Never connected?
+		return $self->{'db'}->connect();
+	}
+	die "End of db_connect should never be reached";
+}
+
+sub set_db {
+	my $self	= shift;
+	$self->{'db'}	= shift;
+}
+
 sub get_dbh {
 	my $self	= shift;
-	my $dbh		= $self->{'config'}->{'fetchers'}->{'db'}->{'dbh'};
-	return $dbh;
+	return $self->{'db'}->{'dbh'};
 }
 
 sub set_dbh {
 	my $self	= shift;
 	my $dbh		= shift;
-	$self->{'config'}->{'fetchers'}->{'db'}->{'dbh'} = $dbh;
+	$self->{'db'}->{'dbh'} = $dbh;
 	return 1;
 }
 

lib/db.pm

@@ -2,7 +2,6 @@ package My::parser::db;
 use strict;
 use warnings;
 use DBI;
-#use Scalar::Util qw(weaken);
 
 sub new {
 	my $class	= shift;
@@ -18,9 +17,7 @@ sub new {
 
 sub init {
 	my $self	= shift;
-	my $parser	= shift; ## not needed for this module as the db-connection is for all modules
-	$self->connect || die "Could not connect to db";
-	return 1;	
+	return $self->{'config'}->get_dbh();
 }
 
 sub connect {
@@ -32,13 +29,19 @@ sub connect {
 	my $db		= $config->get_as_single_val('config','db');
 	my $dbh;
 	if($usr && $pwd && $host && $db) {
+		my $logstr = 'Connecting to DB...';
+		my $i = 0;
 		while(1) {
-			last if($dbh = DBI->connect("DBI:mysql:database=$db;host=$host",$usr,$pwd, { PrintError => 1, mysql_auto_reconnect=>1, AutoCommit => 1 }));
-			sleep(10);
+			last if($dbh = DBI->connect("DBI:mysql:database=$db;host=$host",$usr,$pwd, { PrintError => 1, mysql_auto_reconnect=>0, AutoCommit => 1 }));
+			sleep($i);
+			$i++;
+			unless ($i % 10) {
+				$self->{'config'}->{'logger'}->log($logstr.'timed out, retry #'.$i);
+			}
 		}
 		$config->set_dbh($dbh);
-		#$self->{'config'}->{'logger'}->log("Sucsessfully connected to db"); FIXME add debug to config?
-		return 1;
+		$self->{'config'}->{'logger'}->log($logstr.'done');
+		return $config->get_dbh();
 	} else {
 		$self->{'config'}->{'logger'}->log("Unable to connect to db, not enough parameters");
 		return 0;

lib/exim_parser.pm

@@ -48,9 +48,25 @@ sub parser {
 			m/(\ \[($re_host)\]\ )/gcix && do {
 				$host	= $2;
 			};
-	} elsif($string =~ m/\ rejected (AUTH|MAIL)\ /) {
+	} elsif($string =~ m/\ rejected (EHLO|HELO)\ /) {
 		$_		= $string;
-		$reply		= 'Rejected MAIL or AUTH';
+		$reply		= 'Rejected HELO/EHLO';
+		$hostile	= 1;
+		PARSER:
+			m/(\ \[($re_host)\](\:|\ ))/gcix && do {
+				$host	= $2;
+			};
+	} elsif($string =~ m/\ rejected AUTH\ /) {
+		$_		= $string;
+		$reply		= 'Rejected AUTH';
+		$hostile	= 1;
+		PARSER:
+			m/(\ \[($re_host)\]\ )/gcix && do {
+				$host	= $2;
+			};
+	} elsif($string =~ m/\ rejected MAIL\ /) {
+		$_		= $string;
+		$reply		= 'Rejected MAIL';
 		$hostile	= 1;
 		PARSER:
 			m/(\ \[($re_host)\]\ )/gcix && do {

lib/haproxy.pm

@@ -0,0 +1,45 @@
+package My::parser::haproxy;
+use strict;
+use warnings;
+use File::Tail 0.91;
+use My::parser::haproxy_parser;
+
+sub new {
+	my $class	= shift;
+	my $config	= shift;
+	my $self	= {};
+	bless ($self, $class);
+
+	$self->{'config'}	= $config;
+	$self->{'parser'}	= My::parser::haproxy_parser->new();
+	return $self;
+}
+
+sub parse {
+	my $self	= shift;
+	my @result;
+	while(my $string = $self->fetch) {
+		last unless($string);
+		if (my $line	= $self->{'parser'}->parser($string)) {
+			push(@result,$line);
+		}
+	}
+	return { retval => 0 } unless(scalar(@result)); # nothing to say, nothing to report
+	return { retval => 1, retmsg => 'Here comes the results', lines => \@result };
+}
+
+sub fetch {
+	my $self	= shift;
+	my $fetcher	= $self->{'config'}->get_fetcher('haproxy');
+	die "Fetcher for haproxy went away?" unless($fetcher);
+	my $line;
+	my ($nfound,$timeleft,@pending) = File::Tail::select(undef,undef,undef,1,$fetcher);
+	foreach (@pending) {
+		$line	= $_->read;
+		chomp($line);
+	}
+	return 0 unless($line);
+	return $line;
+}
+
+1;

lib/haproxy_parser.pm

@@ -0,0 +1,53 @@
+package My::parser::haproxy_parser;
+use strict;
+use warnings;
+
+sub new {
+	my $class	= shift;
+	my $self	= {};
+	bless ($self, $class);
+	return $self;
+}
+
+sub parser {
+	my $self	= shift;
+	my $string	= shift;
+	my ($reply,$hostile,$host) = ("No match for $string",0,'');
+	my $re_host	= qr/[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}/;
+	if($string =~ m/SSL handshake failure/) {
+		$_		= $string;
+		$reply		= 'SSL handshake failure';
+		$hostile	= 1;
+		PARSE:
+			m/(\ ($re_host):[0-9]{1,6})/gcix && do {
+				$host	= $2;
+			};
+	} elsif($string =~ m/https\/1: (Timeout|Connection closed) during SSL handshake/) {
+		$_              = $string;
+		$reply          = 'SSL handshake error';
+	} elsif($string =~ m/http(s\~|) http(s|)\/\<NOSRV\>/) {
+		if($string =~ m/-1\/-1\/-1\/-1\/[0-9]{1,20} (400|0) 0/) {
+			#This one seems like someone is doing something bad. Return code 400/0
+			$_		= $string;
+			$hostile	= 1;
+			$reply		= 'Bad request, return code 400/0';
+			PARSE:
+				m/(\ ($re_host):[0-9]{1,6})/gcix && do {
+					$host	= $2;
+				};
+		} else {
+			#Other requests of this type seems to be ... not too bad
+			$reply		= 'Not routed, but probs only random error codes'
+		}
+	} elsif($string =~ m/https\~ /) {
+		#Accepted as https, probably fine..
+		$reply		= 'Routed as https';
+	} elsif($string =~ m/stopped \(cumulated conns/) {
+		#Usual service restart info
+		$hostile	= 0;
+		$reply		= 'haproxy restart information'
+	}
+	return { retval => 1, retmsg => $reply, hostile => $hostile, host => $host, string => $string };
+}
+
+return 1;

lib/localcheck.pm

@@ -12,7 +12,8 @@ sub new {
 sub islocal {
 	my $self	= shift;
 	my $host	= shift;
-	my @local_nets	= ('127\.','10\.','192\.168\.','172\.((1[6-9])|(2[0-9])|(3[0-1]))\.','213\.236\.200\.(7|10)'); # array of regexes matching networks considered local and to be ignored
+	#SSL Labs - 64.41.200.0/24
+	my @local_nets	= ('127\.','10\.','192\.168\.','172\.((1[6-9])|(2[0-9])|(3[0-1]))\.','64\.41\.200\.', '213.236.200.10','65.109.7.147'); # array of regexes matching networks considered local and to be ignored
 	my $local	= 0;
 	foreach my $test(@local_nets) {
 		$local = 1 if($host =~ m/^$test/);

lib/parser.pm

@@ -6,6 +6,7 @@ use My::parser::config;
 use My::parser::geoip;
 use My::parser::localcheck;
 use My::parser::block;
+use My::parser::db;
 
 sub new {
 	my $class	= shift;
@@ -18,6 +19,8 @@ sub new {
 	$self->{'geoip'}		= My::parser::geoip->new($self->{'config'});
 	$self->{'localcheck'}		= My::parser::localcheck->new;
 	$self->{'block'}		= My::parser::block->new($self->{'config'});
+	$self->{'db'}			= My::parser::db->new($self->{'config'});
+	$self->{'config'}->set_db($self->{'db'});
 	return $self;
 }
 
@@ -105,7 +108,6 @@ sub dyninit {
 	my $parser		= shift;
 	if(my $initr = $self->init($fetcher_loaded,$parser)) {
 		$self->{'config'}->set('fetchers',$fetcher_needed,$fetcher_loaded);
-		#$self->{'config'}->{'logger'}->log("Dependency $fetcher_needed initialized"); FIXME add debug in config?
 		return 1;
 	}
 	return 0;

lib/ssh.pm

@@ -15,17 +15,17 @@ sub new {
 
 sub fetch {
 	my $self		= shift;
-	$self->{'dbh'}		= $self->{'config'}->get_dbh unless($self->{'dbh'});
+	my $dbh			= $self->{'config'}->db_connect();
 	my $seq			= $self->{'seq'} || 0;
 	my $retmsg;
 	my @toreturn;
 	unless($seq) {
-		my $seqsth	= $self->{'dbh'}->prepare("SELECT seq FROM logs WHERE program = 'sshd' ORDER BY seq DESC LIMIT 1") or return { retval => 0, retmsg => DBI::errstr, error => 1 };
-		$seqsth->execute or return { retval => 0, retmsg => DBI::errstr, error => 1};
-		$seq            = $seqsth->fetchrow_arrayref->[0] or return { retval => 0, retmsg => DBI::errstr, error => 1};
+		my $seqsth	= $dbh->prepare("SELECT seq FROM logs WHERE program = 'sshd' ORDER BY seq DESC LIMIT 1") or return { retval => 0, retmsg => DBI::errstr, error => 1 };
+		$seqsth->execute() or return { retval => 0, retmsg => DBI::errstr, error => 1};
+		$seq            = $seqsth->fetchrow_arrayref()->[0] or return { retval => 0, retmsg => DBI::errstr, error => 1};
 	}
-	my $sth = $self->{'dbh'}->prepare("SELECT msg,seq FROM logs WHERE program = 'sshd' AND seq > $seq") or return { retval => 1, retmsg => DBI::errstr, error => 1 };
-	$sth->execute or return { retval => 0, retmsg => DBI::errstr, error => 1};
+	my $sth = $dbh->prepare("SELECT msg,seq FROM logs WHERE program = 'sshd' AND seq > $seq") or return { retval => 1, retmsg => DBI::errstr, error => 1 };
+	$sth->execute() or return { retval => 0, retmsg => DBI::errstr, error => 1};
 	while(my $ref = $sth->fetchrow_hashref) {
 		my $string      = $$ref{'msg'};
 		$seq            = $$ref{'seq'};

lib/ssh_parser.pm

@@ -47,7 +47,7 @@ sub parser {
 			m/(from\ ($re_host))	/gcix && do {
 				$host	= $2;
 			};
-	} elsif($string =~ m/(Disconnecting|Received disconnect from|Disconnected from|Connection closed by|Connection reset by) (authenticating |invalid |)(user .* |)$re_host port [0-9]{1,6}(\: Too many authentication failures|)( \[preauth\]|)/) {
+	} elsif($string =~ m/(Disconnecting|Received disconnect from|Disconnected from|Connection closed by|Connection reset by) (invalid |)(user .* |)$re_host port [0-9]{1,6}(\: Too many authentication failures|)( \[preauth\]|)/) {
 		$_		= $string;
 		$reply		= 'Received disconnect';
 		$hostile	= 1;
@@ -183,6 +183,22 @@ sub parser {
 		$reply		= 'Reverse check failed';
 	} elsif($string =~ m/but this does not map back to the address/) {
 		$reply		= 'Reverse map failure';
+	} elsif($string =~ m/Timeout before authentication for connection from/) {
+		$_		= $string;
+		$hostile	= 1;
+		$reply		= 'Timeout before authentication';
+		PARSER:
+			m/from\ ($re_host)\ to\ ($re_host),\ pid/gcix && do {
+				$host	= $1;
+			};
+	} elsif($string =~ m/penalty\: exceeded LoginGraceTime/) {
+		$_		= $string;
+		$hostile	= 1;
+		$reply		= 'exceeded LoginGraceTime';
+		PARSER:
+			m/drop\ connection\ \#([0-9]{1,9})\ from\ \[($re_host)\]:([0-9]{1,9})\ on\ \[($re_host)\]:([0-9]{1,9})\ penalty:\ exceeded\ LoginGraceTime/gcix && do {
+				$host	= $2;
+			};
 	} else {
 		$reply	= 'No match for '.$string;
 	}

lib/stats.pm

@@ -18,8 +18,8 @@ sub new {
 
 sub recent_hostile {
 	my $self	= shift;
-	$self->{'dbh'}	= $self->{'config'}->get_dbh unless($self->{'dbh'});
-	my $sth		= $self->{'dbh'}->prepare("
+	my $dbh		= $self->{'config'}->db_connect();
+	my $sth		= $dbh->prepare("
 		SELECT COUNT(1)
 		FROM reject
 		WHERE time > DATE_SUB(NOW(), INTERVAL 5 MINUTE)")
@@ -48,7 +48,7 @@ sub checker {
 	#### Mapping between checks and column in db
 	my %values		= ('reject' => 'ip','blocks' => 'ip', 'iso' => 'iso', 'asn' => 'asn', 'block_iso' => 'iso', 'block_asn' => 'asn');
 	my %keys		= ('reject' => $host, 'blocks' => $host, 'iso' => $iso, 'block_iso' => $iso, 'asn' => $asn, 'block_asn' => $asn);
-	$self->{'dbh'}		= $self->{'config'}->get_dbh unless($self->{'dbh'});
+	my $dbh			= $self->{'config'}->db_connect();
 	foreach my $time(@times) {
 		foreach my $c(keys %checks) {
 			my $fromcheck = $self->check($keys{$c},$time,$values{$c},$checks{$c});
@@ -72,8 +72,9 @@ sub check {
 	my $retval;
 	my $rows;
 	my $msg;
+	my $dbh                 = $self->{'config'}->db_connect();
 	if($host && $allowed_time && $value && $table) {
-		my $sth		= $self->{'dbh'}->prepare("
+		my $sth		= $dbh->prepare("
 			SELECT COUNT(1)
 			FROM $table 
 			WHERE $value = ? AND