Add config file instead of arguments for everything..

routerstats.config

@@ -1,14 +1,17 @@
 [DEFAULT]
-var_dir = '/var/cache/routerstats/'
+passwd_file = /usr/local/www/routerstats/passwd.client
-httpd_port = 8000
+rrd_file = /usr/local/www/routerstats/test.rrd
-socket_port = 9999
+var_dir = /var/cache/routerstats/
-#passwd_file = 
+port = 9999
-#rrd_file = 
 
 [client]
+host = remoteserver
 
 [socket_server]
 
+
 [httpd]
+port = 8000
 
 [collector]
+logfile = /var/log/ulog/syslogemu.log

routerstats_client.py

@@ -5,6 +5,8 @@ import logging
 import time
 import os
 import sys
+import argparse
+import configparser
 
 import rrdtool
 
@@ -221,39 +223,7 @@ class UpdateRRD:
 		else:
 			logging.error('Not sure what to do here? ' + str(input_dict) + str(self.toupdate))
 
-def main():
+def loop(client, rrdupdater):
-	try:
-		client_host = sys.argv[1]
-	except IndexError:
-		logging.error('Need hostname/ip as first argument')
-		sys.exit()
-	try:
-		client_port = int(sys.argv[2])
-	except (ValueError, IndexError):
-		logging.error('Need port as second argument')
-		sys.exit()
-	try:
-		rrdfile = sys.argv[3]
-	except IndexError:
-		logging.error('Need rrdfile as third argument')
-		sys.exit()
-
-	try:
-		passwd_file = sys.argv[4]
-	except IndexError:
-		logging.error('Need passwd-file as fourth argument')
-		sys.exit()
-
-	#Make sure the file specified is to be found..
-	if not passwd_file.startswith('/'):
-		passwd_file = os.path.abspath(passwd_file)
-
-	if not os.path.isfile(passwd_file):
-		logging.error('Cannot find passwd-file %s', passwd_file)
-		sys.exit()
-
-	rrdupdater = UpdateRRD(rrdfile)
-	client = routerstats_client(client_host, client_port, passwd_file)
 	while True:
 		try:
 			client.connect()
@@ -297,5 +267,62 @@ def main():
 		except KeyboardInterrupt:
 			break
 
+def main():
+	config_section = 'client'
+
+	config = configparser.ConfigParser()
+	parser = argparse.ArgumentParser(exit_on_error=False)
+
+	parser.add_argument('-c', '--config', help='config file to load')
+	parser.add_argument('-d', '--debug', action='store_true', help='enable debug')
+
+	args, remaining_args = parser.parse_known_args()
+
+	if args.debug:
+		logging.root.setLevel(logging.DEBUG)
+
+	logging.debug('Starting as PID ' + str(os.getpid()))
+
+	found = False
+	if args.config:
+		if os.path.isfile(args.config):
+			config.read(args.config)
+			found = True
+		else:
+			logging.error('Specified config file does not exist: %s', args.config)
+	else:
+		logging.debug('Trying to find config')
+		#Try to find in "usual" places
+		for directory in ('/etc/routerstats/', '/usr/local/etc/routerstats/', '/opt/routerstats/', './'):
+			trytoread = directory + 'routerstats.config'
+			if os.path.isfile(trytoread):
+				logging.debug('Reading config file %s', trytoread)
+				config.read(trytoread)
+				found = True
+	if not found:
+		logging.error('No config file found')
+		sys.exit(0)
+
+
+	parser.add_argument('-h', '--host', dest='host', help='ip/hostname to connect to', default=config[config_section]['host'])
+	parser.add_argument('-p', '--port', dest='port', type=int, help='port to connect to', default=config[config_section]['port'])
+	parser.add_argument('-v', '--vardir', dest='vardir', help='directory for storing rrd file', default=config[config_section]['var_dir'])
+	parser.add_argument('-w', '--pwdfile', dest='passwd_file', help='password file', default=config[config_section]['passwd_file'])
+
+	args = parser.parse_args()
+	logging.debug(args)
+
+	#Make sure the file specified is to be found..
+	if not os.path.isfile(args.passwd_file):
+		logging.error('Cannot find passwd-file %s', args.passwd_file)
+		sys.exit()
+
+	if not os.path.isdir(args.vardir):
+		logging.error('Cannot find var dir %s', args.vardir)
+
+	rrdupdater = UpdateRRD(args.vardir + '/routerstats.rrd')
+	client = routerstats_client(args.host, args.port, args.passwd_file)
+	loop(client, rrdupdater)
+
 if __name__ == '__main__':
 	main()

routerstats_collector.py

@@ -11,11 +11,11 @@ from datetime import datetime
 import logging
 import time
 import pickle
+import configparser
+import argparse
 
 from setproctitle import setproctitle
 
-VAR_DIR = '/var/cache/routerstats/'
-
 logging.basicConfig(
     format='%(asctime)s %(funcName)20s %(levelname)-8s %(message)s',
     level=logging.INFO,
@@ -29,7 +29,7 @@ class ReloadLog(Exception):
 	'''Used to reload log file'''
 	pass
 
-def filefetcher(filename: str, collector_queue, signal_queue, sleep_sec=0.5, seek_pos=None):
+def filefetcher(filename: str, output_directory: str, collector_queue, signal_queue, sleep_sec=0.5, seek_pos=None):
 	'''Latch onto a file, putting any new lines onto the queue.'''
 	setproctitle('routerstats-collector file-fetcher')
 	if float(sleep_sec) <= 0.0:
@@ -74,7 +74,7 @@ def filefetcher(filename: str, collector_queue, signal_queue, sleep_sec=0.5, see
 					#Using got_signal with a timeout of sleep_sec to rate limit the loopiness of this loop:)
 					any_signal = got_signal(signal_queue, sleep_sec)
 					if any_signal == 'Quit':
-						shutdown_filefetcher(collector_queue, input_file)
+						shutdown_filefetcher(collector_queue, input_file, output_directory)
 						logging.critical('Shutdown filefetcher')
 						return True
 					now_stat = os.stat(filename)
@@ -95,7 +95,7 @@ def filefetcher(filename: str, collector_queue, signal_queue, sleep_sec=0.5, see
 						seek_pos = 0
 						break
 		except KeyboardInterrupt:
-			shutdown_filefetcher(collector_queue, input_file)
+			shutdown_filefetcher(collector_queue, input_file, output_directory)
 			logging.debug('KeyboardInterrupt, closing file and quitting')
 			return False
 		except FileNotFoundError:
@@ -103,14 +103,14 @@ def filefetcher(filename: str, collector_queue, signal_queue, sleep_sec=0.5, see
 			logging.debug('File gone away')
 			next
 
-def shutdown_filefetcher(output_queue, input_file):
+def shutdown_filefetcher(output_queue, input_file, output_directory):
 	'''Cleanly close filehandles, save log position and queue contents'''
 	cur_pos = input_file.tell()
 	input_file.close()
-	with open(VAR_DIR + 'position', 'w') as output_file:
+	with open(output_directory + 'position', 'w') as output_file:
 		logging.debug('Saving current position ' + str(cur_pos))
 		output_file.write(str(cur_pos))
-	dump_queue(output_queue)
+	dump_queue(output_queue, output_directory + 'dump.pickle')
 
 def got_signal(signal_queue: Queue, sleep_sec: float):
 	'''Read from signal_queue with a timeout of sleep_sec,
@@ -162,7 +162,7 @@ def parse_line(input_line: str) -> dict:
 	logging.debug('Parsed line to ' + str(retval))
 	return retval
 
-def dump_queue(queue_to_dump: Queue):
+def dump_queue(queue_to_dump: Queue, dumpfile):
 	'''Write the contents of a queue to a list that we pickle to a file'''
 	#We use pickle, every entry in the queue is one entry in a list
 	if queue_to_dump.empty():
@@ -176,9 +176,9 @@ def dump_queue(queue_to_dump: Queue):
 		except queue.Empty:
 			break
 	if out_list:
-		logging.debug('Saving ' + str(len(out_list)) + ' entries to dump.pickle')
+		logging.debug('Saving ' + str(len(out_list)) + ' entries to ' + str(dumpfile))
 		to_save = pickle.dumps(out_list)
-		with open(VAR_DIR + 'dump.pickle', 'wb') as output_file:
+		with open(dumpfile, 'wb') as output_file:
 			bytes_written = output_file.write(to_save)
 		logging.debug('Saved ' + str(len(out_list)) + ' entries, taking ' + str(bytes_written) + ' bytes')
 
@@ -188,10 +188,9 @@ def signal_handler(signum, frame):
 		logging.critical('Asked to quit')
 		raise TimeToQuit('Received signal ' + signal.Signals(signum).name)
 
-def load_pickled_file(output_queue):
+def load_pickled_file(output_queue, loadfile):
 	'''Load queue contents from pickled queue structure'''
 	#Does our dump file exist?
-	loadfile = VAR_DIR + 'dump.pickle'
 	if os.path.isfile(loadfile):
 		size = os.stat(loadfile).st_size
 		logging.debug(loadfile + ' exists, loading ' + str(size) + ' bytes.')
@@ -206,11 +205,11 @@ def load_pickled_file(output_queue):
 		logging.debug('Deleting old dump')
 		os.unlink(loadfile)
 
-def load_start_pos(logfile):
+def load_start_pos(logfile, position_file):
 	'''Read start position from file, if it exists'''
 	#Do we have any position we want to start from?
-	if os.path.isfile(VAR_DIR + 'position'):
+	if os.path.isfile(position_file):
-		with open(VAR_DIR + 'position', 'r') as input_file:
+		with open(position_file, 'r') as input_file:
 			tmp_start_pos = input_file.readline()
 			try:
 				tmp_start_pos = int(tmp_start_pos)
@@ -367,11 +366,11 @@ class RequestHandler(socketserver.BaseRequestHandler):
 				logging.error('Peer gone?: ' + str(error))
 			return False
 
-def socket_server(file_parser_result_queue, overflowqueue, socket_server_signal_queue, passwd_file):
+def socket_server(file_parser_result_queue, overflowqueue, socket_server_signal_queue, passwd_file, server_port):
 	'''Socket server sending whatever data is in the queue to any client connecting'''
 	#Multiple connections here is probably a horrible idea:)
 	setproctitle('routerstats-collector socket_server')
-	host, port = '', 9999
+	host, port = '', server_port
 	while True:
 		try:
 			socketserver.TCPServer.allow_reuse_address = True
@@ -415,30 +414,71 @@ def socket_server(file_parser_result_queue, overflowqueue, socket_server_signal_
 
 def main():
 	'''Main thingy'''
+	config_section = 'collector'
 	setproctitle('routerstats-collector main-thread')
+
+	passwd_file = None
+	file_to_follow = None
+	server_port = None
+	var_dir = None
+
+	config = configparser.ConfigParser()
+
+	#parser = argparse.ArgumentParser(exit_on_error=False, prog='routerstats_collector', description='Collecting information from logfile and sending to routerstats_client')
+	parser = argparse.ArgumentParser(exit_on_error=False)
+	parser.add_argument('-c', '--config', help='config file to load')
+	parser.add_argument('-d', '--debug', action='store_true', help='enable debug')
+
+	args, remaining_args = parser.parse_known_args()
+
+	if args.debug:
+		logging.root.setLevel(logging.DEBUG)
+
 	logging.debug('Starting as PID ' + str(os.getpid()))
-	try:
+
-		file_to_follow = sys.argv[1]
+	found = False
-	except IndexError:
+	if args.config:
-		logging.error('Need a log file to tail')
+		if os.path.isfile(args.config):
-		sys.exit()
+			config.read(args.config)
+			found = True
+		else:
+			logging.error('Specified config file does not exist: %s', args.config)
+	else:
+		logging.debug('Trying to find config')
+		#Try to find in "usual" places
+		for directory in ('/etc/routerstats/', '/usr/local/etc/routerstats/', '/opt/routerstats/', './'):
+			trytoread = directory + 'routerstats.config'
+			if os.path.isfile(trytoread):
+				logging.debug('Reading config file %s', trytoread)
+				config.read(trytoread)
+				found = True
+	if not found:
+		logging.error('No config file found')
+		sys.exit(0)
+
+	parser.add_argument('-f', '--file', dest='file_to_follow', help='Log file to follow', default=config[config_section]['logfile'])
+	parser.add_argument('-w', '--pwdfile', dest='passwd_file', help='password file', default=config[config_section]['passwd_file'])
+	parser.add_argument('-p', '--port', dest='server_port', type=int, help='tcp port to listen to', default=config[config_section]['port'])
+	parser.add_argument('-v', '--vardir', dest='var_dir', help='Location for queue dumps', default=config[config_section]['var_dir'])
+
+	args = parser.parse_args()
+	logging.debug(args)
+
 	#Just quit early if file is missing..
-	if os.path.isfile(file_to_follow) is False:
+	if os.path.isfile(args.file_to_follow) is False:
-		logging.error('Could not find file ' + file_to_follow)
+		logging.error('Could not find log file ' + args.file_to_follow)
 		sys.exit()
 
-	try:
+	if not os.path.isfile(args.passwd_file):
-		passwd_file = sys.argv[2]
+		logging.error('Could not find password file %s', args.passwd_file)
-	except IndexError:
-		logging.error('Need passwd-file as second argument')
 		sys.exit()
 
-	if not passwd_file.startswith('/'):
+	if not args.server_port:
-		passwd_file = os.path.abspath(passwd_file)
+		logging.error('No TPC port to bind to')
-		logging.debug('Setting passwd-file to %s', passwd_file)
+		sys.exit()
 
-	if not os.path.isfile(passwd_file):
+	if not os.path.isdir(args.var_dir):
-		logging.error('Could not find file %s', passwd_file)
+		logging.error('Could not find var dir %s', args.var_dir)
 		sys.exit()
 
 	file_parser_result_queue = Queue()
@@ -449,10 +489,10 @@ def main():
 	started_processes = []
 	dead_processes = 0
 
-	load_pickled_file(file_parser_result_queue)
+	load_pickled_file(file_parser_result_queue, args.var_dir + '/dump.pickle')
-	start_pos = load_start_pos(file_to_follow)
+	start_pos = load_start_pos(args.file_to_follow, args.var_dir + '/position')
 
-	file_parser_process = Process(target=filefetcher, daemon=True, args=(file_to_follow, file_parser_result_queue, file_parser_signal_queue, 0.5, start_pos))
+	file_parser_process = Process(target=filefetcher, daemon=True, args=(args.file_to_follow, args.var_dir, file_parser_result_queue, file_parser_signal_queue, 0.5, start_pos))
 	file_parser_process.start()
 	logging.debug('Started filefetcher as pid ' + str(file_parser_process.pid))
 	started_processes.append((file_parser_process, file_parser_signal_queue))
@@ -464,7 +504,7 @@ def main():
 	#This means any "malicious" connections will wipe the history
 	#We're fine with this
 
-	socket_server_process = Process(target=socket_server, daemon=True, args=(file_parser_result_queue, overflowqueue, socket_server_signal_queue, passwd_file))
+	socket_server_process = Process(target=socket_server, daemon=True, args=(file_parser_result_queue, overflowqueue, socket_server_signal_queue, args.passwd_file, args.server_port))
 	socket_server_process.start()
 	logging.debug('Socket server started as pid ' + str(socket_server_process.pid))
 	started_processes.append((socket_server_process, socket_server_signal_queue))
@@ -489,7 +529,7 @@ def main():
 					if dead_processes >= len(started_processes):
 						logging.error('All processes has gone away :/')
 						sys.exit()
-			time.sleep(0.1)
+			time.sleep(0.5)
 		except (KeyboardInterrupt, TimeToQuit):
 			for p in started_processes:
 				if p[0].is_alive():

routerstats_collector.service

@@ -2,7 +2,7 @@
 Description=Routerstats collector service
 
 [Service]
-ExecStart=/usr/bin/env python3 /usr/local/bin/routerstats_collector.py /var/log/ulog/syslogemu.log /var/cache/routerstats/passwd.client
+ExecStart=/usr/bin/env python3 /usr/local/bin/routerstats_collector.py
 #So STDOUT and STDERR are not buffered
 Environment=PYTHONUNBUFFERED=1
 Restart=on-failure

routerstats_httpd.py

@@ -4,7 +4,10 @@ import logging
 from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer
 import io
 import sys
+import os
 from functools import partial
+import argparse
+import configparser
 
 import rrdtool
 
@@ -48,7 +51,7 @@ class RequestHandler(SimpleHTTPRequestHandler):
 				else:
 					title = 'monthly'
 					startstr = '--start=end-1M'
-				data = rrdtool.graphv("-", startstr, "--title=" + title, "DEF:in=" + self.rrdfilename + ":net_dnat:AVERAGE", "DEF:out=" + self.rrdfilename + ":loc-net:AVERAGE", "CDEF:result_in=in,UN,0,in,IF", "CDEF:tmp_out=out,UN,0,out,IF", "CDEF:result_out=tmp_out,-1,*", "AREA:result_in#00ff00:in", "AREA:result_out#0000ff:out")
+				data = rrdtool.graphv("-", startstr, "--title=" + title, "DEF:in=" + self.rrdfile + ":net_dnat:AVERAGE", "DEF:out=" + self.rrdfile + ":loc-net:AVERAGE", "CDEF:result_in=in,UN,0,in,IF", "CDEF:tmp_out=out,UN,0,out,IF", "CDEF:result_out=tmp_out,-1,*", "AREA:result_in#00ff00:in", "AREA:result_out#0000ff:out")
 				#, "--width", "1024", "--height", "600"
 				self.wfile.write(data['image'])
 				break
@@ -87,23 +90,65 @@ class RequestHandler(SimpleHTTPRequestHandler):
 
 class CustomHandler(RequestHandler):
 	'''wrapping the wrapped one, so we can wrap in an init'''
-	def __init__(self, rrdfilename, *args, **kwargs):
+	def __init__(self, rrdfile, *args, **kwargs):
-		self.rrdfilename = rrdfilename
+		self.rrdfile = rrdfile
 		super().__init__(*args, **kwargs)
 
 
-def server(rrdfilename="test.rrd"):
+def server(rrdfile, port):
-	server_address = ('', 8000)
+	server_address = ('', port)
 	server_class = ThreadingHTTPServer
-	handler_class = partial(CustomHandler, rrdfilename)
+	handler_class = partial(CustomHandler, rrdfile)
 	httpd = server_class(server_address, handler_class)
 	httpd.serve_forever()
 
 def main():
-	if len(sys.argv) > 1:
+	config_section = 'httpd'
-		server(sys.argv[1])
+
+	config = configparser.ConfigParser()
+	parser = argparse.ArgumentParser(exit_on_error=False)
+
+	parser.add_argument('-c', '--config', help='config file to load')
+	parser.add_argument('-d', '--debug', action='store_true', help='enable debug')
+
+	args, remaining_args = parser.parse_known_args()
+
+	if args.debug:
+		logging.root.setLevel(logging.DEBUG)
+
+	logging.debug('Starting as PID ' + str(os.getpid()))
+
+	found = False
+	if args.config:
+		if os.path.isfile(args.config):
+			config.read(args.config)
+			found = True
 		else:
-		server()
+			logging.error('Specified config file does not exist: %s', args.config)
+	else:
+		logging.debug('Trying to find config')
+		#Try to find in "usual" places
+		for directory in ('/etc/routerstats/', '/usr/local/etc/routerstats/', '/opt/routerstats/', './'):
+			trytoread = directory + 'routerstats.config'
+			if os.path.isfile(trytoread):
+				logging.debug('Reading config file %s', trytoread)
+				config.read(trytoread)
+				found = True
+	if not found:
+		logging.error('No config file found')
+		sys.exit(0)
+
+	parser.add_argument('-v', '--vardir', help='directory storing rrd file', default=config[config_section]['var_dir'])
+	parser.add_argument('-p', '--port', help='port to bind to', type=int, default=config[config_section]['port'])
+
+	args = parser.parse_args()
+	logging.debug(args)
+
+	if not os.path.isfile(args.vardir + 'routerstats.rrd'):
+		logging.error('Cannot find rrd file %s', args.vardir + 'routerstats.rrd' )
+		sys.exit()
+
+	server(rrdfile=args.vardir + 'routerstats.rrd', port=args.port)
 
 if __name__ == '__main__':
 	logging.debug('Welcome')