Add config file instead of arguments for everything..
This commit is contained in:
@ -1,14 +1,17 @@
|
||||
[DEFAULT]
|
||||
var_dir = '/var/cache/routerstats/'
|
||||
httpd_port = 8000
|
||||
socket_port = 9999
|
||||
#passwd_file =
|
||||
#rrd_file =
|
||||
passwd_file = /usr/local/www/routerstats/passwd.client
|
||||
rrd_file = /usr/local/www/routerstats/test.rrd
|
||||
var_dir = /var/cache/routerstats/
|
||||
port = 9999
|
||||
|
||||
[client]
|
||||
host = remoteserver
|
||||
|
||||
[socket_server]
|
||||
|
||||
|
||||
[httpd]
|
||||
port = 8000
|
||||
|
||||
[collector]
|
||||
logfile = /var/log/ulog/syslogemu.log
|
||||
|
||||
@ -5,6 +5,8 @@ import logging
|
||||
import time
|
||||
import os
|
||||
import sys
|
||||
import argparse
|
||||
import configparser
|
||||
|
||||
import rrdtool
|
||||
|
||||
@ -221,39 +223,7 @@ class UpdateRRD:
|
||||
else:
|
||||
logging.error('Not sure what to do here? ' + str(input_dict) + str(self.toupdate))
|
||||
|
||||
def main():
|
||||
try:
|
||||
client_host = sys.argv[1]
|
||||
except IndexError:
|
||||
logging.error('Need hostname/ip as first argument')
|
||||
sys.exit()
|
||||
try:
|
||||
client_port = int(sys.argv[2])
|
||||
except (ValueError, IndexError):
|
||||
logging.error('Need port as second argument')
|
||||
sys.exit()
|
||||
try:
|
||||
rrdfile = sys.argv[3]
|
||||
except IndexError:
|
||||
logging.error('Need rrdfile as third argument')
|
||||
sys.exit()
|
||||
|
||||
try:
|
||||
passwd_file = sys.argv[4]
|
||||
except IndexError:
|
||||
logging.error('Need passwd-file as fourth argument')
|
||||
sys.exit()
|
||||
|
||||
#Make sure the file specified is to be found..
|
||||
if not passwd_file.startswith('/'):
|
||||
passwd_file = os.path.abspath(passwd_file)
|
||||
|
||||
if not os.path.isfile(passwd_file):
|
||||
logging.error('Cannot find passwd-file %s', passwd_file)
|
||||
sys.exit()
|
||||
|
||||
rrdupdater = UpdateRRD(rrdfile)
|
||||
client = routerstats_client(client_host, client_port, passwd_file)
|
||||
def loop(client, rrdupdater):
|
||||
while True:
|
||||
try:
|
||||
client.connect()
|
||||
@ -297,5 +267,62 @@ def main():
|
||||
except KeyboardInterrupt:
|
||||
break
|
||||
|
||||
def main():
|
||||
config_section = 'client'
|
||||
|
||||
config = configparser.ConfigParser()
|
||||
parser = argparse.ArgumentParser(exit_on_error=False)
|
||||
|
||||
parser.add_argument('-c', '--config', help='config file to load')
|
||||
parser.add_argument('-d', '--debug', action='store_true', help='enable debug')
|
||||
|
||||
args, remaining_args = parser.parse_known_args()
|
||||
|
||||
if args.debug:
|
||||
logging.root.setLevel(logging.DEBUG)
|
||||
|
||||
logging.debug('Starting as PID ' + str(os.getpid()))
|
||||
|
||||
found = False
|
||||
if args.config:
|
||||
if os.path.isfile(args.config):
|
||||
config.read(args.config)
|
||||
found = True
|
||||
else:
|
||||
logging.error('Specified config file does not exist: %s', args.config)
|
||||
else:
|
||||
logging.debug('Trying to find config')
|
||||
#Try to find in "usual" places
|
||||
for directory in ('/etc/routerstats/', '/usr/local/etc/routerstats/', '/opt/routerstats/', './'):
|
||||
trytoread = directory + 'routerstats.config'
|
||||
if os.path.isfile(trytoread):
|
||||
logging.debug('Reading config file %s', trytoread)
|
||||
config.read(trytoread)
|
||||
found = True
|
||||
if not found:
|
||||
logging.error('No config file found')
|
||||
sys.exit(0)
|
||||
|
||||
|
||||
parser.add_argument('-h', '--host', dest='host', help='ip/hostname to connect to', default=config[config_section]['host'])
|
||||
parser.add_argument('-p', '--port', dest='port', type=int, help='port to connect to', default=config[config_section]['port'])
|
||||
parser.add_argument('-v', '--vardir', dest='vardir', help='directory for storing rrd file', default=config[config_section]['var_dir'])
|
||||
parser.add_argument('-w', '--pwdfile', dest='passwd_file', help='password file', default=config[config_section]['passwd_file'])
|
||||
|
||||
args = parser.parse_args()
|
||||
logging.debug(args)
|
||||
|
||||
#Make sure the file specified is to be found..
|
||||
if not os.path.isfile(args.passwd_file):
|
||||
logging.error('Cannot find passwd-file %s', args.passwd_file)
|
||||
sys.exit()
|
||||
|
||||
if not os.path.isdir(args.vardir):
|
||||
logging.error('Cannot find var dir %s', args.vardir)
|
||||
|
||||
rrdupdater = UpdateRRD(args.vardir + '/routerstats.rrd')
|
||||
client = routerstats_client(args.host, args.port, args.passwd_file)
|
||||
loop(client, rrdupdater)
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
@ -11,11 +11,11 @@ from datetime import datetime
|
||||
import logging
|
||||
import time
|
||||
import pickle
|
||||
import configparser
|
||||
import argparse
|
||||
|
||||
from setproctitle import setproctitle
|
||||
|
||||
VAR_DIR = '/var/cache/routerstats/'
|
||||
|
||||
logging.basicConfig(
|
||||
format='%(asctime)s %(funcName)20s %(levelname)-8s %(message)s',
|
||||
level=logging.INFO,
|
||||
@ -29,7 +29,7 @@ class ReloadLog(Exception):
|
||||
'''Used to reload log file'''
|
||||
pass
|
||||
|
||||
def filefetcher(filename: str, collector_queue, signal_queue, sleep_sec=0.5, seek_pos=None):
|
||||
def filefetcher(filename: str, output_directory: str, collector_queue, signal_queue, sleep_sec=0.5, seek_pos=None):
|
||||
'''Latch onto a file, putting any new lines onto the queue.'''
|
||||
setproctitle('routerstats-collector file-fetcher')
|
||||
if float(sleep_sec) <= 0.0:
|
||||
@ -74,7 +74,7 @@ def filefetcher(filename: str, collector_queue, signal_queue, sleep_sec=0.5, see
|
||||
#Using got_signal with a timeout of sleep_sec to rate limit the loopiness of this loop:)
|
||||
any_signal = got_signal(signal_queue, sleep_sec)
|
||||
if any_signal == 'Quit':
|
||||
shutdown_filefetcher(collector_queue, input_file)
|
||||
shutdown_filefetcher(collector_queue, input_file, output_directory)
|
||||
logging.critical('Shutdown filefetcher')
|
||||
return True
|
||||
now_stat = os.stat(filename)
|
||||
@ -95,7 +95,7 @@ def filefetcher(filename: str, collector_queue, signal_queue, sleep_sec=0.5, see
|
||||
seek_pos = 0
|
||||
break
|
||||
except KeyboardInterrupt:
|
||||
shutdown_filefetcher(collector_queue, input_file)
|
||||
shutdown_filefetcher(collector_queue, input_file, output_directory)
|
||||
logging.debug('KeyboardInterrupt, closing file and quitting')
|
||||
return False
|
||||
except FileNotFoundError:
|
||||
@ -103,14 +103,14 @@ def filefetcher(filename: str, collector_queue, signal_queue, sleep_sec=0.5, see
|
||||
logging.debug('File gone away')
|
||||
next
|
||||
|
||||
def shutdown_filefetcher(output_queue, input_file):
|
||||
def shutdown_filefetcher(output_queue, input_file, output_directory):
|
||||
'''Cleanly close filehandles, save log position and queue contents'''
|
||||
cur_pos = input_file.tell()
|
||||
input_file.close()
|
||||
with open(VAR_DIR + 'position', 'w') as output_file:
|
||||
with open(output_directory + 'position', 'w') as output_file:
|
||||
logging.debug('Saving current position ' + str(cur_pos))
|
||||
output_file.write(str(cur_pos))
|
||||
dump_queue(output_queue)
|
||||
dump_queue(output_queue, output_directory + 'dump.pickle')
|
||||
|
||||
def got_signal(signal_queue: Queue, sleep_sec: float):
|
||||
'''Read from signal_queue with a timeout of sleep_sec,
|
||||
@ -162,7 +162,7 @@ def parse_line(input_line: str) -> dict:
|
||||
logging.debug('Parsed line to ' + str(retval))
|
||||
return retval
|
||||
|
||||
def dump_queue(queue_to_dump: Queue):
|
||||
def dump_queue(queue_to_dump: Queue, dumpfile):
|
||||
'''Write the contents of a queue to a list that we pickle to a file'''
|
||||
#We use pickle, every entry in the queue is one entry in a list
|
||||
if queue_to_dump.empty():
|
||||
@ -176,9 +176,9 @@ def dump_queue(queue_to_dump: Queue):
|
||||
except queue.Empty:
|
||||
break
|
||||
if out_list:
|
||||
logging.debug('Saving ' + str(len(out_list)) + ' entries to dump.pickle')
|
||||
logging.debug('Saving ' + str(len(out_list)) + ' entries to ' + str(dumpfile))
|
||||
to_save = pickle.dumps(out_list)
|
||||
with open(VAR_DIR + 'dump.pickle', 'wb') as output_file:
|
||||
with open(dumpfile, 'wb') as output_file:
|
||||
bytes_written = output_file.write(to_save)
|
||||
logging.debug('Saved ' + str(len(out_list)) + ' entries, taking ' + str(bytes_written) + ' bytes')
|
||||
|
||||
@ -188,10 +188,9 @@ def signal_handler(signum, frame):
|
||||
logging.critical('Asked to quit')
|
||||
raise TimeToQuit('Received signal ' + signal.Signals(signum).name)
|
||||
|
||||
def load_pickled_file(output_queue):
|
||||
def load_pickled_file(output_queue, loadfile):
|
||||
'''Load queue contents from pickled queue structure'''
|
||||
#Does our dump file exist?
|
||||
loadfile = VAR_DIR + 'dump.pickle'
|
||||
if os.path.isfile(loadfile):
|
||||
size = os.stat(loadfile).st_size
|
||||
logging.debug(loadfile + ' exists, loading ' + str(size) + ' bytes.')
|
||||
@ -206,11 +205,11 @@ def load_pickled_file(output_queue):
|
||||
logging.debug('Deleting old dump')
|
||||
os.unlink(loadfile)
|
||||
|
||||
def load_start_pos(logfile):
|
||||
def load_start_pos(logfile, position_file):
|
||||
'''Read start position from file, if it exists'''
|
||||
#Do we have any position we want to start from?
|
||||
if os.path.isfile(VAR_DIR + 'position'):
|
||||
with open(VAR_DIR + 'position', 'r') as input_file:
|
||||
if os.path.isfile(position_file):
|
||||
with open(position_file, 'r') as input_file:
|
||||
tmp_start_pos = input_file.readline()
|
||||
try:
|
||||
tmp_start_pos = int(tmp_start_pos)
|
||||
@ -367,11 +366,11 @@ class RequestHandler(socketserver.BaseRequestHandler):
|
||||
logging.error('Peer gone?: ' + str(error))
|
||||
return False
|
||||
|
||||
def socket_server(file_parser_result_queue, overflowqueue, socket_server_signal_queue, passwd_file):
|
||||
def socket_server(file_parser_result_queue, overflowqueue, socket_server_signal_queue, passwd_file, server_port):
|
||||
'''Socket server sending whatever data is in the queue to any client connecting'''
|
||||
#Multiple connections here is probably a horrible idea:)
|
||||
setproctitle('routerstats-collector socket_server')
|
||||
host, port = '', 9999
|
||||
host, port = '', server_port
|
||||
while True:
|
||||
try:
|
||||
socketserver.TCPServer.allow_reuse_address = True
|
||||
@ -415,30 +414,71 @@ def socket_server(file_parser_result_queue, overflowqueue, socket_server_signal_
|
||||
|
||||
def main():
|
||||
'''Main thingy'''
|
||||
config_section = 'collector'
|
||||
setproctitle('routerstats-collector main-thread')
|
||||
|
||||
passwd_file = None
|
||||
file_to_follow = None
|
||||
server_port = None
|
||||
var_dir = None
|
||||
|
||||
config = configparser.ConfigParser()
|
||||
|
||||
#parser = argparse.ArgumentParser(exit_on_error=False, prog='routerstats_collector', description='Collecting information from logfile and sending to routerstats_client')
|
||||
parser = argparse.ArgumentParser(exit_on_error=False)
|
||||
parser.add_argument('-c', '--config', help='config file to load')
|
||||
parser.add_argument('-d', '--debug', action='store_true', help='enable debug')
|
||||
|
||||
args, remaining_args = parser.parse_known_args()
|
||||
|
||||
if args.debug:
|
||||
logging.root.setLevel(logging.DEBUG)
|
||||
|
||||
logging.debug('Starting as PID ' + str(os.getpid()))
|
||||
try:
|
||||
file_to_follow = sys.argv[1]
|
||||
except IndexError:
|
||||
logging.error('Need a log file to tail')
|
||||
sys.exit()
|
||||
|
||||
found = False
|
||||
if args.config:
|
||||
if os.path.isfile(args.config):
|
||||
config.read(args.config)
|
||||
found = True
|
||||
else:
|
||||
logging.error('Specified config file does not exist: %s', args.config)
|
||||
else:
|
||||
logging.debug('Trying to find config')
|
||||
#Try to find in "usual" places
|
||||
for directory in ('/etc/routerstats/', '/usr/local/etc/routerstats/', '/opt/routerstats/', './'):
|
||||
trytoread = directory + 'routerstats.config'
|
||||
if os.path.isfile(trytoread):
|
||||
logging.debug('Reading config file %s', trytoread)
|
||||
config.read(trytoread)
|
||||
found = True
|
||||
if not found:
|
||||
logging.error('No config file found')
|
||||
sys.exit(0)
|
||||
|
||||
parser.add_argument('-f', '--file', dest='file_to_follow', help='Log file to follow', default=config[config_section]['logfile'])
|
||||
parser.add_argument('-w', '--pwdfile', dest='passwd_file', help='password file', default=config[config_section]['passwd_file'])
|
||||
parser.add_argument('-p', '--port', dest='server_port', type=int, help='tcp port to listen to', default=config[config_section]['port'])
|
||||
parser.add_argument('-v', '--vardir', dest='var_dir', help='Location for queue dumps', default=config[config_section]['var_dir'])
|
||||
|
||||
args = parser.parse_args()
|
||||
logging.debug(args)
|
||||
|
||||
#Just quit early if file is missing..
|
||||
if os.path.isfile(file_to_follow) is False:
|
||||
logging.error('Could not find file ' + file_to_follow)
|
||||
if os.path.isfile(args.file_to_follow) is False:
|
||||
logging.error('Could not find log file ' + args.file_to_follow)
|
||||
sys.exit()
|
||||
|
||||
try:
|
||||
passwd_file = sys.argv[2]
|
||||
except IndexError:
|
||||
logging.error('Need passwd-file as second argument')
|
||||
if not os.path.isfile(args.passwd_file):
|
||||
logging.error('Could not find password file %s', args.passwd_file)
|
||||
sys.exit()
|
||||
|
||||
if not passwd_file.startswith('/'):
|
||||
passwd_file = os.path.abspath(passwd_file)
|
||||
logging.debug('Setting passwd-file to %s', passwd_file)
|
||||
if not args.server_port:
|
||||
logging.error('No TPC port to bind to')
|
||||
sys.exit()
|
||||
|
||||
if not os.path.isfile(passwd_file):
|
||||
logging.error('Could not find file %s', passwd_file)
|
||||
if not os.path.isdir(args.var_dir):
|
||||
logging.error('Could not find var dir %s', args.var_dir)
|
||||
sys.exit()
|
||||
|
||||
file_parser_result_queue = Queue()
|
||||
@ -449,10 +489,10 @@ def main():
|
||||
started_processes = []
|
||||
dead_processes = 0
|
||||
|
||||
load_pickled_file(file_parser_result_queue)
|
||||
start_pos = load_start_pos(file_to_follow)
|
||||
load_pickled_file(file_parser_result_queue, args.var_dir + '/dump.pickle')
|
||||
start_pos = load_start_pos(args.file_to_follow, args.var_dir + '/position')
|
||||
|
||||
file_parser_process = Process(target=filefetcher, daemon=True, args=(file_to_follow, file_parser_result_queue, file_parser_signal_queue, 0.5, start_pos))
|
||||
file_parser_process = Process(target=filefetcher, daemon=True, args=(args.file_to_follow, args.var_dir, file_parser_result_queue, file_parser_signal_queue, 0.5, start_pos))
|
||||
file_parser_process.start()
|
||||
logging.debug('Started filefetcher as pid ' + str(file_parser_process.pid))
|
||||
started_processes.append((file_parser_process, file_parser_signal_queue))
|
||||
@ -464,7 +504,7 @@ def main():
|
||||
#This means any "malicious" connections will wipe the history
|
||||
#We're fine with this
|
||||
|
||||
socket_server_process = Process(target=socket_server, daemon=True, args=(file_parser_result_queue, overflowqueue, socket_server_signal_queue, passwd_file))
|
||||
socket_server_process = Process(target=socket_server, daemon=True, args=(file_parser_result_queue, overflowqueue, socket_server_signal_queue, args.passwd_file, args.server_port))
|
||||
socket_server_process.start()
|
||||
logging.debug('Socket server started as pid ' + str(socket_server_process.pid))
|
||||
started_processes.append((socket_server_process, socket_server_signal_queue))
|
||||
@ -489,7 +529,7 @@ def main():
|
||||
if dead_processes >= len(started_processes):
|
||||
logging.error('All processes has gone away :/')
|
||||
sys.exit()
|
||||
time.sleep(0.1)
|
||||
time.sleep(0.5)
|
||||
except (KeyboardInterrupt, TimeToQuit):
|
||||
for p in started_processes:
|
||||
if p[0].is_alive():
|
||||
|
||||
@ -2,7 +2,7 @@
|
||||
Description=Routerstats collector service
|
||||
|
||||
[Service]
|
||||
ExecStart=/usr/bin/env python3 /usr/local/bin/routerstats_collector.py /var/log/ulog/syslogemu.log /var/cache/routerstats/passwd.client
|
||||
ExecStart=/usr/bin/env python3 /usr/local/bin/routerstats_collector.py
|
||||
#So STDOUT and STDERR are not buffered
|
||||
Environment=PYTHONUNBUFFERED=1
|
||||
Restart=on-failure
|
||||
|
||||
@ -4,7 +4,10 @@ import logging
|
||||
from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer
|
||||
import io
|
||||
import sys
|
||||
import os
|
||||
from functools import partial
|
||||
import argparse
|
||||
import configparser
|
||||
|
||||
import rrdtool
|
||||
|
||||
@ -48,7 +51,7 @@ class RequestHandler(SimpleHTTPRequestHandler):
|
||||
else:
|
||||
title = 'monthly'
|
||||
startstr = '--start=end-1M'
|
||||
data = rrdtool.graphv("-", startstr, "--title=" + title, "DEF:in=" + self.rrdfilename + ":net_dnat:AVERAGE", "DEF:out=" + self.rrdfilename + ":loc-net:AVERAGE", "CDEF:result_in=in,UN,0,in,IF", "CDEF:tmp_out=out,UN,0,out,IF", "CDEF:result_out=tmp_out,-1,*", "AREA:result_in#00ff00:in", "AREA:result_out#0000ff:out")
|
||||
data = rrdtool.graphv("-", startstr, "--title=" + title, "DEF:in=" + self.rrdfile + ":net_dnat:AVERAGE", "DEF:out=" + self.rrdfile + ":loc-net:AVERAGE", "CDEF:result_in=in,UN,0,in,IF", "CDEF:tmp_out=out,UN,0,out,IF", "CDEF:result_out=tmp_out,-1,*", "AREA:result_in#00ff00:in", "AREA:result_out#0000ff:out")
|
||||
#, "--width", "1024", "--height", "600"
|
||||
self.wfile.write(data['image'])
|
||||
break
|
||||
@ -87,23 +90,65 @@ class RequestHandler(SimpleHTTPRequestHandler):
|
||||
|
||||
class CustomHandler(RequestHandler):
|
||||
'''wrapping the wrapped one, so we can wrap in an init'''
|
||||
def __init__(self, rrdfilename, *args, **kwargs):
|
||||
self.rrdfilename = rrdfilename
|
||||
def __init__(self, rrdfile, *args, **kwargs):
|
||||
self.rrdfile = rrdfile
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
|
||||
def server(rrdfilename="test.rrd"):
|
||||
server_address = ('', 8000)
|
||||
def server(rrdfile, port):
|
||||
server_address = ('', port)
|
||||
server_class = ThreadingHTTPServer
|
||||
handler_class = partial(CustomHandler, rrdfilename)
|
||||
handler_class = partial(CustomHandler, rrdfile)
|
||||
httpd = server_class(server_address, handler_class)
|
||||
httpd.serve_forever()
|
||||
|
||||
def main():
|
||||
if len(sys.argv) > 1:
|
||||
server(sys.argv[1])
|
||||
config_section = 'httpd'
|
||||
|
||||
config = configparser.ConfigParser()
|
||||
parser = argparse.ArgumentParser(exit_on_error=False)
|
||||
|
||||
parser.add_argument('-c', '--config', help='config file to load')
|
||||
parser.add_argument('-d', '--debug', action='store_true', help='enable debug')
|
||||
|
||||
args, remaining_args = parser.parse_known_args()
|
||||
|
||||
if args.debug:
|
||||
logging.root.setLevel(logging.DEBUG)
|
||||
|
||||
logging.debug('Starting as PID ' + str(os.getpid()))
|
||||
|
||||
found = False
|
||||
if args.config:
|
||||
if os.path.isfile(args.config):
|
||||
config.read(args.config)
|
||||
found = True
|
||||
else:
|
||||
logging.error('Specified config file does not exist: %s', args.config)
|
||||
else:
|
||||
server()
|
||||
logging.debug('Trying to find config')
|
||||
#Try to find in "usual" places
|
||||
for directory in ('/etc/routerstats/', '/usr/local/etc/routerstats/', '/opt/routerstats/', './'):
|
||||
trytoread = directory + 'routerstats.config'
|
||||
if os.path.isfile(trytoread):
|
||||
logging.debug('Reading config file %s', trytoread)
|
||||
config.read(trytoread)
|
||||
found = True
|
||||
if not found:
|
||||
logging.error('No config file found')
|
||||
sys.exit(0)
|
||||
|
||||
parser.add_argument('-v', '--vardir', help='directory storing rrd file', default=config[config_section]['var_dir'])
|
||||
parser.add_argument('-p', '--port', help='port to bind to', type=int, default=config[config_section]['port'])
|
||||
|
||||
args = parser.parse_args()
|
||||
logging.debug(args)
|
||||
|
||||
if not os.path.isfile(args.vardir + 'routerstats.rrd'):
|
||||
logging.error('Cannot find rrd file %s', args.vardir + 'routerstats.rrd' )
|
||||
sys.exit()
|
||||
|
||||
server(rrdfile=args.vardir + 'routerstats.rrd', port=args.port)
|
||||
|
||||
if __name__ == '__main__':
|
||||
logging.debug('Welcome')
|
||||
|
||||
Reference in New Issue
Block a user