diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0d20b64 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +*.pyc diff --git a/app.py b/app.py index 9feab7b..094a49e 100644 --- a/app.py +++ b/app.py @@ -1,33 +1,37 @@ #!/usr/bin/env python2 # -*- coding: utf-8 -*- # # This file is part of precise-tools # Copyright (C) 2017 Bryan Davis and contributors # # This program is free software: you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by the Free # Software Foundation, either version 3 of the License, or (at your option) # any later version. # # This program is distributed in the hope that it will be useful, but WITHOUT # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or # FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for # more details. # # You should have received a copy of the GNU General Public License along # with this program. If not, see . + import flask +from . import jobs + app = flask.Flask(__name__) @app.route('/') def home(): - return flask.render_template('home.html') + tools = set().union(jobs.tools_from_accounting(7), jobs.tools_from_grid()) + return flask.render_template('home.html', tools=tools) if __name__ == '__main__': app.run() # vim:sw=4:ts=4:sts=4:et: diff --git a/jobs.py b/jobs.py new file mode 100644 index 0000000..a93e204 --- /dev/null +++ b/jobs.py @@ -0,0 +1,73 @@ +# -*- coding: utf-8 -*- +# +# This file is part of precise-tools +# Copyright (C) 2017 Bryan Davis and contributors +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see . + +import http.client +import json + +from . import utils + +ACCOUNTING_FIELDS = [ + 'qname', 'hostname', 'group', 'owner', 'job_name', 'job_number', + 'account', 'priority', 'submission_time', 'start_time', 'end_time', + 'failed', 'exit_status', 'ru_wallclock', 'ru_utime', 'ru_stime', + 'ru_maxrss', 'ru_ixrss', 'ru_ismrss', 'ru_idrss', 'ru_isrss', 'ru_minflt', + 'ru_majflt', 'ru_nswap', 'ru_inblock', 'ru_oublock', 'ru_msgsnd', + 'ru_msgrcv', 'ru_nsignals', 'ru_nvcsw', 'ru_nivcsw', 'project', + 'department', 'granted_pe', 'slots', 'task_number', 'cpu', 'mem', 'io', + 'category', 'iow', 'pe_taskid', 'maxvemem', 'arid', 'ar_submission_time', +] + + +def tools_from_accounting(days): + delta = datetime.timedelta(days=days) + cutoff = (datetime.datetime.now() - delta).timestamp() + tools = [] + for line in utils.tail('/data/project/.system/accounting', 45000 * days): + parts = line.split(':') + job = dict(zip(ACCOUNTING_FIELDS, parts) + if int(job['end_time']) < cutoff: + continue + if 'release=precise' in job['category'] and job['owner'] not in tools: + tools.append(job['owner']) + return tools + + +def is_precise_host(hostname): + if hostname[-4:].startswith('12'): + return True + + +def tools_from_grid(): + tools = [] + conn = http.client.HTTPConnection('tools.wmflabs.org') + conn.request( + 'GET', '/gridengine-status', + headers={ + 'User-Agent': 'https://tools.wmflabs.org/precise-tools/' + } + ) + res = conn.getresponse().read() + if not res: + return [] + grid_info = json.loads(res)['data']['attributes'] + for host, info in grid_info.iteritems(): + if is_precise_host(host) + if info['jobs']: + tools.extend([ + job['job_owner'] for job in info['jobs'].values()]) + return tools diff --git a/templates/home.html b/templates/home.html index c88b55f..d6dea48 100644 --- a/templates/home.html +++ b/templates/home.html @@ -1,4 +1,9 @@ {% extends "layout.html" %} {% block body %} -

Hello world

+

Tools running jobs on Precise OGE hosts in last 7 days

+ {% endblock %} diff --git a/app.py b/utils.py similarity index 54% copy from app.py copy to utils.py index 9feab7b..dfc005e 100644 --- a/app.py +++ b/utils.py @@ -1,33 +1,40 @@ -#!/usr/bin/env python2 # -*- coding: utf-8 -*- # # This file is part of precise-tools # Copyright (C) 2017 Bryan Davis and contributors # # This program is free software: you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by the Free # Software Foundation, either version 3 of the License, or (at your option) # any later version. # # This program is distributed in the hope that it will be useful, but WITHOUT # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or # FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for # more details. # # You should have received a copy of the GNU General Public License along # with this program. If not, see . -import flask +import subprocess -app = flask.Flask(__name__) +def tail(filename, lines): + """Get last n lines from the filename as an iterator.""" + # Inspired by http://stackoverflow.com/a/4418193/8171 + cmd = ['tail', '-%d' % lines, filename] + proc = subprocess.Popen( + cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + while True: + line = proc.stdout.readline() + if line == '' and proc.poll() is not None: + break + yield line -@app.route('/') -def home(): - return flask.render_template('home.html') - - -if __name__ == '__main__': - app.run() - -# vim:sw=4:ts=4:sts=4:et: + if proc.returncode == 0: + raise StopIteration + else: + raise subprocess.CalledProcessError( + returncode=proc.returncode, + command=' '.join(cmd) + )