diff --git a/app.py b/app.py index d858a0e..9a809df 100644 --- a/app.py +++ b/app.py @@ -1,305 +1,319 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- # # This file is part of the Keystone browser # # Copyright (c) 2017 Bryan Davis and contributors # # This program is free software: you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by the Free # Software Foundation, either version 3 of the License, or (at your option) # any later version. # # This program is distributed in the hope that it will be useful, but WITHOUT # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or # FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for # more details. # # You should have received a copy of the GNU General Public License along # with this program. If not, see . import flask import werkzeug.middleware.proxy_fix from keystone_browser import zones from keystone_browser import glance from keystone_browser import keystone from keystone_browser import ldap from keystone_browser import nova from keystone_browser import puppetclasses from keystone_browser import proxies from keystone_browser import stats from keystone_browser import utils from keystone_browser import cinder from keystone_browser import neutron app = flask.Flask(__name__) app.wsgi_app = werkzeug.middleware.proxy_fix.ProxyFix(app.wsgi_app) @app.route("/") def home(): ctx = {} try: cached = "purge" not in flask.request.args ctx.update( { "usage": stats.usage(cached), } ) except Exception: app.logger.exception("Error collecting information for projects") return flask.render_template("home.html", **ctx) @app.route("/project/") def projects(): ctx = {} try: + cached = "purge" not in flask.request.args ctx.update( { - "projects": keystone.all_projects(), + "projects": keystone.all_projects(cached), } ) except Exception: app.logger.exception("Error collecting information for projects") return flask.render_template("projects.html", **ctx) @app.route("/server/") def servers(): ctx = {} try: + cached = "purge" not in flask.request.args ctx.update( { - "servers": nova.all_servers(), + "servers": nova.all_servers(cached), } ) except Exception: app.logger.exception("Error collecting information for projects") return flask.render_template("servers.html", **ctx) @app.route("/project/") def project(name): cached = "purge" not in flask.request.args ctx = { "project": name, } try: users = keystone.project_users_by_role(name) admins = users["admin"] + users["projectadmin"] ctx.update( { "project": name, - "admins": ldap.get_users_by_uid(admins), - "users": ldap.get_users_by_uid(users["user"]), - "servers": nova.project_servers(name), - "flavors": nova.flavors(name), - "images": glance.images(), + "admins": ldap.get_users_by_uid(admins, cached), + "users": ldap.get_users_by_uid(users["user"], cached), + "servers": nova.project_servers(name, cached), + "flavors": nova.flavors(name, cached), + "images": glance.images(cached), "proxies": proxies.project_proxies(name, cached), "zones": zones.all_a_records(name, cached), - "limits": nova.limits(name), + "limits": nova.limits(name, cached), "volumes": cinder.project_volumes(name, cached), - "cinder_limits": cinder.limits(name), - "neutron_limits": neutron.limits(name), + "cinder_limits": cinder.limits(name, cached), + "neutron_limits": neutron.limits(name, cached), } ) except Exception: app.logger.exception( 'Error collecting information for project "%s"', name ) return flask.render_template("project.html", **ctx) @app.route("/user/") def user(uid): ctx = { "uid": uid, } try: + cached = "purge" not in flask.request.args ctx.update( { - "user": ldap.get_users_by_uid([uid]), - "projects": keystone.projects_for_user(uid), + "user": ldap.get_users_by_uid([uid], cached), + "projects": keystone.projects_for_user(uid, cached), } ) if ctx["user"]: ctx["user"] = ctx["user"][0] except Exception: app.logger.exception('Error collecting information for user "%s"', uid) return flask.render_template("user.html", **ctx) @app.route("/server/") def server(fqdn): name, project, tld = fqdn.split(".", 2) ctx = { "fqdn": fqdn, "project": project, } try: + cached = "purge" not in flask.request.args ctx.update( { - "server": nova.server(fqdn), - "flavors": nova.flavors(project), - "images": glance.images(), - "puppetclasses": puppetclasses.classes(project, fqdn), - "hiera": puppetclasses.hiera(project, fqdn), + "server": nova.server(fqdn, cached), + "flavors": nova.flavors(project, cached), + "images": glance.images(cached), + "puppetclasses": puppetclasses.classes(project, fqdn, cached), + "hiera": puppetclasses.hiera(project, fqdn, cached), } ) if "user_id" in ctx["server"]: - user = ldap.get_users_by_uid([ctx["server"]["user_id"]]) + user = ldap.get_users_by_uid([ctx["server"]["user_id"]], cached) if user: ctx["owner"] = user[0] except Exception: app.logger.exception( 'Error collecting information for server "%s"', fqdn ) return flask.render_template("server.html", **ctx) @app.route("/puppetclass/") def all_puppetclasses(): ctx = {} try: - ctx.update({"puppetclasses": puppetclasses.all_classes()}) + cached = "purge" not in flask.request.args + ctx.update({"puppetclasses": puppetclasses.all_classes(cached)}) except Exception: app.logger.exception("Error collecting the list of puppet classes") return flask.render_template("puppetclasses.html", **ctx) @app.route("/puppetclass/") def puppetclass(classname): ctx = { "puppetclass": classname, } try: - ctx.update({"data": puppetclasses.prefixes(classname)}) + cached = "purge" not in flask.request.args + ctx.update({"data": puppetclasses.prefixes(classname, cached)}) except Exception: app.logger.exception( 'Error collecting information for puppet class "%s"', classname ) return flask.render_template("puppetclass.html", **ctx) @app.route("/hierakey/") def hierakey(hierakey): ctx = { "hierakey": hierakey, } try: - ctx.update({"data": puppetclasses.hieraprefixes(hierakey)}) + cached = "purge" not in flask.request.args + ctx.update({"data": puppetclasses.hieraprefixes(hierakey, cached)}) except Exception: app.logger.exception( 'Error collecting information for hiera key "%s"', hierakey ) return flask.render_template("hierakey.html", **ctx) @app.route("/proxy/") def all_proxies(): cached = "purge" not in flask.request.args ctx = { "proxies": proxies.all_proxies(cached), } return flask.render_template("proxies.html", **ctx) @app.route("/api/projects.json") def api_projects_json(): - return flask.jsonify(projects=keystone.all_projects()) + cached = "purge" not in flask.request.args + return flask.jsonify(projects=keystone.all_projects(cached)) @app.route("/api/projects.txt") def api_projects_txt(): + cached = "purge" not in flask.request.args return flask.Response( - "\n".join(sorted(keystone.all_projects())), mimetype="text/plain" + "\n".join(sorted(keystone.all_projects(cached))), mimetype="text/plain" ) @app.route("/api/dsh/project/") def api_dsh_project(name): - servers = nova.project_servers(name) + cached = "purge" not in flask.request.args + servers = nova.project_servers(name, cached) dsh = [ "{}.{}.eqiad1.wikimedia.cloud".format(server["name"], name) for server in servers ] return flask.Response("\n".join(sorted(dsh)), mimetype="text/plain") @app.route("/api/dsh/servers") def api_dsh_servers(): - servers = nova.all_servers() + cached = "purge" not in flask.request.args + servers = nova.all_servers(cached) dsh = [ "{}.{}.eqiad1.wikimedia.cloud".format( server["name"], server["tenant_id"] ) for server in servers ] return flask.Response("\n".join(sorted(dsh)), mimetype="text/plain") @app.route("/api/dsh/puppetclass/") def api_dsh_puppet(name): - data = puppetclasses.prefixes(name) + cached = "purge" not in flask.request.args + data = puppetclasses.prefixes(name, cached) dsh = [] for project, d in data.items(): if project == "admin": continue try: - servers = nova.project_servers(project) + cached = "purge" not in flask.request.args + servers = nova.project_servers(project, cached) except Exception: app.logger.exception( "Error collecting the list of servers for %s", project ) servers = [] for prefix in d["prefixes"]: if prefix.endswith(".wmflabs") or prefix.endswith(".cloud"): dsh.append(prefix) else: dsh.extend( [ "{}.{}.eqiad1.wikimedia.cloud".format( server["name"], project ) for server in servers if server["name"].startswith(prefix) ] ) return flask.Response("\n".join(sorted(set(dsh))), mimetype="text/plain") @app.route("/api/hierakey/") def api_hierakey(hierakey): - return flask.jsonify(servers=puppetclasses.hieraprefixes(hierakey)) + cached = "purge" not in flask.request.args + return flask.jsonify(servers=puppetclasses.hieraprefixes(hierakey, cached)) @app.errorhandler(404) def page_not_found(e): return flask.redirect(flask.url_for("projects")) @app.template_filter("contains") def contains(haystack, needle): return needle in haystack @app.template_filter("extract_hostname") def extract_hostname(backend): """Extract a hostname from a backend description.""" return proxies.parse_backend(backend).get("hostname", "404") @app.template_test() def ipv4addr(s): """Is the given string an IPv4 address?""" return utils.is_ipv4(s) diff --git a/keystone_browser/cinder.py b/keystone_browser/cinder.py index df10946..84019d5 100644 --- a/keystone_browser/cinder.py +++ b/keystone_browser/cinder.py @@ -1,78 +1,80 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- # # This file is part of the Keystone browser # # Copyright (c) 2021 Taavi Väänänen # # This program is free software: you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by the Free # Software Foundation, either version 3 of the License, or (at your option) # any later version. # # This program is distributed in the hope that it will be useful, but WITHOUT # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or # FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for # more details. # # You should have received a copy of the GNU General Public License along # with this program. If not, see . import functools from cinderclient import client from . import cache from . import keystone @functools.lru_cache(maxsize=None) def cinder_client(project, region): return client.Client( version="3", session=keystone.session(project), timeout=2, region_name=region, ) @functools.lru_cache() def get_regions(): ks_client = keystone.keystone_client() region_recs = ks_client.regions.list() return [region.id for region in region_recs] def project_volumes(project, cached=True): key = 'cinder:project-volumes:{}'.format(project) data = None if cached: data = cache.CACHE.load(key) if data is None: data = [] for region in get_regions(): cinder = cinder_client(project, region) data.extend( [ volume._info for volume in cinder.volumes.list( detailed=True, ) ] ) cache.CACHE.save(key, data, 300) return data -def limits(project): +def limits(project, cached=True): """Get a dict of limit details.""" key = "cinder:limits:{}".format(project) - data = cache.CACHE.load(key) + data = None + if cached: + data = cache.CACHE.load(key) if data is None: data = {} for region in get_regions(): cinder = cinder_client(project, region) data[region] = cinder.quotas.get(project, usage=True).to_dict() cache.CACHE.save(key, data, 3600) return data diff --git a/keystone_browser/glance.py b/keystone_browser/glance.py index 4c240b7..20bf443 100644 --- a/keystone_browser/glance.py +++ b/keystone_browser/glance.py @@ -1,53 +1,55 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- # # This file is part of the Keystone browser # # Copyright (c) 2017 Bryan Davis and contributors # # This program is free software: you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by the Free # Software Foundation, either version 3 of the License, or (at your option) # any later version. # # This program is distributed in the hope that it will be useful, but WITHOUT # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or # FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for # more details. # # You should have received a copy of the GNU General Public License along # with this program. If not, see . import functools import glanceclient from . import cache from . import keystone @functools.lru_cache(maxsize=1) def glance_client(): return glanceclient.Client( version="2", session=keystone.session(), interface="public", ) -def images(): +def images(cached=True): """Get a dict of image details indexed by id.""" # Images not appearing in this dict? Make sure that the 'observer' project # can see them: # for img in $(openstack image list --private -f value|awk '{print $1}') # do # glance member-create $img observer; # glance member-update $img observer accepted; # done key = "glance:images" - data = cache.CACHE.load(key) + data = None + if cached: + data = cache.CACHE.load(key) if data is None: glance = glance_client() data = {i["id"]: i for i in glance.images.list()} cache.CACHE.save(key, data, 3600) return data diff --git a/keystone_browser/keystone.py b/keystone_browser/keystone.py index 884e88d..1637983 100644 --- a/keystone_browser/keystone.py +++ b/keystone_browser/keystone.py @@ -1,112 +1,116 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- # # This file is part of the Keystone browser # # Copyright (c) 2017 Bryan Davis and contributors # # This program is free software: you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by the Free # Software Foundation, either version 3 of the License, or (at your option) # any later version. # # This program is distributed in the hope that it will be useful, but WITHOUT # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or # FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for # more details. # # You should have received a copy of the GNU General Public License along # with this program. If not, see . import collections import functools from keystoneauth1 import session as keystone_session from keystoneauth1.identity import v3 from keystoneclient.v3 import client from . import cache ROLES = collections.OrderedDict( [ ("admin", "2cd63d467f754404bf3746fe63ee0698"), ("glanceadmin", "1102f4ff63c3435793d0e4340bf4b04e"), ("observer", "47a8370618ea42d49f7047774e75d262"), ("projectadmin", "4d8cad783d6342efa8414d7d36fbc034"), ("user", "f473273fac7146b3bdbf22e5d4504f95"), ] ) @functools.lru_cache(maxsize=None) def session(project="observer"): """Get a session for the novaobserver user scoped to the given project.""" # TODO: read settings from /etc/novaobserver.yaml once we get it mounted # into the kubernetes pods () auth = v3.Password( auth_url="http://cloudcontrol1003.wikimedia.org:5000/v3", password="Fs6Dq2RtG8KwmM2Z", username="novaobserver", project_id=project, user_domain_name="Default", project_domain_name="Default", ) return keystone_session.Session(auth=auth) def keystone_client(): return client.Client( session=session(), interface="public", timeout=2, ) -def all_projects(): +def all_projects(cached=True): """Get a list of all project names.""" key = "keystone:all_projects" - data = cache.CACHE.load(key) + data = None + if cached: + data = cache.CACHE.load(key) if data is None: keystone = keystone_client() # Ignore the magic 'admin' project data = [ p.name for p in keystone.projects.list(enabled=True) if p.name != "admin" ] cache.CACHE.save(key, data, 300) return data def project_users_by_role(name): """Get a dict of lists of user ids indexed by role name.""" key = "keystone:project_users_by_role:{}".format(name) data = cache.CACHE.load(key) if data is None: keystone = keystone_client() # Ignore novaadmin & novaobserver in all user lists seen = ["novaadmin", "novaobserver"] data = {} for role_name, role_id in ROLES.items(): data[role_name] = [ r.user["id"] for r in keystone.role_assignments.list( project=name, role=role_id ) if r.user["id"] not in seen ] seen += data[role_name] cache.CACHE.save(key, data, 300) return data -def projects_for_user(uid): +def projects_for_user(uid, cached=True): """Get a list of projects that a user belongs to.""" key = "keystone:projects_for_user:{}".format(uid) - data = cache.CACHE.load(key) + data = None + if data: + data = cache.CACHE.load(key) if data is None: keystone = keystone_client() data = [p.name for p in keystone.projects.list(enabled=True, user=uid)] cache.CACHE.save(key, data, 300) return data diff --git a/keystone_browser/ldap.py b/keystone_browser/ldap.py index aeac495..712bb99 100644 --- a/keystone_browser/ldap.py +++ b/keystone_browser/ldap.py @@ -1,112 +1,114 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- # # This file is part of the Keystone browser # # Copyright (c) 2017 Bryan Davis and contributors # # This program is free software: you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by the Free # Software Foundation, either version 3 of the License, or (at your option) # any later version. # # This program is distributed in the hope that it will be useful, but WITHOUT # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or # FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for # more details. # # You should have received a copy of the GNU General Public License along # with this program. If not, see . import hashlib import ldap3 from . import cache def ldap_conn(): """Get an ldap connection Return value can be used as a context manager """ servers = ldap3.ServerPool( [ ldap3.Server("ldap-labs.eqiad.wikimedia.org"), ldap3.Server("ldap-labs.codfw.wikimedia.org"), ], ldap3.ROUND_ROBIN, active=True, exhaust=True, ) return ldap3.Connection(servers, read_only=True, auto_bind=True) def in_list(attr, items): """Make a search filter that will match all entries having attr with values in the given list. Similar to an SQL ``WHERE attr in ()`` clause. >>> in_list('uid', ['a', 'b', 'c']) '(|(uid=a)(uid=b)(uid=c))' """ return "(|{})".format( "".join(["({}={})".format(attr, item) for item in items]) ) -def get_users_by_uid(uids): +def get_users_by_uid(uids, cached=True): """Get a list of dicts of user information.""" if not uids: return [] key = "ldap:get_users_by_uid:{}".format( hashlib.sha1("|".join(uids).encode("utf-8")).hexdigest() ) - data = cache.CACHE.load(key) + data = None + if cached: + data = cache.CACHE.load(key) if data is None: data = [] with ldap_conn() as conn: results = conn.extend.standard.paged_search( "ou=people,dc=wikimedia,dc=org", in_list("uid", uids), ldap3.SUBTREE, attributes=["uid", "cn"], paged_size=1000, time_limit=5, generator=True, ) for resp in results: attribs = resp.get("attributes") # LDAP attributes come back as a dict of lists. We know that # there is only one value for each list, so unwrap it data.append( { "uid": attribs["uid"][0], "cn": attribs["cn"][0], } ) cache.CACHE.save(key, data, 3600) return data def user_count(): """Get the count of all users in LDAP.""" key = "ldap:user_count" total_entries = cache.CACHE.load(key) if total_entries is None: total_entries = 0 with ldap_conn() as conn: results = conn.extend.standard.paged_search( "ou=people,dc=wikimedia,dc=org", "(objectclass=posixaccount)", ldap3.SUBTREE, attributes=None, paged_size=1000, time_limit=5, generator=True, ) for resp in results: total_entries += 1 cache.CACHE.save(key, total_entries, 3600) return total_entries diff --git a/keystone_browser/neutron.py b/keystone_browser/neutron.py index f05af13..1447bf2 100644 --- a/keystone_browser/neutron.py +++ b/keystone_browser/neutron.py @@ -1,56 +1,58 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- # # This file is part of the Keystone browser # # Copyright (c) 2021 Taavi Väänänen # # This program is free software: you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by the Free # Software Foundation, either version 3 of the License, or (at your option) # any later version. # # This program is distributed in the hope that it will be useful, but WITHOUT # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or # FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for # more details. # # You should have received a copy of the GNU General Public License along # with this program. If not, see . import functools from neutronclient.v2_0 import client from . import cache from . import keystone @functools.lru_cache(maxsize=None) def neutron_client(project, region): return client.Client( session=keystone.session(project), timeout=2, region_name=region, ) @functools.lru_cache() def get_regions(): ks_client = keystone.keystone_client() region_recs = ks_client.regions.list() return [region.id for region in region_recs] -def limits(project): +def limits(project, cached=True): """Get a dict of limit details.""" key = "neutron:limits:{}".format(project) - data = cache.CACHE.load(key) + data = None + if cached: + data = cache.CACHE.load(key) if data is None: data = {} for region in get_regions(): neutron = neutron_client(project, region) data[region] = neutron.show_quota_details(project) cache.CACHE.save(key, data, 3600) return data diff --git a/keystone_browser/nova.py b/keystone_browser/nova.py index dd4ef1b..09dcefe 100644 --- a/keystone_browser/nova.py +++ b/keystone_browser/nova.py @@ -1,140 +1,150 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- # # This file is part of the Keystone browser # # Copyright (c) 2017 Bryan Davis and contributors # # This program is free software: you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by the Free # Software Foundation, either version 3 of the License, or (at your option) # any later version. # # This program is distributed in the hope that it will be useful, but WITHOUT # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or # FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for # more details. # # You should have received a copy of the GNU General Public License along # with this program. If not, see . import functools from novaclient import client from . import cache from . import keystone @functools.lru_cache(maxsize=None) def nova_client(project, region): return client.Client( "2.12", session=keystone.session(project), endpoint_type="public", timeout=2, region_name=region, ) @functools.lru_cache() def get_regions(): ks_client = keystone.keystone_client() region_recs = ks_client.regions.list() return [region.id for region in region_recs] -def project_servers(project): +def project_servers(project, cached=True): """Get a list of information about servers in the given project. Data returned for each server is described at https://developer.openstack.org/api-ref/compute/?expanded=list-servers-detailed-detail#listServers """ key = "nova:project_servers:{}".format(project) - data = cache.CACHE.load(key) + data = None + if cached: + data = cache.CACHE.load(key) if data is None: data = [] for region in get_regions(): nova = nova_client(project, region) data.extend( [ s._info for s in nova.servers.list( detailed=True, sort_keys=["display_name"], sort_dirs=["asc"], ) ] ) cache.CACHE.save(key, data, 300) return data -def flavors(project): +def flavors(project, cached=True): """Get a dict of flavor details indexed by id.""" key = "nova:flavors:{}".format(project) - data = cache.CACHE.load(key) + data = None + if cached: + data = cache.CACHE.load(key) if data is None: data = {} for region in get_regions(): nova = nova_client(project, region) for f in nova.flavors.list(): data[f._info["id"]] = f._info cache.CACHE.save(key, data, 3600) return data -def limits(project): +def limits(project, cached=True): """Get a dict of limit details.""" key = "nova:limits:{}".format(project) - data = cache.CACHE.load(key) + data = None + if cached: + data = cache.CACHE.load(key) if data is None: data = {} for region in get_regions(): nova = nova_client(project, region) data[region] = nova.limits.get().to_dict() cache.CACHE.save(key, data, 3600) return data -def all_servers(): +def all_servers(cached=True): """Get a list of all servers in all projects.""" key = "keystone:all_servers" - data = cache.CACHE.load(key) + data = None + if cached: + data = cache.CACHE.load(key) if data is None: data = [] all_projects = keystone.all_projects() for project in all_projects: if project != "admin": data += project_servers(project) cache.CACHE.save(key, data, 300) return data -def server(fqdn): +def server(fqdn, cached=True): """Get information about a server by fqdn.""" key = "nova:server:{}".format(fqdn) - data = cache.CACHE.load(key) + data = None + if cached: + data = cache.CACHE.load(key) if data is None: name, project, _ = fqdn.split(".", 2) servers = [] for region in get_regions(): nova = nova_client(project, region) reg_servers = nova.servers.list( detailed=True, search_opts={ "name": "^{}$".format(name), }, ) if reg_servers: servers.extend(reg_servers) if servers: data = servers[0]._info else: data = {} cache.CACHE.save(key, data, 300) return data