diff --git a/Procfile b/Procfile index b3fbbab..79e57c0 100644 --- a/Procfile +++ b/Procfile @@ -1,2 +1,2 @@ -dry-run: python3 -m glaab -vvv --dry-run -approve: timeout 3m python3 -m glaab -v +dry-run: python3 -m glaab -vvv --reject-after 90 --dry-run +approve: timeout 3m python3 -m glaab -v --reject-after 90 diff --git a/src/glaab/cli.py b/src/glaab/cli.py index 8bbfb26..62c2529 100644 --- a/src/glaab/cli.py +++ b/src/glaab/cli.py @@ -1,94 +1,110 @@ # Copyright (c) 2023 Wikimedia Foundation and contributors. # All Rights Reserved. # # This file is part of GitLab Account Approval Bot. # # GitLab Account Approval Bot is free software: you can redistribute it and/or # modify it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or (at your # option) any later version. # # GitLab Account Approval Bot is distributed in the hope that it will be # useful, but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General # Public License for more details. # # You should have received a copy of the GNU General Public License along with # GitLab Account Approval Bot. If not, see . import logging import sys import time import click import coloredlogs import requests.exceptions from . import gitlab from . import mediawiki from . import utils from .version import __version__ logger = logging.getLogger(__name__) @click.command() @click.version_option(version=__version__) @click.option( "-v", "--verbose", count=True, help="Increase debug logging verbosity", ) @click.option( "--dry-run", is_flag=True, default=False, help="Do not actually change anything", ) -def main(verbose, dry_run): +@click.option( + "--reject-after", + type=int, + metavar="N", + help="Reject pending accounts after N days", +) +def main(verbose, dry_run, reject_after): """Approve pending GitLab accounts for trusted contributors.""" coloredlogs.install( level=max(logging.DEBUG, logging.WARNING - (10 * verbose)), fmt="%(asctime)s %(name)s %(levelname)s: %(message)s", datefmt="%Y-%m-%dT%H:%M:%SZ", level_styles=coloredlogs.DEFAULT_LEVEL_STYLES | { "debug": {}, "info": {"color": "green"}, }, field_styles=coloredlogs.DEFAULT_FIELD_STYLES | { "asctime": {"color": "yellow"}, }, ) logging.captureWarnings(True) sys.excepthook = utils.log_uncaught_exception gl = gitlab.Client.default_client() mw = mediawiki.Client.default_client() click.echo(click.style("Searching for pending accounts...", fg="yellow")) for gitlab_user in gl.users_pending_approval(): username = gitlab_user["username"] try: if utils.is_trusted(gitlab_user): if not dry_run: gl.approve_user(gitlab_user) mw.log_account_approval(gitlab_user) else: logger.warning( "Cowardly refusing to approve user %s.", username, ) click.echo(click.style(f"{username}: Trusted", fg="green")) + elif reject_after and utils.is_expired(gitlab_user, reject_after): + if not dry_run: + gl.reject_user(gitlab_user) + mw.log_account_rejection(gitlab_user) + else: + logger.warning( + "Cowardly refusing to reject user %s.", + username, + ) + click.echo(click.style(f"{username}: rejected", fg="red")) else: click.echo(click.style(f"{username}: Untrusted", fg="red")) except (TimeoutError, requests.exceptions.Timeout): logger.exception("Timeout processing user %s", username) # Give Phab a little break time.sleep(3) if __name__ == "__main__": # pragma: nocover main() diff --git a/src/glaab/gitlab.py b/src/glaab/gitlab.py index 8994bcd..8e49f1e 100644 --- a/src/glaab/gitlab.py +++ b/src/glaab/gitlab.py @@ -1,171 +1,186 @@ # Copyright (c) 2023 Wikimedia Foundation and contributors. # All Rights Reserved. # # This file is part of GitLab Account Approval Bot. # # GitLab Account Approval Bot is free software: you can redistribute it and/or # modify it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or (at your # option) any later version. # # GitLab Account Approval Bot is distributed in the hope that it will be # useful, but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General # Public License for more details. # # You should have received a copy of the GNU General Public License along with # GitLab Account Approval Bot. If not, see . import json import logging import requests from . import settings logger = logging.getLogger(__name__) class APIError(Exception): def __init__(self, message, code, result): self.message = message self.code = code self.result = result def __str__(self): return f"{self.message} ({self.code})" class Client: """GitLab client.""" _default_instance = None @classmethod def default_client(cls): """Get a GitLab client using the default credentials.""" if cls._default_instance is None: logger.debug("Creating default instance") cls._default_instance = cls( settings.GITLAB_URL, settings.GITLAB_ACCESS_TOKEN, ) return cls._default_instance def __init__(self, url, token): """Initialize instance.""" self.url = url self.token = token self.session = requests.Session() self.session.headers = { "PRIVATE-TOKEN": self.token, "Content-Type": "application/json", "User-Agent": "{name} ({url}) python-requests/{vers}".format( name="GitLab Account Approval Bot", url="https://wikitech.wikimedia.org/wiki/Tool:Gitlab-account-approval", vers=requests.__version__, ), } # Talking to GitLab from Toolforge can be flaky, but for idempotent # lookups we can smooth over some of the rough spots by retrying # requests. retries = requests.packages.urllib3.util.retry.Retry( total=5, backoff_factor=0.1, ) self.session.mount( self.url, requests.adapters.HTTPAdapter(max_retries=retries), ) def http_request(self, verb, path, payload=None, params=None): url = f"{self.url}/api/v4/{path}" return self.session.request( method=verb, url=url, params=params, json=payload, timeout=(1, 5), ) def json_request(self, verb, path, payload=None, params=None): r = self.http_request(verb, path, payload, params) if 200 <= r.status_code < 300: return r.json() err_msg = r.content try: err_json = r.json() if "message" in err_json: err_msg = err_json["message"] if "error" in err_json: err_msg = err_json["error"] except json.decoder.JSONDecodeError: logger.exception( "Failed to parse error message from %s/api/v4/%s: %s", self.url, path, err_msg, ) raise APIError(err_msg, r.status_code, r) def post(self, path, payload=None): resp = self.json_request("POST", path, payload=payload) logger.debug("POST %s: %s", path, resp) return resp def get(self, path, params=None): resp = self.json_request("GET", path, params=params) logger.debug("GET %s: %s", path, resp) return resp def _paginated(self, path, params=None, filter=None): # noqa: A002 shadow """Get a generator over items from a paginated endpoint.""" r = self.http_request("GET", path, params=params) while r: r.raise_for_status() for item in r.json(): if filter is None or filter(item): yield item if not r.links.get("next"): break r = self.session.request( method="GET", url=r.links["next"]["url"], timeout=(1, 5), ) def users_pending_approval(self): """Get a generator over GitLab accounts pending approval.""" return self._paginated( "users", { # Cognative danger: x=false parameters are ignored by the # service, so setting things like "active=false" only serves # to confuse humans. "exclude_external": "true", "exclude_internal": "true", "without_project_bots": "true", # T368761: keyset-based pagination "pagination": "keyset", "per_page": 500, "order_by": "created_at", "sort": "asc", }, # Upstream: filter=lambda u: u["state"] == "blocked_pending_approval", ) def approve_user(self, gitlab_user): """Mark a user as approved.""" uid = gitlab_user["id"] username = gitlab_user["username"] try: self.post(f"users/{uid}/approve") return True except APIError: logger.exception( "Failed to approve user %s (%s)", uid, username, ) return False + + def reject_user(self, gitlab_user): + """Mark a user as rejected.""" + uid = gitlab_user["id"] + username = gitlab_user["username"] + try: + self.post(f"users/{uid}/reject") + return True + except APIError: + logger.exception( + "Failed to reject user %s (%s)", + uid, + username, + ) + return False diff --git a/src/glaab/mediawiki.py b/src/glaab/mediawiki.py index c5873ed..2150bf8 100644 --- a/src/glaab/mediawiki.py +++ b/src/glaab/mediawiki.py @@ -1,95 +1,105 @@ # Copyright (c) 2023 Wikimedia Foundation and contributors. # All Rights Reserved. # # This file is part of GitLab Account Approval Bot. # # GitLab Account Approval Bot is free software: you can redistribute it and/or # modify it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or (at your # option) any later version. # # GitLab Account Approval Bot is distributed in the hope that it will be # useful, but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General # Public License for more details. # # You should have received a copy of the GNU General Public License along with # GitLab Account Approval Bot. If not, see . import datetime import logging import mwclient from . import settings logger = logging.getLogger(__name__) class Client: """MediaWiki client.""" _default_instance = None @classmethod def default_client(cls): """Get a MediaWiki client using the default credentials.""" if cls._default_instance is None: logger.debug("Creating default instance") cls._default_instance = cls( settings.MEDIAWIKI_HOST, settings.MEDIAWIKI_CONSUMER_TOKEN, settings.MEDIAWIKI_CONSUMER_SECRET, settings.MEDIAWIKI_ACCESS_TOKEN, settings.MEDIAWIKI_ACCESS_SECRET, settings.MEDIAWIKI_LOG_PAGE, ) return cls._default_instance def __init__( self, host, consumer_token, consumer_secret, access_token, access_secret, log_page, ): self.mwsite = mwclient.Site( host, consumer_token=consumer_token, consumer_secret=consumer_secret, access_token=access_token, access_secret=access_secret, clients_useragent="glaab (https://wikitech.wikimedia.org/wiki/Tool:Gitlab-account-approval)", ) self.log_page = log_page - def log_account_approval(self, gitlab_user): - """Log an account approval action on-wiki.""" - now = datetime.datetime.utcnow() + def _log_action(self, log, summary): + """Log an action on-wiki.""" + now = datetime.datetime.now() target_section = now.strftime("=== %Y-%m-%d ===") - username = gitlab_user["username"] - logline = ( - f"* {now.hour:02d}:{now.minute:02d} " - f"[[gitlab:{username}|@{username}]] was approved." - ) - summary = f"@{username} was approved." + logline = f"* {now.hour:02d}:{now.minute:02d} {log}" page = self.mwsite.Pages[self.log_page] text = page.text() lines = text.split("\n") first_header = 0 for pos, line in enumerate(lines): if line.startswith("=== "): first_header = pos break if lines[first_header] == target_section: lines.insert(first_header + 1, logline) else: lines.insert(first_header, "") lines.insert(first_header, logline) lines.insert(first_header, target_section) page.save("\n".join(lines), summary=summary, bot=True) + + def log_account_approval(self, gitlab_user): + """Log an account approval action on-wiki.""" + username = gitlab_user["username"] + log = f"[[gitlab:{username}|@{username}]] was approved." + summary = f"@{username} was approved." + return self._log_action(log, summary) + + def log_account_rejection(self, gitlab_user): + """Log an account rejection action on-wiki.""" + username = gitlab_user["username"] + created = gitlab_user["created_at"] + log = f'"{username}" was rejected ' f"(pending since {created})." + summary = f"{username} was rejected." + return self._log_action(log, summary) diff --git a/src/glaab/utils.py b/src/glaab/utils.py index f27572c..cc6fde2 100644 --- a/src/glaab/utils.py +++ b/src/glaab/utils.py @@ -1,127 +1,146 @@ # Copyright (c) 2023 Wikimedia Foundation and contributors. # All Rights Reserved. # # This file is part of GitLab Account Approval Bot. # # GitLab Account Approval Bot is free software: you can redistribute it and/or # modify it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or (at your # option) any later version. # # GitLab Account Approval Bot is distributed in the hope that it will be # useful, but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General # Public License for more details. # # You should have received a copy of the GNU General Public License along with # GitLab Account Approval Bot. If not, see . +import datetime import functools import logging import sys from . import gerrit from . import gitlab from . import ldap from . import phabricator from . import settings logger = logging.getLogger(__name__) unhandled_logger = logging.getLogger("glaab.unhandled") gerrit_client = gerrit.RESTClient.default_client() gitlab_client = gitlab.Client.default_client() ldap_client = ldap.Client.default_client() phab_client = phabricator.Client.default_client() def is_trusted(gitlab_user): """Should the given user be considered trusted?""" logger.info("Checking %s", gitlab_user["username"]) ldap_user = get_developer(gitlab_user) if is_trusted_developer(ldap_user): return True if is_trusted_gerrit_user(ldap_user): return True phab_user = get_phabricator_user(ldap_user) if phab_user is not None and is_trusted_phab_user(phab_user): return True return False def get_developer(gitlab_user): """Get the Developer account record for a gitlab user.""" cn = None for ident in gitlab_user["identities"]: if ident["provider"] == "openid_connect": cn = ident["extern_uid"] return ldap_client.developer(cn)["attributes"] def is_trusted_developer(ldap_user): """Is the given Developer account trusted by the community?""" groups = ldap_user.get("memberOf") uid = ldap_user["uid"][0] if not groups: logger.debug("%s is not a member of any groups", uid) return False for group in settings.LDAP_TRUSTED_GROUPS: if group in ldap_user.get("memberOf"): logger.info("%s is a memberOf %s", uid, group) return True return False def get_phabricator_user(ldap_user): """Get the Phabricator account for a Developer account record.""" users = phab_client.user_external_lookup( ldap_user["cn"], [ldap_user.get("wikimediaGlobalAccountName")], ) if users: return users[0] return None @functools.cache def trusted_phab_user_phids(): """Get a list of trusted Phabricator user PHIDs.""" return phab_client.project_members( settings.PHABRICATOR_TRUSTED_GROUP, ) def is_trusted_phab_user(phab_user): """Is the given Phabricator user trusted by the community?""" if phab_user["phid"] in trusted_phab_user_phids(): logger.info("%s is a trusted Phabricator user", phab_user["userName"]) return True return False @functools.cache def trusted_gerrit_users(): """Get a dict of Developer accounts trusted by Gerrit.""" return gerrit_client.all_members(settings.GERRIT_TRUSTED_GROUP) def is_trusted_gerrit_user(ldap_user): """Is the given Developer account trusted by Gerrit?""" if ldap_user["uid"][0] in trusted_gerrit_users(): logger.info("%s is a trusted Gerrit user", ldap_user["uid"]) return True return False +def is_expired(gitlab_user, days): + """Is the given GitLab user more than N days old?""" + now = datetime.datetime.now() + created_at = datetime.datetime.strptime( + gitlab_user["created_at"], + "%Y-%m-%dT%H:%M:%S.%fZ", + ) + elapsed = now - created_at + if elapsed.days > days: + logger.info( + "%s was more than %s days ago.", + gitlab_user["created_at"], + days, + ) + return True + return False + + def log_uncaught_exception(exc_type, exc_value, exc_traceback): """`sys.excepthook` handler that logs via the logging module.""" if issubclass(exc_type, KeyboardInterrupt): # Ignore ^C exits sys.__excepthook__(exc_type, exc_value, exc_traceback) return unhandled_logger.critical( "Uncaught exception", exc_info=(exc_type, exc_value, exc_traceback), )