diff --git a/docs/scripts.rst b/docs/scripts.rst index 43f34a3..6b15576 100644 --- a/docs/scripts.rst +++ b/docs/scripts.rst @@ -1,130 +1,130 @@ ####### Scripts ####### .. _scap: scap ==== **scap** is the driver script for syncing the MediaWiki versions and configuration files currently staged on the deploy server to the rest of the cluster. .. program-output:: ../bin/scap --help .. seealso:: * :func:`scap.Scap` * :func:`scap.tasks.check_php_syntax` - * :func:`scap.tasks.compile_wikiversions_cdb` + * :func:`scap.tasks.compile_wikiversions` * :func:`scap.tasks.sync_common` * :func:`scap.tasks.sync_wikiversions` sync-common =========== **sync-common** uses rsync to fetch MediaWiki code and configuration to the local host. It is typically called automatically on hosts during the execution of scap_. .. program-output:: ../bin/sync-common --help .. seealso:: * :func:`scap.SyncCommon` * :func:`scap.tasks.sync_common` sync-dblist =========== **sync-dblist** synchronizes dblist files to the cluster. .. program-output:: ../bin/sync-dblist --help .. seealso:: * :func:`scap.SyncDblist` sync-dir ======== **sync-dir** synchronizes a directory from the staging directory to the cluster. .. program-output:: ../bin/sync-dir --help .. seealso:: * :func:`scap.SyncDir` sync-docroot ============ **sync-docroot** synchronizes common/docroot and common/w to the cluster. .. program-output:: ../bin/sync-docroot --help .. seealso:: * :func:`scap.SyncDocroot` sync-file ========= **sync-file** synchronizes a file from the staging directory to the cluster. .. program-output:: ../bin/sync-file --help .. seealso:: * :func:`scap.SyncFile` sync-wikiversions ================= **sync-wikiversions** compiles wikiversions.json into a CDB database and then syncs both the JSON and CDB versions to the rest of the cluster. .. program-output:: ../bin/sync-wikiversions --help .. seealso:: * :func:`scap.SyncWikiversions` - * :func:`scap.tasks.compile_wikiversions_cdb` + * :func:`scap.tasks.compile_wikiversions` * :func:`scap.tasks.sync_wikiversions` mwversionsinuse =============== **mwversionsinuse** examines wikiversions.json to find the current active MediaWiki versions. .. program-output:: ../bin/mwversionsinuse --help .. seealso:: * :func:`scap.MWVersionsInUse` scap-purge-l10n-cache ===================== **scap-purge-l10n-cache** deletes localization files (CDB and JSON) across the cluster. .. program-output:: ../bin/scap-purge-l10n-cache --help .. seealso:: * :func:`scap.PurgeL10nCache` * :func:`scap.tasks.purge_l10n_cache` compile-wikiversions ==================== **compile-wikiversions** compiles wikiversions.json into wikiversions.cdb. .. program-output:: ../bin/compile-wikiversions --help .. seealso:: * :func:`scap.CompileWikiversions` - * :func:`scap.tasks.compile_wikiversions_cdb` + * :func:`scap.tasks.compile_wikiversions` scap-rebuild-cdbs ================= **scap-rebuild-cdbs** rebuilds localization cache CDB files from JSON files. .. program-output:: ../bin/scap-rebuild-cdbs --help .. seealso:: * :func:`scap.RebuildCdbs` * :func:`scap.tasks.merge_cdb_updates` mw-update-l10n ============== **mw-update-l10n** generates localization cache files. .. program-output:: ../bin/mw-update-l10n --help .. seealso:: * :func:`scap.UpdateL10n` * :func:`scap.tasks.update_localization_cache` diff --git a/scap/main.py b/scap/main.py index 7861ca4..be59301 100644 --- a/scap/main.py +++ b/scap/main.py @@ -1,575 +1,575 @@ # -*- coding: utf-8 -*- """ scap.main ~~~~~~~~~~ Command wrappers for scap tasks """ import argparse import errno import multiprocessing import netifaces import os import psutil import subprocess from . import cli from . import log from . import ssh from . import tasks from . import utils class AbstractSync(cli.Application): """Base class for applications that want to sync one or more files from the deployment server to the rest of the cluster.""" soft_errors = False def _process_arguments(self, args, extra_args): if hasattr(args, 'message'): args.message = ' '.join(args.message) or '(no message)' return args, extra_args @cli.argument('message', nargs='*', help='Log message for SAL') def main(self, *extra_args): """Perform a sync operation to the cluster.""" print utils.logo() self._assert_auth_sock() with utils.lock(self.config['lock_file']): self._before_cluster_sync() # Update proxies proxies = self._get_proxy_list() with log.Timer('sync-proxies', self.get_stats()): update_proxies = ssh.Job(proxies, user=self.config['ssh_user']) update_proxies.command(self._proxy_sync_command()) update_proxies.progress('sync-proxies') succeeded, failed = update_proxies.run() if failed: self.get_logger().warning( '%d proxies had sync errors', failed) self.soft_errors = True # Update apaches with log.Timer('sync-apaches', self.get_stats()): update_apaches = ssh.Job(self._get_target_list(), user=self.config['ssh_user']) update_apaches.exclude_hosts(proxies) update_apaches.shuffle() update_apaches.command(self._apache_sync_command(proxies)) update_apaches.progress('sync-common') succeeded, failed = update_apaches.run() if failed: self.get_logger().warning( '%d apaches had sync errors', failed) self.soft_errors = True self._after_cluster_sync() self._after_lock_release() if self.soft_errors: return 1 else: return 0 def _before_cluster_sync(self): pass def _get_proxy_list(self): """Get list of sync proxy hostnames that should be updated before the rest of the cluster.""" return utils.read_dsh_hosts_file(self.config['dsh_proxies']) def _proxy_sync_command(self): """Synchronization command to run on the proxy hosts.""" cmd = [self.get_script_path('sync-common'), '--no-update-l10n'] if self.verbose: cmd.append('--verbose') return cmd def _get_target_list(self): """Get list of hostnames that should be updated from the proxies.""" return utils.read_dsh_hosts_file(self.config['dsh_targets']) def _apache_sync_command(self, proxies): """Synchronization command to run on the apache hosts. :param proxies: List of proxy hostnames """ return self._proxy_sync_command() + proxies def _after_cluster_sync(self): pass def _after_lock_release(self): pass class CompileWikiversions(cli.Application): """Compile wikiversions.json to wikiversions.cdb.""" def main(self, *extra_args): self._run_as('mwdeploy') self._assert_current_user('mwdeploy') - tasks.compile_wikiversions_cdb('deploy', self.config) + tasks.compile_wikiversions('deploy', self.config) return 0 class MWVersionsInUse(cli.Application): """Get a list of the active MediaWiki versions.""" @cli.argument('--withdb', action='store_true', help='Add `=wikidb` with some wiki using the version.') def main(self, *extra_args): versions = self.active_wikiversions() if self.arguments.withdb: output = ['%s=%s' % (version, wikidb) for version, wikidb in versions.items()] else: output = [str(version) for version in versions.keys()] print ' '.join(output) return 0 def _process_arguments(self, args, extra_args): """Log warnings about unexpected arguments but don't exit.""" if extra_args: self.get_logger().warning( 'Unexpected argument(s) ignored: %s', extra_args) return args, extra_args class PurgeL10nCache(cli.Application): """Purge the localization cache for an inactive MediaWiki version.""" @cli.argument('--version', required=True, help='MediaWiki version (eg 1.23wmf16)') def main(self, *extra_args): if self.arguments.version.startswith('php-'): self.arguments.version = self.arguments.version[4:] if self.arguments.version in self.active_wikiversions(): self.get_logger().error( 'Version %s is in use' % self.arguments.version) return 1 tasks.purge_l10n_cache(self.arguments.version, self.config) self.announce('Purged l10n cache for %s' % self.arguments.version) return 0 class RebuildCdbs(cli.Application): """Rebuild localization cache CDB files from the JSON versions.""" @cli.argument('--no-progress', action='store_true', dest='mute', help='Do not show progress indicator.') def main(self, *extra_args): self._run_as('mwdeploy') self._assert_current_user('mwdeploy') # Leave some of the cores free for apache processes use_cores = max(multiprocessing.cpu_count() / 2, 1) # Rebuild the CDB files from the JSON versions for version, wikidb in self.active_wikiversions().items(): cache_dir = os.path.join(self.config['deploy_dir'], 'php-%s' % version, 'cache', 'l10n') tasks.merge_cdb_updates( cache_dir, use_cores, True, self.arguments.mute) class Scap(AbstractSync): """Deploy MediaWiki to the cluster. #. Validate php syntax of wmf-config and multiversion #. Sync deploy directory on localhost with staging area #. Compile wikiversions.json to cdb in deploy directory #. Update l10n files in staging area #. Compute git version information #. Ask scap proxies to sync with master server #. Ask apaches to sync with fastest rsync server #. Ask apaches to rebuild l10n CDB files #. Update wikiversions.cdb on localhost #. Ask apaches to sync wikiversions.cdb #. Restart HHVM across the cluster """ @cli.argument('-r', '--restart', action='store_true', dest='restart', help='Restart HHVM process on target hosts.') @cli.argument('message', nargs='*', help='Log message for SAL') def main(self, *extra_args): super(Scap, self).main(*extra_args) def _before_cluster_sync(self): self.announce('Started scap: %s', self.arguments.message) # Validate php syntax of wmf-config and multiversion tasks.check_valid_syntax( '%(stage_dir)s/wmf-config' % self.config, '%(stage_dir)s/multiversion' % self.config) # Sync deploy directory on localhost with staging area tasks.sync_common(self.config) # Bug 63659: Compile deploy_dir/wikiversions.json to cdb utils.sudo_check_call('mwdeploy', self.get_script_path('compile-wikiversions'), self.get_logger()) # Update list of extension message files and regenerate the # localisation cache. with log.Timer('mw-update-l10n', self.get_stats()): for version, wikidb in self.active_wikiversions().items(): tasks.update_localization_cache( version, wikidb, self.verbose, self.config) # Compute git version information with log.Timer('cache_git_info', self.get_stats()): for version, wikidb in self.active_wikiversions().items(): tasks.cache_git_info(version, self.config) def _after_cluster_sync(self): target_hosts = self._get_target_list() # Ask apaches to rebuild l10n CDB files with log.Timer('scap-rebuild-cdbs', self.get_stats()): rebuild_cdbs = ssh.Job(target_hosts, user=self.config['ssh_user']) rebuild_cdbs.shuffle() rebuild_cdbs.command('sudo -u mwdeploy -n -- %s' % self.get_script_path('scap-rebuild-cdbs')) rebuild_cdbs.progress('scap-rebuild-cdbs') succeeded, failed = rebuild_cdbs.run() if failed: self.get_logger().warning( '%d hosts had scap-rebuild-cdbs errors', failed) self.soft_errors = True # Update and sync wikiversions.cdb succeeded, failed = tasks.sync_wikiversions(target_hosts, self.config) if failed: self.get_logger().warning( '%d hosts had sync_wikiversions errors', failed) self.soft_errors = True if self.arguments.restart: # Restart HHVM across the cluster succeeded, failed = tasks.restart_hhvm( target_hosts, self.config, # Use a batch size of 5% of the total target list len(target_hosts) // 20) if failed: self.get_logger().warning( '%d hosts failed to restart HHVM', failed) self.soft_errors = True self.get_stats().increment('deploy.restart') def _after_lock_release(self): self.announce('Finished scap: %s (duration: %s)', self.arguments.message, utils.human_duration(self.get_duration())) self.get_stats().increment('deploy.scap') self.get_stats().increment('deploy.all') def _handle_keyboard_interrupt(self, ex): self.announce('scap aborted: %s (duration: %s)', self.arguments.message, utils.human_duration(self.get_duration())) return 1 def _handle_exception(self, ex): self.get_logger().warn('Unhandled error:', exc_info=True) self.announce('scap failed: %s %s (duration: %s)', type(ex).__name__, ex, utils.human_duration(self.get_duration())) return 1 def _before_exit(self, exit_status): if self.config: self.get_stats().timing('scap.scap', self.get_duration() * 1000) return exit_status class SyncCommon(cli.Application): """Sync local MediaWiki deployment directory with deploy server state.""" @cli.argument('--no-update-l10n', action='store_false', dest='update_l10n', help='Do not update l10n cache files.') @cli.argument('-i', '--include', default=None, action='append', help='Rsync include pattern to limit transfer to.' 'End directories with a trailing `/***`. Can be used multiple times.') @cli.argument('servers', nargs=argparse.REMAINDER, help='Rsync server(s) to copy from') def main(self, *extra_args): tasks.sync_common( self.config, include=self.arguments.include, sync_from=self.arguments.servers, verbose=self.verbose ) if self.arguments.update_l10n: utils.sudo_check_call( 'mwdeploy', self.get_script_path('scap-rebuild-cdbs') + ' --no-progress', self.get_logger() ) return 0 class SyncDblist(AbstractSync): """Sync dblist files to the cluster.""" def _proxy_sync_command(self): cmd = [self.get_script_path('sync-common'), '--no-update-l10n', '--include', '*.dblist'] if self.verbose: cmd.append('--verbose') return cmd def _after_lock_release(self): self.announce('Synchronized database lists: %s (duration: %s)', self.arguments.message, utils.human_duration(self.get_duration())) self.get_stats().increment('deploy.sync-dblist') self.get_stats().increment('deploy.all') class SyncDir(AbstractSync): """Sync a directory to the cluster.""" @cli.argument('dir', help='Directory to sync') @cli.argument('message', nargs='*', help='Log message for SAL') def main(self, *extra_args): super(SyncDir, self).main(*extra_args) def _before_cluster_sync(self): # assert file exists abspath = os.path.join( self.config['stage_dir'], self.arguments.dir) if not os.path.isdir(abspath): raise IOError(errno.ENOENT, 'Directory not found', abspath) relpath = os.path.relpath(abspath, self.config['stage_dir']) self.include = '%s/***' % relpath tasks.check_valid_syntax(abspath) def _proxy_sync_command(self): cmd = [self.get_script_path('sync-common'), '--no-update-l10n'] if '/' in self.include: parts = self.include.split('/') for i in range(1, len(parts)): # Include parent directories in sync command or the default # exclude will block them and by extension block the target # file. cmd.extend(['--include', '/'.join(parts[:i])]) cmd.extend(['--include', self.include]) if self.verbose: cmd.append('--verbose') return cmd def _after_lock_release(self): self.announce('Synchronized %s: %s (duration: %s)', self.arguments.dir, self.arguments.message, utils.human_duration(self.get_duration())) self.get_stats().increment('deploy.sync-dir') self.get_stats().increment('deploy.all') class SyncDocroot(AbstractSync): def _proxy_sync_command(self): cmd = [ self.get_script_path('sync-common'), '--no-update-l10n', '--include', 'docroot/***', '--include', 'w/***', ] if self.verbose: cmd.append('--verbose') return cmd def _after_lock_release(self): self.announce('Synchronized docroot and w: %s (duration: %s)', self.arguments.message, utils.human_duration(self.get_duration())) self.get_stats().increment('deploy.sync-docroot') self.get_stats().increment('deploy.all') class SyncFile(AbstractSync): """Sync a specific file to the cluster.""" @cli.argument('file', help='File to sync') @cli.argument('message', nargs='*', help='Log message for SAL') def main(self, *extra_args): super(SyncFile, self).main(*extra_args) def _before_cluster_sync(self): # assert file exists abspath = os.path.join( self.config['stage_dir'], self.arguments.file) if not os.path.isfile(abspath): raise IOError(errno.ENOENT, 'File not found', abspath) # Warn when syncing a symlink. if os.path.islink(abspath): self.get_logger().warning( '%s: did you mean to sync a symbolic link?', abspath) self.include = os.path.relpath(abspath, self.config['stage_dir']) if abspath.endswith(('.php', '.inc', '.phtml', '.php5')): subprocess.check_call('/usr/bin/php -l %s' % abspath, shell=True) utils.check_php_opening_tag(abspath) elif abspath.endswith('.json'): utils.check_valid_json_file(abspath) def _proxy_sync_command(self): cmd = [self.get_script_path('sync-common'), '--no-update-l10n'] if '/' in self.include: parts = self.include.split('/') for i in range(1, len(parts)): # Include parent directories in sync command or the default # exclude will block them and by extension block the target # file. cmd.extend(['--include', '/'.join(parts[:i])]) cmd.extend(['--include', self.include]) if self.verbose: cmd.append('--verbose') return cmd def _after_lock_release(self): self.announce('Synchronized %s: %s (duration: %s)', self.arguments.file, self.arguments.message, utils.human_duration(self.get_duration())) self.get_stats().increment('deploy.sync-file') self.get_stats().increment('deploy.all') class SyncWikiversions(cli.Application): """Rebuild and sync wikiversions.cdb to the cluster.""" def _process_arguments(self, args, extra_args): args.message = ' '.join(args.message) or '(no message)' return args, extra_args @cli.argument('message', nargs='*', help='Log message for SAL') def main(self, *extra_args): self._assert_auth_sock() # check for the presence of ExtensionMessages and l10n cache # for every branch of mediawiki that is referenced in wikiversions.json # to avoid syncing a branch that is lacking these critical files. for version, wikidb in self.active_wikiversions().items(): ext_msg = os.path.join(self.config['stage_dir'], 'wmf-config', 'ExtensionMessages-%s.php' % version) err_msg = 'ExtensionMessages not found in {}' % ext_msg utils.check_file_exists(ext_msg, err_msg) cache_file = os.path.join(self.config['stage_dir'], 'php-%s' % version, 'cache', 'l10n', 'l10n_cache-en.cdb') err_msg = 'l10n cache missing for {}' % version utils.check_file_exists(cache_file, err_msg) mw_install_hosts = utils.read_dsh_hosts_file( self.config['dsh_targets']) tasks.sync_wikiversions(mw_install_hosts, self.config) self.announce( 'rebuilt wikiversions.cdb and synchronized wikiversions files: %s', self.arguments.message) self.get_stats().increment('deploy.sync-wikiversions') self.get_stats().increment('deploy.all') class UpdateL10n(cli.Application): """Update localization files""" def main(self, *extra_args): for version, wikidb in self.active_wikiversions().items(): tasks.update_localization_cache( version, wikidb, self.verbose, self.config) class RestartHHVM(cli.Application): """Restart the HHVM fcgi process on the local server #. Depool the server if registered with pybal #. Wait for pending requests to complete #. Restart HHVM process #. Re-pool the server if needed """ def main(self, *extra_args): self._run_as('mwdeploy') self._assert_current_user('mwdeploy') try: hhvm_pid = utils.read_pid(self.config['hhvm_pid_file']) except IOError: self.get_logger().debug('HHVM pid not found', exc_info=True) return 0 else: if not psutil.pid_exists(hhvm_pid): self.get_logger().debug('HHVM not running') return 0 try: # Check for pybal interface have_pybal = netifaces.ifaddresses(self.config['pybal_interface']) except ValueError: self.get_logger().debug('Pybal interface not found', exc_info=True) have_pybal = False if have_pybal: # Depool by gracefully shutting down apache (SIGWINCH) try: apache_pid = utils.read_pid(self.config['apache_pid_file']) except IOError: self.get_logger().debug('Apache pid not found', exc_info=True) pass else: utils.sudo_check_call('root', '/usr/sbin/apache2ctl graceful-stop', self.get_logger()) # Wait for Apache to stop hard after GracefulShutdownTimeout # seconds or when requests actually complete psutil.Process(apache_pid).wait() # Restart HHVM utils.sudo_check_call('root', '/sbin/restart hhvm', self.get_logger()) if have_pybal: utils.sudo_check_call('root', '/usr/sbin/service apache2 start', self.get_logger()) return 0 class HHVMGracefulAll(cli.Application): """Perform a rolling restart of HHVM across the cluster.""" def _process_arguments(self, args, extra_args): if hasattr(args, 'message'): args.message = ' '.join(args.message) or '(no message)' return args, extra_args @cli.argument('message', nargs='*', help='Log message for SAL') def main(self, *extra_args): exit_code = 0 self.announce('Restarting HHVM: %s', self.arguments.message) target_hosts = utils.read_dsh_hosts_file(self.config['dsh_targets']) succeeded, failed = tasks.restart_hhvm( target_hosts, self.config, # Use a batch size of 5% of the total target list len(target_hosts) // 20) if failed: self.get_logger().warning( '%d hosts failed to restart HHVM', failed) self.get_stats().increment('deploy.fail') exit_code = 1 self.announce('Finished HHVM restart: %s (duration: %s)', self.arguments.message, utils.human_duration(self.get_duration())) self.get_stats().increment('deploy.restart') return exit_code diff --git a/scap/tasks.py b/scap/tasks.py index cadf242..699a769 100644 --- a/scap/tasks.py +++ b/scap/tasks.py @@ -1,555 +1,573 @@ # -*- coding: utf-8 -*- """ scap.tasks ~~~~~~~~~~ Contains functions implementing scap tasks """ import errno import glob import itertools import json import logging import multiprocessing import os import shutil import socket import subprocess from . import cdblib from . import log from . import ssh from . import utils DEFAULT_RSYNC_ARGS = [ '/usr/bin/rsync', '--archive', '--delete-delay', '--delay-updates', '--compress', '--delete', '--exclude=**/.svn/lock', '--exclude=**/.git/objects', '--exclude=**/.git/**/objects', '--exclude=**/cache/l10n/*.cdb', '--no-perms', ] def cache_git_info(version, cfg): """Create JSON cache files of git branch information. :param version: MediaWiki version (eg '1.23wmf15') :param cfg: Dict of global configuration values :raises: :class:`IOError` if version directory is not found """ branch_dir = os.path.join(cfg['stage_dir'], 'php-%s' % version) if not os.path.isdir(branch_dir): raise IOError(errno.ENOENT, 'Invalid branch directory', branch_dir) # Create cache directory if needed cache_dir = os.path.join(branch_dir, 'cache', 'gitinfo') if not os.path.isdir(cache_dir): os.mkdir(cache_dir) # Create cache for branch info = utils.git_info(branch_dir) cache_file = utils.git_info_filename(branch_dir, branch_dir, cache_dir) with open(cache_file, 'w') as f: json.dump(info, f) # Create cache for each extension and skin for dirname in ['extensions', 'skins']: dir = os.path.join(branch_dir, dirname) for subdir in utils.iterate_subdirectories(dir): try: info = utils.git_info(subdir) except IOError: pass else: cache_file = utils.git_info_filename( subdir, branch_dir, cache_dir) with open(cache_file, 'w') as f: json.dump(info, f) def check_valid_syntax(*paths): """Run php -l in parallel on `paths`; raise CalledProcessError if nonzero exit.""" logger = logging.getLogger('check_php_syntax') quoted_paths = ["'%s'" % x for x in paths] cmd = ( "find %s -name '*.php' -or -name '*.inc' -or -name '*.phtml' " " -or -name '*.php5' | xargs -n1 -P%d -exec php -l >/dev/null" ) % (' '.join(quoted_paths), multiprocessing.cpu_count()) logger.debug('Running command: `%s`', cmd) subprocess.check_call(cmd, shell=True) # Check for anything that isn't a shebang before '), sort_keys=True, indent=4 ).strip('{}\n') - with open(php_file, 'wt') as fp: + tmp_php_file = '%s.tmp' % php_file + try: + os.unlink(tmp_php_file) + except OSError: + pass + + with open(tmp_php_file, 'wt') as fp: fp.write(php_code) fp.flush() os.fsync(fp.fileno()) + + if not os.path.isfile(tmp_php_file): + raise IOError( + errno.ENOENT, 'Failed to create php wikiversions', tmp_php_file) + + os.rename(tmp_php_file, php_file) os.chmod(php_file, 0664) logger.info('Compiled %s to %s', json_file, php_file) def merge_cdb_updates(directory, pool_size, trust_mtime=False, mute=False): """Update l10n CDB files using JSON data. :param directory: L10n cache directory :param pool_size: Number of parallel processes to use :param trust_mtime: Trust file modification time? :param mute: Disable progress indicator """ logger = logging.getLogger('merge_cdb_updates') cache_dir = os.path.realpath(directory) upstream_dir = os.path.join(cache_dir, 'upstream') files = [os.path.splitext(os.path.basename(f))[0] for f in glob.glob('%s/*.json' % upstream_dir)] if not files: logger.warning('Directory %s is empty', upstream_dir) return 0 pool = multiprocessing.Pool(pool_size) updated = 0 if mute: reporter = log.MuteReporter() else: reporter = log.ProgressReporter('l10n merge') reporter.expect(len(files)) reporter.start() for i, result in enumerate(pool.imap_unordered( update_l10n_cdb_wrapper, itertools.izip( itertools.repeat(cache_dir), files, itertools.repeat(trust_mtime))), 1): if result: updated += 1 reporter.add_success() reporter.finish() logger.info('Updated %d CDB files(s) in %s', updated, cache_dir) def purge_l10n_cache(version, cfg): """Purge the localization cache for a given version. :param version: MediaWiki version (eg '1.23wmf15') :param cfg: Dict of global configuration values :raises: :class:`IOError` if l10n cache dirs for the given version are not found """ branch_dir = 'php-%s' % version staged_l10n = os.path.join(cfg['stage_dir'], branch_dir, 'cache/l10n') deployed_l10n = os.path.join(cfg['deploy_dir'], branch_dir, 'cache/l10n') if not os.path.isdir(staged_l10n): raise IOError(errno.ENOENT, 'Invalid l10n dir', staged_l10n) if not os.path.isdir(deployed_l10n): raise IOError(errno.ENOENT, 'Invalid l10n dir', deployed_l10n) # Purge from staging directory locally # Shell is needed on subprocess to allow wildcard expansion # --force option given to rm to ignore missing files subprocess.check_call( 'sudo -u l10nupdate -n -- /bin/rm ' '--recursive --force %s/*' % staged_l10n, shell=True) # Purge from deploy directroy across cluster # --force option given to rm to ignore missing files as before purge = ssh.Job(user=cfg['ssh_user']).role(cfg['dsh_targets']) purge.command('sudo -u mwdeploy -n -- /bin/rm ' '--recursive --force %s/*' % deployed_l10n) purge.progress('l10n purge').run() def sync_common(cfg, include=None, sync_from=None, verbose=False): """Sync local deploy dir with upstream rsync server's copy Rsync from ``server::common`` to the local deploy directory. If a list of servers is given in ``sync_from`` we will attempt to select the "best" one to sync from. If no servers are given or all servers given have issues we will fall back to using the server named by ``master_rsync`` in the configuration data. :param cfg: Dict of global configuration values. :param include: List of rsync include patterns to limit the sync to. If ``None`` is given the entire ``common`` module on the target rsync server will be transferred. Rsync syntax for syncing a directory is ``/***``. :param sync_from: List of rsync servers to fetch from. """ logger = logging.getLogger('sync_common') if not os.path.isdir(cfg['deploy_dir']): raise Exception(( 'rsync target directory %s not found. Ask root to create it ' '(should belong to mwdeploy:mwdeploy).') % cfg['deploy_dir']) server = None if sync_from: server = utils.find_nearest_host(sync_from) if server is None: server = cfg['master_rsync'] server = server.strip() # Execute rsync fetch locally via sudo rsync = ['sudo', '-u', 'mwdeploy', '-n', '--'] + DEFAULT_RSYNC_ARGS if verbose: rsync.append('--verbose') if include: for path in include: rsync.append('--include=/%s' % path) # Exclude everything not explicitly included rsync.append('--exclude=*') rsync.append('%s::common' % server) rsync.append(cfg['deploy_dir']) logger.info('Copying to %s from %s', socket.getfqdn(), server) logger.debug('Running rsync command: `%s`', ' '.join(rsync)) stats = log.Stats(cfg['statsd_host'], int(cfg['statsd_port'])) with log.Timer('rsync common', stats): subprocess.check_call(rsync) # Bug 58618: Invalidate local configuration cache by updating the # timestamp of wmf-config/InitialiseSettings.php settings_path = os.path.join( cfg['deploy_dir'], 'wmf-config', 'InitialiseSettings.php') logger.debug('Touching %s', settings_path) subprocess.check_call(('sudo', '-u', 'mwdeploy', '-n', '--', '/usr/bin/touch', settings_path)) def sync_wikiversions(hosts, cfg): """Rebuild and sync wikiversions.cdb to the cluster. :param hosts: List of hosts to sync to :param cfg: Dict of global configuration values """ stats = log.Stats(cfg['statsd_host'], int(cfg['statsd_port'])) with log.Timer('sync_wikiversions', stats): - compile_wikiversions_cdb('stage', cfg) + compile_wikiversions('stage', cfg) rsync = ssh.Job(hosts, user=cfg['ssh_user']).shuffle() rsync.command('sudo -u mwdeploy -n -- /usr/bin/rsync -l ' '%(master_rsync)s::common/wikiversions*.{json,cdb,php} ' '%(deploy_dir)s' % cfg) return rsync.progress('sync_wikiversions').run() def update_l10n_cdb(cache_dir, cdb_file, trust_mtime=False): """Update a localization CDB database. :param cache_dir: L10n cache directory :param cdb_file: L10n CDB database :param trust_mtime: Trust file modification time? """ logger = logging.getLogger('update_l10n_cdb') md5_path = os.path.join(cache_dir, 'upstream', '%s.MD5' % cdb_file) if not os.path.exists(md5_path): logger.warning('skipped %s; no md5 file', cdb_file) return False json_path = os.path.join(cache_dir, 'upstream', '%s.json' % cdb_file) if not os.path.exists(json_path): logger.warning('skipped %s; no json file', cdb_file) return False cdb_path = os.path.join(cache_dir, cdb_file) json_mtime = os.path.getmtime(json_path) if os.path.exists(cdb_path): if trust_mtime: cdb_mtime = os.path.getmtime(cdb_path) # If the CDB was built by this process in a previous sync, the CDB # file mtime will have been set equal to the json file mtime. need_rebuild = cdb_mtime != json_mtime else: upstream_md5 = open(md5_path).read(100).strip() local_md5 = utils.md5_file(cdb_path) need_rebuild = local_md5 != upstream_md5 else: need_rebuild = True if need_rebuild: with open(json_path) as f: data = json.load(f) # Write temp cdb file tmp_cdb_path = '%s.tmp' % cdb_path with open(tmp_cdb_path, 'wb') as fp: writer = cdblib.Writer(fp) for key, value in data.items(): writer.put(key.encode('utf-8'), value.encode('utf-8')) writer.finalize() os.fsync(fp.fileno()) if not os.path.isfile(tmp_cdb_path): raise IOError(errno.ENOENT, 'Failed to create CDB', tmp_cdb_path) # Move temp file over old file os.chmod(tmp_cdb_path, 0664) os.rename(tmp_cdb_path, cdb_path) # Set timestamp to match upstream json os.utime(cdb_path, (json_mtime, json_mtime)) return True return False def update_l10n_cdb_wrapper(args): """Wrapper for update_l10n_cdb to be used in contexts where only a single argument can be provided. :param args: Sequence of arguments to pass to update_l10n_cdb """ try: return update_l10n_cdb(*args) except: # Log detailed error; multiprocessing will truncate the stack trace logging.getLogger('update_l10n_cdb_wrapper').exception( 'Failure processing %s', args) raise def _call_rebuildLocalisationCache(wikidb, out_dir, use_cores=1, lang=None, force=False, quiet=False): """Helper for update_localization_cache :param wikidb: Wiki running given version :param out_dir: The output directory :param use_cores: The number of cores to run in :param lang: The --lang option, or None to omit :param force: Whether to pass --force :param quiet: Whether to pass --quiet """ logger = logging.getLogger('update_localization_cache') with utils.sudo_temp_dir('www-data', 'scap_l10n_') as temp_dir: # Seed the temporary directory with the current CDB and/or PHP files. # The unspeakable horror that is '[pc][hd][pb]' is a portable way of # matching one or more .cdb / .php files. We only need this # monstronsity during a temporary migration period. Slap Ori if it # is still around after 1-Sep-2015. if glob.glob('%s/*.[pc][hd][pb]' % out_dir): utils.sudo_check_call('www-data', "cp '%(out_dir)s/'*.[pc][hd][pb] '%(temp_dir)s'" % { 'temp_dir': temp_dir, 'out_dir': out_dir }, logger) # Generate the files into a temporary directory as www-data utils.sudo_check_call('www-data', '/usr/local/bin/mwscript rebuildLocalisationCache.php ' '--wiki="%(wikidb)s" --outdir="%(temp_dir)s" ' '--threads=%(use_cores)s %(lang)s %(force)s %(quiet)s' % { 'wikidb': wikidb, 'temp_dir': temp_dir, 'use_cores': use_cores, 'lang': '--lang ' + lang if lang else '', 'force': '--force' if force else '', 'quiet': '--quiet' if quiet else '' }, logger) # Copy the files into the real directory as l10nupdate utils.sudo_check_call('l10nupdate', 'cp -r "%(temp_dir)s"/* "%(out_dir)s"' % { 'temp_dir': temp_dir, 'out_dir': out_dir }, logger) def update_localization_cache(version, wikidb, verbose, cfg): """Update the localization cache for a given MW version. :param version: MediaWiki version :param wikidb: Wiki running given version :param verbose: Provide verbose output :param cfg: Global configuration """ logger = logging.getLogger('update_localization_cache') # Calculate the number of parallel threads # Leave a couple of cores free for other stuff use_cores = max(multiprocessing.cpu_count() - 2, 1) verbose_messagelist = '' force_rebuild = False quiet_rebuild = True if verbose: verbose_messagelist = '--verbose' quiet_rebuild = False extension_messages = os.path.join( cfg['stage_dir'], 'wmf-config', 'ExtensionMessages-%s.php' % version) if not os.path.exists(extension_messages): # Touch the extension_messages file to prevent php require errors logger.info('Creating empty %s', extension_messages) open(extension_messages, 'a').close() cache_dir = os.path.join( cfg['stage_dir'], 'php-%s' % version, 'cache', 'l10n') if not os.path.exists(os.path.join(cache_dir, 'l10n_cache-en.cdb')): # mergeMessageFileList.php needs a l10n file logger.info('Bootstrapping l10n cache for %s', version) _call_rebuildLocalisationCache(wikidb, cache_dir, use_cores, lang='en', quiet=True) # Force subsequent cache rebuild to overwrite bootstrap version force_rebuild = True logger.info('Updating ExtensionMessages-%s.php', version) new_extension_messages = subprocess.check_output( 'sudo -u www-data -n -- /bin/mktemp', shell=True).strip() utils.sudo_check_call('www-data', '/usr/local/bin/mwscript mergeMessageFileList.php ' '--wiki="%s" --list-file="%s/wmf-config/extension-list" ' '--output="%s" %s' % ( wikidb, cfg['stage_dir'], new_extension_messages, verbose_messagelist), logger) utils.sudo_check_call('www-data', 'chmod 0664 "%s"' % new_extension_messages, logger) logger.debug('Copying %s to %s' % ( new_extension_messages, extension_messages)) shutil.copyfile(new_extension_messages, extension_messages) utils.sudo_check_call('www-data', 'rm "%s"' % new_extension_messages, logger) # Update ExtensionMessages-*.php in the local copy. deploy_dir = os.path.realpath(cfg['deploy_dir']) stage_dir = os.path.realpath(cfg['stage_dir']) if stage_dir != deploy_dir: logger.debug('Copying ExtensionMessages-*.php to local copy') utils.sudo_check_call('mwdeploy', 'cp "%s" "%s/wmf-config/"' % ( extension_messages, cfg['deploy_dir']), logger) # Rebuild all the CDB files for each language logger.info('Updating LocalisationCache for %s ' 'using %s thread(s)' % (version, use_cores)) _call_rebuildLocalisationCache(wikidb, cache_dir, use_cores, force=force_rebuild, quiet=quiet_rebuild) # Include JSON versions of the CDB files and add MD5 files logger.info('Generating JSON versions and md5 files') utils.sudo_check_call('l10nupdate', '/usr/local/bin/refreshCdbJsonFiles ' '--directory="%s" --threads=%s %s' % ( cache_dir, use_cores, verbose_messagelist), logger) def restart_hhvm(hosts, cfg, batch_size=1): """Restart HHVM on the given hosts. :param hosts: List of hosts to sync to :param cfg: Dict of global configuration values :param batch_size: Number of hosts to restart in parallel """ stats = log.Stats(cfg['statsd_host'], int(cfg['statsd_port'])) with log.Timer('restart_hhvm', stats): restart = ssh.Job(hosts, user=cfg['ssh_user']).shuffle() restart.command('sudo -u mwdeploy -n -- %s' % os.path.join(cfg['bin_dir'], 'scap-hhvm-restart')) return restart.progress('restart_hhvm').run(batch_size=batch_size)