diff --git a/bin/sync-l10n b/bin/sync-l10n new file mode 100755 --- /dev/null +++ b/bin/sync-l10n @@ -0,0 +1,17 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Synchronize the localization files for a MediaWiki branch +# +# Copyright © 2015 Wikimedia Foundation and contributors + +import os +import sys + +# Add scap package to search path +script = os.path.realpath(sys.argv[0]) +scap_src = os.path.dirname(os.path.dirname(script)) +sys.path.append(scap_src) + +import scap +scap.SyncL10n.run() diff --git a/docs/scap2/commands.rst b/docs/scap2/commands.rst --- a/docs/scap2/commands.rst +++ b/docs/scap2/commands.rst @@ -61,6 +61,15 @@ * :func:`scap.SyncFile` +sync-l10n +--------- +:command:`sync-l10n` synchronizes the localization files for a given +MediaWiki version to the cluster and rebuilds the associated cache files. + +.. program-output:: ../bin/sync-l10n --help +.. seealso:: + * :func:`scap.SyncL10n` + sync-wikiversions ----------------- :command:`sync-wikiversions` compiles wikiversions.json into a CDB database and then diff --git a/scap/__init__.py b/scap/__init__.py --- a/scap/__init__.py +++ b/scap/__init__.py @@ -18,6 +18,7 @@ SyncDir, SyncDocroot, SyncFile, + SyncL10n, SyncMaster, SyncWikiversions, UpdateL10n, @@ -44,6 +45,7 @@ 'SyncDir', 'SyncDocroot', 'SyncFile', + 'SyncL10n', 'SyncMaster', 'SyncWikiversions', 'UpdateL10n', @@ -65,5 +67,6 @@ SyncDocroot, SyncFile, SyncMaster, + SyncL10n, SyncWikiversions, UpdateL10n)) # Ignore unused import warning diff --git a/scap/main.py b/scap/main.py --- a/scap/main.py +++ b/scap/main.py @@ -214,6 +214,7 @@ class RebuildCdbs(cli.Application): """Rebuild localization cache CDB files from the JSON versions.""" + @cli.argument('--version', help='MediaWiki version (eg 1.27.0-wmf.7)') @cli.argument('--no-progress', action='store_true', dest='mute', help='Do not show progress indicator.') @cli.argument('--staging', action='store_true', @@ -234,8 +235,23 @@ # Leave some of the cores free for apache processes use_cores = max(multiprocessing.cpu_count() / 2, 1) + self.versions = self.active_wikiversions(source_tree) + + if self.arguments.version: + version = self.arguments.version + if version.startswith('php-'): + version = version[4:] + + # Assert version is active + if version not in self.versions: + raise IOError( + errno.ENOENT, 'Version not active', version) + + # Replace dict of active versions with the single version selected + self.versions = {version: self.versions[version]} + # Rebuild the CDB files from the JSON versions - for version, wikidb in self.active_wikiversions(source_tree).items(): + for version, wikidb in self.versions.items(): cache_dir = os.path.join(root_dir, 'php-%s' % version, 'cache', 'l10n') tasks.merge_cdb_updates( @@ -508,6 +524,72 @@ self.get_stats().increment('deploy.all') +class SyncL10n(AbstractSync): + """Sync l10n files for a given branch and rebuild cache files.""" + + @cli.argument('version', help='MediaWiki version (eg 1.27.0-wmf.7)') + def main(self, *extra_args): + return super(SyncL10n, self).main(*extra_args) + + def _before_cluster_sync(self): + if self.arguments.version.startswith('php-'): + self.arguments.version = self.arguments.version[4:] + + # Assert version is active + if self.arguments.version not in self.active_wikiversions(): + raise IOError( + errno.ENOENT, 'Version not active', self.arguments.version) + + # Assert l10n cache dir for version exists + abspath = os.path.join( + self.config['stage_dir'], + 'php-%s/cache/l10n' % self.arguments.version) + if not os.path.isdir(abspath): + raise IOError(errno.ENOENT, 'Directory not found', abspath) + + self._check_sync_flag(abspath) + + relpath = os.path.relpath(abspath, self.config['stage_dir']) + self.include = '%s/***' % relpath + + def _proxy_sync_command(self): + cmd = [self.get_script_path('sync-common'), '--no-update-l10n'] + + parts = self.include.split('/') + for i in range(1, len(parts)): + # Include parent directories in sync command or the default + # exclude will block them and by extension block the target + # file. + cmd.extend(['--include', '/'.join(parts[:i])]) + + cmd.extend(['--include', self.include]) + if self.verbose: + cmd.append('--verbose') + return cmd + + def _after_cluster_sync(self): + # Rebuild l10n CDB files + target_hosts = self._get_target_list() + with log.Timer('scap-rebuild-cdbs', self.get_stats()): + rebuild_cdbs = ssh.Job(target_hosts, user=self.config['ssh_user']) + rebuild_cdbs.shuffle() + rebuild_cdbs.command('sudo -u mwdeploy -n -- %s --version %s' % ( + self.get_script_path('scap-rebuild-cdbs'), + self.arguments.version)) + rebuild_cdbs.progress('scap-rebuild-cdbs') + succeeded, failed = rebuild_cdbs.run() + if failed: + self.get_logger().warning( + '%d hosts had scap-rebuild-cdbs errors', failed) + self.soft_errors = True + + def _after_lock_release(self): + self.announce('sync-l10nupdate completed (%s) (duration: %s)', + self.arguments.version, + utils.human_duration(self.get_duration())) + self.get_stats().increment('l10nupdate-sync') + + class SyncWikiversions(AbstractSync): """Rebuild and sync wikiversions.php to the cluster."""