From 629395f511a59aaf8bb242c3151d996a500414ce Mon Sep 17 00:00:00 2001 From: Erik Bernhardson Date: Thu, 9 May 2013 11:50:05 -0700 Subject: [PATCH] Database updates for respecting oversight within Echo Provides the first step of adding and populating a new database field for Echo oversight deployment. The new field is populated via a maintenance script and Event::loadFromRow will accept both new and old results. Everything will still run when the 2 now unused fields are later dropped from the db. Bug: 48059 Change-Id: I24d4b61a061f94ed9aaaa6087f33b2ab37f773cd --- Hooks.php | 4 +- db_patches/patch-add-echo_event-event_page_id.sql | 1 + echo.sql | 3 +- formatters/PageLinkFormatter.php | 18 +++- includes/DbEchoBackend.php | 6 +- maintenance/updatePageIdFromTitle.php | 108 +++++++++++++++++++++ maintenance/updatePageLinkedExtraData.php | 100 +++++++++++++++++++ model/Event.php | 26 ++++- 8 files changed, 249 insertions(+), 17 deletions(-) create mode 100644 db_patches/patch-add-echo_event-event_page_id.sql mode change 100644 => 100755 echo.sql create mode 100644 maintenance/updatePageIdFromTitle.php create mode 100644 maintenance/updatePageLinkedExtraData.php diff --git a/Hooks.php b/Hooks.php index 0f65134..a42bbbf 100644 --- a/Hooks.php +++ b/Hooks.php @@ -84,6 +84,7 @@ class EchoHooks { $updater->dropExtensionField( 'echo_event', 'event_timestamp', "$dir/db_patches/patch-drop-echo_event-event_timestamp.sql" ); $updater->addExtensionField( 'echo_email_batch', 'eeb_event_hash', "$dir/db_patches/patch-email_batch-new-field.sql" ); + $updater->addExtensionField( 'echo_event', 'event_page_id', "$dir/db_patches/patch-add-echo_event-event_page_id.sql" ); return true; } @@ -562,8 +563,7 @@ class EchoHooks { 'title' => $title, 'agent' => $wgUser, 'extra' => array( - 'link-from-namespace' => $linksUpdate->mTitle->getNamespace(), - 'link-from-title' => $linksUpdate->mTitle->getDBkey(), + 'link-from-page-id' => $linksUpdate->mTitle->getArticleId(), ) ) ); $max--; diff --git a/db_patches/patch-add-echo_event-event_page_id.sql b/db_patches/patch-add-echo_event-event_page_id.sql new file mode 100644 index 0000000..3c4ae71 --- /dev/null +++ b/db_patches/patch-add-echo_event-event_page_id.sql @@ -0,0 +1 @@ +ALTER TABLE /*_*/echo_event ADD event_page_id int unsigned; diff --git a/echo.sql b/echo.sql old mode 100644 new mode 100755 index eb90f68..771ace5 --- a/echo.sql +++ b/echo.sql @@ -8,7 +8,8 @@ CREATE TABLE /*_*/echo_event ( event_agent_ip varchar(39) binary null, -- IP address who triggered it, if any event_page_namespace int unsigned null, event_page_title varchar(255) binary null, - event_extra BLOB NULL + event_extra BLOB NULL, + event_page_id int unsigned null ) /*$wgDBTableOptions*/; CREATE INDEX /*i*/event_type ON /*_*/echo_event (event_type); diff --git a/formatters/PageLinkFormatter.php b/formatters/PageLinkFormatter.php index 25c5b46..be00a5e 100644 --- a/formatters/PageLinkFormatter.php +++ b/formatters/PageLinkFormatter.php @@ -63,7 +63,7 @@ class EchoPageLinkFormatter extends EchoBasicFormatter { * @return bool */ private function isTitleSet( $extra ) { - if ( isset( $extra['link-from-namespace'], $extra['link-from-title'] ) ) { + if ( isset( $extra['link-from-namespace'], $extra['link-from-title'] ) || isset( $extra['link-from-page-id'] ) ) { return true; } else { return false; @@ -76,6 +76,11 @@ class EchoPageLinkFormatter extends EchoBasicFormatter { * @return string */ private function getTitleHash( $extra ) { + if ( isset( $extra['link-from-page-id'] ) ) { + $title = Title::newFromId( $extra['link-from-page-id'] ); + return md5( $title->getNamespace() . '-' . $title->getDBkey() ); + } + return md5( $extra['link-from-namespace'] . '-' . $extra['link-from-title'] ); } @@ -91,12 +96,15 @@ class EchoPageLinkFormatter extends EchoBasicFormatter { // 'A' part in this message: link from page A and X others case 'link-from-page': if ( $this->isTitleSet( $extra ) ) { - $message->params( - Title::makeTitle( + if ( isset( $extra['link-from-page-id'] ) ) { + $title = Title::newFromId( $extra['link-from-page-id'] ); + } else { + $title = Title::makeTitle( $extra['link-from-namespace'], $extra['link-from-title'] - ) - ); + ); + } + $message->params( $title ); } else { $message->params( '' ); } diff --git a/includes/DbEchoBackend.php b/includes/DbEchoBackend.php index 8936902..3a9db17 100644 --- a/includes/DbEchoBackend.php +++ b/includes/DbEchoBackend.php @@ -98,8 +98,7 @@ class MWDbEchoBackend extends MWEchoBackend { 'event_agent_id', 'event_agent_ip', 'event_extra', - 'event_page_namespace', - 'event_page_title' + 'event_page_id' ), array( 'notification_event=event_id', @@ -118,8 +117,7 @@ class MWDbEchoBackend extends MWEchoBackend { 'event_agent_id', 'event_agent_ip', 'event_extra', - 'event_page_namespace', - 'event_page_title' + 'event_page_id' ), array( 'eeb_event_id=event_id', diff --git a/maintenance/updatePageIdFromTitle.php b/maintenance/updatePageIdFromTitle.php new file mode 100644 index 0000000..a35f29b --- /dev/null +++ b/maintenance/updatePageIdFromTitle.php @@ -0,0 +1,108 @@ +batchSize; + $maxEventId = null; + + while ( $count === $this->batchSize ) { + $conditions = array( + 'event_page_title IS NOT NULL', + 'event_page_id IS NULL', + ); + if ( $maxEventId !== null ) { + $conditions[] = "event_id > $maxEventId"; + } + $res = $dbr->select( + 'echo_event', + array( 'event_id', 'event_page_namespace', 'event_page_title', 'event_extra' ), + $conditions, + __METHOD__, + array( + 'LIMIT' => $this->batchSize, + 'ORDER BY event_id ASC', + ) + ); + + $event = array(); + $count = 0; + foreach ( $res as $row ) { + $count++; + $title = Title::newFromText( $row->event_page_title, $row->event_page_namespace ); + if ( $title === null ) { + // unexpected, but not fatal. I suppose this means an invalid title was + // somehow stored to the event. Continue will leave event_page_id null which is + // appropriate for this case. + continue; + } + $pageId = $title->getArticleId(); + if ( $pageId ) { + // If the title has a proper id from the database, store it + $update = array( + 'event_page_id' => $pageId, + ); + } else { + // For titles that do not refer to a WikiPage stored in the database + // move the title/namespace into event_extra + $extra = $row->event_extra ? unserialize( $row->event_extra ) : array(); + $extra['page_title'] = $row->event_page_title; + $extra['page_namespace'] = $row->event_page_namespace; + + $update = array( + 'event_extra' => serialize( $extra ), + ); + } + // They should come to us in order anyways, but this is safe + if ( $maxEventId === null || $row->event_id > $maxEventId ) { + $maxEventId = $row->event_id; + } + $event[$row->event_id] = $update; + } + + if ( $event ) { + $dbw->begin(); + + foreach ( $event as $id => $update ) { + $dbw->update( + 'echo_event', + $update, + array( 'event_id' => $id ), + __METHOD__ + ); + } + + $dbw->commit(); + $this->output( "Processing $count events\n" ); + wfWaitForSlaves( false, false, $wgEchoCluster ); + } + } + + $this->output( "Completed\n" ); + } + +} + +$maintClass = 'updatePageIdFromTitle'; // Tells it to run the class +require_once( RUN_MAINTENANCE_IF_MAIN ); diff --git a/maintenance/updatePageLinkedExtraData.php b/maintenance/updatePageLinkedExtraData.php new file mode 100644 index 0000000..f0a3133 --- /dev/null +++ b/maintenance/updatePageLinkedExtraData.php @@ -0,0 +1,100 @@ + link-from-page-id + * + * @ingroup Maintenance + */ +class updatePageLinkedExtraData extends Maintenance { + + protected $batchSize = 500; + + public function execute() { + global $wgEchoCluster; + + $dbw = MWEchoDbFactory::getDB( DB_MASTER ); + $dbr = MWEchoDbFactory::getDB( DB_SLAVE ); + + $count = $this->batchSize; + $maxEventId = null; + + while ( $count === $this->batchSize ) { + $conditions = array( + 'event_extra IS NOT NULL', + "event_type = 'page-linked'", + ); + if ( $maxEventId !== null ) { + $conditions[] = "event_id > $maxEventId"; + } + $res = $dbr->select( + 'echo_event', + array( 'event_id', 'event_extra' ), + $conditions, + __METHOD__, + array( + 'LIMIT' => $this->batchSize, + 'ORDER BY event_id ASC', + ) + ); + + $event = array(); + $count = 0; + foreach ( $res as $row ) { + $count++; + $extra = unserialize( $row->event_extra ); + if ( ! isset( $extra['link-from-title'], $extra['link-from-namespace'] ) ) { + continue; + } + + $title = Title::newFromText( $extra['link-from-title'], $extra['link-from-namespace'] ); + unset( $extra['link-from-title'], $extra['link-from-namespace'] ); + + // Link from page is always from a content page, if null or no article id it was + // somehow invalid. + if ( $title !== null && $title->getArticleId() ) { + $extra['link-from-page-id'] = $title->getArticleId(); + } + + // They should come to us in order anyways, but this is safe + if ( $maxEventId === null || $row->event_id > $maxEventId ) { + $maxEventId = $row->event_id; + } + $event[$row->event_id] = array( + 'event_extra' => serialize( $extra ), + ); + } + + if ( $event ) { + $dbw->begin(); + + foreach ( $event as $id => $update ) { + $dbw->update( + 'echo_event', + $update, + array( 'event_id' => $id ), + __METHOD__ + ); + } + + $dbw->commit(); + $this->output( "Processing $count events\n" ); + wfWaitForSlaves( false, false, $wgEchoCluster ); + } + } + + $this->output( "Completed\n" ); + } + +} + +$maintClass = 'updatePageLinkedExtraData'; // Tells it to run the class +require_once( RUN_MAINTENANCE_IF_MAIN ); diff --git a/model/Event.php b/model/Event.php index 669cf07..d25e6c3 100644 --- a/model/Event.php +++ b/model/Event.php @@ -140,8 +140,6 @@ class EchoEvent { 'event_variant' => $this->variant, ); - $row['event_extra'] = $this->serializeExtra(); - if ( $this->agent ) { if ( $this->agent->isAnon() ) { $row['event_agent_ip'] = $this->agent->getName(); @@ -151,10 +149,20 @@ class EchoEvent { } if ( $this->title ) { - $row['event_page_namespace'] = $this->title->getNamespace(); - $row['event_page_title'] = $this->title->getDBkey(); + $pageId = $this->title->getArticleId(); + if ( $pageId ) { + $row['event_page_id'] = $pageId; + } else { + if ( $this->extra === null ) { + $this->extra = array(); + } + $this->extra['page_namespace'] = $this->title->getNamespace(); + $this->extra['page_title'] = $this->title->getDBkey(); + } } + $row['event_extra'] = $this->serializeExtra(); + $this->id = $wgEchoBackend->createEvent( $row ); } @@ -185,11 +193,19 @@ class EchoEvent { $this->agent = User::newFromName( $row->event_agent_ip, false ); } - if ( $row->event_page_title !== null ) { + if ( $row->event_page_id !== null ) { + $this->title = Title::newFromId( $row->event_page_id ); + } elseif ( isset( $row->event_page_title ) ) { + // BC compat with orig Echo deployment $this->title = Title::makeTitleSafe( $row->event_page_namespace, $row->event_page_title ); + } elseif ( isset( $this->extra['page_title'] ) ) { + $this->title = Title::makeTitleSafe( + $this->extra['page_title'], + $this->extra['page_namespace'] + ); } } -- 1.7.9.5