@timestamp	2017-02-22T00:27:55
t@version	1
t_id	AVpjN6lKlL3ktRIGMTlF
t_index	logstash-2017.02.22
#_score	1
t_type	mediawiki
?backtrace	  {
  "file": "/srv/mediawiki/php-1.29.0-wmf.12/includes/libs/rdbms/lbfactory/LBFactory.php",
  "line": 413,
  "function": "waitForReplication",
  "class": "Wikimedia\\Rdbms\\LBFactory",
  "type": "->",
  "args": [
    "array"
  ]
},
{
  "file": "/srv/mediawiki/php-1.29.0-wmf.12/extensions/GeoData/includes/Hooks.php",
  "line": 197,
  "function": "commitAndWaitForReplication",
  "class": "Wikimedia\\Rdbms\\LBFactory",
  "type": "->",
  "args": [
    "string",
    "integer"
  ]
},
{
  "file": "/srv/mediawiki/php-1.29.0-wmf.12/extensions/GeoData/includes/Hooks.php",
  "line": 126,
  "function": "doLinksUpdate",
  "class": "GeoData\\Hooks",
  "type": "::",
  "args": [
    "array",
    "integer",
    "integer"
  ]
},
{
  "file": "/srv/mediawiki/php-1.29.0-wmf.12/includes/Hooks.php",
  "line": 186,
  "function": "onLinksUpdateComplete",
  "class": "GeoData\\Hooks",
  "type": "::",
  "args": [
    "LinksUpdate",
    "integer"
  ]
},
{
  "file": "/srv/mediawiki/php-1.29.0-wmf.12/includes/deferred/LinksUpdate.php",
  "line": 186,
  "function": "run",
  "class": "Hooks",
  "type": "::",
  "args": [
    "string",
    "array"
  ]
},
{
  "file": "/srv/mediawiki/php-1.29.0-wmf.12/includes/libs/rdbms/database/Database.php",
  "line": 2558,
  "function": "Closure$LinksUpdate::doUpdate",
  "args": [
    "integer"
  ]
},
{
  "file": "/srv/mediawiki/php-1.29.0-wmf.12/includes/libs/rdbms/loadbalancer/LoadBalancer.php",
  "line": 1197,
  "function": "runOnTransactionIdleCallbacks",
  "class": "Database",
  "type": "->",
  "args": [
    "integer"
  ]
},
{
  "file": "/srv/mediawiki/php-1.29.0-wmf.12/includes/libs/rdbms/loadbalancer/LoadBalancer.php",
  "line": 1424,
  "function": "Closure$LoadBalancer::runMasterPostTrxCallbacks",
  "args": [
    "DatabaseMysqli"
  ]
},
{
  "file": "/srv/mediawiki/php-1.29.0-wmf.12/includes/libs/rdbms/loadbalancer/LoadBalancer.php",
  "line": 1206,
  "function": "forEachOpenMasterConnection",
  "class": "LoadBalancer",
  "type": "->",
  "args": [
    "Closure$LoadBalancer::runMasterPostTrxCallbacks;1850198745"
  ]
},
{
  "file": "/srv/mediawiki/php-1.29.0-wmf.12/includes/libs/rdbms/lbfactory/LBFactory.php",
  "line": 236,
  "function": "runMasterPostTrxCallbacks",
  "class": "LoadBalancer",
  "type": "->",
  "args": [
    "integer"
  ]
},
{
  "function": "Closure$Wikimedia\\Rdbms\\LBFactory::commitMasterChanges",
  "args": [
    "LoadBalancer"
  ]
},
{
  "file": "/srv/mediawiki/php-1.29.0-wmf.12/includes/libs/rdbms/lbfactory/LBFactoryMulti.php",
  "line": 422,
  "function": "call_user_func_array",
  "args": [
    "Closure$Wikimedia\\Rdbms\\LBFactory::commitMasterChanges;1404996366",
    "array"
  ]
},
{
  "file": "/srv/mediawiki/php-1.29.0-wmf.12/includes/libs/rdbms/lbfactory/LBFactory.php",
  "line": 238,
  "function": "forEachLB",
  "class": "Wikimedia\\Rdbms\\LBFactoryMulti",
  "type": "->",
  "args": [
    "Closure$Wikimedia\\Rdbms\\LBFactory::commitMasterChanges;1404996366"
  ]
},
{
  "file": "/srv/mediawiki/php-1.29.0-wmf.12/includes/jobqueue/JobRunner.php",
  "line": 539,
  "function": "commitMasterChanges",
  "class": "Wikimedia\\Rdbms\\LBFactory",
  "type": "->",
  "args": [
    "string"
  ]
},
{
  "file": "/srv/mediawiki/php-1.29.0-wmf.12/includes/jobqueue/JobRunner.php",
  "line": 289,
  "function": "commitMasterChanges",
  "class": "JobRunner",
  "type": "->",
  "args": [
    "Wikimedia\\Rdbms\\LBFactoryMulti",
    "RefreshLinksJob",
    "string"
  ]
},
{
  "file": "/srv/mediawiki/php-1.29.0-wmf.12/includes/jobqueue/JobRunner.php",
  "line": 187,
  "function": "executeJob",
  "class": "JobRunner",
  "type": "->",
  "args": [
    "RefreshLinksJob",
    "Wikimedia\\Rdbms\\LBFactoryMulti",
    "BufferingStatsdDataFactory",
    "integer"
  ]
},
{
  "file": "/srv/mediawiki/rpc/RunJobs.php",
  "line": 47,
  "function": "run",
  "class": "JobRunner",
  "type": "->",
  "args": [
    "array"
  ]
}
?caught_by	  other
tchannel	exception
tclass	mediawiki
#code	0
texception_id	752438cac75f3bd4fc10f1d9
tfile	/srv/mediawiki/php-1.29.0-wmf.12/includes/libs/rdbms/lbfactory/LBFactory.php
thost	mw1162
thttp_method	POST
tip	127.0.0.1
tlevel	ERROR
#line	372
tmessage	Could not wait for replica DBs to catch up to db1041
tmessage_checksum	4a89fb665bca11c28756a62656353ec5
tmwversion	1.29.0-wmf.12
tnormalized_message	{"id":"752438cac75f3bd4fc10f1d9","type":"DBReplicationWaitError","file":"/srv/mediawiki/php-1.29.0-wmf.12/includes/libs/rdbms/lbfactory/LBFactory.php","line":372,"message":"Could not wait for replica DBs to catch up to db1041","code":0,"url":"/rpc/RunJobs
private	true
treferrer	 - 
treqId	752438cac75f3bd4fc10f1d9
tserver	127.0.0.1
ttags	syslog, es, es, exception-json
ttype	mediawiki
turl	/rpc/RunJobs.php?wiki=svwiki&type=refreshLinks&maxtime=30&maxmem=300M
twiki	svwiki

{
  "_index": "logstash-2017.02.22",
  "_type": "mediawiki",
  "_id": "AVpjN6lKlL3ktRIGMTlF",
  "_score": 1,
  "_source": {
    "message": "Could not wait for replica DBs to catch up to db1041",
    "@version": 1,
    "@timestamp": "2017-02-22T00:27:55.000Z",
    "type": "mediawiki",
    "host": "mw1162",
    "level": "ERROR",
    "tags": [
      "syslog",
      "es",
      "es",
      "exception-json"
    ],
    "channel": "exception",
    "normalized_message": "{\"id\":\"752438cac75f3bd4fc10f1d9\",\"type\":\"DBReplicationWaitError\",\"file\":\"/srv/mediawiki/php-1.29.0-wmf.12/includes/libs/rdbms/lbfactory/LBFactory.php\",\"line\":372,\"message\":\"Could not wait for replica DBs to catch up to db1041\",\"code\":0,\"url\":\"/rpc/RunJobs",
    "url": "/rpc/RunJobs.php?wiki=svwiki&type=refreshLinks&maxtime=30&maxmem=300M",
    "ip": "127.0.0.1",
    "http_method": "POST",
    "server": "127.0.0.1",
    "referrer": null,
    "wiki": "svwiki",
    "mwversion": "1.29.0-wmf.12",
    "reqId": "752438cac75f3bd4fc10f1d9",
    "private": true,
    "file": "/srv/mediawiki/php-1.29.0-wmf.12/includes/libs/rdbms/lbfactory/LBFactory.php",
    "line": 372,
    "code": 0,
    "caught_by": "other",
    "backtrace": [
      {
        "file": "/srv/mediawiki/php-1.29.0-wmf.12/includes/libs/rdbms/lbfactory/LBFactory.php",
        "line": 413,
        "function": "waitForReplication",
        "class": "Wikimedia\\Rdbms\\LBFactory",
        "type": "->",
        "args": [
          "array"
        ]
      },
      {
        "file": "/srv/mediawiki/php-1.29.0-wmf.12/extensions/GeoData/includes/Hooks.php",
        "line": 197,
        "function": "commitAndWaitForReplication",
        "class": "Wikimedia\\Rdbms\\LBFactory",
        "type": "->",
        "args": [
          "string",
          "integer"
        ]
      },
      {
        "file": "/srv/mediawiki/php-1.29.0-wmf.12/extensions/GeoData/includes/Hooks.php",
        "line": 126,
        "function": "doLinksUpdate",
        "class": "GeoData\\Hooks",
        "type": "::",
        "args": [
          "array",
          "integer",
          "integer"
        ]
      },
      {
        "file": "/srv/mediawiki/php-1.29.0-wmf.12/includes/Hooks.php",
        "line": 186,
        "function": "onLinksUpdateComplete",
        "class": "GeoData\\Hooks",
        "type": "::",
        "args": [
          "LinksUpdate",
          "integer"
        ]
      },
      {
        "file": "/srv/mediawiki/php-1.29.0-wmf.12/includes/deferred/LinksUpdate.php",
        "line": 186,
        "function": "run",
        "class": "Hooks",
        "type": "::",
        "args": [
          "string",
          "array"
        ]
      },
      {
        "file": "/srv/mediawiki/php-1.29.0-wmf.12/includes/libs/rdbms/database/Database.php",
        "line": 2558,
        "function": "Closure$LinksUpdate::doUpdate",
        "args": [
          "integer"
        ]
      },
      {
        "file": "/srv/mediawiki/php-1.29.0-wmf.12/includes/libs/rdbms/loadbalancer/LoadBalancer.php",
        "line": 1197,
        "function": "runOnTransactionIdleCallbacks",
        "class": "Database",
        "type": "->",
        "args": [
          "integer"
        ]
      },
      {
        "file": "/srv/mediawiki/php-1.29.0-wmf.12/includes/libs/rdbms/loadbalancer/LoadBalancer.php",
        "line": 1424,
        "function": "Closure$LoadBalancer::runMasterPostTrxCallbacks",
        "args": [
          "DatabaseMysqli"
        ]
      },
      {
        "file": "/srv/mediawiki/php-1.29.0-wmf.12/includes/libs/rdbms/loadbalancer/LoadBalancer.php",
        "line": 1206,
        "function": "forEachOpenMasterConnection",
        "class": "LoadBalancer",
        "type": "->",
        "args": [
          "Closure$LoadBalancer::runMasterPostTrxCallbacks;1850198745"
        ]
      },
      {
        "file": "/srv/mediawiki/php-1.29.0-wmf.12/includes/libs/rdbms/lbfactory/LBFactory.php",
        "line": 236,
        "function": "runMasterPostTrxCallbacks",
        "class": "LoadBalancer",
        "type": "->",
        "args": [
          "integer"
        ]
      },
      {
        "function": "Closure$Wikimedia\\Rdbms\\LBFactory::commitMasterChanges",
        "args": [
          "LoadBalancer"
        ]
      },
      {
        "file": "/srv/mediawiki/php-1.29.0-wmf.12/includes/libs/rdbms/lbfactory/LBFactoryMulti.php",
        "line": 422,
        "function": "call_user_func_array",
        "args": [
          "Closure$Wikimedia\\Rdbms\\LBFactory::commitMasterChanges;1404996366",
          "array"
        ]
      },
      {
        "file": "/srv/mediawiki/php-1.29.0-wmf.12/includes/libs/rdbms/lbfactory/LBFactory.php",
        "line": 238,
        "function": "forEachLB",
        "class": "Wikimedia\\Rdbms\\LBFactoryMulti",
        "type": "->",
        "args": [
          "Closure$Wikimedia\\Rdbms\\LBFactory::commitMasterChanges;1404996366"
        ]
      },
      {
        "file": "/srv/mediawiki/php-1.29.0-wmf.12/includes/jobqueue/JobRunner.php",
        "line": 539,
        "function": "commitMasterChanges",
        "class": "Wikimedia\\Rdbms\\LBFactory",
        "type": "->",
        "args": [
          "string"
        ]
      },
      {
        "file": "/srv/mediawiki/php-1.29.0-wmf.12/includes/jobqueue/JobRunner.php",
        "line": 289,
        "function": "commitMasterChanges",
        "class": "JobRunner",
        "type": "->",
        "args": [
          "Wikimedia\\Rdbms\\LBFactoryMulti",
          "RefreshLinksJob",
          "string"
        ]
      },
      {
        "file": "/srv/mediawiki/php-1.29.0-wmf.12/includes/jobqueue/JobRunner.php",
        "line": 187,
        "function": "executeJob",
        "class": "JobRunner",
        "type": "->",
        "args": [
          "RefreshLinksJob",
          "Wikimedia\\Rdbms\\LBFactoryMulti",
          "BufferingStatsdDataFactory",
          "integer"
        ]
      },
      {
        "file": "/srv/mediawiki/rpc/RunJobs.php",
        "line": 47,
        "function": "run",
        "class": "JobRunner",
        "type": "->",
        "args": [
          "array"
        ]
      }
    ],
    "exception_id": "752438cac75f3bd4fc10f1d9",
    "class": "mediawiki",
    "message_checksum": "4a89fb665bca11c28756a62656353ec5"
  },
  "fields": {
    "@timestamp": [
      1487723275000
    ]
  }
}