2019-07-02 16:48:26,748 DEBUG zuul.stack_dump: Thread: 139978193487616 File "/usr/lib/python2.7/threading.py", line 783, in __bootstrap self.__bootstrap_inner() File "/usr/lib/python2.7/threading.py", line 810, in __bootstrap_inner self.run() File "/usr/lib/python2.7/threading.py", line 763, in run self.__target(*self.__args, **self.__kwargs) File "/usr/lib/python2.7/dist-packages/paste/httpserver.py", line 866, in worker_thread_callback runnable = self.queue.get() File "/usr/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/lib/python2.7/threading.py", line 340, in wait waiter.acquire() Thread: 139978185094912 File "/usr/lib/python2.7/threading.py", line 783, in __bootstrap self.__bootstrap_inner() File "/usr/lib/python2.7/threading.py", line 810, in __bootstrap_inner self.run() File "/usr/lib/python2.7/threading.py", line 763, in run self.__target(*self.__args, **self.__kwargs) File "/usr/lib/python2.7/dist-packages/paste/httpserver.py", line 866, in worker_thread_callback runnable = self.queue.get() File "/usr/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/lib/python2.7/threading.py", line 340, in wait waiter.acquire() Thread: 139977681794816 File "/usr/lib/python2.7/threading.py", line 783, in __bootstrap self.__bootstrap_inner() File "/usr/lib/python2.7/threading.py", line 810, in __bootstrap_inner self.run() File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/connection/gerrit.py", line 128, in run self._handleEvent() File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/connection/gerrit.py", line 116, in _handleEvent event.change_number, event.patch_number, refresh=True) File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/source/gerrit.py", line 173, in _getChange self._updateChange(change, history) File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/source/gerrit.py", line 345, in _updateChange dep = self._getChange(dep_num, dep_ps, refresh=True) File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/source/gerrit.py", line 173, in _getChange self._updateChange(change, history) File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/source/gerrit.py", line 345, in _updateChange dep = self._getChange(dep_num, dep_ps, refresh=True) File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/source/gerrit.py", line 173, in _getChange self._updateChange(change, history) File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/source/gerrit.py", line 345, in _updateChange dep = self._getChange(dep_num, dep_ps, refresh=True) File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/source/gerrit.py", line 173, in _getChange self._updateChange(change, history) File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/source/gerrit.py", line 345, in _updateChange dep = self._getChange(dep_num, dep_ps, refresh=True) File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/source/gerrit.py", line 173, in _getChange self._updateChange(change, history) File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/source/gerrit.py", line 345, in _updateChange dep = self._getChange(dep_num, dep_ps, refresh=True) File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/source/gerrit.py", line 173, in _getChange self._updateChange(change, history) File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/source/gerrit.py", line 345, in _updateChange dep = self._getChange(dep_num, dep_ps, refresh=True) File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/source/gerrit.py", line 173, in _getChange self._updateChange(change, history) File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/source/gerrit.py", line 345, in _updateChange dep = self._getChange(dep_num, dep_ps, refresh=True) File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/source/gerrit.py", line 173, in _getChange self._updateChange(change, history) File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/source/gerrit.py", line 345, in _updateChange dep = self._getChange(dep_num, dep_ps, refresh=True) File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/source/gerrit.py", line 173, in _getChange self._updateChange(change, history) File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/source/gerrit.py", line 345, in _updateChange dep = self._getChange(dep_num, dep_ps, refresh=True) File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/source/gerrit.py", line 173, in _getChange self._updateChange(change, history) File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/source/gerrit.py", line 345, in _updateChange dep = self._getChange(dep_num, dep_ps, refresh=True) File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/source/gerrit.py", line 173, in _getChange self._updateChange(change, history) File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/source/gerrit.py", line 345, in _updateChange dep = self._getChange(dep_num, dep_ps, refresh=True) File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/source/gerrit.py", line 173, in _getChange self._updateChange(change, history) File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/source/gerrit.py", line 345, in _updateChange dep = self._getChange(dep_num, dep_ps, refresh=True) File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/source/gerrit.py", line 173, in _getChange self._updateChange(change, history) File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/source/gerrit.py", line 305, in _updateChange change): File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/source/gerrit.py", line 210, in _getDependsOnFromCommit records.extend(self.connection.simpleQuery(query)) File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/connection/gerrit.py", line 348, in simpleQuery chunk, more_changes = _query_chunk(query) File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/connection/gerrit.py", line 320, in _query_chunk out, err = self._ssh(cmd) File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/connection/gerrit.py", line 388, in _ssh out = stdout.read() File "/usr/lib/python2.7/dist-packages/paramiko/file.py", line 150, in read new_data = self._read(self._DEFAULT_BUFSIZE) File "/usr/lib/python2.7/dist-packages/paramiko/channel.py", line 1217, in _read return self.channel.recv(size) File "/usr/lib/python2.7/dist-packages/paramiko/channel.py", line 596, in recv out = self.in_buffer.read(nbytes, self.timeout) File "/usr/lib/python2.7/dist-packages/paramiko/buffered_pipe.py", line 147, in read self._cv.wait(timeout) File "/usr/lib/python2.7/threading.py", line 340, in wait waiter.acquire() Thread: 139978755536640 File "/usr/lib/python2.7/threading.py", line 783, in __bootstrap self.__bootstrap_inner() File "/usr/lib/python2.7/threading.py", line 810, in __bootstrap_inner self.run() File "/usr/lib/python2.7/threading.py", line 763, in run self.__target(*self.__args, **self.__kwargs) File "/usr/share/python/zuul/local/lib/python2.7/site-packages/gear/__init__.py", line 830, in _doPollLoop self._pollLoop() File "/usr/share/python/zuul/local/lib/python2.7/site-packages/gear/__init__.py", line 856, in _pollLoop ret = poll.poll() Thread: 139979045463808 File "/usr/bin/zuul-server", line 10, in sys.exit(main()) File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/cmd/server.py", line 239, in main server.main() File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/cmd/server.py", line 211, in main signal.pause() File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/cmd/__init__.py", line 40, in stack_dump_handler log_str += "".join(traceback.format_stack(stack_frame)) Thread: 139978705180416 File "/usr/lib/python2.7/threading.py", line 783, in __bootstrap self.__bootstrap_inner() File "/usr/lib/python2.7/threading.py", line 810, in __bootstrap_inner self.run() File "/usr/lib/python2.7/threading.py", line 763, in run self.__target(*self.__args, **self.__kwargs) File "/usr/lib/python2.7/dist-packages/paste/httpserver.py", line 866, in worker_thread_callback runnable = self.queue.get() File "/usr/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/lib/python2.7/threading.py", line 340, in wait waiter.acquire() Thread: 139978161956608 File "/usr/lib/python2.7/threading.py", line 783, in __bootstrap self.__bootstrap_inner() File "/usr/lib/python2.7/threading.py", line 810, in __bootstrap_inner self.run() File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/connection/gerrit.py", line 204, in run self._run() File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/connection/gerrit.py", line 180, in _run self._listen(stdout, stderr) File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/connection/gerrit.py", line 160, in _listen ret = poll.poll(self.poll_timeout) Thread: 139978713573120 File "/usr/lib/python2.7/threading.py", line 783, in __bootstrap self.__bootstrap_inner() File "/usr/lib/python2.7/threading.py", line 810, in __bootstrap_inner self.run() File "/usr/lib/python2.7/threading.py", line 763, in run self.__target(*self.__args, **self.__kwargs) File "/usr/lib/python2.7/dist-packages/paste/httpserver.py", line 866, in worker_thread_callback runnable = self.queue.get() File "/usr/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/lib/python2.7/threading.py", line 340, in wait waiter.acquire() Thread: 139978721965824 File "/usr/lib/python2.7/threading.py", line 783, in __bootstrap self.__bootstrap_inner() File "/usr/lib/python2.7/threading.py", line 810, in __bootstrap_inner self.run() File "/usr/lib/python2.7/threading.py", line 763, in run self.__target(*self.__args, **self.__kwargs) File "/usr/lib/python2.7/dist-packages/paste/httpserver.py", line 866, in worker_thread_callback runnable = self.queue.get() File "/usr/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/lib/python2.7/threading.py", line 340, in wait waiter.acquire() Thread: 139978210273024 File "/usr/lib/python2.7/threading.py", line 783, in __bootstrap self.__bootstrap_inner() File "/usr/lib/python2.7/threading.py", line 810, in __bootstrap_inner self.run() File "/usr/lib/python2.7/threading.py", line 763, in run self.__target(*self.__args, **self.__kwargs) File "/usr/lib/python2.7/dist-packages/paste/httpserver.py", line 866, in worker_thread_callback runnable = self.queue.get() File "/usr/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/lib/python2.7/threading.py", line 340, in wait waiter.acquire() Thread: 139978201880320 File "/usr/lib/python2.7/threading.py", line 783, in __bootstrap self.__bootstrap_inner() File "/usr/lib/python2.7/threading.py", line 810, in __bootstrap_inner self.run() File "/usr/lib/python2.7/threading.py", line 763, in run self.__target(*self.__args, **self.__kwargs) File "/usr/lib/python2.7/dist-packages/paste/httpserver.py", line 866, in worker_thread_callback runnable = self.queue.get() File "/usr/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/lib/python2.7/threading.py", line 340, in wait waiter.acquire() Thread: 139977631590144 File "/usr/lib/python2.7/threading.py", line 783, in __bootstrap self.__bootstrap_inner() File "/usr/lib/python2.7/threading.py", line 810, in __bootstrap_inner self.run() File "/usr/lib/python2.7/threading.py", line 763, in run self.__target(*self.__args, **self.__kwargs) File "/usr/share/python/zuul/local/lib/python2.7/site-packages/gear/__init__.py", line 830, in _doPollLoop self._pollLoop() File "/usr/share/python/zuul/local/lib/python2.7/site-packages/gear/__init__.py", line 856, in _pollLoop ret = poll.poll() Thread: 139977639982848 File "/usr/lib/python2.7/threading.py", line 783, in __bootstrap self.__bootstrap_inner() File "/usr/lib/python2.7/threading.py", line 810, in __bootstrap_inner self.run() File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/webapp.py", line 61, in run self.server.serve_forever() File "/usr/lib/python2.7/dist-packages/paste/httpserver.py", line 1084, in serve_forever self.handle_request() File "/usr/lib/python2.7/SocketServer.py", line 276, in handle_request fd_sets = _eintr_retry(select.select, [self], [], [], timeout) File "/usr/lib/python2.7/SocketServer.py", line 155, in _eintr_retry return func(*args) Thread: 139977669089024 File "/usr/lib/python2.7/threading.py", line 783, in __bootstrap self.__bootstrap_inner() File "/usr/lib/python2.7/threading.py", line 810, in __bootstrap_inner self.run() File "/usr/lib/python2.7/dist-packages/paramiko/transport.py", line 1627, in run ptype, m = self.packetizer.read_message() File "/usr/lib/python2.7/dist-packages/paramiko/packet.py", line 341, in read_message header = self.read_all(self.__block_size_in, check_rekey=True) File "/usr/lib/python2.7/dist-packages/paramiko/packet.py", line 204, in read_all x = self.__socket.recv(n) Thread: 139978218665728 File "/usr/lib/python2.7/threading.py", line 783, in __bootstrap self.__bootstrap_inner() File "/usr/lib/python2.7/threading.py", line 810, in __bootstrap_inner self.run() File "/usr/lib/python2.7/threading.py", line 763, in run self.__target(*self.__args, **self.__kwargs) File "/usr/lib/python2.7/dist-packages/paste/httpserver.py", line 866, in worker_thread_callback runnable = self.queue.get() File "/usr/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/lib/python2.7/threading.py", line 340, in wait waiter.acquire() Thread: 139978747143936 File "/usr/lib/python2.7/threading.py", line 783, in __bootstrap self.__bootstrap_inner() File "/usr/lib/python2.7/threading.py", line 810, in __bootstrap_inner self.run() File "/usr/lib/python2.7/threading.py", line 763, in run self.__target(*self.__args, **self.__kwargs) File "/usr/share/python/zuul/local/lib/python2.7/site-packages/gear/__init__.py", line 744, in _doConnectLoop self.connections_condition.wait() File "/usr/lib/python2.7/threading.py", line 340, in wait waiter.acquire() Thread: 139977228015360 File "/usr/lib/python2.7/threading.py", line 783, in __bootstrap self.__bootstrap_inner() File "/usr/lib/python2.7/threading.py", line 810, in __bootstrap_inner self.run() File "/usr/lib/python2.7/threading.py", line 763, in run self.__target(*self.__args, **self.__kwargs) File "/usr/lib/python2.7/dist-packages/paste/httpserver.py", line 866, in worker_thread_callback runnable = self.queue.get() File "/usr/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/lib/python2.7/threading.py", line 340, in wait waiter.acquire() Thread: 139977279141632 File "/usr/lib/python2.7/threading.py", line 783, in __bootstrap self.__bootstrap_inner() File "/usr/lib/python2.7/threading.py", line 810, in __bootstrap_inner self.run() File "/usr/lib/python2.7/threading.py", line 763, in run self.__target(*self.__args, **self.__kwargs) File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/rpclistener.py", line 68, in run job = self.worker.getJob() File "/usr/share/python/zuul/local/lib/python2.7/site-packages/gear/__init__.py", line 1898, in getJob job = self.job_queue.get() File "/usr/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/lib/python2.7/threading.py", line 340, in wait waiter.acquire() Thread: 139978738751232 File "/usr/lib/python2.7/threading.py", line 783, in __bootstrap self.__bootstrap_inner() File "/usr/lib/python2.7/threading.py", line 810, in __bootstrap_inner self.run() File "/usr/lib/python2.7/dist-packages/paramiko/transport.py", line 1627, in run ptype, m = self.packetizer.read_message() File "/usr/lib/python2.7/dist-packages/paramiko/packet.py", line 341, in read_message header = self.read_all(self.__block_size_in, check_rekey=True) File "/usr/lib/python2.7/dist-packages/paramiko/packet.py", line 204, in read_all x = self.__socket.recv(n) Thread: 139977623197440 File "/usr/lib/python2.7/threading.py", line 783, in __bootstrap self.__bootstrap_inner() File "/usr/lib/python2.7/threading.py", line 810, in __bootstrap_inner self.run() File "/usr/lib/python2.7/threading.py", line 763, in run self.__target(*self.__args, **self.__kwargs) File "/usr/share/python/zuul/local/lib/python2.7/site-packages/gear/__init__.py", line 744, in _doConnectLoop self.connections_condition.wait() File "/usr/lib/python2.7/threading.py", line 340, in wait waiter.acquire() Thread: 139978979469056 File "/usr/lib/python2.7/threading.py", line 783, in __bootstrap self.__bootstrap_inner() File "/usr/lib/python2.7/threading.py", line 810, in __bootstrap_inner self.run() File "/usr/lib/python2.7/threading.py", line 763, in run self.__target(*self.__args, **self.__kwargs) File "/usr/share/python/zuul/local/lib/python2.7/site-packages/gear/__init__.py", line 744, in _doConnectLoop self.connections_condition.wait() File "/usr/lib/python2.7/threading.py", line 340, in wait waiter.acquire() Thread: 139978170349312 File "/usr/lib/python2.7/threading.py", line 783, in __bootstrap self.__bootstrap_inner() File "/usr/lib/python2.7/threading.py", line 810, in __bootstrap_inner self.run() File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/scheduler.py", line 968, in run self.wake_event.wait() File "/usr/lib/python2.7/threading.py", line 621, in wait self.__cond.wait(timeout) File "/usr/lib/python2.7/threading.py", line 340, in wait waiter.acquire() Thread: 139978971076352 File "/usr/lib/python2.7/threading.py", line 783, in __bootstrap self.__bootstrap_inner() File "/usr/lib/python2.7/threading.py", line 810, in __bootstrap_inner self.run() File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/launcher/gearman.py", line 47, in run self.wake_event.wait(300) File "/usr/lib/python2.7/threading.py", line 621, in wait self.__cond.wait(timeout) File "/usr/lib/python2.7/threading.py", line 359, in wait _sleep(delay) Thread: 139977270748928 File "/usr/lib/python2.7/threading.py", line 783, in __bootstrap self.__bootstrap_inner() File "/usr/lib/python2.7/threading.py", line 810, in __bootstrap_inner self.run() File "/usr/lib/python2.7/threading.py", line 763, in run self.__target(*self.__args, **self.__kwargs) File "/usr/lib/python2.7/dist-packages/paste/httpserver.py", line 866, in worker_thread_callback runnable = self.queue.get() File "/usr/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/lib/python2.7/threading.py", line 340, in wait waiter.acquire() Thread: 139978987861760 File "/usr/lib/python2.7/threading.py", line 783, in __bootstrap self.__bootstrap_inner() File "/usr/lib/python2.7/threading.py", line 810, in __bootstrap_inner self.run() File "/usr/lib/python2.7/threading.py", line 763, in run self.__target(*self.__args, **self.__kwargs) File "/usr/share/python/zuul/local/lib/python2.7/site-packages/gear/__init__.py", line 830, in _doPollLoop self._pollLoop() File "/usr/share/python/zuul/local/lib/python2.7/site-packages/gear/__init__.py", line 856, in _pollLoop ret = poll.poll()