exception raised when searching with multithreads
haoxian-zhao opened this issue · comments
not sure whether bulbs support multithread but it blow up when I try to run below code, would you mind having a look and see whether this is an issue with bulbs or I am doing the threading wrongly (it run ok if parallel set to False)? thanks
import collections
import time
from bulbs.titan import Graph, Config
import thread
def slice( names, size ):
m = collections.defaultdict( list )
for i, name in enumerate( names ):
m[ i/size ].append( name )
return m.values()
def search( names, graph ):
return [ list( graph.vertices.index.lookup( username=name ) ) for name in names ]
def titan():
config = Config( 'http://localhost:8182/graphs/graph' )
parallel = True
graph = Graph( config )
for x in slice( map( lambda x:'name{0}'.format( x ), xrange( 100 ) ), 30 ):
if parallel:
thread.start_new_thread( search, (x, graph) )
else:
search( x, graph )
titan()
while 1:
pass
Unhandled exception in thread started by <function search at 0x10b95c050>
Traceback (most recent call last):
File "test.py", line 21, in search
return [ list( graph.vertices.index.lookup( username=name ) ) for name in names ]
File "/Users/xyz/.env/zyx/lib/python2.7/site-packages/bulbs-0.3.14_20130803-py2.7.egg/bulbs/titan/index.py", line 270, in lookup
Unhandled exception in thread started by <function search at 0x10b95c050> resp = self.client.lookup_vertex(self.index_name,key,value)
File "/Users/xyz/.env/zyx/lib/python2.7/site-packages/bulbs-0.3.14_20130803-py2.7.egg/bulbs/titan/client.py", line 337, in lookup_vertex
return self.request.get(path,params)
Traceback (most recent call last):
File "test.py", line 21, in search
return [ list( graph.vertices.index.lookup( username=name ) ) for name in names ]
File "/Users/xyz/.env/zyx/lib/python2.7/site-packages/bulbs-0.3.14_20130803-py2.7.egg/bulbs/titan/index.py", line 270, in lookup
resp = self.client.lookup_vertex(self.index_name,key,value)
File "/Users/xyz/.env/zyx/lib/python2.7/site-packages/bulbs-0.3.14_20130803-py2.7.egg/bulbs/titan/client.py", line 337, in lookup_vertex
return self.request.get(path,params)
File "/Users/xyz/.env/zyx/lib/python2.7/site-packages/bulbs-0.3.14_20130803-py2.7.egg/bulbs/rest.py", line 98, in get
Unhandled exception in thread started by <function search at 0x10b95c050>
Traceback (most recent call last):
return self.request(GET, path, params)
File "/Users/xyz/.env/zyx/lib/python2.7/site-packages/bulbs-0.3.14_20130803-py2.7.egg/bulbs/rest.py", line 181, in request
http_resp = self.http.request(uri, method, body, headers)
File "/Users/xyz/.env/zyx/lib/python2.7/site-packages/bulbs-0.3.14_20130803-py2.7.egg/bulbs/rest.py", line 98, in get
return self.request(GET, path, params)
File "/Users/xyz/.env/zyx/lib/python2.7/site-packages/httplib2/init.py", line 1570, in request
File "/Users/xyz/.env/zyx/lib/python2.7/site-packages/bulbs-0.3.14_20130803-py2.7.egg/bulbs/rest.py", line 181, in request
File "test.py", line 21, in search
return [ list( graph.vertices.index.lookup( username=name ) ) for name in names ](response, content) = self._request(conn, authority, uri, request_uri, method, body, headers, redirections, cachekey)
File "/Users/xyz/.env/zyx/lib/python2.7/site-packages/httplib2/init.py", line 1317, in _request
File "/Users/xyz/.env/zyx/lib/python2.7/site-packages/bulbs-0.3.14_20130803-py2.7.egg/bulbs/titan/index.py", line 270, in lookup
http_resp = self.http.request(uri, method, body, headers)
resp = self.client.lookup_vertex(self.index_name,key,value)
File "/Users/xyz/.env/zyx/lib/python2.7/site-packages/bulbs-0.3.14_20130803-py2.7.egg/bulbs/titan/client.py", line 337, in lookup_vertex
File "/Users/xyz/.env/zyx/lib/python2.7/site-packages/httplib2/init.py", line 1570, in request
(response, content) = self._request(conn, authority, uri, request_uri, method, body, headers, redirections, cachekey)
File "/Users/xyz/.env/zyx/lib/python2.7/site-packages/httplib2/init.py", line 1317, in _request
(response, content) = self._conn_request(conn, request_uri, method, body, headers)
File "/Users/xyz/.env/zyx/lib/python2.7/site-packages/httplib2/init.py", line 1286, in _conn_request
response = conn.getresponse()
File "/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/httplib.py", line 1027, in getresponse
return self.request.get(path,params)
response.begin()
File "/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/httplib.py", line 407, in begin
version, status, reason = self._read_status()
File "/Users/xyz/.env/zyx/lib/python2.7/site-packages/bulbs-0.3.14_20130803-py2.7.egg/bulbs/rest.py", line 98, in get
return self.request(GET, path, params)
File "/Users/xyz/.env/zyx/lib/python2.7/site-packages/bulbs-0.3.14_20130803-py2.7.egg/bulbs/rest.py", line 181, in request
http_resp = self.http.request(uri, method, body, headers)
File "/Users/xyz/.env/zyx/lib/python2.7/site-packages/httplib2/init.py", line 1570, in request
(response, content) = self._request(conn, authority, uri, request_uri, method, body, headers, redirections, cachekey)
File "/Users/xyz/.env/zyx/lib/python2.7/site-packages/httplib2/init.py", line 1317, in _request
(response, content) = self._conn_request(conn, request_uri, method, body, headers)
File "/Users/xyz/.env/zyx/lib/python2.7/site-packages/httplib2/init.py", line 1290, in _conn_request
conn.connect()
File "/Users/xyz/.env/zyx/lib/python2.7/site-packages/httplib2/init.py", line 913, in connect
raise socket.error, msg
socket File "/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/httplib.py", line 365, in _read_status
(response, content) = self._conn_request(conn, request_uri, method, body, headers)
File "/Users/xyz/.env/zyx/lib/python2.7/site-packages/httplib2/init.py", line 1290, in _conn_request
conn.connect()
File "/Users/xyz/.env/zyx/lib/python2.7/site-packages/httplib2/init.py", line 913, in connect
raise socket.error, msg
socket.error: . line = self.fp.readline()
getaddrinfo returns an empty listerror
File "/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/socket.py", line 430, in readline
: data = recv(1)
socket.error: [Errno 57] Socket is not connectedgetaddrinfo returns an empty list
Bulbs does not support multithreading at the moment, but it would not take too much to get it there.
However, in this case (an in most cases), a Gremlin script that encapsulates all the lookups/search into a single request will be more performant than issuing multiple requests in multiple threads because you won't have the HTTP overhead.
See these examples: