DDGS Ratelimit
thierryteisseire opened this issue · comments
When using DuckDuckGo Search
: Search the web with DuckDuckGo. Getting a ratelimit error as seen in LangSmith. If someone had the same or have got an idea on why?
Thanks all!
DuckDuckGoSearchException('_get_url() https://links.duckduckgo.com/d.js DuckDuckGoSearchException: Ratelimit')Traceback (most recent call last):
File "/usr/local/lib/python3.11/site-packages/duckduckgo_search/duckduckgo_search.py", line 49, in _get_url
raise DuckDuckGoSearchException("Ratelimit")
duckduckgo_search.exceptions.DuckDuckGoSearchException: Ratelimit
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/usr/local/lib/python3.11/site-packages/langchain_core/runnables/base.py", line 1595, in _atransform_stream_with_config
chunk: Output = await asyncio.create_task( # type: ignore[call-arg]
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/langchain_core/tracers/log_stream.py", line 238, in tap_output_aiter
async for chunk in output:
File "/usr/local/lib/python3.11/site-packages/langgraph/pregel/init.py", line 525, in _atransform
_interrupt_or_proceed(done, inflight, step)
File "/usr/local/lib/python3.11/site-packages/langgraph/pregel/init.py", line 700, in _interrupt_or_proceed
raise exc
File "/usr/local/lib/python3.11/site-packages/langgraph/pregel/init.py", line 838, in _aconsume
async for _ in iterator:
File "/usr/local/lib/python3.11/site-packages/langchain_core/runnables/base.py", line 4144, in astream
async for item in self.bound.astream(
File "/usr/local/lib/python3.11/site-packages/langchain_core/runnables/base.py", line 2449, in astream
async for chunk in self.atransform(input_aiter(), config, **kwargs):
File "/usr/local/lib/python3.11/site-packages/langchain_core/runnables/base.py", line 2432, in atransform
async for chunk in self._atransform_stream_with_config(
File "/usr/local/lib/python3.11/site-packages/langchain_core/runnables/base.py", line 1595, in _atransform_stream_with_config
chunk: Output = await asyncio.create_task( # type: ignore[call-arg]
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/langchain_core/tracers/log_stream.py", line 238, in tap_output_aiter
async for chunk in output:
File "/usr/local/lib/python3.11/site-packages/langchain_core/runnables/base.py", line 2402, in _atransform
async for output in final_pipeline:
File "/usr/local/lib/python3.11/site-packages/langchain_core/runnables/passthrough.py", line 269, in atransform
async for chunk in self._atransform_stream_with_config(
File "/usr/local/lib/python3.11/site-packages/langchain_core/runnables/base.py", line 1560, in _atransform_stream_with_config
final_input: Optional[Input] = await py_anext(input_for_tracing, None)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/langchain_core/utils/aiter.py", line 62, in anext_impl
return await anext(iterator)
^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/langchain_core/utils/aiter.py", line 97, in tee_peer
item = await iterator.anext()
^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/langchain_core/runnables/base.py", line 3711, in atransform
async for output in self._atransform_stream_with_config(
File "/usr/local/lib/python3.11/site-packages/langchain_core/runnables/base.py", line 1595, in _atransform_stream_with_config
chunk: Output = await asyncio.create_task( # type: ignore[call-arg]
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/langchain_core/tracers/log_stream.py", line 238, in tap_output_aiter
async for chunk in output:
File "/usr/local/lib/python3.11/site-packages/langchain_core/runnables/base.py", line 3680, in _atransform
output = await acall_func_with_variable_args(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/backend/app/agent_types/openai_agent.py", line 70, in call_tool
responses = await tool_executor.abatch(actions)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/langchain_core/runnables/base.py", line 4119, in abatch
return await self.bound.abatch(
^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/langchain_core/runnables/base.py", line 568, in abatch
return await gather_with_concurrency(configs[0].get("max_concurrency"), *coros)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/langchain_core/runnables/utils.py", line 51, in gather_with_concurrency
return await asyncio.gather(*coros)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/langchain_core/runnables/base.py", line 565, in ainvoke
return await self.ainvoke(input, config, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/langchain_core/runnables/base.py", line 3527, in ainvoke
return await self._acall_with_config(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/langchain_core/runnables/base.py", line 1295, in _acall_with_config
output: Output = await asyncio.create_task(coro, context=context) # type: ignore
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/langchain_core/runnables/base.py", line 3474, in _ainvoke
output = await acall_func_with_variable_args(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/langgraph/prebuilt/tool_executor.py", line 73, in _aexecute
output = await tool.ainvoke(tool_invocation.tool_input, config=config)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/langchain_core/tools.py", line 238, in ainvoke
return await self.arun(
^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/langchain_core/tools.py", line 492, in arun
raise e
File "/usr/local/lib/python3.11/site-packages/langchain_core/tools.py", line 451, in arun
else await self._arun(*tool_args, **tool_kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/langchain_core/tools.py", line 303, in _arun
return await run_in_executor(None, self._run, *args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/langchain_core/runnables/config.py", line 493, in run_in_executor
return await asyncio.get_running_loop().run_in_executor(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/concurrent/futures/thread.py", line 58, in run
result = self.fn(*self.args, **self.kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/langchain_community/tools/ddg_search/tool.py", line 39, in _run
return self.api_wrapper.run(query)
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/langchain_community/utilities/duckduckgo_search.py", line 81, in run
results = self._ddgs_text(query)
^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/langchain_community/utilities/duckduckgo_search.py", line 57, in _ddgs_text
return [r for r in ddgs_gen]
^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/langchain_community/utilities/duckduckgo_search.py", line 57, in
return [r for r in ddgs_gen]
^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/duckduckgo_search/duckduckgo_search.py", line 95, in text
for i, result in enumerate(results, start=1):
File "/usr/local/lib/python3.11/site-packages/duckduckgo_search/duckduckgo_search.py", line 146, in _text_api
resp = self._get_url("GET", "https://links.duckduckgo.com/d.js", params=payload)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/duckduckgo_search/duckduckgo_search.py", line 53, in _get_url
raise DuckDuckGoSearchException(f"_get_url() {url} {type(ex).name}: {ex}") from ex
duckduckgo_search.exceptions.DuckDuckGoSearchException: _get_url() https://links.duckduckgo.com/d.js DuckDuckGoSearchException: Ratelimit
I also faced the issue : Upgrading duckduckgo-search package to the latest version made it work on my side. I do not have more information about what the issue was however.
cool, thanks; did you update the pyproject.toml to 5.0?
I upgraded duckduckgo-search = "^5.1.0".
The requirements in my pyproject.toml are however slightly different so I would need to test with main to confirm that it works with the current state.
I followed the thread here as reference : joaomdmoura/crewAI#136
I confirm it works! Just did poetry update duckduckgo-search..thanks!