os.unlink(entry.name, dir_fd=topfd) TypeError: 'NoneType' object is not callable
starlitsky2010 opened this issue · comments
starlitsky2010 commented
When I use code_eval.py to do evaluation I found the following problem.
Traceback (most recent call last):
File "/opt/conda/lib/python3.10/weakref.py", line 667, in _exitfunc
f()
File "/opt/conda/lib/python3.10/weakref.py", line 591, in __call__
return info.func(*info.args, **(info.kwargs or {}))
File "/opt/conda/lib/python3.10/tempfile.py", line 858, in _cleanup
cls._rmtree(name, ignore_errors=ignore_errors)
File "/opt/conda/lib/python3.10/tempfile.py", line 854, in _rmtree
_shutil.rmtree(name, onerror=onerror)
File "/opt/conda/lib/python3.10/shutil.py", line 728, in rmtree
_rmtree_safe_fd(fd, path, onerror)
File "/opt/conda/lib/python3.10/shutil.py", line 682, in _rmtree_safe_fd
os.unlink(entry.name, dir_fd=topfd)
TypeError: 'NoneType' object is not callable
I found /root/.cache/huggingface/modules/evaluate_modules/metrics/evaluate-metric--code_eval/78d307ea938083398db7d9815f03ed661e9c15f60d77880ce007a8a02648f176/execute.py
def reliability_guard(maximum_memory_bytes=None):
"""
This disables various destructive functions and prevents the generated code
from interfering with the test (e.g. fork bomb, killing other processes,
removing filesystem files, etc.)
WARNING
This function is NOT a security sandbox. Untrusted code, including, model-
generated code, should not be blindly executed outside of one. See the
Codex paper for more information about OpenAI's code sandbox, and proceed
with caution.
"""
...
os.replace = None
os.unlink = None
os.fchmod = None
...
So, the os.unlink should be None. But in the
metrics/code_eval/execute.py
def unsafe_execute(check_program, result, timeout):
with create_tempdir():
# These system calls are needed when cleaning up tempdir.
import os
import shutil
rmtree = shutil.rmtree
rmdir = os.rmdir
chdir = os.chdir
then, we found it called
@contextlib.contextmanager
def create_tempdir():
with tempfile.TemporaryDirectory() as dirname:
with chdir(dirname):
yield dirname
For tempfile.TemporaryDirectory(), when exit this function, it will call
--> def rmtree(path, ignore_errors=False, onerror=None):
--> _rmtree_safe_fd(fd, path, onerror
-->
def _rmtree_safe_fd(topfd, path, onerror):
try:
with os.scandir(topfd) as scandir_it:
entries = list(scandir_it)
except OSError as err:
err.filename = path
onerror(os.scandir, path, sys.exc_info())
return
for entry in entries:
fullname = os.path.join(path, entry.name)
try:
is_dir = entry.is_dir(follow_symlinks=False)
except OSError:
is_dir = False
else:
if is_dir:
try:
orig_st = entry.stat(follow_symlinks=False)
is_dir = stat.S_ISDIR(orig_st.st_mode)
except OSError:
onerror(os.lstat, fullname, sys.exc_info())
continue
if is_dir:
try:
dirfd = os.open(entry.name, os.O_RDONLY, dir_fd=topfd)
dirfd_closed = False
except OSError:
onerror(os.open, fullname, sys.exc_info())
else:
try:
if os.path.samestat(orig_st, os.fstat(dirfd)):
_rmtree_safe_fd(dirfd, fullname, onerror)
try:
os.close(dirfd)
dirfd_closed = True
os.rmdir(entry.name, dir_fd=topfd)
except OSError:
onerror(os.rmdir, fullname, sys.exc_info())
else:
try:
# This can only happen if someone replaces
# a directory with a symlink after the call to
# os.scandir or stat.S_ISDIR above.
raise OSError("Cannot call rmtree on a symbolic "
"link")
except OSError:
onerror(os.path.islink, fullname, sys.exc_info())
finally:
if not dirfd_closed:
os.close(dirfd)
else:
try:
os.unlink(entry.name, dir_fd=topfd) # NOTICE!!! It will run this position with os.unlink is None and lead an error mentioned above.
except OSError:
onerror(os.unlink, fullname, sys.exc_info())
My modification is :
--- a/metrics/code_eval/execute.py
+++ b/metrics/code_eval/execute.py
@@ -64,6 +64,7 @@ def unsafe_execute(check_program, result, timeout):
rmtree = shutil.rmtree
rmdir = os.rmdir
chdir = os.chdir
+ osunlink = os.unlink
# Disable functionalities that can make destructive changes to the test.
reliability_guard()
@@ -84,6 +85,7 @@ def unsafe_execute(check_program, result, timeout):
shutil.rmtree = rmtree
os.rmdir = rmdir
os.chdir = chdir
+ os.unlink = osunlink
Is there any better method?
Thanks