CMake: do not run generate_code.py if flatc is cross-compiled
SpaceIm opened this issue · comments
This logic can't work when you cross-build flatc:
Lines 543 to 561 in c1daa6b
Build errors:
[36/36] Linking CXX executable bin/flatc
FAILED: bin/flatc
: && /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/c++ -stdlib=libc++ -std=c++11 -Wall -pedantic -Wextra -Wno-unused-parameter -fsigned-char -O3 -DNDEBUG -arch arm64 -isysroot /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX12.1.sdk -Wl,-headerpad_max_install_names source_subfolder/CMakeFiles/flatc.dir/src/idl_parser.cpp.o source_subfolder/CMakeFiles/flatc.dir/src/idl_gen_text.cpp.o source_subfolder/CMakeFiles/flatc.dir/src/reflection.cpp.o source_subfolder/CMakeFiles/flatc.dir/src/util.cpp.o source_subfolder/CMakeFiles/flatc.dir/src/idl_gen_cpp.cpp.o source_subfolder/CMakeFiles/flatc.dir/src/idl_gen_csharp.cpp.o source_subfolder/CMakeFiles/flatc.dir/src/idl_gen_dart.cpp.o source_subfolder/CMakeFiles/flatc.dir/src/idl_gen_kotlin.cpp.o source_subfolder/CMakeFiles/flatc.dir/src/idl_gen_go.cpp.o source_subfolder/CMakeFiles/flatc.dir/src/idl_gen_java.cpp.o source_subfolder/CMakeFiles/flatc.dir/src/idl_gen_ts.cpp.o source_subfolder/CMakeFiles/flatc.dir/src/idl_gen_php.cpp.o source_subfolder/CMakeFiles/flatc.dir/src/idl_gen_python.cpp.o source_subfolder/CMakeFiles/flatc.dir/src/idl_gen_lobster.cpp.o source_subfolder/CMakeFiles/flatc.dir/src/idl_gen_lua.cpp.o source_subfolder/CMakeFiles/flatc.dir/src/idl_gen_rust.cpp.o source_subfolder/CMakeFiles/flatc.dir/src/idl_gen_fbs.cpp.o source_subfolder/CMakeFiles/flatc.dir/src/idl_gen_grpc.cpp.o source_subfolder/CMakeFiles/flatc.dir/src/idl_gen_json_schema.cpp.o source_subfolder/CMakeFiles/flatc.dir/src/idl_gen_swift.cpp.o source_subfolder/CMakeFiles/flatc.dir/src/flatc.cpp.o source_subfolder/CMakeFiles/flatc.dir/src/flatc_main.cpp.o source_subfolder/CMakeFiles/flatc.dir/src/bfbs_gen_lua.cpp.o source_subfolder/CMakeFiles/flatc.dir/src/code_generators.cpp.o source_subfolder/CMakeFiles/flatc.dir/grpc/src/compiler/cpp_generator.cc.o source_subfolder/CMakeFiles/flatc.dir/grpc/src/compiler/go_generator.cc.o source_subfolder/CMakeFiles/flatc.dir/grpc/src/compiler/java_generator.cc.o source_subfolder/CMakeFiles/flatc.dir/grpc/src/compiler/python_generator.cc.o source_subfolder/CMakeFiles/flatc.dir/grpc/src/compiler/swift_generator.cc.o source_subfolder/CMakeFiles/flatc.dir/grpc/src/compiler/ts_generator.cc.o -o bin/flatc && cd /Users/spaceim/.conan/data/flatbuffers/2.0.5/_/_/build/f1a36a2aea2da1148eb4bddd758d8db183c6db83/source_subfolder && scripts/generate_code.py --flatc /Users/spaceim/.conan/data/flatbuffers/2.0.5/_/_/build/f1a36a2aea2da1148eb4bddd758d8db183c6db83/bin/flatc
Traceback (most recent call last):
File "/Users/spaceim/.conan/data/flatbuffers/2.0.5/_/_/build/f1a36a2aea2da1148eb4bddd758d8db183c6db83/source_subfolder/scripts/generate_code.py", line 118, in <module>
flatc(
File "/Users/spaceim/.conan/data/flatbuffers/2.0.5/_/_/build/f1a36a2aea2da1148eb4bddd758d8db183c6db83/source_subfolder/scripts/generate_code.py", line 77, in flatc
subprocess.run(cmd, cwd=cwd)
File "/usr/local/Cellar/python@3.9/3.9.10/Frameworks/Python.framework/Versions/3.9/lib/python3.9/subprocess.py", line 505, in run
with Popen(*popenargs, **kwargs) as process:
File "/usr/local/Cellar/python@3.9/3.9.10/Frameworks/Python.framework/Versions/3.9/lib/python3.9/subprocess.py", line 951, in __init__
self._execute_child(args, executable, preexec_fn, close_fds,
File "/usr/local/Cellar/python@3.9/3.9.10/Frameworks/Python.framework/Versions/3.9/lib/python3.9/subprocess.py", line 1821, in _execute_child
raise child_exception_type(errno_num, err_msg, err_filename)
OSError: [Errno 86] Bad CPU type in executable: '/Users/spaceim/.conan/data/flatbuffers/2.0.5/_/_/build/f1a36a2aea2da1148eb4bddd758d8db183c6db83/bin/flatc'
ninja: build stopped: subcommand failed.
Good point. I should probably add a cmake option to disable it, or figure out a way to determine the flatc architect isn't the same as the host. Thoughts?
What is the purpose of this script? It doesn't seem to be useful for a pure build & install of C++ flatbuffers, flatc and headers. Does it generate files for other parts of the build already controlled by options like tests?
What happen when python3 is not found? Script is not called, so it's not deterministic, can it break something else?
The purpose is to force the generation of tests/example schema whenever a change to flatc (and thus the code generators) occur. So it is a post-build target of flatc. It doesn't run if you don't have python3 (if(Python3_Interpreter_FOUND)). It won't break anything if you don't run it, it will just fail CI if you make edits to flatc or generators without running it.
Its true that if you aren't developing/editing flatc or the generators it doesn't matter to run this script during the "pure build & install". However, I couldn't figure out a nice way to determine those two build cases. Previously, we had a 'Please run the script manually' message when filling out a PR, but only like 50% of people did. So this script was my attempt to make that 100%.
Thanks for the explanation.
This condition may be sufficient (assuming folks submitting PRs run CMake, don't disable tests, and commit the files generated by this script).
if(FLATBUFFERS_BUILD_TESTS AND NOT CMAKE_CROSSCOMPILING)
# Look if we have python 3.5 installed so that we can run the generate code
# python script after flatc is built.
find_package(Python3 3.5 COMPONENTS Interpreter)
if(Python3_Interpreter_FOUND)
set(GENERATION_OPTS --flatc "${FLATBUFFERS_FLATC_EXECUTABLE}")
if(FLATBUFFERS_BUILD_LEGACY)
# Need to set --cpp-std c++-0x options
set(GENERATION_OPTS ${GENERATION_OPTS}--cpp-0x)
endif()
if(FLATBUFFERS_SKIP_MONSTER_EXTRA)
set(GENERATION_OPTS ${GENERATION_OPTS} --skip-monster-extra)
endif()
add_custom_command(
TARGET flatc
POST_BUILD
COMMAND ${Python3_EXECUTABLE} scripts/generate_code.py ${GENERATION_OPTS} --skip-gen-reflection
WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}"
COMMENT "Running scripts/generate_code.py..."
VERBATIM)
else()
message("No Python3 interpreter found! Unable to generate files automatically.")
endif()
endif()
Previously, we had a 'Please run the script manually' message when filling out a PR, but only like 50% of people did. So this script was my attempt to make that 100%.
If that is the case, what about adding a CI job that:
- runs the script
- checks git status that there were no changes
- prints an error to the user if there were any (or you can maybe choose to automatically commit the files that are expected to change)
Yeah, we can do something like that.