support machine-readable (json) output
Gankra opened this issue · comments
Thankfully I had the insight to make all the error reporting gathered up into types, so it should be relatively easy to select a different renderer. Not sure if the output should be derived, hand-rolled with json!, or based on miette's JsonReportHandler. We aren't setup for miette at the moment, and I'm not sure if it's source/span stuff is super applicable with us generating the source ourselves and shelling out to real compilers already.
This should be configurable with --output-format=json (defaults to human, the only other choice).
Maybe use the same json format as libtest? https://github.com/rust-lang/rust/blob/master/library/test/src/formatters/json.rs Or maybe the junit format, which libtest also supports? https://github.com/rust-lang/rust/blob/master/library/test/src/formatters/junit.rs This may allow existing tools to integrate with abi-checker more easily. Not sure if any existing tool would want to integrate with it though.
I completely forgot you mentionned some relevant pre-existing formats. I most focused on getting things into some structure so that serde could spew it out easily.
As of #13, --output-format=json will partially produce json output to stdout (with interactive output coming on stderr), but I don't have stdout 100% clean of random build.rs dumps.
Some sample output:
json output
{
"summary": {
"num_tests": 231,
"num_passed": 47,
"num_busted": 2,
"num_failed": 0,
"num_skipped": 182
},
"config": {},
"tests": [
{
"key": {
"test_name": "bool",
"convention": "c",
"caller_id": "rustc",
"callee_id": "cc"
},
"rules": {
"run": "Check",
"check": {
"Pass": "Check"
}
},
"results": {
"ran_to": "Check",
"source": {
"Ok": {
"caller_src": "generated_impls/rustc/bool_c_rustc_caller.rs",
"callee_src": "generated_impls/cc/bool_c_cc_callee.c"
}
},
"build": {
"Ok": {
"caller_lib": "bool_c_rustc_caller",
"callee_lib": "bool_c_cc_callee"
}
},
"link": {
"Ok": {
"test_bin": "target/temp/bool_rustc_calls_cc_harness.dll"
}
},
"run": {
"Ok": {
"caller_inputs": { ... }
....
}
}
},
"check": {
"all_passed": true,
"subtest_names": [
"bool::c::rustc_calls_cc::bool_val_in",
"bool::c::rustc_calls_cc::bool_val_out",
"bool::c::rustc_calls_cc::bool_val_in_out",
"bool::c::rustc_calls_cc::bool_ref_in",
"bool::c::rustc_calls_cc::bool_ref_out",
"bool::c::rustc_calls_cc::bool_ref_in_out",
"bool::c::rustc_calls_cc::bool_val_in_2",
"bool::c::rustc_calls_cc::bool_val_in_3",
"bool::c::rustc_calls_cc::bool_val_in_4",
"bool::c::rustc_calls_cc::bool_val_in_5",
"bool::c::rustc_calls_cc::bool_val_in_6",
"bool::c::rustc_calls_cc::bool_val_in_7",
"bool::c::rustc_calls_cc::bool_val_in_8",
"bool::c::rustc_calls_cc::bool_val_in_9",
"bool::c::rustc_calls_cc::bool_val_in_10",
"bool::c::rustc_calls_cc::bool_val_in_11",
"bool::c::rustc_calls_cc::bool_val_in_12",
"bool::c::rustc_calls_cc::bool_val_in_13",
"bool::c::rustc_calls_cc::bool_val_in_14",
"bool::c::rustc_calls_cc::bool_val_in_15",
"bool::c::rustc_calls_cc::bool_val_in_16",
"bool::c::rustc_calls_cc::bool_struct_in_1",
"bool::c::rustc_calls_cc::bool_struct_in_2",
"bool::c::rustc_calls_cc::bool_struct_in_3",
"bool::c::rustc_calls_cc::bool_struct_in_4",
"bool::c::rustc_calls_cc::bool_struct_in_5",
"bool::c::rustc_calls_cc::bool_struct_in_6",
"bool::c::rustc_calls_cc::bool_struct_in_7",
"bool::c::rustc_calls_cc::bool_struct_in_8",
"bool::c::rustc_calls_cc::bool_struct_in_9",
"bool::c::rustc_calls_cc::bool_struct_in_10",
"bool::c::rustc_calls_cc::bool_struct_in_11",
"bool::c::rustc_calls_cc::bool_struct_in_12",
"bool::c::rustc_calls_cc::bool_struct_in_13",
"bool::c::rustc_calls_cc::bool_struct_in_14",
"bool::c::rustc_calls_cc::bool_struct_in_15",
"bool::c::rustc_calls_cc::bool_struct_in_16",
"bool::c::rustc_calls_cc::bool_ref_struct_in_1",
"bool::c::rustc_calls_cc::bool_ref_struct_in_2",
"bool::c::rustc_calls_cc::bool_ref_struct_in_3",
"bool::c::rustc_calls_cc::bool_ref_struct_in_4",
"bool::c::rustc_calls_cc::bool_ref_struct_in_5",
"bool::c::rustc_calls_cc::bool_ref_struct_in_6",
"bool::c::rustc_calls_cc::bool_ref_struct_in_7",
"bool::c::rustc_calls_cc::bool_ref_struct_in_8",
"bool::c::rustc_calls_cc::bool_ref_struct_in_9",
"bool::c::rustc_calls_cc::bool_ref_struct_in_10",
"bool::c::rustc_calls_cc::bool_ref_struct_in_11",
"bool::c::rustc_calls_cc::bool_ref_struct_in_12",
"bool::c::rustc_calls_cc::bool_ref_struct_in_13",
"bool::c::rustc_calls_cc::bool_ref_struct_in_14",
"bool::c::rustc_calls_cc::bool_ref_struct_in_15",
"bool::c::rustc_calls_cc::bool_ref_struct_in_16",
"bool::c::rustc_calls_cc::bool_val_in_0_perturbed_small",
"bool::c::rustc_calls_cc::bool_val_in_1_perturbed_small",
"bool::c::rustc_calls_cc::bool_val_in_2_perturbed_small",
"bool::c::rustc_calls_cc::bool_val_in_3_perturbed_small",
"bool::c::rustc_calls_cc::bool_val_in_0_perturbed_big",
"bool::c::rustc_calls_cc::bool_val_in_1_perturbed_big",
"bool::c::rustc_calls_cc::bool_val_in_2_perturbed_big",
"bool::c::rustc_calls_cc::bool_val_in_3_perturbed_big",
"bool::c::rustc_calls_cc::bool_val_in_4_perturbed_big",
"bool::c::rustc_calls_cc::bool_val_in_5_perturbed_big",
"bool::c::rustc_calls_cc::bool_val_in_6_perturbed_big",
"bool::c::rustc_calls_cc::bool_val_in_7_perturbed_big",
"bool::c::rustc_calls_cc::bool_val_in_8_perturbed_big",
"bool::c::rustc_calls_cc::bool_val_in_9_perturbed_big",
"bool::c::rustc_calls_cc::bool_val_in_10_perturbed_big",
"bool::c::rustc_calls_cc::bool_val_in_11_perturbed_big",
"bool::c::rustc_calls_cc::bool_val_in_12_perturbed_big",
"bool::c::rustc_calls_cc::bool_val_in_13_perturbed_big",
"bool::c::rustc_calls_cc::bool_val_in_14_perturbed_big",
"bool::c::rustc_calls_cc::bool_val_in_15_perturbed_big",
"bool::c::rustc_calls_cc::bool_struct_in_0_perturbed_small",
"bool::c::rustc_calls_cc::bool_struct_in_1_perturbed_small",
"bool::c::rustc_calls_cc::bool_struct_in_2_perturbed_small",
"bool::c::rustc_calls_cc::bool_struct_in_3_perturbed_small",
"bool::c::rustc_calls_cc::bool_struct_in_0_perturbed_big",
"bool::c::rustc_calls_cc::bool_struct_in_1_perturbed_big",
"bool::c::rustc_calls_cc::bool_struct_in_2_perturbed_big",
"bool::c::rustc_calls_cc::bool_struct_in_3_perturbed_big",
"bool::c::rustc_calls_cc::bool_struct_in_4_perturbed_big",
"bool::c::rustc_calls_cc::bool_struct_in_5_perturbed_big",
"bool::c::rustc_calls_cc::bool_struct_in_6_perturbed_big",
"bool::c::rustc_calls_cc::bool_struct_in_7_perturbed_big",
"bool::c::rustc_calls_cc::bool_struct_in_8_perturbed_big",
"bool::c::rustc_calls_cc::bool_struct_in_9_perturbed_big",
"bool::c::rustc_calls_cc::bool_struct_in_10_perturbed_big",
"bool::c::rustc_calls_cc::bool_struct_in_11_perturbed_big",
"bool::c::rustc_calls_cc::bool_struct_in_12_perturbed_big",
"bool::c::rustc_calls_cc::bool_struct_in_13_perturbed_big",
"bool::c::rustc_calls_cc::bool_struct_in_14_perturbed_big",
"bool::c::rustc_calls_cc::bool_struct_in_15_perturbed_big",
"bool::c::rustc_calls_cc::bool_ref_struct_in_0_perturbed_small",
"bool::c::rustc_calls_cc::bool_ref_struct_in_1_perturbed_small",
"bool::c::rustc_calls_cc::bool_ref_struct_in_2_perturbed_small",
"bool::c::rustc_calls_cc::bool_ref_struct_in_3_perturbed_small",
"bool::c::rustc_calls_cc::bool_ref_struct_in_0_perturbed_big",
"bool::c::rustc_calls_cc::bool_ref_struct_in_1_perturbed_big",
"bool::c::rustc_calls_cc::bool_ref_struct_in_2_perturbed_big",
"bool::c::rustc_calls_cc::bool_ref_struct_in_3_perturbed_big",
"bool::c::rustc_calls_cc::bool_ref_struct_in_4_perturbed_big",
"bool::c::rustc_calls_cc::bool_ref_struct_in_5_perturbed_big",
"bool::c::rustc_calls_cc::bool_ref_struct_in_6_perturbed_big",
"bool::c::rustc_calls_cc::bool_ref_struct_in_7_perturbed_big",
"bool::c::rustc_calls_cc::bool_ref_struct_in_8_perturbed_big",
"bool::c::rustc_calls_cc::bool_ref_struct_in_9_perturbed_big",
"bool::c::rustc_calls_cc::bool_ref_struct_in_10_perturbed_big",
"bool::c::rustc_calls_cc::bool_ref_struct_in_11_perturbed_big",
"bool::c::rustc_calls_cc::bool_ref_struct_in_12_perturbed_big",
"bool::c::rustc_calls_cc::bool_ref_struct_in_13_perturbed_big",
"bool::c::rustc_calls_cc::bool_ref_struct_in_14_perturbed_big",
"bool::c::rustc_calls_cc::bool_ref_struct_in_15_perturbed_big"
],
"subtest_checks": [
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
},
{
"Ok": null
}
]
}
},
"conclusion": "Passed"
},
...I've added some really minimal/bad support for rustc's format with --output-format=rustc-json
--output-format=json has also been significantly cleaned up, and now spits out the values of each field more usefully, although running on windows results in an output json that is ~1 million lines.
I've also temporarily(?) forked cc-rs to remove all uses of stdout, so now stdout should be clean and only contain the "true" final report. Fork is here: https://github.com/Gankra/cc-rs
With all of these, I think this is basically good enough, feel free to file issues for refinements.