weird --storage-layout=generic behavior on popcount test

[karmacoma]

Using this example:

// SPDX-License-Identifier: MIT
pragma solidity ^0.8.15;
import "forge-std/Test.sol";
import {console2} from "forge-std/console2.sol";

contract Test43 is Test {
    // assume we have infinite money so let's just put this in storage
    uint8[256] public pop8 = [
        0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4,
        1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5,
        1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5,
        2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,
        1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5,
        2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,
        2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,
        3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7,
        1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5,
        2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,
        2, 3, 3, 4, 3, 4, 4, 5, 4, 4, 4, 5, 4, 5, 5, 7, // <-- bug, last number should be 6
        3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7,
        2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,
        3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7,
        3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7,
        4, 5, 5, 6, 5, 6, 6, 7, 5, 6, 6, 7, 6, 7, 7, 8
    ];


    function popcount_buggy(uint256 x) public view returns (uint256) {
        uint256 c = 0;
        for (uint8 i = 0; i < 32; i++) {
            c += pop8[x & 0xff];
            x >>= 8;
        }
        return c;
    }

    function popcount_slow(uint256 x) public pure returns (uint256) {
        uint256 c = 0;
        for (uint256 i = 0; i < 256; i++) {
            c += x & 1;
            x >>= 1;
        }
        return c;
    }

    /// @custom:halmos --loop 256
    // --storage-layout=generic
    function prove_popcount_equiv(uint256 x) external {
        assertEq(popcount_buggy(x), popcount_slow(x));
    }

    function test_popcount_sanity() public {
        assertEq(popcount_buggy(0), 0);
        assertEq(popcount_buggy(1), 1);

        assertEq(this.popcount_slow(0), 0);
        assertEq(this.popcount_slow(1), 1);
    }

    function validate_counterexample(uint256 cex) internal {
        uint256 buggy_result = popcount_buggy(cex);
        uint256 expected_result = popcount_slow(cex);
        console2.log(buggy_result);
        console2.log(expected_result);
        assertEq(buggy_result, expected_result);
    }

    /// @custom:halmos --loop 256 --storage-layout=generic
    function test_validate_counterexample_storage_layout_generic() public {
        uint256 cex = 0xd3985728720ee3efddac47bd80000bfffffed67efffb7e0a167e97100001d542;
        validate_counterexample(cex);
    }

    /// @custom:halmos --loop 256 --storage-layout=solidity
    function test_validate_counterexample_storage_layout_solidity() public {
        uint256 cex = 0xd3985728720ee3efddac47bd80000bfffffed67efffb7e0a167e97100001d542;
        validate_counterexample(cex);
    }
}

with foundry:

Running 2 tests for test/43_popcount.t.sol:Test43
[PASS] test_validate_counterexample_storage_layout_generic() (gas: 142004)
Logs:
  138
  138

[PASS] test_validate_counterexample_storage_layout_solidity() (gas: 141983)
Logs:
  138
  138

Test result: ok. 2 passed; 0 failed; 0 skipped; finished in 2.07ms

with halmos 0.1.8:

Running 2 tests for test/43_popcount.t.sol:Test43
0
138
[FAIL] test_validate_counterexample_storage_layout_generic() (paths: 0/1, time: 1.55s, bounds: [])
Counterexample: ∅
138
138
[PASS] test_validate_counterexample_storage_layout_solidity() (paths: 1/1, time: 1.46s, bounds: [])
Symbolic test result: 1 passed; 1 failed; time: 4.18s

[Daejun Park]

the function-level custom tag is not considered in the test contract creation or the setup function execution, which is desired.

here, the pop8 array is initialized during the test contract creation, for which the default storage layout (solidity) is used.
as a result, the tests with --storage-layout=generic will not work as expected, because the pop8 array is initialized in a different storage layout.

to prevent this, --storage-layout should be put in the contract-level custom tag, if compound-type storage variables are initialized during the test contract creation or the setup. otherwise, it could result in the mixed use of two storage layouts, leading to unexpected behaviors.

perhaps, it would be better to not allow --storage-layout in the function-level tag at all. wdyt?

[Daejun Park]

related to: #184

[karmacoma]

got it, thanks for investigating @daejunpark! closing this one then, improving handling of function level tags is a separate task