split each test group into separate files (#457)

pull/466/head
Will Song 4 years ago committed by GitHub
parent ff94960bd1
commit 70fecc9938
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 5
      lib/Echidna/Solidity.hs
  2. 105
      src/test/Common.hs
  3. 340
      src/test/Spec.hs
  4. 42
      src/test/Tests/Compile.hs
  5. 28
      src/test/Tests/Config.hs
  6. 37
      src/test/Tests/Encoding.hs
  7. 138
      src/test/Tests/Integration.hs
  8. 13
      src/test/Tests/Research.hs
  9. 23
      src/test/Tests/Seed.hs

@ -54,6 +54,8 @@ import qualified Data.List.NonEmpty.Extra as NEE
import qualified Data.HashMap.Strict as M
import qualified Data.Text as T
data Filter = Blacklist [Text] | Whitelist [Text] deriving Show
-- | Things that can go wrong trying to load a Solidity file for Echidna testing. Read the 'Show'
-- instance for more detailed explanations.
data SolException = BadAddr Addr
@ -70,6 +72,7 @@ data SolException = BadAddr Addr
| DeploymentFailed
| NoCryticCompile
| InvalidMethodFilters Filter
makePrisms ''SolException
instance Show SolException where
show = \case
@ -90,8 +93,6 @@ instance Show SolException where
instance Exception SolException
data Filter = Blacklist [Text] | Whitelist [Text] deriving Show
-- | Configuration for loading Solidity for Echidna testing.
data SolConf = SolConf { _contractAddr :: Addr -- ^ Contract address to use
, _deployer :: Addr -- ^ Contract deployer address to use

@ -0,0 +1,105 @@
module Common
( testConfig
, runContract
, testContract
, testContract'
, solnFor
, solved
, passed
, solvedLen
, solvedWith
, getGas
, gasInRange
, countCorpus
, testsEmpty
, coverageEmpty
) where
import Prelude hiding (lookup)
import Test.Tasty (TestTree)
import Test.Tasty.HUnit (testCase, assertBool)
import Control.Lens (view, set, (.~))
import Control.Monad.Reader (runReaderT)
import Control.Monad.Random (getRandom)
import Data.Function ((&))
import Data.List (find)
import Data.List.NonEmpty (NonEmpty(..))
import Data.Map (lookup, empty)
import Data.Maybe (isJust)
import Data.Text (Text)
import Echidna (prepareContract)
import Echidna.Campaign (campaign)
import Echidna.Config (EConfig, _econfig, parseConfig, defaultConfig, sConf, cConf)
import Echidna.Solidity (quiet)
import Echidna.Types.Campaign (Campaign, TestState(..), testLimit, shrinkLimit, tests, gasInfo, corpus, coverage)
import Echidna.Types.Signature (SolCall)
import Echidna.Types.Tx (Tx(..), TxCall(..), call)
testConfig :: EConfig
testConfig = defaultConfig & sConf . quiet .~ True
& cConf . testLimit .~ 10000
& cConf . shrinkLimit .~ 4000
runContract :: FilePath -> Maybe String -> EConfig -> IO Campaign
runContract f c cfg =
flip runReaderT cfg $ do
g <- getRandom
(v, w, ts, d, txs) <- prepareContract cfg (f :| []) c g
-- start ui and run tests
campaign (pure ()) v w ts d txs
testContract :: FilePath -> Maybe FilePath -> [(String, Campaign -> Bool)] -> TestTree
testContract fp cfg = testContract' fp Nothing cfg True
testContract' :: FilePath -> Maybe String -> Maybe FilePath -> Bool -> [(String, Campaign -> Bool)] -> TestTree
testContract' fp n cfg s as = testCase fp $ do
c <- set (sConf . quiet) True <$> maybe (pure testConfig) (fmap _econfig . parseConfig) cfg
let c' = c & sConf . quiet .~ True
& (if s then cConf . testLimit .~ 10000 else id)
& (if s then cConf . shrinkLimit .~ 4000 else id)
res <- runContract fp n c'
mapM_ (\(t,f) -> assertBool t $ f res) as
getResult :: Text -> Campaign -> Maybe TestState
getResult t = fmap snd <$> find ((t ==) . either fst (("ASSERTION " <>) . fst) . fst) . view tests
solnFor :: Text -> Campaign -> Maybe [Tx]
solnFor t c = case getResult t c of
Just (Large _ s) -> Just s
Just (Solved s) -> Just s
_ -> Nothing
solved :: Text -> Campaign -> Bool
solved t = isJust . solnFor t
passed :: Text -> Campaign -> Bool
passed t c = case getResult t c of
Just (Open _) -> True
Just Passed -> True
_ -> False
solvedLen :: Int -> Text -> Campaign -> Bool
solvedLen i t = (== Just i) . fmap length . solnFor t
-- NOTE: this just verifies a call was found in the solution. Doesn't care about ordering/seq length
solvedWith :: SolCall -> Text -> Campaign -> Bool
solvedWith c t = maybe False (any $ (== SolCall c) . view call) . solnFor t
getGas :: Text -> Campaign -> Maybe (Int, [Tx])
getGas t = lookup t . view gasInfo
gasInRange :: Text -> Int -> Int -> Campaign -> Bool
gasInRange t l h c = case getGas t c of
Just (g, _) -> g >= l && g <= h
_ -> False
countCorpus :: Int -> Campaign -> Bool
countCorpus n c = length (view corpus c) == n
testsEmpty :: Campaign -> Bool
testsEmpty c = null (view tests c)
coverageEmpty :: Campaign -> Bool
coverageEmpty c = view coverage c == empty

@ -1,97 +1,30 @@
{-# LANGUAGE LambdaCase #-}
{-# LANGUAGE FlexibleContexts #-}
{-# LANGUAGE NamedFieldPuns #-}
import Test.Tasty
import Test.Tasty.HUnit
import Test.Tasty.QuickCheck(Arbitrary(..), Gen, (===), property, testProperty, resize)
import EVM.ABI (AbiValue(..))
import EVM.Types (Addr)
import qualified EVM.Concrete(Word(..))
import Echidna
import Echidna.Types.Campaign (Campaign, CampaignConf(..), TestState(..), tests, gasInfo, testLimit, shrinkLimit, knownCoverage, corpus, coverage)
import Echidna.Campaign (campaign)
import Echidna.Config (EConfig, EConfigWithUsage(..), _econfig, defaultConfig, parseConfig, sConf, cConf)
import Echidna.Solidity
import Echidna.Types.Signature (SolCall)
import Echidna.Types.Tx (TxCall(..), Tx(..), call)
import Data.Aeson (encode, decode)
import Control.Lens
import Control.Monad (liftM2, void, when)
import Control.Monad.Catch (MonadCatch(..))
import Control.Monad.Random (getRandom)
import Control.Monad.Reader (runReaderT)
import Data.Map (lookup, empty)
import Data.Maybe (isJust, maybe)
import Data.Text (Text, unpack, pack)
import Data.List (find, isInfixOf)
import Test.Tasty (defaultMain, testGroup)
import System.Directory (withCurrentDirectory)
import System.Process (readProcess)
import qualified Data.List.NonEmpty as NE
import Tests.Compile (compilationTests)
import Tests.Config (configTests)
import Tests.Encoding (encodingJSONTests)
import Tests.Integration (integrationTests)
import Tests.Research (researchTests)
import Tests.Seed (seedTests)
main :: IO ()
main = withCurrentDirectory "./examples/solidity" . defaultMain $
testGroup "Echidna" [ configTests
, compilationTests
, seedTests
, integrationTests
, researchTests
, encodingJSONTests
]
-- Configuration Tests
configTests :: TestTree
configTests = testGroup "Configuration tests" $
[ testCase file $ void $ parseConfig file | file <- files ] ++
[ testCase "parse \"coverage: true\"" $ do
config <- _econfig <$> parseConfig "coverage/test.yaml"
assertCoverage config $ Just mempty
, testCase "coverage disabled by default" $
assertCoverage defaultConfig Nothing
, testCase "default.yaml" $ do
EConfigWithUsage _ bad unset <- parseConfig "basic/default.yaml"
assertBool ("unused options: " ++ show bad) $ null bad
let unset' = unset & sans "seed"
assertBool ("unset options: " ++ show unset') $ null unset'
]
where files = ["basic/config.yaml", "basic/default.yaml"]
assertCoverage config value = (config ^. cConf . knownCoverage) @?= value
-- Compilation Tests
compilationTests :: TestTree
compilationTests = testGroup "Compilation and loading tests"
[ loadFails "bad/nocontract.sol" (Just "c") "failed to warn on contract not found" $
\case ContractNotFound{} -> True; _ -> False
, loadFails "bad/nobytecode.sol" Nothing "failed to warn on abstract contract" $
\case NoBytecode{} -> True; _ -> False
, loadFails "bad/nofuncs.sol" Nothing "failed to warn on no functions found" $
\case NoFuncs{} -> True; _ -> False
, loadFails "bad/notests.sol" Nothing "failed to warn on no tests found" $
\case NoTests{} -> True; _ -> False
, loadFails "bad/onlytests.sol" Nothing "failed to warn on no non-tests found" $
\case OnlyTests{} -> True; _ -> False
, loadFails "bad/testargs.sol" Nothing "failed to warn on test args found" $
\case TestArgsFound{} -> True; _ -> False
, loadFails "bad/consargs.sol" Nothing "failed to warn on cons args found" $
\case ConstructorArgs{} -> True; _ -> False
, loadFails "bad/revert.sol" Nothing "failed to warn on a failed deployment" $
\case DeploymentFailed{} -> True; _ -> False
]
loadFails :: FilePath -> Maybe Text -> String -> (SolException -> Bool) -> TestTree
loadFails fp c e p = testCase fp . catch tryLoad $ assertBool e . p where
tryLoad = runReaderT (loadWithCryticCompile (fp NE.:| []) c >> pure ()) testConfig
testGroup "Echidna"
[ configTests
, compilationTests
, seedTests
, integrationTests
, researchTests
, encodingJSONTests
]
-- Extraction Tests
-- We need to rethink this test
{-
extractionTests :: TestTree
@ -107,246 +40,3 @@ extractionTests = testGroup "Constant extraction/generation testing"
] $ \(t, c) -> liftIO . assertBool ("failed to extract " ++ t ++ " " ++ show (c,is)) $ elem c is
]
-}
seedTests :: TestTree
seedTests =
testGroup "Seed reproducibility testing"
[ testCase "different seeds" $ assertBool "results are the same" . not =<< same 0 1
, testCase "same seeds" $ assertBool "results differ" =<< same 0 0
]
where cfg s = defaultConfig & sConf . quiet .~ True
& cConf .~ CampaignConf 600 False False 20 0 Nothing (Just s) 0.15 Nothing (1,1,1)
gen s = view tests <$> runContract "basic/flags.sol" Nothing (cfg s)
same s t = liftM2 (==) (gen s) (gen t)
-- Integration Tests
integrationTests :: TestTree
integrationTests = testGroup "Solidity Integration Testing"
[ testContract "basic/true.sol" Nothing
[ ("echidna_true failed", passed "echidna_true") ]
, testContract "basic/flags.sol" Nothing
[ ("echidna_alwaystrue failed", passed "echidna_alwaystrue")
, ("echidna_revert_always failed", passed "echidna_revert_always")
, ("echidna_sometimesfalse passed", solved "echidna_sometimesfalse")
, ("echidna_sometimesfalse didn't shrink optimally", solvedLen 2 "echidna_sometimesfalse")
]
, testContract "basic/flags.sol" (Just "basic/whitelist.yaml")
[ ("echidna_alwaystrue failed", passed "echidna_alwaystrue")
, ("echidna_revert_always failed", passed "echidna_revert_always")
, ("echidna_sometimesfalse passed", passed "echidna_sometimesfalse")
]
, testContract "basic/flags.sol" (Just "basic/whitelist_all.yaml")
[ ("echidna_alwaystrue failed", passed "echidna_alwaystrue")
, ("echidna_revert_always failed", passed "echidna_revert_always")
, ("echidna_sometimesfalse passed", solved "echidna_sometimesfalse")
]
, testContract "basic/flags.sol" (Just "basic/blacklist.yaml")
[ ("echidna_alwaystrue failed", passed "echidna_alwaystrue")
, ("echidna_revert_always failed", passed "echidna_revert_always")
, ("echidna_sometimesfalse passed", passed "echidna_sometimesfalse")
]
, testContract "basic/revert.sol" Nothing
[ ("echidna_fails_on_revert passed", solved "echidna_fails_on_revert")
, ("echidna_fails_on_revert didn't shrink to one transaction",
solvedLen 1 "echidna_fails_on_revert")
, ("echidna_revert_is_false didn't shrink to f(-1, 0x0, 0xdeadbeef)",
solvedWith ("f", [AbiInt 256 (-1), AbiAddress 0, AbiAddress 0xdeadbeef]) "echidna_fails_on_revert")
]
, testContract "basic/nearbyMining.sol" (Just "coverage/test.yaml")
[ ("echidna_findNearby passed", solved "echidna_findNearby") ]
, testContract' "basic/smallValues.sol" Nothing (Just "coverage/test.yaml") False
[ ("echidna_findSmall passed", solved "echidna_findSmall") ]
, testContract "basic/multisender.sol" (Just "basic/multisender.yaml") $
[ ("echidna_all_sender passed", solved "echidna_all_sender")
, ("echidna_all_sender didn't shrink optimally", solvedLen 3 "echidna_all_sender")
] ++ (["s1", "s2", "s3"] <&> \n ->
("echidna_all_sender solved without " ++ unpack n, solvedWith (n, []) "echidna_all_sender"))
, testContract "basic/memory-reset.sol" Nothing
[ ("echidna_memory failed", passed "echidna_memory") ]
, testContract "basic/contractAddr.sol" Nothing
[ ("echidna_address failed", solved "echidna_address") ]
, testContract "basic/contractAddr.sol" (Just "basic/contractAddr.yaml")
[ ("echidna_address failed", passed "echidna_address") ]
, testContract "basic/constants.sol" Nothing
[ ("echidna_found failed", solved "echidna_found") ]
, testContract "basic/constants2.sol" Nothing
[ ("echidna_found32 failed", solved "echidna_found32") ]
, testContract "basic/constants3.sol" Nothing
[ ("echidna_found_sender failed", solved "echidna_found_sender") ]
, testContract "basic/rconstants.sol" Nothing
[ ("echidna_found failed", solved "echidna_found") ]
, testContract' "basic/cons-create-2.sol" (Just "C") Nothing True
[ ("echidna_state failed", solved "echidna_state") ]
-- single.sol is really slow and kind of unstable. it also messes up travis.
-- , testContract "coverage/single.sol" (Just "coverage/test.yaml")
-- [ ("echidna_state failed", solved "echidna_state") ]
, testContract "coverage/multi.sol" Nothing
[ ("echidna_state3 failed", solved "echidna_state3") ]
, testContract "basic/balance.sol" (Just "basic/balance.yaml")
[ ("echidna_balance failed", passed "echidna_balance") ]
, testContract "basic/library.sol" (Just "basic/library.yaml")
[ ("echidna_library_call failed", solved "echidna_library_call") ]
, testContract "basic/fallback.sol" Nothing
[ ("echidna_fallback failed", solved "echidna_fallback") ]
, testContract "basic/darray.sol" Nothing
[ ("echidna_darray passed", solved "echidna_darray")
, ("echidna_darray didn't shrink optimally", solvedLen 1 "echidna_darray") ]
, testContract "basic/propGasLimit.sol" (Just "basic/propGasLimit.yaml")
[ ("echidna_runForever passed", solved "echidna_runForever") ]
, testContract "basic/assert.sol" (Just "basic/assert.yaml")
[ ("echidna_set0 passed", solved "ASSERTION set0")
, ("echidna_set1 failed", passed "ASSERTION set1") ]
, testContract "basic/assert.sol" (Just "basic/benchmark.yaml")
[ ("coverage is empty", not . coverageEmpty )
, ("tests are not empty", testsEmpty ) ]
, testContract "basic/constants.sol" (Just "basic/benchmark.yaml")
[ ("coverage is empty", not . coverageEmpty )
, ("tests are not empty", testsEmpty ) ]
, testContract "basic/time.sol" (Just "basic/time.yaml")
[ ("echidna_timepassed passed", solved "echidna_timepassed") ]
, testContract "basic/construct.sol" Nothing
[ ("echidna_construct passed", solved "echidna_construct") ]
, testContract "basic/gasprice.sol" Nothing
[ ("echidna_state passed", solved "echidna_state") ]
, let fp = "basic_multicontract/contracts/Foo.sol"; cfg = Just "basic_multicontract/echidna_config.yaml" in
testCase fp $
do sv <- readProcess "solc" ["--version"] ""
when ("Version: 0.4.25" `isInfixOf` sv) $ do
c <- set (sConf . quiet) True <$> maybe (pure testConfig) (fmap _econfig . parseConfig) cfg
res <- runContract fp (Just "Foo") c
assertBool "echidna_test passed" $ solved "echidna_test" res
, testContract' "basic/multi-abi.sol" (Just "B") (Just "basic/multi-abi.yaml") True
[ ("echidna_test passed", solved "echidna_test") ]
, testContract "abiv2/Ballot.sol" Nothing
[ ("echidna_test passed", solved "echidna_test") ]
, testContract "abiv2/Dynamic.sol" Nothing
[ ("echidna_test passed", solved "echidna_test") ]
, testContract "abiv2/Dynamic2.sol" Nothing
[ ("echidna_test passed", solved "echidna_test") ]
, testContract "abiv2/MultiTuple.sol" Nothing
[ ("echidna_test passed", solved "echidna_test") ]
, testContract "basic/array-mutation.sol" Nothing
[ ("echidna_mutated passed", solved "echidna_mutated") ]
, testContract "basic/darray-mutation.sol" Nothing
[ ("echidna_mutated passed", solved "echidna_mutated") ]
, testContract "basic/gasuse.sol" (Just "basic/gasuse.yaml")
[ ("echidna_true failed", passed "echidna_true")
, ("g gas estimate wrong", gasInRange "g" 15000000 40000000)
, ("f_close1 gas estimate wrong", gasInRange "f_close1" 1800 2000)
, ("f_open1 gas estimate wrong", gasInRange "f_open1" 18000 23000)
, ("push_b gas estimate wrong", gasInRange "push_b" 39000 45000)
]
, testContract "coverage/boolean.sol" (Just "coverage/boolean.yaml")
[ ("echidna_true failed", passed "echidna_true")
, ("unexpected corpus count ", countCorpus 6)]
, testContract "basic/payable.sol" Nothing
[ ("echidna_payable failed", solved "echidna_payable") ]
]
researchTests :: TestTree
researchTests = testGroup "Research-based Integration Testing"
[ testContract "research/harvey_foo.sol" Nothing
[ ("echidna_assert failed", solved "echidna_assert") ]
, testContract "research/harvey_baz.sol" Nothing
[ ("echidna_all_states failed", solved "echidna_all_states") ]
]
testConfig :: EConfig
testConfig = defaultConfig & sConf . quiet .~ True
& cConf . testLimit .~ 10000
& cConf . shrinkLimit .~ 4000
testContract :: FilePath -> Maybe FilePath -> [(String, Campaign -> Bool)] -> TestTree
testContract fp cfg = testContract' fp Nothing cfg True
testContract' :: FilePath -> Maybe String -> Maybe FilePath -> Bool -> [(String, Campaign -> Bool)] -> TestTree
testContract' fp n cfg s as = testCase fp $ do
c <- set (sConf . quiet) True <$> maybe (pure testConfig) (fmap _econfig . parseConfig) cfg
let c' = c & sConf . quiet .~ True
& (if s then cConf . testLimit .~ 10000 else id)
& (if s then cConf . shrinkLimit .~ 4000 else id)
res <- runContract fp n c'
mapM_ (\(t,f) -> assertBool t $ f res) as
runContract :: FilePath -> Maybe String -> EConfig -> IO Campaign
runContract f c cfg =
flip runReaderT cfg $ do
g <- getRandom
(v, w, ts, d, txs) <- prepareContract cfg (f NE.:| []) c g
-- start ui and run tests
campaign (pure ()) v w ts d txs
getResult :: Text -> Campaign -> Maybe TestState
getResult t = fmap snd <$> find ((t ==) . either fst (("ASSERTION " <>) . fst) . fst) . view tests
getGas :: Text -> Campaign -> Maybe (Int, [Tx])
getGas t = Data.Map.lookup t . view gasInfo
gasInRange :: Text -> Int -> Int -> Campaign -> Bool
gasInRange t l h c = case getGas t c of
Just (g, _) -> g >= l && g <= h
_ -> False
countCorpus :: Int -> Campaign -> Bool
countCorpus n c = length (view corpus c) == n
testsEmpty :: Campaign -> Bool
testsEmpty c = null (view tests c)
coverageEmpty :: Campaign -> Bool
coverageEmpty c = view coverage c == empty
solnFor :: Text -> Campaign -> Maybe [Tx]
solnFor t c = case getResult t c of
Just (Large _ s) -> Just s
Just (Solved s) -> Just s
_ -> Nothing
solved :: Text -> Campaign -> Bool
solved t = isJust . solnFor t
passed :: Text -> Campaign -> Bool
passed t c = case getResult t c of
Just (Open _) -> True
Just Passed -> True
_ -> False
solvedLen :: Int -> Text -> Campaign -> Bool
solvedLen i t = (== Just i) . fmap length . solnFor t
-- NOTE: this just verifies a call was found in the solution. Doesn't care about ordering/seq length
solvedWith :: SolCall -> Text -> Campaign -> Bool
solvedWith c t = maybe False (any $ (== SolCall c) . view call) . solnFor t
-- Encoding JSON tests
instance Arbitrary Addr where
arbitrary = fromInteger <$> arbitrary
instance Arbitrary EVM.Concrete.Word where
arbitrary = fromInteger <$> arbitrary
instance Arbitrary TxCall where
arbitrary = do
s <- arbitrary
cs <- resize 32 arbitrary
return $ SolCall (pack s, cs)
instance Arbitrary Tx where
arbitrary = let a :: Arbitrary a => Gen a
a = arbitrary in
Tx <$> a <*> a <*> a <*> a <*> a <*> a <*> a
encodingJSONTests :: TestTree
encodingJSONTests =
testGroup "Tx JSON encoding"
[ testProperty "decode . encode = id" $ property $ do
t <- arbitrary :: Gen Tx
return $ decode (encode t) === Just t
]

@ -0,0 +1,42 @@
{-# LANGUAGE Rank2Types #-}
module Tests.Compile (compilationTests) where
import Test.Tasty (TestTree, testGroup)
import Test.Tasty.HUnit (testCase, assertBool)
import Common (testConfig)
import Control.Lens (Prism', preview)
import Control.Monad.Catch (catch)
import Control.Monad.Reader (runReaderT)
import Data.List.NonEmpty (NonEmpty(..))
import Data.Maybe (isJust)
import Data.Text (Text)
import Echidna.Solidity (SolException, _ContractNotFound, _NoBytecode, _NoFuncs, _NoTests, _OnlyTests, _TestArgsFound, _ConstructorArgs, _DeploymentFailed, loadWithCryticCompile)
compilationTests :: TestTree
compilationTests = testGroup "Compilation and loading tests"
[ loadFails "bad/nocontract.sol" (Just "c") "failed to warn on contract not found" $
pmatch _ContractNotFound
, loadFails "bad/nobytecode.sol" Nothing "failed to warn on abstract contract" $
pmatch _NoBytecode
, loadFails "bad/nofuncs.sol" Nothing "failed to warn on no functions found" $
pmatch _NoFuncs
, loadFails "bad/notests.sol" Nothing "failed to warn on no tests found" $
pmatch _NoTests
, loadFails "bad/onlytests.sol" Nothing "failed to warn on no non-tests found" $
pmatch _OnlyTests
, loadFails "bad/testargs.sol" Nothing "failed to warn on test args found" $
pmatch _TestArgsFound
, loadFails "bad/consargs.sol" Nothing "failed to warn on cons args found" $
pmatch _ConstructorArgs
, loadFails "bad/revert.sol" Nothing "failed to warn on a failed deployment" $
pmatch _DeploymentFailed
]
loadFails :: FilePath -> Maybe Text -> String -> (SolException -> Bool) -> TestTree
loadFails fp c e p = testCase fp . catch tryLoad $ assertBool e . p where
tryLoad = runReaderT (loadWithCryticCompile (fp :| []) c >> pure ()) testConfig
pmatch :: Prism' s a -> s -> Bool
pmatch p = isJust . preview p

@ -0,0 +1,28 @@
module Tests.Config (configTests) where
import Test.Tasty (TestTree, testGroup)
import Test.Tasty.HUnit (testCase, assertBool, (@?=))
import Control.Lens ((^.), sans)
import Control.Monad (void)
import Data.Function ((&))
import Echidna.Config (EConfigWithUsage(..), defaultConfig, parseConfig, cConf)
import Echidna.Types.Campaign (knownCoverage)
configTests :: TestTree
configTests = testGroup "Configuration tests" $
[ testCase file . void $ parseConfig file | file <- files ] ++
[ testCase "parse \"coverage: true\"" $ do
config <- _econfig <$> parseConfig "coverage/test.yaml"
assertCoverage config $ Just mempty
, testCase "coverage disabled by default" $
assertCoverage defaultConfig Nothing
, testCase "default.yaml" $ do
EConfigWithUsage _ bad unset <- parseConfig "basic/default.yaml"
assertBool ("unused options: " ++ show bad) $ null bad
let unset' = unset & sans "seed"
assertBool ("unset options: " ++ show unset') $ null unset'
]
where files = ["basic/config.yaml", "basic/default.yaml"]
assertCoverage config value = (config ^. cConf . knownCoverage) @?= value

@ -0,0 +1,37 @@
module Tests.Encoding (encodingJSONTests) where
import Prelude hiding (Word)
import Test.Tasty (TestTree, testGroup)
import Test.Tasty.QuickCheck (Arbitrary(..), Gen, (===), property, testProperty, resize)
import Data.Aeson (encode, decode)
import Data.Text (pack)
import Echidna.Types.Tx (TxCall(..), Tx(..))
import EVM.Concrete (Word)
import EVM.Types (Addr)
instance Arbitrary Addr where
arbitrary = fromInteger <$> arbitrary
instance Arbitrary Word where
arbitrary = fromInteger <$> arbitrary
instance Arbitrary TxCall where
arbitrary = do
s <- arbitrary
cs <- resize 32 arbitrary
return $ SolCall (pack s, cs)
instance Arbitrary Tx where
arbitrary = Tx <$> a <*> a <*> a <*> a <*> a <*> a <*> a
where a :: Arbitrary a => Gen a
a = arbitrary
encodingJSONTests :: TestTree
encodingJSONTests =
testGroup "Tx JSON encoding"
[ testProperty "decode . encode = id" $ property $ do
t <- arbitrary :: Gen Tx
return $ decode (encode t) === Just t
]

@ -0,0 +1,138 @@
module Tests.Integration (integrationTests) where
import Test.Tasty (TestTree, testGroup)
import Test.Tasty.HUnit (testCase, assertBool)
import Common (runContract, testContract, testContract', testConfig, passed, solved, solvedLen, solvedWith, coverageEmpty, testsEmpty, gasInRange, countCorpus)
import Control.Lens (set)
import Control.Monad (when)
import Data.Functor ((<&>))
import Data.List (isInfixOf)
import Data.Text (unpack)
import Echidna.Config (_econfig, parseConfig, sConf)
import Echidna.Solidity (quiet)
import EVM.ABI (AbiValue(..))
import System.Process (readProcess)
integrationTests :: TestTree
integrationTests = testGroup "Solidity Integration Testing"
[ testContract "basic/true.sol" Nothing
[ ("echidna_true failed", passed "echidna_true") ]
, testContract "basic/flags.sol" Nothing
[ ("echidna_alwaystrue failed", passed "echidna_alwaystrue")
, ("echidna_revert_always failed", passed "echidna_revert_always")
, ("echidna_sometimesfalse passed", solved "echidna_sometimesfalse")
, ("echidna_sometimesfalse didn't shrink optimally", solvedLen 2 "echidna_sometimesfalse")
]
, testContract "basic/flags.sol" (Just "basic/whitelist.yaml")
[ ("echidna_alwaystrue failed", passed "echidna_alwaystrue")
, ("echidna_revert_always failed", passed "echidna_revert_always")
, ("echidna_sometimesfalse passed", passed "echidna_sometimesfalse")
]
, testContract "basic/flags.sol" (Just "basic/whitelist_all.yaml")
[ ("echidna_alwaystrue failed", passed "echidna_alwaystrue")
, ("echidna_revert_always failed", passed "echidna_revert_always")
, ("echidna_sometimesfalse passed", solved "echidna_sometimesfalse")
]
, testContract "basic/flags.sol" (Just "basic/blacklist.yaml")
[ ("echidna_alwaystrue failed", passed "echidna_alwaystrue")
, ("echidna_revert_always failed", passed "echidna_revert_always")
, ("echidna_sometimesfalse passed", passed "echidna_sometimesfalse")
]
, testContract "basic/revert.sol" Nothing
[ ("echidna_fails_on_revert passed", solved "echidna_fails_on_revert")
, ("echidna_fails_on_revert didn't shrink to one transaction",
solvedLen 1 "echidna_fails_on_revert")
, ("echidna_revert_is_false didn't shrink to f(-1, 0x0, 0xdeadbeef)",
solvedWith ("f", [AbiInt 256 (-1), AbiAddress 0, AbiAddress 0xdeadbeef]) "echidna_fails_on_revert")
]
, testContract "basic/nearbyMining.sol" (Just "coverage/test.yaml")
[ ("echidna_findNearby passed", solved "echidna_findNearby") ]
, testContract' "basic/smallValues.sol" Nothing (Just "coverage/test.yaml") False
[ ("echidna_findSmall passed", solved "echidna_findSmall") ]
, testContract "basic/multisender.sol" (Just "basic/multisender.yaml") $
[ ("echidna_all_sender passed", solved "echidna_all_sender")
, ("echidna_all_sender didn't shrink optimally", solvedLen 3 "echidna_all_sender")
] ++ (["s1", "s2", "s3"] <&> \n ->
("echidna_all_sender solved without " ++ unpack n, solvedWith (n, []) "echidna_all_sender"))
, testContract "basic/memory-reset.sol" Nothing
[ ("echidna_memory failed", passed "echidna_memory") ]
, testContract "basic/contractAddr.sol" Nothing
[ ("echidna_address failed", solved "echidna_address") ]
, testContract "basic/contractAddr.sol" (Just "basic/contractAddr.yaml")
[ ("echidna_address failed", passed "echidna_address") ]
, testContract "basic/constants.sol" Nothing
[ ("echidna_found failed", solved "echidna_found") ]
, testContract "basic/constants2.sol" Nothing
[ ("echidna_found32 failed", solved "echidna_found32") ]
, testContract "basic/constants3.sol" Nothing
[ ("echidna_found_sender failed", solved "echidna_found_sender") ]
, testContract "basic/rconstants.sol" Nothing
[ ("echidna_found failed", solved "echidna_found") ]
, testContract' "basic/cons-create-2.sol" (Just "C") Nothing True
[ ("echidna_state failed", solved "echidna_state") ]
-- single.sol is really slow and kind of unstable. it also messes up travis.
-- , testContract "coverage/single.sol" (Just "coverage/test.yaml")
-- [ ("echidna_state failed", solved "echidna_state") ]
, testContract "coverage/multi.sol" Nothing
[ ("echidna_state3 failed", solved "echidna_state3") ]
, testContract "basic/balance.sol" (Just "basic/balance.yaml")
[ ("echidna_balance failed", passed "echidna_balance") ]
, testContract "basic/library.sol" (Just "basic/library.yaml")
[ ("echidna_library_call failed", solved "echidna_library_call") ]
, testContract "basic/fallback.sol" Nothing
[ ("echidna_fallback failed", solved "echidna_fallback") ]
, testContract "basic/darray.sol" Nothing
[ ("echidna_darray passed", solved "echidna_darray")
, ("echidna_darray didn't shrink optimally", solvedLen 1 "echidna_darray") ]
, testContract "basic/propGasLimit.sol" (Just "basic/propGasLimit.yaml")
[ ("echidna_runForever passed", solved "echidna_runForever") ]
, testContract "basic/assert.sol" (Just "basic/assert.yaml")
[ ("echidna_set0 passed", solved "ASSERTION set0")
, ("echidna_set1 failed", passed "ASSERTION set1") ]
, testContract "basic/assert.sol" (Just "basic/benchmark.yaml")
[ ("coverage is empty", not . coverageEmpty )
, ("tests are not empty", testsEmpty ) ]
, testContract "basic/constants.sol" (Just "basic/benchmark.yaml")
[ ("coverage is empty", not . coverageEmpty )
, ("tests are not empty", testsEmpty ) ]
, testContract "basic/time.sol" (Just "basic/time.yaml")
[ ("echidna_timepassed passed", solved "echidna_timepassed") ]
, testContract "basic/construct.sol" Nothing
[ ("echidna_construct passed", solved "echidna_construct") ]
, testContract "basic/gasprice.sol" Nothing
[ ("echidna_state passed", solved "echidna_state") ]
, let fp = "basic_multicontract/contracts/Foo.sol"; cfg = Just "basic_multicontract/echidna_config.yaml" in
testCase fp $
do sv <- readProcess "solc" ["--version"] ""
when ("Version: 0.4.25" `isInfixOf` sv) $ do
c <- set (sConf . quiet) True <$> maybe (pure testConfig) (fmap _econfig . parseConfig) cfg
res <- runContract fp (Just "Foo") c
assertBool "echidna_test passed" $ solved "echidna_test" res
, testContract' "basic/multi-abi.sol" (Just "B") (Just "basic/multi-abi.yaml") True
[ ("echidna_test passed", solved "echidna_test") ]
, testContract "abiv2/Ballot.sol" Nothing
[ ("echidna_test passed", solved "echidna_test") ]
, testContract "abiv2/Dynamic.sol" Nothing
[ ("echidna_test passed", solved "echidna_test") ]
, testContract "abiv2/Dynamic2.sol" Nothing
[ ("echidna_test passed", solved "echidna_test") ]
, testContract "abiv2/MultiTuple.sol" Nothing
[ ("echidna_test passed", solved "echidna_test") ]
, testContract "basic/array-mutation.sol" Nothing
[ ("echidna_mutated passed", solved "echidna_mutated") ]
, testContract "basic/darray-mutation.sol" Nothing
[ ("echidna_mutated passed", solved "echidna_mutated") ]
, testContract "basic/gasuse.sol" (Just "basic/gasuse.yaml")
[ ("echidna_true failed", passed "echidna_true")
, ("g gas estimate wrong", gasInRange "g" 15000000 40000000)
, ("f_close1 gas estimate wrong", gasInRange "f_close1" 1800 2000)
, ("f_open1 gas estimate wrong", gasInRange "f_open1" 18000 23000)
, ("push_b gas estimate wrong", gasInRange "push_b" 39000 45000)
]
, testContract "coverage/boolean.sol" (Just "coverage/boolean.yaml")
[ ("echidna_true failed", passed "echidna_true")
, ("unexpected corpus count ", countCorpus 6)]
, testContract "basic/payable.sol" Nothing
[ ("echidna_payable failed", solved "echidna_payable") ]
]

@ -0,0 +1,13 @@
module Tests.Research (researchTests) where
import Test.Tasty (TestTree, testGroup)
import Common (testContract, solved)
researchTests :: TestTree
researchTests = testGroup "Research-based Integration Testing"
[ testContract "research/harvey_foo.sol" Nothing
[ ("echidna_assert failed", solved "echidna_assert") ]
, testContract "research/harvey_baz.sol" Nothing
[ ("echidna_all_states failed", solved "echidna_all_states") ]
]

@ -0,0 +1,23 @@
module Tests.Seed (seedTests) where
import Test.Tasty (TestTree, testGroup)
import Test.Tasty.HUnit (testCase, assertBool)
import Common (runContract)
import Control.Lens (view, (.~))
import Control.Monad (liftM2)
import Data.Function ((&))
import Echidna.Config (defaultConfig, sConf, cConf)
import Echidna.Solidity (quiet)
import Echidna.Types.Campaign (CampaignConf(..), tests)
seedTests :: TestTree
seedTests =
testGroup "Seed reproducibility testing"
[ testCase "different seeds" $ assertBool "results are the same" . not =<< same 0 1
, testCase "same seeds" $ assertBool "results differ" =<< same 0 0
]
where cfg s = defaultConfig & sConf . quiet .~ True
& cConf .~ CampaignConf 600 False False 20 0 Nothing (Just s) 0.15 Nothing (1,1,1)
gen s = view tests <$> runContract "basic/flags.sol" Nothing (cfg s)
same s t = liftM2 (==) (gen s) (gen t)
Loading…
Cancel
Save