criterion-1.1.4.0: Robust, reliable performance measurement and analysis

Copyright | (c) 2009-2014 Bryan O'Sullivan
License | BSD-style
Maintainer | bos@serpentine.com
Stability | experimental
Portability | GHC
Safe Haskell | Trustworthy
Language | Haskell98

Criterion.Analysis

Description

Analysis code for benchmarks.

Synopsis

- data Outliers = Outliers {
  - samplesSeen :: !Int64
  - lowSevere :: !Int64
  - lowMild :: !Int64
  - highMild :: !Int64
  - highSevere :: !Int64
  }
- data OutlierEffect
  - = Unaffected
  - | Slight
  - | Moderate
  - | Severe
- data OutlierVariance = OutlierVariance {
  - ovEffect :: OutlierEffect
  - ovDesc :: String
  - ovFraction :: Double
  }
- data SampleAnalysis = SampleAnalysis {
  - anRegress :: [Regression]
  - anOverhead :: Double
  - anMean :: Estimate
  - anStdDev :: Estimate
  - anOutlierVar :: OutlierVariance
  }
- analyseSample :: Int -> String -> Vector Measured -> ExceptT String Criterion Report
- scale :: Double -> SampleAnalysis -> SampleAnalysis
- analyseMean :: Sample -> Int -> Criterion Double
- countOutliers :: Outliers -> Int64
- classifyOutliers :: Sample -> Outliers
- noteOutliers :: Outliers -> Criterion ()
- outlierVariance :: Estimate -> Estimate -> Double -> OutlierVariance
- resolveAccessors :: [String] -> Either String [(String, Measured -> Maybe Double)]
- validateAccessors :: [String] -> String -> Either String [(String, Measured -> Maybe Double)]
- regress :: GenIO -> [String] -> String -> Vector Measured -> ExceptT String Criterion Regression

# Documentation

data Outliers #

Outliers from sample data, calculated using the boxplot technique.

Constructors

Outliers |
Fields samplesSeen :: !Int64 lowSevere :: !Int64 More than 3 times the interquartile range (IQR) below the first quartile. lowMild :: !Int64 Between 1.5 and 3 times the IQR below the first quartile. highMild :: !Int64 Between 1.5 and 3 times the IQR above the third quartile. highSevere :: !Int64 More than 3 times the IQR above the third quartile.

Instances

Eq Outliers # |
Methods (==) :: Outliers -> Outliers -> Bool # (/=) :: Outliers -> Outliers -> Bool #
Data Outliers # |
Methods gfoldl :: (forall d b. Data d => c (d -> b) -> d -> c b) -> (forall g. g -> c g) -> Outliers -> c Outliers # gunfold :: (forall b r. Data b => c (b -> r) -> c r) -> (forall r. r -> c r) -> Constr -> c Outliers # toConstr :: Outliers -> Constr # dataTypeOf :: Outliers -> DataType # dataCast1 :: Typeable (* -> *) t => (forall d. Data d => c (t d)) -> Maybe (c Outliers) # dataCast2 :: Typeable (* -> * -> *) t => (forall d e. (Data d, Data e) => c (t d e)) -> Maybe (c Outliers) # gmapT :: (forall b. Data b => b -> b) -> Outliers -> Outliers # gmapQl :: (r -> r' -> r) -> r -> (forall d. Data d => d -> r') -> Outliers -> r # gmapQr :: (r' -> r -> r) -> r -> (forall d. Data d => d -> r') -> Outliers -> r # gmapQ :: (forall d. Data d => d -> u) -> Outliers -> [u] # gmapQi :: Int -> (forall d. Data d => d -> u) -> Outliers -> u # gmapM :: Monad m => (forall d. Data d => d -> m d) -> Outliers -> m Outliers # gmapMp :: MonadPlus m => (forall d. Data d => d -> m d) -> Outliers -> m Outliers # gmapMo :: MonadPlus m => (forall d. Data d => d -> m d) -> Outliers -> m Outliers #
Read Outliers # |
Methods readsPrec :: Int -> ReadS Outliers # readList :: ReadS [Outliers] # readPrec :: ReadPrec Outliers # readListPrec :: ReadPrec [Outliers] #
Show Outliers # |
Methods showsPrec :: Int -> Outliers -> ShowS # show :: Outliers -> String # showList :: [Outliers] -> ShowS #
Generic Outliers # |
Associated Types type Rep Outliers :: * -> * # Methods from :: Outliers -> Rep Outliers x # to :: Rep Outliers x -> Outliers #
Monoid Outliers # |
Methods mempty :: Outliers # mappend :: Outliers -> Outliers -> Outliers # mconcat :: [Outliers] -> Outliers #
NFData Outliers # |
Methods rnf :: Outliers -> () #
FromJSON Outliers # |
Methods parseJSON :: Value -> Parser Outliers # parseJSONList :: Value -> Parser [Outliers] #
ToJSON Outliers # |
Methods toJSON :: Outliers -> Value # toEncoding :: Outliers -> Encoding # toJSONList :: [Outliers] -> Value # toEncodingList :: [Outliers] -> Encoding #
Binary Outliers # |
Methods put :: Outliers -> Put # get :: Get Outliers # putList :: [Outliers] -> Put #
type Rep Outliers # |
type Rep Outliers = D1 (MetaData "Outliers" "Criterion.Types" "criterion-1.1.4.0-BRaaIZoaUKgGjxk9ibCKy1" False) (C1 (MetaCons "Outliers" PrefixI True) ((:*:) ((:*:) (S1 (MetaSel (Just Symbol "samplesSeen") NoSourceUnpackedness SourceStrict DecidedUnpack) (Rec0 Int64)) (S1 (MetaSel (Just Symbol "lowSevere") NoSourceUnpackedness SourceStrict DecidedUnpack) (Rec0 Int64))) ((:*:) (S1 (MetaSel (Just Symbol "lowMild") NoSourceUnpackedness SourceStrict DecidedUnpack) (Rec0 Int64)) ((:*:) (S1 (MetaSel (Just Symbol "highMild") NoSourceUnpackedness SourceStrict DecidedUnpack) (Rec0 Int64)) (S1 (MetaSel (Just Symbol "highSevere") NoSourceUnpackedness SourceStrict DecidedUnpack) (Rec0 Int64))))))

data OutlierEffect #

A description of the extent to which outliers in the sample data affect the sample mean and standard deviation.

Constructors

Unaffected | Less than 1% effect.
Slight | Between 1% and 10%.
Moderate | Between 10% and 50%.
Severe | Above 50% (i.e. measurements are useless).

Instances

Eq OutlierEffect # |
Methods (==) :: OutlierEffect -> OutlierEffect -> Bool # (/=) :: OutlierEffect -> OutlierEffect -> Bool #
Data OutlierEffect # |
Methods gfoldl :: (forall d b. Data d => c (d -> b) -> d -> c b) -> (forall g. g -> c g) -> OutlierEffect -> c OutlierEffect # gunfold :: (forall b r. Data b => c (b -> r) -> c r) -> (forall r. r -> c r) -> Constr -> c OutlierEffect # toConstr :: OutlierEffect -> Constr # dataTypeOf :: OutlierEffect -> DataType # dataCast1 :: Typeable (* -> *) t => (forall d. Data d => c (t d)) -> Maybe (c OutlierEffect) # dataCast2 :: Typeable (* -> * -> *) t => (forall d e. (Data d, Data e) => c (t d e)) -> Maybe (c OutlierEffect) # gmapT :: (forall b. Data b => b -> b) -> OutlierEffect -> OutlierEffect # gmapQl :: (r -> r' -> r) -> r -> (forall d. Data d => d -> r') -> OutlierEffect -> r # gmapQr :: (r' -> r -> r) -> r -> (forall d. Data d => d -> r') -> OutlierEffect -> r # gmapQ :: (forall d. Data d => d -> u) -> OutlierEffect -> [u] # gmapQi :: Int -> (forall d. Data d => d -> u) -> OutlierEffect -> u # gmapM :: Monad m => (forall d. Data d => d -> m d) -> OutlierEffect -> m OutlierEffect # gmapMp :: MonadPlus m => (forall d. Data d => d -> m d) -> OutlierEffect -> m OutlierEffect # gmapMo :: MonadPlus m => (forall d. Data d => d -> m d) -> OutlierEffect -> m OutlierEffect #
Ord OutlierEffect # |
Methods compare :: OutlierEffect -> OutlierEffect -> Ordering # (<) :: OutlierEffect -> OutlierEffect -> Bool # (<=) :: OutlierEffect -> OutlierEffect -> Bool # (>) :: OutlierEffect -> OutlierEffect -> Bool # (>=) :: OutlierEffect -> OutlierEffect -> Bool # max :: OutlierEffect -> OutlierEffect -> OutlierEffect # min :: OutlierEffect -> OutlierEffect -> OutlierEffect #
Read OutlierEffect # |
Methods readsPrec :: Int -> ReadS OutlierEffect # readList :: ReadS [OutlierEffect] # readPrec :: ReadPrec OutlierEffect # readListPrec :: ReadPrec [OutlierEffect] #
Show OutlierEffect # |
Methods showsPrec :: Int -> OutlierEffect -> ShowS # show :: OutlierEffect -> String # showList :: [OutlierEffect] -> ShowS #
Generic OutlierEffect # |
Associated Types type Rep OutlierEffect :: * -> * # Methods from :: OutlierEffect -> Rep OutlierEffect x # to :: Rep OutlierEffect x -> OutlierEffect #
NFData OutlierEffect # |
Methods rnf :: OutlierEffect -> () #
FromJSON OutlierEffect # |
Methods parseJSON :: Value -> Parser OutlierEffect # parseJSONList :: Value -> Parser [OutlierEffect] #
ToJSON OutlierEffect # |
Methods toJSON :: OutlierEffect -> Value # toEncoding :: OutlierEffect -> Encoding # toJSONList :: [OutlierEffect] -> Value # toEncodingList :: [OutlierEffect] -> Encoding #
Binary OutlierEffect # |
Methods put :: OutlierEffect -> Put # get :: Get OutlierEffect # putList :: [OutlierEffect] -> Put #
type Rep OutlierEffect # |
type Rep OutlierEffect = D1 (MetaData "OutlierEffect" "Criterion.Types" "criterion-1.1.4.0-BRaaIZoaUKgGjxk9ibCKy1" False) ((:+:) ((:+:) (C1 (MetaCons "Unaffected" PrefixI False) U1) (C1 (MetaCons "Slight" PrefixI False) U1)) ((:+:) (C1 (MetaCons "Moderate" PrefixI False) U1) (C1 (MetaCons "Severe" PrefixI False) U1)))

data OutlierVariance #

Analysis of the extent to which outliers in a sample affect its standard deviation (and to some extent, its mean).

Constructors

OutlierVariance |
Fields ovEffect :: OutlierEffect Qualitative description of effect. ovDesc :: String Brief textual description of effect. ovFraction :: Double Quantitative description of effect (a fraction between 0 and 1).

Instances

Eq OutlierVariance # |
Methods (==) :: OutlierVariance -> OutlierVariance -> Bool # (/=) :: OutlierVariance -> OutlierVariance -> Bool #
Data OutlierVariance # |
Methods gfoldl :: (forall d b. Data d => c (d -> b) -> d -> c b) -> (forall g. g -> c g) -> OutlierVariance -> c OutlierVariance # gunfold :: (forall b r. Data b => c (b -> r) -> c r) -> (forall r. r -> c r) -> Constr -> c OutlierVariance # toConstr :: OutlierVariance -> Constr # dataTypeOf :: OutlierVariance -> DataType # dataCast1 :: Typeable (* -> *) t => (forall d. Data d => c (t d)) -> Maybe (c OutlierVariance) # dataCast2 :: Typeable (* -> * -> *) t => (forall d e. (Data d, Data e) => c (t d e)) -> Maybe (c OutlierVariance) # gmapT :: (forall b. Data b => b -> b) -> OutlierVariance -> OutlierVariance # gmapQl :: (r -> r' -> r) -> r -> (forall d. Data d => d -> r') -> OutlierVariance -> r # gmapQr :: (r' -> r -> r) -> r -> (forall d. Data d => d -> r') -> OutlierVariance -> r # gmapQ :: (forall d. Data d => d -> u) -> OutlierVariance -> [u] # gmapQi :: Int -> (forall d. Data d => d -> u) -> OutlierVariance -> u # gmapM :: Monad m => (forall d. Data d => d -> m d) -> OutlierVariance -> m OutlierVariance # gmapMp :: MonadPlus m => (forall d. Data d => d -> m d) -> OutlierVariance -> m OutlierVariance # gmapMo :: MonadPlus m => (forall d. Data d => d -> m d) -> OutlierVariance -> m OutlierVariance #
Read OutlierVariance # |
Methods readsPrec :: Int -> ReadS OutlierVariance # readList :: ReadS [OutlierVariance] # readPrec :: ReadPrec OutlierVariance # readListPrec :: ReadPrec [OutlierVariance] #
Show OutlierVariance # |
Methods showsPrec :: Int -> OutlierVariance -> ShowS # show :: OutlierVariance -> String # showList :: [OutlierVariance] -> ShowS #
Generic OutlierVariance # |
Associated Types type Rep OutlierVariance :: * -> * # Methods from :: OutlierVariance -> Rep OutlierVariance x # to :: Rep OutlierVariance x -> OutlierVariance #
NFData OutlierVariance # |
Methods rnf :: OutlierVariance -> () #
FromJSON OutlierVariance # |
Methods parseJSON :: Value -> Parser OutlierVariance # parseJSONList :: Value -> Parser [OutlierVariance] #
ToJSON OutlierVariance # |
Methods toJSON :: OutlierVariance -> Value # toEncoding :: OutlierVariance -> Encoding # toJSONList :: [OutlierVariance] -> Value # toEncodingList :: [OutlierVariance] -> Encoding #
Binary OutlierVariance # |
Methods put :: OutlierVariance -> Put # get :: Get OutlierVariance # putList :: [OutlierVariance] -> Put #
type Rep OutlierVariance # |
type Rep OutlierVariance = D1 (MetaData "OutlierVariance" "Criterion.Types" "criterion-1.1.4.0-BRaaIZoaUKgGjxk9ibCKy1" False) (C1 (MetaCons "OutlierVariance" PrefixI True) ((:*:) (S1 (MetaSel (Just Symbol "ovEffect") NoSourceUnpackedness NoSourceStrictness DecidedLazy) (Rec0 OutlierEffect)) ((:*:) (S1 (MetaSel (Just Symbol "ovDesc") NoSourceUnpackedness NoSourceStrictness DecidedLazy) (Rec0 String)) (S1 (MetaSel (Just Symbol "ovFraction") NoSourceUnpackedness NoSourceStrictness DecidedLazy) (Rec0 Double)))))

data SampleAnalysis #

Result of a bootstrap analysis of a non-parametric sample.

Constructors

SampleAnalysis |
Fields anRegress :: [Regression] Estimates calculated via linear regression. anOverhead :: Double Estimated measurement overhead, in seconds. Estimation is performed via linear regression. anMean :: Estimate Estimated mean. anStdDev :: Estimate Estimated standard deviation. anOutlierVar :: OutlierVariance Description of the effects of outliers on the estimated variance.

Instances

Eq SampleAnalysis # |
Methods (==) :: SampleAnalysis -> SampleAnalysis -> Bool # (/=) :: SampleAnalysis -> SampleAnalysis -> Bool #
Data SampleAnalysis # |
Methods gfoldl :: (forall d b. Data d => c (d -> b) -> d -> c b) -> (forall g. g -> c g) -> SampleAnalysis -> c SampleAnalysis # gunfold :: (forall b r. Data b => c (b -> r) -> c r) -> (forall r. r -> c r) -> Constr -> c SampleAnalysis # toConstr :: SampleAnalysis -> Constr # dataTypeOf :: SampleAnalysis -> DataType # dataCast1 :: Typeable (* -> *) t => (forall d. Data d => c (t d)) -> Maybe (c SampleAnalysis) # dataCast2 :: Typeable (* -> * -> *) t => (forall d e. (Data d, Data e) => c (t d e)) -> Maybe (c SampleAnalysis) # gmapT :: (forall b. Data b => b -> b) -> SampleAnalysis -> SampleAnalysis # gmapQl :: (r -> r' -> r) -> r -> (forall d. Data d => d -> r') -> SampleAnalysis -> r # gmapQr :: (r' -> r -> r) -> r -> (forall d. Data d => d -> r') -> SampleAnalysis -> r # gmapQ :: (forall d. Data d => d -> u) -> SampleAnalysis -> [u] # gmapQi :: Int -> (forall d. Data d => d -> u) -> SampleAnalysis -> u # gmapM :: Monad m => (forall d. Data d => d -> m d) -> SampleAnalysis -> m SampleAnalysis # gmapMp :: MonadPlus m => (forall d. Data d => d -> m d) -> SampleAnalysis -> m SampleAnalysis # gmapMo :: MonadPlus m => (forall d. Data d => d -> m d) -> SampleAnalysis -> m SampleAnalysis #
Read SampleAnalysis # |
Methods readsPrec :: Int -> ReadS SampleAnalysis # readList :: ReadS [SampleAnalysis] # readPrec :: ReadPrec SampleAnalysis # readListPrec :: ReadPrec [SampleAnalysis] #
Show SampleAnalysis # |
Methods showsPrec :: Int -> SampleAnalysis -> ShowS # show :: SampleAnalysis -> String # showList :: [SampleAnalysis] -> ShowS #
Generic SampleAnalysis # |
Associated Types type Rep SampleAnalysis :: * -> * # Methods from :: SampleAnalysis -> Rep SampleAnalysis x # to :: Rep SampleAnalysis x -> SampleAnalysis #
NFData SampleAnalysis # |
Methods rnf :: SampleAnalysis -> () #
FromJSON SampleAnalysis # |
Methods parseJSON :: Value -> Parser SampleAnalysis # parseJSONList :: Value -> Parser [SampleAnalysis] #
ToJSON SampleAnalysis # |
Methods toJSON :: SampleAnalysis -> Value # toEncoding :: SampleAnalysis -> Encoding # toJSONList :: [SampleAnalysis] -> Value # toEncodingList :: [SampleAnalysis] -> Encoding #
Binary SampleAnalysis # |
Methods put :: SampleAnalysis -> Put # get :: Get SampleAnalysis # putList :: [SampleAnalysis] -> Put #
type Rep SampleAnalysis # |
type Rep SampleAnalysis = D1 (MetaData "SampleAnalysis" "Criterion.Types" "criterion-1.1.4.0-BRaaIZoaUKgGjxk9ibCKy1" False) (C1 (MetaCons "SampleAnalysis" PrefixI True) ((:*:) ((:*:) (S1 (MetaSel (Just Symbol "anRegress") NoSourceUnpackedness NoSourceStrictness DecidedLazy) (Rec0 [Regression])) (S1 (MetaSel (Just Symbol "anOverhead") NoSourceUnpackedness NoSourceStrictness DecidedLazy) (Rec0 Double))) ((:*:) (S1 (MetaSel (Just Symbol "anMean") NoSourceUnpackedness NoSourceStrictness DecidedLazy) (Rec0 Estimate)) ((:*:) (S1 (MetaSel (Just Symbol "anStdDev") NoSourceUnpackedness NoSourceStrictness DecidedLazy) (Rec0 Estimate)) (S1 (MetaSel (Just Symbol "anOutlierVar") NoSourceUnpackedness NoSourceStrictness DecidedLazy) (Rec0 OutlierVariance))))))

analyseSample #

Arguments

:: Int | Experiment number.
-> String | Experiment name.
-> Vector Measured | Sample data.
-> ExceptT String Criterion Report |

Perform an analysis of a measurement.

scale #

Arguments

:: Double | Value to multiply by.
-> SampleAnalysis |
-> SampleAnalysis |

Multiply the `Estimate`s in an analysis by the given value, using `scale`.

analyseMean #

Arguments

:: Sample |
-> Int | Number of iterations used to compute the sample.
-> Criterion Double |

Display the mean of a `Sample`, and characterise the outliers present in the sample.

countOutliers :: Outliers -> Int64 #

Count the total number of outliers in a sample.

classifyOutliers :: Sample -> Outliers #

Classify outliers in a data set, using the boxplot technique.

noteOutliers :: Outliers -> Criterion () #

Display a report of the `Outliers` present in a `Sample`.

outlierVariance #

Arguments

:: Estimate | Bootstrap estimate of sample mean.
-> Estimate | Bootstrap estimate of sample standard deviation.
-> Double | Number of original iterations.
-> OutlierVariance |

Compute the extent to which outliers in the sample data affect the sample mean and standard deviation.

resolveAccessors :: [String] -> Either String [(String, Measured -> Maybe Double)] #

Given a list of accessor names (see `measureKeys`), return either a mapping from accessor name to function or an error message if any names are wrong.

validateAccessors #

Arguments

:: [String] | Predictor names.
-> String | Responder name.
-> Either String [(String, Measured -> Maybe Double)] |

Given predictor and responder names, do some basic validation, then hand back the relevant accessors.

regress #

Arguments

:: GenIO |
-> [String] | Predictor names.
-> String | Responder name.
-> Vector Measured |
-> ExceptT String Criterion Regression |

Regress the given predictors against the responder.

Errors may be returned under various circumstances, such as invalid names or lack of needed data.

See `olsRegress` for details of the regression performed.