benchmarkrunner.h

Go to the documentation of this file.

 
#pragma once
 
 
#include <promeki/config.h>
#if PROMEKI_ENABLE_CORE
#include <cstdint>
#include <functional>
#include <promeki/function.h>
#include <promeki/namespace.h>
#include <promeki/string.h>
#include <promeki/list.h>
#include <promeki/map.h>
#include <promeki/error.h>
#include <promeki/elapsedtimer.h>
#include <promeki/statsaccumulator.h>
#include <promeki/util.h>
 
PROMEKI_NAMESPACE_BEGIN
 
class JsonObject;
class BenchmarkRunner;
 
class BenchmarkState {
        public:
                class Iterator {
                        public:
                                Iterator(BenchmarkState *state, uint64_t pos) : _state(state), _pos(pos) {}
 
                                uint64_t operator*() const { return _pos; }
 
                                Iterator &operator++() {
                                        ++_pos;
                                        if (_state != nullptr && _pos >= _state->_iterations) {
                                                _state->finishTiming();
                                        }
                                        return *this;
                                }
 
                                bool operator!=(const Iterator &other) const { return _pos != other._pos; }
 
                        private:
                                BenchmarkState *_state;
                                uint64_t        _pos;
                };
 
                explicit BenchmarkState(uint64_t iterations) : _iterations(iterations) {}
 
                uint64_t iterations() const { return _iterations; }
 
                Iterator begin() {
                        ensureTimerStarted();
                        return Iterator(this, 0);
                }
 
                Iterator end() { return Iterator(this, _iterations); }
 
                bool keepRunning() {
                        if (_counter == 0) ensureTimerStarted();
                        if (_counter >= _iterations) {
                                finishTiming();
                                return false;
                        }
                        _counter++;
                        return true;
                }
 
                void pauseTiming() {
                        if (!_timerStarted || _paused) return;
                        _paused = true;
                        _pauseStartNs = _timer.elapsedNs();
                        return;
                }
 
                void resumeTiming() {
                        if (!_paused) return;
                        _paused = false;
                        _pausedNs += _timer.elapsedNs() - _pauseStartNs;
                        return;
                }
 
                void setItemsProcessed(uint64_t n) { _itemsProcessed = n; }
 
                void setBytesProcessed(uint64_t n) { _bytesProcessed = n; }
 
                void setLabel(const String &label) { _label = label; }
 
                void setCounter(const String &key, double value) {
                        _counters.insert(key, value);
                        return;
                }
 
                const String &label() const { return _label; }
 
                uint64_t itemsProcessed() const { return _itemsProcessed; }
 
                uint64_t bytesProcessed() const { return _bytesProcessed; }
 
                const Map<String, double> &counters() const { return _counters; }
 
                int64_t effectiveNs() const { return _effectiveNs; }
 
                int64_t wallNs() const { return _wallNs; }
 
                bool completed() const { return _completed; }
 
        private:
                friend class BenchmarkRunner;
 
                void ensureTimerStarted() {
                        if (_timerStarted) return;
                        _timerStarted = true;
                        _timer.start();
                }
 
                void finishTiming() {
                        if (_completed) return;
                        _completed = true;
                        if (_paused) {
                                _pausedNs += _timer.elapsedNs() - _pauseStartNs;
                                _paused = false;
                        }
                        _wallNs = _timer.elapsedNs();
                        _effectiveNs = _wallNs - _pausedNs;
                        if (_effectiveNs < 0) _effectiveNs = 0;
                        return;
                }
 
                void captureIfUnfinished() {
                        // If the case used range-for, iteration exits without
                        // calling keepRunning(), so finishTiming() hasn't run.
                        if (_completed) return;
                        if (_timerStarted) {
                                finishTiming();
                        } else {
                                // Case never touched the state at all — leave
                                // _completed false so the runner can flag it.
                        }
                }
 
                uint64_t            _iterations = 0;
                uint64_t            _counter = 0;
                bool                _timerStarted = false;
                bool                _paused = false;
                bool                _completed = false;
                ElapsedTimer        _timer;
                int64_t             _pauseStartNs = 0;
                int64_t             _pausedNs = 0;
                int64_t             _wallNs = 0;
                int64_t             _effectiveNs = 0;
                uint64_t            _itemsProcessed = 0;
                uint64_t            _bytesProcessed = 0;
                String              _label;
                Map<String, double> _counters;
};
 
class BenchmarkResult {
        public:
                String suite;
                String name;
                String label;
                String description;
                uint64_t iterations = 0;
                uint64_t repeats = 0;
                double avgNsPerIter = 0.0;
                double minNsPerIter = 0.0;
                double maxNsPerIter = 0.0;
                double stddevNsPerIter = 0.0;
                double itemsPerSecond = 0.0;
                double bytesPerSecond = 0.0;
                Map<String, double> custom;
                bool succeeded = true;
                String errorMessage;
 
                JsonObject toJson() const;
 
                static BenchmarkResult fromJson(const JsonObject &obj, Error *err = nullptr);
};
 
class BenchmarkCase {
        public:
                using Function = promeki::Function<void(BenchmarkState &)>;
 
                BenchmarkCase(const String &suite, const String &name, const String &description, Function fn)
                    : _suite(suite), _name(name), _description(description), _fn(std::move(fn)) {}
 
                const String &suite() const { return _suite; }
 
                const String &name() const { return _name; }
 
                const String &description() const { return _description; }
 
                String fullName() const { return _suite + "." + _name; }
 
                void invoke(BenchmarkState &state) const { _fn(state); }
 
        private:
                String   _suite;
                String   _name;
                String   _description;
                Function _fn;
};
 
class BenchmarkRunner {
        public:
                static int registerCase(const BenchmarkCase &theCase);
 
                static const List<BenchmarkCase> &registeredCases();
 
                static String formatRegisteredCases();
 
                BenchmarkRunner();
 
                void setMinTimeMs(unsigned int ms) { _minTimeMs = ms; }
 
                void setWarmupMs(unsigned int ms) { _warmupMs = ms; }
 
                void setMinIterations(uint64_t n) { _minIterations = n; }
 
                void setMaxIterations(uint64_t n) { _maxIterations = n; }
 
                void setRepeats(unsigned int n) { _repeats = n == 0 ? 1 : n; }
 
                void setFilter(const String &pattern) { _filterPattern = pattern; }
 
                void setVerbose(bool verbose) { _verbose = verbose; }
 
                unsigned int minTimeMs() const { return _minTimeMs; }
 
                unsigned int warmupMs() const { return _warmupMs; }
 
                uint64_t minIterations() const { return _minIterations; }
 
                uint64_t maxIterations() const { return _maxIterations; }
 
                unsigned int repeats() const { return _repeats; }
 
                const String &filter() const { return _filterPattern; }
 
                const List<BenchmarkResult> &results() const { return _results; }
 
                int filteredCaseCount() const;
 
                int64_t estimatedDurationMs() const;
 
                Error runAll();
 
                Error runCaseByName(const String &fullName);
 
                BenchmarkResult runCase(const BenchmarkCase &theCase);
 
                void clearResults() { _results.clear(); }
 
                Error writeJson(const String &path) const;
 
                JsonObject toJson() const;
 
                static List<BenchmarkResult> loadBaseline(const String &path, Error *err = nullptr);
 
                String formatTable() const;
 
                String formatComparison(const List<BenchmarkResult> &baseline) const;
 
        private:
                BenchmarkResult measureCase(const BenchmarkCase &theCase);
                uint64_t        calibrateIterations(const BenchmarkCase &theCase);
                void            runOne(const BenchmarkCase &theCase, uint64_t iterations, BenchmarkState &state);
                bool matchesFilter(const BenchmarkCase &theCase) const;
 
                unsigned int _minTimeMs = 500;
                unsigned int _warmupMs = 100;
                uint64_t     _minIterations = 1;
                uint64_t     _maxIterations = 1000000000ULL;
                unsigned int _repeats = 1;
                String       _filterPattern;
                bool         _verbose = false;
                // Column width used for the "running" progress line so
                // each case's result lines up in the same place on
                // screen; computed by runAll() from the filtered case
                // set and read by measureCase() when printing the case
                // name.  0 means "no padding" — measureCase falls back
                // to plain width when called outside a runAll() sweep.
                size_t                _progressNameWidth = 0;
                List<BenchmarkResult> _results;
};
 
#define PROMEKI_REGISTER_BENCHMARK(Suite, Name, Description, Fn)                                                       \
        [[maybe_unused]] static int PROMEKI_CONCAT(__promeki_benchmark_, PROMEKI_UNIQUE_ID) =                          \
                promeki::BenchmarkRunner::registerCase(promeki::BenchmarkCase((Suite), (Name), (Description), (Fn)));
 
PROMEKI_NAMESPACE_END
 
#endif // PROMEKI_ENABLE_CORE