From b4870a6e7eac9ffc1ac0ff8226d96ba802ef783b Mon Sep 17 00:00:00 2001 From: saschawillems Date: Sat, 26 Aug 2017 14:05:48 +0200 Subject: [PATCH] Moved benchmark to separate header, output summary to stdout Refs #269 --- base/benchmark.hpp | 82 ++++++++++++++++++++++++++++++++++++++ base/vulkanexamplebase.cpp | 44 ++------------------ base/vulkanexamplebase.h | 22 +--------- bin/benchmark-all.py | 6 +-- vulkanExamples.sln | 1 + 5 files changed, 92 insertions(+), 63 deletions(-) create mode 100644 base/benchmark.hpp diff --git a/base/benchmark.hpp b/base/benchmark.hpp new file mode 100644 index 00000000..dc3faa06 --- /dev/null +++ b/base/benchmark.hpp @@ -0,0 +1,82 @@ +/* +* Benchmark class +* +* Copyright (C) 2016-2017 by Sascha Willems - www.saschawillems.de +* +* This code is licensed under the MIT license (MIT) (http://opensource.org/licenses/MIT) +*/ + +#include +#include +#include +#include +#include +#include +#include + +namespace vks +{ + class Benchmark { + public: + bool active = false; + uint32_t framesPerIteration = 1000; + uint32_t iterations = 10; + std::vector iterationTimes; + std::string filename = "benchmarkresults.csv"; + + Benchmark() { + active = true; + iterationTimes.resize(iterations); + +#if defined(_WIN32) + AttachConsole(ATTACH_PARENT_PROCESS); + FILE *stream; + freopen_s(&stream, "CONOUT$", "w+", stdout); + freopen_s(&stream, "CONOUT$", "w+", stderr); +#endif + } + + void run(std::function renderFunc) { + static uint32_t currIteration; + for (uint32_t i = 0; i < iterations; i++) { + for (uint32_t f = 0; f < framesPerIteration; f++) { + auto tStart = std::chrono::high_resolution_clock::now(); + renderFunc(); + auto tEnd = std::chrono::high_resolution_clock::now(); + auto tDiff = std::chrono::duration(tEnd - tStart).count(); + double frameTime = (double)tDiff / 1000.0; + iterationTimes[i] += frameTime; + } + } + } + + void saveResults(std::string appinfo, std::string deviceinfo) { + std::ofstream result(filename, std::ios::out); + if (result.is_open()) { + + double tMin = *std::min_element(iterationTimes.begin(), iterationTimes.end()); + double tMax = *std::max_element(iterationTimes.begin(), iterationTimes.end()); + double tAvg = std::accumulate(iterationTimes.begin(), iterationTimes.end(), 0.0) / iterations; + + result << std::fixed << std::setprecision(3); + result << appinfo << std::endl; + result << deviceinfo << std::endl; + result << ",iterations,time(ms),fps" << std::endl;; + for (size_t i = 0; i < iterationTimes.size(); i++) { + result << "," << i << "," << iterationTimes[i] << "," << (framesPerIteration / iterationTimes[i]) << std::endl; + } + result << ",summary" << std::endl; + result << ",,time(ms),fps" << std::endl; + result << ",min," << tMin << "," << (1000.0 / tMin) << std::endl; + result << ",max," << tMax << "," << (1000.0 / tMax) << std::endl; + result << ",avg," << tAvg << "," << (1000.0 / tAvg) << std::endl; + + // Output averages to stdout + std::cout << std::fixed << std::setprecision(3); + std::cout << "best : " << (1000.0 / tMin) << " fps (" << tMin << " ms)" << std::endl; + std::cout << "worst: " << (1000.0 / tMax) << " fps (" << tMax << " ms)" << std::endl; + std::cout << "avg : " << (1000.0 / tAvg) << " fps (" << tAvg << " ms)" << std::endl; + } + } + }; +} \ No newline at end of file diff --git a/base/vulkanexamplebase.cpp b/base/vulkanexamplebase.cpp index 57adf762..55dee6bc 100644 --- a/base/vulkanexamplebase.cpp +++ b/base/vulkanexamplebase.cpp @@ -266,47 +266,12 @@ void VulkanExampleBase::renderFrame() } } -void VulkanExampleBase::benchmarkLoop() -{ - for (uint32_t i = 0; i < benchmark.iterations; i++) { - for (uint32_t f = 0; f < 1000; f++) { - auto tStart = std::chrono::high_resolution_clock::now(); - render(); - auto tEnd = std::chrono::high_resolution_clock::now(); - auto tDiff = std::chrono::duration(tEnd - tStart).count(); - double frameTime = (double)tDiff / 1000; - benchmark.iterationTime[benchmark.currIteration] += frameTime; - } - benchmark.currIteration++; - } - vkDeviceWaitIdle(device); - // Save results as comma separated file - std::ofstream result(benchmark.resultFile + ".csv", std::ios::out); - if (result.is_open()) { - - double tMin = *std::min_element(benchmark.iterationTime.begin(), benchmark.iterationTime.end()); - double tMax = *std::max_element(benchmark.iterationTime.begin(), benchmark.iterationTime.end()); - double tAvg = std::accumulate(benchmark.iterationTime.begin(), benchmark.iterationTime.end(), 0.0) / benchmark.iterations; - - result << std::fixed << std::setprecision(3); - result << title << std::endl; - result << deviceProperties.deviceName << std::endl; - result << ",iterations,time(ms),fps" << std::endl;; - for (size_t i = 0; i < benchmark.iterationTime.size(); i++) { - result << "," << i << "," << benchmark.iterationTime[i] << "," << (1000.0 / benchmark.iterationTime[i]) << std::endl; - } - result << ",summary" << std::endl; - result << ",,time(ms),fps" << std::endl; - result << ",min," << tMin << "," << (1000.0 / tMin) << std::endl; - result << ",max," << tMax << "," << (1000.0 / tMax) << std::endl; - result << ",avg," << tAvg << "," << (1000.0 / tAvg) << std::endl; - } -} - void VulkanExampleBase::renderLoop() { if (benchmark.active) { - benchmarkLoop(); + benchmark.run([=] { render(); }); + vkDeviceWaitIdle(device); + benchmark.saveResults(title, deviceProperties.deviceName); return; } @@ -701,7 +666,7 @@ VulkanExampleBase::VulkanExampleBase(bool enableValidation) benchmark.active = true; // Result file name can be overriden if (args.size() > i + 1) { - benchmark.resultFile = args[i + 1]; + benchmark.filename = args[i + 1]; } // Number of iterations as optional parameter if (args.size() > i + 2) { @@ -709,7 +674,6 @@ VulkanExampleBase::VulkanExampleBase(bool enableValidation) uint32_t iterations = strtol(args[i + 2], &endptr, 10); if (endptr != args[i + 2]) { benchmark.iterations = iterations; }; } - benchmark.init(); } } diff --git a/base/vulkanexamplebase.h b/base/vulkanexamplebase.h index f5faa16b..db31c551 100644 --- a/base/vulkanexamplebase.h +++ b/base/vulkanexamplebase.h @@ -50,6 +50,7 @@ #include "VulkanSwapChain.hpp" #include "VulkanTextOverlay.hpp" #include "camera.hpp" +#include "benchmark.hpp" class VulkanExampleBase { @@ -64,6 +65,7 @@ private: uint32_t destWidth; uint32_t destHeight; bool resizing = false; + vks::Benchmark benchmark; // Called if the window is resized and some resources have to be recreatesd void windowResize(); protected: @@ -151,24 +153,6 @@ public: VkClearColorValue defaultClearColor = { { 0.025f, 0.025f, 0.025f, 1.0f } }; - /** @brief Stores performance metrics for benchmark runs */ - struct Benchmark { - bool active = false; - uint32_t iterations = 10; - uint32_t currIteration = 0; - std::string resultFile = "benchmarkresults"; - std::vector iterationTime; - struct FrameTimes { - double min, max, avg; - } frameTimes; - void init() { - iterationTime.resize(iterations); - frameTimes.min = std::numeric_limits::max(); - frameTimes.max = std::numeric_limits::min(); - frameTimes.avg = 0.0; - } - } benchmark; - float zoom = 0; static std::vector args; @@ -398,8 +382,6 @@ public: // Start the main render loop void renderLoop(); - void benchmarkLoop(); - // Render one frame of a render loop on platforms that sync rendering void renderFrame(); diff --git a/bin/benchmark-all.py b/bin/benchmark-all.py index 66d68f8c..aebfc315 100644 --- a/bin/benchmark-all.py +++ b/bin/benchmark-all.py @@ -70,15 +70,15 @@ print("Benchmarking all examples...") os.makedirs("./benchmark", exist_ok=True) for example in EXAMPLES: - print("Running %s (%d/%d) in benchmark mode" % (example, CURR_INDEX+1, len(EXAMPLES))) + print("---- (%d/%d) Running %s in benchmark mode ----" % (CURR_INDEX+1, len(EXAMPLES), example)) if platform.system() == 'Linux': RESULT_CODE = subprocess.call("./%s %s ./benchmark/%s" % (example, ARGS, example), shell=True) else: RESULT_CODE = subprocess.call("%s %s ./benchmark/%s" % (example, ARGS, example)) if RESULT_CODE == 0: - print("\tResults written to ./benchmark/%s.csv" % example) + print("Results written to ./benchmark/%s.csv" % example) else: - print("\tError, result code = %d" % RESULT_CODE) + print("Error, result code = %d" % RESULT_CODE) CURR_INDEX += 1 print("Benchmark run finished") diff --git a/vulkanExamples.sln b/vulkanExamples.sln index 1bec7321..4be850ca 100644 --- a/vulkanExamples.sln +++ b/vulkanExamples.sln @@ -75,6 +75,7 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "deferredshadows", "deferred EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Base", "Base", "{09B9A54B-FC57-4A98-9671-5706FC3846C9}" ProjectSection(SolutionItems) = preProject + base\benchmark.hpp = base\benchmark.hpp base\camera.hpp = base\camera.hpp base\frustum.hpp = base\frustum.hpp base\keycodes.hpp = base\keycodes.hpp