Moved benchmark to separate header, output summary to stdout

Refs #269

base/benchmark.hpp

@@ -0,0 +1,82 @@
+/*
+* Benchmark class
+*
+* Copyright (C) 2016-2017 by Sascha Willems - www.saschawillems.de
+*
+* This code is licensed under the MIT license (MIT) (http://opensource.org/licenses/MIT)
+*/
+
+#include <vector>
+#include <string>
+#include <algorithm>
+#include <limits>
+#include <functional>
+#include <chrono>
+#include <iomanip>
+
+namespace vks
+{
+	class Benchmark {
+	public:
+		bool active = false;
+		uint32_t framesPerIteration = 1000;
+		uint32_t iterations = 10;
+		std::vector<double> iterationTimes;
+		std::string filename = "benchmarkresults.csv";
+
+		Benchmark() {
+			active = true;
+			iterationTimes.resize(iterations);
+
+#if defined(_WIN32)
+			AttachConsole(ATTACH_PARENT_PROCESS);
+			FILE *stream;
+			freopen_s(&stream, "CONOUT$", "w+", stdout);
+			freopen_s(&stream, "CONOUT$", "w+", stderr);
+#endif
+		}
+
+		void run(std::function<void()> renderFunc) {
+			static uint32_t currIteration;
+			for (uint32_t i = 0; i < iterations; i++) {
+				for (uint32_t f = 0; f < framesPerIteration; f++) {
+					auto tStart = std::chrono::high_resolution_clock::now();
+					renderFunc();
+					auto tEnd = std::chrono::high_resolution_clock::now();
+					auto tDiff = std::chrono::duration<double, std::milli>(tEnd - tStart).count();
+					double frameTime = (double)tDiff / 1000.0;
+					iterationTimes[i] += frameTime;
+				}
+			}
+		}
+
+		void saveResults(std::string appinfo, std::string deviceinfo) {
+			std::ofstream result(filename, std::ios::out);
+			if (result.is_open()) {
+
+				double tMin = *std::min_element(iterationTimes.begin(), iterationTimes.end());
+				double tMax = *std::max_element(iterationTimes.begin(), iterationTimes.end());
+				double tAvg = std::accumulate(iterationTimes.begin(), iterationTimes.end(), 0.0) / iterations;
+
+				result << std::fixed << std::setprecision(3);
+				result << appinfo << std::endl;
+				result << deviceinfo << std::endl;
+				result << ",iterations,time(ms),fps" << std::endl;;
+				for (size_t i = 0; i < iterationTimes.size(); i++) {
+					result << "," << i << "," << iterationTimes[i] << "," << (framesPerIteration / iterationTimes[i]) << std::endl;
+				}
+				result << ",summary" << std::endl;
+				result << ",,time(ms),fps" << std::endl;
+				result << ",min," << tMin << "," << (1000.0 / tMin) << std::endl;
+				result << ",max," << tMax << "," << (1000.0 / tMax) << std::endl;
+				result << ",avg," << tAvg << "," << (1000.0 / tAvg) << std::endl;
+
+				// Output averages to stdout
+				std::cout << std::fixed << std::setprecision(3);
+				std::cout << "best : " << (1000.0 / tMin) << " fps (" << tMin << " ms)" << std::endl;
+				std::cout << "worst: " << (1000.0 / tMax) << " fps (" << tMax << " ms)" << std::endl;
+				std::cout << "avg  : " << (1000.0 / tAvg) << " fps (" << tAvg << " ms)" << std::endl;
+			}
+		}
+	};
+}

base/vulkanexamplebase.cpp

@@ -266,47 +266,12 @@ void VulkanExampleBase::renderFrame()
 	}
 }
 
-void VulkanExampleBase::benchmarkLoop()
-{
-	for (uint32_t i = 0; i < benchmark.iterations; i++) {
-		for (uint32_t f = 0; f < 1000; f++) {
-			auto tStart = std::chrono::high_resolution_clock::now();
-			render();
-			auto tEnd = std::chrono::high_resolution_clock::now();
-			auto tDiff = std::chrono::duration<double, std::milli>(tEnd - tStart).count();
-			double frameTime = (double)tDiff / 1000;
-			benchmark.iterationTime[benchmark.currIteration] += frameTime;
-		}
-		benchmark.currIteration++;
-	}
-	vkDeviceWaitIdle(device);
-	// Save results as comma separated file
-	std::ofstream result(benchmark.resultFile + ".csv", std::ios::out);
-	if (result.is_open()) {
-
-		double tMin = *std::min_element(benchmark.iterationTime.begin(), benchmark.iterationTime.end());
-		double tMax = *std::max_element(benchmark.iterationTime.begin(), benchmark.iterationTime.end());
-		double tAvg = std::accumulate(benchmark.iterationTime.begin(), benchmark.iterationTime.end(), 0.0) / benchmark.iterations;
-
-		result << std::fixed << std::setprecision(3);
-		result << title << std::endl;
-		result << deviceProperties.deviceName << std::endl;
-		result << ",iterations,time(ms),fps" << std::endl;;
-		for (size_t i = 0; i < benchmark.iterationTime.size(); i++) {
-			result << "," << i << "," << benchmark.iterationTime[i] << "," << (1000.0 / benchmark.iterationTime[i]) << std::endl;
-		}
-		result << ",summary" << std::endl;
-		result << ",,time(ms),fps" << std::endl;
-		result << ",min," << tMin << "," << (1000.0 / tMin) << std::endl;
-		result << ",max," << tMax << "," << (1000.0 / tMax) << std::endl;
-		result << ",avg," << tAvg << "," << (1000.0 / tAvg) << std::endl;
-	}
-}
-
 void VulkanExampleBase::renderLoop()
 {
 	if (benchmark.active) {
-		benchmarkLoop();
+		benchmark.run([=] { render(); });
+		vkDeviceWaitIdle(device);
+		benchmark.saveResults(title, deviceProperties.deviceName);
 		return;
 	}
 
@@ -701,7 +666,7 @@ VulkanExampleBase::VulkanExampleBase(bool enableValidation)
 			benchmark.active = true;
 			// Result file name can be overriden
 			if (args.size() > i + 1) {
-				benchmark.resultFile = args[i + 1];
+				benchmark.filename = args[i + 1];
 			}
 			// Number of iterations as optional parameter
 			if (args.size() > i + 2) {
@@ -709,7 +674,6 @@ VulkanExampleBase::VulkanExampleBase(bool enableValidation)
 				uint32_t iterations = strtol(args[i + 2], &endptr, 10);
 				if (endptr != args[i + 2]) { benchmark.iterations = iterations; };
 			}
-			benchmark.init();
 		}
 	}
 	

base/vulkanexamplebase.h

@@ -50,6 +50,7 @@
 #include "VulkanSwapChain.hpp"
 #include "VulkanTextOverlay.hpp"
 #include "camera.hpp"
+#include "benchmark.hpp"
 
 class VulkanExampleBase
 {
@@ -64,6 +65,7 @@ private:
 	uint32_t destWidth;
 	uint32_t destHeight;
 	bool resizing = false;
+	vks::Benchmark benchmark;
 	// Called if the window is resized and some resources have to be recreatesd
 	void windowResize();
 protected:
@@ -151,24 +153,6 @@ public:
 
 	VkClearColorValue defaultClearColor = { { 0.025f, 0.025f, 0.025f, 1.0f } };
 
-	/** @brief Stores performance metrics for benchmark runs */
-	struct Benchmark {
-		bool active = false;
-		uint32_t iterations = 10;
-		uint32_t currIteration = 0;
-		std::string resultFile = "benchmarkresults";
-		std::vector<double> iterationTime;
-		struct FrameTimes {
-			double min, max, avg;
-		} frameTimes;
-		void init() {
-			iterationTime.resize(iterations);
-			frameTimes.min = std::numeric_limits<double>::max();
-			frameTimes.max = std::numeric_limits<double>::min();
-			frameTimes.avg = 0.0;
-		}
-	} benchmark;
-
 	float zoom = 0;
 
 	static std::vector<const char*> args;
@@ -398,8 +382,6 @@ public:
 	// Start the main render loop
 	void renderLoop();
 
-	void benchmarkLoop();
-
 	// Render one frame of a render loop on platforms that sync rendering
 	void renderFrame();
 

bin/benchmark-all.py

@@ -70,15 +70,15 @@ print("Benchmarking all examples...")
 os.makedirs("./benchmark", exist_ok=True)
 
 for example in EXAMPLES:
-	print("Running %s (%d/%d) in benchmark mode" % (example, CURR_INDEX+1, len(EXAMPLES)))
+	print("---- (%d/%d) Running %s in benchmark mode ----" % (CURR_INDEX+1, len(EXAMPLES), example))
 	if platform.system() == 'Linux':
 		RESULT_CODE = subprocess.call("./%s %s ./benchmark/%s" % (example, ARGS, example), shell=True)
 	else:
 		RESULT_CODE = subprocess.call("%s %s ./benchmark/%s" % (example, ARGS, example))
 	if RESULT_CODE == 0:
-		print("\tResults written to ./benchmark/%s.csv" % example)
+		print("Results written to ./benchmark/%s.csv" % example)
 	else:
-		print("\tError, result code = %d" % RESULT_CODE)
+		print("Error, result code = %d" % RESULT_CODE)
 	CURR_INDEX += 1
 
 print("Benchmark run finished")

vulkanExamples.sln

@@ -75,6 +75,7 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "deferredshadows", "deferred
 EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Base", "Base", "{09B9A54B-FC57-4A98-9671-5706FC3846C9}"
 	ProjectSection(SolutionItems) = preProject
+		base\benchmark.hpp = base\benchmark.hpp
 		base\camera.hpp = base\camera.hpp
 		base\frustum.hpp = base\frustum.hpp
 		base\keycodes.hpp = base\keycodes.hpp