Duration based benchmarking (instead of no. of iterations), better argument handling
refs #269
This commit is contained in:
saschawillems
parent
f1f96cfd62
commit
39122785a8
2 changed files with 77 additions and 60 deletions
|
|
@ -21,15 +21,15 @@ namespace vks
|
|||
FILE *stream;
|
||||
public:
|
||||
bool active = false;
|
||||
uint32_t framesPerIteration = 1000;
|
||||
uint32_t iterationCount = 10;
|
||||
//std::vector<double> iterationTimes;
|
||||
struct Iteration {
|
||||
bool outputFrameTimes = true;
|
||||
uint32_t warmup = 1;
|
||||
uint32_t duration = 10;
|
||||
std::vector<double> frameTimes;
|
||||
};
|
||||
std::vector<Iteration> iterations;
|
||||
std::string filename = "benchmarkresults.csv";
|
||||
|
||||
double runtime = 0.0;
|
||||
uint32_t frameCount = 0;
|
||||
|
||||
void run(std::function<void()> renderFunc) {
|
||||
active = true;
|
||||
#if defined(_WIN32)
|
||||
|
|
@ -37,61 +37,60 @@ namespace vks
|
|||
freopen_s(&stream, "CONOUT$", "w+", stdout);
|
||||
freopen_s(&stream, "CONOUT$", "w+", stderr);
|
||||
#endif
|
||||
// "Warm up" run to get more stable frame rates
|
||||
for (uint32_t f = 0; f < framesPerIteration; f++) {
|
||||
renderFunc();
|
||||
}
|
||||
std::cout << std::fixed << std::setprecision(3);
|
||||
|
||||
iterations.resize(iterationCount);
|
||||
for (uint32_t i = 0; i < iterationCount; i++) {
|
||||
iterations[i].frameTimes.resize(framesPerIteration);
|
||||
for (uint32_t f = 0; f < framesPerIteration; f++) {
|
||||
// Warm up phase to get more stable frame rates
|
||||
{
|
||||
double tMeasured = 0.0;
|
||||
while (tMeasured < (warmup * 1000)) {
|
||||
auto tStart = std::chrono::high_resolution_clock::now();
|
||||
renderFunc();
|
||||
auto tEnd = std::chrono::high_resolution_clock::now();
|
||||
auto tDiff = std::chrono::duration<double, std::milli>(tEnd - tStart).count();
|
||||
iterations[i].frameTimes[f] = tDiff;
|
||||
auto tDiff = std::chrono::duration<double, std::milli>(std::chrono::high_resolution_clock::now() - tStart).count();
|
||||
tMeasured += tDiff;
|
||||
};
|
||||
}
|
||||
|
||||
// Benchmark phase
|
||||
{
|
||||
while (runtime < (duration * 1000.0)) {
|
||||
auto tStart = std::chrono::high_resolution_clock::now();
|
||||
renderFunc();
|
||||
auto tDiff = std::chrono::duration<double, std::milli>(std::chrono::high_resolution_clock::now() - tStart).count();
|
||||
runtime += tDiff;
|
||||
frameTimes.push_back(tDiff);
|
||||
frameCount++;
|
||||
};
|
||||
std::cout << "Benchmark finished" << std::endl;
|
||||
std::cout << "runtime: " << (runtime / 1000.0) << std::endl;
|
||||
std::cout << "frames: " << frameCount << std::endl;
|
||||
std::cout << "fps: " << frameCount / (runtime / 1000.0) << std::endl;
|
||||
}
|
||||
}
|
||||
|
||||
void saveResults(std::string appinfo, std::string deviceinfo) {
|
||||
std::ofstream result(filename, std::ios::out);
|
||||
if (result.is_open()) {
|
||||
double tMinAll = std::numeric_limits<double>::max();
|
||||
double tMaxAll = std::numeric_limits<double>::min();
|
||||
double tAvgAll = 0.0;
|
||||
|
||||
result << std::fixed << std::setprecision(4);
|
||||
result << "iteration,min(ms),max(ms),avg(ms),min(fps),max(fps),avg(fps)" << std::endl;
|
||||
uint32_t index = 0;
|
||||
for (auto iteration : iterations) {
|
||||
double tMin = *std::min_element(iteration.frameTimes.begin(), iteration.frameTimes.end());
|
||||
double tMax = *std::max_element(iteration.frameTimes.begin(), iteration.frameTimes.end());
|
||||
double tAvg = std::accumulate(iteration.frameTimes.begin(), iteration.frameTimes.end(), 0.0) / framesPerIteration;
|
||||
if (tMin < tMinAll) {
|
||||
tMinAll = tMin;
|
||||
}
|
||||
if (tMax > tMaxAll) {
|
||||
tMaxAll = tMax;
|
||||
}
|
||||
tAvgAll += tAvg;
|
||||
index++;
|
||||
result << index << "," << tMin << "," << tMax << "," << tAvg << "," << (1000.0 / tMax) << "," << (1000.0 / tMin) << "," << (1000.0 / tAvg) << std::endl;
|
||||
}
|
||||
|
||||
tAvgAll /= static_cast<uint32_t>(iterations.size());
|
||||
result << "summary,min(ms),max(ms),avg(ms),min(fps),max(fps),avg(fps)" << std::endl;
|
||||
result << index << "," << tMinAll << "," << tMaxAll << "," << tAvgAll << "," << (1000.0 / tMaxAll) << "," << (1000.0 / tMinAll) << "," << (1000.0 / tAvgAll) << std::endl;
|
||||
result << "duration (ms),frames" << std::endl;
|
||||
result << runtime << "," << frameCount << std::endl;
|
||||
|
||||
// Output averages to stdout
|
||||
std::cout << std::fixed << std::setprecision(3);
|
||||
std::cout << "best : " << (1000.0 / tMinAll) << " fps (" << tMinAll << " ms)" << std::endl;
|
||||
std::cout << "worst: " << (1000.0 / tMaxAll) << " fps (" << tMaxAll << " ms)" << std::endl;
|
||||
std::cout << "avg : " << (1000.0 / tAvgAll) << " fps (" << tAvgAll << " ms)" << std::endl;
|
||||
if (outputFrameTimes) {
|
||||
result << std::endl << "frame,ms" << std::endl;
|
||||
for (size_t i = 0; i < frameTimes.size(); i++) {
|
||||
result << i << "," << frameTimes[i] << std::endl;
|
||||
}
|
||||
double tMin = *std::min_element(frameTimes.begin(), frameTimes.end());
|
||||
double tMax = *std::max_element(frameTimes.begin(), frameTimes.end());
|
||||
double tAvg = std::accumulate(frameTimes.begin(), frameTimes.end(), 0.0) / (double)frameTimes.size();
|
||||
std::cout << "best : " << (1000.0 / tMin) << " fps (" << tMin << " ms)" << std::endl;
|
||||
std::cout << "worst: " << (1000.0 / tMax) << " fps (" << tMax << " ms)" << std::endl;
|
||||
std::cout << "avg : " << (1000.0 / tAvg) << " fps (" << tAvg << " ms)" << std::endl;
|
||||
std::cout << std::endl;
|
||||
}
|
||||
|
||||
result.flush();
|
||||
#if defined(_WIN32)
|
||||
fclose(stream);
|
||||
FreeConsole();
|
||||
#endif
|
||||
}
|
||||
|
|
|
|||
|
|
@ -668,6 +668,8 @@ VulkanExampleBase::VulkanExampleBase(bool enableValidation)
|
|||
|
||||
settings.validation = enableValidation;
|
||||
|
||||
char* numConvPtr;
|
||||
|
||||
// Parse command line arguments
|
||||
for (size_t i = 0; i < args.size(); i++)
|
||||
{
|
||||
|
|
@ -681,26 +683,42 @@ VulkanExampleBase::VulkanExampleBase(bool enableValidation)
|
|||
settings.fullscreen = true;
|
||||
}
|
||||
if ((args[i] == std::string("-w")) || (args[i] == std::string("-width"))) {
|
||||
char* endptr;
|
||||
uint32_t w = strtol(args[i + 1], &endptr, 10);
|
||||
if (endptr != args[i + 1]) { width = w; };
|
||||
uint32_t w = strtol(args[i + 1], &numConvPtr, 10);
|
||||
if (numConvPtr != args[i + 1]) { width = w; };
|
||||
}
|
||||
if ((args[i] == std::string("-h")) || (args[i] == std::string("-height"))) {
|
||||
char* endptr;
|
||||
uint32_t h = strtol(args[i + 1], &endptr, 10);
|
||||
if (endptr != args[i + 1]) { height = h; };
|
||||
uint32_t h = strtol(args[i + 1], &numConvPtr, 10);
|
||||
if (numConvPtr != args[i + 1]) { height = h; };
|
||||
}
|
||||
if ((args[i] == std::string("-b")) || (args[i] == std::string("-benchmark"))) {
|
||||
// TODO: option toggle for detailed frame rate output
|
||||
benchmark.active = true;
|
||||
// Result file name can be overriden
|
||||
// Warmup time in seconds
|
||||
if (args.size() > i + 1) {
|
||||
benchmark.filename = args[i + 1];
|
||||
uint32_t num = strtol(args[i + 1], &numConvPtr, 10);
|
||||
if (numConvPtr != args[i + 1]) {
|
||||
benchmark.warmup = num;
|
||||
} else {
|
||||
std::cerr << "Warmup time for benchmark mode must be specified as a number!" << std::endl;
|
||||
}
|
||||
// Number of iterations as optional parameter
|
||||
}
|
||||
// Benchmark runtime in seconds
|
||||
if (args.size() > i + 2) {
|
||||
char* endptr;
|
||||
uint32_t iterations = strtol(args[i + 2], &endptr, 10);
|
||||
if (endptr != args[i + 2]) { benchmark.iterationCount = iterations; };
|
||||
uint32_t num = strtol(args[i + 2], &numConvPtr, 10);
|
||||
if (numConvPtr != args[i + 2]) {
|
||||
benchmark.duration = num;
|
||||
}
|
||||
else {
|
||||
std::cerr << "Benchmark run duration must be specified as a number!" << std::endl;
|
||||
}
|
||||
}
|
||||
// Default file name can be overriden
|
||||
if (args.size() > i + 3) {
|
||||
if (args[i + 3][0] == '-') {
|
||||
std::cerr << "Filename for benchmark results must not start with a hyphen!" << std::endl;
|
||||
} else {
|
||||
benchmark.filename = args[i + 3];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue