First steps for a benchmark mode

* Add a button in the graphics options to start a benchmark
* Added logic to play a custom replay for benchmarking
* Added logic to automatically start and end the profiling as the race starts and ends, when in benchmark mode
* Disable the profiler drawings in benchmark mode, they take additional resources and are useless
* Keep the race going when the pause menu is opened, as it would otherwise distort the results
* Added logic to clean up the profiler data each time profiling is switched back from off to on. This avoids multiple profiling sessions piling up in one report, for example when benchmarking tw
* Added a sped up Black Forest replay for use during the benchmark

Missing features and known issues:
* The end screen should display a benchmark result summary instead of a normal end screen
* Entering and leaving the pause menu in benchmark mode makes some parts of the race UI disappear
* The end screen should send the player back to the graphics settings, not the main menu
* Pausing and quitting can leave the profiler enabled when it should not be
* The options in the pause menu should be customized in benchmark mode
* The replay used for benchmarking should be configurable in a data file
* Automatically testing multiple graphics settings and providing an overall summary
* And more advanced options.

Author: Alayan-stk-2

data/gui/screens/options_video.stkgui

@@ -124,6 +124,25 @@
                         I18N="In the video settings" text="Apply new resolution" />
                 </div>
 
+                <!-- ************ BENCHMARK OPTIONS ************ -->
+                <spacer width="5" height="3%"/>
+
+                <label width="100%" I18N="In the video settings" text="Performance tests"/>
+
+                <spacer width="5" height="1%"/>
+
+                <div width="100%" height="fit" layout="horizontal-row" id="outer_box" >
+                    <spacer width="5%" height="100%" />
+                    <button id="benchmarkCurrent" text="Benchmark current settings" I18N="In the video settings" align="center"/>
+                </div>
+<!-- 
+                <spacer width="5" height="1%"/>
+
+                <div width="100%" height="fit" layout="horizontal-row" id="outer_box" >
+                    <spacer width="5%" height="100%" />
+                    <button id="benchmarkStandard" text="Standard Benchmark" I18N="In the video settings" align="center"/>
+                </div>
+ -->
             </box>
         </div>
     </div>

data/replay/benchmark_black_forest.replay

@@ -1024,9 +1024,11 @@ void World::updateWorld(int ticks)
     }
 
     // Don't update world if a menu is shown or the race is over.
-    if (getPhase() == FINISH_PHASE ||
-        (!NetworkConfig::get()->isNetworking() &&
-        getPhase() == IN_GAME_MENU_PHASE))
+    // Exceptions : - Networking (local pause doesn't affect the server or other players)
+    //              - Benchmarking (a pause would mess up measurements)
+    if ((getPhase() == FINISH_PHASE) ||
+        ((getPhase() == IN_GAME_MENU_PHASE) &&
+        (!NetworkConfig::get()->isNetworking() || !RaceManager::get()->isBenchmarking())))
         return;
 
     try

src/modes/world.cpp

@@ -33,6 +33,7 @@
 #include "network/rewind_manager.hpp"
 #include "network/race_event_manager.hpp"
 #include "tracks/track.hpp"
+#include "utils/profiler.hpp"
 #include "utils/stk_process.hpp"
 
 #include <IrrlichtDevice.h>
@@ -352,6 +353,13 @@ void WorldStatus::updateTime(int ticks)
                     m_start_sound->play();
                 }
 
+                if(RaceManager::get()->isBenchmarking())
+                {
+                    // The profiler drawings cost performance
+                    profiler.setDrawing(false);
+                    profiler.toggleStatus();
+                }
+
                 // event
                 onGo();
                 // In artist debug mode, when without opponents,
@@ -430,6 +438,13 @@ void WorldStatus::updateTime(int ticks)
                 stk_config->time2Ticks(stk_config->m_delay_finish_time))
             {
                 m_phase = RESULT_DISPLAY_PHASE;
+                if(RaceManager::get()->isBenchmarking())
+                {
+                    // End profiling
+                    profiler.toggleStatus();
+                    profiler.writeToFile();
+                    profiler.setDrawing(true);
+                }
                 terminateRace();
             }
 

src/modes/world_status.cpp

@@ -144,6 +144,7 @@ RaceManager::RaceManager()
     setRecordRace(false);
     setRaceGhostKarts(false);
     setWatchingReplay(false);
+    setBenchmarking(false);
     setTrack("jungle");
     m_default_ai_list.clear();
     setNumPlayers(0);

src/race/race_manager.cpp

@@ -364,12 +364,11 @@ class RaceManager
 
     /** Determines if saved GP should be continued or not*/
     bool m_continue_saved_gp;
-
     bool m_is_recording_race;
-
     bool m_has_ghost_karts;
-
     bool m_watching_replay;
+    bool m_benchmarking;
+
 public:
     // ----------------------------------------------------------------------------------------
     static RaceManager* get();
@@ -855,6 +854,11 @@ class RaceManager
         m_watching_replay = watch;
     }   // setWatchingReplay
     // ----------------------------------------------------------------------------------------
+    void setBenchmarking(bool benchmark)
+    {
+        m_benchmarking = benchmark;
+    }   // setBenchmarking
+    // ----------------------------------------------------------------------------------------
     bool isRecordingRace() const
     {
         return m_is_recording_race;
@@ -870,6 +874,11 @@ class RaceManager
         return m_watching_replay;
     }   // isWatchingReplay
     // ----------------------------------------------------------------------------------------
+    bool isBenchmarking() const
+    {
+        return m_benchmarking;
+    }   // isBenchmarking
+    // ----------------------------------------------------------------------------------------
     void addSpareTireKart(const std::string& name)
     {
         m_kart_status.push_back(KartStatus(name, 0, -1, -1,

src/race/race_manager.hpp

@@ -121,7 +121,7 @@ RacePausedDialog::RacePausedDialog(const float percentWidth,
                 getWidget("team")->setVisible(false);
         }
     }
-    else
+    else if (!RaceManager::get()->isBenchmarking())
     {
         World::getWorld()->schedulePause(WorldStatus::IN_GAME_MENU_PHASE);
     }

src/states_screens/dialogs/race_paused_dialog.cpp

@@ -30,6 +30,8 @@
 #include "guiengine/widgets/spinner_widget.hpp"
 #include "guiengine/widget.hpp"
 #include "io/file_manager.hpp"
+#include "race/race_manager.hpp"
+#include "replay/replay_play.hpp"
 #include "states_screens/dialogs/custom_video_settings.hpp"
 #include "states_screens/options/options_screen_audio.hpp"
 #include "states_screens/options/options_screen_general.hpp"
@@ -906,6 +908,35 @@ void OptionsScreenVideo::eventCallback(Widget* widget, const std::string& name,
 #endif
         updateScaleRTTsSlider();
     }
+    else if (name == "benchmarkCurrent")
+    {
+        // TODO - Add the possibility to benchmark more tracks and define replay benchmarks in
+        //        a config file
+        const std::string bf_bench("benchmark_black_forest.replay");
+        const bool result = ReplayPlay::get()->addReplayFile(file_manager
+            ->getAsset(FileManager::REPLAY, bf_bench), true/*custom_replay*/);
+
+        if (!result)
+            Log::fatal("OptionsScreenVideo", "Can't open replay for benchmark!");
+        RaceManager::get()->setRaceGhostKarts(true);
+
+        RaceManager::get()->setMinorMode(RaceManager::MINOR_MODE_TIME_TRIAL);
+        ReplayPlay::ReplayData bench_rd = ReplayPlay::get()->getCurrentReplayData();
+        RaceManager::get()->setReverseTrack(bench_rd.m_reverse);
+        RaceManager::get()->setRecordRace(false);
+        RaceManager::get()->setWatchingReplay(true);
+        RaceManager::get()->setDifficulty((RaceManager::Difficulty)bench_rd.m_difficulty);
+
+        // The race manager automatically adds karts for the ghosts
+        RaceManager::get()->setNumKarts(0);
+        RaceManager::get()->setBenchmarking(true);
+        RaceManager::get()->startWatchingReplay(bench_rd.m_track_name, bench_rd.m_laps);
+    }
+    // TODO - Add a standard benchmark testing multiple presets
+    /*else if (name == "benchmarkStandard")
+    {
+        // DO NOTHING FOR NOW
+    }*/
     else if (name == "rememberWinpos")
     {
         CheckBoxWidget* rememberWinpos = getWidget<CheckBoxWidget>("rememberWinpos");

src/states_screens/options/options_screen_video.cpp

@@ -89,11 +89,12 @@ Profiler::Profiler()
     // When initializing profile class during static initialization
     // UserConfigParams::m_max_fps may not be properly initialized with default
     // value, so we use hard-coded default value
-    m_max_frames          = 20 * 120;
+    m_max_frames          = 60 * 1000;
     m_current_frame       = 0;
     m_has_wrapped_around  = false;
+    m_drawing             = true;
     m_threads_used = 1;
-}   // Profile
+}   // Profiler
 
 //-----------------------------------------------------------------------------
 Profiler::~Profiler()
@@ -115,6 +116,27 @@ void Profiler::init()
     m_gpu_times.resize(Q_LAST * m_max_frames);
 }   // init
 
+//------------------------------------------------------------------------------
+/** Reset profiling data held in memory, to allow starting a new clean profiling run
+ */
+void Profiler::reset()
+{
+    for (int i = 0; i < m_threads_used; i++)
+    {
+        ThreadData &td = m_all_threads_data[i];
+        td.m_all_event_data.clear();
+        td.m_event_stack.clear();
+        td.m_ordered_headings.clear();
+    }   // for i in threads
+
+    m_all_threads_data.clear();
+    m_gpu_times.clear();
+    m_all_event_names.clear();
+    m_current_frame       = 0;
+    m_has_wrapped_around  = false;
+    m_freeze_state        = UNFROZEN;
+}   // reset
+
 //-----------------------------------------------------------------------------
 /** Returns a unique index for a thread. If the calling thread is not yet in
  *  the mapping, it will assign a new unique id to this thread. */
@@ -206,6 +228,11 @@ void Profiler::popCPUMarker()
 void Profiler::toggleStatus()
 {
     UserConfigParams::m_profiler_enabled = !UserConfigParams::m_profiler_enabled;
+
+    // Avoid data from multiple profiling sessions from merging in one report
+    if (UserConfigParams::m_profiler_enabled)
+        reset();
+
     // If the profiler would immediately enabled, calls that have started but
     // not finished would not be registered correctly. So set the state to 
     // waiting, so the unfreeze started at the next sync frame (which is
@@ -290,6 +317,9 @@ void Profiler::synchronizeFrame()
 void Profiler::draw()
 {
 #ifndef SERVER_ONLY
+    if (!m_drawing)
+        return;
+
     PROFILER_PUSH_CPU_MARKER("ProfilerDraw", 0xFF, 0xFF, 0x00);
     video::IVideoDriver*    driver = irr_driver->getVideoDriver();
 

src/utils/profiler.cpp

@@ -244,6 +244,9 @@ class Profiler
     /** True if the circular buffer has wrapped around. */
     bool m_has_wrapped_around;
 
+    /** True if the profiler UI should be rendered */
+    bool m_drawing;
+
     /** The maximum number of frames to be buffered. Used to minimise
      *  reallocations. */
     int m_max_frames;
@@ -279,18 +282,21 @@ class Profiler
              Profiler();
     virtual ~Profiler();
     void     init();
+    void     reset();
     void     pushCPUMarker(const char* name="N/A",
                            const video::SColor& color=video::SColor());
     void     popCPUMarker();
-    void     toggleStatus(); 
+    void     toggleStatus();
     void     synchronizeFrame();
     void     draw();
     void     onClick(const core::vector2di& mouse_pos);
     void     writeToFile();
 
     // ------------------------------------------------------------------------
     bool isFrozen() const { return m_freeze_state == FROZEN; }
+    // ------------------------------------------------------------------------
+    void setDrawing(bool drawing) { m_drawing = drawing; }
 
-};
 
+};
 #endif // PROFILER_HPP