Opt-in support for multisampled color rendering

src/esp/bindings/SimBindings.cpp

@@ -134,6 +134,9 @@ void initSimConfigBindings(py::module& m) {
       .def_readwrite(
           "enable_hbao", &SimulatorConfiguration::enableHBAO,
           R"(Whether or not to enable horizon-based ambient occlusion, which provides soft shadows in corners and crevices.)")
+      .def_readwrite(
+          "multisample", &SimulatorConfiguration::multisample,
+          R"(Enable multisampling for color-only sensors. Can be only set if create_renderer is set, mutually exclusive with enable_hbao.)")
       .def(py::self == py::self)
       .def(py::self != py::self);
 

src/esp/gfx/RenderTarget.cpp

@@ -68,8 +68,25 @@ struct RenderTarget::Impl {
           gfx_batch::DepthShader::Flag::UnprojectExistingDepth);
     }
 
+    ESP_CHECK(!(flags_ >= Flag::Multisample) ||
+                  !(flags_ &
+                    (Flag::DepthTextureAttachment | Flag::ObjectIdAttachment |
+                     Flag::HorizonBasedAmbientOcclusion)),
+              "RenderTarget: multisampling cannot be enabled together with "
+              "depth, object ID or HBAO");
+
     if (flags_ & Flag::RgbaAttachment) {
-      colorBuffer_.setStorage(Mn::GL::RenderbufferFormat::RGBA8, size);
+      if (flags_ >= Flag::Multisample) {
+        // TODO some option for sample count? use extra flag bits for that?
+        colorBuffer_.setStorageMultisample(8, Mn::GL::RenderbufferFormat::RGBA8,
+                                           size);
+
+        resolvedColorBuffer_ = Mn::GL::Renderbuffer{};
+        resolvedColorBuffer_.setStorage(Mn::GL::RenderbufferFormat::RGBA8,
+                                        size);
+      } else {
+        colorBuffer_.setStorage(Mn::GL::RenderbufferFormat::RGBA8, size);
+      }
     }
     if (flags_ & Flag::ObjectIdAttachment) {
       objectIdTexture_.setMinificationFilter(Mn::GL::SamplerFilter::Nearest)
@@ -86,13 +103,24 @@ struct RenderTarget::Impl {
     } else {
       // we use the unprojectedDepth_ as the depth buffer
       unprojectedDepth_ = Mn::GL::Renderbuffer{};
-      unprojectedDepth_.setStorage(Mn::GL::RenderbufferFormat::DepthComponent24,
-                                   size);
+      if (flags_ >= Flag::Multisample)
+        // TODO some option for sample count? use extra flag bits for that?
+        unprojectedDepth_.setStorageMultisample(
+            8, Mn::GL::RenderbufferFormat::DepthComponent24, size);
+      else
+        unprojectedDepth_.setStorage(
+            Mn::GL::RenderbufferFormat::DepthComponent24, size);
     }
 
     framebuffer_ = Mn::GL::Framebuffer{{{}, size}};
     if (flags_ & Flag::RgbaAttachment) {
       framebuffer_.attachRenderbuffer(RgbaBufferAttachment, colorBuffer_);
+
+      if (flags_ >= Flag::Multisample) {
+        resolvedFramebuffer_ = Mn::GL::Framebuffer{{{}, size}};
+        resolvedFramebuffer_.attachRenderbuffer(
+            Mn::GL::Framebuffer::ColorAttachment{0}, resolvedColorBuffer_);
+      }
     }
     if (flags_ & Flag::ObjectIdAttachment) {
       framebuffer_.attachTexture(ObjectIdTextureColorAttachment,
@@ -206,13 +234,28 @@ struct RenderTarget::Impl {
                       framebuffer_);
   }
 
+  /* Called by readFrameRgba() and readFrameRgbaGPU() if Flag::Multisample is
+     enabled to perform a multisample resolve from colorBuffer_ to
+     resolvedColorBuffer_. */
+  void resolveMultisample() {
+    CORRADE_INTERNAL_ASSERT(flags_ >= Flag::Multisample);
+
+    Mn::GL::AbstractFramebuffer::blit(framebuffer_, resolvedFramebuffer_,
+                                      framebuffer_.viewport(),
+                                      Mn::GL::FramebufferBlit::Color);
+  }
+
   void blitRgbaTo(Mn::GL::AbstractFramebuffer& target,
                   const Mn::Range2Di& targetRectangle) {
     CORRADE_ASSERT(
         flags_ & Flag::RgbaAttachment,
         "RenderTarget::Impl::blitRgbaToDefault(): this render target "
         "was not created with rgba render buffer enabled.", );
 
+    /* Compared to readFrameRgba() and readFrameRgbaGPU(), in case of
+       Flag::Multisample the blit operation already performs a resolve, no need
+       to call resolveMultisample() in this case */
+
     framebuffer_.mapForRead(RgbaBufferAttachment);
     Mn::GL::AbstractFramebuffer::blit(
         framebuffer_, target, framebuffer_.viewport(), targetRectangle,
@@ -224,8 +267,14 @@ struct RenderTarget::Impl {
                    "RenderTarget::Impl::readFrameRgba(): this render target "
                    "was not created with rgba render buffer enabled.", );
 
-    framebuffer_.mapForRead(RgbaBufferAttachment)
-        .read(framebuffer_.viewport(), view);
+    if (flags_ >= Flag::Multisample) {
+      resolveMultisample();
+      resolvedFramebuffer_.mapForRead(Mn::GL::Framebuffer::ColorAttachment{0})
+          .read(resolvedFramebuffer_.viewport(), view);
+    } else {
+      framebuffer_.mapForRead(RgbaBufferAttachment)
+          .read(framebuffer_.viewport(), view);
+    }
   }
 
   void readFrameDepth(const Mn::MutableImageView2D& view) {
@@ -283,10 +332,15 @@ struct RenderTarget::Impl {
                    "RenderTarget::Impl::readFrameRgbaGPU(): this render target "
                    "was not created with rgba render buffer enabled.", );
 
+    if (flags_ >= Flag::Multisample)
+      resolveMultisample();
+
     if (colorBufferCugl_ == nullptr)
       checkCudaErrors(cudaGraphicsGLRegisterImage(
-          &colorBufferCugl_, colorBuffer_.id(), GL_RENDERBUFFER,
-          cudaGraphicsRegisterFlagsReadOnly));
+          &colorBufferCugl_,
+          (flags_ >= Flag::Multisample ? resolvedColorBuffer_ : colorBuffer_)
+              .id(),
+          GL_RENDERBUFFER, cudaGraphicsRegisterFlagsReadOnly));
 
     checkCudaErrors(cudaGraphicsMapResources(1, &colorBufferCugl_, 0));
 
@@ -378,6 +432,10 @@ struct RenderTarget::Impl {
   Mn::GL::Mesh depthUnprojectionMesh_;
   Mn::GL::Framebuffer depthUnprojectionFrameBuffer_;
 
+  /* Used only if Flag::Multisample is enabled */
+  Mn::GL::Renderbuffer resolvedColorBuffer_{Mn::NoCreate};
+  Mn::GL::Framebuffer resolvedFramebuffer_{Mn::NoCreate};
+
   Flags flags_;
 
   const sensor::VisualSensor* visualSensor_ = nullptr;

src/esp/gfx/RenderTarget.h

@@ -34,29 +34,37 @@ class RenderTarget {
  public:
   enum class Flag {
     /**
-     * create a color attachment for the rgba render buffer
-     * No need to set it for depth sensor, semantic sensor etc. as it makes
-     * the rendering slower
+     * Create a color attachment for the rgba render buffer. No need to set it
+     * for depth sensor, semantic sensor etc. as it makes the rendering slower.
      */
     RgbaAttachment = 1 << 0,
     /**
-     * create a color attachment for the objectId texture
-     * No need to set it for color sensor, depth sensor etc. as it makes the
-     * rendering slower
+     * Create a color attachment for the objectId texture. No need to set it
+     * for color sensor, depth sensor etc. as it makes the rendering slower.
+     * Mutually exclusive with @ref Flag::Multisample.
      */
     ObjectIdAttachment = 1 << 1,
     /**
-     * @brief create a depth attachment for the depth texture, it MUST be set
-     * for the depth sensor.
-     * No need to set it for color sensor, objectId sensor etc. as it makes the
-     * rendering slower (default depth buffer will be used in this case.)
+     * Create a depth attachment for the depth texture. Has to be set for the
+     * depth sensor. No need to set it for color sensor, objectId sensor etc.
+     * as it makes the rendering slower (default depth buffer will be used in
+     * this case). Mutually exclusive with @ref Flag::Multisample.
      */
     DepthTextureAttachment = 1 << 2,
 
     /**
-     * Enable HBAO visual effect that adds soft shadows to corners and crevices.
+     * Enable HBAO visual effect that adds soft shadows to corners and
+     * crevices. Mutually exclusive with @ref Flag::Multisample.
      */
     HorizonBasedAmbientOcclusion = 1 << 3,
+
+    /**
+     * Enable color multisampling. Implies @ref Flag::Multisample, mutually
+     * exclusive with @ref Flag::ObjectIdAttachment,
+     * @ref Flag::DepthTextureAttachment and
+     * @ref Flag::HorizonBasedAmbientOcclusion.
+     */
+    Multisample = RgbaAttachment | (1 << 4)
   };
 
   typedef Corrade::Containers::EnumSet<Flag> Flags;

src/esp/gfx/Renderer.cpp

@@ -64,6 +64,10 @@ struct Renderer::Impl {
       backgroundRenderer_ = std::make_unique<BackgroundRenderer>(context_);
     }
 #endif
+
+    ESP_CHECK(!(flags_ >= Flag::Multisample) ||
+                  !(flags_ & (Flag::HorizonBasedAmbientOcclusion)),
+              "Renderere: multisampling cannot be enabled together with HBAO");
   }
 
   ~Impl() {
@@ -345,6 +349,13 @@ struct Renderer::Impl {
           renderTargetFlags |= RenderTarget::Flag::DepthTextureAttachment;
           renderTargetFlags |= RenderTarget::Flag::HorizonBasedAmbientOcclusion;
         }
+
+        // TODO um wait, does it mean that to render color + depth + sematic,
+        //  three separate sensors and thus three separate render targets get
+        //  made?!
+        if (flags_ >= Renderer::Flag::Multisample)
+          renderTargetFlags |= RenderTarget::Flag::Multisample;
+
         break;
 
       case sensor::SensorType::Depth:

src/esp/gfx/Renderer.h

@@ -53,10 +53,18 @@ class Renderer {
     LeaveContextWithBackgroundRenderer = 1 << 3,
 
     /**
-     * Enable HBAO visual effect that adds soft shadows to corners and crevices.
+     * Enable HBAO visual effect that adds soft shadows to corners and
+     * crevices. Mutually exclusinve with @ref Flag::Multisample.
      */
     HorizonBasedAmbientOcclusion = 1 << 4,
 
+    /**
+     * Enable multisampling for color-only buffers. Results in
+     * @ref RenderTarget::Flag::Multisample being set for sensors that have
+     * only color enabled. Mutually exclusive with
+     * @ref Flag::HorizonBasedAmbientOcclusion.
+     */
+    Multisample = 1 << 5,
   };
 
   typedef Corrade::Containers::EnumSet<Flag> Flags;

src/esp/sim/Simulator.cpp

@@ -112,6 +112,11 @@ void Simulator::reconfigure(const SimulatorConfiguration& cfg) {
             "'withbullet' conda build.)");
 #endif
 
+  ESP_CHECK(!cfg.multisample || !cfg.enableHBAO,
+            "Simulator: multisampling cannot be enabled together with HBAO");
+  ESP_CHECK(!cfg.multisample || cfg.createRenderer,
+            "Simulator: multisampling cannot be enabled with renderer off");
+
   // set metadata mediator's cfg  upon creation or reconfigure
   if (!metadataMediator_) {
     metadataMediator_ = metadata::MetadataMediator::create(cfg);
@@ -197,6 +202,10 @@ void Simulator::reconfigure(const SimulatorConfiguration& cfg) {
         flags |= gfx::Renderer::Flag::HorizonBasedAmbientOcclusion;
       }
 
+      if (config_.multisample) {
+        flags |= gfx::Renderer::Flag::Multisample;
+      }
+
       renderer_ = gfx::Renderer::create(context_.get(), flags);
     }
     flextGLInit(Magnum::GL::Context::current());

src/esp/sim/SimulatorConfiguration.h

@@ -109,6 +109,16 @@ struct SimulatorConfiguration {
    */
   bool enableHBAO = false;
 
+  /**
+   * @brief Enable multisampling for color-only sensors
+   *
+   * Results in @ref Renderer::Flag::Multisample being enabled on the renderer
+   * and @ref RenderTarget::Flag::Multisample enabled for all color-only
+   * sensors. Can be only enabled if @ref createRenderer is enabled, mutually
+   * exclusive with @ref enableHBAO.
+   */
+  bool multisample = false;
+
   ESP_SMART_POINTERS(SimulatorConfiguration)
 };