[Flutter GPU] Add pipeline stencil config.

lib/gpu/lib/src/render_pass.dart

@@ -45,6 +45,32 @@ base class DepthStencilAttachment {
   Texture texture;
 }
 
+base class StencilConfig {
+  StencilConfig({
+    this.compareFunction = CompareFunction.always,
+    this.stencilFailureOperation = StencilOperation.keep,
+    this.depthFailureOperation = StencilOperation.keep,
+    this.depthStencilPassOperation = StencilOperation.keep,
+    this.readMask = 0xFFFFFFFF,
+    this.writeMask = 0xFFFFFFFF,
+  });
+
+  CompareFunction compareFunction;
+  StencilOperation stencilFailureOperation;
+  StencilOperation depthFailureOperation;
+  StencilOperation depthStencilPassOperation;
+  int readMask;
+  int writeMask;
+}
+
+// Note: When modifying this enum, also update
+//       `InternalFlutterGpu_RenderPass_SetStencilConfig` in `gpu/render_pass.cc`.
+enum StencilFace {
+  both,
+  front,
+  back,
+}
+
 base class ColorBlendEquation {
   ColorBlendEquation({
     this.colorBlendOperation = BlendOperation.add,
@@ -220,6 +246,24 @@ base class RenderPass extends NativeFieldWrapperClass1 {
     _setStencilReference(referenceValue);
   }
 
+  void setStencilConfig(StencilConfig configuration,
+      {StencilFace targetFace = StencilFace.both}) {
+    if (configuration.readMask < 0 || configuration.readMask > 0xFFFFFFFF) {
+      throw Exception("The stencil read mask must be in the range [0, 255]");
+    }
+    if (configuration.writeMask < 0 || configuration.writeMask > 0xFFFFFFFF) {
+      throw Exception("The stencil write mask must be in the range [0, 255]");
+    }
+    _setStencilConfig(
+        configuration.compareFunction.index,
+        configuration.stencilFailureOperation.index,
+        configuration.depthFailureOperation.index,
+        configuration.depthStencilPassOperation.index,
+        configuration.readMask,
+        configuration.writeMask,
+        targetFace.index);
+  }
+
   void draw() {
     if (!_draw()) {
       throw Exception("Failed to append draw");
@@ -347,6 +391,17 @@ base class RenderPass extends NativeFieldWrapperClass1 {
       symbol: 'InternalFlutterGpu_RenderPass_SetStencilReference')
   external void _setStencilReference(int referenceValue);
 
+  @Native<Void Function(Pointer<Void>, Int, Int, Int, Int, Int, Int, Int)>(
+      symbol: 'InternalFlutterGpu_RenderPass_SetStencilConfig')
+  external void _setStencilConfig(
+      int compareFunction,
+      int stencilFailureOperation,
+      int depthFailureOperation,
+      int depthStencilPassOperation,
+      int readMask,
+      int writeMask,
+      int target_face);
+
   @Native<Bool Function(Pointer<Void>)>(
       symbol: 'InternalFlutterGpu_RenderPass_Draw')
   external bool _draw();

lib/gpu/render_pass.cc

@@ -62,6 +62,16 @@ RenderPass::GetDepthAttachmentDescriptor() {
   return depth_desc_;
 }
 
+impeller::StencilAttachmentDescriptor&
+RenderPass::GetStencilFrontAttachmentDescriptor() {
+  return stencil_front_desc_;
+}
+
+impeller::StencilAttachmentDescriptor&
+RenderPass::GetStencilBackAttachmentDescriptor() {
+  return stencil_back_desc_;
+}
+
 impeller::VertexBuffer& RenderPass::GetVertexBuffer() {
   return vertex_buffer_;
 }
@@ -518,6 +528,36 @@ void InternalFlutterGpu_RenderPass_SetStencilReference(
   command.stencil_reference = static_cast<uint32_t>(stencil_reference);
 }
 
+void InternalFlutterGpu_RenderPass_SetStencilConfig(
+    flutter::gpu::RenderPass* wrapper,
+    int stencil_compare_operation,
+    int stencil_fail_operation,
+    int depth_fail_operation,
+    int depth_stencil_pass_operation,
+    int read_mask,
+    int write_mask,
+    int target_face) {
+  impeller::StencilAttachmentDescriptor desc;
+  desc.stencil_compare =
+      flutter::gpu::ToImpellerCompareFunction(stencil_compare_operation);
+  desc.stencil_failure =
+      flutter::gpu::ToImpellerStencilOperation(stencil_fail_operation);
+  desc.depth_failure =
+      flutter::gpu::ToImpellerStencilOperation(depth_fail_operation);
+  desc.depth_stencil_pass =
+      flutter::gpu::ToImpellerStencilOperation(depth_stencil_pass_operation);
+  desc.read_mask = static_cast<uint32_t>(read_mask);
+  desc.write_mask = static_cast<uint32_t>(write_mask);
+
+  // Corresponds to the `StencilFace` enum in `gpu/lib/src/render_pass.dart`.
+  if (target_face != 2 /* both or front */) {
+    wrapper->GetStencilFrontAttachmentDescriptor() = desc;
+  }
+  if (target_face != 1 /* both or back */) {
+    wrapper->GetStencilBackAttachmentDescriptor() = desc;
+  }
+}
+
 bool InternalFlutterGpu_RenderPass_Draw(flutter::gpu::RenderPass* wrapper) {
   return wrapper->Draw();
 }

lib/gpu/render_pass.h

@@ -46,6 +46,10 @@ class RenderPass : public RefCountedDartWrappable<RenderPass> {
 
   impeller::DepthAttachmentDescriptor& GetDepthAttachmentDescriptor();
 
+  impeller::StencilAttachmentDescriptor& GetStencilFrontAttachmentDescriptor();
+
+  impeller::StencilAttachmentDescriptor& GetStencilBackAttachmentDescriptor();
+
   impeller::VertexBuffer& GetVertexBuffer();
 
   bool Begin(flutter::gpu::CommandBuffer& command_buffer);
@@ -226,6 +230,17 @@ extern void InternalFlutterGpu_RenderPass_SetStencilReference(
     flutter::gpu::RenderPass* wrapper,
     int stencil_reference);
 
+FLUTTER_GPU_EXPORT
+extern void InternalFlutterGpu_RenderPass_SetStencilConfig(
+    flutter::gpu::RenderPass* wrapper,
+    int stencil_compare_operation,
+    int stencil_fail_operation,
+    int depth_fail_operation,
+    int depth_stencil_pass_operation,
+    int read_mask,
+    int write_mask,
+    int target);
+
 FLUTTER_GPU_EXPORT
 extern bool InternalFlutterGpu_RenderPass_Draw(
     flutter::gpu::RenderPass* wrapper);

testing/dart/gpu_test.dart

@@ -262,6 +262,57 @@ void main() async {
     }
   }, skip: !impellerEnabled);
 
+  test('RenderPass.setStencilConfig doesnt throw for valid values', () async {
+    final state = createSimpleRenderPass();
+
+    state.renderPass.setStencilConfig(gpu.StencilConfig());
+    state.renderPass.setStencilConfig(
+        gpu.StencilConfig(
+            compareFunction: gpu.CompareFunction.notEqual,
+            depthFailureOperation: gpu.StencilOperation.decrementWrap,
+            depthStencilPassOperation: gpu.StencilOperation.incrementWrap,
+            stencilFailureOperation: gpu.StencilOperation.invert,
+            readMask: 0,
+            writeMask: 0),
+        targetFace: gpu.StencilFace.back);
+  }, skip: !impellerEnabled);
+
+  test('RenderPass.setStencilConfig throws for invalid masks', () async {
+    final state = createSimpleRenderPass();
+
+    try {
+      state.renderPass.setStencilConfig(gpu.StencilConfig(readMask: -1));
+      fail('Exception not thrown for invalid stencil read mask.');
+    } catch (e) {
+      expect(
+          e.toString(), contains('The stencil read mask must be in the range'));
+    }
+    try {
+      state.renderPass
+          .setStencilConfig(gpu.StencilConfig(readMask: 0xFFFFFFFF + 1));
+      fail('Exception not thrown for invalid stencil read mask.');
+    } catch (e) {
+      expect(
+          e.toString(), contains('The stencil read mask must be in the range'));
+    }
+
+    try {
+      state.renderPass.setStencilConfig(gpu.StencilConfig(writeMask: -1));
+      fail('Exception not thrown for invalid stencil write mask.');
+    } catch (e) {
+      expect(e.toString(),
+          contains('The stencil write mask must be in the range'));
+    }
+    try {
+      state.renderPass
+          .setStencilConfig(gpu.StencilConfig(writeMask: 0xFFFFFFFF + 1));
+      fail('Exception not thrown for invalid stencil write mask.');
+    } catch (e) {
+      expect(e.toString(),
+          contains('The stencil write mask must be in the range'));
+    }
+  }, skip: !impellerEnabled);
+
   // Renders a green triangle pointing downwards.
   test('Can render triangle', () async {
     final state = createSimpleRenderPass();
@@ -298,4 +349,80 @@ void main() async {
     final ui.Image image = state.renderTexture.asImage();
     await comparer.addGoldenImage(image, 'flutter_gpu_test_triangle.png');
   }, skip: !impellerEnabled);
+
+  // Renders a hollow green triangle pointing downwards.
+  test('Can render hollowed out triangle using stencil ops', () async {
+    final state = createSimpleRenderPass();
+
+    final gpu.RenderPipeline pipeline = createUnlitRenderPipeline();
+    state.renderPass.bindPipeline(pipeline);
+
+    // Configure blending with defaults (just to test the bindings).
+    state.renderPass.setColorBlendEnable(true);
+    state.renderPass.setColorBlendEquation(gpu.ColorBlendEquation());
+
+    final gpu.HostBuffer transients = gpu.gpuContext.createHostBuffer();
+    final gpu.BufferView vertices = transients.emplace(float32(<double>[
+      -0.5, 0.5, //
+      0.0, -0.5, //
+      0.5, 0.5, //
+    ]));
+    final gpu.BufferView innerClipVertInfo =
+        transients.emplace(float32(<double>[
+      0.5, 0, 0, 0, // mvp
+      0, 0.5, 0, 0, // mvp
+      0, 0, 0.5, 0, // mvp
+      0, 0, 0, 1, // mvp
+      0, 1, 0, 1, // color
+    ]));
+    final gpu.BufferView outerGreenVertInfo =
+        transients.emplace(float32(<double>[
+      1, 0, 0, 0, // mvp
+      0, 1, 0, 0, // mvp
+      0, 0, 1, 0, // mvp
+      0, 0, 0, 1, // mvp
+      0, 1, 0, 1, // color
+    ]));
+    state.renderPass.bindVertexBuffer(vertices, 3);
+
+    final gpu.UniformSlot vertInfo =
+        pipeline.vertexShader.getUniformSlot('VertInfo');
+
+    // First, punch out a scaled down triangle in the stencil buffer.
+    // Since the stencil buffer is initialized to 0, we set the stencil ref to 1
+    // and the compare to `equial`, which will result in the stencil test
+    // failing. But on failure, we increment the stencil in order to punch out
+    // the triangle.
+
+    state.renderPass.bindUniform(vertInfo, innerClipVertInfo);
+    state.renderPass.setStencilReference(1);
+    state.renderPass.setStencilConfig(gpu.StencilConfig(
+        compareFunction: gpu.CompareFunction.equal,
+        stencilFailureOperation: gpu.StencilOperation.incrementClamp));
+    state.renderPass.draw();
+
+    // Next, render the outer triangle with the stencil ref set to zero, so that
+    // the stencil test passes everywhere except where the inner triangle was
+    // punched out.
+
+    state.renderPass.setStencilReference(0);
+    // Set the stencil config to turn off the increment. For this golden test
+    // we technically don't need to do this, but we do it here just to exercise
+    // the API.
+    state.renderPass.setStencilConfig(
+        gpu.StencilConfig(compareFunction: gpu.CompareFunction.equal));
+    // TODO(bdero): https://github.com/flutter/flutter/issues/155335
+    //              Re-binding the same uniform with `bindUniform` does not
+    //              replace the previous binding. We shouldn't need to clear the
+    //              command bindings to work around this.
+    state.renderPass.clearBindings();
+    state.renderPass.bindUniform(vertInfo, outerGreenVertInfo);
+    state.renderPass.draw();
+
+    state.commandBuffer.submit();
+
+    final ui.Image image = state.renderTexture.asImage();
+    await comparer.addGoldenImage(
+        image, 'flutter_gpu_test_triangle_stencil.png');
+  }, skip: !impellerEnabled);
 }