Potential 'overflow' issue commited to upstream libwebrtc

rdar://159773684

Reviewed by Jean-Yves Avenard.

Cherry-picking of webmproject/libvpx@ba9dad1

* Source/ThirdParty/libwebrtc/Source/third_party/libvpx/source/libvpx/test/encode_api_test.cc:
* Source/ThirdParty/libwebrtc/Source/third_party/libvpx/source/libvpx/vp9/encoder/vp9_bitstream.c:
* Source/ThirdParty/libwebrtc/Source/third_party/libvpx/source/libvpx/vp9/encoder/vp9_encodeframe.c:
(choose_partitioning):
* Source/ThirdParty/libwebrtc/Source/third_party/libvpx/source/libvpx/vp9/encoder/vp9_pickmode.c:
(vp9_pick_intra_mode):

Originally-landed-as: 297297.446@safari-7622-branch (010e247). rdar://164213400
Canonical link: https://commits.webkit.org/302917@main

Author: youennf

Source/ThirdParty/libwebrtc/Source/third_party/libvpx/source/libvpx/test/encode_api_test.cc

@@ -1743,6 +1743,43 @@ TEST(EncodeAPI, Buganizer331108922BitDepth8) {
   encoder.Encode(/*key_frame=*/false);
 }
 
+// Encode some frames, flip from BEST_QUALITY to REALTIME after 2 frames.
+// This test is taken from the code snippet in issue:441668134.
+TEST(EncodeAPI, Buganizer441668134) {
+  // Get VP9 encoder interface.
+  vpx_codec_iface_t *iface = vpx_codec_vp9_cx();
+  // Initialize encoder configuration with default values.
+  vpx_codec_enc_cfg_t cfg;
+  ASSERT_EQ(vpx_codec_enc_config_default(iface, &cfg, 0), VPX_CODEC_OK);
+  cfg.g_lag_in_frames = 0;
+  cfg.rc_max_quantizer = 0;
+  unsigned long init_flags = 0;
+  vpx_codec_ctx_t ctx;
+  ASSERT_EQ(vpx_codec_enc_init(&ctx, iface, &cfg, init_flags), VPX_CODEC_OK);
+  ASSERT_EQ(vpx_codec_control_(&ctx, VP8E_SET_CPUUSED, 9), 0);
+  ASSERT_EQ(vpx_codec_control_(&ctx, VP9E_SET_DELTA_Q_UV, -15), 0);
+  // Image allocation.
+  vpx_img_fmt_t img_fmt = VPX_IMG_FMT_I420;
+  vpx_image_t *img = vpx_img_alloc(NULL, img_fmt, cfg.g_w, cfg.g_h, 32);
+  for (unsigned int y = 0; y < img->d_h; y++) {
+    for (unsigned int x = 0; x < img->d_w; x++) {
+      img->planes[0][y * img->stride[0] + x] = ((x ^ y) * 127) & 0xFF;
+    }
+  }
+  // Encode some frames.
+  int num_frames = 6;
+  static constexpr int kChoices[6] = { 1, 1, 0, 0, 0, 0 };
+  for (int frame = 0; frame < num_frames; frame++) {
+    vpx_enc_deadline_t deadline = VPX_DL_REALTIME;
+    uint8_t dl_choice = kChoices[frame];
+    if (dl_choice == 1) deadline = VPX_DL_BEST_QUALITY;
+    // Encode frame.
+    ASSERT_EQ(vpx_codec_encode(&ctx, img, frame, 1, 0, deadline), VPX_CODEC_OK);
+  }
+  vpx_img_free(img);
+  vpx_codec_destroy(&ctx);
+}
+
 #if CONFIG_VP9_HIGHBITDEPTH
 TEST(EncodeAPI, Buganizer329674887RowMT0BitDepth12) {
   VP9Encoder encoder(8, 0, VPX_BITS_12, VPX_IMG_FMT_I444);

Source/ThirdParty/libwebrtc/Source/third_party/libvpx/source/libvpx/vp9/encoder/vp9_bitstream.c

@@ -56,6 +56,7 @@ static const struct vp9_token inter_mode_encodings[INTER_MODES] = {
 
 static void write_intra_mode(vpx_writer *w, PREDICTION_MODE mode,
                              const vpx_prob *probs) {
+  assert(!is_inter_mode(mode));
   vp9_write_token(w, vp9_intra_mode_tree, probs, &intra_mode_encodings[mode]);
 }
 

Source/ThirdParty/libwebrtc/Source/third_party/libvpx/source/libvpx/vp9/encoder/vp9_encodeframe.c

@@ -1301,8 +1301,13 @@ static int choose_partitioning(VP9_COMP *cpi, const TileInfo *const tile,
       is_key_frame = 1;
   }
 
-  // Always use 4x4 partition for key frame.
-  const int use_4x4_partition = frame_is_intra_only(cm);
+  // Allow for sub8x8 (4x4) partition on key frames, but only for hybrid mode
+  // (i.e., sf->nonrd_keyframe = 0), where for small blocks rd intra pickmode
+  // (vp9_rd_pick_intra_mode_sb) is used. The nonrd intra pickmode
+  // (vp9_pick_intra_mode) does not currently support sub8x8 blocks. This causes
+  // the issue: 44166813. Assert is added in vp9_pick_intra_mode to check this.
+  const int use_4x4_partition =
+      frame_is_intra_only(cm) && !cpi->sf.nonrd_keyframe;
   const int low_res = (cm->width <= 352 && cm->height <= 288);
   int variance4x4downsample[16];
   int segment_id;

Source/ThirdParty/libwebrtc/Source/third_party/libvpx/source/libvpx/vp9/encoder/vp9_pickmode.c

@@ -1179,6 +1179,7 @@ void vp9_pick_intra_mode(VP9_COMP *cpi, MACROBLOCK *x, RD_COST *rd_cost,
   const PREDICTION_MODE A = vp9_above_block_mode(mic, above_mi, 0);
   const PREDICTION_MODE L = vp9_left_block_mode(mic, left_mi, 0);
   bmode_costs = cpi->y_mode_costs[A][L];
+  assert(bsize >= BLOCK_8X8);
 
   (void)ctx;
   vp9_rd_cost_reset(&best_rdc);