PS-10072-[8.0] Fix sever crash after compressed varchar modify

https://perconadev.atlassian.net/browse/PS-10072

Fix crash on SELECT after instant ALTER of compressed VARCHAR across 256-byte boundary

Bug: ALTER TABLE MODIFY VARCHAR(253)->VARCHAR(254) COLUMN_FORMAT COMPRESSED
     with ALGORITHM=INSTANT/INPLACE succeeds but the subsequent SELECT
     crashes the server with an assertion in row_sel_store_mysql_rec().

Root cause:
  length_prevents_inplace() in sql/field.cc checks whether the column
  length crosses the 256-byte row-format storage boundary.  The check
  compared raw display widths without accounting for the 2-byte
  compression header that COLUMN_FORMAT COMPRESSED prepends to every
  stored value.

  For ASCII VARCHAR(253) the raw width (253) is below 256, and after
  extension to VARCHAR(254) the raw width (254) is still below 256, so
  the ALTER was incorrectly allowed inplace/instant.  In reality the
  effective on-disk sizes are 255 and 256 respectively, which does cross
  the boundary and requires a full table rebuild.  The stale metadata
  triggered an assertion (ut_dbg_assertion_failed) in InnoDB when the
  next SELECT decoded the row.

Fix:
  Before the 256-byte threshold comparison, compute effective lengths
  that include the 2-byte compressed column header:

    from_row_pack_length = from.row_pack_length()
                         + (compressed ? 2 : 0)
    to_row_pack_length   = to.max_display_width_in_bytes()
                         + (compressed ? 2 : 0)

  Both the "from" and "to" sides are adjusted so the check is symmetric.

Test:
  mysql-test/suite/innodb/t/xtradb_compressed_columns.test (PS-10072)
  - Create VARCHAR(253) COLUMN_FORMAT COMPRESSED CHARACTER SET ascii.
  - Insert a 253-byte row.
  - Assert ALGORITHM=INPLACE is rejected with
    ER_ALTER_OPERATION_NOT_SUPPORTED_REASON.
  - Verify the default ALTER (ALGORITHM=COPY) succeeds and the row is
    readable after modification.

mysql-test/suite/innodb/r/xtradb_compressed_columns.result

@@ -324,3 +324,19 @@ UPDATE t1 SET ip_col=2680 WHERE ip_col = 23486;
 UPDATE t1 SET ip_col=23486 WHERE ip_col = 2680;
 include/assert.inc ['value inside lt3 compressed column should be valid']
 DROP TABLE t1;
+
+#
+# Bug PS-10072: ALTER VARCHAR(253->254) on compressed column must not use inplace
+#
+CREATE TABLE t11(
+  c1 VARCHAR(253) COLUMN_FORMAT COMPRESSED
+) CHARACTER SET ascii ENGINE=InnoDB;
+INSERT INTO t11 VALUES (REPEAT('a', 253));
+ALTER TABLE t11 MODIFY c1 VARCHAR(254) COLUMN_FORMAT COMPRESSED, ALGORITHM=INPLACE;
+ERROR 0A000: ALGORITHM=INPLACE is not supported. Reason: Cannot change column type INPLACE. Try ALGORITHM=COPY.
+ALTER TABLE t11 MODIFY c1 VARCHAR(254) COLUMN_FORMAT COMPRESSED;
+SELECT LENGTH(c1) FROM t11;
+LENGTH(c1)
+253
+DROP TABLE t11;
+

mysql-test/suite/innodb/t/xtradb_compressed_columns.test

@@ -230,3 +230,17 @@ UPDATE t1 SET ip_col=23486 WHERE ip_col = 2680;
 --source include/assert.inc
 
 DROP TABLE t1;
+
+--echo #
+--echo # Bug PS-10072: ALTER VARCHAR(253->254) on compressed column must not use inplace
+--echo #
+CREATE TABLE t11(
+  c1 VARCHAR(253) COLUMN_FORMAT COMPRESSED
+) CHARACTER SET ascii ENGINE=InnoDB;
+INSERT INTO t11 VALUES (REPEAT('a', 253));
+--error ER_ALTER_OPERATION_NOT_SUPPORTED_REASON
+ALTER TABLE t11 MODIFY c1 VARCHAR(254) COLUMN_FORMAT COMPRESSED, ALGORITHM=INPLACE;
+ALTER TABLE t11 MODIFY c1 VARCHAR(254) COLUMN_FORMAT COMPRESSED;
+SELECT LENGTH(c1) FROM t11;
+DROP TABLE t11;
+

sql/field.cc

@@ -155,12 +155,26 @@ bool sql_type_prevents_inplace(const Field &from, const Create_field &to) {
 */
 bool length_prevents_inplace(const Field &from, const Create_field &to) {
   DBUG_TRACE;
+  constexpr size_t kCompressedColumnHeaderLength = 2;
+  const size_t to_row_pack_length =
+      to.max_display_width_in_bytes() +
+      (to.column_format() == COLUMN_FORMAT_TYPE_COMPRESSED
+           ? kCompressedColumnHeaderLength
+           : 0);
+  const size_t from_row_pack_length =
+      from.row_pack_length() +
+      (from.column_format() == COLUMN_FORMAT_TYPE_COMPRESSED
+           ? kCompressedColumnHeaderLength
+           : 0);
+
   DBUG_PRINT(
       "inplace",
       ("from:%p, to.field:%p, to.field->row_pack_length():%u, "
-       "to.max_display_width_in_bytes():%zu",
+       "to.max_display_width_in_bytes():%zu, from_row_pack_length:%zu, "
+       "to_row_pack_length:%zu",
        &from, to.field, to.field ? to.field->row_pack_length() : (uint)-1,
-       to.max_display_width_in_bytes()));
+       to.max_display_width_in_bytes(), from_row_pack_length,
+       to_row_pack_length));
 
   if (to.pack_length() < from.pack_length()) {
     DBUG_PRINT(
@@ -170,11 +184,11 @@ bool length_prevents_inplace(const Field &from, const Create_field &to) {
     return true;
   }
 
-  if (to.max_display_width_in_bytes() >= 256 && from.row_pack_length() < 256) {
+  if (to_row_pack_length >= 256 && from_row_pack_length < 256) {
     DBUG_PRINT("inplace",
                ("row_pack_length increases past the 256 threshold, from %u to "
                 "%zu, -> true for '%s'",
-                from.row_pack_length(), to.max_display_width_in_bytes(),
+                from.row_pack_length(), to_row_pack_length,
                 current_thd->query().str));
     DBUG_PRINT("inplace",
                ("from:%p, to.field:%p, to.field->row_pack_length():%u", &from,