diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala index 177b08e949147..33eeb5b9d3363 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala @@ -1016,7 +1016,7 @@ class Analyzer(override val catalogManager: CatalogManager) case s: ExposesMetadataColumns => s.withMetadataColumns() case p: Project => val newProj = p.copy( - projectList = p.metadataOutput ++ p.projectList, + projectList = p.projectList ++ p.metadataOutput, child = addMetadataCol(p.child)) newProj.copyTagsFrom(p) newProj diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala index f984019401e2c..fe7baa5288312 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala @@ -4588,6 +4588,45 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark sql("SELECT /*+ hash(t2) */ * FROM t1 join t2 on c1 = c2") } } + + test("SPARK-41538: Metadata column should be appended at the end of project") { + val tableName = "table_1" + val viewName = "view_1" + withTable(tableName) { + withView(viewName) { + sql(s"CREATE TABLE $tableName (a ARRAY, s STRUCT) USING parquet") + val id = "id1" + sql(s"INSERT INTO $tableName values(ARRAY('a'), named_struct('id', '$id'))") + sql( + s""" + |CREATE VIEW $viewName (id) + |AS WITH source AS ( + | SELECT * FROM $tableName + |), + |renamed AS ( + | SELECT s.id FROM source + |) + |SELECT id FROM renamed + |""".stripMargin) + val query = + s""" + |with foo AS ( + | SELECT '$id' as id + |), + |bar AS ( + | SELECT '$id' as id + |) + |SELECT + | 1 + |FROM foo + |FULL OUTER JOIN bar USING(id) + |FULL OUTER JOIN $viewName USING(id) + |WHERE foo.id IS NOT NULL + |""".stripMargin + checkAnswer(sql(query), Row(1)) + } + } + } } case class Foo(bar: Option[String])