PARQUET-1253: Support for new logical type representation

parquet-column/src/main/java/org/apache/parquet/schema/GroupType.java

@@ -1,14 +1,14 @@
-/* 
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
  * regarding copyright ownership.  The ASF licenses this file
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- * 
+ *
  *   http://www.apache.org/licenses/LICENSE-2.0
- * 
+ *
  * Unless required by applicable law or agreed to in writing,
  * software distributed under the License is distributed on an
  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -47,7 +47,7 @@ public class GroupType extends Type {
    * @param fields the contained fields
    */
   public GroupType(Repetition repetition, String name, List<Type> fields) {
-    this(repetition, name, null, fields, null);
+    this(repetition, name, (LogicalTypeAnnotation) null, fields, null);
   }
 
   /**
@@ -88,6 +88,7 @@ public GroupType(Repetition repetition, String name, OriginalType originalType,
    * @param fields the contained fields
    * @param id the id of the field
    */
+  @Deprecated
   GroupType(Repetition repetition, String name, OriginalType originalType, List<Type> fields, ID id) {
     super(name, repetition, originalType, id);
     this.fields = fields;
@@ -97,6 +98,15 @@ public GroupType(Repetition repetition, String name, OriginalType originalType,
     }
   }
 
+  GroupType(Repetition repetition, String name, LogicalTypeAnnotation logicalTypeAnnotation, List<Type> fields, ID id) {
+    super(name, repetition, logicalTypeAnnotation, id);
+    this.fields = fields;
+    this.indexByName = new HashMap<String, Integer>();
+    for (int i = 0; i < fields.size(); i++) {
+      indexByName.put(fields.get(i).getName(), i);
+    }
+  }
+
   /**
    * @param id the field id
    * @return a new GroupType with the same fields and a new id

parquet-column/src/main/java/org/apache/parquet/schema/LogicalTypeAnnotation.java

@@ -280,6 +280,14 @@ private DecimalLogicalTypeAnnotation(int scale, int precision) {
       this.precision = precision;
     }
 
+    public int getPrecision() {
+      return precision;
+    }
+
+    public int getScale() {
+      return scale;
+    }
+
     @Override
     public LogicalType toLogicalType() {
       return LogicalType.DECIMAL(new DecimalType(scale, precision));

parquet-column/src/main/java/org/apache/parquet/schema/PrimitiveType.java

@@ -389,9 +389,8 @@ abstract public void addValueToPrimitiveConverter(
    * @param primitive STRING, INT64, ...
    * @param name the name of the type
    */
-  public PrimitiveType(Repetition repetition, PrimitiveTypeName primitive,
-                       String name) {
-    this(repetition, primitive, 0, name, null, null, null);
+  public PrimitiveType(Repetition repetition, PrimitiveTypeName primitive, String name) {
+    this(repetition, primitive, 0, name, (LogicalTypeAnnotation) null, null, null);
   }
 
   /**
@@ -401,15 +400,18 @@ public PrimitiveType(Repetition repetition, PrimitiveTypeName primitive,
    * @param name the name of the type
    */
   public PrimitiveType(Repetition repetition, PrimitiveTypeName primitive, int length, String name) {
-    this(repetition, primitive, length, name, null, null, null);
+    this(repetition, primitive, length, name, (LogicalTypeAnnotation) null, null, null);
   }
 
   /**
    * @param repetition OPTIONAL, REPEATED, REQUIRED
    * @param primitive STRING, INT64, ...
    * @param name the name of the type
    * @param originalType (optional) the original type to help with cross schema convertion (LIST, MAP, ...)
+   *
+   * @deprecated use {@link #PrimitiveType(Repetition, PrimitiveTypeName, String, LogicalTypeAnnotation)} instead
    */
+  @Deprecated
   public PrimitiveType(Repetition repetition, PrimitiveTypeName primitive,
                        String name, OriginalType originalType) {
     this(repetition, primitive, 0, name, originalType, null, null);
@@ -436,13 +438,20 @@ public PrimitiveType(Repetition repetition, PrimitiveTypeName primitive,
    * @param originalType (optional) the original type (MAP, DECIMAL, UTF8, ...)
    * @param decimalMeta (optional) metadata about the decimal type
    * @param id the id of the field
+   *
+   * @deprecated use {@link #PrimitiveType(Repetition, PrimitiveTypeName, int, String, LogicalTypeAnnotation, ID)} instead
    */
+  @Deprecated
   public PrimitiveType(Repetition repetition, PrimitiveTypeName primitive,
                        int length, String name, OriginalType originalType,
                        DecimalMetadata decimalMeta, ID id) {
     this(repetition, primitive, length, name, originalType, decimalMeta, id, null);
   }
 
+  /**
+   * @deprecated use {@link #PrimitiveType(Repetition, PrimitiveTypeName, int, String, LogicalTypeAnnotation, ID, ColumnOrder)} instead
+   */
+  @Deprecated
   PrimitiveType(Repetition repetition, PrimitiveTypeName primitive,
       int length, String name, OriginalType originalType,
       DecimalMetadata decimalMeta, ID id, ColumnOrder columnOrder) {
@@ -459,6 +468,37 @@ public PrimitiveType(Repetition repetition, PrimitiveTypeName primitive,
     this.columnOrder = requireValidColumnOrder(columnOrder);
   }
 
+  public PrimitiveType(Repetition repetition, PrimitiveTypeName primitive,
+                       String name, LogicalTypeAnnotation logicalTypeAnnotation) {
+    this(repetition, primitive, 0, name, logicalTypeAnnotation, null, null);
+  }
+
+  public PrimitiveType(Repetition repetition, PrimitiveTypeName primitive,
+                       int length, String name, LogicalTypeAnnotation logicalTypeAnnotation, ID id) {
+    this(repetition, primitive, length, name, logicalTypeAnnotation, id, null);
+  }
+
+  PrimitiveType(Repetition repetition, PrimitiveTypeName primitive,
+                int length, String name, LogicalTypeAnnotation logicalTypeAnnotation,
+                ID id, ColumnOrder columnOrder) {
+    super(name, repetition, logicalTypeAnnotation, id);
+    this.primitive = primitive;
+    this.length = length;
+    if (getOriginalType() == OriginalType.DECIMAL) {
+      LogicalTypeAnnotation.DecimalLogicalTypeAnnotation decimal = (LogicalTypeAnnotation.DecimalLogicalTypeAnnotation) logicalTypeAnnotation;
+      this.decimalMeta = new DecimalMetadata(decimal.getPrecision(), decimal.getScale());
+    } else {
+      this.decimalMeta = null;
+    }
+
+    if (columnOrder == null) {
+      columnOrder = primitive == PrimitiveTypeName.INT96 || getOriginalType() == OriginalType.INTERVAL
+        ? ColumnOrder.undefined()
+        : ColumnOrder.typeDefined();
+    }
+    this.columnOrder = requireValidColumnOrder(columnOrder);
+  }
+
   private ColumnOrder requireValidColumnOrder(ColumnOrder columnOrder) {
     if (primitive == PrimitiveTypeName.INT96) {
       Preconditions.checkArgument(columnOrder.getColumnOrderName() == ColumnOrderName.UNDEFINED,

parquet-column/src/main/java/org/apache/parquet/schema/Type.java

@@ -128,7 +128,7 @@ public boolean isMoreRestrictiveThan(Repetition other) {
    */
   @Deprecated
   public Type(String name, Repetition repetition) {
-    this(name, repetition, null, null);
+    this(name, repetition, (LogicalTypeAnnotation) null, null);
   }
 
   /**
@@ -146,11 +146,18 @@ public Type(String name, Repetition repetition, OriginalType originalType) {
    * @param repetition OPTIONAL, REPEATED, REQUIRED
    * @param originalType (optional) the original type to help with cross schema conversion (LIST, MAP, ...)
    * @param id (optional) the id of the fields.
+   *
+   * @deprecated use {@link #Type(String, Repetition, LogicalTypeAnnotation, ID)} instead
    */
+  @Deprecated
   Type(String name, Repetition repetition, OriginalType originalType, ID id) {
     this(name, repetition, originalType, null, id);
   }
 
+  /**
+   * @deprecated use {@link #Type(String, Repetition, LogicalTypeAnnotation, ID)} instead
+   */
+  @Deprecated
   Type(String name, Repetition repetition, OriginalType originalType, DecimalMetadata decimalMetadata, ID id) {
     super();
     this.name = checkNotNull(name, "name");
@@ -159,6 +166,18 @@ public Type(String name, Repetition repetition, OriginalType originalType) {
     this.id = id;
   }
 
+  public Type(String name, Repetition repetition, LogicalTypeAnnotation logicalTypeAnnotation) {
+    this(name, repetition, logicalTypeAnnotation, null);
+  }
+
+  Type(String name, Repetition repetition, LogicalTypeAnnotation logicalTypeAnnotation, ID id) {
+    super();
+    this.name = checkNotNull(name, "name");
+    this.repetition = checkNotNull(repetition, "repetition");
+    this.logicalTypeAnnotation = logicalTypeAnnotation;
+    this.id = id;
+  }
+
   /**
    * @param id
    * @return the same type with the id field set

parquet-column/src/main/java/org/apache/parquet/schema/Types.java

@@ -23,6 +23,7 @@
 import java.util.List;
 
 import org.apache.parquet.Preconditions;
+import org.apache.parquet.format.DecimalType;
 import org.apache.parquet.schema.ColumnOrder.ColumnOrderName;
 import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName;
 import org.apache.parquet.schema.Type.ID;
@@ -250,14 +251,32 @@ protected final THIS repetition(Type.Repetition repetition) {
      *
      * @param type an {@code OriginalType}
      * @return this builder for method chaining
+     *
+     * @deprecated use {@link #as(LogicalTypeAnnotation)} with the corresponding logical type instead
      */
+    @Deprecated
     public THIS as(OriginalType type) {
       this.logicalTypeAnnotation = LogicalTypeAnnotation.fromOriginalType(type, null);
       return self();
     }
 
+    protected boolean newLogicalTypeSet;
+
+    /**
+     * Adds a type annotation ({@link LogicalTypeAnnotation}) to the type being built.
+     * <p>
+     * Type annotations are used to extend the types that parquet can store, by
+     * specifying how the primitive types should be interpreted. This keeps the
+     * set of primitive types to a minimum and reuses parquet's efficient
+     * encodings. For example, strings are stored as byte arrays (binary) with
+     * a UTF8 annotation.
+     *
+     * @param type an {@code {@link LogicalTypeAnnotation}}
+     * @return this builder for method chaining
+     */
     public THIS as(LogicalTypeAnnotation type) {
       this.logicalTypeAnnotation = type;
+      this.newLogicalTypeSet = true;
       return self();
     }
 
@@ -351,6 +370,9 @@ public THIS length(int length) {
       return self();
     }
 
+    private boolean precisionAlreadySet;
+    private boolean scaleAlreadySet;
+
     /**
      * Adds the precision for a DECIMAL.
      * <p>
@@ -360,9 +382,13 @@ public THIS length(int length) {
      *
      * @param precision an int precision value for the DECIMAL
      * @return this builder for method chaining
+     *
+     * @deprecated use {@link #as(LogicalTypeAnnotation)} with the corresponding decimal type instead
      */
+    @Deprecated
     public THIS precision(int precision) {
       this.precision = precision;
+      precisionAlreadySet = true;
       return self();
     }
 
@@ -378,9 +404,13 @@ public THIS precision(int precision) {
      *
      * @param scale an int scale value for the DECIMAL
      * @return this builder for method chaining
+     *
+     * @deprecated use {@link #as(LogicalTypeAnnotation)} with the corresponding decimal type instead
      */
+    @Deprecated
     public THIS scale(int scale) {
       this.scale = scale;
+      scaleAlreadySet = true;
       return self();
     }
 
@@ -498,11 +528,24 @@ private static long maxPrecision(int numBytes) {
     protected DecimalMetadata decimalMetadata() {
       DecimalMetadata meta = null;
       if (OriginalType.DECIMAL == getOriginalType()) {
+        LogicalTypeAnnotation.DecimalLogicalTypeAnnotation decimalType = (LogicalTypeAnnotation.DecimalLogicalTypeAnnotation) logicalTypeAnnotation;
+        if (newLogicalTypeSet) {
+          if (scaleAlreadySet) {
+            Preconditions.checkArgument(this.scale == decimalType.getScale(),
+              "Decimal scale should match with the scale of the logical type");
+          }
+          if (precisionAlreadySet) {
+            Preconditions.checkArgument(this.precision == decimalType.getPrecision(),
+              "Decimal precision should match with the precision of the logical type");
+          }
+          scale = decimalType.getScale();
+          precision = decimalType.getPrecision();
+        }
         Preconditions.checkArgument(precision > 0,
             "Invalid DECIMAL precision: " + precision);
-        Preconditions.checkArgument(scale >= 0,
-            "Invalid DECIMAL scale: " + scale);
-        Preconditions.checkArgument(scale <= precision,
+        Preconditions.checkArgument(this.scale >= 0,
+            "Invalid DECIMAL scale: " + this.scale);
+        Preconditions.checkArgument(this.scale <= precision,
             "Invalid DECIMAL scale: cannot be greater than precision");
         meta = new DecimalMetadata(precision, scale);
       }

parquet-column/src/test/java/org/apache/parquet/schema/TestTypeBuilders.java

@@ -1,14 +1,14 @@
-/* 
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
  * regarding copyright ownership.  The ASF licenses this file
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- * 
+ *
  *   http://www.apache.org/licenses/LICENSE-2.0
- * 
+ *
  * Unless required by applicable law or agreed to in writing,
  * software distributed under the License is distributed on an
  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -1396,6 +1396,47 @@ public PrimitiveType call() {
     });
   }
 
+  @Test
+  public void testDecimalLogicalType() {
+    PrimitiveType expected = new PrimitiveType(REQUIRED, BINARY, "aDecimal",
+      LogicalTypeAnnotation.DecimalLogicalTypeAnnotation.create(3, 4));
+    PrimitiveType actual = Types.required(BINARY)
+      .as(LogicalTypeAnnotation.DecimalLogicalTypeAnnotation.create(3, 4)).named("aDecimal");
+    Assert.assertEquals(expected, actual);
+  }
+
+  @Test
+  public void testDecimalLogicalTypeWithDeprecatedScale() {
+    PrimitiveType expected = new PrimitiveType(REQUIRED, BINARY, "aDecimal",
+      LogicalTypeAnnotation.DecimalLogicalTypeAnnotation.create(3, 4));
+    PrimitiveType actual = Types.required(BINARY)
+      .as(LogicalTypeAnnotation.DecimalLogicalTypeAnnotation.create(3, 4)).scale(3).named("aDecimal");
+    Assert.assertEquals(expected, actual);
+  }
+
+  @Test
+  public void testDecimalLogicalTypeWithDeprecatedPrecision() {
+    PrimitiveType expected = new PrimitiveType(REQUIRED, BINARY, "aDecimal",
+      LogicalTypeAnnotation.DecimalLogicalTypeAnnotation.create(3, 4));
+    PrimitiveType actual = Types.required(BINARY)
+      .as(LogicalTypeAnnotation.DecimalLogicalTypeAnnotation.create(3, 4)).precision(4).named("aDecimal");
+    Assert.assertEquals(expected, actual);
+  }
+
+  @Test(expected = IllegalArgumentException.class)
+  public void testDecimalLogicalTypeWithDeprecatedScaleMismatch() {
+    Types.required(BINARY)
+      .as(LogicalTypeAnnotation.DecimalLogicalTypeAnnotation.create(3, 4))
+      .scale(4).named("aDecimal");
+  }
+
+  @Test(expected = IllegalArgumentException.class)
+  public void testDecimalLogicalTypeWithDeprecatedPrecisionMismatch() {
+    Types.required(BINARY)
+      .as(LogicalTypeAnnotation.DecimalLogicalTypeAnnotation.create(3, 4))
+      .precision(5).named("aDecimal");
+  }
+
   /**
    * A convenience method to avoid a large number of @Test(expected=...) tests
    * @param message A String message to describe this assertion