Converted Schema to a class (dotnet#1167)

* Created a Schema class for eager schema.
Converted existing row mappers to use Schema.

Author: Zruty0

src/Microsoft.ML.Api/DataViewConstructionUtils.cs

@@ -2,14 +2,14 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 // See the LICENSE file in the project root for more information.
 
+using Microsoft.ML.Runtime.Data;
+using Microsoft.ML.Runtime.Internal.Utilities;
+using Microsoft.ML.Runtime.Model;
 using System;
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
 using System.Reflection;
-using Microsoft.ML.Runtime.Data;
-using Microsoft.ML.Runtime.Internal.Utilities;
-using Microsoft.ML.Runtime.Model;
 
 namespace Microsoft.ML.Runtime.Api
 {
@@ -83,7 +83,7 @@ public sealed class InputRow<TRow> : InputRowBase<TRow>, IRowBackedBy<TRow>
             public override long Position => _position;
 
             public InputRow(IHostEnvironment env, InternalSchemaDefinition schemaDef)
-                : base(env, new SchemaProxy(schemaDef), schemaDef, MakePeeks(schemaDef), c => true)
+                : base(env, new Schema(GetSchemaColumns(schemaDef)), schemaDef, MakePeeks(schemaDef), c => true)
             {
                 _position = -1;
             }
@@ -136,11 +136,11 @@ public abstract class InputRowBase<TRow> : IRow
 
             public long Batch => 0;
 
-            public ISchema Schema { get; }
+            public Schema Schema { get; }
 
             public abstract long Position { get; }
 
-            public InputRowBase(IHostEnvironment env, ISchema schema, InternalSchemaDefinition schemaDef, Delegate[] peeks, Func<int, bool> predicate)
+            public InputRowBase(IHostEnvironment env, Schema schema, InternalSchemaDefinition schemaDef, Delegate[] peeks, Func<int, bool> predicate)
             {
                 Contracts.AssertValue(env);
                 Host = env.Register("Row");
@@ -326,27 +326,30 @@ public abstract class DataViewBase<TRow> : IDataView
         {
             protected readonly IHost Host;
 
-            private readonly SchemaProxy _schema;
+            private readonly Schema _schema;
+            private readonly InternalSchemaDefinition _schemaDefn;
 
             // The array of generated methods that extract the fields of the current row object.
             private readonly Delegate[] _peeks;
 
             public abstract bool CanShuffle { get; }
 
-            public ISchema Schema => _schema;
+            public Schema Schema => _schema;
 
             protected DataViewBase(IHostEnvironment env, string name, InternalSchemaDefinition schemaDefn)
             {
                 Contracts.AssertValue(env);
                 env.AssertNonWhiteSpace(name);
                 Host = env.Register(name);
                 Host.AssertValue(schemaDefn);
-                _schema = new SchemaProxy(schemaDefn);
-                int n = _schema.SchemaDefn.Columns.Length;
+
+                _schemaDefn = schemaDefn;
+                _schema = new Schema(GetSchemaColumns(schemaDefn));
+                int n = schemaDefn.Columns.Length;
                 _peeks = new Delegate[n];
                 for (var i = 0; i < n; i++)
                 {
-                    var currentColumn = _schema.SchemaDefn.Columns[i];
+                    var currentColumn = schemaDefn.Columns[i];
                     _peeks[i] = currentColumn.IsComputed
                         ? currentColumn.Generator
                         : ApiUtils.GeneratePeek<DataViewBase<TRow>, TRow>(currentColumn);
@@ -381,7 +384,7 @@ public abstract class DataViewCursorBase : InputRowBase<TRow>, IRowCursor
 
                 protected DataViewCursorBase(IHostEnvironment env, DataViewBase<TRow> dataView,
                     Func<int, bool> predicate)
-                    : base(env, dataView.Schema, dataView._schema.SchemaDefn, dataView._peeks, predicate)
+                    : base(env, dataView.Schema, dataView._schemaDefn, dataView._peeks, predicate)
                 {
                     Contracts.AssertValue(env);
                     Ch = env.Start("Cursor");
@@ -747,72 +750,20 @@ protected override bool MoveManyCore(long count)
             }
         }
 
-        private sealed class SchemaProxy : ISchema
+        internal static Schema.Column[] GetSchemaColumns(InternalSchemaDefinition schemaDefn)
         {
-            public readonly InternalSchemaDefinition SchemaDefn;
-
-            public SchemaProxy(InternalSchemaDefinition schemaDefn)
-            {
-                SchemaDefn = schemaDefn;
-            }
-
-            public int ColumnCount
-            {
-                get { return SchemaDefn.Columns.Length; }
-            }
-
-            public bool TryGetColumnIndex(string name, out int col)
-            {
-                col = Array.FindIndex(SchemaDefn.Columns, c => c.ColumnName == name);
-                return col >= 0;
-            }
-
-            public string GetColumnName(int col)
+            Contracts.AssertValue(schemaDefn);
+            var columns = new Schema.Column[schemaDefn.Columns.Length];
+            for (int i = 0; i < columns.Length; i++)
             {
-                CheckColumnInRange(col);
-                return SchemaDefn.Columns[col].ColumnName;
+                var col = schemaDefn.Columns[i];
+                var meta = new Schema.Metadata.Builder();
+                foreach (var kvp in col.Metadata)
+                    meta.Add(new Schema.Column(kvp.Value.Kind, kvp.Value.MetadataType, null), kvp.Value.GetGetterDelegate());
+                columns[i] = new Schema.Column(col.ColumnName, col.ColumnType, meta.GetMetadata());
             }
 
-            public ColumnType GetColumnType(int col)
-            {
-                CheckColumnInRange(col);
-                return SchemaDefn.Columns[col].ColumnType;
-            }
-
-            public IEnumerable<KeyValuePair<string, ColumnType>> GetMetadataTypes(int col)
-            {
-                CheckColumnInRange(col);
-                var columnMetadata = SchemaDefn.Columns[col].Metadata;
-                if (columnMetadata == null)
-                    yield break;
-                foreach (var kvp in columnMetadata.Select(x => new KeyValuePair<string, ColumnType>(x.Key, x.Value.MetadataType)))
-                    yield return kvp;
-            }
-
-            public ColumnType GetMetadataTypeOrNull(string kind, int col)
-            {
-                if (string.IsNullOrEmpty(kind))
-                    throw MetadataUtils.ExceptGetMetadata();
-                CheckColumnInRange(col);
-                var column = SchemaDefn.Columns[col];
-                return column.Metadata.ContainsKey(kind) ? column.Metadata[kind].MetadataType : null;
-            }
-
-            public void GetMetadata<TValue>(string kind, int col, ref TValue value)
-            {
-                var metadataType = GetMetadataTypeOrNull(kind, col);
-                if (metadataType == null)
-                    throw MetadataUtils.ExceptGetMetadata();
-
-                var metadata = SchemaDefn.Columns[col].Metadata[kind];
-                metadata.GetGetter<TValue>()(ref value);
-            }
-
-            private void CheckColumnInRange(int columnIndex)
-            {
-                if (columnIndex < 0 || columnIndex >= SchemaDefn.Columns.Length)
-                    throw Contracts.Except("Column index must be between 0 and {0}", SchemaDefn.Columns.Length);
-            }
+            return columns;
         }
     }
 
@@ -833,6 +784,8 @@ public abstract partial class MetadataInfo
 
         public abstract ValueGetter<TDst> GetGetter<TDst>();
 
+        internal abstract Delegate GetGetterDelegate();
+
         protected MetadataInfo(string kind, ColumnType metadataType)
         {
             Contracts.AssertValueOrNull(metadataType);
@@ -951,6 +904,8 @@ public override ValueGetter<TDst> GetGetter<TDst>()
             throw Contracts.ExceptNotImpl("Type '{0}' is not yet supported.", typeT.FullName);
         }
 
+        internal override Delegate GetGetterDelegate() => Utils.MarshalInvoke(GetGetter<int>, MetadataType.RawType);
+
         public class TElement
         {
         }

src/Microsoft.ML.Api/MapTransform.cs

@@ -2,10 +2,10 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 // See the LICENSE file in the project root for more information.
 
-using System;
-using System.IO;
 using Microsoft.ML.Runtime.Data;
 using Microsoft.ML.Runtime.Internal.Utilities;
+using System;
+using System.IO;
 
 namespace Microsoft.ML.Runtime.Api
 {
@@ -25,7 +25,8 @@ internal sealed class MapTransform<TSrc, TDst> : LambdaTransformBase, ITransform
     {
         private const string RegistrationNameTemplate = "MapTransform<{0}, {1}>";
         private readonly Action<TSrc, TDst> _mapAction;
-        private readonly MergedSchema _schema;
+        private readonly InternalSchemaDefinition _addedSchema;
+        private readonly ColumnBindings _bindings;
 
         // Memorized input schema definition. Needed for re-apply.
         private readonly SchemaDefinition _inputSchemaDefinition;
@@ -67,7 +68,8 @@ public MapTransform(IHostEnvironment env, IDataView source, Action<TSrc, TDst> m
                ? InternalSchemaDefinition.Create(typeof(TDst), SchemaDefinition.Direction.Write)
                : InternalSchemaDefinition.Create(typeof(TDst), outputSchemaDefinition);
 
-            _schema = MergedSchema.Create(Source.Schema, outSchema);
+            _addedSchema = outSchema;
+            _bindings = new ColumnBindings(Data.Schema.Create(Source.Schema), DataViewConstructionUtils.GetSchemaColumns(outSchema));
         }
 
         /// <summary>
@@ -83,12 +85,13 @@ private MapTransform(IHostEnvironment env, MapTransform<TSrc, TDst> transform, I
             _mapAction = transform._mapAction;
             _typedSource = TypedCursorable<TSrc>.Create(Host, newSource, false, transform._inputSchemaDefinition);
 
-            _schema = MergedSchema.Create(newSource.Schema, transform._schema.AddedSchema);
+            _addedSchema = transform._addedSchema;
+            _bindings = new ColumnBindings(Data.Schema.Create(newSource.Schema), DataViewConstructionUtils.GetSchemaColumns(_addedSchema));
         }
 
         public bool CanShuffle => Source.CanShuffle;
 
-        public ISchema Schema => _schema;
+        public Schema Schema => _bindings.Schema;
 
         public long? GetRowCount(bool lazy = true)
         {
@@ -104,7 +107,7 @@ public IRowCursor GetRowCursor(Func<int, bool> predicate, IRandom rand = null)
             if (DataViewUtils.TryCreateConsolidatingCursor(out curs, this, predicate, Host, rand))
                 return curs;
 
-            var activeInputs = _schema.GetActiveInput(predicate);
+            var activeInputs = _bindings.GetActiveInput(predicate);
             Func<int, bool> srcPredicate = c => activeInputs[c];
 
             var input = _typedSource.GetCursor(srcPredicate, rand == null ? (int?)null : rand.Next());
@@ -116,7 +119,7 @@ public IRowCursor[] GetRowCursorSet(out IRowCursorConsolidator consolidator, Fun
             Host.CheckValue(predicate, nameof(predicate));
             Host.CheckValueOrNull(rand);
 
-            var activeInputs = _schema.GetActiveInput(predicate);
+            var activeInputs = _bindings.GetActiveInput(predicate);
             Func<int, bool> srcPredicate = c => activeInputs[c];
 
             var inputs = _typedSource.GetCursorSet(out consolidator, srcPredicate, n, rand);
@@ -140,7 +143,7 @@ public IDataTransform ApplyToData(IHostEnvironment env, IDataView newSource)
         public Func<int, bool> GetDependencies(Func<int, bool> predicate)
         {
             Host.CheckValue(predicate, nameof(predicate));
-            var activeInput = _schema.GetActiveInput(predicate);
+            var activeInput = _bindings.GetActiveInput(predicate);
             Func<int, bool> srcPredicate =
                 c =>
                 {
@@ -150,7 +153,7 @@ public Func<int, bool> GetDependencies(Func<int, bool> predicate)
             return _typedSource.GetDependencies(srcPredicate);
         }
 
-        ISchema IRowToRowMapper.InputSchema => Source.Schema;
+        Schema IRowToRowMapper.InputSchema => Source.Schema;
 
         public IRow GetRow(IRow input, Func<int, bool> active, out Action disposer)
         {
@@ -181,9 +184,9 @@ private IRow GetAppendedRow(Func<int, bool> active, TDst dst)
             // REVIEW: This is quite odd (for a cursor to create an IDataView). Consider cleaning up your
             // programming model for this. Note that you don't use the IDataView, only a cursor around a single row that
             // is owned by this cursor. Seems like that cursor implementation could be decoupled from any IDataView class.
-            var appendedDataView = new DataViewConstructionUtils.SingleRowLoopDataView<TDst>(Host, _schema.AddedSchema);
+            var appendedDataView = new DataViewConstructionUtils.SingleRowLoopDataView<TDst>(Host, _addedSchema);
             appendedDataView.SetCurrentRowObject(dst);
-            return appendedDataView.GetRowCursor(i => active(_schema.MapIinfoToCol(i)));
+            return appendedDataView.GetRowCursor(i => active(_bindings.AddedColumnIndices[i]));
         }
 
         private sealed class Cursor : SynchronizedCursorBase<IRowCursor<TSrc>>, IRowCursor
@@ -210,7 +213,7 @@ public Cursor(IHost host, MapTransform<TSrc, TDst> owner, IRowCursor<TSrc> input
                 CursorChannelAttribute.TrySetCursorChannel(host, _dst, Ch);
             }
 
-            public ISchema Schema => _row.Schema;
+            public Schema Schema => _row.Schema;
 
             public bool IsColumnActive(int col)
             {
@@ -257,7 +260,7 @@ private sealed class Row : IRow
 
             public long Position => _input.Position;
 
-            public ISchema Schema { get; }
+            public Schema Schema { get; }
 
             public Row(IRowReadableAs<TSrc> input, MapTransform<TSrc, TDst> parent, Func<int, bool> active, TSrc src, TDst dst)
             {
@@ -281,7 +284,7 @@ public ValueGetter<TValue> GetGetter<TValue>(int col)
             public ValueGetter<TValue> GetGetterCore<TValue>(int col, Action checkIsGood)
             {
                 bool isSrc;
-                int index = _parent._schema.MapColumnIndex(out isSrc, col);
+                int index = _parent._bindings.MapColumnIndex(out isSrc, col);
                 if (isSrc)
                     return _input.GetGetter<TValue>(index);
 

src/Microsoft.ML.Api/MergedSchema.cs

@@ -149,7 +149,7 @@ internal PredictionEngine(IHostEnvironment env, Stream modelStream, bool ignoreM
         {
         }
 
-        private static Func<ISchema, IRowToRowMapper> StreamChecker(IHostEnvironment env, Stream modelStream)
+        private static Func<Schema, IRowToRowMapper> StreamChecker(IHostEnvironment env, Stream modelStream)
         {
             env.CheckValue(modelStream, nameof(modelStream));
             return schema =>
@@ -173,14 +173,14 @@ internal PredictionEngine(IHostEnvironment env, ITransformer transformer, bool i
         {
         }
 
-        private static Func<ISchema, IRowToRowMapper> TransformerChecker(IExceptionContext ectx, ITransformer transformer)
+        private static Func<Schema, IRowToRowMapper> TransformerChecker(IExceptionContext ectx, ITransformer transformer)
         {
             ectx.CheckValue(transformer, nameof(transformer));
             ectx.CheckParam(transformer.IsRowToRowMapper, nameof(transformer), "Must be a row to row mapper");
             return transformer.GetRowToRowMapper;
         }
 
-        private PredictionEngine(IHostEnvironment env, Func<ISchema, IRowToRowMapper> makeMapper, bool ignoreMissingColumns,
+        private PredictionEngine(IHostEnvironment env, Func<Schema, IRowToRowMapper> makeMapper, bool ignoreMissingColumns,
                  SchemaDefinition inputSchemaDefinition, SchemaDefinition outputSchemaDefinition)
         {
             Contracts.CheckValue(env, nameof(env));