Remove the uses of CreatePredictionEngine (dotnet#1637)

* One test reborns

* Remove two mores uses and clean copy-and-paste

* Clean the rest TF tests

* Clean the last CreatePredictionEngine

* Further clean up redundant ctor's of PredictionEngine and their relatives

* Use paths in TestDatasets

* Rollback baselines

* Fix

Author: wschin

src/Microsoft.ML.Api/ComponentCreation.cs

@@ -150,46 +150,6 @@ public static BatchPredictionEngine<TSrc, TDst> CreateBatchPredictionEngine<TSrc
             return new BatchPredictionEngine<TSrc, TDst>(env, dataPipe, ignoreMissingColumns, inputSchemaDefinition, outputSchemaDefinition);
         }
 
-        /// <summary>
-        /// Create an on-demand prediction engine.
-        /// </summary>
-        /// <param name="env">The host environment to use.</param>
-        /// <param name="modelStream">The stream to deserialize the pipeline (transforms and predictor) from.</param>
-        /// <param name="ignoreMissingColumns">Whether to ignore missing columns in the data view.</param>
-        /// <param name="inputSchemaDefinition">The optional input schema. If <c>null</c>, the schema is inferred from the <typeparamref name="TSrc"/> type.</param>
-        /// <param name="outputSchemaDefinition">The optional output schema. If <c>null</c>, the schema is inferred from the <typeparamref name="TDst"/> type.</param>
-        public static PredictionEngine<TSrc, TDst> CreatePredictionEngine<TSrc, TDst>(this IHostEnvironment env, Stream modelStream,
-            bool ignoreMissingColumns = false, SchemaDefinition inputSchemaDefinition = null, SchemaDefinition outputSchemaDefinition = null)
-            where TSrc : class
-            where TDst : class, new()
-        {
-            Contracts.CheckValue(env, nameof(env));
-            env.CheckValue(modelStream, nameof(modelStream));
-            env.CheckValueOrNull(inputSchemaDefinition);
-            env.CheckValueOrNull(outputSchemaDefinition);
-            return new PredictionEngine<TSrc, TDst>(env, modelStream, ignoreMissingColumns, inputSchemaDefinition, outputSchemaDefinition);
-        }
-
-        /// <summary>
-        /// Create an on-demand prediction engine.
-        /// </summary>
-        /// <param name="env">The host environment to use.</param>
-        /// <param name="dataPipe">The transformation pipe that may or may not include a scorer.</param>
-        /// <param name="ignoreMissingColumns">Whether to ignore missing columns in the data view.</param>
-        /// <param name="inputSchemaDefinition">The optional input schema. If <c>null</c>, the schema is inferred from the <typeparamref name="TSrc"/> type.</param>
-        /// <param name="outputSchemaDefinition">The optional output schema. If <c>null</c>, the schema is inferred from the <typeparamref name="TDst"/> type.</param>
-        public static PredictionEngine<TSrc, TDst> CreatePredictionEngine<TSrc, TDst>(this IHostEnvironment env, IDataView dataPipe,
-            bool ignoreMissingColumns = false, SchemaDefinition inputSchemaDefinition = null, SchemaDefinition outputSchemaDefinition = null)
-            where TSrc : class
-            where TDst : class, new()
-        {
-            Contracts.CheckValue(env, nameof(env));
-            env.CheckValue(dataPipe, nameof(dataPipe));
-            env.CheckValueOrNull(inputSchemaDefinition);
-            env.CheckValueOrNull(outputSchemaDefinition);
-            return new PredictionEngine<TSrc, TDst>(env, dataPipe, ignoreMissingColumns, inputSchemaDefinition, outputSchemaDefinition);
-        }
-
         /// <summary>
         /// Create an on-demand prediction engine.
         /// </summary>
@@ -198,7 +158,7 @@ public static PredictionEngine<TSrc, TDst> CreatePredictionEngine<TSrc, TDst>(th
         /// <param name="ignoreMissingColumns">Whether to ignore missing columns in the data view.</param>
         /// <param name="inputSchemaDefinition">The optional input schema. If <c>null</c>, the schema is inferred from the <typeparamref name="TSrc"/> type.</param>
         /// <param name="outputSchemaDefinition">The optional output schema. If <c>null</c>, the schema is inferred from the <typeparamref name="TDst"/> type.</param>
-        public static PredictionEngine<TSrc, TDst> CreatePredictionEngine<TSrc, TDst>(this IHostEnvironment env, ITransformer transformer,
+        internal static PredictionEngine<TSrc, TDst> CreatePredictionEngine<TSrc, TDst>(this IHostEnvironment env, ITransformer transformer,
             bool ignoreMissingColumns = false, SchemaDefinition inputSchemaDefinition = null, SchemaDefinition outputSchemaDefinition = null)
             where TSrc : class
             where TDst : class, new()
@@ -210,23 +170,6 @@ public static PredictionEngine<TSrc, TDst> CreatePredictionEngine<TSrc, TDst>(th
             return new PredictionEngine<TSrc, TDst>(env, transformer, ignoreMissingColumns, inputSchemaDefinition, outputSchemaDefinition);
         }
 
-        /// <summary>
-        /// Create a prediction engine.
-        /// This encapsulates the 'classic' prediction problem, where the input is denoted by the float array of features,
-        /// and the output is a float score. For binary classification predictors that can output probability, there are output
-        /// fields that report the predicted label and probability.
-        /// </summary>
-        /// <param name="env">The host environment to use.</param>
-        /// <param name="modelStream">The model stream to load pipeline from.</param>
-        /// <param name="nFeatures">Number of features.</param>
-        public static SimplePredictionEngine CreateSimplePredictionEngine(this IHostEnvironment env, Stream modelStream, int nFeatures)
-        {
-            Contracts.CheckValue(env, nameof(env));
-            env.CheckValue(modelStream, nameof(modelStream));
-            env.CheckParam(nFeatures > 0, nameof(nFeatures), "Number of features must be positive.");
-            return new SimplePredictionEngine(env, modelStream, nFeatures);
-        }
-
         /// <summary>
         /// Load the transforms (but not loader) from the model steram and apply them to the specified data.
         /// It is acceptable to have no transforms in the model stream: in this case the original

src/Microsoft.ML.Api/PredictionEngine.cs

@@ -140,12 +140,6 @@ public sealed class PredictionEngine<TSrc, TDst>
         private readonly IRowReadableAs<TDst> _outputRow;
         private readonly Action _disposer;
 
-        internal PredictionEngine(IHostEnvironment env, Stream modelStream, bool ignoreMissingColumns,
-            SchemaDefinition inputSchemaDefinition = null, SchemaDefinition outputSchemaDefinition = null)
-            : this(env, StreamChecker(env, modelStream), ignoreMissingColumns, inputSchemaDefinition, outputSchemaDefinition)
-        {
-        }
-
         private static Func<Schema, IRowToRowMapper> StreamChecker(IHostEnvironment env, Stream modelStream)
         {
             env.CheckValue(modelStream, nameof(modelStream));
@@ -158,29 +152,12 @@ private static Func<Schema, IRowToRowMapper> StreamChecker(IHostEnvironment env,
             };
         }
 
-        internal PredictionEngine(IHostEnvironment env, IDataView dataPipe, bool ignoreMissingColumns,
-            SchemaDefinition inputSchemaDefinition = null, SchemaDefinition outputSchemaDefinition = null)
-            : this(env, new TransformWrapper(env, env.CheckRef(dataPipe, nameof(dataPipe))), ignoreMissingColumns, inputSchemaDefinition, outputSchemaDefinition)
-        {
-        }
-
         internal PredictionEngine(IHostEnvironment env, ITransformer transformer, bool ignoreMissingColumns,
             SchemaDefinition inputSchemaDefinition = null, SchemaDefinition outputSchemaDefinition = null)
-            : this(env, TransformerChecker(env, transformer), ignoreMissingColumns, inputSchemaDefinition, outputSchemaDefinition)
-        {
-        }
-
-        private static Func<Schema, IRowToRowMapper> TransformerChecker(IExceptionContext ectx, ITransformer transformer)
-        {
-            ectx.CheckValue(transformer, nameof(transformer));
-            ectx.CheckParam(transformer.IsRowToRowMapper, nameof(transformer), "Must be a row to row mapper");
-            return transformer.GetRowToRowMapper;
-        }
-
-        private PredictionEngine(IHostEnvironment env, Func<Schema, IRowToRowMapper> makeMapper, bool ignoreMissingColumns,
-                 SchemaDefinition inputSchemaDefinition, SchemaDefinition outputSchemaDefinition)
         {
             Contracts.CheckValue(env, nameof(env));
+            env.AssertValue(transformer);
+            var makeMapper = TransformerChecker(env, transformer);
             env.AssertValue(makeMapper);
 
             _inputRow = DataViewConstructionUtils.CreateInputRow<TSrc>(env, inputSchemaDefinition);
@@ -190,6 +167,13 @@ private PredictionEngine(IHostEnvironment env, Func<Schema, IRowToRowMapper> mak
             _outputRow = cursorable.GetRow(outputRow);
         }
 
+        private static Func<Schema, IRowToRowMapper> TransformerChecker(IExceptionContext ectx, ITransformer transformer)
+        {
+            ectx.CheckValue(transformer, nameof(transformer));
+            ectx.CheckParam(transformer.IsRowToRowMapper, nameof(transformer), "Must be a row to row mapper");
+            return transformer.GetRowToRowMapper;
+        }
+
         ~PredictionEngine()
         {
             _disposer?.Invoke();
@@ -222,76 +206,4 @@ public void Predict(TSrc example, ref TDst prediction)
             _outputRow.FillValues(prediction);
         }
     }
-
-    /// <summary>
-    /// This class encapsulates the 'classic' prediction problem, where the input is denoted by the float array of features,
-    /// and the output is a float score. For binary classification predictors that can output probability, there are output
-    /// fields that report the predicted label and probability.
-    /// </summary>
-    public sealed class SimplePredictionEngine
-    {
-        private class Example
-        {
-            // REVIEW: convert to VBuffer once we have support for them.
-            public Float[] Features;
-        }
-
-        /// <summary>
-        /// The prediction output. For every field, if there are no column with the matched name in the scoring pipeline,
-        /// the field will be left intact by the engine (and keep 0 as value unless the user code changes it).
-        /// </summary>
-        public class Prediction
-        {
-            public Float Score;
-            public Float Probability;
-        }
-
-        private readonly PredictionEngine<Example, Prediction> _engine;
-        private readonly int _nFeatures;
-
-        /// <summary>
-        /// Create a prediction engine.
-        /// </summary>
-        /// <param name="env">The host environment to use.</param>
-        /// <param name="modelStream">The model stream to load pipeline from.</param>
-        /// <param name="nFeatures">Number of features.</param>
-        /// <param name="featureColumnName">Name of the features column.</param>
-        internal SimplePredictionEngine(IHostEnvironment env, Stream modelStream, int nFeatures, string featureColumnName = "Features")
-        {
-            Contracts.AssertValue(env);
-            Contracts.AssertValue(modelStream);
-            Contracts.Assert(nFeatures > 0);
-
-            _nFeatures = nFeatures;
-            var schema =
-                new SchemaDefinition
-                {
-                new SchemaDefinition.Column
-                {
-                        MemberName = featureColumnName,
-                        ColumnType = new VectorType(NumberType.Float, nFeatures)
-                }
-            };
-            _engine = new PredictionEngine<Example, Prediction>(env, modelStream, true, schema);
-        }
-
-        /// <summary>
-        /// Score an example.
-        /// </summary>
-        /// <param name="features">The feature array of the example.</param>
-        /// <returns>The prediction object. New object is created on every call.</returns>
-        public Prediction Predict(Float[] features)
-        {
-            Contracts.CheckValue(features, nameof(features));
-            if (features.Length != _nFeatures)
-                throw Contracts.ExceptParam(nameof(features), "Number of features should be {0}, but it is {1}", _nFeatures, features.Length);
-
-            var example = new Example { Features = features };
-            return _engine.Predict(example);
-        }
-        public Prediction Predict(VBuffer<Float> features)
-        {
-            throw Contracts.ExceptNotImpl("VBuffers aren't supported yet.");
-        }
-    }
 }

test/Microsoft.ML.Tests/ScenariosWithDirectInstantiation/IrisPlantClassificationTests.cs

@@ -9,6 +9,7 @@
 using Microsoft.ML.Runtime.Data;
 using Microsoft.ML.Runtime.Learners;
 using Microsoft.ML.Runtime.Model;
+using Microsoft.ML.Runtime.RunTests;
 using Microsoft.ML.Trainers;
 using Microsoft.ML.Transforms.Normalizers;
 using System;
@@ -22,54 +23,43 @@ public partial class ScenariosTests
         [Fact]
         public void TrainAndPredictIrisModelUsingDirectInstantiationTest()
         {
-            string dataPath = GetDataPath("iris.txt");
-            string testDataPath = dataPath;
+            var mlContext = new MLContext(seed: 1, conc: 1);
 
-            var env = new MLContext(seed: 1, conc: 1);
-            // Pipeline
-            var loader = TextLoader.ReadFile(env,
-                new TextLoader.Arguments()
+            var reader = mlContext.Data.TextReader(new TextLoader.Arguments()
+            {
+                HasHeader = false,
+                Column = new[]
                 {
-                    HasHeader = false,
-                    Column = new[]
-                    {
-                            new TextLoader.Column("Label", DataKind.R4, 0),
-                            new TextLoader.Column("SepalLength", DataKind.R4, 1),
-                            new TextLoader.Column("SepalWidth", DataKind.R4, 2),
-                            new TextLoader.Column("PetalLength", DataKind.R4, 3),
-                            new TextLoader.Column("PetalWidth", DataKind.R4, 4)
-                    }
-                }, new MultiFileSource(dataPath));
-
-            IDataView pipeline = new ColumnConcatenatingTransformer(env, "Features",
-                "SepalLength", "SepalWidth", "PetalLength", "PetalWidth").Transform(loader);
-
-            // NormalizingEstimator is not automatically added though the trainer has 'NormalizeFeatures' On/Auto
-            pipeline = NormalizeTransform.CreateMinMaxNormalizer(env, pipeline, "Features");
-
-            // Train
-            var trainer = new SdcaMultiClassTrainer(env, "Label", "Features", advancedSettings: s => s.NumThreads = 1);
-
-            // Explicity adding CacheDataView since caching is not working though trainer has 'Caching' On/Auto
-            var cached = new CacheDataView(env, pipeline, prefetch: null);
-            var trainRoles = new RoleMappedData(cached, label: "Label", feature: "Features");
-            var pred = trainer.Train(trainRoles);
-
-            // Get scorer and evaluate the predictions from test data
-            IDataScorerTransform testDataScorer = GetScorer(env, pipeline, pred, testDataPath);
-            var metrics = Evaluate(env, testDataScorer);
-            CompareMatrics(metrics);
+                    new TextLoader.Column("Label", DataKind.R4, 0),
+                    new TextLoader.Column("SepalLength", DataKind.R4, 1),
+                    new TextLoader.Column("SepalWidth", DataKind.R4, 2),
+                    new TextLoader.Column("PetalLength", DataKind.R4, 3),
+                    new TextLoader.Column("PetalWidth", DataKind.R4, 4)
+                }
+            });
 
-            // Create prediction engine and test predictions
-            var model = env.CreatePredictionEngine<IrisData, IrisPrediction>(testDataScorer);
-            ComparePredictions(model);
+            var pipe = mlContext.Transforms.Concatenate("Features", "SepalLength", "SepalWidth", "PetalLength", "PetalWidth")
+                .Append(mlContext.Transforms.Normalize("Features"))
+                .Append(mlContext.MulticlassClassification.Trainers.StochasticDualCoordinateAscent("Label", "Features", advancedSettings: s => s.NumThreads = 1));
 
-            // Get feature importance i.e. weight vector
-            var summary = pred.GetSummaryInKeyValuePairs(trainRoles.Schema);
-            Assert.Equal(7.76443, Convert.ToDouble(summary[0].Value), 5);
+            // Read training and test data sets
+            string dataPath = GetDataPath(TestDatasets.iris.trainFilename);
+            string testDataPath = dataPath;
+            var trainData = reader.Read(dataPath);
+            var testData = reader.Read(testDataPath);
+
+            // Train the pipeline
+            var trainedModel = pipe.Fit(trainData);
+
+            // Make prediction and then evaluate the trained pipeline
+            var predicted = trainedModel.Transform(testData);
+            var metrics = mlContext.MulticlassClassification.Evaluate(predicted);
+            CompareMatrics(metrics);
+            var predictFunction = trainedModel.MakePredictionFunction<IrisData, IrisPrediction>(mlContext);
+            ComparePredictions(predictFunction);
         }
 
-        private void ComparePredictions(PredictionEngine<IrisData, IrisPrediction> model)
+        private void ComparePredictions(PredictionFunction<IrisData, IrisPrediction> model)
         {
             IrisPrediction prediction = model.Predict(new IrisData()
             {
@@ -108,46 +98,17 @@ private void ComparePredictions(PredictionEngine<IrisData, IrisPrediction> model
             Assert.Equal(0, prediction.PredictedLabels[2], 2);
         }
 
-        private void CompareMatrics(ClassificationMetrics metrics)
+        private void CompareMatrics(MultiClassClassifierEvaluator.Result metrics)
         {
             Assert.Equal(.98, metrics.AccuracyMacro);
             Assert.Equal(.98, metrics.AccuracyMicro, 2);
-            Assert.Equal(.06, metrics.LogLoss, 2);
+            Assert.InRange(metrics.LogLoss, .05, .06);
             Assert.InRange(metrics.LogLossReduction, 94, 96);
-            Assert.Equal(1, metrics.TopKAccuracy);
 
             Assert.Equal(3, metrics.PerClassLogLoss.Length);
             Assert.Equal(0, metrics.PerClassLogLoss[0], 1);
             Assert.Equal(.1, metrics.PerClassLogLoss[1], 1);
             Assert.Equal(.1, metrics.PerClassLogLoss[2], 1);
-
-            ConfusionMatrix matrix = metrics.ConfusionMatrix;
-            Assert.Equal(3, matrix.Order);
-            Assert.Equal(3, matrix.ClassNames.Count);
-            Assert.Equal("0", matrix.ClassNames[0]);
-            Assert.Equal("1", matrix.ClassNames[1]);
-            Assert.Equal("2", matrix.ClassNames[2]);
-
-            Assert.Equal(50, matrix[0, 0]);
-            Assert.Equal(50, matrix["0", "0"]);
-            Assert.Equal(0, matrix[0, 1]);
-            Assert.Equal(0, matrix["0", "1"]);
-            Assert.Equal(0, matrix[0, 2]);
-            Assert.Equal(0, matrix["0", "2"]);
-
-            Assert.Equal(0, matrix[1, 0]);
-            Assert.Equal(0, matrix["1", "0"]);
-            Assert.Equal(48, matrix[1, 1]);
-            Assert.Equal(48, matrix["1", "1"]);
-            Assert.Equal(2, matrix[1, 2]);
-            Assert.Equal(2, matrix["1", "2"]);
-
-            Assert.Equal(0, matrix[2, 0]);
-            Assert.Equal(0, matrix["2", "0"]);
-            Assert.Equal(1, matrix[2, 1]);
-            Assert.Equal(1, matrix["2", "1"]);
-            Assert.Equal(49, matrix[2, 2]);
-            Assert.Equal(49, matrix["2", "2"]);
         }
 
         private ClassificationMetrics Evaluate(IHostEnvironment env, IDataView scoredData)