TrainerEstimatorBaseWithGroupId<TTransformer, TModel>

where TTransformer : ISingleFeaturePredictionTransformer<TModel>

where TArgs : TreeArgs, new()

where TModel : IPredictorProducing<Float>

{

protected readonly TArgs Args;

protected readonly bool AllowGC;

protected TreeEnsemble TrainedEnsemble;

protected int FeatureCount;

private protected RoleMappedData ValidData;

/// <summary>

/// If not null, it's a test data set passed in from training context. It will be converted to one element in

/// <see cref="Tests"/> by calling <see cref="ExamplesToFastTreeBins.GetCompatibleDataset"/> in <see cref="InitializeTests"/>.

/// </summary>

private protected RoleMappedData TestData;

protected IParallelTraining ParallelTraining;

protected OptimizationAlgorithm OptimizationAlgorithm;

protected Dataset TrainSet;

protected Dataset ValidSet;

/// <summary>

/// Data sets used to evaluate the prediction scores produced the trained model during the triaining process.

/// </summary>

protected Dataset[] TestSets;

protected int[] FeatureMap;

/// <summary>

/// In the training process, <see cref="TrainSet"/>, <see cref="ValidSet"/>, <see cref="TestSets"/> would be

/// converted into <see cref="Tests"/> for efficient model evaluation.

/// </summary>

protected List<Test> Tests;

protected TestHistory PruningTest;

protected int[] CategoricalFeatures;

// Test for early stopping.

protected Test TrainTest;

protected Test ValidTest;

protected double[] InitTrainScores;

protected double[] InitValidScores;

protected double[][] InitTestScores;

//protected int Iteration;

protected TreeEnsemble Ensemble;

protected bool HasValidSet => ValidSet != null;

private const string RegisterName = "FastTreeTraining";

// random for active features selection

private Random _featureSelectionRandom;

protected string InnerArgs => CmdParser.GetSettings(Host, Args, new TArgs());

public override TrainerInfo Info { get; }

public bool HasCategoricalFeatures => Utils.Size(CategoricalFeatures) > 0;

private protected virtual bool NeedCalibration => false;

/// <summary>

/// Constructor to use when instantiating the classes deriving from here through the API.

/// </summary>

private protected FastTreeTrainerBase(IHostEnvironment env,

SchemaShape.Column label,

string featureColumn,

string weightColumn,

string groupIdColumn,

int numLeaves,

int numTrees,

int minDatapointsInLeaves,

Action<TArgs> advancedSettings)

: base(Contracts.CheckRef(env, nameof(env)).Register(RegisterName), TrainerUtils.MakeR4VecFeature(featureColumn), label, TrainerUtils.MakeR4ScalarWeightColumn(weightColumn), TrainerUtils.MakeU4ScalarColumn(groupIdColumn))

{

Args = new TArgs();

// set up the directly provided values

// override with the directly provided values.

Args.NumLeaves = numLeaves;

Args.NumTrees = numTrees;

Args.MinDocumentsInLeafs = minDatapointsInLeaves;

//apply the advanced args, if the user supplied any

advancedSettings?.Invoke(Args);

Args.LabelColumn = label.Name;

Args.FeatureColumn = featureColumn;

if (weightColumn != null)

Args.WeightColumn = Optional<string>.Explicit(weightColumn);

if (groupIdColumn != null)

Args.GroupIdColumn = Optional<string>.Explicit(groupIdColumn);

// The discretization step renders this trainer non-parametric, and therefore it does not need normalization.

// Also since it builds its own internal discretized columnar structures, it cannot benefit from caching.

// Finally, even the binary classifiers, being logitboost, tend to not benefit from external calibration.

Info = new TrainerInfo(normalization: false, caching: false, calibration: NeedCalibration, supportValid: true, supportTest: true);

// REVIEW: CLR 4.6 has a bug that is only exposed in Scope, and if we trigger GC.Collect in scope environment

// with memory consumption more than 5GB, GC get stuck in infinite loop.

// Before, we could check a specific type of the environment here, but now it is internal, so we will need another

// mechanism to detect that we are running in Scope.

AllowGC = true;

Initialize(env);

}

/// <summary>

/// Legacy constructor that is used when invoking the classes deriving from this, through maml.

/// </summary>

private protected FastTreeTrainerBase(IHostEnvironment env, TArgs args, SchemaShape.Column label)

: base(Contracts.CheckRef(env, nameof(env)).Register(RegisterName), TrainerUtils.MakeR4VecFeature(args.FeatureColumn), label, TrainerUtils.MakeR4ScalarWeightColumn(args.WeightColumn, args.WeightColumn.IsExplicit))

{

Host.CheckValue(args, nameof(args));

Args = args;

// The discretization step renders this trainer non-parametric, and therefore it does not need normalization.

// Also since it builds its own internal discretized columnar structures, it cannot benefit from caching.

// Finally, even the binary classifiers, being logitboost, tend to not benefit from external calibration.

Info = new TrainerInfo(normalization: false, caching: false, calibration: NeedCalibration, supportValid: true, supportTest: true);

// REVIEW: CLR 4.6 has a bug that is only exposed in Scope, and if we trigger GC.Collect in scope environment

// with memory consumption more than 5GB, GC get stuck in infinite loop.

// Before, we could check a specific type of the environment here, but now it is internal, so we will need another

// mechanism to detect that we are running in Scope.

AllowGC = true;

Initialize(env);

}

protected abstract void PrepareLabels(IChannel ch);

protected abstract void InitializeTests();

protected abstract Test ConstructTestForTrainingData();

protected abstract OptimizationAlgorithm ConstructOptimizationAlgorithm(IChannel ch);

protected abstract TreeLearner ConstructTreeLearner(IChannel ch);

protected abstract ObjectiveFunctionBase ConstructObjFunc(IChannel ch);

protected virtual Float GetMaxLabel()

{

return Float.PositiveInfinity;

}

private void Initialize(IHostEnvironment env)

{

int numThreads = Args.NumThreads ?? Environment.ProcessorCount;

if (Host.ConcurrencyFactor > 0 && numThreads > Host.ConcurrencyFactor)

{

using (var ch = Host.Start("FastTreeTrainerBase"))

{

numThreads = Host.ConcurrencyFactor;

ch.Warning("The number of threads specified in trainer arguments is larger than the concurrency factor "

+ "setting of the environment. Using {0} training threads instead.", numThreads);

}

}

ParallelTraining = Args.ParallelTrainer != null ? Args.ParallelTrainer.CreateComponent(env) : new SingleTrainer();

ParallelTraining.InitEnvironment();

Tests = new List<Test>();

InitializeThreads(numThreads);

}

private protected void ConvertData(RoleMappedData trainData)

{

MetadataUtils.TryGetCategoricalFeatureIndices(trainData.Schema.Schema, trainData.Schema.Feature.Value.Index, out CategoricalFeatures);

var useTranspose = UseTranspose(Args.DiskTranspose, trainData) && (ValidData == null || UseTranspose(Args.DiskTranspose, ValidData));

var instanceConverter = new ExamplesToFastTreeBins(Host, Args.MaxBins, useTranspose, !Args.FeatureFlocks, Args.MinDocumentsInLeafs, GetMaxLabel());

TrainSet = instanceConverter.FindBinsAndReturnDataset(trainData, PredictionKind, ParallelTraining, CategoricalFeatures, Args.CategoricalSplit);

FeatureMap = instanceConverter.FeatureMap;

if (ValidData != null)

ValidSet = instanceConverter.GetCompatibleDataset(ValidData, PredictionKind, CategoricalFeatures, Args.CategoricalSplit);

if (TestData != null)

TestSets = new[] { instanceConverter.GetCompatibleDataset(TestData, PredictionKind, CategoricalFeatures, Args.CategoricalSplit) };

}

private bool UseTranspose(bool? useTranspose, RoleMappedData data)

{

Host.AssertValue(data);

Host.Assert(data.Schema.Feature.HasValue);

if (useTranspose.HasValue)

return useTranspose.Value;

ITransposeDataView td = data.Data as ITransposeDataView;

return td != null && td.TransposeSchema.GetSlotType(data.Schema.Feature.Value.Index) != null;

}

protected void TrainCore(IChannel ch)

{

Contracts.CheckValue(ch, nameof(ch));

// REVIEW:Get rid of this lock then we completly remove all static classes from FastTree such as BlockingThreadPool.

lock (FastTreeShared.TrainLock)

{

using (Timer.Time(TimerEvent.TotalInitialization))

{

CheckArgs(ch);

PrintPrologInfo(ch);

Initialize(ch);

PrintMemoryStats(ch);

}

using (Timer.Time(TimerEvent.TotalTrain))

Train(ch);

if (Args.ExecutionTimes)

PrintExecutionTimes(ch);

TrainedEnsemble = Ensemble;

if (FeatureMap != null)

TrainedEnsemble.RemapFeatures(FeatureMap);

ParallelTraining.FinalizeEnvironment();

}

}

protected virtual bool ShouldStop(IChannel ch, ref IEarlyStoppingCriterion earlyStopping, ref int bestIteration)

{

bestIteration = Ensemble.NumTrees;

return false;

}

protected virtual int GetBestIteration(IChannel ch) => Ensemble.NumTrees;

protected virtual void InitializeThreads(int numThreads)

{

ThreadTaskManager.Initialize(numThreads);

}

protected virtual void PrintExecutionTimes(IChannel ch)

{

ch.Info("Execution time breakdown:\n{0}", Timer.GetString());

}

protected virtual void CheckArgs(IChannel ch)

{

Args.Check(ch);

IntArray.CompatibilityLevel = Args.FeatureCompressionLevel;

// change arguments

if (Args.HistogramPoolSize < 2)

Args.HistogramPoolSize = Args.NumLeaves * 2 / 3;

if (Args.HistogramPoolSize > Args.NumLeaves - 1)

Args.HistogramPoolSize = Args.NumLeaves - 1;

if (Args.BaggingSize > 0)

{

int bagCount = Args.NumTrees / Args.BaggingSize;

if (bagCount * Args.BaggingSize != Args.NumTrees)

throw ch.Except("Number of trees should be a multiple of number bag size");

}

if (!(0 <= Args.GainConfidenceLevel && Args.GainConfidenceLevel < 1))

throw ch.Except("Gain confidence level must be in the range [0,1)");

#if OLD_DATALOAD

#if !NO_STORE

if (_args.offloadBinsToFileStore)

{

if (!string.IsNullOrEmpty(_args.offloadBinsDirectory) && !Directory.Exists(_args.offloadBinsDirectory))

{

try

{

Directory.CreateDirectory(_args.offloadBinsDirectory);

}

catch (Exception e)

{

throw ch.Except(e, "Failure creating bins offload directory {0} - Exception {1}", _args.offloadBinsDirectory, e.Message);

}

}

}

#endif

#endif

}

/// <summary>

/// A virtual method that is used to print header of test graph.

/// Appliations that need printing test graph are supposed to override

/// it to print specific test graph header.

/// </summary>

/// <returns> string representation of test graph header </returns>

protected virtual string GetTestGraphHeader() => string.Empty;

/// <summary>

/// A virtual method that is used to print a single line of test graph.

/// Applications that need printing test graph are supposed to override

/// it to print a specific line of test graph after a new iteration is finished.

/// </summary>

/// <returns> string representation of a line of test graph </returns>

protected virtual string GetTestGraphLine() => string.Empty;

/// <summary>

/// A virtual method that is used to compute test results after each iteration is finished.

/// </summary>

protected virtual void ComputeTests()

{

}

protected void PrintTestGraph(IChannel ch)

{

// we call Tests computing no matter whether we require to print test graph

ComputeTests();

if (!Args.PrintTestGraph)

return;

if (Ensemble.NumTrees == 0)

ch.Info(GetTestGraphHeader());

else

ch.Info(GetTestGraphLine());

return;

}

protected virtual void Initialize(IChannel ch)

{

#region Load/Initialize State

using (Timer.Time(TimerEvent.InitializeLabels))

PrepareLabels(ch);

using (Timer.Time(TimerEvent.InitializeTraining))

{

InitializeEnsemble();

OptimizationAlgorithm = ConstructOptimizationAlgorithm(ch);

}

using (Timer.Time(TimerEvent.InitializeTests))

InitializeTests();

if (AllowGC)

{

GC.Collect(2, GCCollectionMode.Forced);

GC.Collect(2, GCCollectionMode.Forced);

}

#endregion

}

#if !NO_STORE

/// <summary>

/// Calculates the percentage of feature bins that will fit into memory based on current available memory in the machine.

/// </summary>

/// <returns>A float number between 0 and 1 indicating the percentage of features to load.

/// The number will not be smaller than two times the feature fraction value</returns>

private float GetFeaturePercentInMemory(IChannel ch)

{

const float maxFeaturePercentValue = 1.0f;

float availableMemory = GetMachineAvailableBytes();

ch.Info("Available memory in the machine is = {0} bytes", availableMemory.ToString("N", CultureInfo.InvariantCulture));

float minFeaturePercentThreshold = _args.preloadFeatureBinsBeforeTraining ? (float)_args.featureFraction * 2 : (float)_args.featureFraction;

if (minFeaturePercentThreshold >= maxFeaturePercentValue)

{

return maxFeaturePercentValue;

}

// Initial free memory allowance in bytes for single and parallel fastrank modes

float freeMemoryAllowance = 1024 * 1024 * 512;

if (_optimizationAlgorithm.TreeLearner != null)

{

// Get the size of memory in bytes needed by the tree learner internal data structures

freeMemoryAllowance += _optimizationAlgorithm.TreeLearner.GetSizeOfReservedMemory();

}

availableMemory = (availableMemory > freeMemoryAllowance) ? availableMemory - freeMemoryAllowance : 0;

long featureSize = TrainSet.FeatureSetSize;

if (ValidSet != null)

{

featureSize += ValidSet.FeatureSetSize;

}

if (TestSets != null)

{

foreach (var item in TestSets)

{

featureSize += item.FeatureSetSize;

}

}

ch.Info("Total Feature bins size is = {0} bytes", featureSize.ToString("N", CultureInfo.InvariantCulture));

return Math.Min(Math.Max(minFeaturePercentThreshold, availableMemory / featureSize), maxFeaturePercentValue);

}

#endif

protected bool[] GetActiveFeatures()

{

var activeFeatures = Utils.CreateArray(TrainSet.NumFeatures, true);

if (Args.FeatureFraction < 1.0)

{

if (_featureSelectionRandom == null)

_featureSelectionRandom = new Random(Args.FeatureSelectSeed);

for (int i = 0; i < TrainSet.NumFeatures; ++i)

{

if (activeFeatures[i])

activeFeatures[i] = _featureSelectionRandom.NextDouble() <= Args.FeatureFraction;

}

}

return activeFeatures;

}

private string GetDatasetStatistics(Dataset set)

{

long datasetSize = set.SizeInBytes();

int skeletonSize = set.Skeleton.SizeInBytes();

return string.Format("set contains {0} query-doc pairs in {1} queries with {2} features and uses {3} MB ({4} MB for features)",

set.NumDocs, set.NumQueries, set.NumFeatures, datasetSize / 1024 / 1024, (datasetSize - skeletonSize) / 1024 / 1024);

}

protected virtual void PrintMemoryStats(IChannel ch)

{

Contracts.AssertValue(ch);

ch.Trace("Training {0}", GetDatasetStatistics(TrainSet));

if (ValidSet != null)

ch.Trace("Validation {0}", GetDatasetStatistics(ValidSet));

if (TestSets != null)

{

for (int i = 0; i < TestSets.Length; ++i)

ch.Trace("ComputeTests[{1}] {0}",

GetDatasetStatistics(TestSets[i]), i);

}

if (AllowGC)

ch.Trace("GC Total Memory = {0} MB", GC.GetTotalMemory(true) / 1024 / 1024);

Process currentProcess = Process.GetCurrentProcess();

ch.Trace("Working Set = {0} MB", currentProcess.WorkingSet64 / 1024 / 1024);

ch.Trace("Virtual Memory = {0} MB",

currentProcess.VirtualMemorySize64 / 1024 / 1024);

ch.Trace("Private Memory = {0} MB",

currentProcess.PrivateMemorySize64 / 1024 / 1024);

ch.Trace("Peak Working Set = {0} MB", currentProcess.PeakWorkingSet64 / 1024 / 1024);

ch.Trace("Peak Virtual Memory = {0} MB",

currentProcess.PeakVirtualMemorySize64 / 1024 / 1024);

}

protected bool AreSamplesWeighted(IChannel ch)

{

return TrainSet.SampleWeights != null;

}

private void InitializeEnsemble()

{

Ensemble = new TreeEnsemble();

}

/// <summary>

/// Creates weights wrapping (possibly, trivial) for gradient target values.

/// </summary>

protected virtual IGradientAdjuster MakeGradientWrapper(IChannel ch)

{

if (AreSamplesWeighted(ch))

return new QueryWeightsGradientWrapper();

else

return new TrivialGradientWrapper();

}

#if !NO_STORE

/// <summary>

/// Unloads feature bins being used in the current iteration.

/// </summary>

/// <param name="featureToUnload">Boolean array indicating the features to unload</param>

private void UnloadFeatureBins(bool[] featureToUnload)

{

foreach (ScoreTracker scoreTracker in this._optimizationAlgorithm.TrackedScores)

{

for (int i = 0; i < scoreTracker.Dataset.Features.Length; i++)

{

if (featureToUnload[i])

{

// Only return buffers to the pool that were allocated using the pool

// So far only this type of IntArrays below have buffer pool support.

// This is to avoid unexpected leaks in case a new IntArray is added but we are not allocating it from the pool.

if (scoreTracker.Dataset.Features[i].Bins is DenseIntArray ||

scoreTracker.Dataset.Features[i].Bins is DeltaSparseIntArray ||

scoreTracker.Dataset.Features[i].Bins is DeltaRepeatIntArray)

{

scoreTracker.Dataset.Features[i].Bins.ReturnBuffer();

scoreTracker.Dataset.Features[i].Bins = null;

}

}

}

}

}

/// <summary>

/// Worker thread delegate that loads features for the next training iteration

/// </summary>

/// <param name="state">thread state object</param>

private void LazyFeatureLoad(object state)

{

bool[] featuresToLoad = (bool[])state;

foreach (ScoreTracker scoreTracker in this._optimizationAlgorithm.TrackedScores)

{

for (int i = 0; i < scoreTracker.Dataset.Features.Length; i++)

{

if (featuresToLoad[i])

{

// just using the Bins property so feature bins are loaded into memory

IntArray bins = scoreTracker.Dataset.Features[i].Bins;

}

}

}

}

/// <summary>

/// Iterates through the feature sets needed in future tree training iterations (i.e. in ActiveFeatureSetQueue),

/// using the same order as they were enqueued, and it returns the initial active features based on the percentage parameter.

/// </summary>

/// <param name="pctFeatureThreshold">A float value between 0 and 1 indicating maximum percentage of features to return</param>

/// <returns>Array indicating calculated feature list</returns>

private bool[] GetNextFeaturesByThreshold(float pctFeatureThreshold)

{

int totalUniqueFeatureCount = 0;

bool[] nextActiveFeatures = new bool[TrainSet.NumFeatures];

if (pctFeatureThreshold == 1.0f)

{

// return all features to load

return nextActiveFeatures.Select(x => x = true).ToArray();

}

int maxNumberOfFeatures = (int)(pctFeatureThreshold * TrainSet.NumFeatures);

for (int i = 0; i < _activeFeatureSetQueue.Count; i++)

{

bool[] tempActiveFeatures = _activeFeatureSetQueue.ElementAt(i);

for (int j = 0; j < tempActiveFeatures.Length; j++)

{

if (tempActiveFeatures[j] && !nextActiveFeatures[j])

{

nextActiveFeatures[j] = true;

if (totalUniqueFeatureCount++ > maxNumberOfFeatures)

return nextActiveFeatures;

}

}

}

return nextActiveFeatures;

}

/// <summary>

/// Adds several items in the ActiveFeature queue

/// </summary>

/// <param name="numberOfItems">Number of items to add</param>

private void GenerateActiveFeatureLists(int numberOfItems)

{

for (int i = 0; i < numberOfItems; i++)

{

_activeFeatureSetQueue.Enqueue(GetActiveFeatures());

}

}

#endif

protected virtual BaggingProvider CreateBaggingProvider()

{

Contracts.Assert(Args.BaggingSize > 0);

return new BaggingProvider(TrainSet, Args.NumLeaves, Args.RngSeed, Args.BaggingTrainFraction);

}

protected virtual bool ShouldRandomStartOptimizer()

{

return false;

}

protected virtual void Train(IChannel ch)

{

Contracts.AssertValue(ch);

int numTotalTrees = Args.NumTrees;

ch.Info(

"Reserved memory for tree learner: {0} bytes",

OptimizationAlgorithm.TreeLearner.GetSizeOfReservedMemory());

#if !NO_STORE

if (_args.offloadBinsToFileStore)

{

// Initialize feature percent to load before loading any features

_featurePercentToLoad = GetFeaturePercentInMemory(ch);

ch.Info("Using featurePercentToLoad = {0} ", _featurePercentToLoad);

}

#endif

// random starting point

bool revertRandomStart = false;

if (Ensemble.NumTrees < numTotalTrees && ShouldRandomStartOptimizer())

{

ch.Info("Randomizing start point");

OptimizationAlgorithm.TrainingScores.RandomizeScores(Args.RngSeed, false);

revertRandomStart = true;

}

ch.Info("Starting to train ...");

BaggingProvider baggingProvider = Args.BaggingSize > 0 ? CreateBaggingProvider() : null;

#if OLD_DATALOAD

#if !NO_STORE

// Preload

GenerateActiveFeatureLists(_args.numTrees);

Thread featureLoadThread = null;

// Initial feature load

if (_args.offloadBinsToFileStore)

{

FileObjectStore<IntArrayFormatter>.GetDefaultInstance().SealObjectStore();

if (_args.preloadFeatureBinsBeforeTraining)

{

StartFeatureLoadThread(GetNextFeaturesByThreshold(_featurePercentToLoad)).Join();

}

}

#endif

#endif

IEarlyStoppingCriterion earlyStoppingRule = null;

int bestIteration = 0;

int emptyTrees = 0;

using (var pch = Host.StartProgressChannel("FastTree training"))

{

pch.SetHeader(new ProgressHeader("trees"), e => e.SetProgress(0, Ensemble.NumTrees, numTotalTrees));

while (Ensemble.NumTrees < numTotalTrees)

{

using (Timer.Time(TimerEvent.Iteration))

{

#if NO_STORE

bool[] activeFeatures = GetActiveFeatures();

#else

bool[] activeFeatures = _activeFeatureSetQueue.Dequeue();

#endif

if (Args.BaggingSize > 0 && Ensemble.NumTrees % Args.BaggingSize == 0)

{

baggingProvider.GenerateNewBag();

OptimizationAlgorithm.TreeLearner.Partitioning =

baggingProvider.GetCurrentTrainingPartition();

}

#if !NO_STORE

if (_args.offloadBinsToFileStore)

{

featureLoadThread = StartFeatureLoadThread(GetNextFeaturesByThreshold(_featurePercentToLoad));

if (!_args.preloadFeatureBinsBeforeTraining)

featureLoadThread.Join();

}

#endif

// call the weak learner

var tree = OptimizationAlgorithm.TrainingIteration(ch, activeFeatures);

if (tree == null)

{

emptyTrees++;

numTotalTrees--;

}

else if (Args.BaggingSize > 0 && Ensemble.Trees.Count() > 0)

{

ch.Assert(Ensemble.Trees.Last() == tree);

Ensemble.Trees.Last()

.AddOutputsToScores(OptimizationAlgorithm.TrainingScores.Dataset,

OptimizationAlgorithm.TrainingScores.Scores,

baggingProvider.GetCurrentOutOfBagPartition().Documents);

}

CustomizedTrainingIteration(tree);

using (Timer.Time(TimerEvent.Test))

{

PrintIterationMessage(ch, pch);

PrintTestResults(ch);

}

// revert randomized start

if (revertRandomStart)

{

revertRandomStart = false;

ch.Info("Reverting random score assignment");

OptimizationAlgorithm.TrainingScores.RandomizeScores(Args.RngSeed, true);

}

#if !NO_STORE

if (_args.offloadBinsToFileStore)

{

// Unload only features that are not needed for the next iteration

bool[] featuresToUnload = activeFeatures;

if (_args.preloadFeatureBinsBeforeTraining)

{

featuresToUnload =

activeFeatures.Zip(GetNextFeaturesByThreshold(_featurePercentToLoad),

(current, next) => current && !next).ToArray();

}

UnloadFeatureBins(featuresToUnload);

if (featureLoadThread != null &&

_args.preloadFeatureBinsBeforeTraining)

{

// wait for loading the features needed for the next iteration

featureLoadThread.Join();

}

}

#endif

if (ShouldStop(ch, ref earlyStoppingRule, ref bestIteration))

break;

}

}

if (emptyTrees > 0)

{

ch.Warning("{0} of the boosting iterations failed to grow a tree. This is commonly because the " +

"minimum documents in leaf hyperparameter was set too high for this dataset.", emptyTrees);

}

}

if (earlyStoppingRule != null)

{

Contracts.Assert(numTotalTrees == 0 || bestIteration > 0);

// REVIEW: Need to reconcile with future progress reporting changes.

ch.Info("The training is stopped at {0} and iteration {1} is picked",

Ensemble.NumTrees, bestIteration);

}

else

{

bestIteration = GetBestIteration(ch);

}

OptimizationAlgorithm.FinalizeLearning(bestIteration);

Ensemble.PopulateRawThresholds(TrainSet);

ParallelTraining.FinalizeTreeLearner();

}

#if !NO_STORE

/// <summary>

/// Gets the available bytes performance counter on the local machine

/// </summary>

/// <returns>Available bytes number</returns>

private float GetMachineAvailableBytes()

{

using (var availableBytes = new System.Diagnostics.PerformanceCounter("Memory", "Available Bytes", true))

{

return availableBytes.NextValue();

}

}

#endif

// This method is called at the end of each training iteration, with the tree that was learnt on that iteration.

// Note that this tree can be null if no tree was learnt this iteration.

protected virtual void CustomizedTrainingIteration(RegressionTree tree)

{

}

protected virtual void PrintIterationMessage(IChannel ch, IProgressChannel pch)

{

// REVIEW: report some metrics, not just number of trees?

int iteration = Ensemble.NumTrees;

if (iteration % 50 == 49)

pch.Checkpoint(iteration + 1);

}

protected virtual void PrintTestResults(IChannel ch)

{

if (Args.TestFrequency != int.MaxValue && (Ensemble.NumTrees % Args.TestFrequency == 0 || Ensemble.NumTrees == Args.NumTrees))

{

var sb = new StringBuilder();

using (var sw = new StringWriter(sb))

{

foreach (var t in Tests)

{

var results = t.ComputeTests();

sw.Write(t.FormatInfoString());

}

}

if (sb.Length > 0)

ch.Info(sb.ToString());

}

}

protected virtual void PrintPrologInfo(IChannel ch)

{

Contracts.AssertValue(ch);

ch.Trace("Host = {0}", Environment.MachineName);

ch.Trace("CommandLine = {0}", CmdParser.GetSettings(Host, Args, new TArgs()));

ch.Trace("GCSettings.IsServerGC = {0}", System.Runtime.GCSettings.IsServerGC);

ch.Trace("{0}", Args);

}

protected ScoreTracker ConstructScoreTracker(Dataset set)

{

// If not found contruct one

ScoreTracker st = null;

if (set == TrainSet)

st = OptimizationAlgorithm.GetScoreTracker("train", TrainSet, InitTrainScores);

else if (set == ValidSet)

st = OptimizationAlgorithm.GetScoreTracker("valid", ValidSet, InitValidScores);

else

{

for (int t = 0; t < TestSets.Length; ++t)

{

if (set == TestSets[t])

{

double[] initTestScores = InitTestScores?[t];

st = OptimizationAlgorithm.GetScoreTracker(string.Format("test[{0}]", t), TestSets[t], initTestScores);

}

}

}

Contracts.Check(st != null, "unknown dataset passed to ConstructScoreTracker");

return st;

}

private double[] ComputeScoresSmart(IChannel ch, Dataset set)

{

if (!Args.CompressEnsemble)

{