hex.glm

Class GLMUtils

java.lang.Object

hex.glm.GLMUtils

public class GLMUtils
extends java.lang.Object

Constructor Summary

Constructors

Constructor and Description
GLMUtils()

Method Summary

Modifier and Type	Method and Description
static void	addOneRow2ScoringHistory(water.util.TwoDimTable glmSc1, water.util.TwoDimTable earlyStopSc2, int glmColSize, int earlyStopColSize, int glmRowIndex, int earlyStopRowIndex, int rowIndex, boolean addGlmSC, boolean addEarlyStopSC, java.util.List<java.lang.Integer> earlyStopColIndices, water.util.TwoDimTable combined, int overlapSize)
static void	addOneRow2ScoringHistoryRestricted(water.util.TwoDimTable glmSc, water.util.TwoDimTable glmScRestricted, water.util.TwoDimTable earlyStopSc, water.util.TwoDimTable earlyStopScRestricted, int glmColSize, int earlyStopColSize, int glmRowIndex, int earlyStopRowIndex, int rowIndex, boolean addGlmSC, boolean addEarlyStopSC, java.util.List<java.lang.Integer> glmScRestrictedIndices, java.util.List<java.lang.Integer> earlyStopColIndices, java.util.List<java.lang.Integer> earlyStopColIndicesRestricted, water.util.TwoDimTable combined, int overlapSize)
static water.fvec.Frame	buildRIDFrame(GLMModel.GLMParameters parms, water.fvec.Frame train, water.fvec.Frame RIDFrame)
static double	calSmoothNess(double[][] beta, double[][][] penaltyMatrix, int[][] gamColIndices)
static double	calSmoothNess(double[] beta, double[][][] penaltyMatrix, int[][] gamColIndices)
static water.util.TwoDimTable	combineScoringHistory(water.util.TwoDimTable glmSc1, water.util.TwoDimTable earlyStopSc2)
static water.util.TwoDimTable	combineScoringHistoryRestricted(water.util.TwoDimTable glmSc, water.util.TwoDimTable glmScRestricted, water.util.TwoDimTable earlyStopSc, water.util.TwoDimTable earlyStopScRestricted, hex.ScoreKeeper.StoppingMetric stoppingMetric, boolean hasValidationMetrics) Combine scoring history tables into one main scoring history when Control variables or remove offset effects are enabled Scoring History: timestamp duration iterations Unrestricted negative_log_likelihood Unrestricted objective = training metrics calculated during optimization with control variables or offset included (in glmSc) Training RMSE Training LogLoss Training r2 Training AUC Training pr_auc Training Lift Training Classification Error = early stopping training metrics with control variables or offset excluded (in earlyStopSc) Validation RMSE Validation LogLoss Validation r2 Validation AUC Validation pr_auc Validation Lift Validation Classification Error = early stopping validation metrics with control variables or offset excluded (in earlyStopSc) Unrestricted Training AUC Unrestricted Validation AUC = stopping metrics with control variables or offset included (in earlyStopScRestricted)
static water.util.TwoDimTable	combineTableContents(water.util.TwoDimTable glmSc1, water.util.TwoDimTable earlyStopSc2, water.util.TwoDimTable combined, java.util.List<java.lang.Integer> earlyStopColIndices, int indexOfIter, int indexOfIterEarlyStop, int overlapSize)
static water.util.TwoDimTable	combineTableContentsRestricted(water.util.TwoDimTable glmSc, water.util.TwoDimTable glmScRestricted, water.util.TwoDimTable earlyStopSc, water.util.TwoDimTable earlyStopScRestricted, water.util.TwoDimTable combined, java.util.List<java.lang.Integer> glmScRestrictedColIndices, java.util.List<java.lang.Integer> earlyStopColIndices, java.util.List<java.lang.Integer> earlyStopColIndicesRestricted, int indexOfIter, int indexOfIterEarlyStop, int overlapSize)
static GLM.GLMGradientInfo	copyGInfo(GLM.GLMGradientInfo ginfo)
static water.fvec.Frame	expandedCatCS(water.fvec.Frame beta_constraints, GLMModel.GLMParameters parms)
static int[][]	extractAdaptedFrameIndices(water.fvec.Frame adaptFrame, java.lang.String[][] gamColnames, int numOffset) From the gamColnames, this method attempts to translate to the column indices in adaptFrame.
static boolean	findEnumInBetaCS(water.fvec.Frame betaCS, GLMModel.GLMParameters parms)
static java.lang.String[]	genDfbetasNames(GLMModel model)
static java.lang.String[]	generateRowHeaders(water.util.TwoDimTable glmSc1, water.util.TwoDimTable earlyStopSc2, int glmIterIndex, int earlyStopIterIndex)
static GLMModel.GLMParameters[]	genGLMParameters(GLMModel.GLMParameters parms, java.lang.String[] validPreds, java.lang.String[] predictorNames)
static double[]	genNewBeta(int newBetaLength, double[] beta, double[] stdErr)
static java.util.List<java.lang.Integer>	getStoppingMetricIndices(hex.ScoreKeeper.StoppingMetric stoppingMetric, java.lang.String[] earlyStopTableHeader)
static boolean	notZeroLambdas(double[] lambdas)
static void	removePredictors(GLMModel.GLMParameters parms, water.fvec.Frame train)
static void	removeRedCols(double[] row2Array, double[] reducedArray, double[] stdErr)
static void	updateGradGam(double[] gradient, double[][][] penalty_mat, int[][] gamBetaIndices, double[] beta, int[] activeCols)
static void	updateGradGamMultinomial(double[][] gradient, double[][][] penaltyMat, int[][] gamBetaIndices, double[][] beta)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Constructor Detail

GLMUtils

public GLMUtils()

Method Detail

extractAdaptedFrameIndices

public static int[][] extractAdaptedFrameIndices(water.fvec.Frame adaptFrame,
                                                 java.lang.String[][] gamColnames,
                                                 int numOffset)

From the gamColnames, this method attempts to translate to the column indices in adaptFrame.

Parameters:

adaptFrame -

gamColnames -

Returns:

genGLMParameters

public static GLMModel.GLMParameters[] genGLMParameters(GLMModel.GLMParameters parms,
                                                        java.lang.String[] validPreds,
                                                        java.lang.String[] predictorNames)

removePredictors

public static void removePredictors(GLMModel.GLMParameters parms,
                                    water.fvec.Frame train)

expandedCatCS

public static water.fvec.Frame expandedCatCS(water.fvec.Frame beta_constraints,
                                             GLMModel.GLMParameters parms)

findEnumInBetaCS

public static boolean findEnumInBetaCS(water.fvec.Frame betaCS,
                                       GLMModel.GLMParameters parms)

copyGInfo

public static GLM.GLMGradientInfo copyGInfo(GLM.GLMGradientInfo ginfo)

combineScoringHistory

public static water.util.TwoDimTable combineScoringHistory(water.util.TwoDimTable glmSc1,
                                                           water.util.TwoDimTable earlyStopSc2)

getStoppingMetricIndices

public static java.util.List<java.lang.Integer> getStoppingMetricIndices(hex.ScoreKeeper.StoppingMetric stoppingMetric,
                                                                         java.lang.String[] earlyStopTableHeader)

combineScoringHistoryRestricted

public static water.util.TwoDimTable combineScoringHistoryRestricted(water.util.TwoDimTable glmSc,
                                                                     water.util.TwoDimTable glmScRestricted,
                                                                     water.util.TwoDimTable earlyStopSc,
                                                                     water.util.TwoDimTable earlyStopScRestricted,
                                                                     hex.ScoreKeeper.StoppingMetric stoppingMetric,
                                                                     boolean hasValidationMetrics)

Combine scoring history tables into one main scoring history when Control variables or remove offset effects are enabled Scoring History: timestamp duration iterations Unrestricted negative_log_likelihood Unrestricted objective = training metrics calculated during optimization with control variables or offset included (in glmSc) Training RMSE Training LogLoss Training r2 Training AUC Training pr_auc Training Lift Training Classification Error = early stopping training metrics with control variables or offset excluded (in earlyStopSc) Validation RMSE Validation LogLoss Validation r2 Validation AUC Validation pr_auc Validation Lift Validation Classification Error = early stopping validation metrics with control variables or offset excluded (in earlyStopSc) Unrestricted Training AUC Unrestricted Validation AUC = stopping metrics with control variables or offset included (in earlyStopScRestricted)

Parameters:

glmSc -

earlyStopSc -

stoppingMetric -

earlyStopScRestricted -

Returns:

Combined scoring history table

generateRowHeaders

public static java.lang.String[] generateRowHeaders(water.util.TwoDimTable glmSc1,
                                                    water.util.TwoDimTable earlyStopSc2,
                                                    int glmIterIndex,
                                                    int earlyStopIterIndex)

combineTableContents

public static water.util.TwoDimTable combineTableContents(water.util.TwoDimTable glmSc1,
                                                          water.util.TwoDimTable earlyStopSc2,
                                                          water.util.TwoDimTable combined,
                                                          java.util.List<java.lang.Integer> earlyStopColIndices,
                                                          int indexOfIter,
                                                          int indexOfIterEarlyStop,
                                                          int overlapSize)

addOneRow2ScoringHistory

public static void addOneRow2ScoringHistory(water.util.TwoDimTable glmSc1,
                                            water.util.TwoDimTable earlyStopSc2,
                                            int glmColSize,
                                            int earlyStopColSize,
                                            int glmRowIndex,
                                            int earlyStopRowIndex,
                                            int rowIndex,
                                            boolean addGlmSC,
                                            boolean addEarlyStopSC,
                                            java.util.List<java.lang.Integer> earlyStopColIndices,
                                            water.util.TwoDimTable combined,
                                            int overlapSize)

combineTableContentsRestricted

public static water.util.TwoDimTable combineTableContentsRestricted(water.util.TwoDimTable glmSc,
                                                                    water.util.TwoDimTable glmScRestricted,
                                                                    water.util.TwoDimTable earlyStopSc,
                                                                    water.util.TwoDimTable earlyStopScRestricted,
                                                                    water.util.TwoDimTable combined,
                                                                    java.util.List<java.lang.Integer> glmScRestrictedColIndices,
                                                                    java.util.List<java.lang.Integer> earlyStopColIndices,
                                                                    java.util.List<java.lang.Integer> earlyStopColIndicesRestricted,
                                                                    int indexOfIter,
                                                                    int indexOfIterEarlyStop,
                                                                    int overlapSize)

addOneRow2ScoringHistoryRestricted

public static void addOneRow2ScoringHistoryRestricted(water.util.TwoDimTable glmSc,
                                                      water.util.TwoDimTable glmScRestricted,
                                                      water.util.TwoDimTable earlyStopSc,
                                                      water.util.TwoDimTable earlyStopScRestricted,
                                                      int glmColSize,
                                                      int earlyStopColSize,
                                                      int glmRowIndex,
                                                      int earlyStopRowIndex,
                                                      int rowIndex,
                                                      boolean addGlmSC,
                                                      boolean addEarlyStopSC,
                                                      java.util.List<java.lang.Integer> glmScRestrictedIndices,
                                                      java.util.List<java.lang.Integer> earlyStopColIndices,
                                                      java.util.List<java.lang.Integer> earlyStopColIndicesRestricted,
                                                      water.util.TwoDimTable combined,
                                                      int overlapSize)

updateGradGam

public static void updateGradGam(double[] gradient,
                                 double[][][] penalty_mat,
                                 int[][] gamBetaIndices,
                                 double[] beta,
                                 int[] activeCols)

updateGradGamMultinomial

public static void updateGradGamMultinomial(double[][] gradient,
                                            double[][][] penaltyMat,
                                            int[][] gamBetaIndices,
                                            double[][] beta)

calSmoothNess

public static double calSmoothNess(double[] beta,
                                   double[][][] penaltyMatrix,
                                   int[][] gamColIndices)

calSmoothNess

public static double calSmoothNess(double[][] beta,
                                   double[][][] penaltyMatrix,
                                   int[][] gamColIndices)

Parameters:

beta - multinomial number of class by number of predictors

penaltyMatrix -

gamColIndices -

Returns:

genDfbetasNames

public static java.lang.String[] genDfbetasNames(GLMModel model)

genNewBeta

public static double[] genNewBeta(int newBetaLength,
                                  double[] beta,
                                  double[] stdErr)

removeRedCols

public static void removeRedCols(double[] row2Array,
                                 double[] reducedArray,
                                 double[] stdErr)

buildRIDFrame

public static water.fvec.Frame buildRIDFrame(GLMModel.GLMParameters parms,
                                             water.fvec.Frame train,
                                             water.fvec.Frame RIDFrame)

notZeroLambdas

public static boolean notZeroLambdas(double[] lambdas)