Skip to content

Incremental Training Phase 3 Implementation #450

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
yunboouyang wants to merge 7 commits into
base: master
Choose a base branch
from
Open

Incremental Training Phase 3 Implementation #450

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
b97f360
Add incremental training and related unit tests
yunboouyang Oct 24, 2019
d5d9ac9
Add calculation details in unit tests and fix integ test failure
yunboouyang Oct 29, 2019
eecae47
Revise incremental training code to test via GAME training template
yunboouyang Nov 13, 2019
1d00902
Enable Autotune for Incremental Training and Fix Subspace projection …
yunboouyang Jan 10, 2020
d67bb74
Remove L1 regularization
yunboouyang Jan 23, 2020
73b475f
Add full Hessian matrix calculation
yunboouyang Jan 31, 2020
0fc80a4
Fix a RandomEffectCoordinate bug
yunboouyang Feb 12, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 8 additions & 1 deletion ...api/src/integTest/scala/com/linkedin/photon/ml/algorithm/CoordinateFactoryIntegTest.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import com.linkedin.photon.ml.TaskType
import com.linkedin.photon.ml.Types.REId
import com.linkedin.photon.ml.data.{FixedEffectDataset, LocalDataset, RandomEffectDataset}
import com.linkedin.photon.ml.function.{DistributedObjectiveFunction, ObjectiveFunctionHelper, SingleNodeObjectiveFunction}
import com.linkedin.photon.ml.model.{FixedEffectModel, RandomEffectModel}
import com.linkedin.photon.ml.normalization.NormalizationContext
import com.linkedin.photon.ml.optimization.game.{FixedEffectOptimizationConfiguration, RandomEffectOptimizationConfiguration}
import com.linkedin.photon.ml.optimization.{OptimizerConfig, OptimizerType, SingleNodeOptimizationProblem, VarianceComputationType}
Expand All @@ -46,6 +47,7 @@ class CoordinateFactoryIntegTest extends SparkTestUtils {

val mockDataset = mock(classOf[FixedEffectDataset])
val optimizationConfiguration = FixedEffectOptimizationConfiguration(OPTIMIZER_CONFIG)
val priorModelOpt: Option[FixedEffectModel] = None

doReturn(sc).when(mockDataset).sparkContext

Expand All @@ -57,6 +59,7 @@ class CoordinateFactoryIntegTest extends SparkTestUtils {
DOWN_SAMPLER_FACTORY,
MOCK_NORMALIZATION,
VARIANCE_COMPUTATION_TYPE,
priorModelOpt,
INTERCEPT_INDEX)

coordinate match {
Expand All @@ -78,8 +81,10 @@ class CoordinateFactoryIntegTest extends SparkTestUtils {
val mockProjectorsRDD = mock(classOf[RDD[(REId, LinearSubspaceProjector)]])
val mockProblemsRDD = mock(classOf[RDD[(REId, SingleNodeOptimizationProblem[SingleNodeObjectiveFunction])]])
val optimizationConfiguration = RandomEffectOptimizationConfiguration(OPTIMIZER_CONFIG)
val priorModelOpt: Option[RandomEffectModel] = None

doReturn(sc).when(mockDataset).sparkContext
doReturn(sc).when(mockProjectorsRDD).sparkContext
doReturn(mockDataRDD).when(mockDataset).activeData
doReturn(mockDataRDD)
.when(mockDataRDD)
Expand All @@ -97,6 +102,7 @@ class CoordinateFactoryIntegTest extends SparkTestUtils {
DOWN_SAMPLER_FACTORY,
MOCK_NORMALIZATION,
VARIANCE_COMPUTATION_TYPE,
priorModelOpt,
INTERCEPT_INDEX)

coordinate match {
Expand Down Expand Up @@ -124,6 +130,7 @@ class CoordinateFactoryIntegTest extends SparkTestUtils {
DOWN_SAMPLER_FACTORY,
MOCK_NORMALIZATION,
VARIANCE_COMPUTATION_TYPE,
None,
INTERCEPT_INDEX)
}
}
Expand All @@ -139,7 +146,7 @@ object CoordinateFactoryIntegTest {
private val INTERCEPT_INDEX = None

private val OPTIMIZER_CONFIG = OptimizerConfig(OPTIMIZER_TYPE, MAX_ITER, TOLERANCE)
private val MOCK_NORMALIZATION = mock(classOf[NormalizationContext])
private val MOCK_NORMALIZATION = mock(classOf[NormalizationContext], withSettings().serializable())
private val GLM_CONSTRUCTOR = LogisticRegressionModel.apply _
private val LOSS_FUNCTION_FACTORY = ObjectiveFunctionHelper.buildFactory(TRAINING_TASK, TREE_AGGREGATE_DEPTH)
private val DOWN_SAMPLER_FACTORY = DownSamplerHelper.buildFactory(TRAINING_TASK)
Expand Down
4 changes: 3 additions & 1 deletion ...on-api/src/integTest/scala/com/linkedin/photon/ml/data/RandomEffectDatasetIntegTest.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -184,7 +184,7 @@ class RandomEffectDatasetIntegTest extends SparkTestUtils {
val partitioner = new RandomEffectDatasetPartitioner(NUM_PARTITIONS, sc.broadcast(partitionMap))

val projectorsMap = RandomEffectDataset
.generateLinearSubspaceProjectors(keyedGameDatasetRDD, partitioner)
.generateLinearSubspaceProjectors(keyedGameDatasetRDD, partitioner, None)
.collect
.toMap

Expand Down Expand Up @@ -381,6 +381,7 @@ class RandomEffectDatasetIntegTest extends SparkTestUtils {
NUM_PARTITIONS)
val randomEffectDataset = RandomEffectDataset(
dataRDD,
None,
randomEffectDataConfig,
rePartitioner,
None,
Expand Down Expand Up @@ -440,6 +441,7 @@ class RandomEffectDatasetIntegTest extends SparkTestUtils {
Some(activeDataLowerBound))
val randomEffectDataset = RandomEffectDataset(
dataRDD,
None,
randomEffectDataConfig,
rePartitioner,
None,
Expand Down
Loading