From eaff0130aee3a990e138b21edde7d85e5286b4d1 Mon Sep 17 00:00:00 2001 From: Magnus Jensen Date: Wed, 25 Mar 2026 15:22:13 -0600 Subject: [PATCH 1/9] use redis to block double profile work for apple devices setting up --- cmd/fleet/cron.go | 3 +- cmd/fleet/serve.go | 1 + server/fleet/mdm.go | 5 + server/fleet/service.go | 10 + .../advanced_key_value_store.go | 65 +++++++ server/mock/service.go | 7 + server/service/apple_mdm.go | 57 +++++- server/service/apple_mdm_test.go | 183 +++++++++++++++++- .../service/integration_mdm_profiles_test.go | 6 +- server/service/integration_mdm_test.go | 15 +- .../redis_key_value/redis_key_value.go | 39 +++- 11 files changed, 366 insertions(+), 25 deletions(-) create mode 100644 server/mock/redis_advanced/advanced_key_value_store.go diff --git a/cmd/fleet/cron.go b/cmd/fleet/cron.go index ad07c5d92a6..fe0c68541b8 100644 --- a/cmd/fleet/cron.go +++ b/cmd/fleet/cron.go @@ -1462,6 +1462,7 @@ func newAppleMDMProfileManagerSchedule( instanceID string, ds fleet.Datastore, commander *apple_mdm.MDMAppleCommander, + redisKeyValue fleet.AdvancedKeyValueStore, logger *slog.Logger, certProfilesLimit int, ) (*schedule.Schedule, error) { @@ -1478,7 +1479,7 @@ func newAppleMDMProfileManagerSchedule( ctx, name, instanceID, defaultInterval, ds, ds, schedule.WithLogger(logger), schedule.WithJob("manage_apple_profiles", func(ctx context.Context) error { - return service.ReconcileAppleProfiles(ctx, ds, commander, logger, certProfilesLimit) + return service.ReconcileAppleProfiles(ctx, ds, commander, redisKeyValue, logger, certProfilesLimit) }), schedule.WithJob("manage_apple_declarations", func(ctx context.Context) error { return service.ReconcileAppleDeclarations(ctx, ds, commander, logger) diff --git a/cmd/fleet/serve.go b/cmd/fleet/serve.go index d37b32e4164..7ec36efc649 100644 --- a/cmd/fleet/serve.go +++ b/cmd/fleet/serve.go @@ -1258,6 +1258,7 @@ func runServeCmd(cmd *cobra.Command, configManager configpkg.Manager, debug, dev instanceID, ds, apple_mdm.NewMDMAppleCommander(mdmStorage, mdmPushService), + redis_key_value.New(redisPool), logger, config.MDM.CertificateProfilesLimit, ) diff --git a/server/fleet/mdm.go b/server/fleet/mdm.go index bc16f0c6bcb..ad675b8d590 100644 --- a/server/fleet/mdm.go +++ b/server/fleet/mdm.go @@ -31,6 +31,11 @@ const ( StickyMDMEnrollmentKeyPrefix = "sticky_mdm_enrollment_" // + host UUID StickyMDMEnrollmentTTL = 30 * time.Minute + + // MDMProfileProcessingKeyPrefix is used to indicate that a host is currently being processed for MDM profile installation. + // We wrap the key in braces to make Redis hash the keys to the same slot, avoding CrossSlot errors. + MDMProfileProcessingKeyPrefix = "{mdm_profile_processing}" // + :hostUUID + MDMProfileProcessingTTL = 1 * time.Minute // We use a low time here, to avoid letting it sit for too long in case of errors. ) // FleetVarName represents the name of a Fleet variable (without the FLEET_VAR_ prefix). diff --git a/server/fleet/service.go b/server/fleet/service.go index ae8bb1ea4af..b101c498d9d 100644 --- a/server/fleet/service.go +++ b/server/fleet/service.go @@ -1471,6 +1471,16 @@ type KeyValueStore interface { Get(ctx context.Context, key string) (*string, error) } +type AdvancedKeyValueStore interface { + KeyValueStore + + // MGet returns the values for the given keys. + // It returns a map of key to value, where the value is nil if the key doesn't exist. + // Important to use hashes for the keys to land in the same slot. + MGet(ctx context.Context, keys []string) (map[string]*string, error) + Delete(ctx context.Context, key string) error +} + const ( // BatchSetSoftwareInstallerStatusProcessing is the value returned for an ongoing BatchSetSoftwareInstallers operation. BatchSetSoftwareInstallersStatusProcessing = "processing" diff --git a/server/mock/redis_advanced/advanced_key_value_store.go b/server/mock/redis_advanced/advanced_key_value_store.go new file mode 100644 index 00000000000..dc12397ee48 --- /dev/null +++ b/server/mock/redis_advanced/advanced_key_value_store.go @@ -0,0 +1,65 @@ +// Automatically generated by mockimpl. DO NOT EDIT! + +package mock + +import ( + "context" + "sync" + "time" + + "github.com/fleetdm/fleet/v4/server/fleet" +) + +var _ fleet.AdvancedKeyValueStore = (*AdvancedKeyValueStore)(nil) + +type SetFunc func(ctx context.Context, key string, value string, expireTime time.Duration) error + +type GetFunc func(ctx context.Context, key string) (*string, error) + +type MGetFunc func(ctx context.Context, keys []string) (map[string]*string, error) + +type DeleteFunc func(ctx context.Context, key string) error + +type AdvancedKeyValueStore struct { + SetFunc SetFunc + SetFuncInvoked bool + + GetFunc GetFunc + GetFuncInvoked bool + + MGetFunc MGetFunc + MGetFuncInvoked bool + + DeleteFunc DeleteFunc + DeleteFuncInvoked bool + + mu sync.Mutex +} + +func (akv *AdvancedKeyValueStore) Set(ctx context.Context, key string, value string, expireTime time.Duration) error { + akv.mu.Lock() + akv.SetFuncInvoked = true + akv.mu.Unlock() + return akv.SetFunc(ctx, key, value, expireTime) +} + +func (akv *AdvancedKeyValueStore) Get(ctx context.Context, key string) (*string, error) { + akv.mu.Lock() + akv.GetFuncInvoked = true + akv.mu.Unlock() + return akv.GetFunc(ctx, key) +} + +func (akv *AdvancedKeyValueStore) MGet(ctx context.Context, keys []string) (map[string]*string, error) { + akv.mu.Lock() + akv.MGetFuncInvoked = true + akv.mu.Unlock() + return akv.MGetFunc(ctx, keys) +} + +func (akv *AdvancedKeyValueStore) Delete(ctx context.Context, key string) error { + akv.mu.Lock() + akv.DeleteFuncInvoked = true + akv.mu.Unlock() + return akv.DeleteFunc(ctx, key) +} diff --git a/server/mock/service.go b/server/mock/service.go index 6ac07e5cda4..25445ff5832 100644 --- a/server/mock/service.go +++ b/server/mock/service.go @@ -3,14 +3,21 @@ package mock import ( "github.com/fleetdm/fleet/v4/server/fleet" kvmock "github.com/fleetdm/fleet/v4/server/mock/redis" + akvmock "github.com/fleetdm/fleet/v4/server/mock/redis_advanced" svcmock "github.com/fleetdm/fleet/v4/server/mock/service" ) //go:generate go run ./mockimpl/impl.go -o service/service_mock.go "s *Service" "fleet.Service" //go:generate go run ./mockimpl/impl.go -o redis/key_value_store.go "kv *KeyValueStore" "fleet.KeyValueStore" +// We need to use a new folder to avoid multiple of the same functions +//go:generate go run ./mockimpl/impl.go -o redis_advanced/advanced_key_value_store.go "akv *AdvancedKeyValueStore" "fleet.AdvancedKeyValueStore" var _ fleet.Service = new(svcmock.Service) type KVStore struct { kvmock.KeyValueStore } + +type AdvancedKVStore struct { + akvmock.AdvancedKeyValueStore +} diff --git a/server/service/apple_mdm.go b/server/service/apple_mdm.go index d0a976166d8..0d893900465 100644 --- a/server/service/apple_mdm.go +++ b/server/service/apple_mdm.go @@ -3320,7 +3320,7 @@ type MDMAppleCheckinAndCommandService struct { vppInstaller fleet.AppleMDMVPPInstaller mdmLifecycle *mdmlifecycle.HostLifecycle commandHandlers map[string][]fleet.MDMCommandResultsHandler - keyValueStore fleet.KeyValueStore + keyValueStore fleet.AdvancedKeyValueStore newActivityFn mdmlifecycle.NewActivityFunc isPremium bool } @@ -3331,7 +3331,7 @@ func NewMDMAppleCheckinAndCommandService( vppInstaller fleet.AppleMDMVPPInstaller, isPremium bool, logger *slog.Logger, - keyValueStore fleet.KeyValueStore, + keyValueStore fleet.AdvancedKeyValueStore, newActivityFn mdmlifecycle.NewActivityFunc, ) *MDMAppleCheckinAndCommandService { mdmLifecycle := mdmlifecycle.New(ds, logger, newActivityFn) @@ -3406,8 +3406,8 @@ func (svc *MDMAppleCheckinAndCommandService) Authenticate(r *mdm.Request, m *mdm return err } - if !scepRenewalInProgress { - if svc.keyValueStore != nil { + if svc.keyValueStore != nil { + if !scepRenewalInProgress { // Set sticky key for MDM enrollments to avoid updating team id on orbit enrollments err = svc.keyValueStore.Set(r.Context, fleet.StickyMDMEnrollmentKeyPrefix+r.ID, "1", fleet.StickyMDMEnrollmentTTL) if err != nil { @@ -3415,6 +3415,12 @@ func (svc *MDMAppleCheckinAndCommandService) Authenticate(r *mdm.Request, m *mdm svc.logger.ErrorContext(r.Context, "failed to set sticky mdm enrollment key", "err", err, "host_uuid", r.ID) } } + + // Set profile processing flag, is being handled by the apple_mdm worker, it will be cleared later if it's a SCEP renewal. + if err := svc.keyValueStore.Set(r.Context, fleet.MDMProfileProcessingKeyPrefix+":"+r.ID, "1", fleet.MDMProfileProcessingTTL); err != nil { + svc.logger.ErrorContext(r.Context, "failed to set mdm profile processing key", "err", err, "host_uuid", r.ID) + // We do not want to fail here, just log the error to notify of issues + } } return nil @@ -3444,6 +3450,13 @@ func (svc *MDMAppleCheckinAndCommandService) TokenUpdate(r *mdm.Request, m *mdm. if !m.AwaitingConfiguration { // Normal SCEP renewal - device is NOT at Setup Assistant. Clean refs and short-circuit. svc.logger.InfoContext(r.Context, "cleaned SCEP refs, skipping setup experience and mdm lifecycle turn on action", "host_uuid", r.ID) + + // Clean up redis key for profile processing if set. + if svc.keyValueStore != nil { + if err := svc.keyValueStore.Delete(r.Context, fleet.MDMProfileProcessingKeyPrefix+":"+r.ID); err != nil { + svc.logger.ErrorContext(r.Context, "failed to delete mdm profile processing key", "err", err, "host_uuid", r.ID) + } + } return nil } @@ -4895,6 +4908,7 @@ func ReconcileAppleProfiles( ctx context.Context, ds fleet.Datastore, commander *apple_mdm.MDMAppleCommander, + redisKeyValue fleet.AdvancedKeyValueStore, logger *slog.Logger, certProfilesLimit int, ) error { @@ -5239,6 +5253,41 @@ func ReconcileAppleProfiles( }) } + // check if some of the hosts to install already is handled by the apple setup worker + // we want to batch check for 1k hosts at a time to avoid hitting query parameter limits + const isBeingSetupBatchSize = 1000 + for i := 0; i < len(hostProfiles); i += isBeingSetupBatchSize { + end := min(i+isBeingSetupBatchSize, len(hostProfiles)) + batch := hostProfiles[i:end] + hostUUIDs := make([]string, len(batch)) + hostUUIDToHostProfile := make(map[string]*fleet.MDMAppleBulkUpsertHostProfilePayload, len(batch)) + for j, hp := range batch { + hostUUIDs[j] = fleet.MDMProfileProcessingKeyPrefix + ":" + hp.HostUUID + hostUUIDToHostProfile[hp.HostUUID] = hp + } + + setupHostUUIDs, err := redisKeyValue.MGet(ctx, hostUUIDs) + if err != nil { + return ctxerr.Wrap(ctx, err, "filtering hosts being set up") + } + for keyedHostUUID, exists := range setupHostUUIDs { + if exists != nil { + hostUUID := strings.TrimPrefix(keyedHostUUID, fleet.MDMProfileProcessingKeyPrefix+":") + logger.DebugContext(ctx, "skipping profile reconciliation for host being set up", "host_uuid", hostUUID) + hp, ok := hostUUIDToHostProfile[hostUUID] + if !ok { + logger.DebugContext(ctx, "expected host uuid to be present but was not, do not skip profile reconciliation", "host_uuid", hostUUID) + continue + } + // Clear out host profile status and installTargets to avoid iterating over them in ProcessAndEnqueueProfiles + hp.Status = nil + hp.CommandUUID = "" + hostProfilesToInstallMap[fleet.HostProfileUUID{HostUUID: hp.HostUUID, ProfileUUID: hp.ProfileUUID}] = hp + delete(installTargets, hp.ProfileUUID) + } + } + } + // delete all profiles that have a matching identifier to be installed. // This is to prevent sending both a `RemoveProfile` and an // `InstallProfile` for the same identifier, which can cause race diff --git a/server/service/apple_mdm_test.go b/server/service/apple_mdm_test.go index 659a7f7ab34..78aeab408df 100644 --- a/server/service/apple_mdm_test.go +++ b/server/service/apple_mdm_test.go @@ -2808,6 +2808,7 @@ func TestMDMAppleReconcileAppleProfiles(t *testing.T) { ctx := context.Background() mdmStorage := &mdmmock.MDMAppleStore{} ds := new(mock.Store) + kv := new(mock.AdvancedKVStore) pushFactory, _ := newMockAPNSPushProviderFactory() pusher := nanomdm_pushsvc.New( mdmStorage, @@ -3161,7 +3162,7 @@ func TestMDMAppleReconcileAppleProfiles(t *testing.T) { failedCount++ require.Len(t, payload, 0) } - err := ReconcileAppleProfiles(ctx, ds, cmdr, slog.New(slog.DiscardHandler), 0) + err := ReconcileAppleProfiles(ctx, ds, cmdr, kv, slog.New(slog.DiscardHandler), 0) require.NoError(t, err) require.Equal(t, 1, failedCount) checkAndReset(t, true, &ds.ListMDMAppleProfilesToInstallAndRemoveFuncInvoked) @@ -3208,7 +3209,7 @@ func TestMDMAppleReconcileAppleProfiles(t *testing.T) { } enqueueFailForOp = fleet.MDMOperationTypeRemove - err := ReconcileAppleProfiles(ctx, ds, cmdr, slog.New(slog.DiscardHandler), 0) + err := ReconcileAppleProfiles(ctx, ds, cmdr, kv, slog.New(slog.DiscardHandler), 0) require.NoError(t, err) require.Equal(t, 1, failedCount) checkAndReset(t, true, &ds.ListMDMAppleProfilesToInstallAndRemoveFuncInvoked) @@ -3281,7 +3282,7 @@ func TestMDMAppleReconcileAppleProfiles(t *testing.T) { } enqueueFailForOp = fleet.MDMOperationTypeInstall - err := ReconcileAppleProfiles(ctx, ds, cmdr, slog.New(slog.DiscardHandler), 0) + err := ReconcileAppleProfiles(ctx, ds, cmdr, kv, slog.New(slog.DiscardHandler), 0) require.NoError(t, err) require.Equal(t, 1, failedCount) checkAndReset(t, true, &ds.ListMDMAppleProfilesToInstallAndRemoveFuncInvoked) @@ -3455,7 +3456,7 @@ func TestMDMAppleReconcileAppleProfiles(t *testing.T) { contents1 = originalContents1 expectedContents1 = originalExpectedContents1 }) - err := ReconcileAppleProfiles(ctx, ds, cmdr, slog.New(slog.DiscardHandler), 0) + err := ReconcileAppleProfiles(ctx, ds, cmdr, kv, slog.New(slog.DiscardHandler), 0) require.NoError(t, err) assert.Equal(t, 2, upsertCount) // checkAndReset(t, true, &ds.GetAllCertificateAuthoritiesFuncInvoked) @@ -3483,7 +3484,7 @@ func TestMDMAppleReconcileAppleProfiles(t *testing.T) { ds.GetHostEmailsFunc = func(ctx context.Context, hostUUID string, source string) ([]string, error) { return nil, errors.New("GetHostEmailsFuncError") } - err := ReconcileAppleProfiles(ctx, ds, cmdr, slog.New(slog.Default().Handler()), 0) + err := ReconcileAppleProfiles(ctx, ds, cmdr, kv, slog.New(slog.Default().Handler()), 0) assert.ErrorContains(t, err, "GetHostEmailsFuncError") // checkAndReset(t, true, &ds.GetAllCertificateAuthoritiesFuncInvoked) checkAndReset(t, true, &ds.ListMDMAppleProfilesToInstallAndRemoveFuncInvoked) @@ -3545,7 +3546,7 @@ func TestMDMAppleReconcileAppleProfiles(t *testing.T) { hostUUIDs = append(hostUUIDs, p.HostUUID) } - err := ReconcileAppleProfiles(ctx, ds, cmdr, slog.New(slog.DiscardHandler), 0) + err := ReconcileAppleProfiles(ctx, ds, cmdr, kv, slog.New(slog.DiscardHandler), 0) require.NoError(t, err) assert.Empty(t, hostUUIDs, "all host+profile combinations should be updated") require.Equal(t, 5, failedCount, "number of profiles with bad content") @@ -3563,6 +3564,7 @@ func TestReconcileAppleProfilesCAThrottle(t *testing.T) { ctx := t.Context() mdmStorage := &mdmmock.MDMAppleStore{} ds := new(mock.Store) + kv := new(mock.AdvancedKVStore) pushFactory, _ := newMockAPNSPushProviderFactory() pusher := nanomdm_pushsvc.New( mdmStorage, @@ -3692,7 +3694,7 @@ func TestReconcileAppleProfilesCAThrottle(t *testing.T) { t.Run("limit=0 sends all profiles", func(t *testing.T) { upsertedProfiles = nil bulkUpsertCallCount = 0 - err := ReconcileAppleProfiles(ctx, ds, cmdr, slog.New(slog.DiscardHandler), 0) + err := ReconcileAppleProfiles(ctx, ds, cmdr, kv, slog.New(slog.DiscardHandler), 0) require.NoError(t, err) // All 10 host-profile pairs should be upserted (5 CA + 5 non-CA) @@ -3711,7 +3713,7 @@ func TestReconcileAppleProfilesCAThrottle(t *testing.T) { t.Run("limit=2 throttles CA profiles only", func(t *testing.T) { upsertedProfiles = nil bulkUpsertCallCount = 0 - err := ReconcileAppleProfiles(ctx, ds, cmdr, slog.New(slog.DiscardHandler), 2) + err := ReconcileAppleProfiles(ctx, ds, cmdr, kv, slog.New(slog.DiscardHandler), 2) require.NoError(t, err) // Should have 2 CA + 5 non-CA = 7 host-profile pairs upserted @@ -3749,7 +3751,7 @@ func TestReconcileAppleProfilesCAThrottle(t *testing.T) { return recentProfilesToInstall, nil, nil } - err := ReconcileAppleProfiles(ctx, ds, cmdr, slog.New(slog.DiscardHandler), 2) + err := ReconcileAppleProfiles(ctx, ds, cmdr, kv, slog.New(slog.DiscardHandler), 2) require.NoError(t, err) var caCount, nonCACount int @@ -3787,7 +3789,7 @@ func TestReconcileAppleProfilesCAThrottle(t *testing.T) { return nil, profilesToRemove, nil } - err := ReconcileAppleProfiles(ctx, ds, cmdr, slog.New(slog.DiscardHandler), 2) + err := ReconcileAppleProfiles(ctx, ds, cmdr, kv, slog.New(slog.DiscardHandler), 2) require.NoError(t, err) var removeCount int @@ -3805,6 +3807,167 @@ func TestReconcileAppleProfilesCAThrottle(t *testing.T) { }) } +func TestReconcileAppleProfilesSkipsHostBeingProcessed(t *testing.T) { + ctx := t.Context() + mdmStorage := &mdmmock.MDMAppleStore{} + ds := new(mock.Store) + kv := new(mock.AdvancedKVStore) + pushFactory, _ := newMockAPNSPushProviderFactory() + pusher := nanomdm_pushsvc.New( + mdmStorage, + mdmStorage, + pushFactory, + NewNanoMDMLogger(slog.New(slog.DiscardHandler)), + ) + mdmConfig := config.MDMConfig{ + AppleSCEPCert: "./testdata/server.pem", + AppleSCEPKey: "./testdata/server.key", + } + ds.GetAllMDMConfigAssetsByNameFunc = func(ctx context.Context, assetNames []fleet.MDMAssetName, + _ sqlx.QueryerContext, + ) (map[fleet.MDMAssetName]fleet.MDMConfigAsset, error) { + _, pemCert, pemKey, err := mdmConfig.AppleSCEP() + require.NoError(t, err) + return map[fleet.MDMAssetName]fleet.MDMConfigAsset{ + fleet.MDMAssetCACert: {Value: pemCert}, + fleet.MDMAssetCAKey: {Value: pemKey}, + }, nil + } + + cmdr := apple_mdm.NewMDMAppleCommander(mdmStorage, pusher) + + profileUUID := "a" + uuid.NewString() + profileContent := []byte("regular profile content") + blockedHostUUID := "host-blocked" + nonSetupHostUUID := "host-non-setup" + + ds.AppConfigFunc = func(ctx context.Context) (*fleet.AppConfig, error) { + return &fleet.AppConfig{MDM: fleet.MDM{EnabledAndConfigured: true}}, nil + } + ds.ListMDMAppleProfilesToInstallAndRemoveFunc = func(ctx context.Context) ([]*fleet.MDMAppleProfilePayload, []*fleet.MDMAppleProfilePayload, error) { + return []*fleet.MDMAppleProfilePayload{ + {ProfileUUID: profileUUID, ProfileIdentifier: "com.test.profile", ProfileName: "Test Profile", HostUUID: blockedHostUUID, Scope: fleet.PayloadScopeSystem}, + {ProfileUUID: profileUUID, ProfileIdentifier: "com.test.profile", ProfileName: "Test Profile", HostUUID: nonSetupHostUUID, Scope: fleet.PayloadScopeSystem}, + }, nil, nil + } + ds.GetMDMAppleProfilesContentsFunc = func(ctx context.Context, profileUUIDs []string) (map[string]mobileconfig.Mobileconfig, error) { + return map[string]mobileconfig.Mobileconfig{profileUUID: profileContent}, nil + } + ds.BulkDeleteMDMAppleHostsConfigProfilesFunc = func(ctx context.Context, payload []*fleet.MDMAppleProfilePayload) error { + return nil + } + ds.GetNanoMDMUserEnrollmentFunc = func(ctx context.Context, hostUUID string) (*fleet.NanoEnrollment, error) { + return nil, nil + } + ds.GetGroupedCertificateAuthoritiesFunc = func(ctx context.Context, allCAs bool) (*fleet.GroupedCertificateAuthorities, error) { + return &fleet.GroupedCertificateAuthorities{}, nil + } + ds.AggregateEnrollSecretPerTeamFunc = func(ctx context.Context) ([]*fleet.EnrollSecret, error) { + return []*fleet.EnrollSecret{}, nil + } + ds.BulkUpsertMDMAppleConfigProfilesFunc = func(ctx context.Context, p []*fleet.MDMAppleConfigProfile) error { + return nil + } + mdmStorage.BulkDeleteHostUserCommandsWithoutResultsFunc = func(ctx context.Context, commandToIDs map[string][]string) error { + return nil + } + mdmStorage.EnqueueCommandFunc = func(ctx context.Context, id []string, cmd *mdm.CommandWithSubtype) (map[string]error, error) { + return nil, nil + } + mdmStorage.RetrievePushInfoFunc = func(ctx context.Context, tokens []string) (map[string]*mdm.Push, error) { + res := make(map[string]*mdm.Push, len(tokens)) + for _, t := range tokens { + res[t] = &mdm.Push{PushMagic: "", Token: []byte(t), Topic: ""} + } + return res, nil + } + mdmStorage.RetrievePushCertFunc = func(ctx context.Context, topic string) (*tls.Certificate, string, error) { + cert, err := tls.LoadX509KeyPair("testdata/server.pem", "testdata/server.key") + return &cert, "", err + } + mdmStorage.IsPushCertStaleFunc = func(ctx context.Context, topic string, staleToken string) (bool, error) { + return false, nil + } + mdmStorage.GetAllMDMConfigAssetsByNameFunc = func(ctx context.Context, assetNames []fleet.MDMAssetName, + _ sqlx.QueryerContext, + ) (map[fleet.MDMAssetName]fleet.MDMConfigAsset, error) { + certPEM, err := os.ReadFile("./testdata/server.pem") + require.NoError(t, err) + keyPEM, err := os.ReadFile("./testdata/server.key") + require.NoError(t, err) + return map[fleet.MDMAssetName]fleet.MDMConfigAsset{ + fleet.MDMAssetCACert: {Value: certPEM}, + fleet.MDMAssetCAKey: {Value: keyPEM}, + }, nil + } + + // Track what gets upserted and which hosts get commands enqueued + var upsertedProfiles []*fleet.MDMAppleBulkUpsertHostProfilePayload + var bulkUpsertCallCount int + ds.BulkUpsertMDMAppleHostProfilesFunc = func(ctx context.Context, payload []*fleet.MDMAppleBulkUpsertHostProfilePayload) error { + bulkUpsertCallCount++ + if bulkUpsertCallCount == 1 { + upsertedProfiles = payload + } + return nil + } + + // Simulate an in-memory KV store with TTL support + kvStore := make(map[string]string) + kv.MGetFunc = func(ctx context.Context, keys []string) (map[string]*string, error) { + result := make(map[string]*string, len(keys)) + for _, k := range keys { + if v, ok := kvStore[k]; ok { + result[k] = &v + } else { + result[k] = nil + } + } + return result, nil + } + + // verify host marked as going through setup does not get profiles reconciled + blockedKey := fleet.MDMProfileProcessingKeyPrefix + ":" + blockedHostUUID + kvStore[blockedKey] = "1" + + upsertedProfiles = nil + bulkUpsertCallCount = 0 + err := ReconcileAppleProfiles(ctx, ds, cmdr, kv, slog.New(slog.DiscardHandler), 0) + require.NoError(t, err) + + // Only the non setup host should have profiles with a pending status and command UUID; + // the blocked host should have its status/command cleared. + var pendingHosts []string + var skippedHosts []string + for _, p := range upsertedProfiles { + if p.Status != nil && *p.Status == fleet.MDMDeliveryPending && p.CommandUUID != "" { + pendingHosts = append(pendingHosts, p.HostUUID) + } else if p.Status == nil && p.CommandUUID == "" { + skippedHosts = append(skippedHosts, p.HostUUID) + } + } + assert.Contains(t, pendingHosts, nonSetupHostUUID, "non setup host should have profiles enqueued") + assert.NotContains(t, pendingHosts, blockedHostUUID, "blocked host should NOT have profiles enqueued") + assert.Contains(t, skippedHosts, blockedHostUUID, "blocked host should be skipped with nil status") + + // expire the key, the host that didn't get profiles before should do now + delete(kvStore, blockedKey) // simulate TTL expiry + + upsertedProfiles = nil + bulkUpsertCallCount = 0 + err = ReconcileAppleProfiles(ctx, ds, cmdr, kv, slog.New(slog.DiscardHandler), 0) + require.NoError(t, err) + + pendingHosts = nil + for _, p := range upsertedProfiles { + if p.Status != nil && *p.Status == fleet.MDMDeliveryPending && p.CommandUUID != "" { + pendingHosts = append(pendingHosts, p.HostUUID) + } + } + assert.Contains(t, pendingHosts, nonSetupHostUUID, "non setup host should still have profiles enqueued") + assert.Contains(t, pendingHosts, blockedHostUUID, "previously blocked host should now have profiles enqueued after key expiry") +} + func TestAppleMDMFileVaultEscrowFunctions(t *testing.T) { svc := Service{} diff --git a/server/service/integration_mdm_profiles_test.go b/server/service/integration_mdm_profiles_test.go index 0933fd98b04..be588779a70 100644 --- a/server/service/integration_mdm_profiles_test.go +++ b/server/service/integration_mdm_profiles_test.go @@ -34,6 +34,7 @@ import ( "github.com/fleetdm/fleet/v4/server/ptr" "github.com/fleetdm/fleet/v4/server/service/contract" "github.com/fleetdm/fleet/v4/server/service/integrationtest/scep_server" + "github.com/fleetdm/fleet/v4/server/service/redis_key_value" "github.com/fleetdm/fleet/v4/server/test" "github.com/google/uuid" "github.com/jmoiron/sqlx" @@ -5222,6 +5223,7 @@ func (s *integrationMDMTestSuite) TestBatchSetMDMProfilesBackwardsCompat() { func (s *integrationMDMTestSuite) TestMDMBatchSetProfilesKeepsReservedNames() { t := s.T() ctx := context.Background() + kv := redis_key_value.New(s.redisPool) checkMacProfs := func(teamID *uint, names ...string) { var count int @@ -5263,7 +5265,7 @@ func (s *integrationMDMTestSuite) TestMDMBatchSetProfilesKeepsReservedNames() { if len(secrets) == 0 { require.NoError(t, s.ds.ApplyEnrollSecrets(ctx, nil, []*fleet.EnrollSecret{{Secret: t.Name()}})) } - require.NoError(t, ReconcileAppleProfiles(ctx, s.ds, s.mdmCommander, s.logger, 0)) + require.NoError(t, ReconcileAppleProfiles(ctx, s.ds, s.mdmCommander, kv, s.logger, 0)) // turn on disk encryption and os updates s.DoJSON("PATCH", "/api/latest/fleet/config", json.RawMessage(`{ @@ -5343,7 +5345,7 @@ func (s *integrationMDMTestSuite) TestMDMBatchSetProfilesKeepsReservedNames() { require.Equal(t, "14.6.1", tmResp.Team.Config.MDM.MacOSUpdates.MinimumVersion.Value) require.Equal(t, true, tmResp.Team.Config.MDM.MacOSUpdates.UpdateNewHosts.Value) - require.NoError(t, ReconcileAppleProfiles(ctx, s.ds, s.mdmCommander, s.logger, 0)) + require.NoError(t, ReconcileAppleProfiles(ctx, s.ds, s.mdmCommander, kv, s.logger, 0)) checkMacProfs(&tmResp.Team.ID, servermdm.ListFleetReservedMacOSProfileNames()...) checkWinProfs(&tmResp.Team.ID, servermdm.ListFleetReservedWindowsProfileNames()...) diff --git a/server/service/integration_mdm_test.go b/server/service/integration_mdm_test.go index 65ce68ce5db..07505850125 100644 --- a/server/service/integration_mdm_test.go +++ b/server/service/integration_mdm_test.go @@ -84,6 +84,7 @@ import ( "github.com/fleetdm/fleet/v4/server/service/integrationtest/scep_server" "github.com/fleetdm/fleet/v4/server/service/mock" "github.com/fleetdm/fleet/v4/server/service/osquery_utils" + "github.com/fleetdm/fleet/v4/server/service/redis_key_value" "github.com/fleetdm/fleet/v4/server/service/schedule" "github.com/fleetdm/fleet/v4/server/test" "github.com/fleetdm/fleet/v4/server/worker" @@ -338,7 +339,7 @@ func (s *integrationMDMTestSuite) SetupSuite() { s.onProfileJobDone() }() } - err = ReconcileAppleProfiles(ctx, ds, mdmCommander, logger, 0) + err = ReconcileAppleProfiles(ctx, ds, mdmCommander, redis_key_value.New(s.redisPool), logger, 0) require.NoError(s.T(), err) return err }), @@ -12021,6 +12022,7 @@ func (s *integrationMDMTestSuite) TestSilentMigrationGotchas() { func (s *integrationMDMTestSuite) TestAPNsPushCron() { t := s.T() ctx := context.Background() + kv := redis_key_value.New(s.redisPool) s.Do("POST", "/api/v1/fleet/mdm/profiles/batch", batchSetMDMProfilesRequest{Profiles: []fleet.MDMProfileBatchPayload{ {Name: "N1", Contents: mobileconfigForTest("N1", "I1")}, @@ -12050,13 +12052,13 @@ func (s *integrationMDMTestSuite) TestAPNsPushCron() { } // trigger the reconciliation schedule - err := ReconcileAppleProfiles(ctx, s.ds, s.mdmCommander, s.logger, 0) + err := ReconcileAppleProfiles(ctx, s.ds, s.mdmCommander, kv, s.logger, 0) require.NoError(t, err) require.Len(t, recordedPushes, 1) recordedPushes = nil // triggering the schedule again doesn't send any more pushes - err = ReconcileAppleProfiles(ctx, s.ds, s.mdmCommander, s.logger, 0) + err = ReconcileAppleProfiles(ctx, s.ds, s.mdmCommander, kv, s.logger, 0) require.NoError(t, err) require.Len(t, recordedPushes, 0) recordedPushes = nil @@ -12088,6 +12090,7 @@ func (s *integrationMDMTestSuite) TestAPNsPushCron() { func (s *integrationMDMTestSuite) TestAPNsPushWithNotNow() { t := s.T() ctx := context.Background() + kv := redis_key_value.New(s.redisPool) // macOS host, MDM on _, macDevice := createHostThenEnrollMDM(s.ds, s.server.URL, t) @@ -12111,7 +12114,7 @@ func (s *integrationMDMTestSuite) TestAPNsPushWithNotNow() { } // trigger the reconciliation schedule - err := ReconcileAppleProfiles(ctx, s.ds, s.mdmCommander, s.logger, 0) + err := ReconcileAppleProfiles(ctx, s.ds, s.mdmCommander, kv, s.logger, 0) require.NoError(t, err) require.Len(t, recordedPushes, 1) recordedPushes = nil @@ -12132,7 +12135,7 @@ func (s *integrationMDMTestSuite) TestAPNsPushWithNotNow() { }}, http.StatusNoContent) // trigger the reconciliation schedule - err = ReconcileAppleProfiles(ctx, s.ds, s.mdmCommander, s.logger, 0) + err = ReconcileAppleProfiles(ctx, s.ds, s.mdmCommander, kv, s.logger, 0) require.NoError(t, err) require.Len(t, recordedPushes, 1) recordedPushes = nil @@ -12152,7 +12155,7 @@ func (s *integrationMDMTestSuite) TestAPNsPushWithNotNow() { assert.Nil(t, cmd) // A 'NotNow' command will not trigger a new push. Device is expected to check in again when conditions change. - err = ReconcileAppleProfiles(ctx, s.ds, s.mdmCommander, s.logger, 0) + err = ReconcileAppleProfiles(ctx, s.ds, s.mdmCommander, kv, s.logger, 0) require.NoError(t, err) require.Len(t, recordedPushes, 0) recordedPushes = nil diff --git a/server/service/redis_key_value/redis_key_value.go b/server/service/redis_key_value/redis_key_value.go index 010c24c19cc..a51a8100ec0 100644 --- a/server/service/redis_key_value/redis_key_value.go +++ b/server/service/redis_key_value/redis_key_value.go @@ -13,8 +13,8 @@ import ( redigo "github.com/gomodule/redigo/redis" ) -// RedisKeyValue is a basic key/value store with SET and GET operations -// Items are removed via expiration (defined in the SET operation). +// RedisKeyValue is a key/value store with basic SET and GET operations and advanced operations +// Items are removed via expiration (defined in the SET operation), or via the DEL command type RedisKeyValue struct { pool fleet.RedisPool testPrefix string // for tests, the key prefix to use to avoid conflicts @@ -56,3 +56,38 @@ func (r *RedisKeyValue) Get(ctx context.Context, key string) (*string, error) { } return &res, nil } + +func (r *RedisKeyValue) MGet(ctx context.Context, keys []string) (map[string]*string, error) { + conn := redis.ConfigureDoer(r.pool, r.pool.Get()) + defer conn.Close() + + redisKeys := make([]interface{}, len(keys)) + for i, key := range keys { + redisKeys[i] = r.testPrefix + prefix + key + } + + res, err := redigo.Strings(conn.Do("MGET", redisKeys...)) + if err != nil { + return nil, ctxerr.Wrap(ctx, err, "redis failed to mget") + } + + result := make(map[string]*string, len(keys)) + for i, key := range keys { + if res[i] == "" { + result[key] = nil + } else { + result[key] = &res[i] + } + } + return result, nil +} + +func (r *RedisKeyValue) Delete(ctx context.Context, key string) error { + conn := redis.ConfigureDoer(r.pool, r.pool.Get()) + defer conn.Close() + + if _, err := redigo.Int(conn.Do("DEL", r.testPrefix+prefix+key)); err != nil { + return ctxerr.Wrap(ctx, err, "redis failed to delete") + } + return nil +} From 4f3417063798481c138af4a186c0b0ef1026e35f Mon Sep 17 00:00:00 2001 From: Magnus Jensen Date: Wed, 25 Mar 2026 16:59:47 -0600 Subject: [PATCH 2/9] fix mocks and logic bug --- cmd/fleet/cron_test.go | 3 ++- server/service/apple_mdm.go | 18 ++++++++++-------- server/service/apple_mdm_test.go | 8 ++++++++ .../service/integration_mdm_profiles_test.go | 2 -- server/service/integration_mdm_test.go | 7 ++++++- 5 files changed, 26 insertions(+), 12 deletions(-) diff --git a/cmd/fleet/cron_test.go b/cmd/fleet/cron_test.go index 015fbaf57db..74ef69c26dd 100644 --- a/cmd/fleet/cron_test.go +++ b/cmd/fleet/cron_test.go @@ -26,10 +26,11 @@ func TestNewAppleMDMProfileManagerWithoutConfig(t *testing.T) { ctx := context.Background() mdmStorage := &mdmmock.MDMAppleStore{} ds := new(mock.Store) + kv := new(mock.AdvancedKVStore) cmdr := apple_mdm.NewMDMAppleCommander(mdmStorage, nil) logger := slog.New(slog.DiscardHandler) - sch, err := newAppleMDMProfileManagerSchedule(ctx, "foo", ds, cmdr, logger, 0) + sch, err := newAppleMDMProfileManagerSchedule(ctx, "foo", ds, cmdr, kv, logger, 0) require.NotNil(t, sch) require.NoError(t, err) } diff --git a/server/service/apple_mdm.go b/server/service/apple_mdm.go index 0d893900465..d58bf85e6aa 100644 --- a/server/service/apple_mdm.go +++ b/server/service/apple_mdm.go @@ -5260,10 +5260,10 @@ func ReconcileAppleProfiles( end := min(i+isBeingSetupBatchSize, len(hostProfiles)) batch := hostProfiles[i:end] hostUUIDs := make([]string, len(batch)) - hostUUIDToHostProfile := make(map[string]*fleet.MDMAppleBulkUpsertHostProfilePayload, len(batch)) + hostUUIDToHostProfiles := make(map[string][]*fleet.MDMAppleBulkUpsertHostProfilePayload, len(batch)) for j, hp := range batch { hostUUIDs[j] = fleet.MDMProfileProcessingKeyPrefix + ":" + hp.HostUUID - hostUUIDToHostProfile[hp.HostUUID] = hp + hostUUIDToHostProfiles[hp.HostUUID] = append(hostUUIDToHostProfiles[hp.HostUUID], hp) } setupHostUUIDs, err := redisKeyValue.MGet(ctx, hostUUIDs) @@ -5274,16 +5274,18 @@ func ReconcileAppleProfiles( if exists != nil { hostUUID := strings.TrimPrefix(keyedHostUUID, fleet.MDMProfileProcessingKeyPrefix+":") logger.DebugContext(ctx, "skipping profile reconciliation for host being set up", "host_uuid", hostUUID) - hp, ok := hostUUIDToHostProfile[hostUUID] + hps, ok := hostUUIDToHostProfiles[hostUUID] if !ok { logger.DebugContext(ctx, "expected host uuid to be present but was not, do not skip profile reconciliation", "host_uuid", hostUUID) continue } - // Clear out host profile status and installTargets to avoid iterating over them in ProcessAndEnqueueProfiles - hp.Status = nil - hp.CommandUUID = "" - hostProfilesToInstallMap[fleet.HostProfileUUID{HostUUID: hp.HostUUID, ProfileUUID: hp.ProfileUUID}] = hp - delete(installTargets, hp.ProfileUUID) + for _, hp := range hps { + // Clear out host profile status and installTargets to avoid iterating over them in ProcessAndEnqueueProfiles + hp.Status = nil + hp.CommandUUID = "" + hostProfilesToInstallMap[fleet.HostProfileUUID{HostUUID: hp.HostUUID, ProfileUUID: hp.ProfileUUID}] = hp + delete(installTargets, hp.ProfileUUID) + } } } } diff --git a/server/service/apple_mdm_test.go b/server/service/apple_mdm_test.go index 78aeab408df..32cfa341264 100644 --- a/server/service/apple_mdm_test.go +++ b/server/service/apple_mdm_test.go @@ -2862,6 +2862,10 @@ func TestMDMAppleReconcileAppleProfiles(t *testing.T) { return baseProfilesToInstall, baseProfilesToRemove, nil } + kv.MGetFunc = func(ctx context.Context, keys []string) (map[string]*string, error) { + return map[string]*string{}, nil + } + ds.GetMDMAppleProfilesContentsFunc = func(ctx context.Context, profileUUIDs []string) (map[string]mobileconfig.Mobileconfig, error) { require.ElementsMatch(t, []string{p1, p2, p4, p5, p7}, profileUUIDs) // only those profiles that are to be installed @@ -3619,6 +3623,10 @@ func TestReconcileAppleProfilesCAThrottle(t *testing.T) { }, nil } + kv.MGetFunc = func(ctx context.Context, keys []string) (map[string]*string, error) { + return make(map[string]*string), nil + } + ds.BulkDeleteMDMAppleHostsConfigProfilesFunc = func(ctx context.Context, payload []*fleet.MDMAppleProfilePayload) error { return nil } diff --git a/server/service/integration_mdm_profiles_test.go b/server/service/integration_mdm_profiles_test.go index be588779a70..6592294e5d4 100644 --- a/server/service/integration_mdm_profiles_test.go +++ b/server/service/integration_mdm_profiles_test.go @@ -6339,8 +6339,6 @@ func (s *integrationMDMTestSuite) TestAppleProfileDeletion() { return err }) - // trigger a profile sync - s.awaitTriggerProfileSchedule(t) installs, removes := checkNextPayloads(t, mdmDevice, false) // verify that we received all profiles s.signedProfilesMatch( diff --git a/server/service/integration_mdm_test.go b/server/service/integration_mdm_test.go index 07505850125..c2b7fc50430 100644 --- a/server/service/integration_mdm_test.go +++ b/server/service/integration_mdm_test.go @@ -141,6 +141,7 @@ type integrationMDMTestSuite struct { proxyCallbackURL string jwtSigningKey *rsa.PrivateKey softwareInstallerStore fleet.SoftwareInstallerStore + keyValueStore fleet.AdvancedKeyValueStore } // appleVPPConfigSrvConf is used to configure the mock server that mocks Apple's VPP endpoints. @@ -303,6 +304,8 @@ func (s *integrationMDMTestSuite) SetupSuite() { softwareTitleIconStore, err := filesystem.NewSoftwareTitleIconStore(iconDir) require.NoError(s.T(), err) + keyValueStore := redis_key_value.New(s.redisPool) + serverConfig := TestServerOpts{ License: &fleet.LicenseInfo{ Tier: fleet.TierPremium, @@ -322,6 +325,7 @@ func (s *integrationMDMTestSuite) SetupSuite() { BootstrapPackageStore: bootstrapPackageStore, androidMockClient: androidMockClient, androidModule: androidSvc, + KeyValueStore: keyValueStore, StartCronSchedules: []TestNewScheduleFunc{ func(ctx context.Context, ds fleet.Datastore) fleet.NewCronScheduleFunc { return func() (fleet.CronSchedule, error) { @@ -339,7 +343,7 @@ func (s *integrationMDMTestSuite) SetupSuite() { s.onProfileJobDone() }() } - err = ReconcileAppleProfiles(ctx, ds, mdmCommander, redis_key_value.New(s.redisPool), logger, 0) + err = ReconcileAppleProfiles(ctx, ds, mdmCommander, keyValueStore, logger, 0) require.NoError(s.T(), err) return err }), @@ -499,6 +503,7 @@ func (s *integrationMDMTestSuite) SetupSuite() { s.mdmCommander = mdmCommander s.logger = serverLogger s.androidAPIClient = androidMockClient + s.keyValueStore = keyValueStore fleetdmSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { status := s.fleetDMNextCSRStatus.Swap(http.StatusOK) From 1ad9aa7a42a61c0696a0ca875b794372ed324efb Mon Sep 17 00:00:00 2001 From: Magnus Jensen Date: Wed, 25 Mar 2026 17:23:24 -0600 Subject: [PATCH 3/9] fix lint --- server/service/redis_key_value/redis_key_value.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/service/redis_key_value/redis_key_value.go b/server/service/redis_key_value/redis_key_value.go index a51a8100ec0..a8af17f5166 100644 --- a/server/service/redis_key_value/redis_key_value.go +++ b/server/service/redis_key_value/redis_key_value.go @@ -61,7 +61,7 @@ func (r *RedisKeyValue) MGet(ctx context.Context, keys []string) (map[string]*st conn := redis.ConfigureDoer(r.pool, r.pool.Get()) defer conn.Close() - redisKeys := make([]interface{}, len(keys)) + redisKeys := make([]any, len(keys)) for i, key := range keys { redisKeys[i] = r.testPrefix + prefix + key } From bce500b3f9f6624b37bfb4242e00b7a1a3679030 Mon Sep 17 00:00:00 2001 From: Magnus Jensen Date: Thu, 26 Mar 2026 10:15:03 -0600 Subject: [PATCH 4/9] fix test cases and run profile installation on manual enrollment as well --- .../service/integration_mdm_profiles_test.go | 30 ++++- server/service/integration_mdm_test.go | 107 +++++++++++++++--- server/worker/apple_mdm.go | 7 ++ 3 files changed, 122 insertions(+), 22 deletions(-) diff --git a/server/service/integration_mdm_profiles_test.go b/server/service/integration_mdm_profiles_test.go index 6592294e5d4..1142a825fd4 100644 --- a/server/service/integration_mdm_profiles_test.go +++ b/server/service/integration_mdm_profiles_test.go @@ -187,12 +187,14 @@ func (s *integrationMDMTestSuite) TestAppleProfileManagement() { }) require.NoError(t, err) + // calls ensure fleet profiles + s.awaitTriggerProfileSchedule(t) + // Create a host and then enroll to MDM. host, mdmDevice := createHostThenEnrollMDM(s.ds, s.server.URL, t) setupPusher(s, t, mdmDevice) - // trigger a profile sync - s.awaitTriggerProfileSchedule(t) + s.awaitRunAppleMDMWorkerSchedule() // run the worker to process the enrollment and queue profiles installs, removes := checkNextPayloads(t, mdmDevice, false) // verify that we received all profiles s.signedProfilesMatch( @@ -211,6 +213,10 @@ func (s *integrationMDMTestSuite) TestAppleProfileManagement() { s.checkMDMProfilesSummaries(t, nil, expectedNoTeamSummary, &expectedNoTeamSummary) s.checkMDMProfilesSummaries(t, &tm.ID, expectedTeamSummary, &expectedTeamSummary) // empty because no hosts in team + // remove the key, to simulate key expiration. + err = s.keyValueStore.Delete(ctx, fleet.MDMProfileProcessingKeyPrefix+":"+host.UUID) + require.NoError(t, err) + // add the host to a team err = s.ds.AddHostsToTeam(ctx, fleet.NewAddHostsToTeamParams(&tm.ID, []uint{host.ID})) require.NoError(t, err) @@ -6330,6 +6336,9 @@ func (s *integrationMDMTestSuite) TestAppleProfileDeletion() { // add global profiles s.Do("POST", "/api/v1/fleet/mdm/apple/profiles/batch", batchSetMDMAppleProfilesRequest{Profiles: globalProfiles}, http.StatusNoContent) + // make sure ensureFleetProfiles has been called + s.awaitTriggerProfileSchedule(t) + // Create a host and then enroll to MDM. host, mdmDevice := createHostThenEnrollMDM(s.ds, s.server.URL, t) // Add IdP email to host @@ -6339,6 +6348,7 @@ func (s *integrationMDMTestSuite) TestAppleProfileDeletion() { return err }) + s.awaitRunAppleMDMWorkerSchedule() installs, removes := checkNextPayloads(t, mdmDevice, false) // verify that we received all profiles s.signedProfilesMatch( @@ -6347,6 +6357,10 @@ func (s *integrationMDMTestSuite) TestAppleProfileDeletion() { ) require.Empty(t, removes) + // Simulate 1 minute expiration for redis key + err = s.keyValueStore.Delete(ctx, fleet.MDMProfileProcessingKeyPrefix+":"+host.UUID) + require.NoError(t, err) + // Add a profile with a Fleet variable. We are also testing that removal of a profile with a Fleet variable works. // A unique command is created for each host when this Fleet variable is used. globalProfilesPlusOne := [][]byte{ @@ -6447,12 +6461,16 @@ func (s *integrationMDMTestSuite) TestAppleProfileDeletion() { return err }) - // trigger a profile sync - s.awaitTriggerProfileSchedule(t) + // Run the worker to process post-enrollment for host2 and install initial profiles + s.awaitRunAppleMDMWorkerSchedule() installs, removes = checkNextPayloads(t, mdmDevice2, false) assert.Len(t, installs, 3) assert.Empty(t, removes) + // Simulate redis key expiration for host2 + err = s.keyValueStore.Delete(ctx, fleet.MDMProfileProcessingKeyPrefix+":"+host2.UUID) + require.NoError(t, err) + // Add a profile again s.Do("POST", "/api/latest/fleet/mdm/profiles/batch", batchSetMDMProfilesRequest{Profiles: []fleet.MDMProfileBatchPayload{ {Name: "N1", Contents: globalProfilesPlusOne[0]}, @@ -8012,6 +8030,10 @@ func (s *integrationMDMTestSuite) TestAppleProfileResendRaceCondition() { host, mdmDevice := createHostThenEnrollMDM(s.ds, s.server.URL, t) setupPusher(s, t, mdmDevice) + // Delete the key to let the reconciler pick up the profiles + err := s.keyValueStore.Delete(ctx, fleet.MDMProfileProcessingKeyPrefix+":"+host.UUID) + require.NoError(t, err) + scimUserID, err := s.ds.CreateScimUser(ctx, &fleet.ScimUser{UserName: "user@example.com"}) require.NoError(t, err) diff --git a/server/service/integration_mdm_test.go b/server/service/integration_mdm_test.go index c2b7fc50430..4635c82fece 100644 --- a/server/service/integration_mdm_test.go +++ b/server/service/integration_mdm_test.go @@ -110,20 +110,23 @@ func TestIntegrationsMDM(t *testing.T) { type integrationMDMTestSuite struct { suite.Suite withServer - fleetCfg config.FleetConfig - fleetDMNextCSRStatus atomic.Value - pushProvider *mock.APNSPushProvider - depStorage nanodep_storage.AllDEPStorage - profileSchedule *schedule.Schedule - androidProfileSchedule *schedule.Schedule - integrationsSchedule *schedule.Schedule - cleanupsSchedule *schedule.Schedule - onProfileJobDone func() // function called when profileSchedule.Trigger() job completed - onAndroidProfileJobDone func() // function called when androidProfileSchedule.Trigger() job completed - onIntegrationsScheduleDone func() // function called when integrationsSchedule.Trigger() job completed - onCleanupScheduleDone func() // function called when cleanupsSchedule.Trigger() job completed - mdmStorage *mysql.NanoMDMStorage - worker *worker.Worker + fleetCfg config.FleetConfig + fleetDMNextCSRStatus atomic.Value + pushProvider *mock.APNSPushProvider + depStorage nanodep_storage.AllDEPStorage + profileSchedule *schedule.Schedule + androidProfileSchedule *schedule.Schedule + integrationsSchedule *schedule.Schedule + cleanupsSchedule *schedule.Schedule + appleMDMWorkerSchedule *schedule.Schedule + onProfileJobDone func() // function called when profileSchedule.Trigger() job completed + onAndroidProfileJobDone func() // function called when androidProfileSchedule.Trigger() job completed + onIntegrationsScheduleDone func() // function called when integrationsSchedule.Trigger() job completed + onCleanupScheduleDone func() // function called when cleanupsSchedule.Trigger() job completed + onAppleMDMWorkerScheduleDone func() // function called when appleMDMWorkerSchedule.Trigger() job completed + mdmStorage *mysql.NanoMDMStorage + worker *worker.Worker + appleMDMWorker *worker.Worker // Flag to skip jobs processing by worker skipWorkerJobs atomic.Bool mdmCommander *apple_mdm.MDMAppleCommander @@ -262,10 +265,16 @@ func (s *integrationMDMTestSuite) SetupSuite() { } workr := worker.NewWorker(s.ds, wlog) workr.TestIgnoreUnknownJobs = true - workr.Register(macosJob, appleMDMJob, vppVerifyJob, softwareWorker) + workr.Register(macosJob, vppVerifyJob, softwareWorker) s.worker = workr + appleMDMWorker := worker.NewWorker(s.ds, wlog) + appleMDMWorker.TestIgnoreUnknownJobs = true + appleMDMWorker.Register(appleMDMJob) + + s.appleMDMWorker = appleMDMWorker + // clear the jobs queue of any pending jobs generated via DB migrations mysql.ExecAdhocSQL(s.T(), s.ds, func(q sqlx.ExtContext) error { _, err := q.ExecContext(context.Background(), "DELETE FROM jobs") @@ -276,6 +285,7 @@ func (s *integrationMDMTestSuite) SetupSuite() { var profileSchedule *schedule.Schedule var cleanupsSchedule *schedule.Schedule var androidProfileSchedule *schedule.Schedule + var appleMDMWorkerSchedule *schedule.Schedule cronLog := slog.New(slog.NewTextHandler(os.Stdout, nil)) if os.Getenv("FLEET_INTEGRATION_TESTS_DISABLE_LOG") != "" { cronLog = slog.New(slog.DiscardHandler) @@ -377,6 +387,24 @@ func (s *integrationMDMTestSuite) SetupSuite() { return profileSchedule, nil } }, + func(ctx context.Context, ds fleet.Datastore) fleet.NewCronScheduleFunc { + return func() (fleet.CronSchedule, error) { + const name = string(fleet.CronAppleMDMWorker) + logger := cronLog + appleMDMWorkerSchedule = schedule.New( + ctx, name, s.T().Name(), 1*time.Hour, ds, ds, + schedule.WithLogger(logger), + schedule.WithJob("apple_mdm_worker", func(ctx context.Context) error { + if s.onAppleMDMWorkerScheduleDone != nil { + defer s.onAppleMDMWorkerScheduleDone() + } + + return s.appleMDMWorker.ProcessJobs(ctx) + }), + ) + return appleMDMWorkerSchedule, nil + } + }, func(ctx context.Context, ds fleet.Datastore) fleet.NewCronScheduleFunc { return func() (fleet.CronSchedule, error) { const name = string(fleet.CronWorkerIntegrations) @@ -499,6 +527,7 @@ func (s *integrationMDMTestSuite) SetupSuite() { s.profileSchedule = profileSchedule s.cleanupsSchedule = cleanupsSchedule s.androidProfileSchedule = androidProfileSchedule + s.appleMDMWorkerSchedule = appleMDMWorkerSchedule s.mdmStorage = mdmStorage s.mdmCommander = mdmCommander s.logger = serverLogger @@ -10708,6 +10737,40 @@ func (s *integrationMDMTestSuite) runIntegrationsSchedule() { <-ch } +func (s *integrationMDMTestSuite) awaitRunAppleMDMWorkerSchedule() { + var wg sync.WaitGroup + wg.Add(1) + s.onAppleMDMWorkerScheduleDone = wg.Done + + var ( + didTrigger bool + err error + ) + for range 10 { + _, didTrigger, err = s.appleMDMWorkerSchedule.Trigger(s.T().Context()) + require.NoError(s.T(), err) + if didTrigger { + break + } + time.Sleep(100 * time.Millisecond) + } + require.True(s.T(), didTrigger, "apple MDM worker schedule did not trigger after 1 second of retries") + + // Add timeout detection + done := make(chan struct{}) + go func() { + wg.Wait() + close(done) + }() + + select { + case <-done: + // Completed successfully + case <-time.After(5 * time.Minute): + s.T().Fatalf("Apple MDM worker schedule jobs timed out after 5 minutes") + } +} + func (s *integrationMDMTestSuite) awaitRunCleanupSchedule() { var wg sync.WaitGroup wg.Add(1) @@ -12036,13 +12099,17 @@ func (s *integrationMDMTestSuite) TestAPNsPushCron() { }}, http.StatusNoContent) // macOS host, MDM on - _, macDevice := createHostThenEnrollMDM(s.ds, s.server.URL, t) + macHost, macDevice := createHostThenEnrollMDM(s.ds, s.server.URL, t) // windows host, MDM on createWindowsHostThenEnrollMDM(s.ds, s.server.URL, t) // linux and darwin, MDM off createOrbitEnrolledHost(t, "linux", "linux_host", s.ds) createOrbitEnrolledHost(t, "darwin", "mac_not_enrolled", s.ds) + // Delete the key to let the reconciler pick up the profiles + err := kv.Delete(ctx, fleet.MDMProfileProcessingKeyPrefix+":"+macHost.UUID) + require.NoError(t, err) + // we're going to modify this mock, make sure we restore its default originalPushMock := s.pushProvider.PushFunc defer func() { s.pushProvider.PushFunc = originalPushMock }() @@ -12057,7 +12124,7 @@ func (s *integrationMDMTestSuite) TestAPNsPushCron() { } // trigger the reconciliation schedule - err := ReconcileAppleProfiles(ctx, s.ds, s.mdmCommander, kv, s.logger, 0) + err = ReconcileAppleProfiles(ctx, s.ds, s.mdmCommander, kv, s.logger, 0) require.NoError(t, err) require.Len(t, recordedPushes, 1) recordedPushes = nil @@ -12105,6 +12172,10 @@ func (s *integrationMDMTestSuite) TestAPNsPushWithNotNow() { createOrbitEnrolledHost(t, "linux", "linux_host", s.ds) createOrbitEnrolledHost(t, "darwin", "mac_not_enrolled", s.ds) + // Delete the key to let the reconciler pick up the profiles + err := kv.Delete(ctx, fleet.MDMProfileProcessingKeyPrefix+":"+macDevice.UUID) + require.NoError(t, err) + // we're going to modify this mock, make sure we restore its default originalPushMock := s.pushProvider.PushFunc defer func() { s.pushProvider.PushFunc = originalPushMock }() @@ -12119,7 +12190,7 @@ func (s *integrationMDMTestSuite) TestAPNsPushWithNotNow() { } // trigger the reconciliation schedule - err := ReconcileAppleProfiles(ctx, s.ds, s.mdmCommander, kv, s.logger, 0) + err = ReconcileAppleProfiles(ctx, s.ds, s.mdmCommander, kv, s.logger, 0) require.NoError(t, err) require.Len(t, recordedPushes, 1) recordedPushes = nil diff --git a/server/worker/apple_mdm.go b/server/worker/apple_mdm.go index cf408ef6aa7..168b7347a0f 100644 --- a/server/worker/apple_mdm.go +++ b/server/worker/apple_mdm.go @@ -112,6 +112,13 @@ func isMacOS(platform string) bool { } func (a *AppleMDM) runPostManualEnrollment(ctx context.Context, args appleMDMArgs) error { + _, err := a.installProfilesForEnrollingHost(ctx, args.HostUUID) + if err != nil { + a.Log.ErrorContext(ctx, "error installing profiles for enrolling host", "host_uuid", args.HostUUID, "err", err) + // We do not return here, as we want to continue with the rest of the logic, and then the reconciler will just pick up the remaining work. + // We do this since this is a speed optimization and not critical to complete enrollment itself. + } + if isMacOS(args.Platform) { if _, err := a.installFleetd(ctx, args.HostUUID); err != nil { return ctxerr.Wrap(ctx, err, "installing post-enrollment packages") From 58208004ba48fab118a725e021e3a66457dc1607 Mon Sep 17 00:00:00 2001 From: Magnus Jensen Date: Thu, 26 Mar 2026 12:01:52 -0600 Subject: [PATCH 5/9] fix additional tests --- server/service/integration_mdm_profiles_test.go | 4 ++++ server/service/integration_mdm_test.go | 10 +++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/server/service/integration_mdm_profiles_test.go b/server/service/integration_mdm_profiles_test.go index 1142a825fd4..ac148d1d100 100644 --- a/server/service/integration_mdm_profiles_test.go +++ b/server/service/integration_mdm_profiles_test.go @@ -631,6 +631,10 @@ func (s *integrationMDMTestSuite) TestAppleProfileRetries() { h, mdmDevice := createHostThenEnrollMDM(s.ds, s.server.URL, t) setupPusher(s, t, mdmDevice) + // we remove the reds key and don't run the apple worker to keep the nature of the test + err = s.keyValueStore.Delete(ctx, fleet.MDMProfileProcessingKeyPrefix+":"+h.UUID) + require.NoError(t, err) + expectedProfileStatuses := map[string]fleet.MDMDeliveryStatus{ "I1": fleet.MDMDeliveryVerifying, "I2": fleet.MDMDeliveryVerifying, diff --git a/server/service/integration_mdm_test.go b/server/service/integration_mdm_test.go index 4635c82fece..c3a92001e4f 100644 --- a/server/service/integration_mdm_test.go +++ b/server/service/integration_mdm_test.go @@ -4370,6 +4370,7 @@ func (s *integrationMDMTestSuite) TestBootstrapPackageStatus() { err := d.device.Enroll() // queues DEP post-enrollment worker job require.NoError(t, err) + s.awaitRunAppleMDMWorkerSchedule() // process worker jobs s.runWorker() @@ -17598,6 +17599,7 @@ func (s *integrationMDMTestSuite) TestCustomSCEPConfig() { func (s *integrationMDMTestSuite) TestCustomSCEPIntegration() { t := s.T() + ctx := context.Background() s.setSkipWorkerJobs(t) scepServer := scep_server.StartTestSCEPServer(t) scepServerURL := scepServer.URL + "/scep" @@ -17695,11 +17697,13 @@ func (s *integrationMDMTestSuite) TestCustomSCEPIntegration() { require.EqualValues(t, fleet.MDMDeliveryVerified, *prof.Status) } + s.awaitTriggerProfileSchedule(t) + // Create a host and then enroll to MDM. host, mdmDevice := createHostThenEnrollMDM(s.ds, s.server.URL, t) setupPusher(s, t, mdmDevice) // trigger a profile sync - s.awaitTriggerProfileSchedule(t) + s.awaitRunAppleMDMWorkerSchedule() // Receive enrollment profiles (we are not checking/testing these here) for { cmd, err := mdmDevice.Idle() @@ -17711,6 +17715,9 @@ func (s *integrationMDMTestSuite) TestCustomSCEPIntegration() { require.NoError(t, err) } + err := s.keyValueStore.Delete(ctx, fleet.MDMProfileProcessingKeyPrefix+":"+host.UUID) + require.NoError(t, err) + // ///////////////////////////////////////// // Upload a profile without defining the CA resp := s.Do("POST", "/api/v1/fleet/mdm/profiles/batch", batchSetMDMProfilesRequest{Profiles: []fleet.MDMProfileBatchPayload{ @@ -19403,6 +19410,7 @@ func (s *integrationMDMTestSuite) TestCancelUpcomingActivity() { key := setOrbitEnrollment(t, mdmHost, s.ds) mdmHost.OrbitNodeKey = &key + s.awaitRunAppleMDMWorkerSchedule() s.runWorker() checkInstallFleetdCommandSent(t, mdmDevice, true) From c4e8701496d086d50153bb4a7f77803a646d4a60 Mon Sep 17 00:00:00 2001 From: Magnus Jensen Date: Thu, 26 Mar 2026 13:26:09 -0600 Subject: [PATCH 6/9] fix more test cases --- server/service/integration_mdm_dep_test.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/server/service/integration_mdm_dep_test.go b/server/service/integration_mdm_dep_test.go index a8a4b584db9..5639010c730 100644 --- a/server/service/integration_mdm_dep_test.go +++ b/server/service/integration_mdm_dep_test.go @@ -578,10 +578,11 @@ func (s *integrationMDMTestSuite) runDEPEnrollReleaseDeviceTest(t *testing.T, de require.NoError(t, err) } - // run the worker to process the DEP enroll request - s.runWorker() // run the cron to assign configuration profiles s.awaitTriggerProfileSchedule(t) + // run the worker to process the DEP enroll request + s.awaitRunAppleMDMWorkerSchedule() + s.runWorker() var seenDeclarativeManagement bool var cmds []*micromdm.CommandPayload @@ -600,7 +601,7 @@ func (s *integrationMDMTestSuite) runDEPEnrollReleaseDeviceTest(t *testing.T, de require.NoError(t, plist.Unmarshal(cmd.Raw, &fullCmd)) // Can be useful for debugging - /* switch cmd.Command.RequestType { + switch cmd.Command.RequestType { case "InstallProfile": fmt.Println(">>>> device received command: ", cmd.CommandUUID, cmd.Command.RequestType, string(fullCmd.Command.InstallProfile.Payload)) case "InstallEnterpriseApplication": @@ -611,7 +612,7 @@ func (s *integrationMDMTestSuite) runDEPEnrollReleaseDeviceTest(t *testing.T, de } default: fmt.Println(">>>> device received command: ", cmd.CommandUUID, cmd.Command.RequestType) - } */ + } cmds = append(cmds, &fullCmd) cmd, err = mdmDevice.Acknowledge(cmd.CommandUUID) @@ -705,6 +706,7 @@ func (s *integrationMDMTestSuite) runDEPEnrollReleaseDeviceTest(t *testing.T, de return err }) + s.awaitRunAppleMDMWorkerSchedule() s.runWorker() // make the device process the commands, it should receive the @@ -823,6 +825,7 @@ func (s *integrationMDMTestSuite) runDEPEnrollReleaseDeviceTest(t *testing.T, de return err }) + s.awaitRunAppleMDMWorkerSchedule() s.runWorker() } else { From d27c57d447aac44386d9e97bcef0b1f2f4bb1834 Mon Sep 17 00:00:00 2001 From: Magnus Jensen Date: Thu, 26 Mar 2026 13:32:41 -0600 Subject: [PATCH 7/9] fix logic bugs --- server/service/apple_mdm.go | 1 - .../service/redis_key_value/redis_key_value.go | 18 +++++++++++++++--- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/server/service/apple_mdm.go b/server/service/apple_mdm.go index d58bf85e6aa..853be1db2dc 100644 --- a/server/service/apple_mdm.go +++ b/server/service/apple_mdm.go @@ -5284,7 +5284,6 @@ func ReconcileAppleProfiles( hp.Status = nil hp.CommandUUID = "" hostProfilesToInstallMap[fleet.HostProfileUUID{HostUUID: hp.HostUUID, ProfileUUID: hp.ProfileUUID}] = hp - delete(installTargets, hp.ProfileUUID) } } } diff --git a/server/service/redis_key_value/redis_key_value.go b/server/service/redis_key_value/redis_key_value.go index a8af17f5166..1ab00a817fd 100644 --- a/server/service/redis_key_value/redis_key_value.go +++ b/server/service/redis_key_value/redis_key_value.go @@ -58,6 +58,10 @@ func (r *RedisKeyValue) Get(ctx context.Context, key string) (*string, error) { } func (r *RedisKeyValue) MGet(ctx context.Context, keys []string) (map[string]*string, error) { + if len(keys) == 0 { + return map[string]*string{}, nil + } + conn := redis.ConfigureDoer(r.pool, r.pool.Get()) defer conn.Close() @@ -66,23 +70,31 @@ func (r *RedisKeyValue) MGet(ctx context.Context, keys []string) (map[string]*st redisKeys[i] = r.testPrefix + prefix + key } - res, err := redigo.Strings(conn.Do("MGET", redisKeys...)) + res, err := redigo.Values(conn.Do("MGET", redisKeys...)) if err != nil { return nil, ctxerr.Wrap(ctx, err, "redis failed to mget") } result := make(map[string]*string, len(keys)) for i, key := range keys { - if res[i] == "" { + if res[i] == nil { result[key] = nil } else { - result[key] = &res[i] + str, err := redigo.String(res[i], nil) + if err != nil { + return nil, ctxerr.Wrap(ctx, err, "redis failed to convert value to string") + } + result[key] = &str } } return result, nil } func (r *RedisKeyValue) Delete(ctx context.Context, key string) error { + if key == "" { + return nil + } + conn := redis.ConfigureDoer(r.pool, r.pool.Get()) defer conn.Close() From 321b3e35610afa5d9bfbffa60ced36dde05a993a Mon Sep 17 00:00:00 2001 From: Magnus Jensen Date: Thu, 26 Mar 2026 16:38:55 -0600 Subject: [PATCH 8/9] fix remaining test cases --- server/service/apple_mdm.go | 21 ++++++- ...ntegration_certificate_authorities_test.go | 3 + server/service/integration_mdm_dep_test.go | 5 ++ .../service/integration_mdm_lifecycle_test.go | 10 ++- .../service/integration_mdm_profiles_test.go | 26 +++++++- .../integration_mdm_setup_experience_test.go | 62 ++++++++++++------- server/service/integration_mdm_test.go | 42 ++++++++++++- .../integration_software_titles_test.go | 1 + .../service/integration_vpp_install_test.go | 1 + 9 files changed, 140 insertions(+), 31 deletions(-) diff --git a/server/service/apple_mdm.go b/server/service/apple_mdm.go index 853be1db2dc..f064744113a 100644 --- a/server/service/apple_mdm.go +++ b/server/service/apple_mdm.go @@ -5280,10 +5280,29 @@ func ReconcileAppleProfiles( continue } for _, hp := range hps { - // Clear out host profile status and installTargets to avoid iterating over them in ProcessAndEnqueueProfiles + // Clear out host profile status and commandUUID to avoid updating the DB with a pending status hp.Status = nil hp.CommandUUID = "" hostProfilesToInstallMap[fleet.HostProfileUUID{HostUUID: hp.HostUUID, ProfileUUID: hp.ProfileUUID}] = hp + + // Also remove this host from installTargets to prevent sending MDM commands for this host. + // Note: user-scoped profiles use user enrollment IDs (not host UUIDs) in EnrollmentIDs, so + // the removal below is a no-op for those profiles, which is acceptable, since they are not enqueued via the worker. + if hp.OperationType == fleet.MDMOperationTypeInstall { + if target, ok := installTargets[hp.ProfileUUID]; ok { + var newEnrollmentIDs []string + for _, id := range target.EnrollmentIDs { + if id != hp.HostUUID { + newEnrollmentIDs = append(newEnrollmentIDs, id) + } + } + if len(newEnrollmentIDs) == 0 { + delete(installTargets, hp.ProfileUUID) + } else { + target.EnrollmentIDs = newEnrollmentIDs + } + } + } } } } diff --git a/server/service/integration_certificate_authorities_test.go b/server/service/integration_certificate_authorities_test.go index 3a2842814ea..37943a851c1 100644 --- a/server/service/integration_certificate_authorities_test.go +++ b/server/service/integration_certificate_authorities_test.go @@ -1584,12 +1584,15 @@ func (s *integrationMDMTestSuite) TestSCEPChallengeExpirationRetriesSmallStep() host, mdmDevice := createHostThenEnrollMDM(s.ds, s.server.URL, t) setupPusher(s, t, mdmDevice) s.awaitTriggerProfileSchedule(t) + s.awaitRunAppleMDMWorkerSchedule() installs, removes := checkNextPayloads(t, mdmDevice, false) s.signedProfilesMatch( defaultProfiles, installs, ) require.Empty(t, removes) + err = s.keyValueStore.Delete(ctx, fleet.MDMProfileProcessingKeyPrefix+":"+host.UUID) + require.NoError(t, err) // setup: start smallstep scep server scepServer := scep_server.StartTestSCEPServer(t) diff --git a/server/service/integration_mdm_dep_test.go b/server/service/integration_mdm_dep_test.go index 5639010c730..01a382a8a6c 100644 --- a/server/service/integration_mdm_dep_test.go +++ b/server/service/integration_mdm_dep_test.go @@ -2126,6 +2126,7 @@ func (s *integrationMDMTestSuite) TestReenrollingADEDeviceAfterRemovingItFromABM checkPostEnrollmentCommands := func(mdmDevice *mdmtest.TestAppleMDMClient, shouldReceive bool) { // run the worker to process the DEP enroll request + s.awaitRunAppleMDMWorkerSchedule() s.runWorker() // run the worker to assign configuration profiles s.awaitTriggerProfileSchedule(t) @@ -2246,6 +2247,10 @@ func (s *integrationMDMTestSuite) TestReenrollingADEDeviceAfterRemovingItFromABM err := mdmDevice.Enroll() require.NoError(t, err) + // Ensure fleet profiles + s.awaitTriggerProfileSchedule(t) + s.awaitRunAppleMDMWorkerSchedule() + // Simulate an osquery enrollment too // set an enroll secret var applyResp applyEnrollSecretSpecResponse diff --git a/server/service/integration_mdm_lifecycle_test.go b/server/service/integration_mdm_lifecycle_test.go index 19e651e2b1e..d3c828c22fc 100644 --- a/server/service/integration_mdm_lifecycle_test.go +++ b/server/service/integration_mdm_lifecycle_test.go @@ -549,8 +549,11 @@ func (s *integrationMDMTestSuite) recordAppleHostStatus( ) ([]*micromdm.CommandPayload, getHostMDMSummaryResponse, getHostMDMResponseTest) { t := s.T() - s.runWorkerUntilDone() + // ensure fleet profiles s.awaitTriggerProfileSchedule(t) + // run worker to process the enroll request + s.awaitRunAppleMDMWorkerSchedule() + s.runWorkerUntilDone() var cmds []*micromdm.CommandPayload @@ -857,8 +860,9 @@ func (s *integrationMDMTestSuite) TestLifecycleSCEPCertExpiration() { ) expectedProfiles := 4 // Fleetd configuration, Fleet root cert, N1, N2 - s.runWorker() s.awaitTriggerProfileSchedule(t) + s.awaitRunAppleMDMWorkerSchedule() + s.runWorker() ackAllCommands := func(mdmDevice *mdmtest.TestAppleMDMClient, wantFleetdInstall, wantBootstrapInstall bool) int { var count int @@ -916,6 +920,7 @@ func (s *integrationMDMTestSuite) TestLifecycleSCEPCertExpiration() { err = manualEnrolledDevice.Enroll() require.NoError(t, err) + s.awaitRunAppleMDMWorkerSchedule() s.runWorker() s.awaitTriggerProfileSchedule(t) require.Equal(t, expectedProfiles+1, ackAllCommands(manualEnrolledDevice, true, false)) // re-enrolled device gets the same commands as before @@ -1115,6 +1120,7 @@ func (s *integrationMDMTestSuite) TestLifecycleSCEPCertExpiration() { iPadMdmDevice := mdmtest.NewTestMDMClientAppleOTA(s.server.URL, enrollSecrets[0].Secret, "iPad8,1", mdmtest.WithLegacyIDeviceEnrollRef("some-legacy-ref")) require.NoError(t, iPadMdmDevice.Enroll()) + s.awaitRunAppleMDMWorkerSchedule() s.runWorker() s.awaitTriggerProfileSchedule(t) require.Equal(t, expectedProfiles-1, ackAllCommands(iPadMdmDevice, false, false)) diff --git a/server/service/integration_mdm_profiles_test.go b/server/service/integration_mdm_profiles_test.go index ac148d1d100..2538bdbbadd 100644 --- a/server/service/integration_mdm_profiles_test.go +++ b/server/service/integration_mdm_profiles_test.go @@ -2461,6 +2461,7 @@ func (s *integrationMDMTestSuite) TestHostMDMAppleProfilesStatus() { t.Logf("[TestHostMDMAppleProfilesStatus] Starting FIRST cron run (after h1, h2 enrolled) at %s", time.Now().Format(time.RFC3339)) s.awaitTriggerProfileSchedule(t) t.Logf("[TestHostMDMAppleProfilesStatus] FIRST cron run completed at %s", time.Now().Format(time.RFC3339)) + s.awaitRunAppleMDMWorkerSchedule() // G3 is user-scoped and the h2 host doesn't have a user-channel yet (and // enrolled just now, so the minimum delay to give up and fail the profile @@ -2489,6 +2490,11 @@ func (s *integrationMDMTestSuite) TestHostMDMAppleProfilesStatus() { assert.Contains(t, enrollmentIds, h1.UUID) assert.Contains(t, enrollmentIds, h2.UUID) + err = s.keyValueStore.Delete(ctx, fleet.MDMProfileProcessingKeyPrefix+":"+h1.UUID) + require.NoError(t, err) + err = s.keyValueStore.Delete(ctx, fleet.MDMProfileProcessingKeyPrefix+":"+h2.UUID) + require.NoError(t, err) + // enroll a couple hosts in team 1 h3, h3UserEnrollment, _ := createManualMDMEnrollWithOrbit(tm1EnrollSec, true) require.NotNil(t, h3.TeamID) @@ -2501,6 +2507,7 @@ func (s *integrationMDMTestSuite) TestHostMDMAppleProfilesStatus() { t.Logf("[TestHostMDMAppleProfilesStatus] Starting SECOND cron run (after h3, h4 enrolled in team1) at %s", time.Now().Format(time.RFC3339)) s.awaitTriggerProfileSchedule(t) t.Logf("[TestHostMDMAppleProfilesStatus] SECOND cron run completed at %s", time.Now().Format(time.RFC3339)) + s.awaitRunAppleMDMWorkerSchedule() // T1.3 is user-scoped and the h4 host doesn't have a user-channel yet (and // enrolled just now, so the minimum delay to give up and send the @@ -2521,6 +2528,10 @@ func (s *integrationMDMTestSuite) TestHostMDMAppleProfilesStatus() { {Identifier: mobileconfig.FleetCARootConfigPayloadIdentifier, OperationType: fleet.MDMOperationTypeInstall, Status: &fleet.MDMDeliveryPending}, }, }) + err = s.keyValueStore.Delete(ctx, fleet.MDMProfileProcessingKeyPrefix+":"+h3.UUID) + require.NoError(t, err) + err = s.keyValueStore.Delete(ctx, fleet.MDMProfileProcessingKeyPrefix+":"+h4.UUID) + require.NoError(t, err) // apply the pending profiles triggerReconcileProfilesMarkVerifying() @@ -2568,6 +2579,7 @@ func (s *integrationMDMTestSuite) TestHostMDMAppleProfilesStatus() { t.Logf("[TestHostMDMAppleProfilesStatus] Starting THIRD cron run (after h3->tm2, h4 user enrolled) at %s", time.Now().Format(time.RFC3339)) s.awaitTriggerProfileSchedule(t) t.Logf("[TestHostMDMAppleProfilesStatus] THIRD cron run completed at %s", time.Now().Format(time.RFC3339)) + s.awaitRunAppleMDMWorkerSchedule() s.assertHostAppleConfigProfiles(map[*fleet.Host][]fleet.HostMDMAppleProfile{ h3: { {Identifier: "T1.1", OperationType: fleet.MDMOperationTypeRemove, Status: &fleet.MDMDeliveryPending}, @@ -5733,6 +5745,8 @@ func (s *integrationMDMTestSuite) TestHostMDMProfilesExcludeLabels() { // hosts are not members of any label yet, so running the cron applies the labels s.awaitTriggerProfileSchedule(t) + s.awaitRunAppleMDMWorkerSchedule() + s.assertHostAppleConfigProfiles(map[*fleet.Host][]fleet.HostMDMAppleProfile{ appleHost: { {Identifier: "A1", OperationType: fleet.MDMOperationTypeInstall, Status: &fleet.MDMDeliveryPending}, @@ -5747,6 +5761,9 @@ func (s *integrationMDMTestSuite) TestHostMDMProfilesExcludeLabels() { }, }) + err = s.keyValueStore.Delete(ctx, fleet.MDMProfileProcessingKeyPrefix+":"+appleHost.UUID) + require.NoError(t, err) + // simulate the reconcile profiles deployment triggerReconcileProfiles() s.assertHostAppleConfigProfiles(map[*fleet.Host][]fleet.HostMDMAppleProfile{ @@ -5900,6 +5917,9 @@ func (s *integrationMDMTestSuite) TestHostMDMProfilesExcludeLabels() { // it also doesn't get installed to a new host not a member of any labels appleHost2, _ := createHostThenEnrollMDM(s.ds, s.server.URL, t) + s.awaitRunAppleMDMWorkerSchedule() + err = s.keyValueStore.Delete(ctx, fleet.MDMProfileProcessingKeyPrefix+":"+appleHost2.UUID) + require.NoError(t, err) triggerReconcileProfiles() s.assertHostAppleConfigProfiles(map[*fleet.Host][]fleet.HostMDMAppleProfile{ appleHost: { @@ -5984,6 +6004,9 @@ func (s *integrationMDMTestSuite) TestMDMProfilesIncludeAnyLabels() { // create an Apple and a Windows host appleHost, _ := createHostThenEnrollMDM(s.ds, s.server.URL, t) windowsHost, _ := createWindowsHostThenEnrollMDM(s.ds, s.server.URL, t) + s.awaitRunAppleMDMWorkerSchedule() + err := s.keyValueStore.Delete(ctx, fleet.MDMProfileProcessingKeyPrefix+":"+appleHost.UUID) + require.NoError(t, err) // create a few labels, we'll use the first five for "exclude any" profiles and the remaining for "include any" labels := make([]*fleet.Label, 10) @@ -5995,7 +6018,7 @@ func (s *integrationMDMTestSuite) TestMDMProfilesIncludeAnyLabels() { // simulate reporting label results for those hosts appleHost.LabelUpdatedAt = time.Now() windowsHost.LabelUpdatedAt = time.Now() - err := s.ds.UpdateHost(ctx, appleHost) + err = s.ds.UpdateHost(ctx, appleHost) require.NoError(t, err) err = s.ds.UpdateHost(ctx, windowsHost) require.NoError(t, err) @@ -6773,6 +6796,7 @@ func (s *integrationMDMTestSuite) TestDeleteMDMProfileCancelsInstalls() { t.Logf("host %d: %s", i+1, h.UUID) } s.awaitTriggerProfileSchedule(t) + s.awaitRunAppleMDMWorkerSchedule() s.assertHostAppleConfigProfiles(map[*fleet.Host][]fleet.HostMDMAppleProfile{ host1: { diff --git a/server/service/integration_mdm_setup_experience_test.go b/server/service/integration_mdm_setup_experience_test.go index 82214f15c53..9555002d028 100644 --- a/server/service/integration_mdm_setup_experience_test.go +++ b/server/service/integration_mdm_setup_experience_test.go @@ -254,10 +254,11 @@ func (s *integrationMDMTestSuite) TestSetupExperienceFlowWithSoftwareAndScriptAu err := mdmDevice.Enroll() require.NoError(t, err) + // ensure fleet profiles + s.awaitTriggerProfileSchedule(t) // run the worker to process the DEP enroll request + s.awaitRunAppleMDMWorkerSchedule() s.runWorker() - // run the worker to assign configuration profiles - s.awaitTriggerProfileSchedule(t) var cmds []*micromdm.CommandPayload cmd, err := mdmDevice.Idle() @@ -781,8 +782,10 @@ func (s *integrationMDMTestSuite) TestSetupExperienceFlowWithFMAAndVersionRollba mdmDevice.SerialNumber = teamDevice.SerialNumber require.NoError(t, mdmDevice.Enroll()) - s.runWorker() + // Ensure fleet profiles s.awaitTriggerProfileSchedule(t) + s.awaitRunAppleMDMWorkerSchedule() + s.runWorker() // Drain the initial MDM commands (InstallProfile × 3 + InstallEnterpriseApplication × 1). var cmds []*micromdm.CommandPayload @@ -948,10 +951,11 @@ func (s *integrationMDMTestSuite) TestSetupExperienceFlowWithSoftwareAndScriptFo err := mdmDevice.Enroll() require.NoError(t, err) + // ensure fleet profiles + s.awaitTriggerProfileSchedule(t) // run the worker to process the DEP enroll request + s.awaitRunAppleMDMWorkerSchedule() s.runWorker() - // run the worker to assign configuration profiles - s.awaitTriggerProfileSchedule(t) var cmds []*micromdm.CommandPayload cmd, err := mdmDevice.Idle() @@ -1147,10 +1151,11 @@ func (s *integrationMDMTestSuite) TestSetupExperienceVPPInstallError() { err := mdmDevice.Enroll() require.NoError(t, err) + // ensure fleet profiles + s.awaitTriggerProfileSchedule(t) // run the worker to process the DEP enroll request + s.awaitRunAppleMDMWorkerSchedule() s.runWorker() - // run the worker to assign configuration profiles - s.awaitTriggerProfileSchedule(t) var cmds []*micromdm.CommandPayload cmd, err := mdmDevice.Idle() @@ -1377,10 +1382,12 @@ func (s *integrationMDMTestSuite) TestSetupExperienceFlowUpdateScript() { err := mdmDevice.Enroll() require.NoError(t, err) + // run the worker to assign fleet profiles + s.awaitTriggerProfileSchedule(t) + // run the worker to process the DEP enroll request + s.awaitRunAppleMDMWorkerSchedule() s.runWorker() - // run the worker to assign configuration profiles - s.awaitTriggerProfileSchedule(t) var cmds []*micromdm.CommandPayload cmd, err := mdmDevice.Idle() @@ -1575,10 +1582,10 @@ func (s *integrationMDMTestSuite) TestSetupExperienceFlowCancelScript() { err := mdmDevice.Enroll() require.NoError(t, err) - // run the worker to process the DEP enroll request - s.runWorker() - // run the worker to assign configuration profiles + // ensure fleet profiles s.awaitTriggerProfileSchedule(t) + // run the worker to process the DEP enroll request + s.awaitRunAppleMDMWorkerSchedule() var cmds []*micromdm.CommandPayload cmd, err := mdmDevice.Idle() @@ -1902,10 +1909,11 @@ func (s *integrationMDMTestSuite) TestSetupExperienceWithLotsOfVPPApps() { err := mdmDevice.Enroll() require.NoError(t, err) + // run the worker to ensure fleet profiles + s.awaitTriggerProfileSchedule(t) // run the worker to process the DEP enroll request + s.awaitRunAppleMDMWorkerSchedule() s.runWorker() - // run the worker to assign configuration profiles - s.awaitTriggerProfileSchedule(t) var cmds []*micromdm.CommandPayload cmd, err := mdmDevice.Idle() @@ -2772,12 +2780,13 @@ func (s *integrationMDMTestSuite) runDEPEnrollReleaseMobileDeviceWithVPPTest(t * require.NoError(t, err) require.True(t, awaitingConfiguration) - // run the worker to process the DEP enroll request - s.runWorker() - // run the cron to assign configuration profiles s.awaitTriggerProfileSchedule(t) + // run the worker to process the DEP enroll request + s.awaitRunAppleMDMWorkerSchedule() + s.runWorker() + var cmds []*micromdm.CommandPayload cmd, err := mdmDevice.Idle() require.NoError(t, err) @@ -2791,7 +2800,7 @@ func (s *integrationMDMTestSuite) runDEPEnrollReleaseMobileDeviceWithVPPTest(t * var profileCustomSeen, profileFleetCASeen, unexpectedProfileSeen bool // Can be useful for debugging - logCommands := false + logCommands := true for cmd != nil { if cmd.Command.RequestType == "DeclarativeManagement" { cmd, err = mdmDevice.Acknowledge(cmd.CommandUUID) @@ -2963,6 +2972,7 @@ func (s *integrationMDMTestSuite) runDEPEnrollReleaseMobileDeviceWithVPPTest(t * return err }) + s.awaitRunAppleMDMWorkerSchedule() s.runWorker() // make the device process the commands, it should receive the VPP Verify. @@ -3023,6 +3033,7 @@ func (s *integrationMDMTestSuite) runDEPEnrollReleaseMobileDeviceWithVPPTest(t * return err }) + s.awaitRunAppleMDMWorkerSchedule() s.runWorker() // make the device process the commands, it should receive the @@ -3144,10 +3155,11 @@ func (s *integrationMDMTestSuite) TestSetupExperienceFlowWithRequireSoftware() { err := mdmDevice.Enroll() require.NoError(t, err) + // ensure fleet profiles + s.awaitTriggerProfileSchedule(t) // run the worker to process the DEP enroll request + s.awaitRunAppleMDMWorkerSchedule() s.runWorker() - // run the worker to assign configuration profiles - s.awaitTriggerProfileSchedule(t) var cmds []*micromdm.CommandPayload cmd, err := mdmDevice.Idle() @@ -3460,10 +3472,11 @@ func (s *integrationMDMTestSuite) TestSetupExperienceFlowWithRequiredSoftwareVPP err := mdmDevice.Enroll() require.NoError(t, err) + // ensure fleet profiles + s.awaitTriggerProfileSchedule(t) // run the worker to process the DEP enroll request + s.awaitRunAppleMDMWorkerSchedule() s.runWorker() - // run the worker to assign configuration profiles - s.awaitTriggerProfileSchedule(t) var cmds []*micromdm.CommandPayload cmd, err := mdmDevice.Idle() @@ -3788,10 +3801,11 @@ func (s *integrationMDMTestSuite) TestSetupExperienceMacOSCustomDisplayNameIcon( err = mdmDevice.Enroll() require.NoError(t, err) + // ensure fleet profiles + s.awaitTriggerProfileSchedule(t) // run the worker to process the DEP enroll request + s.awaitRunAppleMDMWorkerSchedule() s.runWorker() - // run the worker to assign configuration profiles - s.awaitTriggerProfileSchedule(t) var cmds []*micromdm.CommandPayload cmd, err := mdmDevice.Idle() diff --git a/server/service/integration_mdm_test.go b/server/service/integration_mdm_test.go index c3a92001e4f..766227d411f 100644 --- a/server/service/integration_mdm_test.go +++ b/server/service/integration_mdm_test.go @@ -5793,7 +5793,7 @@ func (s *integrationMDMTestSuite) assertHostAppleConfigProfiles(want map[*fleet. gp := gotProfs[i] require.Equal(t, wp.Identifier, gp.Identifier, "host uuid: %s, prof id: %s", h.UUID, gp.Identifier) require.Equal(t, wp.OperationType, gp.OperationType, "host uuid: %s, prof id: %s", h.UUID, gp.Identifier) - require.Equal(t, wp.Status, gp.Status, "host uuid: %s, prof id: %s", h.UUID, gp.Identifier) + require.Equal(t, *wp.Status, *gp.Status, "host uuid: %s, prof id: %s", h.UUID, gp.Identifier) } } } @@ -11391,6 +11391,11 @@ func (s *integrationMDMTestSuite) TestDontIgnoreAnyProfileErrors() { // Create a host and a couple of profiles host, mdmDevice := createHostThenEnrollMDM(s.ds, s.server.URL, t) + s.awaitTriggerProfileSchedule(t) + s.awaitRunAppleMDMWorkerSchedule() + + err = s.keyValueStore.Delete(ctx, fleet.MDMProfileProcessingKeyPrefix+":"+host.UUID) + require.NoError(t, err) globalProfiles := [][]byte{ mobileconfigForTest("N1", "I1"), @@ -11569,10 +11574,17 @@ func (s *integrationMDMTestSuite) TestRemoveFailedProfiles() { require.NotZero(t, createTeamResp.Team.ID) team.ID = createTeamResp.Team.ID - host, mdmDevice := createHostThenEnrollMDM(s.ds, s.server.URL, t) + // Ensure fleet profiles + s.awaitTriggerProfileSchedule(t) + host, mdmDevice := createHostThenEnrollMDM(s.ds, s.server.URL, t) + s.awaitRunAppleMDMWorkerSchedule() ident := uuid.NewString() + // simulate TTL expiration + err := s.keyValueStore.Delete(context.Background(), fleet.MDMProfileProcessingKeyPrefix+":"+host.UUID) + require.NoError(t, err) + mdmDeviceRespond := func(device *mdmtest.TestAppleMDMClient) { cmd, err := device.Idle() require.NoError(t, err) @@ -11647,6 +11659,8 @@ func (s *integrationMDMTestSuite) TestRemoveFailedProfiles() { // Test case where the profile never makes it to the host at all host, _ = createHostThenEnrollMDM(s.ds, s.server.URL, t) ident = uuid.NewString() + err = s.keyValueStore.Delete(context.Background(), fleet.MDMProfileProcessingKeyPrefix+":"+host.UUID) + require.NoError(t, err) globalProfiles = [][]byte{ mobileconfigForTest("N3", ident), @@ -12033,6 +12047,7 @@ func (s *integrationMDMTestSuite) TestSilentMigrationGotchas() { // explicitly run the worker, which will send the install fleetd command // because the device is ADE-enrolled (due to the simulation of it being // ingested from ABM) + s.awaitRunAppleMDMWorkerSchedule() s.runWorker() var installEnterpriseCount int @@ -12041,10 +12056,17 @@ func (s *integrationMDMTestSuite) TestSilentMigrationGotchas() { require.NoError(t, err) for cmd != nil { + if cmd.Command.RequestType == "DeclarativeManagement" { + cmd, err = mdmDevice.Acknowledge(cmd.CommandUUID) + require.NoError(t, err) + continue + } + if cmd.Command.RequestType == "InstallEnterpriseApplication" { installEnterpriseCount++ } else { require.Equal(t, "InstallProfile", cmd.Command.RequestType) + t.Logf("Received InstallProfile command with payload:\n%s", string(cmd.Raw)) installs = append(installs, cmd.Raw) } cmd, err = mdmDevice.Acknowledge(cmd.CommandUUID) @@ -14745,9 +14767,11 @@ func (s *integrationMDMTestSuite) TestVPPAppPolicyAutomation() { orbitHost := createOrbitEnrolledHost(t, "darwin", "nonmdm", s.ds) mdmHost, mdmDevice := createHostThenEnrollMDM(s.ds, s.server.URL, t) setOrbitEnrollment(t, mdmHost, s.ds) + s.awaitRunAppleMDMWorkerSchedule() s.runWorker() checkInstallFleetdCommandSent(t, mdmDevice, true) mdmHost2, mdmDevice2 := createHostThenEnrollMDM(s.ds, s.server.URL, t) + s.awaitRunAppleMDMWorkerSchedule() s.runWorker() checkInstallFleetdCommandSent(t, mdmDevice2, true) key := setOrbitEnrollment(t, mdmHost2, s.ds) @@ -15243,6 +15267,7 @@ func (s *integrationMDMTestSuite) TestVPPAppPolicyAutomation() { require.Equal(t, uint(1), countPendingInstalls) // send an idle request to grab the command uuid + s.awaitRunAppleMDMWorkerSchedule() s.runWorker() var cmdUUID string cmd, err := mdmDevice.Idle() @@ -16029,6 +16054,8 @@ func (s *integrationMDMTestSuite) runSCEPProxyTestWithOptionalSuffix(suffix stri setupPusher(s, t, mdmDevice) // trigger a profile sync s.awaitTriggerProfileSchedule(t) + s.awaitRunAppleMDMWorkerSchedule() + require.NoError(t, s.keyValueStore.Delete(ctx, fleet.MDMProfileProcessingKeyPrefix+":"+host.UUID)) profiles, err := s.ds.GetHostMDMAppleProfiles(ctx, host.UUID) require.NoError(t, err) require.GreaterOrEqual(t, len(profiles), 2) @@ -16319,8 +16346,10 @@ func (s *integrationMDMTestSuite) runSmallstepSCEPProxyTestWithOptionalSuffix(su // Create a host and then enroll to MDM. host, mdmDevice := createHostThenEnrollMDM(s.ds, s.server.URL, t) setupPusher(s, t, mdmDevice) - // trigger a profile sync + // ensure fleet profiles s.awaitTriggerProfileSchedule(t) + s.awaitRunAppleMDMWorkerSchedule() + require.NoError(t, s.keyValueStore.Delete(ctx, fleet.MDMProfileProcessingKeyPrefix+":"+host.UUID)) profiles, err := s.ds.GetHostMDMAppleProfiles(ctx, host.UUID) require.NoError(t, err) require.GreaterOrEqual(t, len(profiles), 2) @@ -16796,6 +16825,7 @@ func (s *integrationMDMTestSuite) TestDigiCertIntegration() { setupPusher(s, t, mdmDevice) // trigger a profile sync s.awaitTriggerProfileSchedule(t) + s.awaitRunAppleMDMWorkerSchedule() profiles, err := s.ds.GetHostMDMAppleProfiles(ctx, host.UUID) require.NoError(t, err) require.GreaterOrEqual(t, len(profiles), 0) @@ -16811,6 +16841,9 @@ func (s *integrationMDMTestSuite) TestDigiCertIntegration() { require.NoError(t, err) } + err = s.keyValueStore.Delete(ctx, fleet.MDMProfileProcessingKeyPrefix+":"+host.UUID) + require.NoError(t, err) + // Add a profile with a bad CA profile := digiCertForTest("N1", "BadCA", "badName") rawRes := s.Do("POST", "/api/latest/fleet/mdm/profiles/batch", batchSetMDMProfilesRequest{Profiles: []fleet.MDMProfileBatchPayload{ @@ -17158,6 +17191,7 @@ func (s *integrationMDMTestSuite) TestDigiCertIntegrationWithHostPlatform() { setupPusher(s, t, mdmDevice) // trigger a profile sync s.awaitTriggerProfileSchedule(t) + s.awaitRunAppleMDMWorkerSchedule() profiles, err := s.ds.GetHostMDMAppleProfiles(ctx, host.UUID) require.NoError(t, err) require.GreaterOrEqual(t, len(profiles), 0) @@ -17172,6 +17206,8 @@ func (s *integrationMDMTestSuite) TestDigiCertIntegrationWithHostPlatform() { _, err = mdmDevice.Acknowledge(cmd.CommandUUID) require.NoError(t, err) } + err = s.keyValueStore.Delete(ctx, fleet.MDMProfileProcessingKeyPrefix+":"+host.UUID) + require.NoError(t, err) // //////////////////////////////// // Test DigiCert CA with HOST_PLATFORM Fleet variable diff --git a/server/service/integration_software_titles_test.go b/server/service/integration_software_titles_test.go index ea39a3301b3..6f448ca9db2 100644 --- a/server/service/integration_software_titles_test.go +++ b/server/service/integration_software_titles_test.go @@ -227,6 +227,7 @@ func (s *integrationMDMTestSuite) TestSoftwareTitleDisplayNames() { // create MDM enrolled macOS host mdmHost, _ := createHostThenEnrollMDM(s.ds, s.server.URL, t) setOrbitEnrollment(t, mdmHost, s.ds) + s.awaitRunAppleMDMWorkerSchedule() s.runWorker() s.DoJSON("POST", "/api/latest/fleet/hosts/transfer", addHostsToTeamRequest{ diff --git a/server/service/integration_vpp_install_test.go b/server/service/integration_vpp_install_test.go index ed7f437f434..a2ed5a946c1 100644 --- a/server/service/integration_vpp_install_test.go +++ b/server/service/integration_vpp_install_test.go @@ -2124,6 +2124,7 @@ func (s *integrationMDMTestSuite) TestVPPAppScheduledUpdates() { http.StatusAccepted, &installResp) // iOS device acknowledges the InstallApplication command. + s.awaitRunAppleMDMWorkerSchedule() s.runWorker() cmd, err := deviceClient.Idle() require.NoError(t, err) From 8e6babb85089aa2663ef60533b608482d5b40ead Mon Sep 17 00:00:00 2001 From: Magnus Jensen Date: Thu, 26 Mar 2026 17:40:09 -0600 Subject: [PATCH 9/9] fix more test cases --- .../service/integration_mdm_commands_test.go | 23 +++++++++++- server/service/integration_mdm_dep_test.go | 8 ++-- .../service/integration_mdm_lifecycle_test.go | 2 +- .../service/integration_mdm_profiles_test.go | 18 ++++++--- .../integration_mdm_release_worker_test.go | 10 ++--- server/service/integration_mdm_test.go | 37 +++++++++++++++---- .../service/integration_vpp_install_test.go | 13 +++++++ 7 files changed, 87 insertions(+), 24 deletions(-) diff --git a/server/service/integration_mdm_commands_test.go b/server/service/integration_mdm_commands_test.go index b9c4a58e5f9..bdce93aa4b4 100644 --- a/server/service/integration_mdm_commands_test.go +++ b/server/service/integration_mdm_commands_test.go @@ -239,9 +239,28 @@ func (s *integrationMDMTestSuite) TestLockUnlockWipeIOSIpadOS() { })) s.setSkipWorkerJobs(t) + // ensure fleet profiles + s.awaitTriggerProfileSchedule(t) + iosHost, iosMDMClient := s.createAppleMobileHostThenDEPEnrollMDM("ios", devices[0].SerialNumber) iPadOSHost, iPadOSMDMClient := s.createAppleMobileHostThenDEPEnrollMDM("ipados", devices[1].SerialNumber) + s.awaitRunAppleMDMWorkerSchedule() + + // empty the command queue for both hosts + cmd, err := iosMDMClient.Idle() + require.NoError(t, err) + for cmd != nil { + cmd, err = iosMDMClient.Acknowledge(cmd.CommandUUID) + require.NoError(t, err) + } + cmd, err = iPadOSMDMClient.Idle() + require.NoError(t, err) + for cmd != nil { + cmd, err = iPadOSMDMClient.Acknowledge(cmd.CommandUUID) + require.NoError(t, err) + } + // We fake set installed_from_dep to emulate the devices was enrolled with DEP. require.NoError(t, s.ds.SetOrUpdateMDMData(t.Context(), iosHost.ID, false, true, s.server.URL, true, t.Name(), "", false)) require.NoError(t, s.ds.SetOrUpdateMDMData(t.Context(), iPadOSHost.ID, false, true, s.server.URL, true, t.Name(), "", false)) @@ -302,7 +321,7 @@ func (s *integrationMDMTestSuite) TestLockUnlockWipeIOSIpadOS() { require.NoError(t, err) // Run device location handler - s.runWorker() + s.awaitRunAppleMDMWorkerSchedule() // refresh the host's status, it is now locked s.DoJSON("GET", fmt.Sprintf("/api/latest/fleet/hosts/%d", tc.host.ID), nil, http.StatusOK, &getHostResp) @@ -365,7 +384,7 @@ func (s *integrationMDMTestSuite) TestLockUnlockWipeIOSIpadOS() { require.NoError(t, err) // Run device location handler - s.runWorker() + s.awaitRunAppleMDMWorkerSchedule() // Get host data s.DoJSON("GET", fmt.Sprintf("/api/latest/fleet/hosts/%d", tc.host.ID), nil, http.StatusOK, &getHostResp) diff --git a/server/service/integration_mdm_dep_test.go b/server/service/integration_mdm_dep_test.go index 01a382a8a6c..0e811ee40f7 100644 --- a/server/service/integration_mdm_dep_test.go +++ b/server/service/integration_mdm_dep_test.go @@ -897,10 +897,10 @@ func (s *integrationMDMTestSuite) TestDEPProfileAssignment() { s.Do("POST", "/api/v1/fleet/mdm/apple/profiles/batch", batchSetMDMAppleProfilesRequest{Profiles: [][]byte{globalProfile}}, http.StatusNoContent) checkPostEnrollmentCommands := func(mdmDevice *mdmtest.TestAppleMDMClient, shouldReceive bool) { - // run the worker to process the DEP enroll request - s.runWorker() - // run the worker to assign configuration profiles + // ensure fleet profiles s.awaitTriggerProfileSchedule(t) + // run the worker to process the DEP enroll request + s.awaitRunAppleMDMWorkerSchedule() var seenDeclarativeManagement bool var fleetdCmd, installProfileCmd *micromdm.CommandPayload @@ -1408,6 +1408,7 @@ func (s *integrationMDMTestSuite) TestDEPProfileAssignment() { // Simulate fleetd re-enrolling automatically. err = mdmDevice.Enroll() require.NoError(t, err) + s.awaitRunAppleMDMWorkerSchedule() // The last activity should have `installed_from_dep=true`. s.lastActivityMatches( @@ -3090,6 +3091,7 @@ func (s *integrationMDMTestSuite) TestSoftwareInventoryForADEMacOSAfterWipeAndRe mdmDevice.SerialNumber = devices[0].SerialNumber err = mdmDevice.Enroll() require.NoError(t, err) + s.awaitRunAppleMDMWorkerSchedule() // Simulate an osquery enrollment too // set an enroll secret diff --git a/server/service/integration_mdm_lifecycle_test.go b/server/service/integration_mdm_lifecycle_test.go index d3c828c22fc..bf4d9115c50 100644 --- a/server/service/integration_mdm_lifecycle_test.go +++ b/server/service/integration_mdm_lifecycle_test.go @@ -1359,7 +1359,7 @@ func (s *integrationMDMTestSuite) TestRefetchAfterReenrollIOSNoDelete() { hwModel, ) require.NoError(t, mdmDevice.Enroll()) - s.runWorker() + s.awaitRunAppleMDMWorkerSchedule() checkInstallFleetdCommandSent(t, mdmDevice, false) // mu.Lock() diff --git a/server/service/integration_mdm_profiles_test.go b/server/service/integration_mdm_profiles_test.go index 2538bdbbadd..68057f38fa0 100644 --- a/server/service/integration_mdm_profiles_test.go +++ b/server/service/integration_mdm_profiles_test.go @@ -1301,7 +1301,7 @@ func (s *integrationMDMTestSuite) TestPuppetMatchPreassignProfiles() { json.RawMessage(jsonMustMarshal(t, map[string]any{"enable_release_device_manually": true})), http.StatusNoContent) - s.runWorker() + s.awaitRunAppleMDMWorkerSchedule() // preassign an empty profile, fails s.Do("POST", "/api/latest/fleet/mdm/apple/profiles/preassign", preassignMDMAppleProfileRequest{MDMApplePreassignProfilePayload: fleet.MDMApplePreassignProfilePayload{ExternalHostIdentifier: "empty", HostUUID: nonMDMHost.UUID, Profile: nil}}, http.StatusUnprocessableEntity) @@ -1378,7 +1378,7 @@ func (s *integrationMDMTestSuite) TestPuppetMatchPreassignProfiles() { // trigger the schedule so profiles are set in their state s.awaitTriggerProfileSchedule(t) - s.runWorker() + s.awaitRunAppleMDMWorkerSchedule() // the mdm host has the same profiles (i1, i2, plus fleetd config and disk encryption) s.assertHostAppleConfigProfiles(map[*fleet.Host][]fleet.HostMDMAppleProfile{ @@ -1535,7 +1535,9 @@ func (s *integrationMDMTestSuite) TestPuppetRun() { host1, _ := createHostThenEnrollMDM(s.ds, s.server.URL, t) host2, _ := createHostThenEnrollMDM(s.ds, s.server.URL, t) host3, _ := createHostThenEnrollMDM(s.ds, s.server.URL, t) - s.runWorker() + // ensure fleet profiles + s.awaitTriggerProfileSchedule(t) + s.awaitRunAppleMDMWorkerSchedule() // Set up a mock Apple DEP API s.enableABM(t.Name()) @@ -1915,8 +1917,11 @@ func (s *integrationMDMTestSuite) TestMDMAppleListConfigProfiles() { testTeam, err := s.ds.NewTeam(ctx, &fleet.Team{Name: "TestTeam"}) require.NoError(t, err) + // ensure fleet profile + s.awaitTriggerProfileSchedule(t) + mdmHost, _ := createHostThenEnrollMDM(s.ds, s.server.URL, t) - s.runWorker() + s.awaitRunAppleMDMWorkerSchedule() t.Run("no profiles", func(t *testing.T) { var listResp listMDMAppleConfigProfilesResponse @@ -7138,8 +7143,11 @@ func (s *integrationMDMTestSuite) TestVerifyUserScopedProfiles() { // cron job hasn't run yet, so no profile exist for the host assertHostProfiles([]hostProfile{}) - // trigger a profile sync + // ensure fleet profiles s.awaitTriggerProfileSchedule(t) + s.awaitRunAppleMDMWorkerSchedule() + + require.NoError(t, s.keyValueStore.Delete(ctx, fleet.MDMProfileProcessingKeyPrefix+":"+host.UUID)) // user-scoped profiles show up as status nil (no user-enrollment yet) assertHostProfiles([]hostProfile{ diff --git a/server/service/integration_mdm_release_worker_test.go b/server/service/integration_mdm_release_worker_test.go index f58d00cb98d..a3883e79b84 100644 --- a/server/service/integration_mdm_release_worker_test.go +++ b/server/service/integration_mdm_release_worker_test.go @@ -156,7 +156,7 @@ func (s *integrationMDMTestSuite) TestReleaseWorker() { mdmDevice := enrollAppleDevice(t, device) // Run worker to start device release (NOTE: Should not release yet) - s.runWorker() + s.awaitRunAppleMDMWorkerSchedule() speedUpQueuedAppleMdmJob(t) // Get install enterprise application command and acknowledge it @@ -175,9 +175,9 @@ func (s *integrationMDMTestSuite) TestReleaseWorker() { }, }) - s.runWorker() // Run after install enterprise command to install profiles. (Should requeue until we trigger profile schedule) + s.awaitRunAppleMDMWorkerSchedule() // Run after install enterprise command to install profiles. (Should requeue until we trigger profile schedule) - s.runWorker() // release device + s.awaitRunAppleMDMWorkerSchedule() // release device // Since moving profile installation to POSTDepEnrollment worker, we can now release the device immediately, as we only wait for sending. // Verify device was released expectDeviceConfiguredSent(t, true) @@ -202,7 +202,7 @@ func (s *integrationMDMTestSuite) TestReleaseWorker() { mdmDevice := enrollAppleDevice(t, device) // Run worker to start device release (NOTE: Should not release yet) - s.runWorker() + s.awaitRunAppleMDMWorkerSchedule() speedUpQueuedAppleMdmJob(t) expectMDMCommandsOfType(t, mdmDevice, []mdmCommandOfType{ @@ -220,7 +220,7 @@ func (s *integrationMDMTestSuite) TestReleaseWorker() { }, }) - s.runWorker() // Run after post dep enrollment to release device. + s.awaitRunAppleMDMWorkerSchedule() // Run after post dep enrollment to release device. // Verify device was not released yet expectDeviceConfiguredSent(t, true) diff --git a/server/service/integration_mdm_test.go b/server/service/integration_mdm_test.go index 766227d411f..b9641b4b998 100644 --- a/server/service/integration_mdm_test.go +++ b/server/service/integration_mdm_test.go @@ -6781,7 +6781,7 @@ func (s *integrationMDMTestSuite) TestSSOWithSCIM() { // Enroll generated the TokenUpdate request to Fleet and enqueued the // Post-DEP enrollment job, it needs to be processed. - s.runWorker() + s.awaitRunAppleMDMWorkerSchedule() // ask for commands and verify that we get AccountConfiguration var accCmd *mdm.Command @@ -11062,6 +11062,9 @@ func (s *integrationMDMTestSuite) TestWindowsFreshEnrollEmptyQuery() { func (s *integrationMDMTestSuite) TestManualEnrollmentCommands() { t := s.T() + // ensure fleet profiles + s.awaitTriggerProfileSchedule(t) + // create a device that's not enrolled into Fleet, it should get a command to // install fleetd mdmDevice := mdmtest.NewTestMDMClientAppleDirect(mdmtest.AppleEnrollInfo{ @@ -11071,7 +11074,7 @@ func (s *integrationMDMTestSuite) TestManualEnrollmentCommands() { }, "MacBookPro16,1") err := mdmDevice.Enroll() require.NoError(t, err) - s.runWorker() + s.awaitRunAppleMDMWorkerSchedule() checkInstallFleetdCommandSent(t, mdmDevice, true) // create a device that's enrolled into Fleet before turning on MDM features, @@ -11084,7 +11087,7 @@ func (s *integrationMDMTestSuite) TestManualEnrollmentCommands() { mdmDevice.UUID = host.UUID err = mdmDevice.Enroll() require.NoError(t, err) - s.runWorker() + s.awaitRunAppleMDMWorkerSchedule() checkInstallFleetdCommandSent(t, mdmDevice, true) } @@ -12966,7 +12969,7 @@ func (s *integrationMDMTestSuite) TestBatchAssociateAppStoreApps() { func (s *integrationMDMTestSuite) TestInvalidCommandUUID() { t := s.T() _, device := createHostThenEnrollMDM(s.ds, s.server.URL, t) - s.runWorker() + s.awaitRunAppleMDMWorkerSchedule() cmd, err := device.Acknowledge("foo") require.NoError(t, err) require.NotNil(t, cmd) @@ -13356,8 +13359,20 @@ func checkInstallFleetdCommandSent(t *testing.T, mdmDevice *mdmtest.TestAppleMDM cmd, err := mdmDevice.Idle() require.NoError(t, err) for cmd != nil { + if cmd.Command.RequestType == "DeclarativeManagement" { + cmd, err = mdmDevice.Acknowledge(cmd.CommandUUID) + require.NoError(t, err) + continue // Do not add to commands as it's not a XML file, so we use a bool to see it once. + } + var fullCmd micromdm.CommandPayload require.NoError(t, plist.Unmarshal(cmd.Raw, &fullCmd)) + if fullCmd.Command.RequestType != "InstallEnterpriseApplication" { + cmd, err = mdmDevice.Acknowledge(cmd.CommandUUID) + require.NoError(t, err) + continue + } + if manifest := fullCmd.Command.InstallEnterpriseApplication.ManifestURL; manifest != nil { foundInstallFleetdCommand = true require.Equal(t, "InstallEnterpriseApplication", cmd.Command.RequestType) @@ -14001,10 +14016,12 @@ func (s *integrationMDMTestSuite) TestVPPApps() { orbitHost := createOrbitEnrolledHost(t, "darwin", "nonmdm", s.ds) mdmHost, mdmDevice := createHostThenEnrollMDM(s.ds, s.server.URL, t) setOrbitEnrollment(t, mdmHost, s.ds) + s.awaitRunAppleMDMWorkerSchedule() s.runWorker() checkInstallFleetdCommandSent(t, mdmDevice, true) selfServiceHost, selfServiceDevice := createHostThenEnrollMDM(s.ds, s.server.URL, t) setOrbitEnrollment(t, selfServiceHost, s.ds) + s.awaitRunAppleMDMWorkerSchedule() s.runWorker() checkInstallFleetdCommandSent(t, selfServiceDevice, true) selfServiceToken := "selfservicetoken" @@ -14014,6 +14031,7 @@ func (s *integrationMDMTestSuite) TestVPPApps() { iPadOSHost, iPadOSMdmClient := s.createAppleMobileHostThenEnrollMDM("ipados") // ensure a valid alternate device token for self-service status access checking later updateDeviceTokenForHost(t, s.ds, mdmHost.ID, "foobar") + s.awaitRunAppleMDMWorkerSchedule() // Add serial number to our fake Apple server s.appleVPPConfigSrvConfig.SerialNumbers = append(s.appleVPPConfigSrvConfig.SerialNumbers, mdmHost.HardwareSerial, @@ -15605,6 +15623,9 @@ func (s *integrationMDMTestSuite) TestOTAEnrollment() { signedReqBody, err := signedData.Finish() require.NoError(t, err) + // ensure fleet profiles + s.awaitTriggerProfileSchedule(t) + t.Run("errors", func(t *testing.T) { t.Run("if no enroll secret is provided", func(t *testing.T) { httpResp := s.DoRawNoAuth("POST", "/api/latest/fleet/ota_enrollment", reqBody, http.StatusForbidden) @@ -15723,7 +15744,7 @@ func (s *integrationMDMTestSuite) TestOTAEnrollment() { ) enrollTime := time.Now().UTC().Truncate(time.Second) require.NoError(t, mdmDevice.Enroll()) - s.runWorker() + s.awaitRunAppleMDMWorkerSchedule() checkInstallFleetdCommandSent(t, mdmDevice, true) hostByIdentifierResp := verifySuccessfulOTAEnrollment(mdmDevice, hwModel, "darwin", enrollTime) @@ -15744,7 +15765,7 @@ func (s *integrationMDMTestSuite) TestOTAEnrollment() { ) enrollTime := time.Now().UTC().Truncate(time.Second) require.NoError(t, mdmDevice.Enroll()) - s.runWorker() + s.awaitRunAppleMDMWorkerSchedule() checkInstallFleetdCommandSent(t, mdmDevice, false) hostByIdentifierResp := verifySuccessfulOTAEnrollment(mdmDevice, hwModel, "ipados", enrollTime) @@ -15761,7 +15782,7 @@ func (s *integrationMDMTestSuite) TestOTAEnrollment() { ) enrollTime := time.Now().UTC().Truncate(time.Second) require.NoError(t, mdmDevice.Enroll()) - s.runWorker() + s.awaitRunAppleMDMWorkerSchedule() checkInstallFleetdCommandSent(t, mdmDevice, true) resp := verifySuccessfulOTAEnrollment(mdmDevice, hwModel, "darwin", enrollTime) @@ -15789,7 +15810,7 @@ func (s *integrationMDMTestSuite) TestOTAEnrollment() { ) enrollTime := time.Now().UTC().Truncate(time.Second) require.NoError(t, mdmDevice.Enroll()) - s.runWorker() + s.awaitRunAppleMDMWorkerSchedule() checkInstallFleetdCommandSent(t, mdmDevice, true) resp := verifySuccessfulOTAEnrollment(mdmDevice, hwModel, "darwin", enrollTime) diff --git a/server/service/integration_vpp_install_test.go b/server/service/integration_vpp_install_test.go index a2ed5a946c1..bc57d1803a5 100644 --- a/server/service/integration_vpp_install_test.go +++ b/server/service/integration_vpp_install_test.go @@ -143,9 +143,11 @@ func (s *integrationMDMTestSuite) TestVPPAppInstallVerification() { orbitHost := createOrbitEnrolledHost(t, "darwin", "nonmdm", s.ds) mdmHost, mdmDevice := createHostThenEnrollMDM(s.ds, s.server.URL, t) setOrbitEnrollment(t, mdmHost, s.ds) + s.awaitRunAppleMDMWorkerSchedule() s.runWorker() checkInstallFleetdCommandSent(t, mdmDevice, true) selfServiceHost, selfServiceDevice := createHostThenEnrollMDM(s.ds, s.server.URL, t) + s.awaitRunAppleMDMWorkerSchedule() s.runWorker() setOrbitEnrollment(t, selfServiceHost, s.ds) selfServiceToken := "selfservicetoken" @@ -156,6 +158,7 @@ func (s *integrationMDMTestSuite) TestVPPAppInstallVerification() { // Create and enroll an iOS device // ensure a valid alternate device token for self-service status access checking later updateDeviceTokenForHost(t, s.ds, mdmHost.ID, "foobar") + s.awaitRunAppleMDMWorkerSchedule() // Add serial number to our fake Apple server s.appleVPPConfigSrvConfig.SerialNumbers = append(s.appleVPPConfigSrvConfig.SerialNumbers, mdmHost.HardwareSerial) @@ -774,6 +777,7 @@ func (s *integrationMDMTestSuite) TestVPPAppInstallVerification() { // Re-enroll host in MDM mdmDevice = enrollMacOSHostInMDMManually(t, mdmHost, s.ds, s.server.URL) + s.awaitRunAppleMDMWorkerSchedule() s.runWorker() checkInstallFleetdCommandSent(t, mdmDevice, true) @@ -835,6 +839,7 @@ func (s *integrationMDMTestSuite) TestVPPAppInstallVerification() { // Enroll iOS device and ipod device, add serial number to fake Apple server, and transfer to team iosHost, iosDevice := s.createAppleMobileHostThenEnrollMDM("ios") ipodHost, ipodDevice := s.createIpodHostThenEnrollMDM() + s.awaitRunAppleMDMWorkerSchedule() s.appleVPPConfigSrvConfig.SerialNumbers = append(s.appleVPPConfigSrvConfig.SerialNumbers, iosDevice.SerialNumber, ipodDevice.SerialNumber) s.Do("POST", "/api/latest/fleet/hosts/transfer", &addHostsToTeamRequest{HostIDs: []uint{iosHost.ID, ipodHost.ID}, TeamID: &team.ID}, http.StatusOK) @@ -1094,6 +1099,7 @@ func (s *integrationMDMTestSuite) TestVPPAppInstallVerification() { for _, data := range ssVppData { // Enroll device, add serial number to fake Apple server, and transfer to team data.host, data.device = s.createAppleMobileHostThenEnrollMDM(data.platform) + s.awaitRunAppleMDMWorkerSchedule() s.appleVPPConfigSrvConfig.SerialNumbers = append(s.appleVPPConfigSrvConfig.SerialNumbers, data.device.SerialNumber) s.Do("POST", "/api/latest/fleet/hosts/transfer", &addHostsToTeamRequest{HostIDs: []uint{data.host.ID}, TeamID: &team.ID}, http.StatusOK) @@ -1231,6 +1237,7 @@ func (s *integrationMDMTestSuite) TestVPPAppActivitiesOnCancelInstall() { // create a control host that will not be used in the test, should be unaffected controlHost, controlDevice := createHostThenEnrollMDM(s.ds, s.server.URL, t) setOrbitEnrollment(t, controlHost, s.ds) + s.awaitRunAppleMDMWorkerSchedule() s.runWorker() checkInstallFleetdCommandSent(t, controlDevice, true) // Add serial number to our fake Apple server @@ -1247,6 +1254,7 @@ func (s *integrationMDMTestSuite) TestVPPAppActivitiesOnCancelInstall() { // (so the VPP installs are not activated when they are cancelled) mdmHost, mdmDevice := createHostThenEnrollMDM(s.ds, s.server.URL, t) setOrbitEnrollment(t, mdmHost, s.ds) + s.awaitRunAppleMDMWorkerSchedule() s.runWorker() checkInstallFleetdCommandSent(t, mdmDevice, true) // Add serial number to our fake Apple server @@ -1315,6 +1323,7 @@ func (s *integrationMDMTestSuite) TestVPPAppActivitiesOnCancelInstall() { // they are cancelled) mdmHost2, mdmDevice2 := createHostThenEnrollMDM(s.ds, s.server.URL, t) setOrbitEnrollment(t, mdmHost2, s.ds) + s.awaitRunAppleMDMWorkerSchedule() s.runWorker() checkInstallFleetdCommandSent(t, mdmDevice2, true) // Add serial number to our fake Apple server @@ -1562,6 +1571,7 @@ func (s *integrationMDMTestSuite) TestInHouseAppInstall() { // Enroll iPhone iosHost, iosDevice := s.createAppleMobileHostThenEnrollMDM("ios") s.appleVPPConfigSrvConfig.SerialNumbers = append(s.appleVPPConfigSrvConfig.SerialNumbers, iosDevice.SerialNumber) + s.awaitRunAppleMDMWorkerSchedule() // Create a label clr := createLabelResponse{} @@ -1728,6 +1738,7 @@ func (s *integrationMDMTestSuite) TestInHouseAppSelfInstall() { // Enroll iPhone iosHost, iosDevice := s.createAppleMobileHostThenEnrollMDM("ios") s.appleVPPConfigSrvConfig.SerialNumbers = append(s.appleVPPConfigSrvConfig.SerialNumbers, iosDevice.SerialNumber) + s.awaitRunAppleMDMWorkerSchedule() // Upload in-house app for iOS, not available in self-service for now s.uploadSoftwareInstaller(t, &fleet.UploadSoftwareInstallerPayload{Filename: "ipa_test.ipa"}, http.StatusOK, "") @@ -2625,6 +2636,7 @@ func (s *integrationMDMTestSuite) TestVPPAppInstallVerificationXcodeSpecialCase( // create a host that will receive the VPP install commands mdmHost, mdmDevice := createHostThenEnrollMDM(s.ds, s.server.URL, t) setOrbitEnrollment(t, mdmHost, s.ds) + s.awaitRunAppleMDMWorkerSchedule() s.runWorker() checkInstallFleetdCommandSent(t, mdmDevice, true) @@ -2777,6 +2789,7 @@ func (s *integrationMDMTestSuite) TestVPPAppInstallVerificationXcodeSpecialCase( // trigger install of both apps together on a different host mdmHost2, mdmDevice2 := createHostThenEnrollMDM(s.ds, s.server.URL, t) mdmHost2.OrbitNodeKey = ptr.String(setOrbitEnrollment(t, mdmHost2, s.ds)) + s.awaitRunAppleMDMWorkerSchedule() s.runWorker() checkInstallFleetdCommandSent(t, mdmDevice2, true)