diff --git a/cmd/nelm/release_install.go b/cmd/nelm/release_install.go index 122b62ed..e4925417 100644 --- a/cmd/nelm/release_install.go +++ b/cmd/nelm/release_install.go @@ -59,7 +59,7 @@ func newReleaseInstallCommand(ctx context.Context, afterAllCommandsBuiltFuncs ma } } - if err := action.ReleaseInstall(ctx, cfg.ReleaseName, cfg.ReleaseNamespace, cfg.ReleaseInstallOptions); err != nil { + if _, err := action.ReleaseInstall(ctx, cfg.ReleaseName, cfg.ReleaseNamespace, cfg.ReleaseInstallOptions); err != nil { return fmt.Errorf("install: %w", err) } diff --git a/pkg/action/release_install.go b/pkg/action/release_install.go index 1f75bfe5..441bb9c3 100644 --- a/pkg/action/release_install.go +++ b/pkg/action/release_install.go @@ -150,7 +150,28 @@ type runRollbackPlanResult struct { FailedResourceOps []*plan.Operation } -func ReleaseInstall(ctx context.Context, releaseName, releaseNamespace string, opts ReleaseInstallOptions) error { +type ReleaseInstallResultV1 struct { + APIVersion string `json:"apiVersion"` + Release *ReleaseInstallResultRelease `json:"release"` + Resources []*spec.ResourceSpec `json:"resources,omitempty"` +} + +type ReleaseInstallResultRelease struct { + Name string `json:"name"` + Namespace string `json:"namespace"` + Revision int `json:"revision"` + Status helmrelease.Status `json:"status"` +} + +type releaseInstallOutcome struct { + err error + result *ReleaseInstallResultV1 +} + +// Install or upgrade the Helm release. Returns a result unless the release failed with critical +// errors. The result is also returned when the release was skipped as already up to date +// (Release.Status is StatusSkipped) and when non-critical errors are returned alongside it. +func ReleaseInstall(ctx context.Context, releaseName, releaseNamespace string, opts ReleaseInstallOptions) (*ReleaseInstallResultV1, error) { if opts.LegacyProgressReportCh != nil { defer close(opts.LegacyProgressReportCh) } @@ -164,37 +185,38 @@ func ReleaseInstall(ctx context.Context, releaseName, releaseNamespace string, o ctx, _ = context.WithTimeoutCause(ctx, opts.Timeout, fmt.Errorf("context timed out: action timed out after %s", opts.Timeout.String())) defer ctxCancelFn(fmt.Errorf("context canceled: action finished")) - actionCh := make(chan error, 1) + actionCh := make(chan *releaseInstallOutcome, 1) go func() { - actionCh <- releaseInstall(ctx, ctxCancelFn, releaseName, releaseNamespace, opts) + result, err := releaseInstall(ctx, ctxCancelFn, releaseName, releaseNamespace, opts) + actionCh <- &releaseInstallOutcome{err: err, result: result} }() for { select { - case err := <-actionCh: - return err + case outcome := <-actionCh: + return outcome.result, outcome.err case <-ctx.Done(): - return context.Cause(ctx) + return nil, context.Cause(ctx) } } } -func releaseInstall(ctx context.Context, ctxCancelFn context.CancelCauseFunc, releaseName, releaseNamespace string, opts ReleaseInstallOptions) error { +func releaseInstall(ctx context.Context, ctxCancelFn context.CancelCauseFunc, releaseName, releaseNamespace string, opts ReleaseInstallOptions) (*ReleaseInstallResultV1, error) { usePlan := opts.PlanArtifactPath != "" currentDir, err := os.Getwd() if err != nil { - return fmt.Errorf("get current working directory: %w", err) + return nil, fmt.Errorf("get current working directory: %w", err) } homeDir, err := os.UserHomeDir() if err != nil { - return fmt.Errorf("get home directory: %w", err) + return nil, fmt.Errorf("get home directory: %w", err) } opts, err = applyReleaseInstallOptionsDefaults(opts, currentDir, homeDir) if err != nil { - return fmt.Errorf("build release install options: %w", err) + return nil, fmt.Errorf("build release install options: %w", err) } if opts.SecretKey != "" { @@ -209,13 +231,13 @@ func releaseInstall(ctx context.Context, ctxCancelFn context.CancelCauseFunc, re planArtifact, err = plan.ReadPlanArtifact(ctx, opts.PlanArtifactPath, opts.SecretKey, opts.SecretWorkDir) if err != nil { - return fmt.Errorf("read plan artifact from %s: %w", opts.PlanArtifactPath, err) + return nil, fmt.Errorf("read plan artifact from %s: %w", opts.PlanArtifactPath, err) } log.Default.Debug(ctx, "Validate plan artifact") if err := plan.ValidatePlanArtifact(planArtifact, opts.PlanArtifactLifetime); err != nil { - return fmt.Errorf("validate plan artifact: %w", err) + return nil, fmt.Errorf("validate plan artifact: %w", err) } releaseNamespace = planArtifact.Release.Namespace @@ -238,12 +260,12 @@ func releaseInstall(ctx context.Context, ctxCancelFn context.CancelCauseFunc, re KubeContextNamespace: releaseNamespace, // TODO: unset it everywhere }) if err != nil { - return fmt.Errorf("construct kube config: %w", err) + return nil, fmt.Errorf("construct kube config: %w", err) } clientFactory, err := kube.NewClientFactory(ctx, kubeConfig) if err != nil { - return fmt.Errorf("construct kube client factory: %w", err) + return nil, fmt.Errorf("construct kube client factory: %w", err) } var helmRegistryClient *registry.Client @@ -263,7 +285,7 @@ func releaseInstall(ctx context.Context, ctxCancelFn context.CancelCauseFunc, re helmRegistryClient, err = registry.NewClient(helmRegistryClientOpts...) if err != nil { - return fmt.Errorf("construct registry client: %w", err) + return nil, fmt.Errorf("construct registry client: %w", err) } } @@ -272,13 +294,13 @@ func releaseInstall(ctx context.Context, ctxCancelFn context.CancelCauseFunc, re SQLConnection: opts.ReleaseStorageSQLConnection, }) if err != nil { - return fmt.Errorf("construct release storage: %w", err) + return nil, fmt.Errorf("construct release storage: %w", err) } var lockManager *lock.LockManager if !opts.LegacyNoReleaseLock { if m, err := lock.NewLockManager(ctx, releaseNamespace, false, clientFactory); err != nil { - return fmt.Errorf("construct lock manager: %w", err) + return nil, fmt.Errorf("construct lock manager: %w", err) } else { lockManager = m } @@ -286,7 +308,7 @@ func releaseInstall(ctx context.Context, ctxCancelFn context.CancelCauseFunc, re if !opts.NoCreateNamespace { if err := createReleaseNamespace(ctx, clientFactory, releaseNamespace); err != nil { - return fmt.Errorf("create release namespace: %w", err) + return nil, fmt.Errorf("create release namespace: %w", err) } } @@ -294,7 +316,7 @@ func releaseInstall(ctx context.Context, ctxCancelFn context.CancelCauseFunc, re if lockManager != nil { if lock, err := lockManager.LockRelease(ctx, releaseName); err != nil { - return fmt.Errorf("lock release: %w", err) + return nil, fmt.Errorf("lock release: %w", err) } else { defer func() { _ = lockManager.Unlock(lock) @@ -306,7 +328,7 @@ func releaseInstall(ctx context.Context, ctxCancelFn context.CancelCauseFunc, re history, err := release.BuildHistory(releaseName, releaseStorage, release.HistoryOptions{}) if err != nil { - return fmt.Errorf("build release history: %w", err) + return nil, fmt.Errorf("build release history: %w", err) } releases := history.Releases() @@ -328,7 +350,7 @@ func releaseInstall(ctx context.Context, ctxCancelFn context.CancelCauseFunc, re if usePlan { if planArtifact.Release.Revision != newRevision { - return fmt.Errorf("plan artifact release revision mismatch: expected %d, got %d", + return nil, fmt.Errorf("plan artifact release revision mismatch: expected %d, got %d", planArtifact.Release.Revision, newRevision) } @@ -383,7 +405,7 @@ func releaseInstall(ctx context.Context, ctxCancelFn context.CancelCauseFunc, re TempDirPath: opts.TempDirPath, }) if err != nil { - return fmt.Errorf("render chart: %w", err) + return nil, fmt.Errorf("render chart: %w", err) } log.Default.Debug(ctx, "Build transformed resource specs") @@ -393,7 +415,7 @@ func releaseInstall(ctx context.Context, ctxCancelFn context.CancelCauseFunc, re spec.NewDropInvalidAnnotationsAndLabelsTransformer(), }) if err != nil { - return fmt.Errorf("build transformed resource specs: %w", err) + return nil, fmt.Errorf("build transformed resource specs: %w", err) } log.Default.Debug(ctx, "Build releasable resource specs") @@ -409,7 +431,7 @@ func releaseInstall(ctx context.Context, ctxCancelFn context.CancelCauseFunc, re releasableResSpecs, err := spec.BuildReleasableResourceSpecs(ctx, releaseNamespace, transformedResSpecs, patchers) if err != nil { - return fmt.Errorf("build releasable resource specs: %w", err) + return nil, fmt.Errorf("build releasable resource specs: %w", err) } newRelease, err = release.NewRelease(releaseName, releaseNamespace, newRevision, deployType, releasableResSpecs, renderChartResult.Chart, renderChartResult.ReleaseConfig, release.ReleaseOptions{ @@ -418,7 +440,7 @@ func releaseInstall(ctx context.Context, ctxCancelFn context.CancelCauseFunc, re Notes: renderChartResult.Notes, }) if err != nil { - return fmt.Errorf("construct new release: %w", err) + return nil, fmt.Errorf("construct new release: %w", err) } log.Default.Debug(ctx, "Convert previous release to resource specs") @@ -427,7 +449,7 @@ func releaseInstall(ctx context.Context, ctxCancelFn context.CancelCauseFunc, re if prevRelease != nil { prevRelResSpecs, err = release.ReleaseToResourceSpecs(prevRelease, releaseNamespace, false) if err != nil { - return fmt.Errorf("convert previous release to resource specs: %w", err) + return nil, fmt.Errorf("convert previous release to resource specs: %w", err) } } @@ -435,7 +457,7 @@ func releaseInstall(ctx context.Context, ctxCancelFn context.CancelCauseFunc, re newRelResSpecs, err := release.ReleaseToResourceSpecs(newRelease, releaseNamespace, false) if err != nil { - return fmt.Errorf("convert new release to resource specs: %w", err) + return nil, fmt.Errorf("convert new release to resource specs: %w", err) } log.Default.Debug(ctx, "Build resources") @@ -449,13 +471,13 @@ func releaseInstall(ctx context.Context, ctxCancelFn context.CancelCauseFunc, re NoPodLogs: opts.NoPodLogs, }) if err != nil { - return fmt.Errorf("build resources: %w", err) + return nil, fmt.Errorf("build resources: %w", err) } log.Default.Debug(ctx, "Locally validate resources") if err := resource.ValidateLocal(ctx, releaseNamespace, instResources, opts.ResourceValidationOptions); err != nil { - return fmt.Errorf("locally validate resources: %w", err) + return nil, fmt.Errorf("locally validate resources: %w", err) } log.Default.Debug(ctx, "Build resource infos") @@ -468,7 +490,7 @@ func releaseInstall(ctx context.Context, ctxCancelFn context.CancelCauseFunc, re if lastDeployedOrLastRelease != nil { lastDeployedOrLastRelResSpecs, err = release.ReleaseToResourceSpecs(lastDeployedOrLastRelease, releaseNamespace, false) if err != nil { - return fmt.Errorf("convert last deployed or last release to resource specs: %w", err) + return nil, fmt.Errorf("convert last deployed or last release to resource specs: %w", err) } } @@ -478,20 +500,20 @@ func releaseInstall(ctx context.Context, ctxCancelFn context.CancelCauseFunc, re LastDeployedOrLastRelResourceSpecs: lastDeployedOrLastRelResSpecs, }) if err != nil { - return fmt.Errorf("build resource infos: %w", err) + return nil, fmt.Errorf("build resource infos: %w", err) } log.Default.Debug(ctx, "Remotely validate resources") if err := plan.ValidateRemote(releaseName, releaseNamespace, instResInfos, opts.ForceAdoption); err != nil { - return fmt.Errorf("remotely validate resources: %w", err) + return nil, fmt.Errorf("remotely validate resources: %w", err) } log.Default.Debug(ctx, "Build release infos") relInfos, err = plan.BuildReleaseInfos(ctx, deployType, releases, newRelease) if err != nil { - return fmt.Errorf("build release infos: %w", err) + return nil, fmt.Errorf("build release infos: %w", err) } log.Default.Debug(ctx, "Build install plan") @@ -502,19 +524,19 @@ func releaseInstall(ctx context.Context, ctxCancelFn context.CancelCauseFunc, re if err != nil { handleBuildPlanErr(ctx, installPlan, err, opts.InstallGraphPath, opts.TempDirPath, "release-install-graph.dot") - return fmt.Errorf("%w: install: %w", ErrBuildPlan, err) + return nil, fmt.Errorf("%w: install: %w", ErrBuildPlan, err) } } if opts.InstallGraphPath != "" { if err := savePlanAsDot(installPlan, opts.InstallGraphPath); err != nil { - return fmt.Errorf("save release install graph: %w", err) + return nil, fmt.Errorf("save release install graph: %w", err) } } result, err := release.IsReleaseUpToDate(prevRelease, newRelease) if err != nil { - return fmt.Errorf("check if release is up to date: %w", err) + return nil, fmt.Errorf("check if release is up to date: %w", err) } releaseIsUpToDate := result.UpToDate @@ -537,7 +559,7 @@ func releaseInstall(ctx context.Context, ctxCancelFn context.CancelCauseFunc, re Revision: newRelease.Version, Status: helmrelease.StatusSkipped, }); err != nil { - return fmt.Errorf("save release install report: %w", err) + return nil, fmt.Errorf("save release install report: %w", err) } } @@ -556,7 +578,7 @@ func releaseInstall(ctx context.Context, ctxCancelFn context.CancelCauseFunc, re log.Default.Info(ctx, color.Style{color.Bold, color.Green}.Render(fmt.Sprintf("Skipped release %q (namespace: %q): cluster resources already as desired", releaseName, releaseNamespace))) - return nil + return newReleaseInstallResult(releaseName, releaseNamespace, prevRelease.Version, helmrelease.StatusSkipped, instResInfos), nil } taskStore := kdutil.NewConcurrent(statestore.NewTaskStore()) @@ -710,14 +732,18 @@ func releaseInstall(ctx context.Context, ctxCancelFn context.CancelCauseFunc, re allErrs := &util.MultiError{} allErrs.Add(criticalErrs, nonCriticalErrs) - return fmt.Errorf("failed release %q (namespace: %q): %w", releaseName, releaseNamespace, allErrs) - } else if nonCriticalErrs.HasErrors() { - return fmt.Errorf("succeeded release %q (namespace: %q), but non-critical errors encountered: %w", releaseName, releaseNamespace, nonCriticalErrs) + return nil, fmt.Errorf("failed release %q (namespace: %q): %w", releaseName, releaseNamespace, allErrs) + } + + installResult := newReleaseInstallResult(releaseName, releaseNamespace, newRelease.Version, helmrelease.StatusDeployed, instResInfos) + + if nonCriticalErrs.HasErrors() { + return installResult, fmt.Errorf("succeeded release %q (namespace: %q), but non-critical errors encountered: %w", releaseName, releaseNamespace, nonCriticalErrs) } log.Default.Info(ctx, color.Style{color.Bold, color.Green}.Render(fmt.Sprintf("Succeeded release %q (namespace: %q)", releaseName, releaseNamespace))) - return nil + return installResult, nil } func applyReleaseInstallOptionsDefaults(opts ReleaseInstallOptions, currentDir, homeDir string) (ReleaseInstallOptions, error) { @@ -838,6 +864,32 @@ func createReleaseNamespace(ctx context.Context, clientFactory kube.ClientFactor return nil } +func newReleaseInstallResult(releaseName, releaseNamespace string, revision int, status helmrelease.Status, instResInfos []*plan.InstallableResourceInfo) *ReleaseInstallResultV1 { + // There is one InstallableResourceInfo per deploy stage, but each resource must be returned once. + uniqResInfos := lo.UniqBy(instResInfos, func(info *plan.InstallableResourceInfo) string { + return info.ID() + }) + + resSpecs := lo.Map(uniqResInfos, func(info *plan.InstallableResourceInfo, _ int) *spec.ResourceSpec { + return info.LocalResource.ResourceSpec + }) + + sort.SliceStable(resSpecs, func(i, j int) bool { + return spec.ResourceSpecSortHandler(resSpecs[i], resSpecs[j]) + }) + + return &ReleaseInstallResultV1{ + APIVersion: "v1", + Release: &ReleaseInstallResultRelease{ + Name: releaseName, + Namespace: releaseNamespace, + Revision: revision, + Status: status, + }, + Resources: resSpecs, + } +} + func runRollbackPlan(ctx context.Context, releaseName, releaseNamespace string, failedRelease, prevDeployedRelease *helmrelease.Release, taskStore *kdutil.Concurrent[*statestore.TaskStore], logStore *kdutil.Concurrent[*logstore.LogStore], informerFactory *kdutil.Concurrent[*informer.InformerFactory], history *release.History, clientFactory *kube.ClientFactory, opts runRollbackPlanOptions) (result *runRollbackPlanResult, nonCritErrs, critErrs *util.MultiError) { critErrs = &util.MultiError{} nonCritErrs = &util.MultiError{} diff --git a/pkg/action/release_install_ai_test.go b/pkg/action/release_install_ai_test.go index 7ea6c09d..41195288 100644 --- a/pkg/action/release_install_ai_test.go +++ b/pkg/action/release_install_ai_test.go @@ -19,8 +19,11 @@ import ( "k8s.io/client-go/kubernetes" "github.com/werf/nelm/pkg/common" + helmrelease "github.com/werf/nelm/pkg/helm/pkg/release" "github.com/werf/nelm/pkg/kube" "github.com/werf/nelm/pkg/legacy/progrep" + "github.com/werf/nelm/pkg/plan" + "github.com/werf/nelm/pkg/resource" "github.com/werf/nelm/pkg/resource/spec" ) @@ -266,10 +269,95 @@ func TestAI_CreateReleaseNamespaceRealCreateFailurePropagates(t *testing.T) { assert.Equal(t, createNamespaceCall{dryRun: false, kind: "Namespace"}, kubeClient.calls[2]) } +func TestAI_NewReleaseInstallResultDeduplicatesMultiStageResources(t *testing.T) { + hookInfo := newTestInstallableResourceInfo("batch/v1", "Job", "myhook", "mynamespace", "mynamespace", "mychart/templates/hook.yaml", common.StoreAsHook, map[string]string{ + "helm.sh/hook": "pre-install,post-install", + }) + hookInfo.Stage = common.StagePreInstall + + postHookInfo := *hookInfo + postHookInfo.Stage = common.StagePostInstall + + cmInfo := newTestInstallableResourceInfo("v1", "ConfigMap", "mycm", "mynamespace", "mynamespace", "mychart/templates/configmap.yaml", common.StoreAsRegular, nil) + cmInfo.Stage = common.StageInstall + + instResInfos := []*plan.InstallableResourceInfo{hookInfo, &postHookInfo, cmInfo} + + result := newReleaseInstallResult("myrelease", "mynamespace", 1, helmrelease.StatusDeployed, instResInfos) + + require.Len(t, result.Resources, 2) + assert.Same(t, hookInfo.LocalResource.ResourceSpec, result.Resources[0]) + assert.Equal(t, "myhook", result.Resources[0].Name) + assert.Equal(t, "mycm", result.Resources[1].Name) +} + +func TestAI_NewReleaseInstallResultKeepsResourceSpecData(t *testing.T) { + releaseAnnotations := map[string]string{ + "meta.helm.sh/release-name": "myrelease", + "meta.helm.sh/release-namespace": "mynamespace", + } + + instResInfos := []*plan.InstallableResourceInfo{ + newTestInstallableResourceInfo("v1", "ConfigMap", "mycm", "mynamespace", "mynamespace", "mychart/templates/configmap.yaml", common.StoreAsRegular, releaseAnnotations), + newTestInstallableResourceInfo("v1", "ConfigMap", "othercm", "othernamespace", "mynamespace", "mychart/templates/other.yaml", common.StoreAsRegular, nil), + } + + result := newReleaseInstallResult("myrelease", "mynamespace", 1, helmrelease.StatusDeployed, instResInfos) + + require.Len(t, result.Resources, 2) + + assert.Same(t, instResInfos[0].LocalResource.ResourceSpec, result.Resources[0]) + assert.Equal(t, "mychart/templates/configmap.yaml", result.Resources[0].FilePath) + assert.Equal(t, releaseAnnotations, result.Resources[0].Annotations) + assert.Equal(t, "myrelease", result.Resources[0].Unstruct.GetAnnotations()["meta.helm.sh/release-name"]) + + assert.Empty(t, result.Resources[0].Namespace) + assert.Equal(t, "othernamespace", result.Resources[1].Namespace) +} + +func TestAI_NewReleaseInstallResultSortsResources(t *testing.T) { + instResInfos := []*plan.InstallableResourceInfo{ + newTestInstallableResourceInfo("v1", "Service", "mysvc", "mynamespace", "mynamespace", "mychart/templates/service.yaml", common.StoreAsRegular, nil), + newTestInstallableResourceInfo("v1", "ConfigMap", "mycm", "mynamespace", "mynamespace", "mychart/templates/configmap.yaml", common.StoreAsRegular, nil), + newTestInstallableResourceInfo("batch/v1", "Job", "myhook", "mynamespace", "mynamespace", "mychart/templates/hook.yaml", common.StoreAsHook, nil), + newTestInstallableResourceInfo("apiextensions.k8s.io/v1", "CustomResourceDefinition", "mycrd", "", "mynamespace", "mychart/crds/mycrd.yaml", common.StoreAsNone, nil), + } + + result := newReleaseInstallResult("myrelease", "mynamespace", 1, helmrelease.StatusDeployed, instResInfos) + + require.Len(t, result.Resources, 4) + + assert.Equal(t, common.StoreAsNone, result.Resources[0].StoreAs) + assert.Equal(t, "mycrd", result.Resources[0].Name) + + assert.Equal(t, common.StoreAsHook, result.Resources[1].StoreAs) + assert.Equal(t, "myhook", result.Resources[1].Name) + + assert.Equal(t, common.StoreAsRegular, result.Resources[2].StoreAs) + assert.Equal(t, "mycm", result.Resources[2].Name) + + assert.Equal(t, common.StoreAsRegular, result.Resources[3].StoreAs) + assert.Equal(t, "mysvc", result.Resources[3].Name) +} + +func TestAI_NewReleaseInstallResultWithoutResources(t *testing.T) { + result := newReleaseInstallResult("myrelease", "mynamespace", 7, helmrelease.StatusSkipped, nil) + + require.NotNil(t, result) + assert.Equal(t, "v1", result.APIVersion) + assert.Empty(t, result.Resources) + + require.NotNil(t, result.Release) + assert.Equal(t, "myrelease", result.Release.Name) + assert.Equal(t, "mynamespace", result.Release.Namespace) + assert.Equal(t, 7, result.Release.Revision) + assert.Equal(t, helmrelease.StatusSkipped, result.Release.Status) +} + func TestAI_ReleaseInstall_ClosesProgressReportChannelOnEarlyError(t *testing.T) { reportCh := make(chan progrep.ProgressReport, 1) - err := ReleaseInstall(context.Background(), "rel", "ns", ReleaseInstallOptions{ + _, err := ReleaseInstall(context.Background(), "rel", "ns", ReleaseInstallOptions{ AutoRollback: true, LegacyProgressReportCh: reportCh, TempDirPath: t.TempDir(), @@ -311,3 +399,30 @@ func newForbiddenErr(resource, name string) error { func newNotFoundErr(resource, name string) error { return apierrors.NewNotFound(schema.GroupResource{Resource: resource}, name) } + +func newTestInstallableResourceInfo(apiVersion, kind, name, namespace, releaseNamespace, filePath string, storeAs common.StoreAs, annotations map[string]string) *plan.InstallableResourceInfo { + unstruct := &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": apiVersion, + "kind": kind, + "metadata": map[string]interface{}{ + "name": name, + "namespace": namespace, + }, + }, + } + + if len(annotations) > 0 { + unstruct.SetAnnotations(annotations) + } + + resSpec := spec.NewResourceSpec(unstruct, releaseNamespace, spec.ResourceSpecOptions{ + FilePath: filePath, + StoreAs: storeAs, + }) + + return &plan.InstallableResourceInfo{ + ResourceMeta: resSpec.ResourceMeta, + LocalResource: &resource.InstallableResource{ResourceSpec: resSpec}, + } +}