diff --git a/db-connector.go b/db-connector.go index 3f96e5d0..043b6159 100755 --- a/db-connector.go +++ b/db-connector.go @@ -14077,11 +14077,10 @@ func SetDatastoreKeyBulk(ctx context.Context, allKeys []CacheKeyData) ([]Datasto } if len(cacheData.Enrichments) > 0 && len(cacheData.Value) == 0 { - if debug { + if debug { log.Printf("[DEBUG] Having enrichments with empty value doesn't make sense, skipping enrichments for key %s in category %s", cacheData.Key, cacheData.Category) } - cacheData.Value = config.Value } @@ -14102,7 +14101,7 @@ func SetDatastoreKeyBulk(ctx context.Context, allKeys []CacheKeyData) ([]Datasto } cacheData.Enrichments = newObservables - if debug { + if debug { log.Printf("Found new enrichments: %d for key %s in category %s", len(newObservables), cacheData.Key, cacheData.Category) os.Exit(3) } @@ -14187,7 +14186,7 @@ func SetDatastoreKeyBulk(ctx context.Context, allKeys []CacheKeyData) ([]Datasto if len(cacheData.Enrichments) == 0 && len(config.Enrichments) > 0 { cacheData.Enrichments = config.Enrichments - } + } if len(cacheData.Tags) == 0 { cacheData.Tags = config.Tags diff --git a/health.go b/health.go index 226a83e9..efa98d31 100644 --- a/health.go +++ b/health.go @@ -4,10 +4,10 @@ import ( "bytes" "context" "encoding/base64" + // "encoding/json" "errors" "fmt" - "github.com/goccy/go-json" "io" "io/ioutil" "log" @@ -19,6 +19,8 @@ import ( "strings" "time" + "github.com/goccy/go-json" + "github.com/Masterminds/semver" "github.com/frikky/kin-openapi/openapi3" uuid "github.com/satori/go.uuid" diff --git a/shared.go b/shared.go index 1d25650b..3d821b02 100644 --- a/shared.go +++ b/shared.go @@ -873,14 +873,14 @@ func HandleGetOrgs(resp http.ResponseWriter, request *http.Request) { return } - if project.Environment == "cloud" { - gceProject := os.Getenv("SHUFFLE_GCEPROJECT") - if gceProject != "shuffler" && gceProject != sandboxProject && len(gceProject) > 0 { - log.Printf("[DEBUG] Redirecting GET ORGS request to main site handler (shuffler.io)") - RedirectUserRequest(resp, request) - return - } - } + // if project.Environment == "cloud" { + // gceProject := os.Getenv("SHUFFLE_GCEPROJECT") + // if gceProject != "shuffler" && gceProject != sandboxProject && len(gceProject) > 0 { + // log.Printf("[DEBUG] Redirecting GET ORGS request to main site handler (shuffler.io)") + // RedirectUserRequest(resp, request) + // return + // } + // } user, err := HandleApiAuthentication(resp, request) if err != nil { @@ -947,15 +947,15 @@ func HandleGetOrg(resp http.ResponseWriter, request *http.Request) { return } - // Checking if it's a special region. All user-specific requests should - if project.Environment == "cloud" { - gceProject := os.Getenv("SHUFFLE_GCEPROJECT") - if gceProject != "shuffler" && gceProject != sandboxProject && len(gceProject) > 0 { - log.Printf("[DEBUG] Redirecting GET ORG request to main site handler (shuffler.io)") - RedirectUserRequest(resp, request) - return - } - } + // // Checking if it's a special region. All user-specific requests should + // if project.Environment == "cloud" { + // gceProject := os.Getenv("SHUFFLE_GCEPROJECT") + // if gceProject != "shuffler" && gceProject != sandboxProject && len(gceProject) > 0 { + // log.Printf("[DEBUG] Redirecting GET ORG request to main site handler (shuffler.io)") + // RedirectUserRequest(resp, request) + // return + // } + // } var fileId string location := strings.Split(request.URL.String(), "/") @@ -1436,14 +1436,14 @@ func HandleGetSubOrgs(resp http.ResponseWriter, request *http.Request) { // Checking if it's a special region. All user-specific requests should // go through shuffler.io and not subdomains - if project.Environment == "cloud" { - gceProject := os.Getenv("SHUFFLE_GCEPROJECT") - if gceProject != "shuffler" && gceProject != sandboxProject && len(gceProject) > 0 { - log.Printf("[DEBUG] Redirecting GET SUBORG request to main site handler (shuffler.io)") - RedirectUserRequest(resp, request) - return - } - } + // if project.Environment == "cloud" { + // gceProject := os.Getenv("SHUFFLE_GCEPROJECT") + // if gceProject != "shuffler" && gceProject != sandboxProject && len(gceProject) > 0 { + // log.Printf("[DEBUG] Redirecting GET SUBORG request to main site handler (shuffler.io)") + // RedirectUserRequest(resp, request) + // return + // } + // } ctx := GetContext(request) user, err := HandleApiAuthentication(resp, request) @@ -5888,17 +5888,17 @@ func HandleUpdateUser(resp http.ResponseWriter, request *http.Request) { return } - if project.Environment == "cloud" { - // Checking if it's a special region. All user-specific requests should - // go through shuffler.io and not subdomains - - gceProject := os.Getenv("SHUFFLE_GCEPROJECT") - if gceProject != "shuffler" && gceProject != sandboxProject && len(gceProject) > 0 { - log.Printf("[DEBUG] Redirecting Update User request to main site handler (shuffler.io)") - RedirectUserRequest(resp, request) - return - } - } + // if project.Environment == "cloud" { + // // Checking if it's a special region. All user-specific requests should + // // go through shuffler.io and not subdomains + + // gceProject := os.Getenv("SHUFFLE_GCEPROJECT") + // if gceProject != "shuffler" && gceProject != sandboxProject && len(gceProject) > 0 { + // log.Printf("[DEBUG] Redirecting Update User request to main site handler (shuffler.io)") + // RedirectUserRequest(resp, request) + // return + // } + // } userInfo, err := HandleApiAuthentication(resp, request) if err != nil { @@ -9989,15 +9989,15 @@ func HandleGetUsers(resp http.ResponseWriter, request *http.Request) { return } - if project.Environment == "cloud" { - // Checking if it's a special region. All user-specific requests should - gceProject := os.Getenv("SHUFFLE_GCEPROJECT") - if gceProject != "shuffler" && gceProject != sandboxProject && len(gceProject) > 0 { - log.Printf("[DEBUG] Redirecting Get Users request to main site handler (shuffler.io)") - RedirectUserRequest(resp, request) - return - } - } + // if project.Environment == "cloud" { + // // Checking if it's a special region. All user-specific requests should + // gceProject := os.Getenv("SHUFFLE_GCEPROJECT") + // if gceProject != "shuffler" && gceProject != sandboxProject && len(gceProject) > 0 { + // log.Printf("[DEBUG] Redirecting Get Users request to main site handler (shuffler.io)") + // RedirectUserRequest(resp, request) + // return + // } + // } user, err := HandleApiAuthentication(resp, request) if err != nil { @@ -12496,34 +12496,34 @@ func HandleChangeUserOrg(resp http.ResponseWriter, request *http.Request) { log.Printf("[AUDIT] Api authentication failed in change org (local): %s", userErr) } - if project.Environment == "cloud" { - // Checking if it's a special region. All user-specific requests should - // Clean up the users' cache for different parts - gceProject := os.Getenv("SHUFFLE_GCEPROJECT") - if gceProject != "shuffler" && gceProject != sandboxProject && len(gceProject) > 0 { - - DeleteCache(ctx, fmt.Sprintf("%s_workflows", user.Id)) - DeleteCache(ctx, fmt.Sprintf("apps_%s", user.Id)) - DeleteCache(ctx, fmt.Sprintf("user_%s", user.Username)) - DeleteCache(ctx, fmt.Sprintf("user_%s", user.Id)) - DeleteCache(ctx, fmt.Sprintf("%s", user.ApiKey)) - DeleteCache(ctx, fmt.Sprintf("Users_%s", user.ApiKey)) - DeleteCache(ctx, fmt.Sprintf("session_%s", user.Session)) - - log.Printf("[DEBUG] Redirecting ORGCHANGE request to main site handler (shuffler.io)") - RedirectUserRequest(resp, request) - - DeleteCache(ctx, fmt.Sprintf("%s_workflows", user.Id)) - DeleteCache(ctx, fmt.Sprintf("apps_%s", user.Id)) - DeleteCache(ctx, fmt.Sprintf("user_%s", user.Username)) - DeleteCache(ctx, fmt.Sprintf("user_%s", user.Id)) - DeleteCache(ctx, fmt.Sprintf("Users_%s", user.ApiKey)) - DeleteCache(ctx, fmt.Sprintf("%s", user.ApiKey)) - DeleteCache(ctx, fmt.Sprintf("session_%s", user.Session)) - - return - } - } + // if project.Environment == "cloud" { + // // Checking if it's a special region. All user-specific requests should + // // Clean up the users' cache for different parts + // gceProject := os.Getenv("SHUFFLE_GCEPROJECT") + // if gceProject != "shuffler" && gceProject != sandboxProject && len(gceProject) > 0 { + + // DeleteCache(ctx, fmt.Sprintf("%s_workflows", user.Id)) + // DeleteCache(ctx, fmt.Sprintf("apps_%s", user.Id)) + // DeleteCache(ctx, fmt.Sprintf("user_%s", user.Username)) + // DeleteCache(ctx, fmt.Sprintf("user_%s", user.Id)) + // DeleteCache(ctx, fmt.Sprintf("%s", user.ApiKey)) + // DeleteCache(ctx, fmt.Sprintf("Users_%s", user.ApiKey)) + // DeleteCache(ctx, fmt.Sprintf("session_%s", user.Session)) + + // log.Printf("[DEBUG] Redirecting ORGCHANGE request to main site handler (shuffler.io)") + // RedirectUserRequest(resp, request) + + // DeleteCache(ctx, fmt.Sprintf("%s_workflows", user.Id)) + // DeleteCache(ctx, fmt.Sprintf("apps_%s", user.Id)) + // DeleteCache(ctx, fmt.Sprintf("user_%s", user.Username)) + // DeleteCache(ctx, fmt.Sprintf("user_%s", user.Id)) + // DeleteCache(ctx, fmt.Sprintf("Users_%s", user.ApiKey)) + // DeleteCache(ctx, fmt.Sprintf("%s", user.ApiKey)) + // DeleteCache(ctx, fmt.Sprintf("session_%s", user.Session)) + + // return + // } + // } if userErr != nil { resp.WriteHeader(401) @@ -12864,25 +12864,25 @@ func HandleCreateSubOrg(resp http.ResponseWriter, request *http.Request) { DeleteCache(ctx, fmt.Sprintf("user_%s", inneruser.Username)) DeleteCache(ctx, fmt.Sprintf("user_%s", inneruser.Id)) - // Added another to ensure we handle empty cursor DeleteCache(ctx, fmt.Sprintf("%s__childorgs", inneruser.ActiveOrg.Id)) - DeleteCache(ctx, fmt.Sprintf("%s_childorgs", inneruser.ActiveOrg.Id)) } + DeleteCache(ctx, fmt.Sprintf("%s__childorgs", parentOrg.Id)) + // Delete parent org cache as well from the org region DeleteCache(ctx, fmt.Sprintf("Organizations_%s", parentOrg.Id)) // Checking if it's a special region. All user-specific requests should // go through shuffler.io and not subdomains - if project.Environment == "cloud" { - gceProject := os.Getenv("SHUFFLE_GCEPROJECT") - if gceProject != "shuffler" && gceProject != sandboxProject && len(gceProject) > 0 { - log.Printf("[DEBUG] Redirecting Create Suborg request to main site handler (shuffler.io)") + // if project.Environment == "cloud" { + // gceProject := os.Getenv("SHUFFLE_GCEPROJECT") + // if gceProject != "shuffler" && gceProject != sandboxProject && len(gceProject) > 0 { + // log.Printf("[DEBUG] Redirecting Create Suborg request to main site handler (shuffler.io)") - RedirectUserRequest(resp, request) - return - } - } + // RedirectUserRequest(resp, request) + // return + // } + // } body, err := ioutil.ReadAll(request.Body) if err != nil { @@ -13049,7 +13049,6 @@ func HandleCreateSubOrg(resp http.ResponseWriter, request *http.Request) { Id: orgId, }) - DeleteCache(ctx, fmt.Sprintf("%s_childorgs", parentOrg.Id)) DeleteCache(ctx, fmt.Sprintf("Organizations_%s", parentOrg.Id)) err = SetOrg(ctx, *parentOrg, parentOrg.Id) @@ -13093,7 +13092,6 @@ func HandleCreateSubOrg(resp http.ResponseWriter, request *http.Request) { newOrg.Users = append(newOrg.Users, loopUser) } - DeleteCache(ctx, fmt.Sprintf("%s_childorgs", newOrg.Id)) err = SetOrg(ctx, newOrg, newOrg.Id) if err != nil { log.Printf("[WARNING] Failed setting new org %s: %s", newOrg.Id, err) @@ -13342,15 +13340,15 @@ func HandleEditOrg(resp http.ResponseWriter, request *http.Request) { // Checking if it's a special region. All user-specific requests should // go through shuffler.io and not subdomains - if project.Environment == "cloud" { - gceProject := os.Getenv("SHUFFLE_GCEPROJECT") - if gceProject != "shuffler" && gceProject != sandboxProject && len(gceProject) > 0 { - log.Printf("[DEBUG] Redirecting Edit Org request to main site handler (shuffler.io)") + // if project.Environment == "cloud" { + // gceProject := os.Getenv("SHUFFLE_GCEPROJECT") + // if gceProject != "shuffler" && gceProject != sandboxProject && len(gceProject) > 0 { + // log.Printf("[DEBUG] Redirecting Edit Org request to main site handler (shuffler.io)") - RedirectUserRequest(resp, request) - return - } - } + // RedirectUserRequest(resp, request) + // return + // } + // } user, err := HandleApiAuthentication(resp, request) if err != nil { @@ -16427,7 +16425,6 @@ func HandleLogin(resp http.ResponseWriter, request *http.Request) { newCookie.Domain = ".shuffler.io" http.SetCookie(resp, newCookie) } - loginData = fmt.Sprintf(`{"success": true, "cookies": [{"key": "session_token", "value": "%s", "expiration": %d}], "region_url": "%s"}`, userdata.Session, expiration.Unix(), regionUrl) newData, err := json.Marshal(returnValue) @@ -20239,6 +20236,8 @@ func HandleListCacheKeys(resp http.ResponseWriter, request *http.Request) { var orgId string category := "" location := strings.Split(request.URL.String(), "/") + + log.Printf("I am here 1") if location[1] == "api" { if len(location) <= 4 { log.Printf("Path too short: %d", len(location)) @@ -20267,7 +20266,7 @@ func HandleListCacheKeys(resp http.ResponseWriter, request *http.Request) { if orgOk && len(orgQuery) > 0 { orgId = orgQuery[0] } - + log.Printf("I am here 2") if usererr != nil { if len(category) == 0 || category == "default" { log.Printf("[WARNING] No category provided in request. Returning 400.") @@ -20294,6 +20293,8 @@ func HandleListCacheKeys(resp http.ResponseWriter, request *http.Request) { } } + log.Printf("I am here 3 with orgId %s and category %s", orgId, category) + ctx := GetContext(request) org, err := GetOrg(ctx, orgId) if err != nil { @@ -20323,7 +20324,7 @@ func HandleListCacheKeys(resp http.ResponseWriter, request *http.Request) { isSuccess := true if keyList, keyOk := request.URL.Query()["key"]; keyOk && len(keyList) > 0 { key := keyList[0] - + log.Printf("I am here 4") cacheId := fmt.Sprintf("%s_%s", user.ActiveOrg.Id, key) if len(category) > 0 { cacheId = fmt.Sprintf("%s_%s_%s", user.ActiveOrg.Id, key, category) @@ -20340,7 +20341,7 @@ func HandleListCacheKeys(resp http.ResponseWriter, request *http.Request) { *cacheItem, } } else { - + log.Printf("I am here 5") if debug { log.Printf("[DEBUG] Looking for keys in org %s and category %s", org.Id, category) } @@ -20351,11 +20352,15 @@ func HandleListCacheKeys(resp http.ResponseWriter, request *http.Request) { } } + log.Printf("retrived keys are: %+v", keys) + // This is NOT required unless automation/other config is set. foundCategories := []string{} categoryConfig := &DatastoreCategoryUpdate{} + log.Printf("I am here 6") if len(category) > 0 && category != "default" { foundCategories = append(foundCategories, category) + log.Printf("I am here 7") categoryConfig, err = GetDatastoreCategoryConfig(ctx, org.Id, category) if err != nil { //if debug { @@ -20363,6 +20368,7 @@ func HandleListCacheKeys(resp http.ResponseWriter, request *http.Request) { //} } } else { + log.Printf("I am here 8") allCategories, err := GetDatastoreCategories(ctx, org.Id) if err == nil { for _, cat := range allCategories { @@ -20387,6 +20393,7 @@ func HandleListCacheKeys(resp http.ResponseWriter, request *http.Request) { } } + log.Printf("I am here 9 with found categories: %#v", foundCategories) if orgId != user.ActiveOrg.Id { if !categoryConfig.Settings.Public { sourceExecution, sourceExecutionOk := request.URL.Query()["execution_id"] @@ -20421,6 +20428,7 @@ func HandleListCacheKeys(resp http.ResponseWriter, request *http.Request) { } } + log.Printf("I am here 10") // Cleanup just in case categoryConfig = &DatastoreCategoryUpdate{} for keyIndex, _ := range keys { @@ -20449,10 +20457,11 @@ func HandleListCacheKeys(resp http.ResponseWriter, request *http.Request) { Categories: foundCategories, } + log.Printf("I am here 11 with new return: %#v", len(newReturn.Keys)) outputTypeList, outputTypeOk := request.URL.Query()["type"] if outputTypeOk && len(outputTypeList) > 0 { outputType := outputTypeList[0] - + log.Printf("I am here 12 with output type %s", outputType) if outputType == "ndjson" || outputType == "csv" || outputType == "raw" { outputString := "" for _, key := range newReturn.Keys { @@ -20473,6 +20482,7 @@ func HandleListCacheKeys(resp http.ResponseWriter, request *http.Request) { return } else if outputType == "values" || outputType == "json" { + log.Printf("I am here 13 with output type %s", outputType) newOutput := []string{} for _, key := range newReturn.Keys { if len(key.Value) == 0 { @@ -20523,6 +20533,7 @@ func HandleListCacheKeys(resp http.ResponseWriter, request *http.Request) { } } + log.Printf("I am here 14 with new return amount %d and total amount %d", newReturn.Amount, newReturn.TotalAmount) categoryCount, err := GetCacheKeyCount(ctx, orgId, category) if err != nil { log.Printf("[WARNING] Failed to get cache key count for org %s: %s", org.Id, err) @@ -20869,7 +20880,7 @@ func HandleDeleteCacheKeyPost(resp http.ResponseWriter, request *http.Request) { cacheId := fmt.Sprintf("%s_%s", selectedOrg, tmpData.Key) cacheData, err := GetDatastoreKey(ctx, cacheId, tmpData.Category) if err != nil || len(cacheData.Key) == 0 { - log.Printf("[ERROR] Failed to DELETE cache key '%s' for org %s (delete) in category '%s'. Does it exist?", tmpData.Key, tmpData.OrgId, tmpData.Category) + log.Printf("[ERROR] Failed to DELETE cache key '%s' for org %s (delete) in category '%s' with error: %v. Does it exist?", tmpData.Key, tmpData.OrgId, tmpData.Category, err) resp.WriteHeader(400) result := ResultChecker{ @@ -21401,10 +21412,10 @@ func HandleSetDatastoreKey(resp http.ResponseWriter, request *http.Request) { } tmpData = append(tmpData, CacheKeyData{ - OrgId: tmpDataOverride.OrgId, - Key: tmpDataOverride.Key, - Category: tmpDataOverride.Category, - Tags: tmpDataOverride.Tags, + OrgId: tmpDataOverride.OrgId, + Key: tmpDataOverride.Key, + Category: tmpDataOverride.Category, + Tags: tmpDataOverride.Tags, Enrichments: tmpDataOverride.Enrichments, Value: parsedValue, @@ -22771,7 +22782,7 @@ func HandleRetValidation(ctx context.Context, workflowExecution WorkflowExecutio } } - if debug { + if debug { log.Printf("[DEBUG][%s] Single action execution check finished. Result len: %d, Errors: %#v", workflowExecution.ExecutionId, len(returnBody.Result), returnBody.Errors) } @@ -32494,14 +32505,14 @@ func HandleDeleteOrg(resp http.ResponseWriter, request *http.Request) { // Checking if it's a special region. All user-specific requests should // go through shuffler.io and not subdomains - if project.Environment == "cloud" { - gceProject := os.Getenv("SHUFFLE_GCEPROJECT") - if gceProject != "shuffler" && gceProject != sandboxProject && len(gceProject) > 0 { - log.Printf("[DEBUG] Redirecting DELETE ORG request to main site handler (shuffler.io)") - RedirectUserRequest(resp, request) - return - } - } + // if project.Environment == "cloud" { + // gceProject := os.Getenv("SHUFFLE_GCEPROJECT") + // if gceProject != "shuffler" && gceProject != sandboxProject && len(gceProject) > 0 { + // log.Printf("[DEBUG] Redirecting DELETE ORG request to main site handler (shuffler.io)") + // RedirectUserRequest(resp, request) + // return + // } + // } var fileId string location := strings.Split(request.URL.String(), "/") @@ -32659,7 +32670,7 @@ func HandleDeleteOrg(resp http.ResponseWriter, request *http.Request) { parentOrg.ChildOrgs = newChildOrg - suborgCacheKey := fmt.Sprintf("%s_childorgs", parentOrg.Id) + suborgCacheKey := fmt.Sprintf("%s__childorgs", parentOrg.Id) DeleteCache(ctx, suborgCacheKey) DeleteCache(ctx, fmt.Sprintf("Organizations_%s", subOrg.Id)) parentOrg.SyncUsage.MultiTenant.Counter = int64(len(newChildOrg)) + 1