From 1fa6fa89c1a5883d3d2952fbde22aa910d694197 Mon Sep 17 00:00:00 2001 From: siffalo Date: Wed, 29 Jul 2026 06:57:45 -0400 Subject: [PATCH] Update fetch.py We at Crocus LLC have a recommended requirement from South Carolina Department of Education to use lightbeam in an application we're developing for them. It's been working well to date, both sending and fetching data. However, when we get into advanced use cases of the Ed-Fi API via lightbeam, it lacks support for resource endpoint fetch/query parameters by reference, only by name. The most critical fetch/query parameter we need at the moment is minChangeVersion, and of course it's a parameter by reference, not by name, for all resource endpoints. So either we shift away from lightbeam in these advanced use cases, we use our own fork of lightbeam (not clear at the moment whether a custom lightbeam deployment would be acceptable at SCDE), or we try to get lightbeam to include support for query parameters by reference. You know your swagger model better than I and my morning with it, so this could probably be optimized, but just getting a suggested code change to you, if you'd consider support for query parameters by reference. With this change, I can use minChangeVersion successfully, and of course all other query parameters by reference. --- lightbeam/fetch.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/lightbeam/fetch.py b/lightbeam/fetch.py index 02d1bfd..882cb8a 100644 --- a/lightbeam/fetch.py +++ b/lightbeam/fetch.py @@ -34,6 +34,28 @@ async def get_records(self, do_write=True, log_status_counts=True): namespace = self.lightbeam.get_namespace_for_endpoint(endpoint) supported_params = swagger.get("paths", {}).get(f"/{namespace}/{endpoint}", {}).get("get", {}).get("parameters", []) supported_param_names = [ x["name"] for x in supported_params if "name" in x.keys() and "in" in x.keys() and x["in"]=="query" ] + + supported_param_refs = [ x["$ref"] for x in supported_params if "$ref" in x.keys()] + supported_component_params = [] + + for p in supported_param_refs: + + node = {} + + for part in p.split("/"): + if (part == "#"): + continue + if not(node): + node = swagger.get(part, {}) + else: + node = node.get(part, {}) + + if node: + supported_component_params.append(node) + + supported_param_names_by_ref = [ x["name"] for x in supported_component_params if "name" in x.keys() and "in" in x.keys() and x["in"]=="query" ] + supported_param_names = supported_param_names + supported_param_names_by_ref + if not set(params.keys()).issubset(set(supported_param_names)): self.logger.warn(f"Query contains keys that are not params for the endpoint {endpoint}... skipping! (Supported params: {(', '.join(supported_param_names))})") continue