diff --git a/.gitignore b/.gitignore index c9fffc88..0bb3d8d7 100644 --- a/.gitignore +++ b/.gitignore @@ -82,6 +82,7 @@ pom.xml.versionsBackup target/ lexer/target service/target +service/unittests.txt model/target application-*.properties diff --git a/model/swagger.yml b/model/swagger.yml index 9f0839c8..6a0fad1e 100644 --- a/model/swagger.yml +++ b/model/swagger.yml @@ -13,7 +13,7 @@ info: tags: - name: 1. all products - description: search any classe of product and resolve product's identifiers + description: search any classes of product and resolve product's identifiers - name: 2. product references description: explore the product hierarchy - name: 3. by product classes @@ -233,7 +233,27 @@ paths: - $ref: "#/components/parameters/Limit" - $ref: "#/components/parameters/Sort" - $ref: "#/components/parameters/SearchAfter" - + + /products/{identifier}/tools: + get: + tags: + - 1. all products + summary: | + for the given product, returns deep links to all tools that support the display of that product + operationId: product-cross-links + responses: + "200": + $ref: "#/components/responses/Singular" + "400": + $ref: "#/components/responses/Error" + "404": + $ref: "#/components/responses/Error" + "500": + $ref: "#/components/responses/Error" + "501": + $ref: "#/components/responses/Error" + parameters: + - $ref: "#/components/parameters/Identifier" /properties: get: @@ -462,9 +482,9 @@ components: description: | syntax: fields=field1,field2,... - behavior: this parameter and the headder Accept: type determine what content is packaged for the result. While the types application/csv, application/kvp+json, and text/csv return only the fields requesteted, all of the other types have a minimal set of fields that must be returned. Duplicating a minimally required field in this parameter has not effect. The types vnd.nasa.pds.pds4+json and vnd.nasa.pds.pds4+xml have a complete set of fields that must be returned; meaning this parameter does not impact their content. When fields is not used, then the minimal set of fields, or all when minimal is an empty set, is returned. + behavior: this parameter and the header Accept: type determine what content is packaged for the result. While the types application/csv, application/kvp+json, and text/csv return only the fields requested, all of the other types have a minimal set of fields that must be returned. Duplicating a minimally required field in this parameter has not effect. The types vnd.nasa.pds.pds4+json and vnd.nasa.pds.pds4+xml have a complete set of fields that must be returned; meaning this parameter does not impact their content. When fields is not used, then the minimal set of fields, or all when minimal is an empty set, is returned. - notes: the blob fields are blocked unless specifically requrested and only for the */csv and application/kvp+csv types. + notes: the blob fields are blocked unless specifically requested and only for the */csv and application/kvp+csv types. required: false schema: type: array @@ -498,11 +518,11 @@ components: description: | syntax: lidvid or lid - behavior (lid): returns one or more items whose lid matches this lid exactly. If the endpoint ends with the identifier or /latest then a signle result is returned and it is the highest version. If the endpoint ends with /all then all versions of the lid are returned. + behavior (lid): returns one or more items whose lid matches this lid exactly. If the endpoint ends with the identifier or /latest then a single result is returned and it is the highest version. If the endpoint ends with /all then all versions of the lid are returned. behavior (lidvid): returns one and only one item whose lidvid matches this lidvid exactly. - note: the current lid/lidvid resolution will match all the lids that start with lid. In other words, it acts like a glob of foobar*. It behavesn this way from first character to the last + note: the current lid/lidvid resolution will match all the lids that start with lid. In other words, it acts like a glob of foobar*. It behaves this way from first character to the last note: simple sorting of the lidvid is being done to select the latest from the end of the list. However, the versions 1.0, 2.0, and 13.0 will sort to 1.0, 13.0, and 2.0 so the end of the list may not be the latest. required: true @@ -846,7 +866,7 @@ components: $ref: '#/components/schemas/metadata' properties: type: object - description: propertie's values extracted from pds4 model, as a dictionnary, syntax for property name is {namespace}:{property}[{namespace}:{property}]* + description: properties' values extracted from pds4 model, as a dictionary, syntax for property name is {namespace}:{property}[{namespace}:{property}]* additionalProperties: $ref: '#/components/schemas/propertyArrayValues' xml: diff --git a/service/src/main/java/gov/nasa/pds/api/registry/controllers/ProductsController.java b/service/src/main/java/gov/nasa/pds/api/registry/controllers/ProductsController.java index 8980e1ce..45c2a417 100644 --- a/service/src/main/java/gov/nasa/pds/api/registry/controllers/ProductsController.java +++ b/service/src/main/java/gov/nasa/pds/api/registry/controllers/ProductsController.java @@ -2,6 +2,7 @@ import java.lang.reflect.InvocationTargetException; import java.io.IOException; +import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.HashMap; @@ -15,6 +16,7 @@ import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.HttpStatus; +import org.springframework.http.HttpHeaders; import org.springframework.http.ResponseEntity; import org.springframework.stereotype.Controller; import org.springframework.web.context.request.RequestContextHolder; @@ -32,10 +34,11 @@ import gov.nasa.pds.api.registry.model.api_responses.RawMultipleProductResponse; import gov.nasa.pds.api.registry.model.api_responses.WyriwygBusinessObject; import gov.nasa.pds.api.registry.model.identifiers.PdsProductIdentifier; +import gov.nasa.pds.api.registry.model.tools.CrossLinksLoader; +import gov.nasa.pds.api.registry.model.tools.CrossLinks; import gov.nasa.pds.api.registry.search.RegistrySearchRequestBuilder; - @Controller public class ProductsController implements ProductsApi { @@ -46,6 +49,8 @@ public class ProductsController implements ProductsApi { private final ObjectMapper objectMapper; private OpenSearchClient openSearchClient; private SearchRequest presetSearchRequest; + private CrossLinksLoader crossLinksLoader; + private CrossLinks crossLinks; // TODO move that at a better place, it is not specific to this controller private static Map> formatters = @@ -84,6 +89,11 @@ public ProductsController(ConnectionContext connectionContext, this.connectionContext = connectionContext; this.openSearchClient = this.connectionContext.getOpenSearchClient(); + /** + * Initialize CrossLinksLoader for it to load the cross-links configuration on-load + */ + this.crossLinksLoader = new CrossLinksLoader(); + this.crossLinks = this.crossLinksLoader.loadConfiguration(); } private ResponseEntity formatSingleProduct(HashMap product, @@ -186,7 +196,6 @@ public ResponseEntity selectByLidvid(String identifier, List fie } - @Override public ResponseEntity selectByLidvidLatest(String identifier, List fields) throws UnhandledException, NotFoundException, AcceptFormatNotSupportedException { @@ -204,6 +213,31 @@ public ResponseEntity selectByLidvidLatest(String identifier, List productCrossLinks(String identifier) + throws UnhandledException, NotFoundException, AcceptFormatNotSupportedException { + + // Get product metadata that we're cross-linking between services + HashMap product = new HashMap<>(); + + List fields = new ArrayList<>(); + + try { + PdsProductIdentifier pdsIdentifier = PdsProductIdentifier.fromString(identifier); + + if (pdsIdentifier.isLidvid()) { + product = this.getLidVid(pdsIdentifier, fields); + } else { + product = this.getLatestLidVid(pdsIdentifier, fields); + } + } catch (IOException | OpenSearchException e) { + throw new UnhandledException(e); + } + + // Return all the crossLinks for it + return new ResponseEntity(this.crossLinks.getLinks(product), new HttpHeaders(), HttpStatus.OK); + } + @Override public ResponseEntity selectByLidvidAll(String identifier, List fields, Integer limit, List sort, List searchAfter) throws UnhandledException, @@ -271,6 +305,7 @@ private HashMap getLidVid(PdsProductIdentifier identifier, List< // https://stackoverflow.com/questions/2390662/java-how-do-i-get-a-class-literal-from-a-generic-type SearchResponse searchResponse = this.openSearchClient.search(searchRequest, HashMap.class); + if (searchResponse.hits().total().value() == 0) { throw new NotFoundException("No product found with identifier " + identifier.toString()); } @@ -331,6 +366,4 @@ private RawMultipleProductResponse getAllLidVid(PdsProductIdentifier identifier, } - - } diff --git a/service/src/main/java/gov/nasa/pds/api/registry/model/EntityProduct.java b/service/src/main/java/gov/nasa/pds/api/registry/model/EntityProduct.java index c0aee1a3..4a0db0bb 100644 --- a/service/src/main/java/gov/nasa/pds/api/registry/model/EntityProduct.java +++ b/service/src/main/java/gov/nasa/pds/api/registry/model/EntityProduct.java @@ -112,20 +112,20 @@ public List getRef_lid_target() { return ref_lid_target; } - private static String getFirstElmentOrNull(List l) { + private static String getFirstElementOrNull(List l) { return l != null ? l.get(0) : null; } public String getPDS4FileRef() { - return EntityProduct.getFirstElmentOrNull(this.pds4FileReference); + return EntityProduct.getFirstElementOrNull(this.pds4FileReference); } public String getStartDateTime() { - return EntityProduct.getFirstElmentOrNull(this.start_date_time); + return EntityProduct.getFirstElementOrNull(this.start_date_time); } public String getStopDateTime() { - return EntityProduct.getFirstElmentOrNull(this.stop_date_time); + return EntityProduct.getFirstElementOrNull(this.stop_date_time); } public List getModificationDate() { diff --git a/service/src/main/java/gov/nasa/pds/api/registry/model/tools/CrossLinks.java b/service/src/main/java/gov/nasa/pds/api/registry/model/tools/CrossLinks.java new file mode 100644 index 00000000..fc84412a --- /dev/null +++ b/service/src/main/java/gov/nasa/pds/api/registry/model/tools/CrossLinks.java @@ -0,0 +1,359 @@ +package gov.nasa.pds.api.registry.model.tools; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonMappingException; +import com.fasterxml.jackson.databind.JsonNode; + +/** + * This class solves the problem: given this lidvid, what are all the + * PDS tools and services that can display this product and what are + * their deep-links? + * + * This class provides a core method, getLinks(product), to map a product's + * metadata fields into the available deep-links. + * + * The inner datatypes and classes are for storing a jackson/json representation + * of a cross-link configuration loaded by ./CrossLinksLoader.java + * + * @see gov.nasa.pds.api.registry.controllers.ProductsController#productCrossLinks(String identifier) + * @see gov.nasa.pds.api.registry.model.tools.CrossLinksLoader + * @author tariqksoliman + */ +public class CrossLinks { + /** + * Unused. cross-links.json. injectableParams is present in the cross-links.json configuration + * merely as a convenience so that writers of it can quickly see which + * parameters get {param} replaced in the base urls. + */ + private String[] injectableParams; + public void setInjectableParams(String[] i) { + this.injectableParams = i; + } + public String[] getInjectableParams() { + return injectableParams; + } + + /** + * cross-links.json. List of tools configured for cross linking. + */ + private List tools; + public void setTools(List t) { + this.tools = t; + } + public List getTools() { + return tools; + } + + /** + * Gets a value from a product's opensearch document. + * Accounts for values like "[hello,world]" and will return the first + * element of that parsed-into array. + * + * @param product Map of product's opensearch document/metadata + * @param field Which field to get + * @return Value of product[field]. If it's an array, returns first element + */ + public String get(HashMap product, String field) { + Object obj = product.get(field); + String value = obj != null ? obj.toString() : null; + if( value != null ) { + String array[] = value.replace("[", "").replace("]", "").split(","); + return array[0]; + } + return ""; + } + + /** + * Core method. Given a product's opensearch document/metadata, + * and given the current pre-loaded cross-links configuration, + * returns deep-links to all tools that support showing that product. + * + * @param product Map of product's opensearch document/metadata + * @param field Which field to get + * @return [{ tooL: , description: , link: }, ...] + */ + public Object getLinks(HashMap product) { + + /** + * First pull out all the supported metadata from the product + */ + Map values = new HashMap<>(); + String lidvid = this.get(product, "lidvid"); + String filename = this.get(product, "pds:File/pds:file_name"); + + // Parse out file extension injectable params from the filename + int filenameLastIndexOf = -1; + String filenameWithoutFileExtension = ""; + String fileExtension = ""; + if( filename != null ) { + filenameLastIndexOf = filename.lastIndexOf('.'); + if( filenameLastIndexOf > -1 ) { + filenameWithoutFileExtension = filename.substring(0, filenameLastIndexOf); + fileExtension = filename.substring(filenameLastIndexOf + 1); + } + } + + // Populate the values Map so that we can iterate it later + values.put("lid", this.get(product, "lid")); + values.put("vid", this.get(product, "vid")); + values.put("lidvid", lidvid); + values.put("mission", this.get(product, "pds:Investigation_Area/pds:name")); + values.put("spacecraft", this.get(product, "pds:Observing_System/pds:name")); + values.put("bundle", lidvid.split(":")[3]); + values.put("collection", lidvid.split(":")[4]); + values.put("target", this.get(product, "pds:Target_Identification/pds:name")); + values.put("filename", filename); + values.put("filenameWithoutFileExtension", filenameWithoutFileExtension); + values.put("fileExtension", fileExtension); + values.put("fileRef", this.get(product, "ops:Data_File_Info/ops:file_ref")); + values.put("productClass", this.get(product, "product_class")); + values.put("productType", this.get(product, "msn:Mission_Information/msn:product_type_name")); + values.put("nodeName", this.get(product, "ops:Harvest_Info/ops:node_name")); + + List> response = new ArrayList<>(); + + /** + * For each tool from the configuration, if possible, form a link for it + */ + for (Tool t : getTools()) { + Map linkObj = formToolLink(t, values, product); + if( linkObj != null ) { + response.add(linkObj); + } + } + return response; + } + + /** + * Helper method for getLinks(product) + * + * @see #getLinks(product) + * + * @param t A tool's cross-links configuration + * @param values key/values of all injectable params. + * @param product Map of product's opensearch document/metadata + * @return { tooL: , description: , link: } + */ + private Map formToolLink(Tool t, Map values, HashMap product) { + + Map l = new HashMap<>(); + l.put("tool", t.getName()); + l.put("description", t.getDescription()); + + /** + * getBase() gets the base url template from the tool + * + * For example: "https://pds-imaging.jpl.nasa.gov/beta/record?lidvid={lidvid}&mission={mission}" + */ + String link = t.getBase(); + + /** + * Just because a tool is configured, it does not mean that is always has + * a deep-link for a given lidvid. We'll accept or reject is as needed below + */ + boolean accept = false; + boolean reject = false; + + // Iterate each field: lid, vid, lidvid, mission, spacecraft... + for (String field : values.keySet()) { + String value = (String) values.get(field); + + // Aliases + /** + * aliases are a tool's cross-link configuration that allows mappings to different values. + * + * For example: "Mars2020" may be the value that gets used in the registry for the mission name + * but a deep-link may require "m20" instead. Thus an alias like the following would support that: + * { + * "field": "mission", + * "alias": "m20", + * "from": ["mars_2020", "m2020", "mars2020", "Mars2020"] + * } + */ + for( Alias a : t.getAliases()) { + // Only map an alias if the current field matches the alias field + if( a.getField().equals(field) ) { + // Only map if the current value exists in "from" + if( a.getFrom().contains(value) ) { + // If so use the alias + value = a.getAlias(); + // Note that we do not break here, multiple aliases can be applied to the same field + } + } + } + + // AcceptOnly + /** + * acceptOnly are a tool's cross-link configuration that specifies a hard requirement to be accepted + * as a working deep-link. + * + * For example: { "field": "mission", "match": "m20" } would accept if values[mission] == "m20" + * + * Note that this works after aliases are applied. + * Note the acceptOnly is ORed together -- only one acceptOnly needs to match -- not all of them + */ + for( Accept a : t.getAcceptOnly()) { + if( a.getField().equals(field) ) { + if( value.matches(a.getMatch()) ) { + accept = true; + } + } + } + // If acceptOnly is an empty List, accept regardless. + if( t.getAcceptOnly().size() == 0 ) { + accept = true; + } + + // Reject + /** + * Works just like acceptOnly but if any single match is found, the + * tool is rejected from being deep-linked to. + */ + for( Reject a : t.getReject()) { + if( a.getField().equals(field) ) { + if( value.matches(a.getMatch()) ) { + reject = true; + } + } + } + + // Continually replace the next {field} for the final link + link = link.replaceAll("\\{" + field + "\\}", value); + + } + + l.put("link", link); + + if( accept == true && reject == false ) { + return l; + } + + // If it's null, the caller won't add it + return null; + + } + + private static class Tool { + private String name; + public void setName(String n) { + this.name = n; + } + public String getName() { + return name; + } + + private String base; + public void setBase(String b) { + this.base = b; + } + public String getBase() { + return base; + } + + private String description; + public void setDescription(String d) { + this.description = d; + } + public String getDescription() { + return description; + } + + private List aliases; + public void setAliases(List a) { + this.aliases = a; + } + public List getAliases() { + return aliases; + } + + + private List acceptOnly; + public void setAcceptOnly(List a) { + this.acceptOnly = a; + } + public List getAcceptOnly() { + return acceptOnly; + } + + private List reject; + public void setReject(List r) { + this.reject = r; + } + public List getReject() { + return reject; + } + } + + private static class Alias { + private String field; + public void setField(String f) { + this.field = f; + } + public String getField() { + return field; + } + + private String alias; + public void setAlias(String a) { + this.alias = a; + } + public String getAlias() { + return alias; + } + + private List from; + public void setFrom(List f) { + this.from = f; + } + public List getFrom() { + return from; + } + + } + + private static class Accept { + private String field; + public void setField(String f) { + this.field = f; + } + public String getField() { + return field; + } + + private String match; + public void setMatch(String m) { + this.match = m; + } + public String getMatch() { + return match; + } + + } + + private static class Reject { + private String field; + public void setField(String f) { + this.field = f; + } + public String getField() { + return field; + } + + private String match; + public void setMatch(String m) { + this.match = m; + } + public String getMatch() { + return match; + } + + } +} + + diff --git a/service/src/main/java/gov/nasa/pds/api/registry/model/tools/CrossLinksLoader.java b/service/src/main/java/gov/nasa/pds/api/registry/model/tools/CrossLinksLoader.java new file mode 100644 index 00000000..1de33721 --- /dev/null +++ b/service/src/main/java/gov/nasa/pds/api/registry/model/tools/CrossLinksLoader.java @@ -0,0 +1,45 @@ +package gov.nasa.pds.api.registry.model.tools; + +import java.io.File; +import java.io.IOException; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonMappingException; +import com.fasterxml.jackson.databind.JsonNode; + +/** + * This is a wrapper class for ./CrossLinks.java. Use this loader to do a + * one-time pull of CrossLink's cross-links.json configuration file. + * + * Use like: + * this.crossLinksLoader = new CrossLinksLoader(); + * this.crossLinks = this.crossLinksLoader.loadConfiguration(); + * + * @see gov.nasa.pds.api.registry.controllers.ProductsController#Constructor(ConnectionContext connectionContext, ErrorMessageFactory errorMessageFactory, ObjectMapper objectMapper) + * @see gov.nasa.pds.api.registry.controllers.ProductsController#productCrossLinks(String identifier) + * @see gov.nasa.pds.api.registry.model.tools.CrossLinks + * @author tariqksoliman + */ +public class CrossLinksLoader { + private CrossLinks config; + + public CrossLinks loadConfiguration() { + if (config == null) { + ObjectMapper mapper = new ObjectMapper(); + try { + config = mapper.readValue(new File("src/main/resources/cross-links.json"), CrossLinks.class); + System.out.println(config); + } catch (IOException e) { + // Handle exception + System.out.println(e); + } + } + return config; + } +} \ No newline at end of file diff --git a/service/src/main/resources/cross-links.json b/service/src/main/resources/cross-links.json new file mode 100644 index 00000000..5e3d6e9a --- /dev/null +++ b/service/src/main/resources/cross-links.json @@ -0,0 +1,52 @@ +{ + "injectableParams": [ + "vid", + "lid", + "lidvid", + "mission", + "spacecraft", + "bundle", + "collection", + "target", + "filename", + "filenameWithoutFileExtension", + "fileExtension", + "fileRef", + "productClass", + "productType", + "nodeName" + ], + "tools": [ + { + "name": "analysts_notebook", + "base": "https://an.rsl.wustl.edu/{mission}/AN/an3.aspx?it=B1&ii={filenameWithoutFileExtension}", + "description": "A tool for accessing the science data archives from NASA landed Mars and lunar missions.", + "aliases": [ + { + "field": "mission", + "alias": "m20", + "from": ["mars_2020", "m2020", "mars2020", "Mars2020"] + }, + { + "field": "mission", + "alias": "ins", + "from": ["InSight"] + } + ], + "acceptOnly": [ + { "field": "mission", "match": "m20" }, + { "field": "mission", "match": "ins" }, + { "field": "fileExtension", "match": "\\.IMG" } + ], + "reject": [{ "field": "lidvid", "match": ".*mars2020_helicam.*" }] + }, + { + "name": "atlas4", + "base": "https://pds-imaging.jpl.nasa.gov/beta/record?lidvid={lidvid}", + "description": "The Official Image Search of the PDS Cartography and Imaging Sciences Node (PDSIMG)", + "aliases": [], + "acceptOnly": [], + "reject": [] + } + ] +}