draft changes to cs-api for DB based provider

tx/cs/cs-api.js

@@ -499,13 +499,45 @@ class CodeSystemProvider {
   async doesFilter(prop, op, value) { return false; }
 
   /**
-   * gets a single context in which filters will be evaluated. The application doesn't make use of this context;
-   * it's only use is to be passed back to the CodeSystem provider so it can make use of it - if it wants
+   * @return true if the cs provider handles excludes when building filters. If true, and the value set is a clean include+exclude,
+   * the handleExclude will be called between getPrepContext and executeFilters
+   */
+  handlesExcludes() {
+    return false;
+  }
+
+  handlesOffset() {
+
+  }
+  /**
+   * gets a single context in which filters will be evaluated. The server doesn't doesn't make use of this context;
+   * it's only use is to be passed back to the CodeSystem provider so it can make use of it to organise the filter process
+   *
+   * The function is passed several pieces of information about the use of the filters that can help it optimise the
+   * behaviour:
+   *   - iterate: whether the value set is being expanded, or instead that membership is just being checked (expand vs validate-code).
+   *       But note, though, that when iterating, only the first filter set (see executeFilters) will be iterated - the rest will
+   *       have filterCheck called
+   *   - excludeInactive: whether to exclude inactive codes from the results. Note that the expand worker will check this anyway,
+   *       so it can be ignored, but it's more efficient to never return inactive codes if they're going to be ignored
+   *   - params: a handle to the parameters passed from the client. The provider doesn't need to do anything because of these
+   *       but it might decide how to optimise loading based on e.g languages, properties, designations, etc. The server will
+   *       reprocess these anyway, so it can be ignored, but again, efficiency
+   *   - offset & count: if the user is paging through the expansion, their offset and count request. Note that if the
+   *       provider does anything with these, it needs to return true from handlesOffset() so the expand worker doesn't try
+   *       to reprocess the offset and count. Note that there is information in the params about offset and count, but
+   *       the provider should ignore these, as it only gets to check offset and count when the conditions are correct
    *
    * @param {boolean} iterate true if the conceptSets that result from this will be iterated, and false if they'll be used to locate a single code
-   * @returns {FilterExecutionContext} filter (or null, it no use for this)
-   * */
-  async getPrepContext(iterate) { return new FilterExecutionContext(iterate); }
+   * @param {TxParameters} params: information from the request that the user made, to help optimise loading
+   * @param {boolean} excludeInactive: whether the server will use inactive codes or not
+   * @param {int} offset if handlesOffset() and !iterate, and if the value set is a simple one that only uses this provider, then this is the applicable offset. -1 if not applicable
+   * @param {int} count if handlesOffset() and !iterate, and if the value set is a simple one that only uses this provider, then this is the applicable count. -1 if not applicable
+   * @returns {FilterExecutionContext} filter
+   *
+   **/
+  async getPrepContext(iterate, params, excludeInactive, offset = -1, count = -1) { return new FilterExecutionContext(iterate); }
+
 
   /**
    * executes a text search filter (whatever that means) and returns a FilterConceptSet
@@ -532,7 +564,7 @@ class CodeSystemProvider {
   } // ? must override?
 
   /**
-   * Get a FilterConceptSet for a value set filter
+   * inform the CS provider about a filter
    *
    * throws an exception if the search filter can't be handled
    *
@@ -543,6 +575,28 @@ class CodeSystemProvider {
    **/
   async filter(filterContext, prop, op, value) { throw new Error("Must override"); } // well, only if any filters are actually supported
 
+  /**
+   * if handlesExcludes(), then inform the CS provider about an applicable set of exclude filters
+   *
+   * this might be called more than once. For each iteration, all of the filters apply
+   *
+   * the objects each have prop, op, and value.
+   *
+   * throws an exception if the search filter can't be handled
+   *
+   * @param {FilterExecutionContext} filterContext filtering context
+   * @param {Object[]} filters
+   **/
+  async filterExcludeFilters(filterContext, filters) { throw new Error("Must override"); } // well, only if any filters are actually supported
+
+  /**
+   * if handlesExcludes(), then inform the CS provider about an applicable set of excluded codes
+   *
+   * @param {FilterExecutionContext} }filterContext - filter context
+   * @param {String[]} code list of codes to exclude
+   */
+  async filterExcludeConcepts(filterContext, code) { throw new Error("Must override"); } // well, only if any filters are actually supported
+
   /**
    * called once all the filters have been handled, and iteration is about to happen.
    * this function returns one more filters. If there were multiple filters, but only

tx/workers/expand.js

@@ -601,7 +601,7 @@ class ValueSetExpander {
     }
   }
 
-  async checkSource(cset, exp, filter, srcURL, ts) {
+  async checkSource(cset, exp, filter, srcURL, ts, vsInfo) {
     this.worker.deadCheck('checkSource');
     Extensions.checkNoModifiers(cset, 'ValueSetExpander.checkSource', 'set');
     let imp = false;
@@ -628,6 +628,10 @@ class ValueSetExpander {
       if (cs == null) {
         // nothing
       } else {
+        if (vsInfo && vsInfo.isSimple) {
+          vsInfo.csDoExcludes = cs.handlesExcludes();
+          vsInfo.csDoOffset = cs.handlesOffset();
+        }
         if (cs.contentMode() !== 'complete') {
           if (cs.contentMode() === 'not-present') {
             throw new Issue('error', 'business-rule', null, null, 'The code system definition for ' + cset.system + ' has no content, so this expansion cannot be performed', 'invalid');
@@ -660,7 +664,7 @@ class ValueSetExpander {
     }
   }
 
-  async includeCodes(cset, path, vsSrc, filter, expansion, excludeInactive, notClosed) {
+  async includeCodes(cset, path, vsSrc, compose, filter, expansion, excludeInactive, notClosed, vsInfo) {
     this.worker.deadCheck('processCodes#1');
     const valueSets = [];
 
@@ -752,6 +756,7 @@ class ValueSetExpander {
             }
             const prep = await cs.getPrepContext(true);
             const ctxt = await cs.searchFilter(prep, filter, false);
+            await cs.filterExclude(prep, )
             let set = await cs.executeFilters(prep);
             this.worker.opContext.log('iterate filters');
             while (await cs.filterMore(ctxt, set)) {
@@ -799,10 +804,22 @@ class ValueSetExpander {
         if (cset.filter) {
           this.worker.opContext.log('prepare filters');
           const fcl = cset.filter;
-          const prep = await cs.getPrepContext(true);
+          const prep = await cs.getPrepContext(true,
+            this.params, excludeInactive, vsInfo.csDoOffset ? this.offset : -1, cs.handlesOffset() && vsInfo.csDoExcludes ? this.count : -1);
+
           if (!filter.isNull) {
             await cs.searchFilter(filter, prep, true);
           }
+          if (vsInfo.csDoExcludes) {
+            for (let exc of compose.exclude || []) {
+              if (exc.filter) {
+                await cs.filterExcludeFilters(prep, this.excludeFilterList(exc));
+              }
+              if (exc.concept) {
+                await cs.filterExcludeConcepts(prep, exc.concept.map(c => c.code));
+              }
+            }
+          }
 
           if (cs.specialEnumeration()) {
             Extensions.addString(expansion, "http://hl7.org/fhir/StructureDefinition/valueset-unclosed", 'The code System "' + cs.system() + " has a grammar and so has infinite members. This extension is based on " + cs.specialEnumeration());
@@ -1106,32 +1123,34 @@ class ValueSetExpander {
     }
   }
 
-  async handleCompose(source, filter, expansion, notClosed) {
+  async handleCompose(source, filter, expansion, notClosed, vsInfo) {
     this.worker.opContext.log('compose #1');
 
     const ts = new Map();
     for (const c of source.jsonObj.compose.include || []) {
       this.worker.deadCheck('handleCompose#2');
-      await this.checkSource(c, expansion, filter, source.url, ts);
+      await this.checkSource(c, expansion, filter, source.url, ts, vsInfo);
     }
     for (const c of source.jsonObj.compose.exclude || []) {
       this.worker.deadCheck('handleCompose#3');
       this.hasExclusions = true;
-      await this.checkSource(c, expansion, filter, source.url, ts);
+      await this.checkSource(c, expansion, filter, source.url, ts, null);
     }
 
     this.worker.opContext.log('compose #2');
 
-    let i = 0;
-    for (const c of source.jsonObj.compose.exclude || []) {
-      this.worker.deadCheck('handleCompose#4');
-      await this.excludeCodes(c, "ValueSet.compose.exclude["+i+"]", source, filter, expansion, this.excludeInactives(source), notClosed);
+    if (!vsInfo.csDoExcludes) {
+      let i = 0;
+      for (const c of source.jsonObj.compose.exclude || []) {
+        this.worker.deadCheck('handleCompose#4');
+        await this.excludeCodes(c, "ValueSet.compose.exclude["+i+"]", source, source.jsonObj.compose, filter, expansion, this.excludeInactives(source), notClosed);
+      }
     }
 
-    i = 0;
+    let i = 0;
     for (const c of source.jsonObj.compose.include || []) {
       this.worker.deadCheck('handleCompose#5');
-      await this.includeCodes(c, "ValueSet.compose.include["+i+"]", source, filter, expansion, this.excludeInactives(source), notClosed);
+      await this.includeCodes(c, "ValueSet.compose.include["+i+"]", source, filter, expansion, this.excludeInactives(source), notClosed, vsInfo);
       i++;
     }
   }
@@ -1259,10 +1278,11 @@ class ValueSetExpander {
 
     let notClosed = { value :  false};
 
+    let vsInfo = this.scanValueSet(source.jsonObj.compose);
     try {
       if (source.jsonObj.compose && Extensions.checkNoModifiers(source.jsonObj.compose, 'ValueSetExpander.Expand', 'compose')
           && this.worker.checkNoLockedDate(source.url, source.jsonObj.compose)) {
-        await this.handleCompose(source, filter, exp, notClosed);
+        await this.handleCompose(source, filter, exp, notClosed, vsInfo);
       }
 
       const unused = new Set([...this.requiredSupplements].filter(s => !this.usedSupplements.has(s)));
@@ -1338,7 +1358,7 @@ class ValueSetExpander {
         const c = list[i];
         if (this.map.has(this.keyC(c))) {
           o++;
-          if (o > this.offset && (this.count < 0 || t < this.count)) {
+          if ((vsInfo.csDoOffset) || (o > this.offset && (this.count < 0 || t < this.count))) {
             t++;
             if (!exp.contains) {
               exp.contains = [];
@@ -1533,6 +1553,48 @@ class ValueSetExpander {
     return undefined;
   }
 
+  /**
+   * we have a look at the value set compose to see what we have.
+   * If it's all one code system(|version), and has no value set dependencies,
+   * then we call it simple - this will affect how it can be handled later
+   *
+   * @param compose
+   * @returns {undefined}
+   */
+  scanValueSet(compose) {
+    let result = { isSimple : false, hasExcludes : true, csset : new Set(), csDoExcludes : false, csDoOffset : false};
+    let simple = true;
+    for (let inc of compose.include) {
+      if (!this.isSimpleInclude(inc, result.csset, false)) {
+        simple = false;
+      }
+    }
+    for (let exc of compose.exclude) {
+      if (!this.isSimpleInclude(exc, result.csset, true)) {
+        simple = false;
+      }
+      result.hasExcludes = true;
+    }
+    if (simple && result.csset.size == 1) {
+      result.isSimple = true;
+    }
+    return result;
+  }
+
+  isSimpleInclude(inc, set, isExclude) {
+    set.add(inc.system+"|"+inc.version);
+    return (!inc.valueset || inc.valueset.length == 0) && ((inc.filter && inc.filter.length > 0) || (isExclude && inc.concept && inc.filter.concept > 0));
+  }
+
+  excludeFilterList(exc) {
+    const results = [];
+
+    for (const f of exc.filter || []) {
+      results.push({ prop: f.property, op: f.op, value: f.value });
+    }
+
+    return results;
+  }
 }
 
 class ExpandWorker extends TerminologyWorker {