From bd184b30d96690013727b18c70e14771647a2502 Mon Sep 17 00:00:00 2001 From: "LaCivita, Jeremy" Date: Thu, 31 Mar 2022 17:13:09 -0400 Subject: [PATCH 1/2] added util to generate firebolt specification --- src/cli.mjs | 27 ++++--- util/specification/index.mjs | 136 +++++++++++++++++++++++++++++++++++ 2 files changed, 155 insertions(+), 8 deletions(-) create mode 100644 util/specification/index.mjs diff --git a/src/cli.mjs b/src/cli.mjs index b1e944b5..6d218b86 100755 --- a/src/cli.mjs +++ b/src/cli.mjs @@ -5,8 +5,10 @@ import docs from '../util/docs/index.mjs' import validate from '../util/validate/index.mjs' import openrpc from '../util/openrpc/index.mjs' import declarations from '../util/declarations/index.mjs' +import specification from '../util/specification/index.mjs' import nopt from 'nopt' import path from 'path' +import { sign } from 'crypto' const knownOpts = { 'task': [String, null], @@ -15,7 +17,9 @@ const knownOpts = { 'output': [path], 'shared-schemas': [path], 'as-path': Boolean, - 'static-modules': String + 'static-modules': String, + 'base': [path], + 'sdk': [String, Array] } const shortHands = { 't': '--task', @@ -25,8 +29,11 @@ const shortHands = { 'o': '--output', 'ap': '--as-path', 'sm': '--static-modules', - 'ss': '--shared-schemas' + 'ss': '--shared-schemas', + 'b': '--base', + 'k': '--sdk' } + // Last 2 arguments are the defaults. const parsedArgs = nopt(knownOpts, shortHands, process.argv, 2) const signOff = () => console.log('This has been a presentation of Firebolt \u{1F525} \u{1F529}') @@ -34,19 +41,23 @@ const signOff = () => console.log('This has been a presentation of Firebolt \u{1 const util = parsedArgs.task if (util === 'sdk') { - sdk(parsedArgs).done(signOff) + sdk(parsedArgs).done(signOff) } else if (util === 'docs') { - docs(parsedArgs).done(signOff) + docs(parsedArgs).done(signOff) } else if (util === 'validate') { - validate(parsedArgs).done(signOff) + validate(parsedArgs).done(signOff) } else if (util === 'openrpc') { - openrpc(parsedArgs).done(signOff) + openrpc(parsedArgs).done(signOff) } else if (util === 'declarations') { - declarations(parsedArgs).done(signOff) -} else { + declarations(parsedArgs).done(signOff) +} +else if (util === 'specification') { + specification(parsedArgs).done(signOff) +} +else { console.log("Invalid build type") } \ No newline at end of file diff --git a/util/specification/index.mjs b/util/specification/index.mjs new file mode 100644 index 00000000..a26e39cf --- /dev/null +++ b/util/specification/index.mjs @@ -0,0 +1,136 @@ +import fs from 'fs' + +const run = ({ + base: base, + sdk: sdk, + output: output + }) => { + + console.log(base) + console.log(sdk) + console.log(output) + +// const base = base //JSON.parse(fs.readFileSync('src/json/firebolt-specification-base.json')) + + const sdk_names = { + 'core': 'node_modules/@firebolt-js/sdk/dist/firebolt-open-rpc.json', + 'manage': 'node_modules/@firebolt-js/manage/dist/firebolt-open-rpc.json', + 'discovery': 'node_modules/@firebolt-js/discovery/dist/firebolt-open-rpc.json' + } + + const sdks = { + methods: [] + } + + Object.keys(sdk_names).map(name => { + const sdk = JSON.parse(fs.readFileSync(sdk_names[name])) + + // make sure there are no duplicate methods across SDKs + sdk.methods.forEach(item => { + if (!item.name.startsWith('rpc.')) { + if (sdks.methods.find(existingItem => existingItem.name === item.name)) { + throw new Error(`Error: '${name}' SDK has a duplicate method '${item.name}.`) + } + } + }) + + sdks.methods.push(...sdk.methods.filter(m => !m.name.startsWith('rpc.'))) + }) + + base = JSON.parse(fs.readFileSync(base)) + + const getTag = (method, tag) => method.tags ? method.tags.find(t => t.name == tag) || {} : {} + const getCapabilities = (method, role) => getTag(method, 'capabilities') ? getTag(method, 'capabilities')['x-' + role] || [] : [] + const hasAnyCapabilities = method => hasCapabilities(method, 'uses') || hasCapabilities(method, 'provides') || hasCapabilities(method, 'manages') + const hasCapabilities = (method, role) => getCapabilities(method, role).length > 0 + const hasNoCapabilities = method => !hasAnyCapabilities(method) + const isValidCapability = capability => base.capabilities.find(c => c.id === capability) + const getInvalidCapabilities = method => getCapabilities(method, 'uses').filter(c => !isValidCapability(c)).concat(getCapabilities(method, 'provides').filter(c => !isValidCapability(c))).concat(getCapabilities(method, 'manages').filter(c => !isValidCapability(c))) + const hasInvalidCapabilities = method => (getInvalidCapabilities(method).length > 0) + const isPrivateCapability = (capability, role) => !(base.capabilities.find(c => c.id === capability) || {use: { public: true}, manage: { public: true }, provide: { public: true }})[role].public + const getPrivateCapabilities = (method) => getCapabilities(method, 'uses').filter(c => isPrivateCapability(c, 'use')).concat(getCapabilities(method, 'provides').filter(c => isPrivateCapability(c, 'provide'))).concat(getCapabilities(method, 'manages').filter(c => isPrivateCapability(c, 'manage'))) + const hasPrivateCapabilities = (method) => (getPrivateCapabilities(method).length > 0) + + let problems + let errorCount = 0 + let warningCount = 0 + + problems = sdks.methods.filter(hasNoCapabilities) + warningCount += problems.length + if (problems.length) + console.warn('\nThe following methods have no capabilities tag: \n\t- ' + problems.map(m => m.name).join('\n\t- ')) + + problems = sdks.methods.filter(hasInvalidCapabilities) + errorCount += problems.length + if (problems.length) + console.error('\nThe following methods have invalid capabilities: \n\t- ' + problems.map(m => m.name + `: ${getInvalidCapabilities(m).join(', ')}`).join('\n\t- ')) + + problems = sdks.methods.filter(hasPrivateCapabilities) + errorCount += problems.length + if (problems.length) + console.error('\nThe following methods have private capabilities: \n\t- ' + problems.map(m => m.name + `: ${getPrivateCapabilities(m).join(', ')}`).join('\n\t- ')) + + if (errorCount > 0) { + console.warn(`\nFound ${errorCount} problems`) + } + else { + const specification = JSON.parse(JSON.stringify(base)) + specification.apis = sdks.methods.filter(hasAnyCapabilities).map( method => { + const methodInfo = { + "method": method.name, + "type": "firebolt" + } + + if (hasCapabilities(method, 'uses')) { + methodInfo.uses = getCapabilities(method, 'uses') + } + + if (hasCapabilities(method, 'provides')) { + methodInfo.provides = getCapabilities(method, 'provides') + } + + if (hasCapabilities(method, 'manages')) { + methodInfo.manages = getCapabilities(method, 'manages') + } + + return methodInfo + }) + + const orderedCompare = (a, b, order) => order.indexOf(a) - order.indexOf(b) + const typeCompare = (a ,b) => orderedCompare(a.type, b.type, ['firebolt', 'external', 'w3c']) + + const levelCompare = (a ,b) => { + const result = orderedCompare(a.level, b.level, ['must', 'should', 'could']) + if (result === 0) { + return a.id.localeCompare(b.id) + } + else { + return result + } + } + const capabilityCompare = (a, b) => { + const result = typeCompare(a, b) + + if (result === 0) { + return levelCompare(a, b) + } + else { + result + } + } + + specification.capabilities.sort( capabilityCompare ) + specification.apis.sort( (a, b) => a.method.localeCompare(b.method) ) + + fs.writeFileSync(output, JSON.stringify(specification, null, '\t')) + console.log('\nWrote Firebolt Specification:\n' + output) + } + + console.log('\n') + + return { + done: callback => callback() + } +} + +export default run \ No newline at end of file From be99aedf336798de3d31cbeb76449615eabcd264 Mon Sep 17 00:00:00 2001 From: "LaCivita, Jeremy" Date: Wed, 13 Apr 2022 12:43:45 -0400 Subject: [PATCH 2/2] Support merging `apis` --- util/shared/helpers.mjs | 41 ++++++++++++++++++++---------------- util/specification/index.mjs | 29 ++++++++++++++++++++----- util/validate/index.mjs | 2 +- 3 files changed, 48 insertions(+), 24 deletions(-) diff --git a/util/shared/helpers.mjs b/util/shared/helpers.mjs index 52f6afd8..f10aa5b1 100644 --- a/util/shared/helpers.mjs +++ b/util/shared/helpers.mjs @@ -52,24 +52,29 @@ const logHeader = message => console.log(`\x1b[0m\x1b[7m\x1b[32m${message}\x1b[0 // TODO: Convert to "stream" style fs functions const recursiveFileDirectoryList = dirOrFile => { return h((push, next) => { - fs.stat(dirOrFile, (err, stat) => { - if (!stat || stat.isFile()) { - stat && push(err, dirOrFile) - push(null, h.nil) - } else { - // Add the directory itself to the ouput stream. - push(null, dirOrFile) - fs.readdir(dirOrFile, (_err, files) => { - next(h(files) - .map(file => { - file = path.join(dirOrFile, file) - return recursiveFileDirectoryList(file) - }) - .merge() - ) - }) - } - }) + if (!dirOrFile) { + push(null, h.nil) + } + else { + fs.stat(dirOrFile, (err, stat) => { + if (!stat || stat.isFile()) { + stat && push(err, dirOrFile) + push(null, h.nil) + } else { + // Add the directory itself to the ouput stream. + push(null, dirOrFile) + fs.readdir(dirOrFile, (_err, files) => { + next(h(files) + .map(file => { + file = path.join(dirOrFile, file) + return recursiveFileDirectoryList(file) + }) + .merge() + ) + }) + } + }) + } }) } diff --git a/util/specification/index.mjs b/util/specification/index.mjs index a26e39cf..b349ef5f 100644 --- a/util/specification/index.mjs +++ b/util/specification/index.mjs @@ -75,7 +75,8 @@ const run = ({ } else { const specification = JSON.parse(JSON.stringify(base)) - specification.apis = sdks.methods.filter(hasAnyCapabilities).map( method => { + specification.apis = specification.apis || [] + specification.apis.push(...sdks.methods.filter(hasAnyCapabilities).map( method => { const methodInfo = { "method": method.name, "type": "firebolt" @@ -94,10 +95,19 @@ const run = ({ } return methodInfo - }) + })) const orderedCompare = (a, b, order) => order.indexOf(a) - order.indexOf(b) - const typeCompare = (a ,b) => orderedCompare(a.type, b.type, ['firebolt', 'external', 'w3c']) + const typeCompare = (a ,b) => { + if (a.isExtension && b.isExtension) + return 0 + else if (a.isExtension) + return 1 + else if (b.isExtension) + return -1 + else + return 0 + } const levelCompare = (a ,b) => { const result = orderedCompare(a.level, b.level, ['must', 'should', 'could']) @@ -115,12 +125,21 @@ const run = ({ return levelCompare(a, b) } else { - result + return result } } + const methodCompare = (a, b) => { + const result = orderedCompare(a.type, b.type, ['firebolt', 'w3c', 'extension']) + + if (result === 0) + return a.method.localeCompare(b.method) + else + return result + } + specification.capabilities.sort( capabilityCompare ) - specification.apis.sort( (a, b) => a.method.localeCompare(b.method) ) + specification.apis.sort( methodCompare ) fs.writeFileSync(output, JSON.stringify(specification, null, '\t')) console.log('\nWrote Firebolt Specification:\n' + output) diff --git a/util/validate/index.mjs b/util/validate/index.mjs index 66c81fcf..be393152 100644 --- a/util/validate/index.mjs +++ b/util/validate/index.mjs @@ -56,7 +56,7 @@ const run = ({ // Set up the ajv instance const ajv = new Ajv() addFormats(ajv) - const errorCounter = 0 + let errorCounter = 0 const combinedSchemas = combineStreamObjects(schemaFetcher(sharedSchemasFolder), schemaFetcher(schemasFolder), schemaFetcher(externalFolder)) const allModules = localModules(modulesFolder, markdownFolder, disableTransforms, false) // Validate private modules