diff --git a/.gitignore b/.gitignore index 0ed53a26a..36ee55c90 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,7 @@ _mytests .idea .zed docs/changelog.md +.generated # Yarn files .yarn/install-state.gz diff --git a/docs/reference.md b/docs/reference.md index 6a56fa1db..e751da934 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -419,22 +419,26 @@ DESCRIPTION Manages runtime data operations inside of a running Actor. SUBCOMMANDS - actor set-value Sets or removes record into the - default key-value store associated with the Actor run. - actor push-data Saves data to Actor's run - default dataset. - actor get-value Gets a value from the default - key-value store associated with the Actor run. - actor get-public-url Get an HTTP URL that allows - public access to a key-value store item. - actor get-input Gets the Actor input value from - the default key-value store associated with the Actor - run. - actor charge Charge for a specific event in - the pay-per-event Actor run. - actor calculate-memory Calculates the Actor’s dynamic - memory usage based on a memory expression from - actor.json, input data, and run options. + actor set-value Sets or removes record + into the default key-value store associated with + the Actor run. + actor push-data Saves data to Actor's + run default dataset. + actor get-value Gets a value from the + default key-value store associated with the Actor + run. + actor get-public-url Get an HTTP URL that + allows public access to a key-value store item. + actor get-input Gets the Actor input + value from the default key-value store associated + with the Actor run. + actor charge Charge for a specific + event in the pay-per-event Actor run. + actor calculate-memory Calculates the Actor’s + dynamic memory usage based on a memory expression + from actor.json, input data, and run options. + actor generate-schema-types Generate TypeScript + types from Actor schemas. ``` ##### `apify actor calculate-memory` @@ -491,6 +495,42 @@ FLAGS charging without actually charging ``` +##### `apify actor generate-schema-types` + +```sh +DESCRIPTION + Generate TypeScript types from Actor schemas. + + Generates types from the input schema and, when no custom path is provided, + also from the dataset schema defined in '.actor/actor.json' under + "storages.dataset". + + Reads the input schema from one of these locations (in priority order): + 1. Object in '.actor/actor.json' under "input" key + 2. JSON file path in '.actor/actor.json' "input" key + 3. .actor/INPUT_SCHEMA.json + 4. INPUT_SCHEMA.json + + Optionally specify custom schema path to use. + +USAGE + $ apify actor generate-schema-types [path] + [--all-optional] [-o ] [--strict] + +ARGUMENTS + path Optional path to the input schema file. If not provided, searches + default locations. + +FLAGS + --all-optional Mark all properties as optional in + generated types. + -o, --output= Directory where the generated files + should be outputted. Defaults to src/.generated/actor/ to + stay within the typical tsconfig rootDir. + --strict Whether generated interfaces should be + strict (no index signature [key: string]: unknown). +``` + ##### `apify actor get-input` ```sh diff --git a/package.json b/package.json index b0e1f5cc2..a0d1c1993 100644 --- a/package.json +++ b/package.json @@ -99,6 +99,7 @@ "istextorbinary": "~9.5.0", "jju": "~1.4.0", "js-levenshtein": "^1.1.6", + "json-schema-to-typescript": "^15.0.4", "lodash.clonedeep": "^4.5.0", "mime": "~4.1.0", "open": "~11.0.0", diff --git a/scripts/generate-cli-docs.ts b/scripts/generate-cli-docs.ts index 5fbb13b41..addf7e913 100644 --- a/scripts/generate-cli-docs.ts +++ b/scripts/generate-cli-docs.ts @@ -34,6 +34,7 @@ const categories: Record = { { command: Commands.actor }, { command: Commands.actorCalculateMemory }, { command: Commands.actorCharge }, + { command: Commands.actorGenerateSchemaTypes }, { command: Commands.actorGetInput }, { command: Commands.actorGetPublicUrl }, { command: Commands.actorGetValue }, diff --git a/src/commands/_register.ts b/src/commands/_register.ts index 61f6f9d63..16657acd8 100644 --- a/src/commands/_register.ts +++ b/src/commands/_register.ts @@ -2,6 +2,7 @@ import type { BuiltApifyCommand } from '../lib/command-framework/apify-command.j import { ActorIndexCommand } from './actor/_index.js'; import { ActorCalculateMemoryCommand } from './actor/calculate-memory.js'; import { ActorChargeCommand } from './actor/charge.js'; +import { ActorGenerateSchemaTypesCommand } from './actor/generate-schema-types.js'; import { ActorGetInputCommand } from './actor/get-input.js'; import { ActorGetPublicUrlCommand } from './actor/get-public-url.js'; import { ActorGetValueCommand } from './actor/get-value.js'; @@ -77,6 +78,7 @@ export const actorCommands = [ ActorGetInputCommand, ActorChargeCommand, ActorCalculateMemoryCommand, + ActorGenerateSchemaTypesCommand, // top-level HelpCommand, diff --git a/src/commands/actor/_index.ts b/src/commands/actor/_index.ts index 6bc63a38f..ecfa66841 100644 --- a/src/commands/actor/_index.ts +++ b/src/commands/actor/_index.ts @@ -1,6 +1,7 @@ import { ApifyCommand } from '../../lib/command-framework/apify-command.js'; import { ActorCalculateMemoryCommand } from './calculate-memory.js'; import { ActorChargeCommand } from './charge.js'; +import { ActorGenerateSchemaTypesCommand } from './generate-schema-types.js'; import { ActorGetInputCommand } from './get-input.js'; import { ActorGetPublicUrlCommand } from './get-public-url.js'; import { ActorGetValueCommand } from './get-value.js'; @@ -21,6 +22,7 @@ export class ActorIndexCommand extends ApifyCommand { ActorGetInputCommand, ActorChargeCommand, ActorCalculateMemoryCommand, + ActorGenerateSchemaTypesCommand, ]; async run() { diff --git a/src/commands/actor/generate-schema-types.ts b/src/commands/actor/generate-schema-types.ts new file mode 100644 index 000000000..c50cf9784 --- /dev/null +++ b/src/commands/actor/generate-schema-types.ts @@ -0,0 +1,283 @@ +import { mkdir, writeFile } from 'node:fs/promises'; +import path from 'node:path'; +import process from 'node:process'; + +import type { JSONSchema4 } from 'json-schema'; +import { compile, type Options } from 'json-schema-to-typescript'; + +import { ApifyCommand } from '../../lib/command-framework/apify-command.js'; +import { Args } from '../../lib/command-framework/args.js'; +import { Flags } from '../../lib/command-framework/flags.js'; +import { CommandExitCodes, LOCAL_CONFIG_PATH } from '../../lib/consts.js'; +import { + readAndValidateInputSchema, + readDatasetSchema, + readOutputSchema, + readStorageSchema, +} from '../../lib/input_schema.js'; +import { error, info, success, warning } from '../../lib/outputs.js'; +import { + clearAllRequired, + makePropertiesRequired, + prepareFieldsSchemaForCompilation, + prepareKvsCollectionsForCompilation, + prepareOutputSchemaForCompilation, + stripTitles, +} from '../../lib/schema-transforms.js'; + +export const BANNER_COMMENT = ` +// biome-ignore-all lint: generated +// biome-ignore-all format: generated +/* eslint-disable */ +/* prettier-ignore-start */ +/* + * This file was automatically generated by json-schema-to-typescript. + * DO NOT MODIFY IT BY HAND. Instead, modify the source JSONSchema file, + * and run apify actor generate-schema-types to regenerate this file. + */ +`; + +export class ActorGenerateSchemaTypesCommand extends ApifyCommand { + static override name = 'generate-schema-types' as const; + + static override hiddenAliases = ['generate-types']; + + static override description = `Generate TypeScript types from Actor schemas. + +Generates types from the input schema and, when no custom path is provided, +also from the Dataset, Output (experimental), and Key-Value Store (experimental) +schemas defined in '${LOCAL_CONFIG_PATH}'. + +Reads the input schema from one of these locations (in priority order): + 1. Object in '${LOCAL_CONFIG_PATH}' under "input" key + 2. JSON file path in '${LOCAL_CONFIG_PATH}' "input" key + 3. .actor/INPUT_SCHEMA.json + 4. INPUT_SCHEMA.json + +Optionally specify custom schema path to use.`; + + static override flags = { + output: Flags.string({ + char: 'o', + description: + 'Directory where the generated files should be outputted. Defaults to src/.generated/actor/ to stay within the typical tsconfig rootDir.', + required: false, + default: path.join('src', '.generated', 'actor'), + }), + strict: Flags.boolean({ + description: 'Whether generated interfaces should be strict (no index signature [key: string]: unknown).', + required: false, + default: true, + }), + 'all-optional': Flags.boolean({ + description: 'Mark all properties as optional in generated types.', + required: false, + default: false, + }), + }; + + static override args = { + path: Args.string({ + required: false, + description: 'Optional path to the input schema file. If not provided, searches default locations.', + }), + }; + + async run() { + const cwd = process.cwd(); + + const { inputSchema } = await readAndValidateInputSchema({ + forcePath: this.args.path, + cwd, + action: 'Generating types from', + }); + + const name = 'input'; + + const schemaToCompile = this.flags.allOptional + ? clearAllRequired(inputSchema) + : makePropertiesRequired(inputSchema); + + const compileOptions: Partial = { + bannerComment: BANNER_COMMENT, + maxItems: -1, + unknownAny: true, + format: true, + additionalProperties: !this.flags.strict, + $refOptions: { resolve: { external: false, file: false, http: false } }, + }; + + const result = await compile(stripTitles(schemaToCompile) as JSONSchema4, name, compileOptions); + + const outputDir = path.resolve(cwd, this.flags.output); + await mkdir(outputDir, { recursive: true }); + + const outputFile = path.join(outputDir, `${name}.ts`); + await writeFile(outputFile, result, 'utf-8'); + + success({ message: `Generated types written to ${outputFile}` }); + + // When no custom path is provided, also generate types from additional schemas + if (!this.args.path) { + const schemaResults = await Promise.allSettled([ + this.generateDatasetTypes({ cwd, outputDir, compileOptions }), + this.generateOutputTypes({ cwd, outputDir, compileOptions }), + this.generateKvsTypes({ cwd, outputDir, compileOptions }), + ]); + + const schemaLabels = ['Dataset', 'Output', 'Key-Value Store']; + let anyFailed = false; + + for (const [i, schemaResult] of schemaResults.entries()) { + if (schemaResult.status === 'rejected') { + anyFailed = true; + error({ + message: `Failed to generate types for ${schemaLabels[i]} schema: ${schemaResult.reason instanceof Error ? schemaResult.reason.message : String(schemaResult.reason)}`, + }); + } + } + + if (anyFailed) { + process.exitCode = CommandExitCodes.BuildFailed; + } + } + } + + private async generateDatasetTypes({ + cwd, + outputDir, + compileOptions, + }: { + cwd: string; + outputDir: string; + compileOptions: Partial; + }) { + const datasetResult = readDatasetSchema({ cwd }); + + if (!datasetResult) { + return; + } + + const { datasetSchema, datasetSchemaPath } = datasetResult; + + if (datasetSchemaPath) { + info({ message: `[experimental] Generating types from Dataset schema at ${datasetSchemaPath}` }); + } else { + info({ message: `[experimental] Generating types from Dataset schema embedded in '${LOCAL_CONFIG_PATH}'` }); + } + + const prepared = prepareFieldsSchemaForCompilation(datasetSchema); + + if (!prepared) { + warning({ message: 'Dataset schema has no fields defined, skipping type generation.' }); + return; + } + + const datasetName = 'dataset'; + + const schemaToCompile = this.flags.allOptional ? clearAllRequired(prepared) : prepared; + + const result = await compile(stripTitles(schemaToCompile) as JSONSchema4, datasetName, compileOptions); + + const outputFile = path.join(outputDir, `${datasetName}.ts`); + await writeFile(outputFile, result, 'utf-8'); + + success({ message: `Generated types written to ${outputFile}` }); + } + + private async generateOutputTypes({ + cwd, + outputDir, + compileOptions, + }: { + cwd: string; + outputDir: string; + compileOptions: Partial; + }) { + const outputResult = readOutputSchema({ cwd }); + + if (!outputResult) { + return; + } + + const { outputSchema, outputSchemaPath } = outputResult; + + if (outputSchemaPath) { + info({ message: `[experimental] Generating types from Output schema at ${outputSchemaPath}` }); + } else { + info({ message: `[experimental] Generating types from Output schema embedded in '${LOCAL_CONFIG_PATH}'` }); + } + + const prepared = prepareOutputSchemaForCompilation(outputSchema); + + if (!prepared) { + warning({ message: 'Output schema has no properties defined, skipping type generation.' }); + return; + } + + const outputName = 'output'; + + const schemaToCompile = this.flags.allOptional ? clearAllRequired(prepared) : prepared; + + const result = await compile(stripTitles(schemaToCompile) as JSONSchema4, outputName, compileOptions); + + const outputFile = path.join(outputDir, `${outputName}.ts`); + await writeFile(outputFile, result, 'utf-8'); + + success({ message: `Generated types written to ${outputFile}` }); + } + + private async generateKvsTypes({ + cwd, + outputDir, + compileOptions, + }: { + cwd: string; + outputDir: string; + compileOptions: Partial; + }) { + const kvsResult = readStorageSchema({ cwd, key: 'keyValueStore', label: 'Key-Value Store' }); + + if (!kvsResult) { + return; + } + + const { schema: kvsSchema, schemaPath: kvsSchemaPath } = kvsResult; + + if (kvsSchemaPath) { + info({ message: `[experimental] Generating types from Key-Value Store schema at ${kvsSchemaPath}` }); + } else { + info({ + message: `[experimental] Generating types from Key-Value Store schema embedded in '${LOCAL_CONFIG_PATH}'`, + }); + } + + const collections = prepareKvsCollectionsForCompilation(kvsSchema); + + if (collections.length === 0) { + warning({ + message: 'Key-Value Store schema has no collections with JSON schemas, skipping type generation.', + }); + return; + } + + const parts: string[] = []; + + for (const { name, schema } of collections) { + const schemaToCompile = this.flags.allOptional ? clearAllRequired(schema) : schema; + + const compiled = await compile(stripTitles(schemaToCompile) as JSONSchema4, name, { + ...compileOptions, + // Only the first collection gets the banner comment + bannerComment: parts.length === 0 ? (compileOptions.bannerComment as string) : '', + }); + + parts.push(compiled); + } + + const outputFile = path.join(outputDir, 'key-value-store.ts'); + await writeFile(outputFile, parts.join('\n'), 'utf-8'); + + success({ message: `Generated types written to ${outputFile}` }); + } +} diff --git a/src/commands/validate-schema.ts b/src/commands/validate-schema.ts index eef2e1b3c..9bb4e556f 100644 --- a/src/commands/validate-schema.ts +++ b/src/commands/validate-schema.ts @@ -1,13 +1,10 @@ import process from 'node:process'; -import { validateInputSchema } from '@apify/input_schema'; - import { ApifyCommand } from '../lib/command-framework/apify-command.js'; import { Args } from '../lib/command-framework/args.js'; import { LOCAL_CONFIG_PATH } from '../lib/consts.js'; -import { readInputSchema } from '../lib/input_schema.js'; -import { info, success } from '../lib/outputs.js'; -import { Ajv2019 } from '../lib/utils.js'; +import { readAndValidateInputSchema } from '../lib/input_schema.js'; +import { success } from '../lib/outputs.js'; export class ValidateInputSchemaCommand extends ApifyCommand { static override name = 'validate-schema' as const; @@ -30,23 +27,12 @@ Optionally specify custom schema path to validate.`; static override hiddenAliases = ['vis']; async run() { - const { inputSchema, inputSchemaPath } = await readInputSchema({ + await readAndValidateInputSchema({ forcePath: this.args.path, cwd: process.cwd(), + action: 'Validating', }); - if (!inputSchema) { - throw new Error(`Input schema has not been found at ${inputSchemaPath}.`); - } - - if (inputSchemaPath) { - info({ message: `Validating input schema stored at ${inputSchemaPath}` }); - } else { - info({ message: `Validating input schema embedded in '${LOCAL_CONFIG_PATH}'` }); - } - - const validator = new Ajv2019({ strict: false }); - validateInputSchema(validator, inputSchema); // This one throws an error in a case of invalid schema. success({ message: 'Input schema is valid.' }); } } diff --git a/src/lib/input_schema.ts b/src/lib/input_schema.ts index a35d21e93..08ed69204 100644 --- a/src/lib/input_schema.ts +++ b/src/lib/input_schema.ts @@ -7,8 +7,8 @@ import deepClone from 'lodash.clonedeep'; import { KEY_VALUE_STORE_KEYS } from '@apify/consts'; import { validateInputSchema } from '@apify/input_schema'; -import { ACTOR_SPECIFICATION_FOLDER } from './consts.js'; -import { warning } from './outputs.js'; +import { ACTOR_SPECIFICATION_FOLDER, LOCAL_CONFIG_PATH } from './consts.js'; +import { info, warning } from './outputs.js'; import { Ajv2019, getJsonFileContent, getLocalConfig, getLocalKeyValueStorePath } from './utils.js'; const DEFAULT_INPUT_SCHEMA_PATHS = [ @@ -70,6 +70,154 @@ export const readInputSchema = async ( }; }; +/** + * Reads and validates input schema, logging appropriate info messages. + * Throws an error if the schema is not found or invalid. + * + * @param options.forcePath - Optional path to force reading from + * @param options.cwd - Current working directory + * @param options.action - Action description for the info message (e.g., "Validating", "Generating types from") + * @returns The validated input schema and its path + */ +export const readAndValidateInputSchema = async ({ + forcePath, + cwd, + action, +}: { + forcePath?: string; + cwd: string; + action: string; +}): Promise<{ inputSchema: Record; inputSchemaPath: string | null }> => { + const { inputSchema, inputSchemaPath } = await readInputSchema({ + forcePath, + cwd, + }); + + if (!inputSchema) { + throw new Error(`Input schema has not been found at ${inputSchemaPath}.`); + } + + if (inputSchemaPath) { + info({ message: `${action} input schema at ${inputSchemaPath}` }); + } else { + info({ message: `${action} input schema embedded in '${LOCAL_CONFIG_PATH}'` }); + } + + const validator = new Ajv2019({ strict: false }); + validateInputSchema(validator, inputSchema); + + return { inputSchema, inputSchemaPath }; +}; + +/** + * Read a storage schema (Dataset or Key-Value Store) from the Actor config. + * + * Resolves `storages.` from `.actor/actor.json`: + * - If it's an object, uses it directly as the embedded schema. + * - If it's a string, resolves the file path relative to `.actor/`. + * - If it's missing, returns `null`. + */ +export const readStorageSchema = ({ + cwd, + key, + label, +}: { + cwd: string; + key: string; + label: string; +}): { schema: Record; schemaPath: string | null } | null => { + const localConfig = getLocalConfig(cwd); + + const ref = (localConfig?.storages as Record | undefined)?.[key]; + + if (typeof ref === 'object' && ref !== null) { + return { + schema: ref as Record, + schemaPath: null, + }; + } + + if (typeof ref === 'string') { + const fullPath = join(cwd, ACTOR_SPECIFICATION_FOLDER, ref); + const schema = getJsonFileContent(fullPath); + + if (!schema) { + warning({ + message: `${label} schema file not found at ${fullPath} (referenced in '${LOCAL_CONFIG_PATH}').`, + }); + return null; + } + + return { + schema, + schemaPath: fullPath, + }; + } + + return null; +}; + +/** + * Read the Dataset schema from the Actor config. + * Thin wrapper around `readStorageSchema` for backwards compatibility. + */ +export const readDatasetSchema = ( + { cwd }: { cwd: string } = { cwd: process.cwd() }, +): { datasetSchema: Record; datasetSchemaPath: string | null } | null => { + const result = readStorageSchema({ cwd, key: 'dataset', label: 'Dataset' }); + + if (!result) { + return null; + } + + return { + datasetSchema: result.schema, + datasetSchemaPath: result.schemaPath, + }; +}; + +/** + * Read the Output schema from the Actor config. + * + * Resolves `output` from `.actor/actor.json`: + * - If it's an object, uses it directly as the embedded schema. + * - If it's a string, resolves the file path relative to `.actor/`. + * - If it's missing, returns `null`. + */ +export const readOutputSchema = ( + { cwd }: { cwd: string } = { cwd: process.cwd() }, +): { outputSchema: Record; outputSchemaPath: string | null } | null => { + const localConfig = getLocalConfig(cwd); + + const outputRef = localConfig?.output; + + if (typeof outputRef === 'object' && outputRef !== null) { + return { + outputSchema: outputRef as Record, + outputSchemaPath: null, + }; + } + + if (typeof outputRef === 'string') { + const fullPath = join(cwd, ACTOR_SPECIFICATION_FOLDER, outputRef); + const schema = getJsonFileContent(fullPath); + + if (!schema) { + warning({ + message: `Output schema file not found at ${fullPath} (referenced in '${LOCAL_CONFIG_PATH}').`, + }); + return null; + } + + return { + outputSchema: schema, + outputSchemaPath: fullPath, + }; + } + + return null; +}; + /** * Goes to the Actor directory and creates INPUT.json file from the input schema prefills. diff --git a/src/lib/schema-transforms.ts b/src/lib/schema-transforms.ts new file mode 100644 index 000000000..8c36555d9 --- /dev/null +++ b/src/lib/schema-transforms.ts @@ -0,0 +1,208 @@ +import deepClone from 'lodash.clonedeep'; + +/** + * Transforms a JSON schema so that all properties without a `default` value are marked as required. + * Properties that have a `default` are left optional, since Apify fills them in at runtime. + * Recurses into nested object properties. + */ +export function makePropertiesRequired(schema: Record): Record { + const clone = deepClone(schema); + + if (!clone.properties || typeof clone.properties !== 'object') { + return clone; + } + + const properties = clone.properties as Record>; + const requiredSet = new Set(Array.isArray(clone.required) ? (clone.required as string[]) : []); + + for (const [key, prop] of Object.entries(properties)) { + if (prop.default === undefined) { + requiredSet.add(key); + } else { + requiredSet.delete(key); + } + + if (prop.type === 'object' && prop.properties) { + properties[key] = makePropertiesRequired(prop) as Record; + } + } + + clone.required = Array.from(requiredSet); + + return clone; +} + +/** + * Deep clones a schema and recursively removes all `required` arrays, + * making every property optional at all nesting levels. + */ +export function clearAllRequired(schema: Record): Record { + const clone = deepClone(schema); + + delete clone.required; + + if (clone.properties && typeof clone.properties === 'object') { + const properties = clone.properties as Record>; + for (const [key, prop] of Object.entries(properties)) { + if (prop.type === 'object' && prop.properties) { + properties[key] = clearAllRequired(prop) as Record; + } + } + } + + return clone; +} + +/** + * Recursively strips `title` from all properties in a schema. + * + * When a nested property has a `title`, `json-schema-to-typescript` extracts it + * as a separate named `export interface`. Stripping titles forces all nested types + * to be inlined, ensuring only one exported interface per schema. + */ +export function stripTitles(schema: Record): Record { + const clone = deepClone(schema); + + delete clone.title; + + if (clone.properties && typeof clone.properties === 'object') { + const properties = clone.properties as Record>; + for (const [key, prop] of Object.entries(properties)) { + if (prop && typeof prop === 'object') { + properties[key] = stripTitles(prop) as Record; + } + } + } + + if (clone.items && typeof clone.items === 'object') { + clone.items = stripTitles(clone.items as Record); + } + + // Recurse into composition keywords (arrays of sub-schemas) + for (const keyword of ['allOf', 'anyOf', 'oneOf'] as const) { + if (Array.isArray(clone[keyword])) { + clone[keyword] = (clone[keyword] as Record[]).map((subSchema) => + subSchema && typeof subSchema === 'object' ? stripTitles(subSchema) : subSchema, + ); + } + } + + // Recurse into definitions / $defs (objects mapping names to sub-schemas) + for (const keyword of ['definitions', '$defs'] as const) { + if (clone[keyword] && typeof clone[keyword] === 'object' && !Array.isArray(clone[keyword])) { + const defs = clone[keyword] as Record>; + for (const [key, def] of Object.entries(defs)) { + if (def && typeof def === 'object') { + defs[key] = stripTitles(def) as Record; + } + } + } + } + + // Recurse into additionalProperties when it is a schema object + if (clone.additionalProperties && typeof clone.additionalProperties === 'object') { + clone.additionalProperties = stripTitles(clone.additionalProperties as Record); + } + + return clone; +} + +/** + * Extracts and prepares the `fields` sub-schema from a Dataset or KVS schema for compilation. + * Returns `null` if the schema has no compilable fields (empty or missing). + */ +export function prepareFieldsSchemaForCompilation(schema: Record): Record | null { + const fields = schema.fields as Record | undefined; + + if (!fields || typeof fields !== 'object' || !fields.properties || typeof fields.properties !== 'object') { + return null; + } + + const clone = deepClone(fields); + + if (!clone.type) { + clone.type = 'object'; + } + + return clone; +} + +/** + * Prepares an Output schema for compilation by stripping non-JSON-Schema keys. + * + * Output schemas have `properties` at the top level where each property always has + * `type: "string"` and a `template` field (URL construction pattern). + * We strip `template` since it's not valid JSON Schema. + * + * Returns `null` if the schema has no compilable properties. + */ +export function prepareOutputSchemaForCompilation(schema: Record): Record | null { + const properties = schema.properties as Record> | undefined; + + if (!properties || typeof properties !== 'object' || Object.keys(properties).length === 0) { + return null; + } + + const clonedProperties = deepClone(properties); + + // Strip non-JSON-Schema keys (like `template`) from each property + for (const prop of Object.values(clonedProperties)) { + if (prop && typeof prop === 'object') { + delete prop.template; + } + } + + const result: Record = { + type: schema.type || 'object', + properties: clonedProperties, + }; + + if (Array.isArray(schema.required)) { + result.required = [...schema.required]; + } + + return result; +} + +/** + * Extracts compilable JSON schemas from KVS collections. + * + * KVS schemas use `collections` where each collection can have a `jsonSchema` (Draft 07). + * Only collections with `jsonSchema` are returned, as non-JSON collections (e.g. images) + * have no type to generate. + * + * Returns an array of `{ name, schema }` pairs, or an empty array if none are found. + */ +export function prepareKvsCollectionsForCompilation( + schema: Record, +): { name: string; schema: Record }[] { + const collections = schema.collections as Record> | undefined; + + if (!collections || typeof collections !== 'object') { + return []; + } + + const result: { name: string; schema: Record }[] = []; + + for (const [name, collection] of Object.entries(collections)) { + if (!collection || typeof collection !== 'object') { + continue; + } + + const jsonSchema = collection.jsonSchema as Record | undefined; + + if (!jsonSchema || typeof jsonSchema !== 'object' || Object.keys(jsonSchema).length === 0) { + continue; + } + + const clone = deepClone(jsonSchema); + + if (!clone.type) { + clone.type = 'object'; + } + + result.push({ name, schema: clone }); + } + + return result; +} diff --git a/test/__setup__/dataset-schemas/empty-fields.json b/test/__setup__/dataset-schemas/empty-fields.json new file mode 100644 index 000000000..ff5a5ea32 --- /dev/null +++ b/test/__setup__/dataset-schemas/empty-fields.json @@ -0,0 +1,17 @@ +{ + "actorSpecification": 1, + "fields": {}, + "views": { + "overview": { + "title": "Overview", + "transformation": { "fields": ["title", "url"] }, + "display": { + "component": "table", + "properties": { + "title": { "label": "Title", "format": "text" }, + "url": { "label": "URL", "format": "link" } + } + } + } + } +} diff --git a/test/__setup__/dataset-schemas/no-fields.json b/test/__setup__/dataset-schemas/no-fields.json new file mode 100644 index 000000000..a6f09f4ca --- /dev/null +++ b/test/__setup__/dataset-schemas/no-fields.json @@ -0,0 +1,16 @@ +{ + "actorSpecification": 1, + "views": { + "overview": { + "title": "Overview", + "transformation": { "fields": ["title", "url"] }, + "display": { + "component": "table", + "properties": { + "title": { "label": "Title", "format": "text" }, + "url": { "label": "URL", "format": "link" } + } + } + } + } +} diff --git a/test/__setup__/dataset-schemas/no-type.json b/test/__setup__/dataset-schemas/no-type.json new file mode 100644 index 000000000..5edc42ee5 --- /dev/null +++ b/test/__setup__/dataset-schemas/no-type.json @@ -0,0 +1,11 @@ +{ + "actorSpecification": 1, + "fields": { + "properties": { + "name": { "type": "string" }, + "count": { "type": "integer" } + }, + "required": ["name"] + }, + "views": {} +} diff --git a/test/__setup__/dataset-schemas/paths.ts b/test/__setup__/dataset-schemas/paths.ts new file mode 100644 index 000000000..fef20f343 --- /dev/null +++ b/test/__setup__/dataset-schemas/paths.ts @@ -0,0 +1,9 @@ +import { fileURLToPath } from 'node:url'; + +export const validDatasetSchemaPath = fileURLToPath(new URL('./valid.json', import.meta.url)); + +export const emptyFieldsDatasetSchemaPath = fileURLToPath(new URL('./empty-fields.json', import.meta.url)); + +export const noTypeDatasetSchemaPath = fileURLToPath(new URL('./no-type.json', import.meta.url)); + +export const noFieldsDatasetSchemaPath = fileURLToPath(new URL('./no-fields.json', import.meta.url)); diff --git a/test/__setup__/dataset-schemas/valid.json b/test/__setup__/dataset-schemas/valid.json new file mode 100644 index 000000000..1acaf0816 --- /dev/null +++ b/test/__setup__/dataset-schemas/valid.json @@ -0,0 +1,29 @@ +{ + "actorSpecification": 1, + "fields": { + "type": "object", + "properties": { + "title": { + "type": "string", + "title": "Title", + "description": "Item title" + }, + "url": { "type": "string", "title": "URL" }, + "price": { "type": "integer", "title": "Price" } + }, + "required": ["title", "url"] + }, + "views": { + "overview": { + "title": "Overview", + "transformation": { "fields": ["title", "url"] }, + "display": { + "component": "table", + "properties": { + "title": { "label": "Title", "format": "text" }, + "url": { "label": "URL", "format": "link" } + } + } + } + } +} diff --git a/test/__setup__/input-schemas/complex.json b/test/__setup__/input-schemas/complex.json new file mode 100644 index 000000000..24361c6f4 --- /dev/null +++ b/test/__setup__/input-schemas/complex.json @@ -0,0 +1,80 @@ +{ + "$schema": "https://apify-projects.github.io/actor-json-schemas/input.ide.json?v=0.6", + "title": "Example Actor Input Schema", + "description": "A sample input schema demonstrating different input field types.", + "type": "object", + "schemaVersion": 1, + "properties": { + "startUrls": { + "title": "Start URLs", + "description": "List of URLs to start crawling", + "type": "array", + "editor": "requestListSources", + "prefill": [ + { "url": "https://example.com" }, + { "url": "https://example.org" } + ] + }, + "searchQuery": { + "title": "Search query", + "description": "The keyword or phrase to search for", + "type": "string", + "editor": "textfield", + "minLength": 3, + "default": "apify" + }, + "maxItems": { + "title": "Maximum items to fetch", + "description": "Limit the number of items the Actor will process", + "type": "integer", + "editor": "number", + "minimum": 1, + "default": 100 + }, + "includeImages": { + "title": "Include images", + "description": "Whether to include image data in the results", + "type": "boolean", + "editor": "checkbox", + "default": false + }, + "crawlerType": { + "title": "Crawler type", + "description": "Select the crawling engine to use", + "type": "string", + "editor": "select", + "enum": ["cheerio", "puppeteer", "playwright"], + "enumTitles": [ + "Cheerio crawler", + "Puppeteer browser", + "Playwright browser" + ], + "default": "cheerio" + }, + "proxyConfig": { + "title": "Proxy configuration", + "description": "Optional proxy settings to use while crawling", + "type": "object", + "editor": "json", + "properties": { + "useApifyProxy": { + "title": "Use Apify Proxy", + "description": "Enable Apify Proxy", + "type": "boolean" + }, + "customProxyUrls": { + "title": "Custom proxy URLs", + "description": "List of custom proxy URLs", + "type": "array", + "editor": "json", + "items": { + "type": "string" + } + } + }, + "required": ["useApifyProxy"], + "additionalProperties": false + } + }, + "required": ["startUrls", "searchQuery"] +} diff --git a/test/__setup__/input-schemas/paths.ts b/test/__setup__/input-schemas/paths.ts index 96728bb73..60e4cd9d3 100644 --- a/test/__setup__/input-schemas/paths.ts +++ b/test/__setup__/input-schemas/paths.ts @@ -13,3 +13,5 @@ export const prefillsInputSchemaPath = fileURLToPath(new URL('./prefills.json', export const unparsableInputSchemaPath = fileURLToPath(new URL('./unparsable.json', import.meta.url)); export const validInputSchemaPath = fileURLToPath(new URL('./valid.json', import.meta.url)); + +export const complexInputSchemaPath = fileURLToPath(new URL('./complex.json', import.meta.url)); diff --git a/test/__setup__/kvs-schemas/paths.ts b/test/__setup__/kvs-schemas/paths.ts new file mode 100644 index 000000000..f30ee8ac5 --- /dev/null +++ b/test/__setup__/kvs-schemas/paths.ts @@ -0,0 +1,3 @@ +import { fileURLToPath } from 'node:url'; + +export const validKvsSchemaPath = fileURLToPath(new URL('./valid.json', import.meta.url)); diff --git a/test/__setup__/kvs-schemas/valid.json b/test/__setup__/kvs-schemas/valid.json new file mode 100644 index 000000000..69e068d27 --- /dev/null +++ b/test/__setup__/kvs-schemas/valid.json @@ -0,0 +1,24 @@ +{ + "actorKeyValueStoreSchemaVersion": 1, + "title": "Test KVS", + "collections": { + "screenshots": { + "title": "Screenshots", + "contentTypes": ["image/png"], + "keyPrefix": "screenshot-" + }, + "results": { + "title": "Results", + "contentTypes": ["application/json"], + "key": "RESULTS", + "jsonSchema": { + "type": "object", + "properties": { + "totalItems": { "type": "integer" }, + "summary": { "type": "string" } + }, + "required": ["totalItems"] + } + } + } +} diff --git a/test/__setup__/output-schemas/no-properties.json b/test/__setup__/output-schemas/no-properties.json new file mode 100644 index 000000000..51e58ef06 --- /dev/null +++ b/test/__setup__/output-schemas/no-properties.json @@ -0,0 +1,8 @@ +{ + "actorOutputSchemaVersion": 1, + "views": { + "overview": { + "title": "Overview" + } + } +} diff --git a/test/__setup__/output-schemas/paths.ts b/test/__setup__/output-schemas/paths.ts new file mode 100644 index 000000000..54271d043 --- /dev/null +++ b/test/__setup__/output-schemas/paths.ts @@ -0,0 +1,5 @@ +import { fileURLToPath } from 'node:url'; + +export const validOutputSchemaPath = fileURLToPath(new URL('./valid.json', import.meta.url)); + +export const noPropertiesOutputSchemaPath = fileURLToPath(new URL('./no-properties.json', import.meta.url)); diff --git a/test/__setup__/output-schemas/valid.json b/test/__setup__/output-schemas/valid.json new file mode 100644 index 000000000..5dc75c0d3 --- /dev/null +++ b/test/__setup__/output-schemas/valid.json @@ -0,0 +1,24 @@ +{ + "actorOutputSchemaVersion": 1, + "properties": { + "productPage": { + "type": "string", + "title": "Product Page", + "description": "URL of the product page", + "template": "https://example.com/products/{{productId}}" + }, + "screenshot": { + "type": "string", + "title": "Screenshot", + "description": "URL of the screenshot", + "template": "https://example.com/screenshots/{{screenshotId}}" + }, + "report": { + "type": "string", + "title": "Report", + "description": "URL of the report", + "template": "https://example.com/reports/{{reportId}}" + } + }, + "required": ["productPage", "screenshot"] +} diff --git a/test/local/commands/actor/generate-schema-types.test.ts b/test/local/commands/actor/generate-schema-types.test.ts new file mode 100644 index 000000000..fcc8811ce --- /dev/null +++ b/test/local/commands/actor/generate-schema-types.test.ts @@ -0,0 +1,873 @@ +import { mkdir, readFile, writeFile } from 'node:fs/promises'; +import { basename, join } from 'node:path'; + +import { ActorGenerateSchemaTypesCommand } from '../../../../src/commands/actor/generate-schema-types.js'; +import { testRunCommand } from '../../../../src/lib/command-framework/apify-command.js'; +import { + clearAllRequired, + makePropertiesRequired, + prepareFieldsSchemaForCompilation, + prepareKvsCollectionsForCompilation, + prepareOutputSchemaForCompilation, +} from '../../../../src/lib/schema-transforms.js'; +import { validDatasetSchemaPath } from '../../../__setup__/dataset-schemas/paths.js'; +import { useConsoleSpy } from '../../../__setup__/hooks/useConsoleSpy.js'; +import { useTempPath } from '../../../__setup__/hooks/useTempPath.js'; +import { + complexInputSchemaPath, + defaultsInputSchemaPath, + unparsableInputSchemaPath, +} from '../../../__setup__/input-schemas/paths.js'; +import { validKvsSchemaPath } from '../../../__setup__/kvs-schemas/paths.js'; +import { noPropertiesOutputSchemaPath, validOutputSchemaPath } from '../../../__setup__/output-schemas/paths.js'; + +const { lastErrorMessage, logMessages } = useConsoleSpy(); + +async function setupActorConfig( + basePath: string, + { + inputSchema, + datasetSchemaRef, + outputSchemaRef, + kvsSchemaRef, + }: { + inputSchema?: Record; + datasetSchemaRef?: string | Record; + outputSchemaRef?: string | Record; + kvsSchemaRef?: string | Record; + }, +) { + const actorDir = join(basePath, '.actor'); + await mkdir(actorDir, { recursive: true }); + + // Always provide a minimal input schema so the command doesn't fail + const minimalInput = inputSchema ?? { + title: 'Test', + type: 'object', + schemaVersion: 1, + properties: { + foo: { title: 'Foo', description: 'A foo field', type: 'string', default: 'bar', editor: 'textfield' }, + }, + }; + + await writeFile(join(actorDir, 'input_schema.json'), JSON.stringify(minimalInput, null, '\t')); + + const actorJson: Record = { + actorSpecification: 1, + name: 'test-actor', + version: '0.1', + input: './input_schema.json', + }; + + const storages: Record = {}; + + if (datasetSchemaRef !== undefined) { + if (typeof datasetSchemaRef === 'string') { + const content = await readFile(datasetSchemaRef, 'utf-8'); + const fileName = basename(datasetSchemaRef); + await writeFile(join(actorDir, fileName), content); + storages.dataset = `./${fileName}`; + } else { + storages.dataset = datasetSchemaRef; + } + } + + if (kvsSchemaRef !== undefined) { + if (typeof kvsSchemaRef === 'string') { + const content = await readFile(kvsSchemaRef, 'utf-8'); + const fileName = `kvs-${basename(kvsSchemaRef)}`; + await writeFile(join(actorDir, fileName), content); + storages.keyValueStore = `./${fileName}`; + } else { + storages.keyValueStore = kvsSchemaRef; + } + } + + if (Object.keys(storages).length > 0) { + actorJson.storages = storages; + } + + if (outputSchemaRef !== undefined) { + if (typeof outputSchemaRef === 'string') { + const content = await readFile(outputSchemaRef, 'utf-8'); + const fileName = `output-${basename(outputSchemaRef)}`; + await writeFile(join(actorDir, fileName), content); + actorJson.output = `./${fileName}`; + } else { + actorJson.output = outputSchemaRef; + } + } + + await writeFile(join(actorDir, 'actor.json'), JSON.stringify(actorJson, null, '\t')); +} + +describe('apify actor generate-schema-types', () => { + const { joinPath, beforeAllCalls, afterAllCalls } = useTempPath('generate-schema-types', { + create: true, + remove: true, + cwd: true, + cwdParent: false, + }); + + beforeEach(async () => { + await beforeAllCalls(); + }); + + afterEach(async () => { + await afterAllCalls(); + }); + + it('should generate types from a valid schema', async () => { + const outputDir = joinPath('output'); + + await testRunCommand(ActorGenerateSchemaTypesCommand, { + args_path: complexInputSchemaPath, + flags_output: outputDir, + }); + + expect(lastErrorMessage()).include('Generated types written to'); + + const generatedFile = await readFile(joinPath('output', 'input.ts'), 'utf-8'); + expect(generatedFile).toContain('export interface'); + expect(generatedFile).toContain('searchQuery'); + }); + + it('should use default output directory when not specified', async () => { + await testRunCommand(ActorGenerateSchemaTypesCommand, { + args_path: complexInputSchemaPath, + }); + + expect(lastErrorMessage()).include('Generated types written to'); + expect(lastErrorMessage()).include(join('.generated', 'actor', 'input.ts')); + + const generatedFile = await readFile(joinPath('src', '.generated', 'actor', 'input.ts'), 'utf-8'); + expect(generatedFile).toContain('export interface'); + }); + + it('should generate strict types by default (no index signature)', async () => { + const outputDir = joinPath('output-strict'); + + await testRunCommand(ActorGenerateSchemaTypesCommand, { + args_path: complexInputSchemaPath, + flags_output: outputDir, + }); + + const generatedFile = await readFile(joinPath('output-strict', 'input.ts'), 'utf-8'); + expect(generatedFile).not.toContain('[key: string]: unknown'); + }); + + it('should generate non-strict types when -s is used', async () => { + const outputDir = joinPath('output-non-strict'); + + await testRunCommand(ActorGenerateSchemaTypesCommand, { + args_path: defaultsInputSchemaPath, + flags_output: outputDir, + flags_strict: false, + }); + + const generatedFile = await readFile(joinPath('output-non-strict', 'input.ts'), 'utf-8'); + // Verify the file is generated with the interface + expect(generatedFile).toContain('export interface'); + }); + + it('should fail when schema file does not exist', async () => { + const outputDir = joinPath('output-missing'); + + await testRunCommand(ActorGenerateSchemaTypesCommand, { + args_path: '/non/existent/schema.json', + flags_output: outputDir, + }); + + expect(lastErrorMessage()).include('Input schema has not been found'); + }); + + it('should fail when schema is not valid JSON', async () => { + const outputDir = joinPath('output-invalid'); + + await testRunCommand(ActorGenerateSchemaTypesCommand, { + args_path: unparsableInputSchemaPath, + flags_output: outputDir, + }); + + expect(lastErrorMessage()).toBeTruthy(); + }); + + it('should use custom output directory with -o flag', async () => { + const outputDir = joinPath('custom-output'); + await testRunCommand(ActorGenerateSchemaTypesCommand, { + args_path: defaultsInputSchemaPath, + flags_output: outputDir, + }); + + expect(lastErrorMessage()).include('Generated types written to'); + expect(lastErrorMessage()).include('custom-output'); + }); + + it('should generate required properties by default for fields without defaults', async () => { + const outputDir = joinPath('output-required'); + + await testRunCommand(ActorGenerateSchemaTypesCommand, { + args_path: complexInputSchemaPath, + flags_output: outputDir, + }); + + const generatedFile = await readFile(joinPath('output-required', 'input.ts'), 'utf-8'); + + // startUrls has no default -> required (no ?) + expect(generatedFile).toMatch(/startUrls:/); + expect(generatedFile).not.toMatch(/startUrls\?:/); + + // proxyConfig has no default -> required (no ?) + expect(generatedFile).toMatch(/proxyConfig:/); + expect(generatedFile).not.toMatch(/proxyConfig\?:/); + + // searchQuery is in original required array but has default: "apify" -> optional (has ?) + expect(generatedFile).toMatch(/searchQuery\?:/); + + // maxItems has default: 100 -> optional (has ?) + expect(generatedFile).toMatch(/maxItems\?:/); + + // includeImages has default: false -> optional (has ?) + expect(generatedFile).toMatch(/includeImages\?:/); + + // crawlerType has default: "cheerio" -> optional (has ?) + expect(generatedFile).toMatch(/crawlerType\?:/); + }); + + it('should make all properties optional with --all-optional flag', async () => { + const outputDir = joinPath('output-all-optional'); + + await testRunCommand(ActorGenerateSchemaTypesCommand, { + args_path: complexInputSchemaPath, + flags_output: outputDir, + 'flags_all-optional': true, + }); + + const generatedFile = await readFile(joinPath('output-all-optional', 'input.ts'), 'utf-8'); + + // With --all-optional, ALL properties should be optional - including originally required ones + expect(generatedFile).toMatch(/startUrls\?:/); + expect(generatedFile).toMatch(/searchQuery\?:/); + expect(generatedFile).toMatch(/maxItems\?:/); + expect(generatedFile).toMatch(/includeImages\?:/); + expect(generatedFile).toMatch(/proxyConfig\?:/); + + // Nested required properties should also become optional + expect(generatedFile).toMatch(/useApifyProxy\?:/); + expect(generatedFile).not.toMatch(/useApifyProxy:/); // ensure it's not non-optional + }); + + describe('dataset schema', () => { + it('should generate types from dataset schema referenced in actor.json', async () => { + const outputDir = joinPath('ds-output'); + await setupActorConfig(joinPath(), { datasetSchemaRef: validDatasetSchemaPath }); + + await testRunCommand(ActorGenerateSchemaTypesCommand, { + flags_output: outputDir, + }); + + const generatedFile = await readFile(joinPath('ds-output', 'dataset.ts'), 'utf-8'); + expect(generatedFile).toContain('export interface'); + expect(generatedFile).toContain('title'); + expect(generatedFile).toContain('url'); + expect(generatedFile).toContain('price'); + }); + + it('should generate types from dataset schema embedded in actor.json', async () => { + const outputDir = joinPath('ds-output-embedded'); + await setupActorConfig(joinPath(), { + datasetSchemaRef: { + actorSpecification: 1, + fields: { + type: 'object', + properties: { + name: { type: 'string' }, + value: { type: 'integer' }, + }, + required: ['name'], + }, + views: {}, + }, + }); + + await testRunCommand(ActorGenerateSchemaTypesCommand, { + flags_output: outputDir, + }); + + const generatedFile = await readFile(joinPath('ds-output-embedded', 'dataset.ts'), 'utf-8'); + expect(generatedFile).toContain('export interface'); + expect(generatedFile).toContain('name'); + expect(generatedFile).toContain('value'); + }); + + it('should skip when dataset fields are empty', async () => { + const outputDir = joinPath('ds-output-empty'); + await setupActorConfig(joinPath(), { + datasetSchemaRef: { + actorSpecification: 1, + fields: {}, + views: {}, + }, + }); + + await testRunCommand(ActorGenerateSchemaTypesCommand, { + flags_output: outputDir, + }); + + const errorMessages = logMessages.error.join('\n'); + expect(errorMessages).toContain('no fields defined'); + }); + + it('should not generate dataset types when path argument is provided', async () => { + const outputDir = joinPath('ds-output-path-arg'); + await setupActorConfig(joinPath(), { datasetSchemaRef: validDatasetSchemaPath }); + + await testRunCommand(ActorGenerateSchemaTypesCommand, { + args_path: complexInputSchemaPath, + flags_output: outputDir, + }); + + const errorMessages = logMessages.error.join('\n'); + expect(errorMessages).not.toContain('Dataset schema'); + }); + }); + + describe('output schema', () => { + it('should generate types from output schema referenced in actor.json', async () => { + const outputDir = joinPath('out-output'); + await setupActorConfig(joinPath(), { outputSchemaRef: validOutputSchemaPath }); + + await testRunCommand(ActorGenerateSchemaTypesCommand, { + flags_output: outputDir, + }); + + const generatedFile = await readFile(joinPath('out-output', 'output.ts'), 'utf-8'); + expect(generatedFile).toContain('export interface'); + expect(generatedFile).toContain('productPage'); + expect(generatedFile).toContain('screenshot'); + expect(generatedFile).toContain('report'); + // template fields should be stripped + expect(generatedFile).not.toContain('template'); + expect(generatedFile).not.toContain('{{'); + }); + + it('should generate types from output schema embedded in actor.json', async () => { + const outputDir = joinPath('out-output-embedded'); + await setupActorConfig(joinPath(), { + outputSchemaRef: { + actorOutputSchemaVersion: 1, + properties: { + resultPage: { type: 'string', template: 'https://example.com/{{id}}' }, + dataExport: { type: 'string', template: 'https://example.com/export/{{id}}' }, + }, + required: ['resultPage'], + }, + }); + + await testRunCommand(ActorGenerateSchemaTypesCommand, { + flags_output: outputDir, + }); + + const generatedFile = await readFile(joinPath('out-output-embedded', 'output.ts'), 'utf-8'); + expect(generatedFile).toContain('export interface'); + expect(generatedFile).toContain('resultPage'); + expect(generatedFile).toContain('dataExport'); + }); + + it('should skip when output schema has no properties', async () => { + const outputDir = joinPath('out-output-empty'); + await setupActorConfig(joinPath(), { outputSchemaRef: noPropertiesOutputSchemaPath }); + + await testRunCommand(ActorGenerateSchemaTypesCommand, { + flags_output: outputDir, + }); + + const errorMessages = logMessages.error.join('\n'); + expect(errorMessages).toContain('no properties defined'); + }); + + it('should not generate output types when path argument is provided', async () => { + const outputDir = joinPath('out-output-path-arg'); + await setupActorConfig(joinPath(), { outputSchemaRef: validOutputSchemaPath }); + + await testRunCommand(ActorGenerateSchemaTypesCommand, { + args_path: complexInputSchemaPath, + flags_output: outputDir, + }); + + const errorMessages = logMessages.error.join('\n'); + expect(errorMessages).not.toContain('Output schema'); + }); + }); + + describe('key-value store schema', () => { + it('should generate types from KVS schema referenced in actor.json', async () => { + const outputDir = joinPath('kvs-output'); + await setupActorConfig(joinPath(), { kvsSchemaRef: validKvsSchemaPath }); + + await testRunCommand(ActorGenerateSchemaTypesCommand, { + flags_output: outputDir, + }); + + const generatedFile = await readFile(joinPath('kvs-output', 'key-value-store.ts'), 'utf-8'); + expect(generatedFile).toContain('export interface'); + // Only "results" collection has jsonSchema; "screenshots" does not + expect(generatedFile).toContain('totalItems'); + expect(generatedFile).toContain('summary'); + }); + + it('should generate types from KVS schema embedded in actor.json', async () => { + const outputDir = joinPath('kvs-output-embedded'); + await setupActorConfig(joinPath(), { + kvsSchemaRef: { + actorKeyValueStoreSchemaVersion: 1, + title: 'Test KVS', + collections: { + metrics: { + title: 'Metrics', + contentTypes: ['application/json'], + key: 'METRICS', + jsonSchema: { + type: 'object', + properties: { + runCount: { type: 'integer' }, + avgDuration: { type: 'number' }, + }, + required: ['runCount'], + }, + }, + }, + }, + }); + + await testRunCommand(ActorGenerateSchemaTypesCommand, { + flags_output: outputDir, + }); + + const generatedFile = await readFile(joinPath('kvs-output-embedded', 'key-value-store.ts'), 'utf-8'); + expect(generatedFile).toContain('export interface'); + expect(generatedFile).toContain('runCount'); + expect(generatedFile).toContain('avgDuration'); + }); + + it('should skip when no collections have jsonSchema', async () => { + const outputDir = joinPath('kvs-output-no-json'); + await setupActorConfig(joinPath(), { + kvsSchemaRef: { + actorKeyValueStoreSchemaVersion: 1, + title: 'Image KVS', + collections: { + images: { + title: 'Images', + contentTypes: ['image/png'], + keyPrefix: 'img-', + }, + }, + }, + }); + + await testRunCommand(ActorGenerateSchemaTypesCommand, { + flags_output: outputDir, + }); + + const errorMessages = logMessages.error.join('\n'); + expect(errorMessages).toContain('no collections with JSON schemas'); + }); + + it('should not generate KVS types when path argument is provided', async () => { + const outputDir = joinPath('kvs-output-path-arg'); + await setupActorConfig(joinPath(), { kvsSchemaRef: validKvsSchemaPath }); + + await testRunCommand(ActorGenerateSchemaTypesCommand, { + args_path: complexInputSchemaPath, + flags_output: outputDir, + }); + + const errorMessages = logMessages.error.join('\n'); + expect(errorMessages).not.toContain('Key-Value Store schema'); + }); + }); +}); + +describe('prepareFieldsSchemaForCompilation', () => { + it('should extract fields sub-schema', () => { + const schema = { + actorSpecification: 1, + fields: { + type: 'object', + properties: { + title: { type: 'string' }, + }, + required: ['title'], + }, + views: {}, + }; + + const result = prepareFieldsSchemaForCompilation(schema); + expect(result).toEqual({ + type: 'object', + properties: { title: { type: 'string' } }, + required: ['title'], + }); + }); + + it('should inject type: "object" when missing from fields', () => { + const schema = { + actorSpecification: 1, + fields: { + properties: { + name: { type: 'string' }, + }, + }, + views: {}, + }; + + const result = prepareFieldsSchemaForCompilation(schema); + expect(result).not.toBeNull(); + expect(result!.type).toBe('object'); + }); + + it('should return null for empty fields', () => { + const schema = { + actorSpecification: 1, + fields: {}, + views: {}, + }; + + const result = prepareFieldsSchemaForCompilation(schema); + expect(result).toBeNull(); + }); + + it('should return null when fields key is missing', () => { + const schema = { + actorSpecification: 1, + views: {}, + }; + + const result = prepareFieldsSchemaForCompilation(schema); + expect(result).toBeNull(); + }); + + it('should not mutate the original schema', () => { + const schema = { + actorSpecification: 1, + fields: { + properties: { + title: { type: 'string' }, + }, + }, + views: {}, + }; + + prepareFieldsSchemaForCompilation(schema); + expect((schema.fields as any).type).toBeUndefined(); + }); +}); + +describe('prepareOutputSchemaForCompilation', () => { + it('should extract properties and strip template fields', () => { + const schema = { + actorOutputSchemaVersion: 1, + properties: { + page: { type: 'string', template: 'https://example.com/{{id}}', title: 'Page' }, + report: { type: 'string', template: 'https://example.com/report/{{id}}' }, + }, + required: ['page'], + }; + + const result = prepareOutputSchemaForCompilation(schema); + expect(result).toEqual({ + type: 'object', + properties: { + page: { type: 'string', title: 'Page' }, + report: { type: 'string' }, + }, + required: ['page'], + }); + }); + + it('should return null when properties are missing', () => { + const schema = { + actorOutputSchemaVersion: 1, + }; + + const result = prepareOutputSchemaForCompilation(schema); + expect(result).toBeNull(); + }); + + it('should return null when properties are empty', () => { + const schema = { + actorOutputSchemaVersion: 1, + properties: {}, + }; + + const result = prepareOutputSchemaForCompilation(schema); + expect(result).toBeNull(); + }); + + it('should not include non-JSON-Schema keys like actorOutputSchemaVersion', () => { + const schema = { + actorOutputSchemaVersion: 1, + properties: { + name: { type: 'string', template: 'https://example.com/{{name}}' }, + }, + }; + + const result = prepareOutputSchemaForCompilation(schema); + expect(result).not.toBeNull(); + expect(result).not.toHaveProperty('actorOutputSchemaVersion'); + }); + + it('should not mutate the original schema', () => { + const schema = { + actorOutputSchemaVersion: 1, + properties: { + name: { type: 'string', template: 'https://example.com/{{name}}' }, + }, + }; + + prepareOutputSchemaForCompilation(schema); + expect(schema).toHaveProperty('actorOutputSchemaVersion'); + expect((schema.properties as any).name).toHaveProperty('template'); + }); +}); + +describe('prepareKvsCollectionsForCompilation', () => { + it('should extract jsonSchema from collections', () => { + const schema = { + actorKeyValueStoreSchemaVersion: 1, + title: 'Test', + collections: { + results: { + contentTypes: ['application/json'], + key: 'RESULTS', + jsonSchema: { + type: 'object', + properties: { count: { type: 'integer' } }, + }, + }, + }, + }; + + const result = prepareKvsCollectionsForCompilation(schema); + expect(result).toHaveLength(1); + expect(result[0].name).toBe('results'); + expect(result[0].schema).toEqual({ + type: 'object', + properties: { count: { type: 'integer' } }, + }); + }); + + it('should skip collections without jsonSchema', () => { + const schema = { + actorKeyValueStoreSchemaVersion: 1, + title: 'Test', + collections: { + images: { + contentTypes: ['image/png'], + keyPrefix: 'img-', + }, + results: { + contentTypes: ['application/json'], + key: 'RESULTS', + jsonSchema: { + type: 'object', + properties: { count: { type: 'integer' } }, + }, + }, + }, + }; + + const result = prepareKvsCollectionsForCompilation(schema); + expect(result).toHaveLength(1); + expect(result[0].name).toBe('results'); + }); + + it('should return empty array when no collections exist', () => { + const schema = { + actorKeyValueStoreSchemaVersion: 1, + title: 'Test', + }; + + const result = prepareKvsCollectionsForCompilation(schema); + expect(result).toEqual([]); + }); + + it('should return empty array when no collections have jsonSchema', () => { + const schema = { + actorKeyValueStoreSchemaVersion: 1, + title: 'Test', + collections: { + images: { + contentTypes: ['image/png'], + keyPrefix: 'img-', + }, + }, + }; + + const result = prepareKvsCollectionsForCompilation(schema); + expect(result).toEqual([]); + }); + + it('should inject type: "object" when missing from jsonSchema', () => { + const schema = { + actorKeyValueStoreSchemaVersion: 1, + title: 'Test', + collections: { + data: { + contentTypes: ['application/json'], + key: 'DATA', + jsonSchema: { + properties: { name: { type: 'string' } }, + }, + }, + }, + }; + + const result = prepareKvsCollectionsForCompilation(schema); + expect(result).toHaveLength(1); + expect(result[0].schema.type).toBe('object'); + }); + + it('should not mutate the original schema', () => { + const schema = { + actorKeyValueStoreSchemaVersion: 1, + title: 'Test', + collections: { + data: { + contentTypes: ['application/json'], + key: 'DATA', + jsonSchema: { + properties: { name: { type: 'string' } }, + }, + }, + }, + }; + + prepareKvsCollectionsForCompilation(schema); + expect((schema.collections as any).data.jsonSchema.type).toBeUndefined(); + }); +}); + +describe('makePropertiesRequired', () => { + it('should add properties without defaults to required array', () => { + const schema = { + type: 'object', + properties: { + name: { type: 'string' }, + age: { type: 'number', default: 25 }, + }, + }; + + const result = makePropertiesRequired(schema); + expect(result.required).toEqual(['name']); + }); + + it('should remove existing required entries that have defaults', () => { + const schema = { + type: 'object', + properties: { + name: { type: 'string' }, + age: { type: 'number', default: 25 }, + }, + required: ['age'], + }; + + const result = makePropertiesRequired(schema); + expect(result.required).toContain('name'); + expect(result.required).not.toContain('age'); + }); + + it('should recurse into nested object properties', () => { + const schema = { + type: 'object', + properties: { + nested: { + type: 'object', + properties: { + innerRequired: { type: 'string' }, + innerOptional: { type: 'string', default: 'hello' }, + }, + }, + }, + }; + + const result = makePropertiesRequired(schema); + const { nested } = result.properties as any; + expect(nested.required).toEqual(['innerRequired']); + }); + + it('should not mutate the original schema', () => { + const schema = { + type: 'object', + properties: { + name: { type: 'string' }, + }, + required: [] as string[], + }; + + makePropertiesRequired(schema); + expect(schema.required).toEqual([]); + }); + + it('should return schema unchanged when there are no properties', () => { + const schema = { type: 'object' }; + const result = makePropertiesRequired(schema); + expect(result).toEqual({ type: 'object' }); + }); +}); + +describe('clearAllRequired', () => { + it('should remove top-level required array', () => { + const schema = { + type: 'object', + properties: { + name: { type: 'string' }, + }, + required: ['name'], + }; + + const result = clearAllRequired(schema); + expect(result.required).toBeUndefined(); + }); + + it('should remove required arrays from nested objects', () => { + const schema = { + type: 'object', + properties: { + nested: { + type: 'object', + properties: { + inner: { type: 'string' }, + }, + required: ['inner'], + }, + }, + required: ['nested'], + }; + + const result = clearAllRequired(schema); + expect(result.required).toBeUndefined(); + const { nested } = result.properties as any; + expect(nested.required).toBeUndefined(); + }); + + it('should not mutate the original schema', () => { + const schema = { + type: 'object', + properties: { + name: { type: 'string' }, + }, + required: ['name'], + }; + + clearAllRequired(schema); + expect(schema.required).toEqual(['name']); + }); + + it('should handle schema with no properties', () => { + const schema = { type: 'object', required: ['foo'] }; + const result = clearAllRequired(schema); + expect(result.required).toBeUndefined(); + }); +}); diff --git a/yarn.lock b/yarn.lock index aaf81830b..021701fa4 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5,6 +5,17 @@ __metadata: version: 8 cacheKey: 10c0 +"@apidevtools/json-schema-ref-parser@npm:^11.5.5": + version: 11.9.3 + resolution: "@apidevtools/json-schema-ref-parser@npm:11.9.3" + dependencies: + "@jsdevtools/ono": "npm:^7.1.3" + "@types/json-schema": "npm:^7.0.15" + js-yaml: "npm:^4.1.0" + checksum: 10c0/5745813b3d964279f387677b7a903ba6634cdeaf879ff3a331a694392cbc923763f398506df190be114f2574b8b570baab3e367c2194bb35f50147ff6cf27d7a + languageName: node + linkType: hard + "@apify/actor-memory-expression@npm:^0.1.3": version: 0.1.7 resolution: "@apify/actor-memory-expression@npm:0.1.7" @@ -1067,6 +1078,13 @@ __metadata: languageName: node linkType: hard +"@jsdevtools/ono@npm:^7.1.3": + version: 7.1.3 + resolution: "@jsdevtools/ono@npm:7.1.3" + checksum: 10c0/a9f7e3e8e3bc315a34959934a5e2f874c423cf4eae64377d3fc9de0400ed9f36cb5fd5ebce3300d2e8f4085f557c4a8b591427a583729a87841fda46e6c216b9 + languageName: node + linkType: hard + "@keyv/serialize@npm:^1.1.1": version: 1.1.1 resolution: "@keyv/serialize@npm:1.1.1" @@ -1646,7 +1664,7 @@ __metadata: languageName: node linkType: hard -"@types/lodash@npm:*": +"@types/lodash@npm:*, @types/lodash@npm:^4.17.7": version: 4.17.23 resolution: "@types/lodash@npm:4.17.23" checksum: 10c0/9d9cbfb684e064a2b78aab9e220d398c9c2a7d36bc51a07b184ff382fa043a99b3d00c16c7f109b4eb8614118f4869678dbae7d5c6700ed16fb9340e26cc0bf6 @@ -2324,6 +2342,7 @@ __metadata: istextorbinary: "npm:~9.5.0" jju: "npm:~1.4.0" js-levenshtein: "npm:^1.1.6" + json-schema-to-typescript: "npm:^15.0.4" lint-staged: "npm:^16.0.0" lodash.clonedeep: "npm:^4.5.0" mime: "npm:~4.1.0" @@ -2603,14 +2622,14 @@ __metadata: linkType: hard "b4a@npm:^1.6.4": - version: 1.7.3 - resolution: "b4a@npm:1.7.3" + version: 1.7.4 + resolution: "b4a@npm:1.7.4" peerDependencies: react-native-b4a: "*" peerDependenciesMeta: react-native-b4a: optional: true - checksum: 10c0/ac16d186e00fa0d16de1f1a4af413953bc762d50d5a0e382aaa744a13886600313b7293403ad77fc83f6b1489c3fc2610494d1026754a51d1b7cdac2115a7598 + checksum: 10c0/d6d427bbcb04c07c91c9a57a7cf04832f57301450faa5121abaef799ec9c599da097eb43f7bbdd62657660b8a8f1754b8ca9a604b618d094f1d3bca896e1e703 languageName: node linkType: hard @@ -5815,7 +5834,7 @@ __metadata: languageName: node linkType: hard -"js-yaml@npm:^4.1.1": +"js-yaml@npm:^4.1.0, js-yaml@npm:^4.1.1": version: 4.1.1 resolution: "js-yaml@npm:4.1.1" dependencies: @@ -5833,6 +5852,25 @@ __metadata: languageName: node linkType: hard +"json-schema-to-typescript@npm:^15.0.4": + version: 15.0.4 + resolution: "json-schema-to-typescript@npm:15.0.4" + dependencies: + "@apidevtools/json-schema-ref-parser": "npm:^11.5.5" + "@types/json-schema": "npm:^7.0.15" + "@types/lodash": "npm:^4.17.7" + is-glob: "npm:^4.0.3" + js-yaml: "npm:^4.1.0" + lodash: "npm:^4.17.21" + minimist: "npm:^1.2.8" + prettier: "npm:^3.2.5" + tinyglobby: "npm:^0.2.9" + bin: + json2ts: dist/src/cli.js + checksum: 10c0/1af8a68d5121710f6f2b9998eea062b751ffc45166b0272a17068e64443010b3c6b02360d3446ca696fcb77102bcb82abd717be0869299d1e12bab437287331b + languageName: node + linkType: hard + "json-schema-traverse@npm:^0.4.1": version: 0.4.1 resolution: "json-schema-traverse@npm:0.4.1" @@ -6306,7 +6344,7 @@ __metadata: languageName: node linkType: hard -"minimist@npm:^1.2.0, minimist@npm:^1.2.5, minimist@npm:^1.2.6": +"minimist@npm:^1.2.0, minimist@npm:^1.2.5, minimist@npm:^1.2.6, minimist@npm:^1.2.8": version: 1.2.8 resolution: "minimist@npm:1.2.8" checksum: 10c0/19d3fcdca050087b84c2029841a093691a91259a47def2f18222f41e7645a0b7c44ef4b40e88a1e58a40c84d2ef0ee6047c55594d298146d0eb3f6b737c20ce6 @@ -7122,7 +7160,7 @@ __metadata: languageName: node linkType: hard -"prettier@npm:^3.5.3": +"prettier@npm:^3.2.5, prettier@npm:^3.5.3": version: 3.8.1 resolution: "prettier@npm:3.8.1" bin: @@ -8409,11 +8447,11 @@ __metadata: linkType: hard "text-decoder@npm:^1.1.0": - version: 1.2.3 - resolution: "text-decoder@npm:1.2.3" + version: 1.2.4 + resolution: "text-decoder@npm:1.2.4" dependencies: b4a: "npm:^1.6.4" - checksum: 10c0/569d776b9250158681c83656ef2c3e0a5d5c660c27ca69f87eedef921749a4fbf02095e5f9a0f862a25cf35258379b06e31dee9c125c9f72e273b7ca1a6d1977 + checksum: 10c0/9c75119e6daafbc091672859738be33b612bb48e1e14aeeabdb0bf4e9f81edf4d5656669a2adf1779fdb9b76605838741213e78683d1944cd425c133f16988de languageName: node linkType: hard @@ -8516,7 +8554,7 @@ __metadata: languageName: node linkType: hard -"tinyglobby@npm:^0.2.11, tinyglobby@npm:^0.2.12, tinyglobby@npm:^0.2.15": +"tinyglobby@npm:^0.2.11, tinyglobby@npm:^0.2.12, tinyglobby@npm:^0.2.15, tinyglobby@npm:^0.2.9": version: 0.2.15 resolution: "tinyglobby@npm:0.2.15" dependencies: