From 5b6d730a04b9b34d63922a72c866f6fe42b136d9 Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Thu, 12 Mar 2026 02:45:11 +0100 Subject: [PATCH 1/5] Added mcp server and refactored controllers to use reportService for querying data - Updated geosController to utilize queryGeos from reportService. - Refactored ranksController to use queryRanks from reportService. - Simplified reportController by replacing Firestore queries with queryReport from reportService. - Modified technologiesController to leverage queryTechnologies from reportService. - Refactored versionsController to use queryVersions from reportService. - Introduced reportService.js to centralize query logic for technologies, categories, reports, ranks, geos, and versions. - Added MCP handler to manage Model Context Protocol requests and integrated various tools for technology and report metrics. - Updated package.json and package-lock.json to include new dependencies for MCP and Zod. --- src/controllers/categoriesController.js | 64 +------ src/controllers/geosController.js | 23 ++- src/controllers/ranksController.js | 23 ++- src/controllers/reportController.js | 120 +------------ src/controllers/technologiesController.js | 81 ++------- src/controllers/versionsController.js | 61 ++----- src/index.js | 20 ++- src/mcpHandler.js | 177 +++++++++++++++++++ src/package-lock.json | 202 +++++++++++++++++++++- src/package.json | 4 +- src/utils/reportService.js | 196 +++++++++++++++++++++ 11 files changed, 646 insertions(+), 325 deletions(-) create mode 100644 src/mcpHandler.js create mode 100644 src/utils/reportService.js diff --git a/src/controllers/categoriesController.js b/src/controllers/categoriesController.js index 5b2ed8a..cacf97d 100644 --- a/src/controllers/categoriesController.js +++ b/src/controllers/categoriesController.js @@ -1,60 +1,14 @@ -import { firestore } from '../utils/db.js'; -import { executeQuery, validateArrayParameter } from '../utils/controllerHelpers.js'; +import { handleControllerError } from '../utils/controllerHelpers.js'; +import { queryCategories } from '../utils/reportService.js'; -/** - * List categories with optional filtering and field selection - */ const listCategories = async (req, res) => { - const queryBuilder = async (params) => { - /* - // Validate parameters - const supportedParams = ['category', 'onlyname', 'fields']; - const providedParams = Object.keys(params); - const unsupportedParams = providedParams.filter(param => !supportedParams.includes(param)); - - if (unsupportedParams.length > 0) { - const error = new Error(`Unsupported parameters: ${unsupportedParams.join(', ')}.`); - error.statusCode = 400; - throw error; - } - */ - - const isOnlyNames = params.onlyname || typeof params.onlyname === 'string'; - const hasCustomFields = params.fields && !isOnlyNames; - - let query = firestore.collection('categories').orderBy('category', 'asc'); - - // Apply category filter with validation - const categoryParam = params.category || 'ALL'; - if (categoryParam !== 'ALL') { - const categories = validateArrayParameter(categoryParam, 'category'); - if (categories.length > 0) { - query = query.where('category', 'in', categories); - } - } - - // Apply field selection - if (isOnlyNames) { - query = query.select('category'); - } else if (hasCustomFields) { - const requestedFields = params.fields.split(',').map(f => f.trim()); - query = query.select(...requestedFields); - } - - return query; - }; - - const dataProcessor = (data, params) => { - const isOnlyNames = params.onlyname || typeof params.onlyname === 'string'; - - if (isOnlyNames) { - return data.map(item => item.category); - } - - return data; - }; - - await executeQuery(req, res, 'categories', queryBuilder, dataProcessor); + try { + const data = await queryCategories(req.query); + res.statusCode = 200; + res.end(JSON.stringify(data)); + } catch (error) { + handleControllerError(res, error, 'fetching categories'); + } }; export { listCategories }; diff --git a/src/controllers/geosController.js b/src/controllers/geosController.js index c670aea..5d1013e 100644 --- a/src/controllers/geosController.js +++ b/src/controllers/geosController.js @@ -1,17 +1,14 @@ -import { firestore } from '../utils/db.js'; -import { executeQuery } from '../utils/controllerHelpers.js'; +import { handleControllerError } from '../utils/controllerHelpers.js'; +import { queryGeos } from '../utils/reportService.js'; -/** - * List all geographic locations from database - */ const listGeos = async (req, res) => { - const queryBuilder = async () => { - return firestore.collection('geos').orderBy('mobile_origins', 'desc').select('geo'); - }; - - await executeQuery(req, res, 'geos', queryBuilder); + try { + const data = await queryGeos(); + res.statusCode = 200; + res.end(JSON.stringify(data)); + } catch (error) { + handleControllerError(res, error, 'fetching geos'); + } }; -export { - listGeos -}; +export { listGeos }; diff --git a/src/controllers/ranksController.js b/src/controllers/ranksController.js index 5fd4aae..d89305c 100644 --- a/src/controllers/ranksController.js +++ b/src/controllers/ranksController.js @@ -1,17 +1,14 @@ -import { firestore } from '../utils/db.js'; -import { executeQuery } from '../utils/controllerHelpers.js'; +import { handleControllerError } from '../utils/controllerHelpers.js'; +import { queryRanks } from '../utils/reportService.js'; -/** - * List all rank options from database - */ const listRanks = async (req, res) => { - const queryBuilder = async () => { - return firestore.collection('ranks').orderBy('mobile_origins', 'desc').select('rank'); - }; - - await executeQuery(req, res, 'ranks', queryBuilder); + try { + const data = await queryRanks(); + res.statusCode = 200; + res.end(JSON.stringify(data)); + } catch (error) { + handleControllerError(res, error, 'fetching ranks'); + } }; -export { - listRanks -}; +export { listRanks }; diff --git a/src/controllers/reportController.js b/src/controllers/reportController.js index 3fd4b53..3621058 100644 --- a/src/controllers/reportController.js +++ b/src/controllers/reportController.js @@ -1,132 +1,18 @@ -import { firestoreOld } from '../utils/db.js'; -const firestore = firestoreOld; +import { handleControllerError } from '../utils/controllerHelpers.js'; +import { queryReport } from '../utils/reportService.js'; -import { - REQUIRED_PARAMS, - validateRequiredParams, - sendValidationError, - getLatestDate, - handleControllerError, - validateArrayParameter -} from '../utils/controllerHelpers.js'; - -/** - * Configuration for different report types - */ -const REPORT_CONFIGS = { - adoption: { - table: 'adoption', - dataField: 'adoption' - }, - pageWeight: { - table: 'page_weight', - dataField: 'pageWeight' // TODO: change to page_weight once migrated to new Firestore DB - }, - lighthouse: { - table: 'lighthouse', - dataField: 'lighthouse' - }, - cwv: { - table: 'core_web_vitals', - dataField: 'vitals' - }, - audits: { - table: 'audits', - dataField: 'audits' - } -}; - -/** - * Generic report data controller factory - * Creates controllers for adoption, pageWeight, lighthouse, and cwv data - */ const createReportController = (reportType) => { - const config = REPORT_CONFIGS[reportType]; - if (!config) { - throw new Error(`Unknown report type: ${reportType}`); - } - return async (req, res) => { try { - const params = req.query; - - /* - // Validate supported parameters - const supportedParams = ['technology', 'geo', 'rank', 'start', 'end']; - const providedParams = Object.keys(params); - const unsupportedParams = providedParams.filter(param => !supportedParams.includes(param)); - - if (unsupportedParams.length > 0) { - const error = new Error(`Unsupported parameters: ${unsupportedParams.join(', ')}.`); - error.statusCode = 400; - throw error; - } - */ - - // Validate required parameters using shared utility - const errors = validateRequiredParams(params, []); - - if (errors) { - sendValidationError(res, errors); - return; - } - - // Default technology, geo, and rank to 'ALL' if missing or empty - const technologyParam = params.technology || 'ALL'; - const geoParam = params.geo || 'ALL'; - const rankParam = params.rank || 'ALL'; - - // Validate and process technology array - const techArray = validateArrayParameter(technologyParam, 'technology'); - - // Handle 'latest' date substitution - let startDate = params.start; - if (startDate === 'latest') { - startDate = await getLatestDate(firestore, config.table); - } - - // Build Firestore query - let query = firestore.collection(config.table); - - // Apply required filters - query = query.where('geo', '==', geoParam); - query = query.where('rank', '==', rankParam); - - // Apply technology filter with batch processing - query = query.where('technology', 'in', techArray); - - // Apply version filter with special handling for 'ALL' case - if (params.version && techArray.length === 1) { - //query = query.where('version', '==', params.version); // TODO: Uncomment when migrating to a new data schema - } else { - //query = query.where('version', '==', 'ALL'); - } - - // Apply date filters - if (startDate) query = query.where('date', '>=', startDate); - if (params.end) query = query.where('date', '<=', params.end); - - // Apply field projection to optimize query - query = query.select('date', 'technology', config.dataField); - - // Execute query - const snapshot = await query.get(); - const data = []; - snapshot.forEach(doc => { - data.push(doc.data()); - }); - - // Send response + const data = await queryReport(reportType, req.query); res.statusCode = 200; res.end(JSON.stringify(data)); - } catch (error) { handleControllerError(res, error, `fetching ${reportType} data`); } }; }; -// Export individual controller functions export const listAuditsData = createReportController('audits'); export const listAdoptionData = createReportController('adoption'); export const listCWVTechData = createReportController('cwv'); diff --git a/src/controllers/technologiesController.js b/src/controllers/technologiesController.js index 85c443e..7b3f831 100644 --- a/src/controllers/technologiesController.js +++ b/src/controllers/technologiesController.js @@ -1,75 +1,14 @@ -import { firestore } from '../utils/db.js'; -import { executeQuery, validateTechnologyArray, validateArrayParameter, FIRESTORE_IN_LIMIT } from '../utils/controllerHelpers.js'; +import { handleControllerError } from '../utils/controllerHelpers.js'; +import { queryTechnologies } from '../utils/reportService.js'; -/** - * List technologies with optional filtering and field selection - */ const listTechnologies = async (req, res) => { - const queryBuilder = async (params) => { - /* - // Validate parameters - const supportedParams = ['technology', 'category', 'onlyname', 'fields']; - const providedParams = Object.keys(params); - const unsupportedParams = providedParams.filter(param => !supportedParams.includes(param)); - - if (unsupportedParams.length > 0) { - const error = new Error(`Unsupported parameters: ${unsupportedParams.join(', ')}.`); - error.statusCode = 400; - throw error; - } - */ - - const isOnlyNames = params.onlyname || typeof params.onlyname === 'string'; - const hasCustomFields = params.fields && !isOnlyNames; - - let query = firestore.collection('technologies').orderBy('technology', 'asc'); - - // Apply technology filter with validation - const technologyParam = params.technology || 'ALL'; - if (technologyParam !== 'ALL') { - const technologies = validateTechnologyArray(technologyParam); - if (technologies === null) { - throw new Error(`Too many technologies specified. Maximum ${FIRESTORE_IN_LIMIT} allowed.`); - } - if (technologies.length > 0) { - query = query.where('technology', 'in', technologies); - } - } - - // Apply category filter with validation - if (params.category) { - const categories = validateArrayParameter(params.category, 'category'); - if (categories.length > 0) { - query = query.where('category_obj', 'array-contains-any', categories); - } - } - - // Apply field selection - if (isOnlyNames) { - query = query.select('technology'); - } else if (hasCustomFields) { - const requestedFields = params.fields.split(',').map(f => f.trim()); - query = query.select(...requestedFields); - } else { - query = query.select('technology', 'category', 'description', 'icon', 'origins'); - } - - return query; - }; - - const dataProcessor = (data, params) => { - const isOnlyNames = params.onlyname || typeof params.onlyname === 'string'; - - if (isOnlyNames) { - return data.map(item => item.technology); - } - - return data; - }; - - await executeQuery(req, res, 'technologies', queryBuilder, dataProcessor); + try { + const data = await queryTechnologies(req.query); + res.statusCode = 200; + res.end(JSON.stringify(data)); + } catch (error) { + handleControllerError(res, error, 'fetching technologies'); + } }; -export { - listTechnologies -}; +export { listTechnologies }; diff --git a/src/controllers/versionsController.js b/src/controllers/versionsController.js index d79dfcb..02d3af5 100644 --- a/src/controllers/versionsController.js +++ b/src/controllers/versionsController.js @@ -1,55 +1,14 @@ -import { firestore } from '../utils/db.js'; -import { executeQuery, validateTechnologyArray, FIRESTORE_IN_LIMIT } from '../utils/controllerHelpers.js'; +import { handleControllerError } from '../utils/controllerHelpers.js'; +import { queryVersions } from '../utils/reportService.js'; -/** - * List versions with optional technology filtering - */ const listVersions = async (req, res) => { - const queryBuilder = async (params) => { - /* - // Validate parameters - const supportedParams = ['version', 'technology', 'category', 'onlyname', 'fields']; - const providedParams = Object.keys(params); - const unsupportedParams = providedParams.filter(param => !supportedParams.includes(param)); - - if (unsupportedParams.length > 0) { - const error = new Error(`Unsupported parameters: ${unsupportedParams.join(', ')}.`); - error.statusCode = 400; - throw error; - } - */ - - let query = firestore.collection('versions'); - - // Apply technology filter with validation - const technologyParam = params.technology || 'ALL'; - if (technologyParam !== 'ALL') { - const technologies = validateTechnologyArray(technologyParam); - if (technologies === null) { - throw new Error(`Too many technologies specified. Maximum ${FIRESTORE_IN_LIMIT} allowed.`); - } - if (technologies.length > 0) { - query = query.where('technology', 'in', technologies); - } - } - - // Apply version filter - if (params.version) { - query = query.where('version', '==', params.version); - } - - // Apply field selection - if (params.fields) { - const requestedFields = params.fields.split(',').map(f => f.trim()); - query = query.select(...requestedFields); - } - - return query; - }; - - await executeQuery(req, res, 'versions', queryBuilder); + try { + const data = await queryVersions(req.query); + res.statusCode = 200; + res.end(JSON.stringify(data)); + } catch (error) { + handleControllerError(res, error, 'fetching versions'); + } }; -export { - listVersions -}; +export { listVersions }; diff --git a/src/index.js b/src/index.js index a5831b9..839e26d 100644 --- a/src/index.js +++ b/src/index.js @@ -93,6 +93,23 @@ const isModified = (req, etag) => { // Route handler function const handleRequest = async (req, res) => { try { + // Parse URL path first so we can route /mcp before setting common headers + const pathname = req.path || req.url.split('?')[0]; + + // MCP endpoint — handled before common headers; transport owns the response + if (pathname === '/mcp') { + setCORSHeaders(res); + res.setHeader('Access-Control-Allow-Methods', 'GET, POST, DELETE, OPTIONS'); + if (req.method === 'OPTIONS') { + res.statusCode = 204; + res.end(); + return; + } + const { handleMcp } = await import('./mcpHandler.js'); + await handleMcp(req, res); + return; + } + setCommonHeaders(res); // Handle OPTIONS requests for CORS preflight @@ -102,9 +119,6 @@ const handleRequest = async (req, res) => { return; } - // Parse URL path - robustly handle Express (req.path) or native Node (req.url) - const pathname = req.path || req.url.split('?')[0]; - // Route handling if (pathname === '/' && req.method === 'GET') { // Health check endpoint diff --git a/src/mcpHandler.js b/src/mcpHandler.js new file mode 100644 index 0000000..be6742d --- /dev/null +++ b/src/mcpHandler.js @@ -0,0 +1,177 @@ +import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; +import { StreamableHTTPServerTransport } from '@modelcontextprotocol/sdk/server/streamableHttp.js'; +import { z } from 'zod'; + +import { + queryTechnologies, + queryCategories, + queryReport, + queryRanks, + queryGeos, + queryVersions, +} from './utils/reportService.js'; + +const createMcpServer = () => { + const server = new McpServer({ + name: 'tech-report', + version: '1.0.0', + }); + + server.tool( + 'search_technologies', + 'Search and filter web technologies tracked by HTTP Archive Tech Report. Returns technology metadata including categories, descriptions, and origin counts.', + { + technology: z.string().optional().describe('Comma-separated technology names to filter by (e.g. "WordPress,Drupal")'), + category: z.string().optional().describe('Comma-separated category names to filter by (e.g. "CMS,CDN")'), + sort: z.enum(['name']).optional().describe('Sort results by "name" (defaults to popularity)'), + limit: z.number().optional().describe('Limit the number of results returned'), + }, + async ({ technology, category, sort, limit }) => { + const data = await queryTechnologies({ technology, category, sort, limit }); + return { content: [{ type: 'text', text: JSON.stringify(data) }] }; + } + ); + + server.tool( + 'list_categories', + 'List all technology categories tracked by HTTP Archive Tech Report (e.g. CMS, CDN, JavaScript, Analytics).', + { + category: z.string().optional().describe('Comma-separated category names to filter results'), + sort: z.enum(['name']).optional().describe('Sort results by "name" (defaults to popularity)'), + limit: z.number().optional().describe('Limit the number of results returned'), + }, + async ({ category, sort, limit }) => { + const data = await queryCategories({ category, sort, limit }); + return { content: [{ type: 'text', text: JSON.stringify(data) }] }; + } + ); + + server.tool( + 'get_adoption_metrics', + 'Get web technology adoption metrics over time from HTTP Archive. Returns the percentage of websites using a technology for a given geography, rank segment, and date range.', + { + technology: z.string().describe('Comma-separated technology names (e.g. "WordPress" or "WordPress,Drupal")'), + geo: z.string().optional().describe('Geographic region (e.g. "ALL", "US", "GB"). Defaults to "ALL"'), + rank: z.string().optional().describe('Traffic rank segment (e.g. "ALL", "top 1000", "top 10000"). Defaults to "ALL"'), + start: z.string().optional().describe('Start date in YYYY-MM-DD format, or "latest" for most recent data'), + end: z.string().optional().describe('End date in YYYY-MM-DD format'), + }, + async ({ technology, geo, rank, start, end }) => { + const data = await queryReport('adoption', { technology, geo, rank, start, end }); + return { content: [{ type: 'text', text: JSON.stringify(data) }] }; + } + ); + + server.tool( + 'get_cwv_metrics', + 'Get Core Web Vitals (CWV) metrics for websites using specific web technologies. Returns good/needs improvement/poor rates for LCP, CLS, INP, and TTFB.', + { + technology: z.string().describe('Comma-separated technology names (e.g. "WordPress" or "WordPress,Drupal")'), + geo: z.string().optional().describe('Geographic region (e.g. "ALL", "US", "GB"). Defaults to "ALL"'), + rank: z.string().optional().describe('Traffic rank segment (e.g. "ALL", "top 1000", "top 10000"). Defaults to "ALL"'), + start: z.string().optional().describe('Start date in YYYY-MM-DD format, or "latest" for most recent data'), + end: z.string().optional().describe('End date in YYYY-MM-DD format'), + }, + async ({ technology, geo, rank, start, end }) => { + const data = await queryReport('cwv', { technology, geo, rank, start, end }); + return { content: [{ type: 'text', text: JSON.stringify(data) }] }; + } + ); + + server.tool( + 'get_lighthouse_metrics', + 'Get Google Lighthouse audit scores for websites using specific web technologies. Returns median scores for Performance, Accessibility, Best Practices, and SEO.', + { + technology: z.string().describe('Comma-separated technology names (e.g. "WordPress" or "WordPress,Drupal")'), + geo: z.string().optional().describe('Geographic region (e.g. "ALL", "US", "GB"). Defaults to "ALL"'), + rank: z.string().optional().describe('Traffic rank segment (e.g. "ALL", "top 1000", "top 10000"). Defaults to "ALL"'), + start: z.string().optional().describe('Start date in YYYY-MM-DD format, or "latest" for most recent data'), + end: z.string().optional().describe('End date in YYYY-MM-DD format'), + }, + async ({ technology, geo, rank, start, end }) => { + const data = await queryReport('lighthouse', { technology, geo, rank, start, end }); + return { content: [{ type: 'text', text: JSON.stringify(data) }] }; + } + ); + + server.tool( + 'get_page_weight_metrics', + 'Get page weight and size metrics for websites using specific web technologies. Returns total page weight, JavaScript size, CSS size, and other resource sizes in bytes.', + { + technology: z.string().describe('Comma-separated technology names (e.g. "WordPress" or "WordPress,Drupal")'), + geo: z.string().optional().describe('Geographic region (e.g. "ALL", "US", "GB"). Defaults to "ALL"'), + rank: z.string().optional().describe('Traffic rank segment (e.g. "ALL", "top 1000", "top 10000"). Defaults to "ALL"'), + start: z.string().optional().describe('Start date in YYYY-MM-DD format, or "latest" for most recent data'), + end: z.string().optional().describe('End date in YYYY-MM-DD format'), + }, + async ({ technology, geo, rank, start, end }) => { + const data = await queryReport('pageWeight', { technology, geo, rank, start, end }); + return { content: [{ type: 'text', text: JSON.stringify(data) }] }; + } + ); + + server.tool( + 'get_audits_metrics', + 'Get web performance and quality audit metrics for websites using specific technologies, sourced from HTTP Archive crawl data.', + { + technology: z.string().describe('Comma-separated technology names (e.g. "WordPress" or "WordPress,Drupal")'), + geo: z.string().optional().describe('Geographic region (e.g. "ALL", "US", "GB"). Defaults to "ALL"'), + rank: z.string().optional().describe('Traffic rank segment (e.g. "ALL", "top 1000", "top 10000"). Defaults to "ALL"'), + start: z.string().optional().describe('Start date in YYYY-MM-DD format, or "latest" for most recent data'), + end: z.string().optional().describe('End date in YYYY-MM-DD format'), + }, + async ({ technology, geo, rank, start, end }) => { + const data = await queryReport('audits', { technology, geo, rank, start, end }); + return { content: [{ type: 'text', text: JSON.stringify(data) }] }; + } + ); + + server.tool( + 'list_ranks', + 'List available traffic rank segments for filtering Tech Report data (e.g. "top 1000", "top 10000", "top 100000", "ALL").', + async () => { + const data = await queryRanks(); + return { content: [{ type: 'text', text: JSON.stringify(data) }] }; + } + ); + + server.tool( + 'list_geos', + 'List available geographic regions for filtering Tech Report data (e.g. "ALL", "US", "GB", "IN").', + async () => { + const data = await queryGeos(); + return { content: [{ type: 'text', text: JSON.stringify(data) }] }; + } + ); + + server.tool( + 'list_versions', + 'List technology versions tracked in HTTP Archive Tech Report.', + { + technology: z.string().optional().describe('Comma-separated technology names to filter versions'), + version: z.string().optional().describe('Exact version string to look up'), + }, + async ({ technology, version }) => { + const data = await queryVersions({ technology, version }); + return { content: [{ type: 'text', text: JSON.stringify(data) }] }; + } + ); + + return server; +}; + +export const handleMcp = async (req, res) => { + const transport = new StreamableHTTPServerTransport({ + sessionIdGenerator: undefined, // stateless — safe for Cloud Run + }); + + const server = createMcpServer(); + await server.connect(transport); + + res.on('close', () => { + transport.close(); + server.close(); + }); + + await transport.handleRequest(req, res, req.body); +}; diff --git a/src/package-lock.json b/src/package-lock.json index 81a63c5..7f6e8fc 100644 --- a/src/package-lock.json +++ b/src/package-lock.json @@ -10,7 +10,9 @@ "dependencies": { "@google-cloud/firestore": "8.3.0", "@google-cloud/functions-framework": "^5.0.2", - "@google-cloud/storage": "7.19.0" + "@google-cloud/storage": "7.19.0", + "@modelcontextprotocol/sdk": "^1.0.0", + "zod": "^3.0.0" }, "devDependencies": { "@jest/transform": "^30.2.0", @@ -51,6 +53,7 @@ "integrity": "sha512-e7jT4DxYvIDLk1ZHmU/m/mB19rex9sv0c2ftBtjSBv+kVM/902eh0fINUzD7UwLLNR+jU585GxUJ8/EBfAM5fw==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@babel/code-frame": "^7.27.1", "@babel/generator": "^7.28.5", @@ -695,6 +698,18 @@ "node": ">=6" } }, + "node_modules/@hono/node-server": { + "version": "1.19.11", + "resolved": "https://registry.npmjs.org/@hono/node-server/-/node-server-1.19.11.tgz", + "integrity": "sha512-dr8/3zEaB+p0D2n/IUrlPF1HZm586qgJNXK1a9fhg/PzdtkK7Ksd5l312tJX2yBuALqDYBlG20QEbayqPyxn+g==", + "license": "MIT", + "engines": { + "node": ">=18.14.1" + }, + "peerDependencies": { + "hono": "^4" + } + }, "node_modules/@isaacs/cliui": { "version": "8.0.2", "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", @@ -1141,6 +1156,63 @@ "url": "https://opencollective.com/js-sdsl" } }, + "node_modules/@modelcontextprotocol/sdk": { + "version": "1.27.1", + "resolved": "https://registry.npmjs.org/@modelcontextprotocol/sdk/-/sdk-1.27.1.tgz", + "integrity": "sha512-sr6GbP+4edBwFndLbM60gf07z0FQ79gaExpnsjMGePXqFcSSb7t6iscpjk9DhFhwd+mTEQrzNafGP8/iGGFYaA==", + "license": "MIT", + "dependencies": { + "@hono/node-server": "^1.19.9", + "ajv": "^8.17.1", + "ajv-formats": "^3.0.1", + "content-type": "^1.0.5", + "cors": "^2.8.5", + "cross-spawn": "^7.0.5", + "eventsource": "^3.0.2", + "eventsource-parser": "^3.0.0", + "express": "^5.2.1", + "express-rate-limit": "^8.2.1", + "hono": "^4.11.4", + "jose": "^6.1.3", + "json-schema-typed": "^8.0.2", + "pkce-challenge": "^5.0.0", + "raw-body": "^3.0.0", + "zod": "^3.25 || ^4.0", + "zod-to-json-schema": "^3.25.1" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@cfworker/json-schema": "^4.1.1", + "zod": "^3.25 || ^4.0" + }, + "peerDependenciesMeta": { + "@cfworker/json-schema": { + "optional": true + }, + "zod": { + "optional": false + } + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/ajv-formats": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-3.0.1.tgz", + "integrity": "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==", + "license": "MIT", + "dependencies": { + "ajv": "^8.0.0" + }, + "peerDependencies": { + "ajv": "^8.0.0" + }, + "peerDependenciesMeta": { + "ajv": { + "optional": true + } + } + }, "node_modules/@napi-rs/wasm-runtime": { "version": "0.2.12", "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-0.2.12.tgz", @@ -2225,6 +2297,7 @@ } ], "license": "MIT", + "peer": true, "dependencies": { "baseline-browser-mapping": "^2.9.0", "caniuse-lite": "^1.0.30001759", @@ -2617,6 +2690,23 @@ "dev": true, "license": "MIT" }, + "node_modules/cors": { + "version": "2.8.6", + "resolved": "https://registry.npmjs.org/cors/-/cors-2.8.6.tgz", + "integrity": "sha512-tJtZBBHA6vjIAaF6EnIaq6laBBP9aq/Y3ouVJjEfoHbRBcHBAHYcMh/w8LDrk2PvIMMq8gmopa5D4V8RmbrxGw==", + "license": "MIT", + "dependencies": { + "object-assign": "^4", + "vary": "^1" + }, + "engines": { + "node": ">= 0.10" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, "node_modules/cross-spawn": { "version": "7.0.6", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", @@ -2941,6 +3031,27 @@ "node": ">=6" } }, + "node_modules/eventsource": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/eventsource/-/eventsource-3.0.7.tgz", + "integrity": "sha512-CRT1WTyuQoD771GW56XEZFQ/ZoSfWid1alKGDYMmkt2yl8UXrVR4pspqWNEcqKvVIzg6PAltWjxcSSPrboA4iA==", + "license": "MIT", + "dependencies": { + "eventsource-parser": "^3.0.1" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/eventsource-parser": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/eventsource-parser/-/eventsource-parser-3.0.6.tgz", + "integrity": "sha512-Vo1ab+QXPzZ4tCa8SwIHJFaSzy4R6SHf7BY79rFBDf0idraZWAkYrDjDj8uWaSm3S2TK+hJ7/t1CEmZ7jXw+pg==", + "license": "MIT", + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/execa": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/execa/-/execa-5.1.1.tgz", @@ -3043,6 +3154,24 @@ "url": "https://opencollective.com/express" } }, + "node_modules/express-rate-limit": { + "version": "8.3.1", + "resolved": "https://registry.npmjs.org/express-rate-limit/-/express-rate-limit-8.3.1.tgz", + "integrity": "sha512-D1dKN+cmyPWuvB+G2SREQDzPY1agpBIcTa9sJxOPMCNeH3gwzhqJRDWCXW3gg0y//+LQ/8j52JbMROWyrKdMdw==", + "license": "MIT", + "dependencies": { + "ip-address": "10.1.0" + }, + "engines": { + "node": ">= 16" + }, + "funding": { + "url": "https://github.com/sponsors/express-rate-limit" + }, + "peerDependencies": { + "express": ">= 4.11" + } + }, "node_modules/express/node_modules/mime-db": { "version": "1.54.0", "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", @@ -3786,6 +3915,16 @@ "node": ">= 0.4" } }, + "node_modules/hono": { + "version": "4.12.7", + "resolved": "https://registry.npmjs.org/hono/-/hono-4.12.7.tgz", + "integrity": "sha512-jq9l1DM0zVIvsm3lv9Nw9nlJnMNPOcAtsbsgiUhWcFzPE99Gvo6yRTlszSLLYacMeQ6quHD6hMfId8crVHvexw==", + "license": "MIT", + "peer": true, + "engines": { + "node": ">=16.9.0" + } + }, "node_modules/hosted-git-info": { "version": "7.0.2", "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-7.0.2.tgz", @@ -3972,6 +4111,15 @@ "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", "license": "ISC" }, + "node_modules/ip-address": { + "version": "10.1.0", + "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.1.0.tgz", + "integrity": "sha512-XXADHxXmvT9+CRxhXg56LJovE+bmWnEWB78LB83VZTprKTmaC5QfruXocxzTZ2Kl0DNwKuBdlIhjL8LeY8Sf8Q==", + "license": "MIT", + "engines": { + "node": ">= 12" + } + }, "node_modules/ipaddr.js": { "version": "1.9.1", "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", @@ -4800,6 +4948,15 @@ "url": "https://github.com/chalk/supports-color?sponsor=1" } }, + "node_modules/jose": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/jose/-/jose-6.2.1.tgz", + "integrity": "sha512-jUaKr1yrbfaImV7R2TN/b3IcZzsw38/chqMpo2XJ7i2F8AfM/lA4G1goC3JVEwg0H7UldTmSt3P68nt31W7/mw==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/panva" + } + }, "node_modules/js-tokens": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", @@ -4855,6 +5012,12 @@ "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", "license": "MIT" }, + "node_modules/json-schema-typed": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/json-schema-typed/-/json-schema-typed-8.0.2.tgz", + "integrity": "sha512-fQhoXdcvc3V28x7C7BMs4P5+kNlgUURe2jmUT1T//oBRMDrqy1QPelJimwZGo7Hg9VPV3EQV5Bnq4hbFy2vetA==", + "license": "BSD-2-Clause" + }, "node_modules/json5": { "version": "2.2.3", "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz", @@ -5233,6 +5396,15 @@ "node": ">=8" } }, + "node_modules/object-assign": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", + "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/object-hash": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/object-hash/-/object-hash-3.0.0.tgz", @@ -5469,6 +5641,15 @@ "node": ">= 6" } }, + "node_modules/pkce-challenge": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/pkce-challenge/-/pkce-challenge-5.0.1.tgz", + "integrity": "sha512-wQ0b/W4Fr01qtpHlqSqspcj3EhBvimsdh0KlHhH8HRZnMsEa0ea2fTULOXOS9ccQr3om+GcGRk4e+isrZWV8qQ==", + "license": "MIT", + "engines": { + "node": ">=16.20.0" + } + }, "node_modules/pkg-dir": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-4.2.0.tgz", @@ -7123,6 +7304,25 @@ "funding": { "url": "https://github.com/sponsors/sindresorhus" } + }, + "node_modules/zod": { + "version": "3.25.76", + "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", + "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", + "license": "MIT", + "peer": true, + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + }, + "node_modules/zod-to-json-schema": { + "version": "3.25.1", + "resolved": "https://registry.npmjs.org/zod-to-json-schema/-/zod-to-json-schema-3.25.1.tgz", + "integrity": "sha512-pM/SU9d3YAggzi6MtR4h7ruuQlqKtad8e9S0fmxcMi+ueAK5Korys/aWcV9LIIHTVbj01NdzxcnXSN+O74ZIVA==", + "license": "ISC", + "peerDependencies": { + "zod": "^3.25 || ^4" + } } } } diff --git a/src/package.json b/src/package.json index bae4930..59b6ca7 100644 --- a/src/package.json +++ b/src/package.json @@ -17,7 +17,9 @@ "dependencies": { "@google-cloud/firestore": "8.3.0", "@google-cloud/functions-framework": "^5.0.2", - "@google-cloud/storage": "7.19.0" + "@google-cloud/storage": "7.19.0", + "@modelcontextprotocol/sdk": "^1.0.0", + "zod": "^3.0.0" }, "devDependencies": { "@jest/transform": "^30.2.0", diff --git a/src/utils/reportService.js b/src/utils/reportService.js new file mode 100644 index 0000000..60396d4 --- /dev/null +++ b/src/utils/reportService.js @@ -0,0 +1,196 @@ +import { firestore, firestoreOld } from './db.js'; +import { + getLatestDate, + validateArrayParameter, + validateTechnologyArray, + FIRESTORE_IN_LIMIT, +} from './controllerHelpers.js'; + +const REPORT_CONFIGS = { + adoption: { table: 'adoption', dataField: 'adoption' }, + pageWeight: { table: 'page_weight', dataField: 'pageWeight' }, + lighthouse: { table: 'lighthouse', dataField: 'lighthouse' }, + cwv: { table: 'core_web_vitals', dataField: 'vitals' }, + audits: { table: 'audits', dataField: 'audits' }, +}; + +export const queryTechnologies = async (params = {}) => { + const isOnlyNames = 'onlyname' in params; + const hasCustomFields = params.fields && !isOnlyNames; + + let query = firestore.collection('technologies'); + + const technologyParam = params.technology || 'ALL'; + const technologies = technologyParam !== 'ALL' ? validateTechnologyArray(technologyParam) : []; + + if (technologies.length > 0) { + if (technologyParam !== 'ALL' && validateTechnologyArray(technologyParam) === null) { + const err = new Error(`Too many technologies specified. Maximum ${FIRESTORE_IN_LIMIT} allowed.`); + err.statusCode = 400; + throw err; + } + query = query.where('technology', 'in', technologies); + } + + if (params.category) { + const categories = validateArrayParameter(params.category, 'category'); + if (categories.length > 0) { + query = query.where('category_obj', 'array-contains-any', categories); + } + } + + if (params.sort === 'name') { + query = query.orderBy('technology', 'asc'); + } else { + query = query.orderBy('origins.mobile', 'desc'); + } + + if (isOnlyNames) { + query = query.select('technology'); + } else if (hasCustomFields) { + const requestedFields = params.fields.split(',').map(f => f.trim()); + query = query.select(...requestedFields); + } else { + query = query.select('technology', 'category', 'description', 'icon', 'origins'); + } + + if (params.limit) { + query = query.limit(parseInt(params.limit, 10)); + } + + const snapshot = await query.get(); + const data = []; + snapshot.forEach(doc => data.push(doc.data())); + + if (isOnlyNames) { + return data.map(item => item.technology); + } + return data; +}; + +export const queryCategories = async (params = {}) => { + const isOnlyNames = 'onlyname' in params; + const hasCustomFields = params.fields && !isOnlyNames; + + let query = firestore.collection('categories'); + + const categoryParam = params.category || 'ALL'; + + if (categoryParam !== 'ALL') { + const categories = validateArrayParameter(categoryParam, 'category'); + if (categories.length > 0) { + query = query.where('category', 'in', categories); + } + } + + if (params.sort === 'name') { + query = query.orderBy('category', 'asc'); + } else { + query = query.orderBy('origins.mobile', 'desc'); + } + + if (isOnlyNames) { + query = query.select('category'); + } else if (hasCustomFields) { + const requestedFields = params.fields.split(',').map(f => f.trim()); + query = query.select(...requestedFields); + } + + if (params.limit) { + query = query.limit(parseInt(params.limit, 10)); + } + + const snapshot = await query.get(); + const data = []; + snapshot.forEach(doc => data.push(doc.data())); + + if (isOnlyNames) { + return data.map(item => item.category); + } + return data; +}; + +export const queryReport = async (reportType, params = {}) => { + const config = REPORT_CONFIGS[reportType]; + if (!config) throw new Error(`Unknown report type: ${reportType}`); + + const db = firestoreOld; + const technologyParam = params.technology || 'ALL'; + const geoParam = params.geo || 'ALL'; + const rankParam = params.rank || 'ALL'; + + const techArray = validateArrayParameter(technologyParam, 'technology'); + + let startDate = params.start; + if (startDate === 'latest') { + startDate = await getLatestDate(db, config.table); + } + + let query = db.collection(config.table); + query = query.where('geo', '==', geoParam); + query = query.where('rank', '==', rankParam); + query = query.where('technology', 'in', techArray); + + if (startDate) query = query.where('date', '>=', startDate); + if (params.end) query = query.where('date', '<=', params.end); + + query = query.select('date', 'technology', config.dataField); + + const snapshot = await query.get(); + const data = []; + snapshot.forEach(doc => data.push(doc.data())); + return data; +}; + +export const queryRanks = async () => { + const snapshot = await firestore + .collection('ranks') + .orderBy('mobile_origins', 'desc') + .select('rank') + .get(); + const data = []; + snapshot.forEach(doc => data.push(doc.data())); + return data; +}; + +export const queryGeos = async () => { + const snapshot = await firestore + .collection('geos') + .orderBy('mobile_origins', 'desc') + .select('geo') + .get(); + const data = []; + snapshot.forEach(doc => data.push(doc.data())); + return data; +}; + +export const queryVersions = async (params = {}) => { + let query = firestore.collection('versions'); + + const technologyParam = params.technology || 'ALL'; + if (technologyParam !== 'ALL') { + const technologies = validateTechnologyArray(technologyParam); + if (technologies === null) { + const err = new Error(`Too many technologies specified. Maximum ${FIRESTORE_IN_LIMIT} allowed.`); + err.statusCode = 400; + throw err; + } + if (technologies.length > 0) { + query = query.where('technology', 'in', technologies); + } + } + + if (params.version) { + query = query.where('version', '==', params.version); + } + + if (params.fields) { + const requestedFields = params.fields.split(',').map(f => f.trim()); + query = query.select(...requestedFields); + } + + const snapshot = await query.get(); + const data = []; + snapshot.forEach(doc => data.push(doc.data())); + return data; +}; From 1fb9fedb3600ae65a8c68cdae6184d842322486f Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Sun, 5 Apr 2026 00:08:33 +0200 Subject: [PATCH 2/5] feat(geoBreakdown): add geo breakdown data endpoint and query function --- src/controllers/reportController.js | 12 +++++++++++- src/index.js | 7 +++++++ src/utils/reportService.js | 21 +++++++++++++++++++++ 3 files changed, 39 insertions(+), 1 deletion(-) diff --git a/src/controllers/reportController.js b/src/controllers/reportController.js index 3621058..3fc5e15 100644 --- a/src/controllers/reportController.js +++ b/src/controllers/reportController.js @@ -1,5 +1,5 @@ import { handleControllerError } from '../utils/controllerHelpers.js'; -import { queryReport } from '../utils/reportService.js'; +import { queryReport, queryGeoBreakdown } from '../utils/reportService.js'; const createReportController = (reportType) => { return async (req, res) => { @@ -19,4 +19,14 @@ export const listCWVTechData = createReportController('cwv'); export const listLighthouseData = createReportController('lighthouse'); export const listPageWeightData = createReportController('pageWeight'); +export const listGeoBreakdownData = async (req, res) => { + try { + const data = await queryGeoBreakdown(req.query); + res.statusCode = 200; + res.end(JSON.stringify(data)); + } catch (error) { + handleControllerError(res, error, 'fetching geo breakdown data'); + } +}; + diff --git a/src/index.js b/src/index.js index 839e26d..ec1ad89 100644 --- a/src/index.js +++ b/src/index.js @@ -13,6 +13,7 @@ const controllers = { ranks: null, geos: null, versions: null, + geoBreakdown: null, static: null }; @@ -42,6 +43,9 @@ const getController = async (name) => { case 'versions': controllers[name] = await import('./controllers/versionsController.js'); break; + case 'geoBreakdown': + controllers[name] = await import('./controllers/reportController.js'); + break; case 'static': controllers[name] = await import('./controllers/cdnController.js'); break; @@ -154,6 +158,9 @@ const handleRequest = async (req, res) => { } else if (pathname === '/v1/versions' && req.method === 'GET') { const { listVersions } = await getController('versions'); await listVersions(req, res); + } else if (pathname === '/v1/geo-breakdown' && req.method === 'GET') { + const { listGeoBreakdownData } = await getController('geoBreakdown'); + await listGeoBreakdownData(req, res); } else if (pathname.startsWith('/v1/static/') && req.method === 'GET') { // GCS proxy endpoint for reports files const filePath = decodeURIComponent(pathname.replace('/v1/static/', '')); diff --git a/src/utils/reportService.js b/src/utils/reportService.js index 60396d4..f9863f8 100644 --- a/src/utils/reportService.js +++ b/src/utils/reportService.js @@ -142,6 +142,27 @@ export const queryReport = async (reportType, params = {}) => { return data; }; +export const queryGeoBreakdown = async (params = {}) => { + const config = REPORT_CONFIGS['cwv']; + const db = firestoreOld; + const technologyParam = params.technology || 'ALL'; + const rankParam = params.rank || 'ALL'; + + const techArray = validateArrayParameter(technologyParam, 'technology'); + const snapshotDate = params.end || await getLatestDate(db, config.table); + + let query = db.collection(config.table); + query = query.where('rank', '==', rankParam); + query = query.where('technology', 'in', techArray); + query = query.where('date', '==', snapshotDate); + query = query.select('date', 'technology', 'geo', config.dataField); + + const snapshot = await query.get(); + const data = []; + snapshot.forEach(doc => data.push(doc.data())); + return data; +}; + export const queryRanks = async () => { const snapshot = await firestore .collection('ranks') From f61c5e8c79474e4bee63d58afc170f9554b2c82f Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Tue, 7 Apr 2026 00:33:29 +0200 Subject: [PATCH 3/5] refactor: streamline response handling by consolidating JSON response logic --- src/controllers/categoriesController.js | 5 ++--- src/controllers/geosController.js | 5 ++--- src/controllers/ranksController.js | 5 ++--- src/controllers/reportController.js | 15 ++------------- src/controllers/technologiesController.js | 13 ++----------- src/controllers/versionsController.js | 5 ++--- src/index.js | 4 ++-- src/utils/controllerHelpers.js | 18 +++++++----------- 8 files changed, 21 insertions(+), 49 deletions(-) diff --git a/src/controllers/categoriesController.js b/src/controllers/categoriesController.js index cacf97d..d2dfb22 100644 --- a/src/controllers/categoriesController.js +++ b/src/controllers/categoriesController.js @@ -1,11 +1,10 @@ -import { handleControllerError } from '../utils/controllerHelpers.js'; +import { handleControllerError, sendJSONResponse } from '../utils/controllerHelpers.js'; import { queryCategories } from '../utils/reportService.js'; const listCategories = async (req, res) => { try { const data = await queryCategories(req.query); - res.statusCode = 200; - res.end(JSON.stringify(data)); + sendJSONResponse(req, res, data); } catch (error) { handleControllerError(res, error, 'fetching categories'); } diff --git a/src/controllers/geosController.js b/src/controllers/geosController.js index 5d1013e..839aaaa 100644 --- a/src/controllers/geosController.js +++ b/src/controllers/geosController.js @@ -1,11 +1,10 @@ -import { handleControllerError } from '../utils/controllerHelpers.js'; +import { handleControllerError, sendJSONResponse } from '../utils/controllerHelpers.js'; import { queryGeos } from '../utils/reportService.js'; const listGeos = async (req, res) => { try { const data = await queryGeos(); - res.statusCode = 200; - res.end(JSON.stringify(data)); + sendJSONResponse(req, res, data); } catch (error) { handleControllerError(res, error, 'fetching geos'); } diff --git a/src/controllers/ranksController.js b/src/controllers/ranksController.js index d89305c..98d9f4f 100644 --- a/src/controllers/ranksController.js +++ b/src/controllers/ranksController.js @@ -1,11 +1,10 @@ -import { handleControllerError } from '../utils/controllerHelpers.js'; +import { handleControllerError, sendJSONResponse } from '../utils/controllerHelpers.js'; import { queryRanks } from '../utils/reportService.js'; const listRanks = async (req, res) => { try { const data = await queryRanks(); - res.statusCode = 200; - res.end(JSON.stringify(data)); + sendJSONResponse(req, res, data); } catch (error) { handleControllerError(res, error, 'fetching ranks'); } diff --git a/src/controllers/reportController.js b/src/controllers/reportController.js index 27444f5..25e0969 100644 --- a/src/controllers/reportController.js +++ b/src/controllers/reportController.js @@ -1,22 +1,11 @@ -import { handleControllerError, generateETag, isModified } from '../utils/controllerHelpers.js'; +import { handleControllerError, sendJSONResponse } from '../utils/controllerHelpers.js'; import { queryReport } from '../utils/reportService.js'; const createReportController = (reportType, defaults = {}) => { return async (req, res) => { try { const data = await queryReport(reportType, { ...defaults, ...req.query }); - - // Send response with ETag support - const jsonData = JSON.stringify(data); - const etag = generateETag(jsonData); - res.setHeader('ETag', `"${etag}"`); - if (!isModified(req, etag)) { - res.statusCode = 304; - res.end(); - return; - } - res.statusCode = 200; - res.end(jsonData); + sendJSONResponse(req, res, data); } catch (error) { handleControllerError(res, error, `fetching ${reportType} data`); } diff --git a/src/controllers/technologiesController.js b/src/controllers/technologiesController.js index 7d98022..b991160 100644 --- a/src/controllers/technologiesController.js +++ b/src/controllers/technologiesController.js @@ -1,19 +1,10 @@ -import { handleControllerError, generateETag, isModified } from '../utils/controllerHelpers.js'; +import { handleControllerError, sendJSONResponse } from '../utils/controllerHelpers.js'; import { queryTechnologies } from '../utils/reportService.js'; const listTechnologies = async (req, res) => { try { const data = await queryTechnologies(req.query); - const jsonData = JSON.stringify(data); - const etag = generateETag(jsonData); - res.setHeader('ETag', `"${etag}"`); - if (!isModified(req, etag)) { - res.statusCode = 304; - res.end(); - return; - } - res.statusCode = 200; - res.end(jsonData); + sendJSONResponse(req, res, data); } catch (error) { handleControllerError(res, error, 'fetching technologies'); } diff --git a/src/controllers/versionsController.js b/src/controllers/versionsController.js index 02d3af5..b69fd8e 100644 --- a/src/controllers/versionsController.js +++ b/src/controllers/versionsController.js @@ -1,11 +1,10 @@ -import { handleControllerError } from '../utils/controllerHelpers.js'; +import { handleControllerError, sendJSONResponse } from '../utils/controllerHelpers.js'; import { queryVersions } from '../utils/reportService.js'; const listVersions = async (req, res) => { try { const data = await queryVersions(req.query); - res.statusCode = 200; - res.end(JSON.stringify(data)); + sendJSONResponse(req, res, data); } catch (error) { handleControllerError(res, error, 'fetching versions'); } diff --git a/src/index.js b/src/index.js index 3324b31..e2ccf53 100644 --- a/src/index.js +++ b/src/index.js @@ -1,5 +1,5 @@ import functions from '@google-cloud/functions-framework'; -import { sendJSONResponse, isModified } from './utils/controllerHelpers.js'; +import { sendJSONResponse } from './utils/controllerHelpers.js'; // Dynamic imports for better performance - only load when needed const controllers = { @@ -106,7 +106,7 @@ const handleRequest = async (req, res) => { if (pathname === '/' && req.method === 'GET') { // Health check endpoint const data = { status: 'ok' }; - sendJSONResponse(res, data); + sendJSONResponse(req, res, data); } else if (pathname === '/v1/technologies' && req.method === 'GET') { const { listTechnologies } = await getController('technologies'); await listTechnologies(req, res); diff --git a/src/utils/controllerHelpers.js b/src/utils/controllerHelpers.js index 58583d6..3ff005e 100644 --- a/src/utils/controllerHelpers.js +++ b/src/utils/controllerHelpers.js @@ -104,10 +104,15 @@ const generateETag = (jsonData) => { return crypto.createHash('md5').update(jsonData).digest('hex'); }; -const sendJSONResponse = (res, data, statusCode = 200) => { +const sendJSONResponse = (req, res, data, statusCode = 200) => { const jsonData = JSON.stringify(data); const etag = generateETag(jsonData); res.setHeader('ETag', `"${etag}"`); + if (!isModified(req, etag)) { + res.statusCode = 304; + res.end(); + return; + } res.statusCode = statusCode; res.end(jsonData); }; @@ -145,16 +150,7 @@ const executeQuery = async (req, res, collection, queryBuilder, dataProcessor = } // Send response with ETag support - const jsonData = JSON.stringify(data); - const etag = generateETag(jsonData); - res.setHeader('ETag', `"${etag}"`); - if (!isModified(req, etag)) { - res.statusCode = 304; - res.end(); - return; - } - res.statusCode = 200; - res.end(jsonData); + sendJSONResponse(req, res, data); } catch (error) { // Handle validation errors specifically From 04c3bb7cc5464eaed9adb00abd68d4ebd971cd75 Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Tue, 7 Apr 2026 00:35:27 +0200 Subject: [PATCH 4/5] fix: correct Dependabot auto-merge condition placement in workflow --- .github/workflows/dependabot.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/dependabot.yml b/.github/workflows/dependabot.yml index 94650a0..1b650c3 100644 --- a/.github/workflows/dependabot.yml +++ b/.github/workflows/dependabot.yml @@ -1,5 +1,5 @@ --- -name: Dependabot Auto-Merge +name: Test on: pull_request: @@ -8,7 +8,6 @@ on: jobs: test: - if: github.event.pull_request.user.login == 'dependabot[bot]' && github.repository == 'HTTPArchive/tech-report-apis' runs-on: ubuntu-latest steps: - uses: actions/checkout@v6 @@ -19,6 +18,7 @@ jobs: dependabot: name: Dependabot auto-merge + if: github.event.pull_request.user.login == 'dependabot[bot]' && github.repository == 'HTTPArchive/tech-report-apis' runs-on: ubuntu-latest needs: test From cb43c8db68391ff30bed693348349caa8aecbb47 Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Tue, 7 Apr 2026 00:43:13 +0200 Subject: [PATCH 5/5] refactor: simplify controller imports and route handling logic --- src/index.js | 119 +++++++++++++++------------------------------------ 1 file changed, 35 insertions(+), 84 deletions(-) diff --git a/src/index.js b/src/index.js index e2ccf53..1995023 100644 --- a/src/index.js +++ b/src/index.js @@ -1,59 +1,44 @@ import functions from '@google-cloud/functions-framework'; import { sendJSONResponse } from './utils/controllerHelpers.js'; -// Dynamic imports for better performance - only load when needed -const controllers = { - technologies: null, - categories: null, - adoption: null, - cwvtech: null, - lighthouse: null, - pageWeight: null, - audits: null, - ranks: null, - geos: null, - versions: null, - geoBreakdown: null, - static: null +const CONTROLLER_MODULES = { + technologies: './controllers/technologiesController.js', + categories: './controllers/categoriesController.js', + adoption: './controllers/reportController.js', + cwvtech: './controllers/reportController.js', + lighthouse: './controllers/reportController.js', + pageWeight: './controllers/reportController.js', + audits: './controllers/reportController.js', + geoBreakdown: './controllers/reportController.js', + ranks: './controllers/ranksController.js', + geos: './controllers/geosController.js', + versions: './controllers/versionsController.js', + static: './controllers/cdnController.js', }; -// Helper function to dynamically import controllers +const controllers = {}; + const getController = async (name) => { if (!controllers[name]) { - switch (name) { - case 'technologies': - controllers[name] = await import('./controllers/technologiesController.js'); - break; - case 'categories': - controllers[name] = await import('./controllers/categoriesController.js'); - break; - case 'adoption': - case 'cwvtech': - case 'lighthouse': - case 'pageWeight': - case 'audits': - controllers[name] = await import('./controllers/reportController.js'); - break; - case 'ranks': - controllers[name] = await import('./controllers/ranksController.js'); - break; - case 'geos': - controllers[name] = await import('./controllers/geosController.js'); - break; - case 'versions': - controllers[name] = await import('./controllers/versionsController.js'); - break; - case 'geoBreakdown': - controllers[name] = await import('./controllers/reportController.js'); - break; - case 'static': - controllers[name] = await import('./controllers/cdnController.js'); - break; - } + controllers[name] = await import(CONTROLLER_MODULES[name]); } return controllers[name]; }; +const V1_ROUTES = { + '/v1/technologies': ['technologies', 'listTechnologies'], + '/v1/categories': ['categories', 'listCategories'], + '/v1/adoption': ['adoption', 'listAdoptionData'], + '/v1/cwv': ['cwvtech', 'listCWVTechData'], + '/v1/lighthouse': ['lighthouse', 'listLighthouseData'], + '/v1/page-weight': ['pageWeight', 'listPageWeightData'], + '/v1/audits': ['audits', 'listAuditsData'], + '/v1/ranks': ['ranks', 'listRanks'], + '/v1/geos': ['geos', 'listGeos'], + '/v1/versions': ['versions', 'listVersions'], + '/v1/geo-breakdown': ['geoBreakdown', 'listGeoBreakdownData'], +}; + // Helper function to set CORS headers const setCORSHeaders = (res) => { res.setHeader('Access-Control-Allow-Origin', '*'); @@ -102,46 +87,13 @@ const handleRequest = async (req, res) => { return; } - // Route handling if (pathname === '/' && req.method === 'GET') { - // Health check endpoint - const data = { status: 'ok' }; - sendJSONResponse(req, res, data); - } else if (pathname === '/v1/technologies' && req.method === 'GET') { - const { listTechnologies } = await getController('technologies'); - await listTechnologies(req, res); - } else if (pathname === '/v1/categories' && req.method === 'GET') { - const { listCategories } = await getController('categories'); - await listCategories(req, res); - } else if (pathname === '/v1/adoption' && req.method === 'GET') { - const { listAdoptionData } = await getController('adoption'); - await listAdoptionData(req, res); - } else if (pathname === '/v1/cwv' && req.method === 'GET') { - const { listCWVTechData } = await getController('cwvtech'); - await listCWVTechData(req, res); - } else if (pathname === '/v1/lighthouse' && req.method === 'GET') { - const { listLighthouseData } = await getController('lighthouse'); - await listLighthouseData(req, res); - } else if (pathname === '/v1/page-weight' && req.method === 'GET') { - const { listPageWeightData } = await getController('pageWeight'); - await listPageWeightData(req, res); - } else if (pathname === '/v1/audits' && req.method === 'GET') { - const { listAuditsData } = await getController('audits'); - await listAuditsData(req, res); - } else if (pathname === '/v1/ranks' && req.method === 'GET') { - const { listRanks } = await getController('ranks'); - await listRanks(req, res); - } else if (pathname === '/v1/geos' && req.method === 'GET') { - const { listGeos } = await getController('geos'); - await listGeos(req, res); - } else if (pathname === '/v1/versions' && req.method === 'GET') { - const { listVersions } = await getController('versions'); - await listVersions(req, res); - } else if (pathname === '/v1/geo-breakdown' && req.method === 'GET') { - const { listGeoBreakdownData } = await getController('geoBreakdown'); - await listGeoBreakdownData(req, res); + sendJSONResponse(req, res, { status: 'ok' }); + } else if (req.method === 'GET' && V1_ROUTES[pathname]) { + const [controllerKey, handlerName] = V1_ROUTES[pathname]; + const controller = await getController(controllerKey); + await controller[handlerName](req, res); } else if (pathname.startsWith('/v1/static/') && req.method === 'GET') { - // GCS proxy endpoint for reports files const filePath = decodeURIComponent(pathname.replace('/v1/static/', '')); if (!filePath) { res.statusCode = 400; @@ -151,7 +103,6 @@ const handleRequest = async (req, res) => { const { proxyReportsFile } = await getController('static'); await proxyReportsFile(req, res, filePath); } else { - // 404 Not Found res.statusCode = 404; res.end(JSON.stringify({ error: 'Not Found' })); }