From cce4fe68e6c4242efd00dbf3273e84f09834b275 Mon Sep 17 00:00:00 2001 From: thorwebdev Date: Mon, 15 Jul 2024 17:05:04 +0800 Subject: [PATCH 1/3] feat: add postgres provider. --- pnpm-lock.yaml | 17 ++++++ provider/postgres/README.md | 22 ++++++++ provider/postgres/client.ts | 84 ++++++++++++++++++++++++++++++ provider/postgres/index.ts | 77 +++++++++++++++++++++++++++ provider/postgres/package.json | 30 +++++++++++ provider/postgres/tsconfig.json | 11 ++++ provider/postgres/vitest.config.ts | 3 ++ tsconfig.json | 1 + 8 files changed, 245 insertions(+) create mode 100644 provider/postgres/README.md create mode 100644 provider/postgres/client.ts create mode 100644 provider/postgres/index.ts create mode 100644 provider/postgres/package.json create mode 100644 provider/postgres/tsconfig.json create mode 100644 provider/postgres/vitest.config.ts diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index df4e6e02..f1fe9124 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -646,6 +646,18 @@ importers: specifier: workspace:* version: link:../../lib/provider + provider/postgres: + dependencies: + '@openctx/provider': + specifier: workspace:* + version: link:../../lib/provider + dedent: + specifier: ^1.5.3 + version: 1.5.3 + postgres: + specifier: ^3.4.4 + version: 3.4.4 + provider/prometheus: dependencies: '@openctx/provider': @@ -12492,6 +12504,11 @@ packages: source-map-js: 1.2.0 dev: true + /postgres@3.4.4: + resolution: {integrity: sha512-IbyN+9KslkqcXa8AO9fxpk97PA4pzewvpi2B3Dwy9u4zpV32QicaEdgmF3eSQUzdRk7ttDHQejNgAEr4XoeH4A==} + engines: {node: '>=12'} + dev: false + /prebuild-install@7.1.1: resolution: {integrity: sha512-jAXscXWMcCK8GgCoHOfIr0ODh5ai8mj63L2nWrjuAgXE6tDyYGnx4/8o/rCgU+B4JSyZBKbeZqzhtwtC3ovxjw==} engines: {node: '>=10'} diff --git a/provider/postgres/README.md b/provider/postgres/README.md new file mode 100644 index 00000000..c14050a6 --- /dev/null +++ b/provider/postgres/README.md @@ -0,0 +1,22 @@ +# Postgres context provider for OpenCtx + +This is a context provider for [OpenCtx](https://openctx.org) that brings context from your Postgres DB to code AI and editors. Postgres context provider allows to add details about a specific schema from your Postgres Database. + +**Status:** Experimental + +## Configuration + +```json +"openctx.providers": { + // ...other providers... + "https://openctx.org/npm/@openctx/provider-postgres": { + "DB_URL": "" + } +}, +``` + +## Development + +- [Source code](https://sourcegraph.com/github.com/sourcegraph/openctx/-/tree/provider/postgres) +- [Docs](https://openctx.org/docs/providers/postgres) +- License: Apache 2.0 diff --git a/provider/postgres/client.ts b/provider/postgres/client.ts new file mode 100644 index 00000000..f4395632 --- /dev/null +++ b/provider/postgres/client.ts @@ -0,0 +1,84 @@ +import type { ItemsResult } from '@openctx/provider'; +import dedent from 'dedent'; +import postgres from 'postgres'; + +export class PostgresClient { + private readonly sql: postgres.Sql; + public schemas: string[] = []; + + constructor(DB_URL: string) { + this.sql = postgres(DB_URL); + } + + public getSchemas(): string[] { + return this.schemas; + } + + public async getSchema(schema: string): Promise { + const res = await this.sql< + { ['table_name']: string; [key: string]: any }[] + >` + SELECT table_name, column_name, data_type, character_maximum_length, column_default, is_nullable + FROM information_schema.columns + where table_schema = '${schema}'`; + + const schemaDDL = this.schemaToDDL(res); + + const schemaPrompt = dedent` + The reference database schema for this question is ${schemaDDL}. + IMPORTANT: Be sure you only use the tables and columns from this schema in your answer. + `; + + return [ + { + title: schema, + ai: { content: schemaPrompt }, + }, + ]; + } + + // ----------- Helper function --------------- + + private schemaToDDL( + schema: { ['table_name']: string; [key: string]: any }[] + ) { + const tables: { [key: string]: any } = {}; + for (let row of schema) { + tables[row.table_name] = row; + } + const out = []; + const tableNames = Object.keys(tables); + for (let table of tableNames) { + const sql = [`create table ${table}(\n`]; + const cols = schema.filter((s) => s.table_name === table); + for (let c of cols) { + let colSql = ''; + //if (c.column_name === null || c.column_name === "") continue; + colSql = ` ${c.column_name} ${c.data_type}`; + if (c.is_nullable === 'NO') colSql += ' not null '; + if (c.column_default === 'NO') + colSql += ` default ${c.column_default} `; + colSql += ',\n'; + sql.push(colSql); + } + sql.push(');'); + out.push(sql.join('')); + } + return out.join('\n'); + } + + // ----------- Initialization function --------------- + + public async initializePGData() { + await this.initializeSchemas(); + } + + private async initializeSchemas(): Promise { + const schemas = await this.sql` + select schema_name + from information_schema.schemata; + `; + console.log({ schemas }); + return schemas.map((schema) => schema.schema_name); + } +} diff --git a/provider/postgres/index.ts b/provider/postgres/index.ts new file mode 100644 index 00000000..a52cef1f --- /dev/null +++ b/provider/postgres/index.ts @@ -0,0 +1,77 @@ +import type { + ItemsParams, + ItemsResult, + MentionsParams, + MentionsResult, + MetaResult, +} from '@openctx/provider'; +import { PostgresClient } from './client.js'; + +/** Settings for the Postgres provider. */ +export type Settings = { + /** Database URL. */ + DB_URL: string; +}; + +let pgClient: undefined | PostgresClient = undefined; + +const postgresContext = { + meta(): MetaResult { + return { + name: 'Postgres', + mentions: { label: 'Search by schema name...' }, + }; + }, + + async initializePGClient(settingsInput: Settings) { + if (pgClient === undefined) { + pgClient = new PostgresClient(settingsInput.DB_URL); + await pgClient.initializePGData(); + } + }, + + async mentions( + params: MentionsParams, + settingsInput: Settings + ): Promise { + await this.initializePGClient(settingsInput); + if (!pgClient) { + return []; + } + const userQuery = params.query ?? ''; + const schemas = pgClient.getSchemas(); + const schemaList = schemas.filter((schema) => schema.includes(userQuery)); + if (!schemaList) { + return []; + } + const mentionRes: MentionsResult = []; + for (const schema of schemaList) { + mentionRes.push({ + title: schema, + uri: schema, + data: { + schema, + }, + }); + } + return mentionRes; + }, + + async items( + params: ItemsParams, + settingsInput: Settings + ): Promise { + await this.initializePGClient(settingsInput); + if (!pgClient) { + return []; + } + const schema = params.mention?.data?.schema as string; + let message = params.message || ''; + console.log({ schema, message }); + + const schemaContext = await pgClient.getSchema(schema); + return schemaContext; + }, +}; + +export default postgresContext; diff --git a/provider/postgres/package.json b/provider/postgres/package.json new file mode 100644 index 00000000..a3474650 --- /dev/null +++ b/provider/postgres/package.json @@ -0,0 +1,30 @@ +{ + "name": "@openctx/provider-postgres", + "version": "0.0.6", + "description": "Context from your Postgres for code AI and editors (OpenCtx provider)", + "license": "Apache-2.0", + "repository": { + "type": "git", + "url": "https://github.com/sourcegraph/openctx", + "directory": "provider/postgres" + }, + "type": "module", + "main": "dist/bundle.js", + "types": "dist/index.d.ts", + "files": [ + "dist/bundle.js", + "dist/index.d.ts" + ], + "sideEffects": false, + "scripts": { + "bundle": "tsc --build && esbuild --log-level=error --platform=node --bundle --format=esm --outfile=dist/bundle.js index.ts", + "prepublishOnly": "tsc --build --clean && npm run --silent bundle", + "test": "vitest", + "google-auth": "node --no-warnings=ExperimentalWarning --es-module-specifier-resolution=node --loader ts-node/esm/transpile-only auth.ts" + }, + "dependencies": { + "dedent": "^1.5.3", + "@openctx/provider": "workspace:*", + "postgres": "^3.4.4" + } +} diff --git a/provider/postgres/tsconfig.json b/provider/postgres/tsconfig.json new file mode 100644 index 00000000..4ff7ab81 --- /dev/null +++ b/provider/postgres/tsconfig.json @@ -0,0 +1,11 @@ +{ + "extends": "../../.config/tsconfig.base.json", + "compilerOptions": { + "rootDir": ".", + "outDir": "dist", + "lib": ["ESNext"], + }, + "include": ["*.ts"], + "exclude": ["dist", "vitest.config.ts"], + "references": [{ "path": "../../lib/provider" }], +} diff --git a/provider/postgres/vitest.config.ts b/provider/postgres/vitest.config.ts new file mode 100644 index 00000000..abed6b21 --- /dev/null +++ b/provider/postgres/vitest.config.ts @@ -0,0 +1,3 @@ +import { defineConfig } from 'vitest/config' + +export default defineConfig({}) diff --git a/tsconfig.json b/tsconfig.json index 442280c3..c04b6df3 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -31,6 +31,7 @@ { "path": "provider/hello-world" }, { "path": "provider/links" }, { "path": "provider/notion" }, + { "path": "provider/postgres" }, { "path": "provider/prometheus" }, { "path": "provider/sourcegraph-search" }, { "path": "provider/storybook" }, From a86e529ca70cac2c24f329f95742ef36c8b4cfcd Mon Sep 17 00:00:00 2001 From: thorwebdev Date: Mon, 15 Jul 2024 17:47:00 +0800 Subject: [PATCH 2/3] chore: point to bundel.js --- DEVELOPERS.md | 13 +++++++++++++ provider/postgres/package.json | 5 ++--- web/content/docs/creating-a-provider.mdx | 2 +- 3 files changed, 16 insertions(+), 4 deletions(-) create mode 100644 DEVELOPERS.md diff --git a/DEVELOPERS.md b/DEVELOPERS.md new file mode 100644 index 00000000..e96517b6 --- /dev/null +++ b/DEVELOPERS.md @@ -0,0 +1,13 @@ +## Set up for local development + +1. `pnpm i` +2. `pnpm run build` +3. `pnpm run bundle` + +When developing locally, configure your provider in the client (e.g., VS Code) by using the path to the bundled .js file: + +```json +"openctx.providers": { + "file:////bundle.js": true, +} +``` \ No newline at end of file diff --git a/provider/postgres/package.json b/provider/postgres/package.json index a3474650..f9a4593a 100644 --- a/provider/postgres/package.json +++ b/provider/postgres/package.json @@ -1,6 +1,6 @@ { "name": "@openctx/provider-postgres", - "version": "0.0.6", + "version": "0.0.1", "description": "Context from your Postgres for code AI and editors (OpenCtx provider)", "license": "Apache-2.0", "repository": { @@ -19,8 +19,7 @@ "scripts": { "bundle": "tsc --build && esbuild --log-level=error --platform=node --bundle --format=esm --outfile=dist/bundle.js index.ts", "prepublishOnly": "tsc --build --clean && npm run --silent bundle", - "test": "vitest", - "google-auth": "node --no-warnings=ExperimentalWarning --es-module-specifier-resolution=node --loader ts-node/esm/transpile-only auth.ts" + "test": "vitest" }, "dependencies": { "dedent": "^1.5.3", diff --git a/web/content/docs/creating-a-provider.mdx b/web/content/docs/creating-a-provider.mdx index ace75ad0..20fc3ab7 100644 --- a/web/content/docs/creating-a-provider.mdx +++ b/web/content/docs/creating-a-provider.mdx @@ -60,7 +60,7 @@ You can configure that provider in an OpenCtx client like so: When developing locally, configure your provider in the client (e.g., VS Code) by using the path to the bundled `.js` file: ```json "openctx.providers": { - "file:////index.js": true, + "file:////bundle.js": true, } ``` From 332db837c060ecb96edf193c94bbbc05cfb48ea6 Mon Sep 17 00:00:00 2001 From: thorwebdev Date: Mon, 15 Jul 2024 18:17:15 +0800 Subject: [PATCH 3/3] fix: misc fixes. --- DEVELOPERS.md | 6 +++++- provider/postgres/client.ts | 6 +++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/DEVELOPERS.md b/DEVELOPERS.md index e96517b6..7221602f 100644 --- a/DEVELOPERS.md +++ b/DEVELOPERS.md @@ -10,4 +10,8 @@ When developing locally, configure your provider in the client (e.g., VS Code) b "openctx.providers": { "file:////bundle.js": true, } -``` \ No newline at end of file +``` + +## Debug + +To see console log statements of your provider, open the vscode developer tools: [`cmd + Shift + P`] > "Toggle Developer Tools". \ No newline at end of file diff --git a/provider/postgres/client.ts b/provider/postgres/client.ts index f4395632..913bec10 100644 --- a/provider/postgres/client.ts +++ b/provider/postgres/client.ts @@ -20,7 +20,7 @@ export class PostgresClient { >` SELECT table_name, column_name, data_type, character_maximum_length, column_default, is_nullable FROM information_schema.columns - where table_schema = '${schema}'`; + where table_schema = '${this.sql.unsafe(schema)}'`; const schemaDDL = this.schemaToDDL(res); @@ -73,12 +73,12 @@ export class PostgresClient { await this.initializeSchemas(); } - private async initializeSchemas(): Promise { + private async initializeSchemas() { const schemas = await this.sql` select schema_name from information_schema.schemata; `; console.log({ schemas }); - return schemas.map((schema) => schema.schema_name); + this.schemas = schemas.map((schema) => schema.schema_name); } }