Dev hyper aos migration

jest.config.js

@@ -1,6 +1,10 @@
 module.exports = {
   transform: {'^.+\\.ts?$': 'ts-jest'},
   testEnvironment: 'node',
-  testRegex: '/tests/.*\\.(test|spec)?\\.(ts|tsx)$',
-  moduleFileExtensions: ['ts', 'tsx', 'js', 'jsx', 'json', 'node']
-}
+  // NB: the `(test|spec)` group used to be optional, which made EVERY .ts file under
+  // tests/ a test file — including helpers like setup-env.ts, which then fail as suites
+  // containing no tests.
+  testRegex: '/tests/.*\\.(test|spec)\\.(ts|tsx)$',
+  moduleFileExtensions: ['ts', 'tsx', 'js', 'jsx', 'json', 'node'],
+  setupFiles: ['<rootDir>/tests/setup-env.ts']
+}

operations/deploy-live.hcl

@@ -90,7 +90,7 @@ job "api-service-live" {
         HEXAGON_RESOLUTION="4"
         GEODATADIR="/api-service-live/geo-ip-db/data"
         GEOTMPDIR="/api-service-live/tmp"
-        CU_URL="https://cu.anyone.tech"
+        HB_URL="https://hb.anyone.tech"
         DB_NAME="uns_indexer"
       }
 

operations/deploy-stage.hcl

@@ -90,7 +90,7 @@ job "api-service-stage" {
         HEXAGON_RESOLUTION="4"
         GEODATADIR="/api-service-stage/geo-ip-db/data"
         GEOTMPDIR="/api-service-stage/tmp"
-        CU_URL="https://cu-stage.anyone.tech"
+        HB_URL="https://hb-stage.anyone.tech"
         DB_NAME="uns_indexer"
       }
 

package.json

@@ -15,7 +15,6 @@
   "devDependencies": {
     "@types/cors": "^2.8.17",
     "@types/jest": "^29.5.14",
-    "@types/lodash": "^4.17.20",
     "@types/node": "^20.11.30",
     "@types/pg": "^8.11.10",
     "@types/supertest": "^6.0.2",
@@ -28,7 +27,6 @@
   "dependencies": {
     "@ethers-ext/provider-multicall": "^6.0.0-beta.3",
     "@maxmind/geoip2-node": "^6.1.0",
-    "@permaweb/aoconnect": "^0.0.85",
     "@types/express": "^4.17.21",
     "axios": "^1.6.8",
     "cors": "^2.8.5",
@@ -37,13 +35,10 @@
     "express": "^4.19.1",
     "express-validator": "^7.1.0",
     "h3-js": "^4.1.0",
-    "lodash": "^4.17.21",
     "mongoose": "^8.16.5",
     "pg": "^8.13.1",
-    "reflect-metadata": "^0.2.2",
     "run-script": "^0.1.1",
     "tsx": "^4.20.3",
-    "typeorm": "^0.3.20",
     "winston": "^3.17.0"
   }
 }

src/app.ts

@@ -1,4 +1,3 @@
-import 'reflect-metadata';
 import express from 'express';
 import bodyParser from 'body-parser';
 import cors from 'cors';
@@ -11,7 +10,7 @@ import { H3Service } from './H3Service';
 import { GeoLiteService } from './GeoLiteService';
 import { OperatorRegistryService } from './operator-registry.service';
 import { UnsTokenQueryService } from './uns-token.repository';
-import { initUnsIndexerDataSource } from './data-source';
+import { initUnsIndexerDb } from './data-source';
 import mongoose from 'mongoose';
 dotenv.config();
 
@@ -348,7 +347,7 @@ const bootstrap = async () => {
         console.log(`Connecting to MongoDB at [${mongodbUri}]`);
         await mongoose.connect(mongodbUri);
         try {
-            await initUnsIndexerDataSource();
+            await initUnsIndexerDb();
         } catch (error) {
             console.error(
                 'Failed to initialize UNS indexer data source:',

src/data-source.ts

@@ -1,34 +1,37 @@
-import 'reflect-metadata'
-import { DataSource } from 'typeorm'
+import { Pool } from 'pg'
 
-import { UnsTokenEntity } from './schema/uns-token.entity'
 import { logger } from './util/logger'
 
+/**
+ * Read-only connection to the Postgres database owned by the `uns-record-indexer`
+ * microservice.
+ *
+ * This used to be a TypeORM DataSource with a mirrored UnsTokenEntity. That mirror
+ * declared eleven columns to read three, and had to be kept in step with a schema owned
+ * by another repository — while every feature that justifies an ORM was switched off:
+ * `synchronize: false`, `migrationsRun: false`, migrations authoritative in the indexer,
+ * and a read-only role here. The one query is a full-table select of three columns, so
+ * it is plain `pg` now. That also drops the typeorm -> glob -> minimatch ->
+ * brace-expansion advisory chain, which had no non-breaking fix.
+ */
 const DB_HOST = process.env.DB_HOST || ''
 const DB_PORT = parseInt(process.env.DB_PORT || '5432')
 const DB_USER = process.env.DB_USER || ''
 const DB_PASS = process.env.DB_PASS || ''
 const DB_NAME = process.env.DB_NAME || ''
 
-export const unsIndexerDataSource = new DataSource({
-  type: 'postgres',
-  host: DB_HOST,
-  port: DB_PORT,
-  username: DB_USER,
-  password: DB_PASS,
-  database: DB_NAME,
-  entities: [UnsTokenEntity],
-  synchronize: false,
-  migrationsRun: false,
-  logging: false
-})
+let pool: Pool | null = null
+let initializePromise: Promise<Pool> | null = null
 
-let initializePromise: Promise<DataSource> | null = null
-
-export async function initUnsIndexerDataSource(): Promise<DataSource> {
-  if (unsIndexerDataSource.isInitialized) {
-    return unsIndexerDataSource
+export function unsIndexerPool(): Pool {
+  if (!pool) {
+    throw new Error('UNS indexer pool used before initUnsIndexerDb()')
   }
+  return pool
+}
+
+export async function initUnsIndexerDb(): Promise<Pool> {
+  if (pool) { return pool }
 
   const missing = [
     ['DB_HOST', DB_HOST],
@@ -48,21 +51,30 @@ export async function initUnsIndexerDataSource(): Promise<DataSource> {
 
   if (!initializePromise) {
     logger.info(
-      `Initializing UNS indexer Postgres data source at ` +
+      `Initializing UNS indexer Postgres pool at ` +
         `[${DB_HOST}:${DB_PORT}/${DB_NAME}]...`
     )
-    initializePromise = unsIndexerDataSource
-      .initialize()
-      .then(ds => {
-        logger.info('UNS indexer Postgres data source initialized.')
-        return ds
+    const candidate = new Pool({
+      host: DB_HOST,
+      port: DB_PORT,
+      user: DB_USER,
+      password: DB_PASS,
+      database: DB_NAME
+    })
+    // TypeORM's initialize() opened a connection, so failures surfaced at boot rather
+    // than on the first request. Keep that: a pg Pool is lazy on its own.
+    initializePromise = candidate
+      .query('SELECT 1')
+      .then(() => {
+        pool = candidate
+        logger.info('UNS indexer Postgres pool initialized.')
+        return candidate
       })
       .catch(error => {
         initializePromise = null
-        throw error
+        return candidate.end().catch(() => {}).then(() => { throw error })
       })
   }
 
   return initializePromise
 }
-

src/operator-registry.service.ts

@@ -1,22 +1,13 @@
-import _ from 'lodash'
+import { nodeUrlFromEnv, readView } from './util/ao-read'
 import { logger } from './util/logger'
-import { sendAosDryRun } from './util/send-aos-message'
-
-export interface OperatorRegistryState {
-  ClaimableFingerprintsToOperatorAddresses: { [fingerprint: string]: string }
-  VerifiedFingerprintsToOperatorAddresses: { [fingerprint: string]: string }
-  BlockedOperatorAddresses: { [fingerprint: string]: boolean }
-  RegistrationCreditsFingerprintsToOperatorAddresses: {
-    [fingerprint: string]: string
-  }
-  VerifiedHardwareFingerprints: { [fingerprint: string]: boolean }
-}
 
 export class OperatorRegistryService {
   private readonly operatorRegistryProcessId: string
+  private readonly hbUrl: string
+
   private operatorRegistryCacheTtlSeconds: number = 0
   private operatorRegistryCacheTimestamp: number = 0
-  private operatorRegistryCachedState: OperatorRegistryState | null = null
+  private operatorRegistryCachedOperators: string[] | null = null
 
   constructor() {
     logger.info('Initializing OperatorRegistryService...')
@@ -29,6 +20,12 @@ export class OperatorRegistryService {
       `Using operator registry process ID [${this.operatorRegistryProcessId}]`
     )
 
+    // Fail closed, no default. Replaces CU_URL/GATEWAY_URL/GRAPHQL_URL, two of which
+    // pointed at third-party infrastructure. The outage that forced this migration was
+    // caused by endpoints nobody had set explicitly.
+    this.hbUrl = nodeUrlFromEnv()
+    logger.info(`Reading operator registry from node [${this.hbUrl}]`)
+
     this.operatorRegistryCacheTtlSeconds =
       parseInt(process.env.OPERATOR_REGISTRY_CACHE_TTL_SECONDS || '0')
     if (
@@ -37,7 +34,7 @@ export class OperatorRegistryService {
     ) {
       this.operatorRegistryCacheTtlSeconds = 0
       logger.warn(
-        `Invalid OPERATOR_REGISTRY_CACHE_TTL_SECONDS ` + 
+        `Invalid OPERATOR_REGISTRY_CACHE_TTL_SECONDS ` +
           `[${process.env.OPERATOR_REGISTRY_CACHE_TTL_SECONDS}]. ` +
           `Using default value of 0.`
       )
@@ -49,42 +46,47 @@ export class OperatorRegistryService {
     logger.info('OperatorRegistryService initialized.')
   }
 
-  static async getOperatorRegistryState(
-    operatorRegistryProcessId: string
-  ): Promise<OperatorRegistryState> {
-    const { result } = await sendAosDryRun({
-      processId: operatorRegistryProcessId,
-      tags: [{ name: 'Action', value: 'View-State' }],
-    })
-    const state = JSON.parse(result.Messages[0].Data)
-
-    for (const prop in state) {
-      // NB: Lua returns empty tables as JSON arrays, so we normalize them to
-      //     empty objects as when they are populated they will also be objects
-      if (Array.isArray(state[prop]) && state[prop].length < 1) {
-        state[prop] = {}
-      }
-    }
+  /**
+   * Active operator addresses: unique verified, minus blocked.
+   *
+   * This was a `View-State` dryrun that downloaded all five registry maps so the
+   * service could uniq the values of one and difference them against the keys of
+   * another. The native contract added the `operators` view for exactly this consumer
+   * and does that work on-device, returning a SET (`{[address]: true}`) — already
+   * deduped, already filtered. So the whole lodash pipeline collapses into reading the
+   * keys, and we stop shipping the claimable/credit/hardware maps over the wire to
+   * throw them away.
+   */
+  private async fetchOperators(): Promise<string[]> {
+    const operators = await readView<Record<string, boolean>>(
+      this.hbUrl,
+      this.operatorRegistryProcessId,
+      'operators'
+    )
 
-    return state
+    // NB: Lua serializes an EMPTY table as a JSON array, so an empty registry arrives as
+    //     `[]` rather than `{}`. Object.keys handles both, but be explicit about why the
+    //     shape can vary.
+    return Object.keys(operators)
   }
 
   async getOperators() {
     const now = Date.now()
     const cacheAge = (now - this.operatorRegistryCacheTimestamp) / 1000
     if (
-      !this.operatorRegistryCachedState ||
+      !this.operatorRegistryCachedOperators ||
       cacheAge >= this.operatorRegistryCacheTtlSeconds
     ) {
       logger.info(
         'Fetching operator registry state because the cache is empty or expired'
       )
       try {
-        this.operatorRegistryCachedState = await OperatorRegistryService
-          .getOperatorRegistryState(this.operatorRegistryProcessId)
+        this.operatorRegistryCachedOperators = await this.fetchOperators()
         this.operatorRegistryCacheTimestamp = now
         logger.info(`Operator registry state fetched successfully!`)
       } catch (error: Error | any) {
+        // Deliberately keep serving the stale list: this backs a public endpoint, and a
+        // node blip should degrade to slightly-old data rather than an empty response.
         logger.error(
           `Failed to get Operator Registry State: ${error.message}`,
           error
@@ -96,36 +98,15 @@ export class OperatorRegistryService {
       )
     }
 
-    if (!this.operatorRegistryCachedState) {
+    if (!this.operatorRegistryCachedOperators) {
       logger.error('Operator registry state is not available!')
       return []
     }
 
-    const verifiedOperatorAddresses = _.uniq(
-      Object.values(
-        this.operatorRegistryCachedState.VerifiedFingerprintsToOperatorAddresses
-      )
-    )
-    logger.info(
-      `Found [${verifiedOperatorAddresses.length}] verified operator addresses.`
-    )
-
-    const blockedOperatorAddresses = Object.keys(
-      this.operatorRegistryCachedState.BlockedOperatorAddresses
-    )
-    logger.info(
-      `Found [${blockedOperatorAddresses.length}] blocked operator addresses.`
-    )
-
-    const operatorAddresses = _.difference(
-      verifiedOperatorAddresses,
-      blockedOperatorAddresses
-    )
     logger.info(
-      `Found [${operatorAddresses.length}] operator addresses after ` +
-        `filtering out blocked operator addresses`
+      `Found [${this.operatorRegistryCachedOperators.length}] operator addresses`
     )
 
-    return operatorAddresses
+    return this.operatorRegistryCachedOperators
   }
 }