From 1906566d7bacab4eee9d17af6b1893d3764e0123 Mon Sep 17 00:00:00 2001 From: Jake Wang Date: Sun, 15 Mar 2026 18:27:31 -0700 Subject: [PATCH 1/3] Additional KrogerPipeline Columns --- KrogerPipeline/kroger_catalogue.js | 200 ++++++++++++++++------------- schema.sql | 23 ++++ supabase_import.js | 44 ++++--- 3 files changed, 163 insertions(+), 104 deletions(-) diff --git a/KrogerPipeline/kroger_catalogue.js b/KrogerPipeline/kroger_catalogue.js index ccd28ac..b547346 100644 --- a/KrogerPipeline/kroger_catalogue.js +++ b/KrogerPipeline/kroger_catalogue.js @@ -57,6 +57,7 @@ const statusOnly = args.includes('--status'); const catalogueDir = path.join(outRoot, 'catalogue'); const catalogueFile = path.join(catalogueDir, 'food_catalogue.csv'); +const dryRunFile = path.join(catalogueDir, 'food_catalogue_dryrun.csv'); const maxStores = storesArg === 'all' ? Infinity : parseInt(storesArg || '10', 10); @@ -649,23 +650,38 @@ const FOOD_CATEGORIES = { // --- CSV Schema --- const CSV_HEADERS = [ + // ── Product-level fields ────────────────────────────────────────────────── 'productId', 'upc', 'brand', 'description', 'categories', - 'size', - 'soldBy', - 'temperature', - 'soldInStore', 'countryOrigin', 'aisleLocations', 'itemsFacets', 'image_url', - 'classifier', // ← Walmart classifier tag (PRODUCE, PROTEIN, DAIRY, etc.) - 'search_keyword', // ← the specific term that first found this product - 'store_ids', // ← semicolon-separated locationIds (in order found) - 'price', // ← semicolon-separated prices matching store_ids order + // ── Item-level fields (items[0]) ────────────────────────────────────────── + 'itemId', + 'size', + 'soldBy', + 'soldInStore', + 'favorite', + 'temperature', // indicator string (e.g. "Refrigerated") + 'temperature_heatSensitive', + 'price_regular', // base price from the API + 'price_promo', // promotional price if active + 'fulfillment_inStore', + 'fulfillment_curbside', + 'fulfillment_delivery', + 'fulfillment_shipGrocery', + 'discount_hasDiscount', + 'discount_digital', + 'discount_inStore', + // ── Pipeline metadata ───────────────────────────────────────────────────── + 'classifier', // ← Walmart classifier tag (PRODUCE, PROTEIN, DAIRY, etc.) + 'search_keyword', // ← the specific term that first found this product + 'store_ids', // ← semicolon-separated locationIds (in order found) + 'price', // ← semicolon-separated per-store prices matching store_ids order ]; // --- Token Manager --- @@ -769,26 +785,42 @@ function productToFields(p, classifier = '', searchKeyword = '') { '') : ''; + const fulfillment = items.fulfillment ?? {}; + const discount = items.discount ?? {}; + return { + // Product-level productId: p.productId ?? '', upc: p.upc ?? '', brand: p.brand ?? '', description: p.description ?? '', categories: (p.categories ?? []).join('; '), - size: items.size ?? '', - soldBy: items.soldBy ?? '', - temperature: items.temperature?.indicator ?? '', - soldInStore: items.soldInStore ?? '', countryOrigin: p.countryOrigin ?? '', - aisleLocations: (p.aisleLocations ?? []) - .map((a) => a.description) - .join('; '), + aisleLocations: (p.aisleLocations ?? []).map((a) => a.description).join('; '), itemsFacets: (p.itemsFacets ?? []).join('; '), image_url: imgUrl, - classifier: classifier, // e.g. PRODUCE, PROTEIN, DAIRY … - search_keyword: searchKeyword, // e.g. "chicken breast", "raw almonds" … - store_ids: '', // filled in per store during collection - price: '', // filled in per store during collection + // Item-level (items[0]) + itemId: items.itemId ?? '', + size: items.size ?? '', + soldBy: items.soldBy ?? '', + soldInStore: items.soldInStore ?? '', + favorite: items.favorite ?? '', + temperature: items.temperature?.indicator ?? '', + temperature_heatSensitive: items.temperature?.heatSensitive ?? '', + price_regular: items.price?.regular ?? '', + price_promo: items.price?.promo ?? '', + fulfillment_inStore: fulfillment.inStore ?? '', + fulfillment_curbside: fulfillment.curbside ?? '', + fulfillment_delivery: fulfillment.delivery ?? '', + fulfillment_shipGrocery: fulfillment.shipGrocery ?? '', + discount_hasDiscount: discount.hasDiscount ?? '', + discount_digital: discount.digital ?? '', + discount_inStore: discount.inStore ?? '', + // Pipeline metadata + classifier: classifier, + search_keyword: searchKeyword, + store_ids: '', // filled in during store enrichment (phase 2) + price: '', // per-store prices, semicolon-delimited, matches store_ids order }; } @@ -943,12 +975,6 @@ async function loadCatalogue() { return catalogue; } -/** Write the full catalogue Map to disk */ -function saveCatalogue(catalogue) { - const header = CSV_HEADERS.join(','); - const rows = Array.from(catalogue.values()).map(rowToCsv); - fs.writeFileSync(catalogueFile, [header, ...rows].join('\n'), 'utf8'); -} // --- Core: Run search terms and collect results --- async function runSearchTerms(locationId, tokenMgr) { @@ -1041,102 +1067,104 @@ async function printStatus() { ); } -// finds stores near a zip, queries each one, and compiles the results into food_catalogue.csv -// price[i] corresponds to store_ids[i] - they're kept in the same order +// Two-phase build: +// Phase 1 — search all terms at the FIRST (nearest) store to build a full +// product catalogue with real prices. The Kroger API returns far +// fewer results without a locationId, so anchoring to a real store +// is the only way to get the full ~19k product universe. +// Phase 2 — re-search the same terms at each remaining store to collect +// their prices and append them to store_ids / price columns. async function buildCatalogue(tokenMgr) { const zip = zipcodeArg; + const outFile = isDryRun ? dryRunFile : catalogueFile; + console.log(`\nBuilding food catalogue for stores near ${zip}`); - console.log(` Output: ${catalogueFile}`); - if (isDryRun) console.log(` Dry run: 3 terms per category only\n`); + console.log(` Output: ${outFile}`); + if (isDryRun) console.log(` Dry run: 3 terms per category, ${storesArg || maxStores} stores max\n`); - // 1. Find stores near zip + // Find all stores upfront — we need the first one for phase 1 console.log(`\n Finding stores within ${radiusArg} miles of ${zip}...`); const stores = await fetchStoresNearZip(zip, tokenMgr); - const storeLimit = - maxStores === Infinity ? stores.length : Math.min(maxStores, stores.length); - console.log( - `\n Found ${stores.length} store(s) - querying ${storeLimit}:\n`, - ); + const storeLimit = maxStores === Infinity + ? stores.length + : Math.min(maxStores, stores.length); + + console.log(`\n Found ${stores.length} store(s) - using ${storeLimit}:\n`); stores.slice(0, storeLimit).forEach((s, i) => { - console.log( - ` ${String(i + 1).padStart(2)}. [${s.locationId}] ${s.name} (${s.chain})`, - ); + console.log(` ${String(i + 1).padStart(2)}. [${s.locationId}] ${s.name} (${s.chain})`); console.log(` ${s.address}`); }); - // 2. Load any existing catalogue so a crashed run can be resumed - const catalogue = await loadCatalogue(); - if (catalogue.size > 0) { - console.log( - `\n Resuming from existing catalogue (${catalogue.size.toLocaleString()} products already saved).`, - ); + // ── Phase 1: Full catalogue read anchored to the nearest store ───────────── + // Searching with a real locationId is required to get the full product set. + // Without it the API only returns a fraction of available products (~2k vs ~19k). + const primaryStore = stores[0]; + console.log(`\n Phase 1: Building catalogue from primary store [${primaryStore.locationId}] ${primaryStore.name}...`); + const primaryResults = await runSearchTerms(primaryStore.locationId, tokenMgr); + + const catalogue = new Map(); + for (const [productId, { product: p, classifier, search_keyword }] of primaryResults) { + if (p.items?.[0]?.fulfillment?.inStore === false) continue; + const fields = productToFields(p, classifier, search_keyword); + fields.store_ids = primaryStore.locationId; + fields.price = String(p.items?.[0]?.price?.regular ?? ''); + catalogue.set(productId, fields); } + console.log(`\n Phase 1 complete: ${catalogue.size.toLocaleString()} products from primary store`); + + // Save after phase 1 so we have something if phase 2 crashes + fs.writeFileSync(outFile, [CSV_HEADERS.join(','), ...Array.from(catalogue.values()).map(rowToCsv)].join('\n'), 'utf8'); - let totalNew = 0, - totalUpdated = 0; + // ── Phase 2: Enrich remaining stores ────────────────────────────────────── + // Re-search each additional store to add its prices and store_id. + // Starts at index 1 since the primary store is already done in phase 1. + let totalUpdated = 0; - // 3. Query each store and record price + store_id together - for (let i = 0; i < storeLimit; i++) { + for (let i = 1; i < storeLimit; i++) { const store = stores[i]; - console.log( - `\n [${i + 1}/${storeLimit}] Searching store [${store.locationId}] ${store.name}...`, - ); + console.log(`\n [${i + 1}/${storeLimit}] Enriching from store [${store.locationId}] ${store.name}...`); const storeProducts = await runSearchTerms(store.locationId, tokenMgr); + let updatedThisStore = 0; - let newThisStore = 0, - updatedThisStore = 0; - - for (const [ - productId, - { product: p, classifier, search_keyword }, - ] of storeProducts) { + for (const [productId, { product: p, classifier, search_keyword }] of storeProducts) { const price = String(p.items?.[0]?.price?.regular ?? ''); + const row = catalogue.get(productId); - if (catalogue.has(productId)) { - // already seen from an earlier store - append this store's id and price - const row = catalogue.get(productId); - const existingStores = row.store_ids - ? row.store_ids.split(';').filter(Boolean) - : []; + if (row) { + // product was in global catalogue — append this store's id and price + const existingStores = row.store_ids ? row.store_ids.split(';').filter(Boolean) : []; if (!existingStores.includes(store.locationId)) { row.store_ids = [...existingStores, store.locationId].join(';'); row.price = row.price ? `${row.price};${price}` : price; + updatedThisStore++; } - updatedThisStore++; - } else { - // first time we've seen this product + } else if (p.items?.[0]?.fulfillment?.inStore !== false) { + // product wasn't in the global search — add it if it's sold in store const fields = productToFields(p, classifier, search_keyword); fields.store_ids = store.locationId; fields.price = price; catalogue.set(productId, fields); - newThisStore++; + updatedThisStore++; } } - totalNew += newThisStore; totalUpdated += updatedThisStore; + console.log(`\n Store [${store.locationId}] done: ${updatedThisStore.toLocaleString()} products enriched`); - console.log( - `\n Store [${store.locationId}] done: ${storeProducts.size.toLocaleString()} products, ${newThisStore.toLocaleString()} new, ${updatedThisStore.toLocaleString()} updated`, - ); - - // save after every store so we don't lose progress if something crashes - saveCatalogue(catalogue); - console.log( - ` Catalogue saved (${catalogue.size.toLocaleString()} total products)`, - ); + // write after every store so a crash doesn't lose progress + const header = CSV_HEADERS.join(','); + const rows = Array.from(catalogue.values()).map(rowToCsv); + fs.writeFileSync(outFile, [header, ...rows].join('\n'), 'utf8'); + console.log(` Catalogue saved (${catalogue.size.toLocaleString()} total products)`); } - console.log( - `\n-- Done -------------------------------------------------------------------`, - ); - console.log(` Stores searched : ${storeLimit}`); - console.log(` New products : ${totalNew.toLocaleString()}`); - console.log(` Updated products : ${totalUpdated.toLocaleString()}`); - console.log(` Total in catalogue: ${catalogue.size.toLocaleString()}`); - console.log(` Saved to: ${catalogueFile}\n`); + console.log(`\n-- Done -------------------------------------------------------------------`); + console.log(` Global products : ${catalogue.size.toLocaleString()}`); + console.log(` Stores enriched : ${storeLimit}`); + console.log(` Products with price: ${totalUpdated.toLocaleString()}`); + console.log(` Saved to: ${outFile}\n`); } // --- Main --- diff --git a/schema.sql b/schema.sql index 9ed51cb..650a0ca 100644 --- a/schema.sql +++ b/schema.sql @@ -38,6 +38,29 @@ CREATE TABLE IF NOT EXISTS kroger_ingredients ( search_keyword TEXT ); +-- ── Kroger ingredients v2 ───────────────────────────────────────────────────── +-- Expanded schema capturing all Kroger API fields. +-- price and store_ids are semicolon-delimited text (one entry per store). +-- Re-runs require TRUNCATE first. + +CREATE TABLE IF NOT EXISTS kroger_ingredients2 ( + "productId" BIGINT PRIMARY KEY, + upc BIGINT, + brand TEXT, + name TEXT, + categories TEXT, + "countryOrigin" TEXT, + "aisleLocations" TEXT, + image_url TEXT, + "itemId" BIGINT, + size TEXT, + "soldBy" TEXT, + classifier TEXT, + search_keyword TEXT, + store_ids TEXT, + price TEXT NOT NULL +); + CREATE INDEX IF NOT EXISTS kroger_ingredients_classifiers_idx ON kroger_ingredients USING GIN (classifiers); diff --git a/supabase_import.js b/supabase_import.js index 2e052bf..09172a7 100644 --- a/supabase_import.js +++ b/supabase_import.js @@ -34,7 +34,7 @@ const SUPABASE_KEY = process.env.SUPABASE_SERVICE_KEY; const BATCH_SIZE = 1000; const TABLE_WALMART = 'walmart_ingredients'; -const TABLE_KROGER = 'kroger_ingredients'; +const TABLE_KROGER = 'kroger_ingredients2'; const TABLE_STORES = 'kroger_locations'; const KROGER_TOKEN_URL = 'https://api.kroger.com/v1/connect/oauth2/token'; @@ -86,13 +86,17 @@ function csvParser(filePath) { // Sends one batch of rows to Supabase. In dry-run mode it just logs what // it would have done instead of actually writing anything. -async function insertBatch(table, rows, dryRun) { +// Pass onConflict to upsert instead of insert (e.g. 'productId'). +async function insertBatch(table, rows, dryRun, onConflict = null) { if (dryRun) { - console.log(` [dry-run] would insert ${rows.length} rows → ${table}`); + console.log(` [dry-run] would upsert ${rows.length} rows → ${table}`); return; } - const { error } = await supabase.from(table).insert(rows); - if (error) throw new Error(`${table} insert failed: ${error.message}`); + const query = onConflict + ? supabase.from(table).upsert(rows, { onConflict }) + : supabase.from(table).insert(rows); + const { error } = await query; + if (error) throw new Error(`${table} upsert failed: ${error.message}`); } // ── Walmart ─────────────────────────────────────────────────────────────────── @@ -183,7 +187,7 @@ export async function importKroger({ dryRun = false } = {}) { async function flush() { if (!batch.length) return; - await insertBatch(TABLE_KROGER, batch, dryRun); + await insertBatch(TABLE_KROGER, batch, dryRun, 'productId'); totalInserted += batch.length; batch = []; process.stdout.write( @@ -198,26 +202,30 @@ export async function importKroger({ dryRun = false } = {}) { continue; } - // The price column looks like "2.99;3.49;2.99" — one price per store. - // We just grab the first one as a representative price for the product. - const price = toFloat(split(row.price, ';')[0]); - if (price === null) { + // price is stored as-is (semicolon-delimited per-store prices). + // Skip if completely empty since the column is NOT NULL. + const price = row.price ? row.price.trim() : ''; + if (!price) { skippedMissingFields++; continue; } batch.push({ - name: row.description, + productId: row.productId ? parseInt(row.productId, 10) : null, + upc: row.upc ? parseInt(row.upc, 10) : null, brand: row.brand || null, - price, - // classifier is a single tag like "PRODUCE" — wrap it in an array - // to match the classifiers column type on the Walmart table - classifiers: row.classifier ? [row.classifier] : [], - image: row.image_url || null, + name: row.description, + categories: row.categories || null, + countryOrigin: row.countryOrigin || null, + aisleLocations: row.aisleLocations || null, + image_url: row.image_url || null, + itemId: row.itemId ? parseInt(row.itemId, 10) : null, size: row.size || '', - // store_ids looks like "70400786;70400343" — split into an array - store_id: split(row.store_ids, ';'), + soldBy: row.soldBy || null, + classifier: row.classifier || null, search_keyword: row.search_keyword || null, + store_ids: row.store_ids || null, + price: String(price), }); if (batch.length >= BATCH_SIZE) await flush(); From 5930683b966f73aa4bdf76b7b742cc17f7366463 Mon Sep 17 00:00:00 2001 From: Jake Wang Date: Mon, 16 Mar 2026 14:10:59 -0700 Subject: [PATCH 2/3] Kroger Pipeline Finalizations --- KrogerPipeline/kroger_catalogue.js | 87 +++++++++++++++++++++--------- 1 file changed, 61 insertions(+), 26 deletions(-) diff --git a/KrogerPipeline/kroger_catalogue.js b/KrogerPipeline/kroger_catalogue.js index b547346..a4d97af 100644 --- a/KrogerPipeline/kroger_catalogue.js +++ b/KrogerPipeline/kroger_catalogue.js @@ -666,10 +666,10 @@ const CSV_HEADERS = [ 'soldBy', 'soldInStore', 'favorite', - 'temperature', // indicator string (e.g. "Refrigerated") + 'temperature', // indicator string (e.g. "Refrigerated") 'temperature_heatSensitive', - 'price_regular', // base price from the API - 'price_promo', // promotional price if active + 'price_regular', // base price from the API + 'price_promo', // promotional price if active 'fulfillment_inStore', 'fulfillment_curbside', 'fulfillment_delivery', @@ -678,10 +678,10 @@ const CSV_HEADERS = [ 'discount_digital', 'discount_inStore', // ── Pipeline metadata ───────────────────────────────────────────────────── - 'classifier', // ← Walmart classifier tag (PRODUCE, PROTEIN, DAIRY, etc.) - 'search_keyword', // ← the specific term that first found this product - 'store_ids', // ← semicolon-separated locationIds (in order found) - 'price', // ← semicolon-separated per-store prices matching store_ids order + 'classifier', // ← Walmart classifier tag (PRODUCE, PROTEIN, DAIRY, etc.) + 'search_keyword', // ← the specific term that first found this product + 'store_ids', // ← semicolon-separated locationIds (in order found) + 'price', // ← semicolon-separated per-store prices matching store_ids order ]; // --- Token Manager --- @@ -796,7 +796,9 @@ function productToFields(p, classifier = '', searchKeyword = '') { description: p.description ?? '', categories: (p.categories ?? []).join('; '), countryOrigin: p.countryOrigin ?? '', - aisleLocations: (p.aisleLocations ?? []).map((a) => a.description).join('; '), + aisleLocations: (p.aisleLocations ?? []) + .map((a) => a.description) + .join('; '), itemsFacets: (p.itemsFacets ?? []).join('; '), image_url: imgUrl, // Item-level (items[0]) @@ -820,7 +822,7 @@ function productToFields(p, classifier = '', searchKeyword = '') { classifier: classifier, search_keyword: searchKeyword, store_ids: '', // filled in during store enrichment (phase 2) - price: '', // per-store prices, semicolon-delimited, matches store_ids order + price: '', // per-store prices, semicolon-delimited, matches store_ids order }; } @@ -975,7 +977,6 @@ async function loadCatalogue() { return catalogue; } - // --- Core: Run search terms and collect results --- async function runSearchTerms(locationId, tokenMgr) { const categoryKeys = categoryFilter @@ -1080,19 +1081,23 @@ async function buildCatalogue(tokenMgr) { console.log(`\nBuilding food catalogue for stores near ${zip}`); console.log(` Output: ${outFile}`); - if (isDryRun) console.log(` Dry run: 3 terms per category, ${storesArg || maxStores} stores max\n`); + if (isDryRun) + console.log( + ` Dry run: 3 terms per category, ${storesArg || maxStores} stores max\n`, + ); // Find all stores upfront — we need the first one for phase 1 console.log(`\n Finding stores within ${radiusArg} miles of ${zip}...`); const stores = await fetchStoresNearZip(zip, tokenMgr); - const storeLimit = maxStores === Infinity - ? stores.length - : Math.min(maxStores, stores.length); + const storeLimit = + maxStores === Infinity ? stores.length : Math.min(maxStores, stores.length); console.log(`\n Found ${stores.length} store(s) - using ${storeLimit}:\n`); stores.slice(0, storeLimit).forEach((s, i) => { - console.log(` ${String(i + 1).padStart(2)}. [${s.locationId}] ${s.name} (${s.chain})`); + console.log( + ` ${String(i + 1).padStart(2)}. [${s.locationId}] ${s.name} (${s.chain})`, + ); console.log(` ${s.address}`); }); @@ -1100,21 +1105,38 @@ async function buildCatalogue(tokenMgr) { // Searching with a real locationId is required to get the full product set. // Without it the API only returns a fraction of available products (~2k vs ~19k). const primaryStore = stores[0]; - console.log(`\n Phase 1: Building catalogue from primary store [${primaryStore.locationId}] ${primaryStore.name}...`); - const primaryResults = await runSearchTerms(primaryStore.locationId, tokenMgr); + console.log( + `\n Phase 1: Building catalogue from primary store [${primaryStore.locationId}] ${primaryStore.name}...`, + ); + const primaryResults = await runSearchTerms( + primaryStore.locationId, + tokenMgr, + ); const catalogue = new Map(); - for (const [productId, { product: p, classifier, search_keyword }] of primaryResults) { + for (const [ + productId, + { product: p, classifier, search_keyword }, + ] of primaryResults) { if (p.items?.[0]?.fulfillment?.inStore === false) continue; const fields = productToFields(p, classifier, search_keyword); fields.store_ids = primaryStore.locationId; fields.price = String(p.items?.[0]?.price?.regular ?? ''); catalogue.set(productId, fields); } - console.log(`\n Phase 1 complete: ${catalogue.size.toLocaleString()} products from primary store`); + console.log( + `\n Phase 1 complete: ${catalogue.size.toLocaleString()} products from primary store`, + ); // Save after phase 1 so we have something if phase 2 crashes - fs.writeFileSync(outFile, [CSV_HEADERS.join(','), ...Array.from(catalogue.values()).map(rowToCsv)].join('\n'), 'utf8'); + fs.writeFileSync( + outFile, + [ + CSV_HEADERS.join(','), + ...Array.from(catalogue.values()).map(rowToCsv), + ].join('\n'), + 'utf8', + ); // ── Phase 2: Enrich remaining stores ────────────────────────────────────── // Re-search each additional store to add its prices and store_id. @@ -1123,18 +1145,25 @@ async function buildCatalogue(tokenMgr) { for (let i = 1; i < storeLimit; i++) { const store = stores[i]; - console.log(`\n [${i + 1}/${storeLimit}] Enriching from store [${store.locationId}] ${store.name}...`); + console.log( + `\n [${i + 1}/${storeLimit}] Enriching from store [${store.locationId}] ${store.name}...`, + ); const storeProducts = await runSearchTerms(store.locationId, tokenMgr); let updatedThisStore = 0; - for (const [productId, { product: p, classifier, search_keyword }] of storeProducts) { + for (const [ + productId, + { product: p, classifier, search_keyword }, + ] of storeProducts) { const price = String(p.items?.[0]?.price?.regular ?? ''); const row = catalogue.get(productId); if (row) { // product was in global catalogue — append this store's id and price - const existingStores = row.store_ids ? row.store_ids.split(';').filter(Boolean) : []; + const existingStores = row.store_ids + ? row.store_ids.split(';').filter(Boolean) + : []; if (!existingStores.includes(store.locationId)) { row.store_ids = [...existingStores, store.locationId].join(';'); row.price = row.price ? `${row.price};${price}` : price; @@ -1151,16 +1180,22 @@ async function buildCatalogue(tokenMgr) { } totalUpdated += updatedThisStore; - console.log(`\n Store [${store.locationId}] done: ${updatedThisStore.toLocaleString()} products enriched`); + console.log( + `\n Store [${store.locationId}] done: ${updatedThisStore.toLocaleString()} products enriched`, + ); // write after every store so a crash doesn't lose progress const header = CSV_HEADERS.join(','); const rows = Array.from(catalogue.values()).map(rowToCsv); fs.writeFileSync(outFile, [header, ...rows].join('\n'), 'utf8'); - console.log(` Catalogue saved (${catalogue.size.toLocaleString()} total products)`); + console.log( + ` Catalogue saved (${catalogue.size.toLocaleString()} total products)`, + ); } - console.log(`\n-- Done -------------------------------------------------------------------`); + console.log( + `\n-- Done -------------------------------------------------------------------`, + ); console.log(` Global products : ${catalogue.size.toLocaleString()}`); console.log(` Stores enriched : ${storeLimit}`); console.log(` Products with price: ${totalUpdated.toLocaleString()}`); From 19e5a2900ea32a37e661b65b9c99edb109c8b8a3 Mon Sep 17 00:00:00 2001 From: Jake Wang Date: Mon, 16 Mar 2026 14:19:40 -0700 Subject: [PATCH 3/3] Changes to match kroger_ingredient2 table changes --- schema.sql | 26 ++++++++++++-------------- supabase_import.js | 2 -- 2 files changed, 12 insertions(+), 16 deletions(-) diff --git a/schema.sql b/schema.sql index 650a0ca..975e6e3 100644 --- a/schema.sql +++ b/schema.sql @@ -44,21 +44,19 @@ CREATE TABLE IF NOT EXISTS kroger_ingredients ( -- Re-runs require TRUNCATE first. CREATE TABLE IF NOT EXISTS kroger_ingredients2 ( - "productId" BIGINT PRIMARY KEY, - upc BIGINT, - brand TEXT, - name TEXT, - categories TEXT, - "countryOrigin" TEXT, + "productId" BIGINT PRIMARY KEY, + brand TEXT, + name TEXT, + categories TEXT, + "countryOrigin" TEXT, "aisleLocations" TEXT, - image_url TEXT, - "itemId" BIGINT, - size TEXT, - "soldBy" TEXT, - classifier TEXT, - search_keyword TEXT, - store_ids TEXT, - price TEXT NOT NULL + image_url TEXT, + size TEXT, + "soldBy" TEXT, + classifier TEXT, + search_keyword TEXT, + store_ids TEXT, + price TEXT NOT NULL ); CREATE INDEX IF NOT EXISTS kroger_ingredients_classifiers_idx diff --git a/supabase_import.js b/supabase_import.js index 09172a7..466393b 100644 --- a/supabase_import.js +++ b/supabase_import.js @@ -212,14 +212,12 @@ export async function importKroger({ dryRun = false } = {}) { batch.push({ productId: row.productId ? parseInt(row.productId, 10) : null, - upc: row.upc ? parseInt(row.upc, 10) : null, brand: row.brand || null, name: row.description, categories: row.categories || null, countryOrigin: row.countryOrigin || null, aisleLocations: row.aisleLocations || null, image_url: row.image_url || null, - itemId: row.itemId ? parseInt(row.itemId, 10) : null, size: row.size || '', soldBy: row.soldBy || null, classifier: row.classifier || null,