Search relevance: learned ranking for Lunar search (spec 0084)

.github/workflows/build_panel_assets.yml

@@ -19,6 +19,10 @@ on:
       - "packages/panel/resources/**"
       - "packages/panel/vite.config.ts"
       - "packages/panel/package.json"
+      - "packages/search-relevance/resources/js/**"
+      - "packages/search-relevance/resources/client/src/**"
+      - "packages/search-relevance/vite.config.js"
+      - "packages/search-relevance/package.json"
       - ".github/workflows/build_panel_assets.yml"
 
 permissions:
@@ -58,8 +62,16 @@ jobs:
         run: npx vite build
         working-directory: packages/panel
 
+      # The search relevance add-on ships its compiled bundle the same way;
+      # its build/ directory is tracked, so the split carries it too.
+      - name: Build search relevance add-on bundle
+        run: npm run build --workspace @lunarphp/search-relevance-addon
+
+      - name: Build search relevance storefront client
+        run: npm run build --workspace @lunarphp/search-relevance
+
       - name: Commit built assets
         uses: stefanzweifel/git-auto-commit-action@v5
         with:
           commit_message: "chore(panel): rebuild compiled assets"
-          file_pattern: packages/panel/public/build
+          file_pattern: packages/panel/public/build packages/search-relevance/build packages/search-relevance/resources/client/dist

.github/workflows/publish_npm.yml

@@ -1,7 +1,7 @@
 name: publish-npm
 
-# Publishes the add-on-facing npm packages (@lunarphp/panel and
-# @lunarphp/panel-vite-plugin) alongside the Composer monorepo split.
+# Publishes the add-on-facing npm packages (@lunarphp/panel,
+# @lunarphp/panel-vite-plugin and @lunarphp/search-relevance) alongside the Composer monorepo split.
 # Skips any version already on the registry, so tags that don't bump the
 # npm package versions pass without churn.
 #
@@ -33,6 +33,8 @@ jobs:
             directory: packages/panel/resources/panel-package
           - package: "@lunarphp/panel-vite-plugin"
             directory: packages/panel/resources/package
+          - package: "@lunarphp/search-relevance"
+            directory: packages/search-relevance/resources/client
     steps:
       - name: Checkout code
         uses: actions/checkout@v6

.github/workflows/split_packages.yml

@@ -20,6 +20,7 @@ jobs:
           - "panel-addon-example"
           - "paypal"
           - "search"
+          - "search-relevance"
           - "stripe"
           - "table-rate-shipping"
           - "upgrade"

.github/workflows/tests.yml

@@ -24,7 +24,7 @@ jobs:
         php: ${{ fromJSON(github.event_name == 'pull_request' && '["8.4"]' || '["8.4","8.5"]') }}
         laravel: ["12.*", "13.*"]
         dependency-version: [prefer-stable]
-        testsuite: [core, admin, panel, filament, shipping, stripe, paypal, search, upgrade]
+        testsuite: [core, admin, panel, filament, shipping, stripe, paypal, search, search-relevance, upgrade]
         include:
           - laravel: 12.*
             testbench: 10.*
@@ -196,6 +196,15 @@ jobs:
       - name: Build example add-on bundle
         run: npm run build --workspace @lunarphp/panel-addon-example
 
+      - name: Build search relevance panel add-on bundle
+        run: npm run build --workspace @lunarphp/search-relevance-addon
+
+      - name: Test and build the search relevance storefront client
+        run: |
+          npm test --workspace @lunarphp/search-relevance
+          npm run type-check --workspace @lunarphp/search-relevance
+          npm run build --workspace @lunarphp/search-relevance
+
       # Fails when a publishable npm package's content no longer matches the
       # tarball published under its current version — the reminder to bump.
       # Relies on the panel build above for fresh @lunarphp/panel types.

TODO.md

@@ -34,6 +34,7 @@ Items tagged _(judgement)_ are genuine line-calls worth revisiting.
 
 ## Done
 
+- Search relevance — request/results pipelines in `search`, and the `search-relevance` add-on: logging, scoring, learned reranking, panel section (spec 0084)
 - Attribute field-type configuration schema — field types declare their settings once in core; the panel and Filament bridge both render from the descriptors (spec 0054)
 - Filament admin & bridge hardening — standalone bridge, dead hooks and config, locale nav groups, guard and asset id (spec 0076)
 - First staff account creation in core — `lunar:create-admin` moves out of the Filament admin; panel install offers it (spec 0075)

composer.json

@@ -57,7 +57,8 @@
         "files": [
             "packages/admin/src/helpers.php",
             "packages/core/src/helpers.php",
-            "packages/search/src/helpers.php"
+            "packages/search/src/helpers.php",
+            "packages/search-relevance/src/helpers.php"
         ],
         "psr-4": {
             "Lunar\\Core\\": "packages/core/src",
@@ -74,6 +75,8 @@
             "Lunar\\Panel\\Database\\Factories\\": "packages/panel/database/factories",
             "Lunar\\Paypal\\": "packages/paypal/src/",
             "Lunar\\Search\\": "packages/search/src/",
+            "Lunar\\SearchRelevance\\": "packages/search-relevance/src/",
+            "Lunar\\SearchRelevance\\Database\\Factories\\": "packages/search-relevance/database/factories",
             "Lunar\\Shipping\\": "packages/table-rate-shipping/src",
             "Lunar\\Shipping\\Database\\Factories\\": "packages/table-rate-shipping/database/factories",
             "Lunar\\Stripe\\": "packages/stripe/src/",
@@ -94,6 +97,7 @@
             "Lunar\\Shipping\\Tests\\": "packages/table-rate-shipping/tests",
             "Lunar\\Tests\\Stripe\\": "tests/stripe",
             "Lunar\\Tests\\Search\\": "tests/search",
+            "Lunar\\Tests\\SearchRelevance\\": "tests/search-relevance",
             "Lunar\\Tests\\Upgrade\\": "tests/upgrade",
             "LunarPanelExample\\": "packages/panel-addon-example/src/"
         }
@@ -103,6 +107,7 @@
             "name": [
                 "Table Rate Shipping",
                 "Search",
+                "Search Relevance",
                 "Meilisearch",
                 "Paypal Payments",
                 "Stripe Payments",
@@ -116,6 +121,7 @@
                 "Lunar\\Paypal\\PaypalServiceProvider",
                 "Lunar\\Meilisearch\\MeilisearchServiceProvider",
                 "Lunar\\Search\\SearchServiceProvider",
+                "Lunar\\SearchRelevance\\SearchRelevanceServiceProvider",
                 "Lunar\\Filament\\LunarFilamentServiceProvider",
                 "Lunar\\Admin\\LunarPanelProvider",
                 "Lunar\\Shipping\\ShippingServiceProvider",
@@ -136,6 +142,7 @@
         "lunarphp/panel-addon-example": "self.version",
         "lunarphp/paypal": "self.version",
         "lunarphp/search": "self.version",
+        "lunarphp/search-relevance": "self.version",
         "lunarphp/stripe": "self.version",
         "lunarphp/table-rate-shipping": "self.version",
         "lunarphp/upgrade": "self.version"

package.json

@@ -5,7 +5,9 @@
     "workspaces": [
         "packages/panel/resources/package",
         "packages/panel/resources/panel-package",
-        "packages/panel-addon-example"
+        "packages/panel-addon-example",
+        "packages/search-relevance",
+        "packages/search-relevance/resources/client"
     ],
     "scripts": {
         "check-npm-drift": "node scripts/check-npm-drift.mjs"

packages/core/src/Search/ProductIndexer.php

@@ -61,6 +61,20 @@ public function toSearchableArray(Model $model): array
 
         $data['skus'] = $model->variants->pluck('sku')->toArray();
 
+        // Uppercased with separators stripped, so a partial code typed without
+        // the printed hyphens (HAGMB) still prefix-matches HAG-MB-32A.
+        $data['skus_normalised'] = $model->variants->pluck('sku')
+            ->filter()
+            ->map(fn (string $sku) => strtoupper(preg_replace('/[^A-Za-z0-9]+/', '', $sku)))
+            ->filter()
+            ->values()
+            ->toArray();
+
         return $data;
     }
+
+    public function getExactMatchFields(): array
+    {
+        return ['skus', 'skus_normalised'];
+    }
 }

packages/core/src/Search/ScoutIndexer.php

@@ -54,6 +54,19 @@ public function getFilterableFields(): array
         ];
     }
 
+    /**
+     * Fields that must match exactly, with typo tolerance disabled by the
+     * engine setup (`lunar:meilisearch:setup` applies these). Part-number
+     * style codes match random tokens under default typo tolerance.
+     * search-relevance's part-number retrieval searches these fields alone,
+     * so list every field a shopper types a code into (SKU, supplier part
+     * number, barcode).
+     */
+    public function getExactMatchFields(): array
+    {
+        return [];
+    }
+
     public function toSearchableArray(Model $model): array
     {
         if (! $model->attribute_data) {

packages/meilisearch/src/Console/MeilisearchSetup.php

@@ -69,6 +69,17 @@ public function handle(EngineManager $engine): void
             );
             $this->engine->waitForTask($task['taskUid']);
 
+            $indexer = $model->indexer();
+            $exactFields = method_exists($indexer, 'getExactMatchFields') ? $indexer->getExactMatchFields() : [];
+
+            if ($exactFields) {
+                $this->info("Disable typo tolerance on exact-match fields for {$searchable}");
+                $task = $index->updateTypoTolerance([
+                    'disableOnAttributes' => $exactFields,
+                ]);
+                $this->engine->waitForTask($task['taskUid']);
+            }
+
             $this->newLine();
         }
     }

packages/search-relevance/.github/workflows/close-pull-request.yml

@@ -0,0 +1,13 @@
+name: Close Pull Request
+
+on:
+  pull_request_target:
+    types: [opened]
+
+jobs:
+  run:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: superbrothers/close-pull-request@v3
+      with:
+        comment: "Thank you for your pull request. However, you have submitted this PR on the Lunar Search Relevance Repo which is a read-only sub split of `lunarphp/lunar`. Please submit your PR on the https://github.com/lunarphp/lunar repository.<br><br>Thanks!"

packages/search-relevance/.gitignore

@@ -0,0 +1 @@
+node_modules/

packages/search-relevance/build/manifest.json

@@ -0,0 +1,8 @@
+{
+  "resources/js/addon.ts": {
+    "file": "assets/addon-_4607Chj.js",
+    "name": "addon",
+    "src": "resources/js/addon.ts",
+    "isEntry": true
+  }
+}

packages/search-relevance/composer.json

@@ -0,0 +1,53 @@
+{
+  "name": "lunarphp/search-relevance",
+  "type": "library",
+  "description": "Learned search ranking for LunarPHP: logs searches and shopper events, scores them nightly and reranks results, engine-agnostic.",
+  "keywords": [
+    "lunarphp",
+    "laravel",
+    "ecommerce",
+    "e-commerce",
+    "headless",
+    "store",
+    "shop",
+    "search",
+    "relevance",
+    "ranking"
+  ],
+  "license": "MIT",
+  "authors": [
+    {
+      "name": "Lunar",
+      "homepage": "https://lunarphp.io/"
+    }
+  ],
+  "require": {
+    "php": "^8.4",
+    "lunarphp/core": "self.version",
+    "lunarphp/search": "self.version"
+  },
+  "suggest": {
+    "lunarphp/panel": "Adds the Search relevance section, dashboard widget and settings screen to the admin panel."
+  },
+  "autoload": {
+    "files": [
+      "src/helpers.php"
+    ],
+    "psr-4": {
+      "Lunar\\SearchRelevance\\": "src/",
+      "Lunar\\SearchRelevance\\Database\\Factories\\": "database/factories"
+    }
+  },
+  "extra": {
+    "lunar": {
+      "name": "Search Relevance"
+    },
+    "laravel": {
+      "providers": [
+        "Lunar\\SearchRelevance\\SearchRelevanceServiceProvider"
+      ]
+    }
+  },
+  "minimum-stability": "dev",
+  "prefer-stable": true
+}

packages/search-relevance/config/search-relevance.php

@@ -0,0 +1,107 @@
+<?php
+
+use Lunar\Core\Models\Product;
+use Lunar\SearchRelevance\Normalisers\DefaultQueryNormaliser;
+use Lunar\SearchRelevance\Rankers\BucketedRanker;
+use Lunar\SearchRelevance\Signals\QueryAffinitySignal;
+
+return [
+    /*
+    |--------------------------------------------------------------------------
+    | Mode
+    |--------------------------------------------------------------------------
+    |
+    | off:    nothing is logged or ranked.
+    | shadow: everything is logged and the reranked order is stored alongside
+    |         the shown order, but the engine order is displayed. The default
+    |         after install, so training data accrues from day one and the
+    |         replay command can prove uplift before switching on.
+    | on:     the reranked order is displayed.
+    |
+    | Set in config or LUNAR_SEARCH_RELEVANCE_MODE, like every other store
+    | setting; the admin panel shows the current value but cannot change it.
+    |
+    */
+    'mode' => env('LUNAR_SEARCH_RELEVANCE_MODE', 'shadow'),
+
+    // Searchable models whose results are ranked and logged.
+    'models' => [Product::class],
+
+    // Sort fields that mean the engine's own relevance order. A search sorted
+    // by one of these is ranked like an unsorted one; any other sort is the
+    // shopper's choice and is left alone. Lunar's storefront sends
+    // `relevance:asc` by default.
+    'relevance_sorts' => ['relevance', '_text_match'],
+
+    // Candidate window fetched from the engine and reordered.
+    'window' => 250,
+
+    // Reorder only within buckets of this many hits, so a weak keyword match
+    // can never leap above a strong one.
+    'bucket_size' => 10,
+
+    // Seconds the ranked window is cached for.
+    'cache_ttl' => 300,
+
+    // Learned products the engine did not return are inserted at the head of
+    // the second bucket, at most `max` of them, each at least `min_relative`.
+    'learned_union' => ['max' => 5, 'min_relative' => 0.1],
+
+    // How many shown product ids a logged search records.
+    'impressions_logged' => 50,
+
+    // How long a click attributes a later basket or purchase.
+    'attribution_ttl_minutes' => 30,
+
+    // What identifies a shopper: the Lunar cart (survives login) or the
+    // Laravel session id.
+    'session_key' => 'cart',
+
+    'normaliser' => DefaultQueryNormaliser::class,
+
+    // Bump when normalisation rules change so learned scores relearn cleanly.
+    'normaliser_version' => 1,
+
+    'ranker' => BucketedRanker::class,
+
+    // Signal class => weight. Combined scores are clamped to 0..1.
+    'signals' => [
+        QueryAffinitySignal::class => 1.0,
+    ],
+
+    'scoring' => [
+        'weights' => ['click' => 1, 'basket' => 3, 'purchase' => 5],
+        // Position correction: an event at position p counts p^eta times,
+        // capped at max_position_weight, so the ranking learns preference
+        // rather than position bias. 0 disables it.
+        'position_eta' => 0.7,
+        'max_position_weight' => 5,
+        'half_life_days' => 30,
+        'window_days' => 180,
+        'min_sessions' => 3,
+        'max_products_per_query' => 50,
+        // Daily run time for lunar:search-relevance:score.
+        'schedule' => '02:00',
+    ],
+
+    // Raw queries and events older than this are pruned weekly.
+    'retention_days' => 400,
+
+    'guards' => [
+        // Events endpoint rate limit, "attempts,minutes", per shopper and per IP.
+        'events_rate_limit' => '60,1',
+        // Sessions searching faster than this are ignored by scoring.
+        'max_searches_per_minute' => 30,
+        // Events are only accepted this long after the search they belong to.
+        'event_window_minutes' => 120,
+        // Only sessions that hold a cart or belong to a known customer count
+        // towards learning, so a bot minting fresh sessions gains nothing.
+        'trusted_sessions_only' => true,
+        // Searches from these user agents (case-insensitive substrings) are
+        // neither logged nor ranked.
+        'ignored_user_agents' => [
+            'bot', 'crawl', 'spider', 'slurp', 'curl', 'wget', 'python-requests',
+            'headlesschrome', 'phantomjs', 'lighthouse', 'facebookexternalhit',
+        ],
+    ],
+];