IBX-12271: Implemented Solr 10 support

.github/init_solr.sh

@@ -2,9 +2,9 @@
 
 SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
 
-SOLR_VERSION=${SOLR_VERSION:-'9.8.1'}
+SOLR_VERSION=${SOLR_VERSION:-'9.10.1'}
 
-if [[ "${SOLR_VERSION}" =~ ^9\. ]]; then
+if [[ "${SOLR_VERSION}" =~ ^(9|10)\. ]]; then
     default_config_files[1]="${SCRIPT_DIR}/../src/lib/Resources/config/solr/managed-schema.xml"
     default_config_files[2]="${SCRIPT_DIR}/../src/lib/Resources/config/solr/custom-fields-types-solr9.xml"
 else
@@ -42,7 +42,7 @@ SOLR_CLOUD=${SOLR_CLOUD:-'no'}
 INSTALL_DIR="${SOLR_DIR}/${SOLR_VERSION}"
 HOME_DIR="${INSTALL_DIR}/server/${SOLR_HOME}"
 
-if [[ "${SOLR_VERSION}" =~ ^9\. ]]; then
+if [[ "${SOLR_VERSION}" =~ ^(9|10)\. ]]; then
     TEMPLATE_DIR="${HOME_DIR}/template/conf"
 else
     TEMPLATE_DIR="${HOME_DIR}/template"
@@ -60,7 +60,7 @@ fi
 download() {
     case ${SOLR_VERSION} in
         # PS!!: Append versions and don't remove old ones (except in major versions), used in integration tests from other packages!
-        9.*)
+        9.* | 10.*)
             url="https://archive.apache.org/dist/solr/solr/${SOLR_VERSION}/solr-${SOLR_VERSION}.tgz"
             ;;
         7.7.* | 8.* )
@@ -155,7 +155,10 @@ solr_run() {
     echo "Running with version ${SOLR_VERSION} in standalone mode"
     echo "Starting solr on port ${SOLR_PORT}..."
 
-    if [[ "${SOLR_VERSION}" =~ ^9\. ]]; then
+    if [[ "${SOLR_VERSION}" =~ ^10\. ]]; then
+        # Solr 10: SolrCloud is the default mode, standalone requires --user-managed (SOLR-17467); '-s' was repurposed for --solr-url
+        ./${SOLR_INSTALL_DIR}/bin/solr start --user-managed -p "${SOLR_PORT}" --solr-home "${SOLR_HOME}" || exit_on_error "Can't start Solr"
+    elif [[ "${SOLR_VERSION}" =~ ^9\. ]]; then
         ./${SOLR_INSTALL_DIR}/bin/solr start -p ${SOLR_PORT} -s ${SOLR_HOME} || exit_on_error "Can't start Solr"
     else
         ./${SOLR_INSTALL_DIR}/bin/solr -p ${SOLR_PORT} -s ${SOLR_HOME} -Dsolr.disable.shardsWhitelist=true || exit_on_error "Can't start Solr"
@@ -188,7 +191,12 @@ solr_create_core() {
 
     abs_conf_dir="$(pwd)/${config_dir}"
 
-    ./${SOLR_INSTALL_DIR}/bin/solr create_core ${solr_port_flag} -c ${core_name} -d "${abs_conf_dir}" || exit_on_error "Can't create core"
+    if [[ "${SOLR_VERSION}" =~ ^10\. ]]; then
+        # Solr 10: 'create_core' was removed in favour of 'create' addressed via --solr-url (SOLR-16893)
+        ./${SOLR_INSTALL_DIR}/bin/solr create -c "${core_name}" -d "${abs_conf_dir}" --solr-url "http://localhost:${SOLR_PORT}" || exit_on_error "Can't create core"
+    else
+        ./${SOLR_INSTALL_DIR}/bin/solr create_core ${solr_port_flag} -c ${core_name} -d "${abs_conf_dir}" || exit_on_error "Can't create core"
+    fi
 }
 
 solr_cloud_configure_nodes() {
@@ -225,12 +233,19 @@ solr_cloud_start_nodes() {
         local IFS=':'; read node_name node_port <<< "${node}"
         local node_dir="${HOME_DIR}/${node_name}"
 
+        if [[ "${SOLR_VERSION}" =~ ^10\. ]]; then
+            # Solr 10: SolrCloud is the default mode ('-cloud' removed, SOLR-17467), '-s' was repurposed for --solr-url, '-V' for --verbose
+            local start_args=(start --solr-home "${node_dir}" -p "${node_port}" --verbose)
+        else
+            local start_args=(start -cloud -s "${node_dir}" -p "${node_port}" -V)
+        fi
+
         if [[ ! ${ZOOKEEPER_HOST} ]] ; then
-            ${START_SCRIPT} start -cloud -s ${node_dir} -p ${node_port} -V || exit_on_error "Can't start node '${node_name}'"
+            ${START_SCRIPT} "${start_args[@]}" || exit_on_error "Can't start node '${node_name}'"
             # start script default
             ZOOKEEPER_HOST="localhost:$((node_port+1000))"
         else
-            ${START_SCRIPT} start -cloud -s ${node_dir} -p ${node_port} -z "${ZOOKEEPER_HOST}" -V || exit_on_error "Can't start node '${node_name}'"
+            ${START_SCRIPT} "${start_args[@]}" -z "${ZOOKEEPER_HOST}" || exit_on_error "Can't start node '${node_name}'"
         fi
     done
 }
@@ -258,15 +273,21 @@ solr_cloud_configure_collection() {
     # modify solrconfig.xml to remove section that doesn't agree with our schema
     sed -i.bak '/<updateRequestProcessorChain name="add-unknown-fields-to-the-schema".*/,/<\/updateRequestProcessorChain>/d' ${TEMPLATE_DIR}/solrconfig.xml
     # Adapt autoSoftCommit to have a recommended value
-    sed -i.bak 's/${solr.autoSoftCommit.maxTime:-1}/${solr.autoSoftCommit.maxTime:20}/' "${TEMPLATE_DIR}/solrconfig.xml" || exit_on_error "Can't modify file '${TEMPLATE_DIR}/solrconfig.xml'"
+    # upstream default was '-1' up to Solr 8 and '3000' since Solr 9, so match any value
+    sed -i.bak 's/${solr.autoSoftCommit.maxTime:[^}]*}/${solr.autoSoftCommit.maxTime:20}/' "${TEMPLATE_DIR}/solrconfig.xml" || exit_on_error "Can't modify file '${TEMPLATE_DIR}/solrconfig.xml'"
     # Configure spellcheck component
     sed -i.bak 's/<str name="field">_text_<\/str>/<str name="field">meta_content__text_t<\/str>/' "${TEMPLATE_DIR}/solrconfig.xml"
     # Add spellcheck component to /select handler
     sed -i.bak 's/<requestHandler name="\/select" class="solr.SearchHandler">/<requestHandler name="\/select" class="solr.SearchHandler">\n    <arr name="last-components">\n      <str>spellcheck<\/str>\n    <\/arr>/' "${TEMPLATE_DIR}/solrconfig.xml"
 }
 
 solr_cloud_upload_collection_configuration() {
-    ${ZOOKEEPER_CLI_SCRIPT} -zkhost "${ZOOKEEPER_HOST}" -cmd upconfig -confname ${SOLR_CONFIGURATION_NAME} -confdir ${TEMPLATE_DIR} || exit_on_error "Can't upload configuration to Zookeeper"
+    if [[ "${SOLR_VERSION}" =~ ^10\. ]]; then
+        # Solr 10: zkcli.sh was removed in favour of 'bin/solr zk' (SOLR-14115)
+        ${START_SCRIPT} zk upconfig -z "${ZOOKEEPER_HOST}" -n ${SOLR_CONFIGURATION_NAME} -d ${TEMPLATE_DIR} || exit_on_error "Can't upload configuration to Zookeeper"
+    else
+        ${ZOOKEEPER_CLI_SCRIPT} -zkhost "${ZOOKEEPER_HOST}" -cmd upconfig -confname ${SOLR_CONFIGURATION_NAME} -confdir ${TEMPLATE_DIR} || exit_on_error "Can't upload configuration to Zookeeper"
+    fi
     echo "Uploaded configuration to Zookeeper"
 }
 
@@ -290,7 +311,6 @@ solr_cloud_create_collection() {
         "collection.configName=${SOLR_CONFIGURATION_NAME}"
         "createNodeSet=${nodes}"
         "shards=${shards}"
-        "maxShardsPerNode=${SOLR_MAX_SHARDS_PER_NODE}"
         "replicationFactor=${SOLR_REPLICATION_FACTOR}"
         "router.name=compositeId"
         "numShards=${shards_count}"
@@ -299,6 +319,11 @@ solr_cloud_create_collection() {
         "indent=on"
     )
 
+    # maxShardsPerNode was removed from the Collections API in Solr 9
+    if [[ ! "${SOLR_VERSION}" =~ ^(9|10)\. ]]; then
+        parameters+=("maxShardsPerNode=${SOLR_MAX_SHARDS_PER_NODE}")
+    fi
+
     echo "Creating collection with parameters:"
     echo "$(IFS=$'\n'; echo "${parameters[*]}")"
 
@@ -311,7 +336,7 @@ download
 
 if [ "$SOLR_CLOUD" = "no" ]; then
 
-    if [[ "${SOLR_VERSION}" =~ ^9\. ]]; then
+    if [[ "${SOLR_VERSION}" =~ ^(9|10)\. ]]; then
         TEMPLATE_CONF="template/conf"
     else
         TEMPLATE_CONF="template"

.github/workflows/integration-tests.yaml

@@ -17,14 +17,15 @@ jobs:
                 php-version:
                     - '8.3'
                 solr-version:
-                    - '7.7.3'
-                    - '8.11.2'
-                    - '9.8.1'
+                    - '10.0.0'
                 cores-setup:
                     - 'dedicated'
                     - 'shared'
                     - 'single'
                     - 'cloud'
+                # Solr 10 requires Java 21+
+                java-version:
+                    - '21'
         env:
             CORES_SETUP: ${{ matrix.cores-setup }}
             SOLR_VERSION: ${{ matrix.solr-version }}
@@ -37,7 +38,13 @@ jobs:
               name: "Set up single core"
               run: echo "SOLR_CORES=collection1" >> $GITHUB_ENV
 
-            - uses: actions/checkout@v4
+            - uses: actions/checkout@v7
+
+            - name: Setup Java
+              uses: actions/setup-java@v6
+              with:
+                  distribution: 'temurin'
+                  java-version: ${{ matrix.java-version }}
 
             - name: Setup PHP Action
               uses: shivammathur/setup-php@v2

README.md

@@ -6,6 +6,15 @@ To use this package, [install Ibexa DXP](https://doc.ibexa.co/en/latest/install/
 
 This package contains the Solr search engine implementation for [Ibexa DXP](https://ibexa.co).
 
+## Supported Solr versions
+
+Solr 10 is the only supported version. It requires Java 21
+(see [Solr system requirements](https://solr.apache.org/guide/solr/latest/deployment-guide/system-requirements.html)).
+
+When upgrading a SolrCloud cluster, note that nodes refuse to join
+a cluster containing higher-versioned nodes (rolling upgrades only go up, see
+[Solr 9.10 upgrade notes](https://solr.apache.org/guide/solr/latest/upgrade-notes/major-changes-in-solr-9.html#solr-9-10)).
+
 ### Testing locally
 
 To contribute to this bundle, make sure to run both unit and integration tests (from the `ezplatform-kernel` repository).

bin/generate-solr-config.sh

@@ -3,8 +3,8 @@
 set -e
 
 # Default parameters, if not overloaded by user arguments
-DESTINATION_DIR=.platform/configsets/solr8/conf
-SOLR_VERSION=8.11.1
+DESTINATION_DIR=.platform/configsets/solr10/conf
+SOLR_VERSION=10.0.0
 FORCE=false
 SOLR_INSTALL_DIR=""
 ALLOW_URLS_CLI=""
@@ -20,21 +20,21 @@ Help (this text):
 
 Usage with Ibexa Cloud (arguments here can be skipped as they have default values):
 ./vendor/ibexa/solr/bin/generate-solr-config.sh \\
-  --destination-dir=.platform/configsets/solr8/conf \\
-  --solr-version=8.11.1
+  --destination-dir=.platform/configsets/solr10/conf \\
+  --solr-version=10.0.0
 
 Usage with on-premise version of Solr:
 ./vendor/ibexa/solr/bin/generate-solr-config.sh \\
   --destination-dir=/opt/solr/server/ibexa/template \\
   --solr-install-dir=/opt/solr
 
 Warning:
- This script only supports Solr 7 and higher !!
+ This script only supports Solr 10 !!
 
 
 Arguments:
   [--destination-dir=<dest.dir>]     : Location where solr config should be stored
-                                       Default value is .platform/configsets/solr8/conf
+                                       Default value is .platform/configsets/solr10/conf
   [-f|--force]                       : Overwrite destination-dir if it already exists
   [--solr-install-dir]               : Existing downloaded Solr install to copy base config from.
   [--solr-version]                   : Solr version to download & copy base config from, used only if --solr-install-dir is unset
@@ -82,6 +82,11 @@ done
 
 : "${ALLOW_URLS_CLI:=${ALLOW_URLS:-}}"
 
+# Solr 10 is the only supported version; older ones remain usable for internal CI (init_solr.sh)
+if [[ ! "${SOLR_VERSION}" =~ ^10\. ]]; then
+    echo -e "\033[1;31mWarning: Solr ${SOLR_VERSION} is not supported, use Solr 10 \033[0m"
+fi
+
 if [ `whoami` == "root" ]; then
     echo "Error : Do not run this script as root"
     exit 1
@@ -102,7 +107,13 @@ if [ "$SOLR_INSTALL_DIR" == "" ]; then
     # If we were not provided an existing install directory we'll temporarily download a version of solr to generate config.
     GENERATE_SOLR_TMPDIR=`mktemp -d`
     echo "Downloading solr bundle:"
-    curl https://archive.apache.org/dist/lucene/solr/${SOLR_VERSION}/solr-${SOLR_VERSION}.tgz > $GENERATE_SOLR_TMPDIR/solr-${SOLR_VERSION}.tgz
+    # Solr 9+ releases moved from dist/lucene/solr to dist/solr/solr
+    if [[ "${SOLR_VERSION}" =~ ^(9|10)\. ]]; then
+        SOLR_ARCHIVE_URL="https://archive.apache.org/dist/solr/solr/${SOLR_VERSION}/solr-${SOLR_VERSION}.tgz"
+    else
+        SOLR_ARCHIVE_URL="https://archive.apache.org/dist/lucene/solr/${SOLR_VERSION}/solr-${SOLR_VERSION}.tgz"
+    fi
+    curl ${SOLR_ARCHIVE_URL} > $GENERATE_SOLR_TMPDIR/solr-${SOLR_VERSION}.tgz
 
     echo "Untaring"
     cd $GENERATE_SOLR_TMPDIR
@@ -118,7 +129,7 @@ cp ${SOLR_INSTALL_DIR}/server/solr/configsets/_default/conf/{solrconfig.xml,stop
 
 if [[ ! $DESTINATION_DIR =~ ^\.platform ]]; then
 
-    if [[ "${SOLR_VERSION}" =~ ^9\. ]]; then
+    if [[ "${SOLR_VERSION}" =~ ^(9|10)\. ]]; then
         cp -f ${SOLR_INSTALL_DIR}/server/solr/solr.xml $DESTINATION_DIR/../..
 
         URL_LIST="${ALLOW_URLS_CLI//,/ }"
@@ -147,7 +158,8 @@ fi
 
 # Adapt autoSoftCommit to have a recommended value, and remove add-unknown-fields-to-the-schema
 sed -i.bak '/<updateRequestProcessorChain name="add-unknown-fields-to-the-schema".*/,/<\/updateRequestProcessorChain>/d' $DESTINATION_DIR/solrconfig.xml
-sed -i.bak 's/${solr.autoSoftCommit.maxTime:-1}/${solr.autoSoftCommit.maxTime:20}/' $DESTINATION_DIR/solrconfig.xml
+# upstream default was '-1' up to Solr 8 and '3000' since Solr 9, so match any value
+sed -i.bak 's/${solr.autoSoftCommit.maxTime:[^}]*}/${solr.autoSoftCommit.maxTime:20}/' $DESTINATION_DIR/solrconfig.xml
 # Configure spellcheck component
 sed -i.bak 's/<str name="field">_text_<\/str>/<str name="field">meta_content__text_t<\/str>/' $DESTINATION_DIR/solrconfig.xml
 # Add spellcheck component to /select handler

phpstan-baseline.neon

@@ -225,7 +225,7 @@ parameters:
 		-
 			message: '#^Parameter \#2 \$fieldDefinitionIdentifier of method Ibexa\\Solr\\Query\\Common\\CriterionVisitor\\MapLocation\:\:getSearchFields\(\) expects string, string\|null given\.$#'
 			identifier: argument.type
-			count: 2
+			count: 1
 			path: src/lib/Query/Common/CriterionVisitor/MapLocation/MapLocationDistanceRange.php
 
 		-
@@ -596,12 +596,6 @@ parameters:
 			count: 1
 			path: tests/lib/Search/ResultExtractor/AggregationResultExtractor/RangeAggregationResultExtractorTest.php
 
-		-
-			message: '#^Parameter \#2 \$entries of class Ibexa\\Contracts\\Core\\Repository\\Values\\Content\\Search\\AggregationResult\\RangeAggregationResult constructor expects iterable\<Ibexa\\Contracts\\Core\\Repository\\Values\\Content\\Search\\AggregationResult\\RangeAggregationResultEntry\<string\|null\>\>, array\{Ibexa\\Contracts\\Core\\Repository\\Values\\Content\\Search\\AggregationResult\\RangeAggregationResultEntry\<string\|null\>, Ibexa\\Contracts\\Core\\Repository\\Values\\Content\\Search\\AggregationResult\\RangeAggregationResultEntry\<string\>, Ibexa\\Contracts\\Core\\Repository\\Values\\Content\\Search\\AggregationResult\\RangeAggregationResultEntry\<string\|null\>\} given\.$#'
-			identifier: argument.type
-			count: 1
-			path: tests/lib/Search/ResultExtractor/AggregationResultExtractor/RangeAggregationResultExtractorTest.php
-
 		-
 			message: '#^Parameter \#1 \$aggregation of method Ibexa\\Solr\\ResultExtractor\\AggregationResultExtractor\\TermAggregationKeyMapper\\ContentTypeAggregationKeyMapper\:\:map\(\) expects Ibexa\\Contracts\\Core\\Repository\\Values\\Content\\Query\\Aggregation\\ContentTypeTermAggregation, Ibexa\\Contracts\\Core\\Repository\\Values\\Content\\Query\\Aggregation&PHPUnit\\Framework\\MockObject\\MockObject given\.$#'
 			identifier: argument.type

src/bundle/DependencyInjection/Configuration.php

@@ -18,7 +18,7 @@ class Configuration implements ConfigurationInterface
 {
     public const int SOLR_HTTP_CLIENT_DEFAULT_TIMEOUT = 10;
     public const int SOLR_HTTP_CLIENT_DEFAULT_MAX_RETRIES = 3;
-    public const string SOLR_DEFAULT_VERSION = '7.7.3';
+    public const string SOLR_DEFAULT_VERSION = '10.0.0';
 
     /**
      * Holds default endpoint values.
@@ -113,7 +113,7 @@ protected function addConnectionsSection(ArrayNodeDefinition $node): void
     {
         $node->children()
             ->scalarNode('version')
-                ->info('Version of the Solr Search Engine to use')
+                ->info('Version of the Solr Search Engine in use. Solr 10 is the only supported version')
                 ->defaultValue(self::SOLR_DEFAULT_VERSION)
             ->end()
             ->scalarNode('default_connection')

src/lib/Gateway/DistributionStrategy/CloudDistributionStrategy.php

@@ -11,7 +11,7 @@
 /**
  * Solr Cloud distributed search.
  *
- * @see https://lucene.apache.org/solr/guide/7_7/distributed-requests.html
+ * @see https://solr.apache.org/guide/solr/latest/deployment-guide/solrcloud-distributed-requests.html
  */
 final class CloudDistributionStrategy extends AbstractDistributionStrategy
 {

src/lib/Gateway/DistributionStrategy/StandaloneDistributionStrategy.php

@@ -11,7 +11,7 @@
 /**
  * Standalone setup of distributed search.
  *
- * @see https://lucene.apache.org/solr/guide/7_7/distributed-search-with-index-sharding.html
+ * @see https://solr.apache.org/guide/solr/latest/deployment-guide/user-managed-distributed-search.html
  */
 final class StandaloneDistributionStrategy extends AbstractDistributionStrategy
 {

src/lib/Handler.php

@@ -285,11 +285,12 @@ protected function updateAllElementsWithAdditionalLocation(int $locationId): voi
         foreach ($searchResult->searchHits as $searchHit) {
             try {
                 $contentInfo = $this->contentHandler->loadContentInfo($searchHit->valueObject->id);
+                // load() may throw even though loadContentInfo() succeeded, as the persistence
+                // cache can serve a stale ContentInfo for content deleted a moment ago
+                $contentItems[] = $this->contentHandler->load($contentInfo->id, $contentInfo->currentVersionNo);
             } catch (NotFoundException) {
                 continue;
             }
-
-            $contentItems[] = $this->contentHandler->load($contentInfo->id, $contentInfo->currentVersionNo);
         }
 
         $this->bulkIndexContent($contentItems);
@@ -319,14 +320,20 @@ protected function allItemsWithinLocation(int $locationId): CustomField
     }
 
     /**
-     * @param int $locationId
+     * Matches Content documents having at least one Location outside the given Location subtree.
+     *
+     * Locations are indexed as child documents of Content, so this is a boolean NOT on a block-join
+     * child query. A single complement regex on the multivalued Content path field was used before,
+     * but the Lucene complement operator (~) is unavailable as of Lucene 10 (LUCENE-10010).
      */
-    protected function allItemsWithinLocationWithAdditionalLocation($locationId): CustomField
+    protected function allItemsWithinLocationWithAdditionalLocation(int $locationId): CustomField
     {
+        // The CustomField visitor emits this value inside a quoted '_query_:"…"' clause, where the
+        // quoted-string parsing consumes one escaping level - hence the double backslash before slashes
         return new CustomField(
-            'location_path_string_mid',
+            '_query_',
             Criterion\Operator::EQ,
-            "/@&~(.*\\/{$locationId}\\/.*)/"
+            "{!parent which=document_type_id:content}(+document_type_id:location -path_string_id:/.*\\\\/{$locationId}\\\\/.*/)"
         );
     }