From eb7840ed66547dfd208a5e22cbdde98d00b2d429 Mon Sep 17 00:00:00 2001 From: Shinsuke Sugaya Date: Sat, 22 Aug 2026 19:10:18 +0900 Subject: [PATCH] feat: migrate to Fess 15.8 Pin Fess 15.8.0 / OpenSearch 3.8.0, and bring along the pieces that do not follow those pins on their own. - .env: FESS_VERSION=15.8.0, OPENSEARCH_VERSION=3.8.0. The fess_config.properties base is re-fetched from the fess-15.8.0 tag by render-fess-config.sh with no further change; every key the codesearch overlay redefines still exists in the 15.8 base. - bin/setup.sh: fess-ds-git 15.8.0, downloaded from maven.codelibs.org/release. Fess plugins are no longer published to Maven Central from the 15.8 line on, so the previous URL 404s for 15.8.0 and setup.sh would abort. FESS_PLUGIN_REPO_URL overrides the repository for pinning an older plugin release. - fess_indices/_codesearch: refresh the hand-maintained fork with what upstream added in 15.8 - index.knn (plus knn.derived_source.enabled and the merge policy) in fess.json, and the content_chunk_vector / content_chunk_status document fields in fess/doc.json. The fork is now identical to codelibs/fess@fess-15.8.0 apart from the codesearch tuning it exists for. Codesearch does not use semantic search - the Content Chunk Vector Indexer job and content_chunker.search.enabled are both off by default - but a mapping is only ever applied to a brand-new index, so carrying it now is what keeps enabling it later from requiring a full re-crawl. - bin/verify.sh: read record_count out of the {"response": {...}} envelope that /api/v2/search actually returns. The probe read it at the top level, so it never found a count and silently SKIPped instead of running - the one check that catches the "HTTP 200 with 0 hits" failure mode this script exists for. - README, register_github.sh, compose.yaml: version references, the plugin repository change, and a note that an in-place upgrade keeps the old index mapping. --- .env | 4 +-- README.md | 26 +++++++++++-------- bin/register_github.sh | 6 ++--- bin/render-fess-config.sh | 2 +- bin/setup.sh | 14 +++++++--- bin/verify.sh | 7 ++++- compose.yaml | 2 +- .../fess_indices/_codesearch/fess.json | 3 +++ .../fess_indices/_codesearch/fess/doc.json | 18 +++++++++++++ 9 files changed, 60 insertions(+), 22 deletions(-) diff --git a/.env b/.env index aef6094..30363b0 100644 --- a/.env +++ b/.env @@ -3,5 +3,5 @@ # Docker Compose auto-loads this file for ${VAR} substitution in compose.yaml. # NOTE: version pins only — do NOT put secrets here # (use conf/fess_config.local.properties for those). -FESS_VERSION=15.7.0 -OPENSEARCH_VERSION=3.7.0 +FESS_VERSION=15.8.0 +OPENSEARCH_VERSION=3.8.0 diff --git a/README.md b/README.md index 478b592..659f3fd 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ ## Architecture / Theme Model -- **Theme**: Fess 15.7 static theme system — `theme.default=codesearch` in `system.properties` selects the codesearch theme. No virtual-host routing is needed for theme activation. +- **Theme**: Fess 15.8 static theme system — `theme.default=codesearch` in `system.properties` selects the codesearch theme. No virtual-host routing is needed for theme activation. - **Fess config (`fess_config.properties`)**: `setup.sh` generates `data/fess/opt/fess/fess_config.properties` from the upstream base for the pinned Fess version plus the codesearch overlay (`conf/fess_config.overlay.properties`) and an optional local override (`conf/fess_config.local.properties`). It is mounted at `/opt/fess`, which the image places ahead of its `/etc/fess` default on the classpath, so the generated file takes effect. Only the delta is tracked in git; the base auto-tracks the pinned version. See [Configuration](#configuration). - **Version pins (`.env`)**: `FESS_VERSION` / `OPENSEARCH_VERSION` are the single source of truth for the image tags (`compose.yaml`) and the `fess_config.properties` base. - **system.properties**: The live file (`data/fess/opt/fess/system.properties`) is generated from `data/fess/opt/fess/system.properties.template` by `setup.sh` on first run. The live file is git-ignored. @@ -66,7 +66,7 @@ Once documents are indexed, it also probes the search API by comparing the theme `fessctl` (used in the next steps) authenticates to Fess with an access token. Create one with the `{role}admin-api` permission on the Admin Access Token page ([http://localhost:8080/admin/accesstoken/](http://localhost:8080/admin/accesstoken/)). -For more details, see the [Admin Access Token Guide](https://fess.codelibs.org/15.7/admin/accesstoken-guide.html). +For more details, see the [Admin Access Token Guide](https://fess.codelibs.org/15.8/admin/accesstoken-guide.html). ### Install fessctl @@ -81,7 +81,7 @@ pipx install fessctl # or: uv tool install fessctl ```bash export FESS_ENDPOINT=http://localhost:8080 export FESS_ACCESS_TOKEN= -export FESS_VERSION=15.7.0 +export FESS_VERSION=15.8.0 fessctl ping # reports the search engine status (GREEN when ready) ``` @@ -195,26 +195,30 @@ docker compose -f compose.yaml up -d | `FESS_VERSION` | base ref | |----------------|----------| -| `15.7.0`, `15.6.1`, … | `fess-` (the release tag) | -| `15.7.0-noble`, `15.7.0-al2023` | `fess-` (the OS suffix is dropped) | -| `snapshot`, `snapshot-noble`, `snapshot-al2023`, `15.8.0-SNAPSHOT` | `master` | -| anything else (`latest`, `15.7`, …) | **rejected** — pin an explicit release, or set `FESS_CONFIG_BASE_REF` | +| `15.8.0`, `15.7.0`, … | `fess-` (the release tag) | +| `15.8.0-noble`, `15.8.0-al2023` | `fess-` (the OS suffix is dropped) | +| `snapshot`, `snapshot-noble`, `snapshot-al2023`, `15.9.0-SNAPSHOT` | `master` | +| anything else (`latest`, `15.8`, …) | **rejected** — pin an explicit release, or set `FESS_CONFIG_BASE_REF` | Floating tags are rejected on purpose: `latest` has no matching source ref, so there is no way to render a base config that is guaranteed to match the running image. For a ref this mapping does not cover — an unreleased version, or a maintenance branch — name it explicitly: ```bash -FESS_CONFIG_BASE_REF=15.8.x bash ./bin/render-fess-config.sh +FESS_CONFIG_BASE_REF=15.9.x bash ./bin/render-fess-config.sh ``` -Plugin versions are **not** derived from `FESS_VERSION` — snapshot images have no matching plugin release. Override them per plugin if the pinned one is too old, e.g. `FESS_DS_GIT_VERSION=15.8.0 bash ./bin/setup.sh`. +Plugin versions are **not** derived from `FESS_VERSION` — snapshot images have no matching plugin release. Override them per plugin if the pinned one is too old, e.g. `FESS_DS_GIT_VERSION=15.9.0 bash ./bin/setup.sh`. + +Plugin jars are downloaded from `https://maven.codelibs.org/release`, where Fess plugins are published from the 15.8 line on. Releases up to 15.7.0 are on Maven Central instead, so pinning an older plugin also needs `FESS_PLUGIN_REPO_URL=https://repo1.maven.org/maven2`. ### Index schema (`fess_indices/_codesearch`) `data/fess/usr/share/fess/app/WEB-INF/classes/fess_indices/_codesearch/` is a **hand-maintained fork** of the upstream index schema, selected by `search_engine.type=codesearch`. It carries genuine codesearch tuning that has no upstream equivalent — the `line_number_filter` char filter that strips the `L:` prefix added by the handler script, code-aware `operator_filter` / `dotnum_filter` / `code_stop_filter` tokenization, and the seven codesearch document fields (`domain`, `organization`, `repository`, `path`, `repository_url`, `owner`, `homepage`). -Unlike `fess_config.properties`, it is **not** regenerated per version, so it can drift from upstream. `bin/verify.sh` reports the dangerous direction (a core field the running Fess expects that the fork lacks) as an advisory `WARN`; it currently reports none for 15.7.0. Refresh it by hand when upstream adds document fields. +Unlike `fess_config.properties`, it is **not** regenerated per version, so it can drift from upstream. `bin/verify.sh` reports the dangerous direction (a core field the running Fess expects that the fork lacks) as an advisory `WARN`; it currently reports none for 15.8.0. Refresh it by hand when upstream adds document fields. + +The 15.8 refresh carries over the semantic-chunk mapping Fess 15.8 ships unconditionally: `index.knn` in `fess.json` and the `content_chunk_vector` / `content_chunk_status` document fields. Codesearch does not use semantic search — the `content-chunk-vector-indexer` job and `content_chunker.search.enabled` are both off by default — but the mapping is only ever applied to a brand-new index, so shipping it now is what keeps enabling it later from requiring a full re-crawl. It does make the k-NN plugin a hard requirement; `ghcr.io/codelibs/fess-opensearch` bundles it. -> **Re-index after a major version bump**: a Fess or OpenSearch major upgrade can change the index format. If search returns errors or stops returning results after upgrading, re-crawl your repositories with `fessctl scheduler start default_crawler` to rebuild the index. +> **Re-index after a major version bump**: a Fess or OpenSearch major upgrade can change the index format. If search returns errors or stops returning results after upgrading, re-crawl your repositories with `fessctl scheduler start default_crawler` to rebuild the index. An index created by an older version keeps its original mapping — Fess applies a mapping only to an index that has none yet — so an in-place upgrade will not pick up the fields above until the index is rebuilt. ## Troubleshooting diff --git a/bin/register_github.sh b/bin/register_github.sh index a7b6444..b880b39 100755 --- a/bin/register_github.sh +++ b/bin/register_github.sh @@ -10,7 +10,7 @@ # Connection/auth use fessctl's own environment variables: # FESS_ENDPOINT Fess base URL (default: http://localhost:8080) # FESS_ACCESS_TOKEN admin-api access token (required) -# FESS_VERSION Fess version (default: 15.7.0) +# FESS_VERSION Fess version (default: 15.8.0) # # Requirements: fessctl (https://github.com/codelibs/fessctl), git, python3. set -euo pipefail @@ -37,7 +37,7 @@ Options: Environment (consumed by fessctl): FESS_ENDPOINT Fess base URL (default: http://localhost:8080) FESS_ACCESS_TOKEN Admin-api access token (required) - FESS_VERSION Fess version (default: 15.7.0) + FESS_VERSION Fess version (default: 15.8.0) Examples: FESS_ACCESS_TOKEN=xxxx ./bin/register_github.sh codelibs fess-suggest @@ -78,7 +78,7 @@ command -v git >/dev/null 2>&1 || die "git not found." command -v python3 >/dev/null 2>&1 || die "python3 not found." [ -n "${FESS_ACCESS_TOKEN:-}" ] || die "FESS_ACCESS_TOKEN is not set (an admin-api access token)." : "${FESS_ENDPOINT:=http://localhost:8080}"; export FESS_ENDPOINT -: "${FESS_VERSION:=15.7.0}"; export FESS_VERSION +: "${FESS_VERSION:=15.8.0}"; export FESS_VERSION git_url="https://${domain}/${org}/${repo}.git" diff --git a/bin/render-fess-config.sh b/bin/render-fess-config.sh index da0858d..9a49989 100755 --- a/bin/render-fess-config.sh +++ b/bin/render-fess-config.sh @@ -54,7 +54,7 @@ else [0-9]*.[0-9]*.[0-9]*) base_ref="fess-${fess_release}" ;; *) echo "ERROR: cannot derive the fess_config.properties base ref from FESS_VERSION=${FESS_VERSION}." >&2 - echo " Pin an explicit release in .env (e.g. FESS_VERSION=15.7.0) - a floating tag" >&2 + echo " Pin an explicit release in .env (e.g. FESS_VERSION=15.8.0) - a floating tag" >&2 echo " such as 'latest' has no matching source ref - or set FESS_CONFIG_BASE_REF" >&2 echo " to the codelibs/fess ref that image was built from." >&2 exit 1 diff --git a/bin/setup.sh b/bin/setup.sh index 2b88590..5076076 100755 --- a/bin/setup.sh +++ b/bin/setup.sh @@ -12,12 +12,18 @@ set -euo pipefail base_dir=$(cd "$(dirname "$0")";cd ..;pwd) # fess-script-groovy is no longer downloaded: the Groovy script engine is # bundled in Fess core since 15.0. -# Override a version with e.g. FESS_DS_GIT_VERSION=15.8.0 (these are not +# Override a version with e.g. FESS_DS_GIT_VERSION=15.9.0 (these are not # derived from FESS_VERSION: snapshot images have no matching release plugin). fess_plugins=" -fess-ds-git:${FESS_DS_GIT_VERSION:-15.7.0} +fess-ds-git:${FESS_DS_GIT_VERSION:-15.8.0} " +# Maven repository the plugin jars are downloaded from. Fess plugins moved off +# Maven Central with the 15.8 line, so releases from 15.8.0 on are only on +# maven.codelibs.org; point this at https://repo1.maven.org/maven2 to pull an +# older one. +fess_plugin_repo_url="${FESS_PLUGIN_REPO_URL:-https://maven.codelibs.org/release}" + # fess-themes branch to fetch the codesearch static theme from (default: main). # Override with FESS_THEMES_BRANCH= to test theme changes from another branch. fess_themes_branch="${FESS_THEMES_BRANCH:-main}" @@ -53,7 +59,7 @@ for fess_plugin in ${fess_plugins} ; do plugin_version=$(echo "$fess_plugin" | sed -e "s/.*://") plugin_jar=${plugin_name}-${plugin_version}.jar plugin_file=${plugin_dir}/${plugin_jar} - plugin_url=https://repo1.maven.org/maven2/org/codelibs/fess/${plugin_name}/${plugin_version}/${plugin_jar} + plugin_url=${fess_plugin_repo_url}/org/codelibs/fess/${plugin_name}/${plugin_version}/${plugin_jar} echo "Downloading ${plugin_name} version ${plugin_version}..." # -f, or a 404 page gets written out as a .jar that Fess then fails to load. if ! curl -fsSL "${plugin_url}" -o "${plugin_file}.tmp"; then @@ -61,6 +67,8 @@ for fess_plugin in ${fess_plugins} ; do echo "ERROR: could not download ${plugin_name} ${plugin_version} from" >&2 echo " ${plugin_url}" >&2 echo " Check the version (see FESS_DS_GIT_VERSION in this script)." >&2 + echo " Plugin releases before 15.8.0 are on Maven Central instead:" >&2 + echo " FESS_PLUGIN_REPO_URL=https://repo1.maven.org/maven2" >&2 exit 1 fi if ! unzip -tq "${plugin_file}.tmp" > /dev/null 2>&1; then diff --git a/bin/verify.sh b/bin/verify.sh index dbbfdde..f8098a8 100755 --- a/bin/verify.sh +++ b/bin/verify.sh @@ -162,11 +162,16 @@ case "${status}" in if [ "${have_python}" -eq 0 ]; then skip "python3 not found; cannot compare record counts (HTTP 200 alone does not prove the facets worked)" else + # /api/v2/search wraps its payload in a {"response": {...}} envelope, so + # record_count lives one level down; fall back to the top level for the + # older flat shape. count_of() { curl -fsS "$1" 2>/dev/null | python3 -c ' import json, sys try: - print(int(json.load(sys.stdin).get("record_count", -1))) + doc = json.load(sys.stdin) + body = doc.get("response", doc) + print(int(body.get("record_count", -1))) except Exception: print(-1) ' 2>/dev/null || echo -1 diff --git a/compose.yaml b/compose.yaml index 17514a0..d454ecc 100644 --- a/compose.yaml +++ b/compose.yaml @@ -25,7 +25,7 @@ services: - ./data/fess/usr/share/fess/app/WEB-INF/plugin:/usr/share/fess/app/WEB-INF/plugin - ./data/fess/usr/share/fess/app/WEB-INF/classes/fess_indices/_codesearch:/usr/share/fess/app/WEB-INF/classes/fess_indices/_codesearch # Static theme fetched from fess-themes by setup.sh; mounted at the Fess app - # themes dir (served at /themes/codesearch by the 15.7 container). + # themes dir (served at /themes/codesearch by the 15.8 container). - ./data/fess/themes/codesearch:/usr/share/fess/app/themes/codesearch logging: driver: "json-file" diff --git a/data/fess/usr/share/fess/app/WEB-INF/classes/fess_indices/_codesearch/fess.json b/data/fess/usr/share/fess/app/WEB-INF/classes/fess_indices/_codesearch/fess.json index 6ddf278..6b32d00 100644 --- a/data/fess/usr/share/fess/app/WEB-INF/classes/fess_indices/_codesearch/fess.json +++ b/data/fess/usr/share/fess/app/WEB-INF/classes/fess_indices/_codesearch/fess.json @@ -1,6 +1,9 @@ { "settings": { "index": { + "knn": true, + "knn.derived_source.enabled": false, + "merge": { "policy": { "floor_segment": "16mb", "max_merge_at_once": 30 } }, "codec": "${fess.index.codec}", "refresh_interval": "1s", "number_of_shards": "${fess.index.number_of_shards}", diff --git a/data/fess/usr/share/fess/app/WEB-INF/classes/fess_indices/_codesearch/fess/doc.json b/data/fess/usr/share/fess/app/WEB-INF/classes/fess_indices/_codesearch/fess/doc.json index fe0c5c4..92afea7 100644 --- a/data/fess/usr/share/fess/app/WEB-INF/classes/fess_indices/_codesearch/fess/doc.json +++ b/data/fess/usr/share/fess/app/WEB-INF/classes/fess_indices/_codesearch/fess/doc.json @@ -495,6 +495,21 @@ "type": "text", "index": false }, + "content_chunk_vector": { + "type": "nested", + "properties": { + "vector": { + "type": "knn_vector", + "dimension": "${fess.content_chunker.embedding.dimension}", + "method": { + "name": "${fess.content_chunker.search.knn.method}", + "engine": "${fess.content_chunker.search.knn.engine}", + "space_type": "${fess.content_chunker.search.knn.space_type}", + "parameters": { "m": 16, "ef_construction": 100 } + } + } + } + }, "content": { "type": "text", "analyzer": "standard_analyzer", @@ -513,6 +528,9 @@ "content_length": { "type": "long" }, + "content_chunk_status": { + "type": "keyword" + }, "created": { "type": "date", "format": "date_optional_time"