diff --git a/.github/workflows/idric-core.yml b/.github/workflows/idric-core.yml index bc946a3..37d3dce 100644 --- a/.github/workflows/idric-core.yml +++ b/.github/workflows/idric-core.yml @@ -29,6 +29,9 @@ jobs: - name: Exercise filesystem category and hot views run: sh bin/ci_browser_foundation.grease exercise-filesystem-views + - name: Exercise saved-tab text question mock + run: sh bin/ci_browser_foundation.grease exercise-tab-qa + - name: Install system dependencies through Grease run: sh bin/ci_browser_foundation.grease install-dependencies diff --git a/README.md b/README.md index 283b612..d170526 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,8 @@ The browser core owns resource, tab, event, and task identity; sleeping and waki - `docs/filesystem-views.md` — category links, `_active`, and the requested `hot/` presentation set - `docs/storage-model.md` — identity levels and canonical, proposed, and derived state - `docs/developer-workbench.md` — fixture and memory-pressure harness +- `docs/vector-index.md` — readable multi-model vector views and rebuildable Float32 query caches +- `docs/tab-qa-mock.md` — console questions over saved reading pages with replaceable processing stages - `experiments/category-hyperplanes/README.md` — disposable embedding and explicit affine-separator probe - `experiments/embedding-models/README.md` — pinned tiny ONNX models and an end-to-end filesystem-index comparison diff --git a/bin/ask_saved_pages.grease b/bin/ask_saved_pages.grease new file mode 100755 index 0000000..42e6a6c --- /dev/null +++ b/bin/ask_saved_pages.grease @@ -0,0 +1,30 @@ +#!/bin/sh +set -eu + +repository_root=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +export IB_TAB_QA_REPOSITORY_ROOT="$repository_root" +. "$repository_root/lib/tab_qa.grease" + +case "${1:-}" in + --index) + test "$#" -eq 1 || exit 2 + ib_tab_qa_build_index + ;; + --inspect) + test "$#" -eq 1 || exit 2 + ib_tab_qa_inspect + ;; + --help) + printf '%s\n' \ + "usage: $0 [--index|--inspect|QUESTION]" \ + 'With no argument, read one question from the console.' + ;; + '') + printf 'question> ' >&2 + IFS= read -r question || exit 1 + ib_tab_qa_answer "$question" + ;; + *) + ib_tab_qa_answer "$*" + ;; +esac diff --git a/bin/ci_browser_foundation.grease b/bin/ci_browser_foundation.grease index 27ec674..545b23d 100755 --- a/bin/ci_browser_foundation.grease +++ b/bin/ci_browser_foundation.grease @@ -74,6 +74,19 @@ exercise_filesystem_views() { sh tests/test_filesystem_views.grease } +exercise_tab_qa() { + cd "$repository_root" + sh -n lib/tab_qa.grease + sh -n bin/ask_saved_pages.grease + sh -n bin/mock_tab_model.grease + sh -n bin/mock_tab_qa_reducer.grease + sh -n tests/test_mock_tab_model.grease + sh -n tests/test_tab_qa_pipeline.grease + make -C native/vector-index + sh tests/test_mock_tab_model.grease + sh tests/test_tab_qa_pipeline.grease +} + exercise_workbench() { cd "$repository_root" sh -n tests/test_real_world_url_fixture.grease @@ -199,11 +212,12 @@ case "${1:-}" in exercise-workbench) exercise_workbench ;; exercise-file-store) exercise_file_store ;; exercise-filesystem-views) exercise_filesystem_views ;; + exercise-tab-qa) exercise_tab_qa ;; exercise-scientific-media) exercise_scientific_media ;; exercise-live-arxiv) exercise_live_arxiv ;; exercise-live-arxiv-prepaint) exercise_live_arxiv_prepaint ;; *) - printf 'usage: %s {install-dependencies|build-idric|verify-pdf-harvester|exercise-information|exercise-core|exercise-workbench|exercise-file-store|exercise-filesystem-views|exercise-scientific-media|exercise-live-arxiv|exercise-live-arxiv-prepaint}\n' "$0" >&2 + printf 'usage: %s {install-dependencies|build-idric|verify-pdf-harvester|exercise-information|exercise-core|exercise-workbench|exercise-file-store|exercise-filesystem-views|exercise-tab-qa|exercise-scientific-media|exercise-live-arxiv|exercise-live-arxiv-prepaint}\n' "$0" >&2 exit 2 ;; esac diff --git a/bin/mock_tab_model.grease b/bin/mock_tab_model.grease new file mode 100755 index 0000000..d478ac4 --- /dev/null +++ b/bin/mock_tab_model.grease @@ -0,0 +1,116 @@ +#!/bin/sh +set -eu + +model_name=mock-token-i8-v1 +dimensions=32 + +inspect_model() { + printf '%s\n' \ + "model=$model_name" \ + 'available=True' \ + 'model-kind=deterministic-token-test-double' \ + 'llm=False' \ + 'quantization=int8-counts' \ + "dimensions=$dimensions" \ + 'tasks=embedding,extractive-answer' +} + +embed_text() { + awk -v dimensions="$dimensions" ' + function token_slot(token, alphabet, character, hash, index_in_alphabet, position) { + alphabet = "abcdefghijklmnopqrstuvwxyz0123456789" + hash = 0 + for (position = 1; position <= length(token); position++) { + character = substr(token, position, 1) + index_in_alphabet = index(alphabet, character) + hash = (hash * 37 + index_in_alphabet) % dimensions + } + return hash + 1 + } + { + text = tolower($0) + gsub(/[^[:alnum:]]+/, " ", text) + token_count = split(text, tokens, / +/) + for (token_index = 1; token_index <= token_count; token_index++) { + token = tokens[token_index] + if (length(token) < 2) + continue + slot = token_slot(token) + if (values[slot] < 127) + values[slot]++ + } + } + END { + for (slot = 1; slot <= dimensions; slot++) + printf "%s%d", slot == 1 ? "" : " ", values[slot] + 0 + printf "\n" + } + ' +} + +answer_from_document() { + question=$1 + document=$2 + test -s "$document" || { + printf 'mock tab model: source document is empty: %s\n' "$document" >&2 + return 1 + } + + awk -v question="$question" ' + BEGIN { + normalized_question = tolower(question) + gsub(/[^[:alnum:]]+/, " ", normalized_question) + question_count = split(normalized_question, question_tokens, / +/) + for (question_index = 1; question_index <= question_count; question_index++) + if (length(question_tokens[question_index]) >= 2) + wanted[question_tokens[question_index]] = 1 + } + NF { + if (first_line == "") + first_line = $0 + normalized_line = tolower($0) + gsub(/[^[:alnum:]]+/, " ", normalized_line) + line_count = split(normalized_line, line_tokens, / +/) + score = 0 + for (line_index = 1; line_index <= line_count; line_index++) { + token = line_tokens[line_index] + if (token in wanted && seen_on_line[token] != NR) { + score++ + seen_on_line[token] = NR + } + } + if (best_line == "" || score > best_score) { + best_line = $0 + best_score = score + } + } + END { + if (best_line != "") + print best_line + else if (first_line != "") + print first_line + } + ' "$document" +} + +case "${1:-}" in + inspect) + test "$#" -eq 1 || exit 2 + inspect_model + ;; + embed) + test "$#" -eq 1 || exit 2 + embed_text + ;; + answer) + test "$#" -eq 3 || { + printf 'usage: %s answer QUESTION DOCUMENT\n' "$0" >&2 + exit 2 + } + answer_from_document "$2" "$3" + ;; + *) + printf 'usage: %s {inspect|embed|answer QUESTION DOCUMENT}\n' "$0" >&2 + exit 2 + ;; +esac diff --git a/bin/mock_tab_qa_reducer.grease b/bin/mock_tab_qa_reducer.grease new file mode 100755 index 0000000..5619dc3 --- /dev/null +++ b/bin/mock_tab_qa_reducer.grease @@ -0,0 +1,68 @@ +#!/bin/sh +set -eu + +reducer_name=mock-single-member-weighted-reducer-v1 + +inspect_reducer() { + printf '%s\n' \ + "reducer=$reducer_name" \ + 'available=True' \ + 'ensemble=single-member-test-double' \ + 'member-count=1' \ + 'xgboost=False' +} + +reduce_candidate() { + retrieval_score=$1 + source_document=$2 + report=${IB_TAB_QA_REDUCER_REPORT:?IB_TAB_QA_REDUCER_REPORT must name the reducer report} + candidate=$(sed -n '1p') + + test -n "$candidate" || { + printf 'mock reducer: model candidate is empty\n' >&2 + return 1 + } + test -s "$source_document" || { + printf 'mock reducer: source document is unavailable\n' >&2 + return 1 + } + awk -v value="$retrieval_score" 'BEGIN { + numeric = value ~ /^[-+]?([0-9]+([.][0-9]*)?|[.][0-9]+)([eE][-+]?[0-9]+)?$/ + exit !numeric + }' || { + printf 'mock reducer: retrieval score is not numeric\n' >&2 + return 1 + } + + printf '%s\n' \ + "reducer=$reducer_name" \ + 'ensemble-stage-ran=True' \ + 'member-count=1' \ + 'checks-run=3' \ + 'candidate-nonempty=True' \ + 'source-readable=True' \ + 'score-numeric=True' \ + "aggregate-score=$retrieval_score" > "$report" + + # This one-member reducer deliberately preserves the model text. A later + # voting, bagging, rank-aggregation, or XGBoost adapter occupies this stage. + printf '%s\n' "$candidate" +} + +case "${1:-}" in + inspect) + test "$#" -eq 1 || exit 2 + inspect_reducer + ;; + reduce) + test "$#" -eq 3 || { + printf 'usage: %s reduce RETRIEVAL_SCORE SOURCE_DOCUMENT < candidate.txt\n' "$0" >&2 + exit 2 + } + reduce_candidate "$2" "$3" + ;; + *) + printf 'usage: %s {inspect|reduce RETRIEVAL_SCORE SOURCE_DOCUMENT}\n' "$0" >&2 + exit 2 + ;; +esac diff --git a/docs/tab-qa-mock.md b/docs/tab-qa-mock.md new file mode 100644 index 0000000..5144a44 --- /dev/null +++ b/docs/tab-qa-mock.md @@ -0,0 +1,84 @@ +# Saved-tab question-and-answer mock + +This is the first executable vertical slice of the ChatGPT-like text frontend. +It is a console prompt, not the eventual phone UI, and it answers only from +plain-text documents deliberately present below `~/reading`. + +The stages are explicit: + +1. `mock-token-i8-v1` tokenizes text into a 32-component, bounded 8-bit-count + test vector and selects one source line as an extractive answer. +2. `ib-vector-index` stores one exact cosine index in the existing model-view + tree. Authoritative fixed-width text and its rebuildable Float32 cache are + two query paths over that index. Each query can record whether its dot + products used a GPU; the current C backend records `False` and `compute=cpu`. +3. `mock-single-member-weighted-reducer-v1` is a one-member ensemble test + double. It runs three validations, records an aggregate score, and preserves + the candidate. This is the replaceable slot for voting, bagging, rank + aggregation, XGBoost, or another reducer; it does not claim to run XGBoost. +4. `ib_tab_qa_null_render` is an identity post-processing step before the text + is returned. + +The bundled model is intentionally not described as an LLM. It is a tiny, +queryable deterministic mock that exercises token, embedding, retrieval, +model-output, reducer, and renderer boundaries without a model download. +`IB_TAB_QA_MODEL_COMMAND` can name a later adapter implementing `inspect`, +`embed`, and `answer`. `IB_TAB_QA_REDUCER_COMMAND` can replace the one-member +reducer. The two pinned Hugging Face embedding manifests remain available for +real model views; this mock writes `model-adapter.txt`, never a misleading +`embedding-model.txt` manifest. A model name is bound to one exact adapter +record, including the adapter command's SHA-256, so changed adapter code cannot +silently reuse old vectors. An adapter with external model or vocabulary files +must report their immutable hashes from `inspect` as part of the same record. + +## Filesystem boundaries + +Saved source text stays below `${IB_READING_DIR:-~/reading}`. The rebuildable +index defaults to: + +```text +${XDG_DATA_HOME:-~/.local/share}/ib/views/ + organizing-the-information/vector-spaces/mock-token-i8-v1/reading/ +``` + +Successful questions are appended as one directory per exchange below the +separate requested folder: + +```text +~/questions and answers about tabs that the user has visited/ + -/ + question.txt + model-candidate.txt + reduced-answer.txt + answer.txt + source.tsv + vector-query.tsv + vector-format.txt + reducer.tsv + indexing.tsv + model.tsv + reducer-model.tsv + reading-source.tsv + pipeline.tsv +``` + +The exchange preserves the raw model candidate, reduction evidence, selected +source identity and content hash, upstream reading-source record, exact vector +generation, adapter and reducer command hashes, vector execution report, and +final response separately. It snapshots the selected document for processing +and fails closed if those bytes differ from the indexed corpus record. The +snapshot is temporary: canonical reading text is neither moved nor duplicated +into the Q&A store. The command rejects a Q&A root equal to or nested with the +reading root. + +## Run the mock + +```text +make -C native/vector-index +sh bin/ask_saved_pages.grease --index +sh bin/ask_saved_pages.grease "What do feed-forward networks do to input space?" +``` + +Running the last command without a question displays `question>` and reads one +line from the console. The index is reused until `--index` is run again; corpus +change detection is deliberately left for the next slice. diff --git a/docs/vector-index.md b/docs/vector-index.md index d1eb99b..c26a2cb 100644 --- a/docs/vector-index.md +++ b/docs/vector-index.md @@ -36,6 +36,12 @@ document-id0.8125 `query` memory-maps the Float32 cache. `query-text` performs the same exact scan by parsing the readable file and works with the cache removed. `column` uses the fixed-width layout to print one coordinate across every ID. `compile-cache` atomically recreates the Float32 file from text. A later exact or approximate program can occupy the same process boundary without changing canonical browser state or callers that stream rows and queries. +When `IB_VECTOR_QUERY_REPORT` names a file, either query command also writes a +small execution report containing the compute device, storage path, metric, +dimensions, and number of row dot products. The current portable C scan reports +`dot-product-used-gpu=False`; a future accelerated backend must report its own +execution rather than letting callers infer it. + ## Files One model-specific view lives below the derived view root: diff --git a/lib/tab_qa.grease b/lib/tab_qa.grease new file mode 100644 index 0000000..de9c713 --- /dev/null +++ b/lib/tab_qa.grease @@ -0,0 +1,398 @@ +# Grease source. This is the first console-sized vertical slice of the +# text-and-action workbench over material deliberately saved in ~/reading. + +ib_tab_qa_fail() { + printf 'tab Q&A: %s\n' "$1" >&2 + return 1 +} + +ib_tab_qa_reading_root() { + if test -n "${IB_READING_DIR:-}"; then + printf '%s\n' "$IB_READING_DIR" + else + test -n "${HOME:-}" || return 1 + printf '%s/reading\n' "$HOME" + fi +} + +ib_tab_qa_root() { + if test -n "${IB_TAB_QA_DIR:-}"; then + printf '%s\n' "$IB_TAB_QA_DIR" + else + test -n "${HOME:-}" || return 1 + printf '%s/questions and answers about tabs that the user has visited\n' "$HOME" + fi +} + +ib_tab_qa_views_root() { + if test -n "${IB_VIEWS_DIR:-}"; then + printf '%s\n' "$IB_VIEWS_DIR" + else + test -n "${HOME:-}" || return 1 + data_home=${XDG_DATA_HOME:-$HOME/.local/share} + printf '%s/ib/views\n' "$data_home" + fi +} + +ib_tab_qa_model_command() { + printf '%s\n' "${IB_TAB_QA_MODEL_COMMAND:-$IB_TAB_QA_REPOSITORY_ROOT/bin/mock_tab_model.grease}" +} + +ib_tab_qa_reducer_command() { + printf '%s\n' "${IB_TAB_QA_REDUCER_COMMAND:-$IB_TAB_QA_REPOSITORY_ROOT/bin/mock_tab_qa_reducer.grease}" +} + +ib_tab_qa_vector_program() { + printf '%s\n' "${IB_VECTOR_INDEX_PROGRAM:-$IB_TAB_QA_REPOSITORY_ROOT/native/vector-index/ib-vector-index}" +} + +ib_tab_qa_metadata_value() { + metadata_key=$1 + sed -n "s/^${metadata_key}=//p" | sed -n '1p' +} + +ib_tab_qa_sha256() { + command -v sha256sum >/dev/null 2>&1 || { + ib_tab_qa_fail 'sha256sum is unavailable' + return 1 + } + checksum_line=$(sha256sum "$1") || return 1 + checksum=${checksum_line%% *} + case "$checksum" in + ''|*[!0-9a-f]*) ib_tab_qa_fail "invalid SHA-256 for: $1"; return 1 ;; + esac + test "${#checksum}" -eq 64 || { + ib_tab_qa_fail "invalid SHA-256 for: $1" + return 1 + } + printf '%s\n' "$checksum" +} + +ib_tab_qa_safe_component() { + case "$1" in + ''|.|..|*[!A-Za-z0-9._-]*) return 1 ;; + *) return 0 ;; + esac +} + +ib_tab_qa_safe_relative_document() { + relative_document=$1 + tab_character=$(printf '\t') + case "$relative_document" in + ''|/*|.|..|../*|*/../*|*/..|*"$tab_character"*|*' +'*) return 1 ;; + *) return 0 ;; + esac +} + +ib_tab_qa_separate_roots() { + command -v realpath >/dev/null 2>&1 || return 1 + first_root=$(realpath -m "$1") || return 1 + second_root=$(realpath -m "$2") || return 1 + case "$first_root/" in + "$second_root/"*) return 1 ;; + esac + case "$second_root/" in + "$first_root/"*) return 1 ;; + esac + return 0 +} + +ib_tab_qa_model_metadata() { + model_command=$(ib_tab_qa_model_command) || return 1 + test -x "$model_command" || { + ib_tab_qa_fail "model is unavailable: $model_command" + return 1 + } + model_report=$("$model_command" inspect) || return 1 + model_command_sha256=$(ib_tab_qa_sha256 "$model_command") || return 1 + printf '%s\nadapter-command-sha256=%s\n' "$model_report" "$model_command_sha256" +} + +ib_tab_qa_reducer_metadata() { + reducer_command=$(ib_tab_qa_reducer_command) || return 1 + test -x "$reducer_command" || { + ib_tab_qa_fail "reducer is unavailable: $reducer_command" + return 1 + } + reducer_report=$("$reducer_command" inspect) || return 1 + reducer_command_sha256=$(ib_tab_qa_sha256 "$reducer_command") || return 1 + printf '%s\nreducer-command-sha256=%s\n' "$reducer_report" "$reducer_command_sha256" +} + +ib_tab_qa_model_name() { + model_metadata=$(ib_tab_qa_model_metadata) || return 1 + model_name=$(printf '%s\n' "$model_metadata" | ib_tab_qa_metadata_value model) + ib_tab_qa_safe_component "$model_name" || { + ib_tab_qa_fail "model reported an unsafe name: $model_name" + return 1 + } + printf '%s\n' "$model_name" +} + +ib_tab_qa_index_directory() { + views_root=$(ib_tab_qa_views_root) || return 1 + model_name=$(ib_tab_qa_model_name) || return 1 + printf '%s/organizing-the-information/vector-spaces/%s/reading\n' \ + "$views_root" "$model_name" +} + +ib_tab_qa_build_index() ( + build_work= + cleanup() { + test -z "$build_work" || rm -rf "$build_work" + } + trap cleanup EXIT HUP INT TERM + + reading_root=$(ib_tab_qa_reading_root) || return 1 + views_root=$(ib_tab_qa_views_root) || return 1 + index_directory=$(ib_tab_qa_index_directory) || return 1 + model_command=$(ib_tab_qa_model_command) || return 1 + vector_program=$(ib_tab_qa_vector_program) || return 1 + + test -d "$reading_root" || ib_tab_qa_fail "reading directory is missing: $reading_root" || return 1 + test -x "$model_command" || ib_tab_qa_fail "model is unavailable: $model_command" || return 1 + test -x "$vector_program" || ib_tab_qa_fail "vector backend is unavailable: $vector_program" || return 1 + command -v find >/dev/null 2>&1 || ib_tab_qa_fail 'find is unavailable' || return 1 + command -v sha256sum >/dev/null 2>&1 || ib_tab_qa_fail 'sha256sum is unavailable' || return 1 + + model_metadata=$(ib_tab_qa_model_metadata) || return 1 + model_available=$(printf '%s\n' "$model_metadata" | ib_tab_qa_metadata_value available) + model_name=$(printf '%s\n' "$model_metadata" | ib_tab_qa_metadata_value model) + model_dimensions=$(printf '%s\n' "$model_metadata" | ib_tab_qa_metadata_value dimensions) + test "$model_available" = True || ib_tab_qa_fail "model did not report available: $model_command" || return 1 + ib_tab_qa_safe_component "$model_name" || ib_tab_qa_fail "model reported an unsafe name: $model_name" || return 1 + case "$model_dimensions" in + ''|*[!0-9]*) ib_tab_qa_fail 'model dimensions are not a positive integer' || return 1 ;; + esac + test "$model_dimensions" -gt 0 || ib_tab_qa_fail 'model dimensions must be positive' || return 1 + + sh "$IB_TAB_QA_REPOSITORY_ROOT/bin/ib_views.grease" init "$views_root" + model_view_directory=$(dirname "$index_directory") + test ! -L "$model_view_directory" || ib_tab_qa_fail "model view is a symbolic link: $model_view_directory" || return 1 + mkdir -p "$model_view_directory" + adapter_record="$model_view_directory/model-adapter.txt" + if test -e "$adapter_record" || test -L "$adapter_record"; then + test -f "$adapter_record" && test ! -L "$adapter_record" || \ + ib_tab_qa_fail "model adapter record is not a regular file: $adapter_record" || return 1 + recorded_model_metadata=$(cat "$adapter_record") + test "$recorded_model_metadata" = "$model_metadata" || \ + ib_tab_qa_fail "model name is already bound to different adapter facts: $model_name" || return 1 + else + adapter_temporary=$(mktemp "$model_view_directory/.model-adapter.XXXXXX") || return 1 + printf '%s\n' "$model_metadata" > "$adapter_temporary" + mv -T "$adapter_temporary" "$adapter_record" + fi + + build_work=$(mktemp -d) || return 1 + document_paths="$build_work/document-paths.txt" + vector_rows="$build_work/vector-rows.tsv" + corpus_rows="$build_work/corpus.tsv" + build_report="$build_work/build-report.txt" + : > "$vector_rows" + : > "$corpus_rows" + find "$reading_root" -type f -name document.txt -print | LC_ALL=C sort > "$document_paths" + + document_count=0 + while IFS= read -r document_path; do + test -n "$document_path" || continue + document_id=${document_path#"${reading_root%/}"/} + ib_tab_qa_safe_relative_document "$document_id" || { + ib_tab_qa_fail "unsafe reading document identity: $document_id" + return 1 + } + test -s "$document_path" || ib_tab_qa_fail "reading document is empty: $document_id" || return 1 + document_vector=$("$model_command" embed < "$document_path") || return 1 + vector_dimensions=$(printf '%s\n' "$document_vector" | awk 'NR == 1 { print NF }') + test "$vector_dimensions" = "$model_dimensions" || { + ib_tab_qa_fail "model emitted $vector_dimensions dimensions, expected $model_dimensions" + return 1 + } + document_sha256=$(ib_tab_qa_sha256 "$document_path") || return 1 + document_bytes=$(wc -c < "$document_path" | tr -d '[:space:]') + printf '%s\t%s\n' "$document_id" "$document_vector" >> "$vector_rows" + printf '%s\t%s\t%s\n' "$document_id" "$document_sha256" "$document_bytes" >> "$corpus_rows" + document_count=$((document_count + 1)) + done < "$document_paths" + + test "$document_count" -gt 0 || ib_tab_qa_fail "no document.txt files exist below $reading_root" || return 1 + "$vector_program" build "$index_directory" "$model_dimensions" cosine \ + < "$vector_rows" > "$build_report" + "$vector_program" check "$index_directory" >/dev/null + + corpus_temporary=$(mktemp "$index_directory/.corpus.XXXXXX") || return 1 + cp "$corpus_rows" "$corpus_temporary" + mv -T "$corpus_temporary" "$index_directory/corpus.tsv" + indexing_temporary=$(mktemp "$index_directory/.indexing.XXXXXX") || return 1 + printf '%s\n' \ + 'index-count=1' \ + 'index-1=flat-f32-exact/cosine' \ + 'query-path-count=2' \ + 'query-path-1=readable-fixed-width-text' \ + 'query-path-2=rebuildable-float32-cache' \ + 'collection=reading' \ + "model=$model_name" \ + "dimensions=$model_dimensions" \ + "document-count=$document_count" > "$indexing_temporary" + mv -T "$indexing_temporary" "$index_directory/indexing.tsv" + + sed -n '1,20p' "$build_report" + sed -n '1,20p' "$index_directory/indexing.tsv" +) + +ib_tab_qa_ensure_index() { + index_directory=$(ib_tab_qa_index_directory) || return 1 + adapter_record=$(dirname "$index_directory")/model-adapter.txt + current_model_metadata=$(ib_tab_qa_model_metadata) || return 1 + if test -s "$index_directory/format.txt" && + test -s "$index_directory/corpus.tsv" && + test -s "$index_directory/indexing.tsv" && + test -f "$adapter_record" && test ! -L "$adapter_record" && + test "$(cat "$adapter_record")" = "$current_model_metadata"; then + return 0 + fi + ib_tab_qa_build_index >/dev/null +} + +ib_tab_qa_null_render() { + # This is a mock standing in for any further post-processing we want to do + # before returning the text to the user. Its current behavior is identity. + cat +} + +ib_tab_qa_inspect() { + reading_root=$(ib_tab_qa_reading_root) || return 1 + qa_root=$(ib_tab_qa_root) || return 1 + views_root=$(ib_tab_qa_views_root) || return 1 + index_directory=$(ib_tab_qa_index_directory) || return 1 + pinned_manifest_count=$(find "$IB_TAB_QA_REPOSITORY_ROOT/models/embedding" -type f -name '*.model' | wc -l | tr -d '[:space:]') + printf '%s\n' \ + 'frontend=console-text-stub' \ + "reading-root=$reading_root" \ + "qa-root=$qa_root" \ + "views-root=$views_root" \ + "index-directory=$index_directory" \ + "pinned-embedding-manifest-count=$pinned_manifest_count" + ib_tab_qa_model_metadata + ib_tab_qa_reducer_metadata + if test -s "$index_directory/format.txt"; then + printf 'index-available=True\n' + sed -n '/^index-count=/p; /^query-path-count=/p' "$index_directory/indexing.tsv" + else + printf 'index-available=False\n' + fi +} + +ib_tab_qa_answer() ( + answer_work= + exchange_partial= + cleanup() { + test -z "$answer_work" || rm -rf "$answer_work" + test -z "$exchange_partial" || rm -rf "$exchange_partial" + } + trap cleanup EXIT HUP INT TERM + + question=$1 + test -n "$question" || ib_tab_qa_fail 'question is empty' || return 1 + + reading_root=$(ib_tab_qa_reading_root) || return 1 + qa_root=$(ib_tab_qa_root) || return 1 + ib_tab_qa_separate_roots "$reading_root" "$qa_root" || { + ib_tab_qa_fail 'question-and-answer storage must be separate from the reading corpus' + return 1 + } + ib_tab_qa_ensure_index || return 1 + index_directory=$(ib_tab_qa_index_directory) || return 1 + model_command=$(ib_tab_qa_model_command) || return 1 + reducer_command=$(ib_tab_qa_reducer_command) || return 1 + vector_program=$(ib_tab_qa_vector_program) || return 1 + test -x "$reducer_command" || ib_tab_qa_fail "reducer is unavailable: $reducer_command" || return 1 + + answer_work=$(mktemp -d) || return 1 + printf '%s\n' "$question" | "$model_command" embed > "$answer_work/question-vector.txt" + IB_VECTOR_QUERY_REPORT="$answer_work/vector-query.tsv" \ + "$vector_program" query "$index_directory" 1 \ + < "$answer_work/question-vector.txt" > "$answer_work/results.tsv" + + result_line=$(sed -n '1p' "$answer_work/results.tsv") + test -n "$result_line" || ib_tab_qa_fail 'vector query returned no saved page' || return 1 + source_id=$(printf '%s\n' "$result_line" | awk -F '\t' '{ print $1 }') + retrieval_score=$(printf '%s\n' "$result_line" | awk -F '\t' '{ print $2 }') + ib_tab_qa_safe_relative_document "$source_id" || ib_tab_qa_fail "vector index returned unsafe identity: $source_id" || return 1 + source_document="${reading_root%/}/$source_id" + test -s "$source_document" || ib_tab_qa_fail "retrieved source is unavailable: $source_id" || return 1 + + corpus_record=$(awk -F '\t' -v wanted="$source_id" ' + $1 == wanted { record = $0; matches++ } + END { if (matches == 1) print record; else exit 1 } + ' "$index_directory/corpus.tsv") || { + ib_tab_qa_fail "retrieved source has no unique index provenance: $source_id" + return 1 + } + indexed_source_sha256=$(printf '%s\n' "$corpus_record" | awk -F '\t' '{ print $2 }') + indexed_source_bytes=$(printf '%s\n' "$corpus_record" | awk -F '\t' '{ print $3 }') + cp "$source_document" "$answer_work/source-document.txt" + answer_source_sha256=$(ib_tab_qa_sha256 "$answer_work/source-document.txt") || return 1 + answer_source_bytes=$(wc -c < "$answer_work/source-document.txt" | tr -d '[:space:]') + if test "$answer_source_sha256" != "$indexed_source_sha256" || + test "$answer_source_bytes" != "$indexed_source_bytes"; then + ib_tab_qa_fail "reading source changed since index build: $source_id (run --index)" + return 1 + fi + + reading_source_record=${source_document%/*}/source.tsv + reading_source_present=False + reading_source_sha256= + if test -f "$reading_source_record" && test ! -L "$reading_source_record"; then + cp "$reading_source_record" "$answer_work/reading-source.tsv" + reading_source_sha256=$(ib_tab_qa_sha256 "$answer_work/reading-source.tsv") || return 1 + reading_source_present=True + fi + + "$model_command" answer "$question" "$answer_work/source-document.txt" > "$answer_work/model-candidate.txt" + IB_TAB_QA_REDUCER_REPORT="$answer_work/reducer.tsv" \ + "$reducer_command" reduce "$retrieval_score" "$answer_work/source-document.txt" \ + < "$answer_work/model-candidate.txt" > "$answer_work/reduced-answer.txt" + ib_tab_qa_null_render < "$answer_work/reduced-answer.txt" > "$answer_work/answer.txt" + test -s "$answer_work/answer.txt" || ib_tab_qa_fail 'post-processing returned an empty response' || return 1 + + mkdir -p "$qa_root" + exchange_partial=$(mktemp -d "$qa_root/.partial.XXXXXX") || return 1 + printf '%s\n' "$question" > "$exchange_partial/question.txt" + cp "$answer_work/model-candidate.txt" "$exchange_partial/model-candidate.txt" + cp "$answer_work/reduced-answer.txt" "$exchange_partial/reduced-answer.txt" + cp "$answer_work/answer.txt" "$exchange_partial/answer.txt" + cp "$answer_work/vector-query.tsv" "$exchange_partial/vector-query.tsv" + cp "$answer_work/reducer.tsv" "$exchange_partial/reducer.tsv" + cp "$index_directory/indexing.tsv" "$exchange_partial/indexing.tsv" + cp "$index_directory/format.txt" "$exchange_partial/vector-format.txt" + ib_tab_qa_model_metadata > "$exchange_partial/model.tsv" + ib_tab_qa_reducer_metadata > "$exchange_partial/reducer-model.tsv" + if test "$reading_source_present" = True; then + cp "$answer_work/reading-source.tsv" "$exchange_partial/reading-source.tsv" + fi + { + printf 'document-id\t%s\n' "$source_id" + printf 'retrieval-score\t%s\n' "$retrieval_score" + printf 'document-sha256\t%s\n' "$answer_source_sha256" + printf 'document-bytes\t%s\n' "$answer_source_bytes" + printf 'reading-source-present\t%s\n' "$reading_source_present" + printf 'reading-source-sha256\t%s\n' "$reading_source_sha256" + } > "$exchange_partial/source.tsv" + printf '%s\n' \ + '1=vector-retrieval' \ + '2=model-answer' \ + '3=mock-ensemble-reducer' \ + '4=null-identity-renderer' > "$exchange_partial/pipeline.tsv" + + exchange_base=$(date -u '+%Y%m%dT%H%M%SZ')-$$ + exchange_directory="$qa_root/$exchange_base" + exchange_suffix=0 + while test -e "$exchange_directory"; do + exchange_suffix=$((exchange_suffix + 1)) + exchange_directory="$qa_root/$exchange_base-$exchange_suffix" + done + mv "$exchange_partial" "$exchange_directory" + exchange_partial= + cat "$answer_work/answer.txt" +) diff --git a/native/vector-index/ib_vector_index.c b/native/vector-index/ib_vector_index.c index 3a1c43c..413819e 100644 --- a/native/vector-index/ib_vector_index.c +++ b/native/vector-index/ib_vector_index.c @@ -834,6 +834,35 @@ static int dot_product_text(const unsigned char *stored, const float *query, return 1; } +static int write_query_report(const char *path, const struct manifest *manifest, + int use_text, size_t dot_products) { + if (path == NULL || path[0] == '\0') { + return 0; + } + FILE *file = fopen(path, "wb"); + if (file == NULL) { + return fail_path("cannot open query report", path); + } + int bad = fprintf(file, + "operation=exact-dot-product-scan\n" + "compute=cpu\n" + "dot-product-used-gpu=False\n" + "storage=%s\n" + "metric=%s\n" + "dimensions=%zu\n" + "dot-products=%zu\n", + use_text ? "readable-text" : "float32-cache", + metric_text(manifest->metric), manifest->dimensions, + dot_products) < 0; + if (!bad && flush_file(file, path) != 0) { + bad = 1; + } + if (fclose(file) != 0) { + bad = 1; + } + return bad ? fail("cannot write query report") : 0; +} + static int match_before(float score, size_t row, const struct match *other) { return score > other->score || (score == other->score && row < other->row); } @@ -876,6 +905,7 @@ static int query_index(const char *directory, const char *limit_text, int data_fd = -1; unsigned char *mapped = MAP_FAILED; int status = 1; + const char *query_report_path = getenv("IB_VECTOR_QUERY_REPORT"); if (!parse_size(limit_text, &limit)) { return fail("result count must be a positive integer"); @@ -913,7 +943,7 @@ static int query_index(const char *directory, const char *limit_text, } if (limit == 0) { - status = 0; + status = write_query_report(query_report_path, &manifest, use_text, 0); goto cleanup; } matches = calloc(limit, sizeof(*matches)); @@ -980,7 +1010,8 @@ static int query_index(const char *directory, const char *limit_text, for (size_t i = 0; i < used; ++i) { printf("%s\t%.9g\n", matches[i].id, (double)matches[i].score); } - status = 0; + status = write_query_report(query_report_path, &manifest, use_text, + manifest.count); cleanup: if (mapped != MAP_FAILED) { diff --git a/tests/fixtures/reading/arxiv/1901.09021/document.txt b/tests/fixtures/reading/arxiv/1901.09021/document.txt new file mode 100644 index 0000000..e449eb0 --- /dev/null +++ b/tests/fixtures/reading/arxiv/1901.09021/document.txt @@ -0,0 +1,3 @@ +Tiny color blocks +Figure 1 shows tiny red, green, blue, and white color blocks. +The figure is embedded in the PDF fixture used by the scientific reading path. diff --git a/tests/fixtures/reading/arxiv/1901.09021/source.tsv b/tests/fixtures/reading/arxiv/1901.09021/source.tsv new file mode 100644 index 0000000..725a695 --- /dev/null +++ b/tests/fixtures/reading/arxiv/1901.09021/source.tsv @@ -0,0 +1 @@ +pdf 1901.09021 fixture://arxiv_site/pdf/1901.09021 diff --git a/tests/fixtures/reading/arxiv/2203.11355/document.txt b/tests/fixtures/reading/arxiv/2203.11355/document.txt new file mode 100644 index 0000000..54fb221 --- /dev/null +++ b/tests/fixtures/reading/arxiv/2203.11355/document.txt @@ -0,0 +1,5 @@ +Origami in N dimensions: How feed-forward networks manufacture linear separability +Feed-forward networks fold input space into representations whose classes can be separated. +The paper studies the geometry of the folding process. +Figure 1. Linear regions along a one dimensional input. +Activation space folding. diff --git a/tests/fixtures/reading/arxiv/2203.11355/source.tsv b/tests/fixtures/reading/arxiv/2203.11355/source.tsv new file mode 100644 index 0000000..4e2175c --- /dev/null +++ b/tests/fixtures/reading/arxiv/2203.11355/source.tsv @@ -0,0 +1 @@ +html 2203.11355 fixture://arxiv_site/html/2203.11355 diff --git a/tests/test_mock_tab_model.grease b/tests/test_mock_tab_model.grease new file mode 100755 index 0000000..0992f1f --- /dev/null +++ b/tests/test_mock_tab_model.grease @@ -0,0 +1,34 @@ +#!/bin/sh +set -eu + +repository_root=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +model="$repository_root/bin/mock_tab_model.grease" +document="$repository_root/tests/fixtures/reading/arxiv/2203.11355/document.txt" + +metadata=$("$model" inspect) +printf '%s\n' "$metadata" | grep -Fx 'model=mock-token-i8-v1' +printf '%s\n' "$metadata" | grep -Fx 'available=True' +printf '%s\n' "$metadata" | grep -Fx 'llm=False' +printf '%s\n' "$metadata" | grep -Fx 'quantization=int8-counts' +printf '%s\n' "$metadata" | grep -Fx 'dimensions=32' + +vector=$(printf '%s\n' 'feed-forward networks fold input space' | "$model" embed) +printf '%s\n' "$vector" | awk ' + NF != 32 { exit 1 } + { + nonzero = 0 + for (field = 1; field <= NF; field++) { + if ($field !~ /^[0-9]+$/ || $field < 0 || $field > 127) + exit 1 + if ($field > 0) + nonzero = 1 + } + exit !nonzero + } +' + +answer=$("$model" answer 'What do feed-forward networks do to input space?' "$document") +printf '%s\n' "$answer" | grep -Fx \ + 'Feed-forward networks fold input space into representations whose classes can be separated.' + +printf 'mock tab model tests: ok\n' diff --git a/tests/test_tab_qa_pipeline.grease b/tests/test_tab_qa_pipeline.grease new file mode 100755 index 0000000..238bc26 --- /dev/null +++ b/tests/test_tab_qa_pipeline.grease @@ -0,0 +1,171 @@ +#!/bin/sh +set -eu + +repository_root=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +work_directory=$(mktemp -d) +cleanup() { + rm -rf "$work_directory" +} +trap cleanup EXIT HUP INT TERM + +export HOME="$work_directory/home" +export IB_READING_DIR="$work_directory/reading" +export IB_TAB_QA_DIR="$work_directory/questions and answers about tabs that the user has visited" +export IB_VIEWS_DIR="$work_directory/views" +export IB_VECTOR_INDEX_PROGRAM="$repository_root/native/vector-index/ib-vector-index" +index_directory="$IB_VIEWS_DIR/organizing-the-information/vector-spaces/mock-token-i8-v1/reading" +model_view_directory=$(dirname "$index_directory") +mkdir -p "$HOME" "$IB_READING_DIR" +cp -R "$repository_root/tests/fixtures/reading/." "$IB_READING_DIR/" + +make -C "$repository_root/native/vector-index" >/dev/null + +document_count=$(find "$IB_READING_DIR" -type f -name document.txt | wc -l | tr -d '[:space:]') +test "$document_count" = 2 +find "$IB_READING_DIR" -type f -name document.txt -exec test -s {} \; +grep -F 'Origami in N dimensions' \ + "$repository_root/tests/fixtures/arxiv_site/html/2203.11355" >/dev/null +grep -aF 'Figure 1. Tiny color blocks' \ + "$repository_root/tests/fixtures/arxiv_site/pdf/1901.09021" >/dev/null +grep -F 'Feed-forward networks fold input space' \ + "$IB_READING_DIR/arxiv/2203.11355/document.txt" >/dev/null +grep -F 'tiny red, green, blue, and white color blocks' \ + "$IB_READING_DIR/arxiv/1901.09021/document.txt" >/dev/null +awk -F '\t' 'NR == 1 && $1 == "html" && $3 == "fixture://arxiv_site/html/2203.11355" { found = 1 } END { exit !found }' \ + "$IB_READING_DIR/arxiv/2203.11355/source.tsv" +awk -F '\t' 'NR == 1 && $1 == "pdf" && $3 == "fixture://arxiv_site/pdf/1901.09021" { found = 1 } END { exit !found }' \ + "$IB_READING_DIR/arxiv/1901.09021/source.tsv" + +inspect_before=$(sh "$repository_root/bin/ask_saved_pages.grease" --inspect) +printf '%s\n' "$inspect_before" | grep -Fx 'frontend=console-text-stub' +pinned_manifest_count=$(printf '%s\n' "$inspect_before" | sed -n 's/^pinned-embedding-manifest-count=//p') +test "$pinned_manifest_count" -ge 1 +printf '%s\n' "$inspect_before" | grep -Fx 'available=True' +printf '%s\n' "$inspect_before" | grep -Fx 'llm=False' +printf '%s\n' "$inspect_before" | grep -Fx 'ensemble=single-member-test-double' +printf '%s\n' "$inspect_before" | grep -Fx 'index-available=False' + +sh "$repository_root/bin/ask_saved_pages.grease" --index > "$work_directory/index-build.txt" +grep -Fx 'build=ok' "$work_directory/index-build.txt" +grep -Fx 'count=2' "$work_directory/index-build.txt" +grep -Fx 'index-count=1' "$work_directory/index-build.txt" +grep -Fx 'query-path-count=2' "$work_directory/index-build.txt" +grep -Fx 'query-path-1=readable-fixed-width-text' "$work_directory/index-build.txt" +grep -Fx 'query-path-2=rebuildable-float32-cache' "$work_directory/index-build.txt" +grep -Fx 'document-count=2' "$work_directory/index-build.txt" +test -s "$index_directory/format.txt" +test -s "$index_directory/corpus.tsv" +test -s "$model_view_directory/model-adapter.txt" +grep -E '^adapter-command-sha256=[0-9a-f]{64}$' "$model_view_directory/model-adapter.txt" >/dev/null +test ! -e "$model_view_directory/embedding-model.txt" +test "$(wc -l < "$index_directory/corpus.tsv" | tr -d '[:space:]')" = 2 +vectors_text=$(sed -n 's/^vectors_text //p' "$index_directory/format.txt") +vectors_cache=$(sed -n 's/^vectors_cache //p' "$index_directory/format.txt") +test -s "$index_directory/$vectors_text" +test -s "$index_directory/$vectors_cache" +"$IB_VECTOR_INDEX_PROGRAM" check "$index_directory" >/dev/null +inspect_after=$(sh "$repository_root/bin/ask_saved_pages.grease" --inspect) +printf '%s\n' "$inspect_after" | grep -Fx 'index-available=True' +printf '%s\n' "$inspect_after" | grep -Fx 'index-count=1' +printf '%s\n' "$inspect_after" | grep -Fx 'query-path-count=2' + +changed_model="$work_directory/changed-model.grease" +cp "$repository_root/bin/mock_tab_model.grease" "$changed_model" +printf '%s\n' '# behavior-bearing adapter bytes changed' >> "$changed_model" +chmod +x "$changed_model" +if IB_TAB_QA_MODEL_COMMAND="$changed_model" \ + sh "$repository_root/bin/ask_saved_pages.grease" --index \ + > "$work_directory/changed-model.txt" 2>&1; then + printf 'changed adapter code silently reused the model name\n' >&2 + exit 1 +fi +grep -F 'model name is already bound to different adapter facts' \ + "$work_directory/changed-model.txt" >/dev/null + +question='What do feed-forward networks do to input space?' +printf '%s\n' "$question" | "$repository_root/bin/mock_tab_model.grease" embed \ + > "$work_directory/question-vector.txt" +IB_VECTOR_QUERY_REPORT="$work_directory/cached-query.tsv" \ + "$IB_VECTOR_INDEX_PROGRAM" query "$index_directory" 2 \ + < "$work_directory/question-vector.txt" > "$work_directory/cached-results.tsv" +IB_VECTOR_QUERY_REPORT="$work_directory/readable-query.tsv" \ + "$IB_VECTOR_INDEX_PROGRAM" query-text "$index_directory" 2 \ + < "$work_directory/question-vector.txt" > "$work_directory/readable-results.tsv" +cmp "$work_directory/cached-results.tsv" "$work_directory/readable-results.tsv" +grep -Fx 'dot-product-used-gpu=False' "$work_directory/cached-query.tsv" +grep -Fx 'compute=cpu' "$work_directory/cached-query.tsv" +grep -Fx 'storage=float32-cache' "$work_directory/cached-query.tsv" +grep -Fx 'dot-products=2' "$work_directory/cached-query.tsv" +grep -Fx 'dot-product-used-gpu=False' "$work_directory/readable-query.tsv" +grep -Fx 'storage=readable-text' "$work_directory/readable-query.tsv" + +response=$(printf '%s\n' "$question" | \ + sh "$repository_root/bin/ask_saved_pages.grease" 2> "$work_directory/prompt.txt") +grep -Fx 'question> ' "$work_directory/prompt.txt" +test -n "$response" +printf '%s\n' "$response" | grep -Fx \ + 'Feed-forward networks fold input space into representations whose classes can be separated.' + +test "$IB_TAB_QA_DIR" != "$IB_READING_DIR" +exchange_count=$(find "$IB_TAB_QA_DIR" -mindepth 1 -maxdepth 1 -type d ! -name '.partial.*' | wc -l | tr -d '[:space:]') +test "$exchange_count" = 1 +exchange_directory=$(find "$IB_TAB_QA_DIR" -mindepth 1 -maxdepth 1 -type d ! -name '.partial.*' | sed -n '1p') +test -s "$exchange_directory/question.txt" +test -s "$exchange_directory/answer.txt" +test -s "$exchange_directory/model-candidate.txt" +test -s "$exchange_directory/reduced-answer.txt" +test -s "$exchange_directory/source.tsv" +test -s "$exchange_directory/vector-format.txt" +test -s "$exchange_directory/reading-source.tsv" +grep -Fx 'dot-product-used-gpu=False' "$exchange_directory/vector-query.tsv" +grep -Fx 'compute=cpu' "$exchange_directory/vector-query.tsv" +grep -Fx 'storage=float32-cache' "$exchange_directory/vector-query.tsv" +grep -Fx 'dot-products=2' "$exchange_directory/vector-query.tsv" +grep -Fx 'reducer=mock-single-member-weighted-reducer-v1' "$exchange_directory/reducer.tsv" +grep -Fx 'ensemble-stage-ran=True' "$exchange_directory/reducer.tsv" +grep -Fx 'member-count=1' "$exchange_directory/reducer.tsv" +grep -Fx 'checks-run=3' "$exchange_directory/reducer.tsv" +grep -Fx 'candidate-nonempty=True' "$exchange_directory/reducer.tsv" +grep -E '^aggregate-score=[-+0-9.eE]+$' "$exchange_directory/reducer.tsv" >/dev/null +awk -F '\t' '$1 == "document-id" && $2 == "arxiv/2203.11355/document.txt" { found = 1 } END { exit !found }' \ + "$exchange_directory/source.tsv" +awk -F '\t' '$1 == "document-sha256" && $2 ~ /^[0-9a-f]{64}$/ { found = 1 } END { exit !found }' \ + "$exchange_directory/source.tsv" +awk -F '\t' '$1 == "reading-source-present" && $2 == "True" { found = 1 } END { exit !found }' \ + "$exchange_directory/source.tsv" +awk -F '\t' '$1 == "reading-source-sha256" && $2 ~ /^[0-9a-f]{64}$/ { found = 1 } END { exit !found }' \ + "$exchange_directory/source.tsv" +cmp "$exchange_directory/reading-source.tsv" "$IB_READING_DIR/arxiv/2203.11355/source.tsv" +grep -E '^adapter-command-sha256=[0-9a-f]{64}$' "$exchange_directory/model.tsv" >/dev/null +grep -E '^reducer-command-sha256=[0-9a-f]{64}$' "$exchange_directory/reducer-model.tsv" >/dev/null +cmp "$exchange_directory/reduced-answer.txt" "$exchange_directory/answer.txt" +grep -Fx '4=null-identity-renderer' "$exchange_directory/pipeline.tsv" +grep -Fx 'index-count=1' "$exchange_directory/indexing.tsv" +test "$(find "$IB_READING_DIR" -type f -name document.txt | wc -l | tr -d '[:space:]')" = 2 + +printf '%s\n' 'changed after indexing' >> "$IB_READING_DIR/arxiv/2203.11355/document.txt" +if sh "$repository_root/bin/ask_saved_pages.grease" "$question" \ + > "$work_directory/stale-source.txt" 2>&1; then + printf 'changed reading source was unexpectedly accepted\n' >&2 + exit 1 +fi +grep -F 'reading source changed since index build' "$work_directory/stale-source.txt" >/dev/null +test "$(find "$IB_TAB_QA_DIR" -mindepth 1 -maxdepth 1 -type d ! -name '.partial.*' | wc -l | tr -d '[:space:]')" = 1 + +if IB_TAB_QA_DIR="$IB_READING_DIR/qa" \ + sh "$repository_root/bin/ask_saved_pages.grease" "$question" \ + > "$work_directory/nested-root.txt" 2>&1; then + printf 'nested Q&A root was unexpectedly accepted\n' >&2 + exit 1 +fi +grep -F 'question-and-answer storage must be separate' "$work_directory/nested-root.txt" >/dev/null + +if IB_TAB_QA_DIR="$IB_READING_DIR/../reading/qa" \ + sh "$repository_root/bin/ask_saved_pages.grease" "$question" \ + > "$work_directory/normalized-nested-root.txt" 2>&1; then + printf 'normalized nested Q&A root was unexpectedly accepted\n' >&2 + exit 1 +fi +grep -F 'question-and-answer storage must be separate' "$work_directory/normalized-nested-root.txt" >/dev/null + +printf 'tab Q&A pipeline tests: ok\n' diff --git a/tests/vector-index-smoke.sh b/tests/vector-index-smoke.sh index dad527c..c01a17c 100644 --- a/tests/vector-index-smoke.sh +++ b/tests/vector-index-smoke.sh @@ -24,13 +24,23 @@ grep -Fx 'check=ok' "$temporary/check.txt" grep -Fx 'text_slot_bytes=16' "$temporary/check.txt" printf '%s\n' '0.9 0.1 0' | + IB_VECTOR_QUERY_REPORT="$temporary/cache-query-report.txt" \ "$program" query "$index" 2 > "$temporary/results.txt" sed -n '1s/ .*//p' "$temporary/results.txt" | grep -Fx 'book-page' sed -n '2s/ .*//p' "$temporary/results.txt" | grep -Fx 'mixed-page' +grep -Fx 'operation=exact-dot-product-scan' "$temporary/cache-query-report.txt" +grep -Fx 'compute=cpu' "$temporary/cache-query-report.txt" +grep -Fx 'dot-product-used-gpu=False' "$temporary/cache-query-report.txt" +grep -Fx 'storage=float32-cache' "$temporary/cache-query-report.txt" +grep -Fx 'dimensions=3' "$temporary/cache-query-report.txt" +grep -Fx 'dot-products=3' "$temporary/cache-query-report.txt" printf '%s\n' '0.9 0.1 0' | + IB_VECTOR_QUERY_REPORT="$temporary/text-query-report.txt" \ "$program" query-text "$index" 2 > "$temporary/text-results.txt" cmp "$temporary/results.txt" "$temporary/text-results.txt" +grep -Fx 'dot-product-used-gpu=False' "$temporary/text-query-report.txt" +grep -Fx 'storage=readable-text' "$temporary/text-query-report.txt" "$program" column "$index" 1 > "$temporary/first-coordinate.txt" grep -Fx 'book-page +1.00000000e+00' "$temporary/first-coordinate.txt"