LLM: Played with a reduced-scope non-vector LLM query system

Added Id to crowdin config for compatibility with upcoming change to
crowdin CLI process after switch to codeberg

LICENSE

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2015-2025, Dan Brown and the BookStack project contributors.
+Copyright (c) 2015-2026, Dan Brown and the BookStack project contributors.
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

app/Config/services.php

@@ -22,6 +22,18 @@ return [
     // Callback URL for social authentication methods
     'callback_url' => env('APP_URL', false),
 
+    // LLM Service
+    // Options: openai
+    'llm' => env('LLM_SERVICE', ''),
+
+    // OpenAI API-compatible service details
+    'openai' => [
+        'endpoint' => env('OPENAI_ENDPOINT', 'https://api.openai.com'),
+        'key' => env('OPENAI_KEY', ''),
+        'embedding_model' => env('OPENAI_EMBEDDING_MODEL', 'text-embedding-3-small'),
+        'query_model' => env('OPENAI_QUERY_MODEL', 'gpt-4o'),
+    ],
+
     'github'   => [
         'client_id'     => env('GITHUB_APP_ID', false),
         'client_secret' => env('GITHUB_APP_SECRET', false),

app/Console/Commands/RegenerateVectorsCommand.php

@@ -0,0 +1,46 @@
+<?php
+
+namespace BookStack\Console\Commands;
+
+use BookStack\Entities\EntityProvider;
+use BookStack\Entities\Models\Entity;
+use BookStack\Search\Queries\SearchVector;
+use BookStack\Search\Queries\StoreEntityVectorsJob;
+use Illuminate\Console\Command;
+
+class RegenerateVectorsCommand extends Command
+{
+    /**
+     * The name and signature of the console command.
+     *
+     * @var string
+     */
+    protected $signature = 'bookstack:regenerate-vectors';
+
+    /**
+     * The console command description.
+     *
+     * @var string
+     */
+    protected $description = 'Re-index vectors for all content in the system';
+
+    /**
+     * Execute the console command.
+     */
+    public function handle(EntityProvider $entityProvider)
+    {
+        // TODO - Add confirmation before run regarding deletion/time/effort/api-cost etc...
+        SearchVector::query()->delete();
+
+        $types = $entityProvider->all();
+        foreach ($types as $type => $typeInstance) {
+            $this->info("Creating jobs to store vectors for {$type} data...");
+            /** @var Entity[] $entities  */
+            $typeInstance->newQuery()->chunkById(100, function ($entities) {
+                foreach ($entities as $entity) {
+                    dispatch(new StoreEntityVectorsJob($entity));
+                }
+            });
+        }
+    }
+}

app/Search/Queries/EntityVectorGenerator.php

@@ -0,0 +1,89 @@
+<?php
+
+declare(strict_types=1);
+
+namespace BookStack\Search\Queries;
+
+use BookStack\Activity\Models\Tag;
+use BookStack\Entities\Models\Entity;
+use BookStack\Search\Queries\Services\LlmQueryService;
+use Illuminate\Support\Facades\DB;
+
+class EntityVectorGenerator
+{
+    public function __construct(
+        protected LlmQueryServiceProvider $vectorQueryServiceProvider
+    ) {
+    }
+
+    public function generateAndStore(Entity $entity): void
+    {
+        $vectorService = $this->vectorQueryServiceProvider->get();
+
+        $text = $this->entityToPlainText($entity);
+        $chunks = $this->chunkText($text);
+        $embeddings = $this->chunksToEmbeddings($chunks, $vectorService);
+
+        $this->deleteExistingEmbeddingsForEntity($entity);
+        $this->storeEmbeddings($embeddings, $chunks, $entity);
+    }
+
+    protected function deleteExistingEmbeddingsForEntity(Entity $entity): void
+    {
+        SearchVector::query()
+            ->where('entity_type', '=', $entity->getMorphClass())
+            ->where('entity_id', '=', $entity->id)
+            ->delete();
+    }
+
+    protected function storeEmbeddings(array $embeddings, array $textChunks, Entity $entity): void
+    {
+        $toInsert = [];
+
+        foreach ($embeddings as $index => $embedding) {
+            $text = $textChunks[$index];
+            $toInsert[] = [
+                'entity_id' => $entity->id,
+                'entity_type' => $entity->getMorphClass(),
+                'embedding' => DB::raw('VEC_FROMTEXT("[' . implode(',', $embedding) . ']")'),
+                'text' => $text,
+            ];
+        }
+
+        $chunks = array_chunk($toInsert, 500);
+        foreach ($chunks as $chunk) {
+            SearchVector::query()->insert($chunk);
+        }
+    }
+
+    /**
+     * @param string[] $chunks
+     * @return float[] array
+     */
+    protected function chunksToEmbeddings(array $chunks, LlmQueryService $vectorQueryService): array
+    {
+        $embeddings = [];
+        foreach ($chunks as $index => $chunk) {
+            $embeddings[$index] = $vectorQueryService->generateEmbeddings($chunk);
+        }
+        return $embeddings;
+    }
+
+    /**
+     * @return string[]
+     */
+    protected function chunkText(string $text): array
+    {
+        return (new TextChunker(500, ["\n", '.', ' ', '']))->chunk($text);
+    }
+
+    protected function entityToPlainText(Entity $entity): string
+    {
+        $tags = $entity->tags()->get();
+        $tagText = $tags->map(function (Tag $tag) {
+            return $tag->name . ': ' . $tag->value;
+        })->join('\n');
+
+        return $entity->name . "\n{$tagText}\n" . $entity->{$entity->textField};
+    }
+}

app/Search/Queries/LlmQueryRunner.php

@@ -0,0 +1,40 @@
+<?php
+
+namespace BookStack\Search\Queries;
+
+use BookStack\Entities\Models\Entity;
+use BookStack\Search\SearchRunner;
+use Exception;
+
+class LlmQueryRunner
+{
+    public function __construct(
+        protected LlmQueryServiceProvider $vectorQueryServiceProvider,
+        protected SearchRunner $searchRunner,
+    ) {
+    }
+
+    /**
+     * Transform the given query into an array of terms which can be used
+     * to search for documents to help answer that query.
+     * @return string[]
+     * @throws Exception
+     */
+    public function queryToSearchTerms(string $query): array
+    {
+        $queryService = $this->vectorQueryServiceProvider->get();
+
+        return $queryService->queryToSearchTerms($query);
+    }
+
+    /**
+     * Run a query against the configured LLM to produce a text response.
+     * @param Entity[] $searchResults
+     * @throws Exception
+     */
+    public function run(string $query, array $searchResults): string
+    {
+        $queryService = $this->vectorQueryServiceProvider->get();
+        return $queryService->query($query, $searchResults);
+    }
+}

app/Search/Queries/LlmQueryServiceProvider.php

@@ -0,0 +1,38 @@
+<?php
+
+declare(strict_types=1);
+
+namespace BookStack\Search\Queries;
+
+use BookStack\Http\HttpRequestService;
+use BookStack\Search\Queries\Services\OpenAiLlmQueryService;
+use BookStack\Search\Queries\Services\LlmQueryService;
+
+class LlmQueryServiceProvider
+{
+    public function __construct(
+        protected HttpRequestService $http,
+    ) {
+    }
+
+    public function get(): LlmQueryService
+    {
+        $service = $this->getServiceName();
+
+        if ($service === 'openai') {
+            return new OpenAiLlmQueryService(config('services.openai'), $this->http);
+        }
+
+        throw new \Exception("No '{$service}' LLM service found");
+    }
+
+    protected static function getServiceName(): string
+    {
+        return strtolower(config('services.llm'));
+    }
+
+    public static function isEnabled(): bool
+    {
+        return !empty(static::getServiceName());
+    }
+}

app/Search/Queries/QueryController.php

@@ -0,0 +1,65 @@
+<?php
+
+namespace BookStack\Search\Queries;
+
+use BookStack\Http\Controller;
+use BookStack\Search\SearchOptions;
+use BookStack\Search\SearchRunner;
+use Illuminate\Http\Request;
+
+class QueryController extends Controller
+{
+    public function __construct(
+        protected SearchRunner $searchRunner,
+    ) {
+        // TODO - Check via testing
+        $this->middleware(function ($request, $next) {
+            if (!LlmQueryServiceProvider::isEnabled()) {
+                $this->showPermissionError('/');
+            }
+            return $next($request);
+        });
+    }
+
+    /**
+     * Show the view to start a vector/LLM-based query search.
+     */
+    public function show(Request $request)
+    {
+        $query = $request->get('ask', '');
+
+        // TODO - Set page title
+
+        return view('search.query', [
+            'query' => $query,
+        ]);
+    }
+
+    /**
+     * Perform an LLM-based query search.
+     */
+    public function run(Request $request, LlmQueryRunner $llmRunner)
+    {
+        // TODO - Rate limiting
+        $query = $request->get('query', '');
+
+        return response()->eventStream(function () use ($query, $llmRunner) {
+
+            $searchTerms = $llmRunner->queryToSearchTerms($query);
+            $searchOptions = SearchOptions::fromTermArray($searchTerms);
+            $searchResults = $this->searchRunner->searchEntities($searchOptions, count: 10)['results'];
+
+            $entities = [];
+            foreach ($searchResults as $entity) {
+                $entityKey = $entity->getMorphClass() . ':' . $entity->id;
+                if (!isset($entities[$entityKey])) {
+                    $entities[$entityKey] = $entity;
+                }
+            }
+
+            yield ['view' => view('entities.list', ['entities' => $entities])->render()];
+
+            yield ['result' => $llmRunner->run($query, array_values($entities))];
+        });
+    }
+}

app/Search/Queries/Services/LlmQueryService.php

@@ -0,0 +1,25 @@
+<?php
+
+namespace BookStack\Search\Queries\Services;
+
+use BookStack\Entities\Models\Entity;
+
+interface LlmQueryService
+{
+    /**
+     * Generate embedding vectors from the given chunk of text.
+     * @return float[]
+     */
+    public function generateEmbeddings(string $text): array;
+
+    public function queryToSearchTerms(string $text): array;
+
+    /**
+     * Query the LLM service using the given user input, and
+     * relevant entity content retrieved locally via a search.
+     * Returns the response output text from the LLM.
+     *
+     * @param Entity[] $context
+     */
+    public function query(string $input, array $context): string;
+}

app/Search/Queries/Services/OpenAiLlmQueryService.php

@@ -0,0 +1,97 @@
+<?php
+
+namespace BookStack\Search\Queries\Services;
+
+use BookStack\Http\HttpRequestService;
+
+class OpenAiLlmQueryService implements LlmQueryService
+{
+    protected string $key;
+    protected string $endpoint;
+    protected string $embeddingModel;
+    protected string $queryModel;
+
+    public function __construct(
+        protected array $options,
+        protected HttpRequestService $http,
+    ) {
+        // TODO - Some kind of validation of options
+        $this->key = $this->options['key'] ?? '';
+        $this->endpoint = $this->options['endpoint'] ?? '';
+        $this->embeddingModel = $this->options['embedding_model'] ?? '';
+        $this->queryModel = $this->options['query_model'] ?? '';
+    }
+
+    protected function jsonRequest(string $method, string $uri, array $data): array
+    {
+        $fullUrl = rtrim($this->endpoint, '/') . '/' . ltrim($uri, '/');
+        $client = $this->http->buildClient(60);
+        $request = $this->http->jsonRequest($method, $fullUrl, $data)
+            ->withHeader('Authorization', 'Bearer ' . $this->key);
+
+        $response = $client->sendRequest($request);
+        return json_decode($response->getBody()->getContents(), true);
+    }
+
+    public function generateEmbeddings(string $text): array
+    {
+        $response = $this->jsonRequest('POST', 'v1/embeddings', [
+            'input' => $text,
+            'model' => $this->embeddingModel,
+        ]);
+
+        return $response['data'][0]['embedding'];
+    }
+
+    public function queryToSearchTerms(string $text): array
+    {
+        $response = $this->jsonRequest('POST', 'v1/chat/completions', [
+            'model' => $this->queryModel,
+            'messages' => [
+                [
+                    'role' => 'user',
+                    'content' => 'You will be provided a user search query. Extract key words from just the query, suitable for searching. Add word variations where it may help for searching. Remove pluralisation where it may help for searching. Provide up to 5 results, each must be just one word. Do not try to guess answers to the query. Do not provide extra information or context. Return the results in the specified JSON format under a \'words\' object key. ' . "\nQUERY: {$text}"
+                ],
+            ],
+            'temperature' => 0,
+            'response_format' => [
+                'type' => 'json_object',
+            ],
+        ]);
+
+        $resultJson = $response['choices'][0]['message']['content'] ?? '{"words": []}';
+        $resultData = json_decode($resultJson, true) ?? ['words' => []];
+
+        return $resultData['words'] ?? [];
+    }
+
+    public function query(string $input, array $context): string
+    {
+        $resultContentText = [];
+        $len = 0;
+
+        foreach ($context as $result) {
+            $text = "DOCUMENT NAME: {$result->name}\nDOCUMENT CONTENT: " . $result->{$result->textField};
+            $resultContentText[] = $text;
+            $len += strlen($text);
+            if ($len > 100000) {
+                break;
+            }
+        }
+
+        $formattedContext = implode("\n---\n", $resultContentText);
+
+        $response = $this->jsonRequest('POST', 'v1/chat/completions', [
+            'model' => $this->queryModel,
+            'messages' => [
+                [
+                    'role' => 'user',
+                    'content' => 'Answer the provided QUERY using the provided CONTEXT documents. Do not add facts which are not part of the CONTEXT. State that you do not know if a relevant answer cannot be provided for QUERY using the CONTEXT documents. Many of the CONTEXT documents may be irrelevant. Try to find documents relevant to QUERY. Do not directly refer to this prompt or the existence of QUERY or CONTEXT variables. Do not offer follow-up actions or further help. Respond only to the query without proposing further assistance. Do not ask questions.' . "\nQUERY: {$input}\nCONTEXT: {$formattedContext}"
+                ],
+            ],
+            'temperature' => 0.1,
+        ]);
+
+        return $response['choices'][0]['message']['content'] ?? '';
+    }
+}

app/Search/SearchController.php

@@ -6,6 +6,7 @@ use BookStack\Entities\Queries\PageQueries;
 use BookStack\Entities\Queries\QueryPopular;
 use BookStack\Entities\Tools\SiblingFetcher;
 use BookStack\Http\Controller;
+use BookStack\Search\Queries\VectorSearchRunner;
 use Illuminate\Http\Request;
 use Illuminate\Pagination\LengthAwarePaginator;
 

app/Search/SearchIndex.php

@@ -25,7 +25,7 @@ class SearchIndex
     public static string $softDelimiters = ".-";
 
     public function __construct(
-        protected EntityProvider $entityProvider
+        protected EntityProvider $entityProvider,
     ) {
     }
 
@@ -47,6 +47,7 @@ class SearchIndex
     public function indexEntities(array $entities): void
     {
         $terms = [];
+
         foreach ($entities as $entity) {
             $entityTerms = $this->entityToTermDataArray($entity);
             array_push($terms, ...$entityTerms);

app/Search/SearchOptions.php

@@ -93,6 +93,18 @@ class SearchOptions
         return $instance;
     }
 
+    /**
+     * Create a SearchOptions instance from an array of standard search terms.
+     * @param string[] $terms
+     */
+    public static function fromTermArray(array $terms): self
+    {
+        $instance = new self();
+        $instance->searches = SearchOptionSet::fromValueArray(array_values(array_filter($terms)), TermSearchOption::class);
+        $instance->limitOptions();
+        return $instance;
+    }
+
     /**
      * Decode a search string and add its contents to this instance.
      */

crowdin.yml

@@ -1,3 +1,4 @@
+project_id: "377219"
 project_identifier: bookstack
 base_path: .
 preserve_hierarchy: false

database/migrations/2025_03_24_155748_create_search_vectors_table.php

@@ -0,0 +1,37 @@
+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\Schema;
+
+return new class extends Migration
+{
+    /**
+     * Run the migrations.
+     */
+    public function up(): void
+    {
+        // TODO - Handle compatibility with older databases that don't support vectors
+        Schema::create('search_vectors', function (Blueprint $table) {
+            $table->string('entity_type', 100);
+            $table->integer('entity_id');
+            $table->text('text');
+
+            $table->index(['entity_type', 'entity_id']);
+        });
+
+        $table = DB::getTablePrefix() . 'search_vectors';
+
+        // TODO - Vector size might need to be dynamic
+        DB::statement("ALTER TABLE {$table} ADD COLUMN (embedding VECTOR(1536) NOT NULL)");
+        DB::statement("ALTER TABLE {$table} ADD VECTOR INDEX (embedding) DISTANCE=cosine");
+    }
+
+    /**
+     * Reverse the migrations.
+     */
+    public function down(): void
+    {
+        Schema::dropIfExists('search_vectors');
+    }
+};

dev/docker/Dockerfile

@@ -14,6 +14,9 @@ RUN apt-get update && \
         wait-for-it && \
     rm -rf /var/lib/apt/lists/*
 
+# Mark /app as safe for Git >= 2.35.2
+RUN git config --system --add safe.directory /app
+
 # Install PHP extensions
 RUN docker-php-ext-configure ldap --with-libdir="lib/$(gcc -dumpmachine)" && \
     docker-php-ext-configure gd --with-freetype --with-jpeg && \

package-lock.json

@@ -23,6 +23,7 @@
         "@ssddanbrown/codemirror-lang-twig": "^1.0.0",
         "@types/jest": "^30.0.0",
         "codemirror": "^6.0.2",
+        "eventsource-client": "^1.1.4",
         "idb-keyval": "^6.2.2",
         "markdown-it": "^14.1.0",
         "markdown-it-task-lists": "^2.1.1",
@@ -4797,6 +4798,27 @@
         "node": ">=0.10.0"
       }
     },
+    "node_modules/eventsource-client": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/eventsource-client/-/eventsource-client-1.2.0.tgz",
+      "integrity": "sha512-kDI75RSzO3TwyG/K9w1ap8XwqSPcwi6jaMkNulfVeZmSeUM49U8kUzk1s+vKNt0tGrXgK47i+620Yasn1ccFiw==",
+      "license": "MIT",
+      "dependencies": {
+        "eventsource-parser": "^3.0.0"
+      },
+      "engines": {
+        "node": ">=18.0.0"
+      }
+    },
+    "node_modules/eventsource-parser": {
+      "version": "3.0.6",
+      "resolved": "https://registry.npmjs.org/eventsource-parser/-/eventsource-parser-3.0.6.tgz",
+      "integrity": "sha512-Vo1ab+QXPzZ4tCa8SwIHJFaSzy4R6SHf7BY79rFBDf0idraZWAkYrDjDj8uWaSm3S2TK+hJ7/t1CEmZ7jXw+pg==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=18.0.0"
+      }
+    },
     "node_modules/execa": {
       "version": "5.1.1",
       "resolved": "https://registry.npmjs.org/execa/-/execa-5.1.1.tgz",

package.json

@@ -53,6 +53,7 @@
     "@ssddanbrown/codemirror-lang-twig": "^1.0.0",
     "@types/jest": "^30.0.0",
     "codemirror": "^6.0.2",
+    "eventsource-client": "^1.1.4",
     "idb-keyval": "^6.2.2",
     "markdown-it": "^14.1.0",
     "markdown-it-task-lists": "^2.1.1",

resources/js/components/index.ts

@@ -45,6 +45,7 @@ export {PagePicker} from './page-picker';
 export {PermissionsTable} from './permissions-table';
 export {Pointer} from './pointer';
 export {Popup} from './popup';
+export {QueryManager} from './query-manager';
 export {SettingAppColorScheme} from './setting-app-color-scheme';
 export {SettingColorPicker} from './setting-color-picker';
 export {SettingHomepageControl} from './setting-homepage-control';

resources/js/components/query-manager.ts

@@ -0,0 +1,77 @@
+import {Component} from "./component";
+
+export class QueryManager extends Component {
+    protected input!: HTMLTextAreaElement;
+    protected generatedLoading!: HTMLElement;
+    protected generatedDisplay!: HTMLElement;
+    protected contentLoading!: HTMLElement;
+    protected contentDisplay!: HTMLElement;
+    protected form!: HTMLFormElement;
+    protected fieldset!: HTMLFieldSetElement;
+
+    setup() {
+        this.input = this.$refs.input as HTMLTextAreaElement;
+        this.form = this.$refs.form as HTMLFormElement;
+        this.fieldset = this.$refs.fieldset as HTMLFieldSetElement;
+        this.generatedLoading = this.$refs.generatedLoading;
+        this.generatedDisplay = this.$refs.generatedDisplay;
+        this.contentLoading = this.$refs.contentLoading;
+        this.contentDisplay = this.$refs.contentDisplay;
+
+        this.setupListeners();
+
+        // Start lookup if a query is set
+        if (this.input.value.trim() !== '') {
+            this.runQuery();
+        }
+    }
+
+    protected setupListeners(): void {
+        // Handle form submission
+        this.form.addEventListener('submit', event => {
+            event.preventDefault();
+            this.runQuery();
+        });
+
+        // Allow Ctrl+Enter to run a query
+        this.input.addEventListener('keydown', event => {
+            if (event.key === 'Enter' && event.ctrlKey && this.input.value.trim() !== '') {
+                this.runQuery();
+            }
+        });
+    }
+
+    protected async runQuery(): Promise<void> {
+        this.contentLoading.hidden = false;
+        this.generatedLoading.hidden = false;
+        this.contentDisplay.innerHTML = '';
+        this.generatedDisplay.innerHTML = '';
+        this.fieldset.disabled = true;
+
+        const query = this.input.value.trim();
+        const url = new URL(window.location.href);
+        url.searchParams.set('ask', query);
+        window.history.pushState({}, '', url.toString());
+
+        const es = window.$http.eventSource('/query', 'POST', {query});
+
+        let messageCount = 0;
+        for await (const {data, event, id} of es) {
+            messageCount++;
+            if (messageCount === 1) {
+                // Entity results
+                this.contentDisplay.innerHTML = JSON.parse(data).view;
+                this.contentLoading.hidden = true;
+            } else if (messageCount === 2) {
+                // LLM Output
+                this.generatedDisplay.innerText = JSON.parse(data).result;
+                this.generatedLoading.hidden = true;
+            } else {
+                es.close();
+                break;
+            }
+        }
+
+        this.fieldset.disabled = false;
+    }
+}

resources/js/services/http.ts

@@ -1,3 +1,5 @@
+import {createEventSource, EventSourceClient} from "eventsource-client";
+
 type ResponseData = Record<any, any>|string;
 
 type RequestOptions = {
@@ -59,7 +61,6 @@ export class HttpManager {
     }
 
     createXMLHttpRequest(method: string, url: string, events: Record<string, (e: Event) => void> = {}): XMLHttpRequest {
-        const csrfToken = document.querySelector('meta[name=token]')?.getAttribute('content');
         const req = new XMLHttpRequest();
 
         for (const [eventName, callback] of Object.entries(events)) {
@@ -68,7 +69,7 @@ export class HttpManager {
 
         req.open(method, url);
         req.withCredentials = true;
-        req.setRequestHeader('X-CSRF-TOKEN', csrfToken || '');
+        req.setRequestHeader('X-CSRF-TOKEN', this.getCSRFToken());
 
         return req;
     }
@@ -95,12 +96,11 @@ export class HttpManager {
             requestUrl = urlObj.toString();
         }
 
-        const csrfToken = document.querySelector('meta[name=token]')?.getAttribute('content') || '';
         const requestOptions: RequestInit = {...options, credentials: 'same-origin'};
         requestOptions.headers = {
             ...requestOptions.headers || {},
             baseURL: window.baseUrl(''),
-            'X-CSRF-TOKEN': csrfToken,
+            'X-CSRF-TOKEN': this.getCSRFToken(),
         };
 
         const response = await fetch(requestUrl, requestOptions);
@@ -191,6 +191,33 @@ export class HttpManager {
         return this.dataRequest('DELETE', url, data);
     }
 
+    eventSource(url: string, method: string = 'GET', body: object = {}): EventSourceClient {
+        if (!url.startsWith('http')) {
+            url = window.baseUrl(url);
+        }
+
+        const es = createEventSource({
+            url,
+            method,
+            body: JSON.stringify(body),
+            credentials: 'same-origin',
+            headers: {
+                'Content-Type': 'application/json',
+                'X-CSRF-TOKEN': this.getCSRFToken(),
+            },
+            onDisconnect: () => {
+                console.log('here');
+                es.close();
+            }
+        });
+
+        return es;
+    }
+
+    protected getCSRFToken(): string {
+        return document.querySelector('meta[name=token]')?.getAttribute('content') || '';
+    }
+
     /**
      * Parse the response text for an error response to a user
      * presentable string. Handles a range of errors responses including

resources/sass/_forms.scss

@@ -601,3 +601,29 @@ input.shortcut-input {
   max-width: 120px;
   height: auto;
 }
+
+.query-form {
+  display: flex;
+  flex-direction: row;
+  gap: vars.$m;
+  textarea {
+    font-size: 1.4rem;
+    height: 100px;
+    box-shadow: vars.$bs-card;
+    border-radius: 8px;
+    color: #444;
+  }
+  button {
+    align-self: start;
+    margin: 0;
+    font-size: 1.6rem;
+  }
+  button:disabled {
+    opacity: 0.5;
+    cursor: not-allowed;
+  }
+  textarea:disabled {
+    opacity: 0.5;
+    cursor: not-allowed;
+  }
+}

resources/views/search/query.blade.php

@@ -0,0 +1,52 @@
+@extends('layouts.simple')
+
+@section('body')
+    <div component="query-manager" class="container small pt-xxl">
+
+        <div class="card content-wrap auto-height">
+            <h1 class="list-heading">Start a Query</h1>
+            <form action="{{ url('/query') }}"
+                  refs="query-manager@form"
+                  title="Run Query"
+                  method="post">
+                <fieldset class="query-form" refs="query-manager@fieldset">
+                    <textarea name="query"
+                              refs="query-manager@input"
+                              class="input-fill-width"
+                              rows="5"
+                              placeholder="Enter a query"
+                              autocomplete="off">{{ $query }}</textarea>
+                    <button class="button icon">@icon('search')</button>
+                </fieldset>
+            </form>
+        </div>
+
+        <div class="card content-wrap auto-height pb-xl">
+            <h2 class="list-heading">Generated Response</h2>
+            <div refs="query-manager@generated-loading" hidden>
+                @include('common.loading-icon')
+            </div>
+            <p refs="query-manager@generated-display">
+                <span class="text-muted italic">
+                When you run a query, the relevant content found & shown below will be used to help generate a smart machine generated response.
+                </span>
+            </p>
+        </div>
+
+
+        <div class="card content-wrap auto-height pb-xl">
+            <h2 class="list-heading">Relevant Content</h2>
+            <div refs="query-manager@content-loading" hidden>
+                @include('common.loading-icon')
+            </div>
+            <div class="book-contents">
+                <div refs="query-manager@content-display" class="entity-list">
+                    <p class="text-muted italic mx-m">
+                        Start a query to find relevant matching content.
+                        The items shown here reflect those used to help provide the above response.
+                    </p>
+                </div>
+            </div>
+        </div>
+    </div>
+@stop

routes/web.php

@@ -11,6 +11,7 @@ use BookStack\Exports\Controllers as ExportControllers;
 use BookStack\Http\Middleware\VerifyCsrfToken;
 use BookStack\Permissions\PermissionsController;
 use BookStack\References\ReferenceController;
+use BookStack\Search\Queries\QueryController;
 use BookStack\Search\SearchController;
 use BookStack\Settings as SettingControllers;
 use BookStack\Sorting as SortingControllers;
@@ -196,6 +197,11 @@ Route::middleware('auth')->group(function () {
     Route::get('/search/entity-selector-templates', [SearchController::class, 'templatesForSelector']);
     Route::get('/search/suggest', [SearchController::class, 'searchSuggestions']);
 
+    // Queries
+    Route::get('/query', [QueryController::class, 'show']);
+    Route::get('/query/run', [QueryController::class, 'run']); // TODO - Development only, remove
+    Route::post('/query', [QueryController::class, 'run']);
+
     // User Search
     Route::get('/search/users/select', [UserControllers\UserSearchController::class, 'forSelect']);
     Route::get('/search/users/mention', [UserControllers\UserSearchController::class, 'forMentions']);

tests/Search/TextChunkerTest.php

@@ -0,0 +1,47 @@
+<?php
+
+namespace Search;
+
+use BookStack\Search\Queries\TextChunker;
+use Tests\TestCase;
+
+class TextChunkerTest extends TestCase
+{
+    public function test_it_chunks_text()
+    {
+        $chunker = new TextChunker(3, []);
+        $chunks = $chunker->chunk('123456789');
+
+        $this->assertEquals(['123', '456', '789'], $chunks);
+    }
+
+    public function test_chunk_size_must_be_greater_than_zero()
+    {
+        $this->expectException(\InvalidArgumentException::class);
+        $chunker = new TextChunker(-5, []);
+    }
+
+    public function test_it_works_through_given_delimiters()
+    {
+        $chunker = new TextChunker(5, ['-', '.', '']);
+        $chunks = $chunker->chunk('12-3456.789abcdefg');
+
+        $this->assertEquals(['12', '3456', '789ab', 'cdefg'], $chunks);
+    }
+
+    public function test_it_attempts_to_pack_chunks()
+    {
+        $chunker = new TextChunker(8, [' ', '']);
+        $chunks = $chunker->chunk('123 456 789 abc def');
+
+        $this->assertEquals(['123 456', '789 abc', 'def'], $chunks);
+    }
+
+    public function test_it_attempts_to_pack_using_subchunks()
+    {
+        $chunker = new TextChunker(8, [' ', '-', '']);
+        $chunks = $chunker->chunk('123 456-789abc');
+
+        $this->assertEquals(['123 456', '789abc'], $chunks);
+    }
+}

version

@@ -1 +1 @@
-v25.11-dev
+v26.01-dev