ft

In-program fulltext search with structured ranking and snippet generation. Builds a naive in-memory inverted index from a document set or a store, then runs a query language (keyword conjunctions, phrases, boolean operators, field restriction) ranked by TF-IDF (default) or BM25. The tokenization pipeline — Unicode word segmentation, case folding, stopword removal, and English stemming — runs entirely inside the process, with no persistent index and no external engine. The Index is an opaque in-memory value rebuilt per run.

ft:index

[$ft:index] -> element — Build an Index over a sequence of documents using the default tokenization pipeline.

            [?lib 'cx-stdlib/ft']
[$ft:index-stats [$ft:index ()]]

            {doc-count: 0, term-count: 0, size-bytes: 0, languages: ()}

ft:index-with-opts

[$ft:index-with-opts] -> element — Build an Index with explicit options for language, stopwords, stemmer, tokenizer, and fields.

            [?lib 'cx-stdlib/ft']
[?let [= $idx [$ft:index-with-opts ([doc [title "Database design"] [body "schemas and indexes"]]) {"fields" ("title")}]]
  [$ft:search $idx [$ft:parse-query "body:schemas"] 10]]

            cx-err:CXER1202

ft:search

[$ft:search] -> [sequence element] — Run a query against an Index and return up to limit ranked result elements.

            [?lib 'cx-stdlib/ft']
[?let [= $idx [$ft:index ([doc "database design"], [doc "schema migrations"])]]
  [$ft:search $idx [$ft:parse-query ""] 10]]

()

ft:parse-query

[$ft:parse-query] -> element — Parse an end-user query string (the §2.2.1 data format) into the canonical [query …] element — the only place the string format exists.

            [?lib 'cx-stdlib/ft']
[$ft:parse-query "database"]

            [query [term 'database']]

ft:search-with-opts

[$ft:search-with-opts] -> [sequence element] — Search with options for scoring mode, limit, offset, min-score, and query tokenization.

            [?lib 'cx-stdlib/ft']
[?let [= $idx [$ft:index ([doc id="d1" "database one"], [doc id="d2" "database two"], [doc id="d3" "database three"])]]
  [$count [$ft:search-with-opts $idx [$ft:parse-query "database"] {limit: 2}]]]

ft:search-store

[$ft:search-store] -> [sequence element] — Build an Index from a store's documents and search it in one call.

            [?lib 'cx-stdlib/ft']
[?lib 'cx-stdlib/store']
[?let [= $store [$store:open "mem:///ft-conformance"]]
  [= $_ [$store:put-doc $store [doc id="d1" "database design and analytics"]]]
  [$ft:doc-ids [$ft:search-store $store [$ft:parse-query "database"] 10]]]

            ('d1')

ft:snippet

[$ft:snippet] -> string — Extract a context window around query matches, wrapping each match in tags.

            [?lib 'cx-stdlib/ft']
[?lib 'cx-stdlib/strings']
[$strings:contains [$ft:snippet [doc "the database design covers schemas and indexes"] [$ft:parse-query "database"] 80] ""]

            true

ft:snippet-with-opts

[$ft:snippet-with-opts] -> string — Extract snippets with options for context size, count, ellipsis, and mark delimiters.

            [?lib 'cx-stdlib/ft']
[?lib 'cx-stdlib/strings']
[$strings:contains [$ft:snippet-with-opts [doc "the database design covers schemas"] [$ft:parse-query "database"] {mark-prefix: "[[", mark-suffix: "]]"}] "[[database]]"]

            true

ft:tokenize

[$ft:tokenize] -> [sequence string] — Run the default pipeline over text and return its token sequence — useful for debugging alignment.

            [?lib 'cx-stdlib/ft']
[?lib 'cx-stdlib/strings']
[$contains [$ft:tokenize "the cat in the hat" "en"] "the"]

            false

ft:doc-ids

[$ft:doc-ids] -> [sequence string] — Extract the document IDs from a sequence of search results.

            [?lib 'cx-stdlib/ft']
[?let [= $idx [$ft:index ([doc id="d1" "database design and analytics"], [doc id="d2" "kitchen recipes for soup"])]]
  [$ft:doc-ids [$ft:search $idx [$ft:parse-query "database"] 10]]]

            ('d1')

ft:score-of

[$ft:score-of] -> float — Extract the relevance score from a single search result.

            [?lib 'cx-stdlib/ft']
[?let [= $idx [$ft:index ([doc id="d1" "database design"])]]
  [= $results [$ft:search $idx [$ft:parse-query "database"] 10]]
  [$gte [$ft:score-of [$first $results]] 0.0]]

            true

ft:index-stats

[$ft:index-stats] -> map — Report Index statistics — document count, term count, byte size, and languages.

            [?lib 'cx-stdlib/ft']
[$ft:index-stats [$ft:index ()]]

            {doc-count: 0, term-count: 0, size-bytes: 0, languages: ()}