utils/embedding_utils.cpp
| Line | Branch | Exec | Source |
|---|---|---|---|
| 1 | #include "embedding_utils.hpp" | ||
| 2 | #include "utils/text_utils.hpp" | ||
| 3 | #include <string> | ||
| 4 | #include <vector> | ||
| 5 | |||
| 6 | /** | ||
| 7 | * Constructs an EmbeddingUtils instance with the collaborators it needs for generating and storing embeddings. | ||
| 8 | * | ||
| 9 | * @param ui: Shared pointer to the UserInterface used to display messages and status. | ||
| 10 | * @param curl: Shared pointer to the CurlUtils used to perform HTTP requests. | ||
| 11 | * @param ollama_client: Shared pointer to the OllamaClient used to request embeddings from Ollama. | ||
| 12 | * @param storage_client: Shared pointer to the QdrantClient used to store and retrieve embeddings. | ||
| 13 | * | ||
| 14 | * Initializes the ui, curl, ollama_client, and storage_client members by copying each shared pointer, so this instance | ||
| 15 | * shares ownership of every dependency with the caller. The constructor body is empty, so no other work is performed. | ||
| 16 | * | ||
| 17 | * Note: Because the dependencies are shared, they remain alive as long as any holder still references them. Passing a | ||
| 18 | * null pointer is not checked here, so any later use of a null dependency will fail at the point of use. | ||
| 19 | */ | ||
| 20 | ✗ | EmbeddingUtils::EmbeddingUtils(std::shared_ptr<UserInterface> ui, std::shared_ptr<CurlUtils> curl, | |
| 21 | std::shared_ptr<OllamaClient> ollama_client, | ||
| 22 | ✗ | std::shared_ptr<QdrantClient> storage_client) | |
| 23 | ✗ | : ui(ui), curl(curl), ollama_client(ollama_client), storage_client(storage_client) { | |
| 24 | ✗ | } | |
| 25 | |||
| 26 | /** | ||
| 27 | * Fetches documents from the given URLs, splits them into chunks, generates embeddings for those chunks, and stores | ||
| 28 | * the chunks with their embeddings in Qdrant. | ||
| 29 | * | ||
| 30 | * @param urls: The web addresses whose content should be fetched and embedded. | ||
| 31 | * | ||
| 32 | * Runs a four-stage pipeline. First, each URL is fetched with the CurlUtils instance and its content is collected in | ||
| 33 | * order. Second, every fetched document is split with TextUtils::split_text using a chunk size of 250 and an overlap of | ||
| 34 | * 0, and the resulting chunks are appended to a single list. Third, the OllamaClient generates one embedding per chunk. | ||
| 35 | * Fourth, the QdrantClient stores the chunks together with their embeddings. Before each stage, a progress message is | ||
| 36 | * shown, using the interactive UI when it is running and the non-interactive output otherwise. When all stages finish, | ||
| 37 | * a success message is displayed through the same mechanism. | ||
| 38 | * | ||
| 39 | * Note: The method assumes that the embeddings returned by the OllamaClient correspond index for index to the chunks | ||
| 40 | * passed to it, because the storage call pairs them by position. No length check is performed. Fetch failures are not | ||
| 41 | * detected here, so an empty or error string returned by CurlUtils is split and embedded like any other content. | ||
| 42 | * Exceptions thrown by the curl, Ollama, Qdrant, or UI components are not caught and propagate to the caller, which | ||
| 43 | * means a failure in any stage stops the pipeline before the later stages run. | ||
| 44 | */ | ||
| 45 | ✗ | void EmbeddingUtils::create_embedings(const std::vector<std::string>& urls) { | |
| 46 | ✗ | std::string message{}; | |
| 47 | |||
| 48 | ✗ | message = "⏳ Fetching documents from URLs...\n"; | |
| 49 | ✗ | if (this->ui->backend_running() == true) { | |
| 50 | ✗ | this->ui->print_message(message); | |
| 51 | } else { | ||
| 52 | ✗ | UserInterface::print_message_noninteractive(message); | |
| 53 | } | ||
| 54 | |||
| 55 | ✗ | std::vector<std::string> all_documents; | |
| 56 | ✗ | for (const auto& url : urls) { | |
| 57 | ✗ | std::string content = this->curl->fetch_web_content(url); | |
| 58 | ✗ | all_documents.push_back(content); | |
| 59 | ✗ | } | |
| 60 | |||
| 61 | ✗ | message = "⏳ Splitting documents into chunks...\n"; | |
| 62 | ✗ | if (this->ui->backend_running() == true) { | |
| 63 | ✗ | this->ui->print_message(message); | |
| 64 | } else { | ||
| 65 | ✗ | UserInterface::print_message_noninteractive(message); | |
| 66 | } | ||
| 67 | |||
| 68 | ✗ | std::vector<std::string> document_chunks; | |
| 69 | ✗ | for (const auto& doc : all_documents) { | |
| 70 | ✗ | auto chunks = TextUtils::split_text(doc, 250, 0); | |
| 71 | ✗ | document_chunks.insert(document_chunks.end(), chunks.begin(), chunks.end()); | |
| 72 | ✗ | } | |
| 73 | |||
| 74 | ✗ | message = "⏳ Generating embeddings...\n"; | |
| 75 | ✗ | if (this->ui->backend_running() == true) { | |
| 76 | ✗ | this->ui->print_message(message); | |
| 77 | } else { | ||
| 78 | ✗ | UserInterface::print_message_noninteractive(message); | |
| 79 | } | ||
| 80 | ✗ | auto embeddings = this->ollama_client->generate_embeddings(document_chunks); | |
| 81 | |||
| 82 | ✗ | message = "⏳ Storing embeddings in Qdrant...\n"; | |
| 83 | ✗ | if (this->ui->backend_running() == true) { | |
| 84 | ✗ | this->ui->print_message(message); | |
| 85 | } else { | ||
| 86 | ✗ | UserInterface::print_message_noninteractive(message); | |
| 87 | } | ||
| 88 | ✗ | this->storage_client->store(document_chunks, embeddings); | |
| 89 | |||
| 90 | ✗ | message = "Embeddings created.\n"; | |
| 91 | ✗ | if (this->ui->backend_running() == true) { | |
| 92 | ✗ | this->ui->print_success(message); | |
| 93 | } else { | ||
| 94 | ✗ | UserInterface::print_success_noninteractive(message); | |
| 95 | } | ||
| 96 | ✗ | } | |
| 97 |