From f505bb7a0f7bbc22511f3de20a4944fa15d59588 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E2=80=9EBAS8243=E2=80=9C?= <gerd.embruch@uni-hamburg.de>
Date: Wed, 31 Jul 2024 20:55:49 +0200
Subject: [PATCH] added a second LLM query to check if the given answer based
 on docs. Now the source attribute is more reliable

---
 .codiumai.toml                   | 142 -------------------------------
 __tests__/manualREST/ollama.rest |  26 +++---
 controllers/Embeddings.js        |   7 ++
 utils/handleAI.js                |  50 +++++++----
 utils/handleDB.js                |   3 +-
 5 files changed, 57 insertions(+), 171 deletions(-)
 delete mode 100644 .codiumai.toml

diff --git a/.codiumai.toml b/.codiumai.toml
deleted file mode 100644
index d5c7370..0000000
--- a/.codiumai.toml
+++ /dev/null
@@ -1,142 +0,0 @@
-#.codiumai.toml
-[tests]
-
-## Testing framework to use - this can affect the content of the generated tests
-## as well as the test run command.
-## Possible values are:
-##  Python: Pytest, Unittest
-##  Javascript / Typescript: Jest, Mocha, Vitest, Karma, Jasmine, QUnit, React Testing Library
-##    HOWEVER: running tests in JS / TS is at the moment only supported
-##    for Jest, Mocha, Vitest, and React Testing Library
-# framework = "Jest"
-framework = "Vitest"
-
-## An additional Javascript utility library used to test your code, if any. 
-## Possible values are None, Testing Library, Enzyme, or Chai. Not applicable to Python projects.
-# utility_library = "Testing Library"
-
-## A hint to the test generator about whether to use mocks or not. Possible values are true or false.
-# use_mocks = false
-
-## How many tests should be generated by default. Fewer tests is faster.
-## Does not apply at the moment to extend-suite tests.
-# num_desired_tests = 6
-num_desired_tests = 6
-
-## A multiline string, delimited with triple-quotes (""") serving as an extra instruction
-## that the AI model will take into consideration.
-## This will appear as "General instructions" in the
-## configuration section in the tests panel.
-# plan_instructions = """
-# Each line should have a comment explaining it.
-# Each comment should start with the comment number (1., 2. etc.)
-# """
-plan_instructions = """
-1. make use of supertest
-2. consider supertest and vitest as already imported
-3. encapsulate the tests in a describe block
-4. use the beforeAll hook to make the request with BEFORE_ALL_TIMEOUT as the second parameter for timeout
-5. use the response object to make assertions
-6. use the expect function to make assertions
-7. encasulate each asserrtion with a it block
-8. make sure to test the GET, POST, PUT, and DELETE endpoints
-9. use the es6 import statement instead of require
-10. whenever you create a random e-mail address use the domain local.local instead of example.com
-"""
-
-## A multiline string, delimited with triple-quotes (""") serving as an example test that represents
-## what you would like the generated tests to look like in terms of style, setup, etc.
-# example_test = """
-# describe("something", () => {
-#   it("says 'bar'", () => {
-#     // given
-#
-#     // when
-#     const res = something.say();
-#    
-#     // Then
-#     expect(res).to.equal("bar");
-#   });
-# });
-# """
-example_test = """
-describe('user registration', () => {
-  describe('given the inputs are valid', () => {
-    // set response by running route
-    let response;
-    beforeAll(async () => {
-      response = await supertest(app)
-        .post('/users/signup')
-        .send({
-          name: 'John Doe',
-          username: 'johndoe',
-          email: 'john.doe@local.local',
-          password: 'StrongPass1!',
-          confirmPassword: 'StrongPass1!'
-        });
-    }, BEFORE_ALL_TIMEOUT);
-    it('should return 200', () => {
-      expect(response.status).toBe(200);
-    });
-    it('should respond with a proper message', () => {
-      expect(response.body.message).toEqual('Check your emails for the verification link.');
-    });
-  });
-});
-"""
-
-
-[tests.javascript]
-
-## When running Javascript / Typescript tests, use this directory as the test process "current working directory".
-## This is a path relative to the location of the config file.
-## Default: The directory containing the config file.
-## Note: the typical setup is to place the config file in the same directory as the relevant 'package.json' file,
-## and leave this commented-out.
-# overrideTestRunCwd = "./test"
-
-## This is the command that's used to run tests.
-## PLEASE READ CAREFULLY:
-##
-## When running tests, CodiumAI generates a temporary file that contains the test code for a single test,
-## and runs that file.
-## When the tests are done, the temporary file is deleted.
-## For component-oriented tests (when you click "test this class" or "test this function"), the temporary file
-## is created next to the file being tested.
-## For extend-suite tests (when you click "add more tests" on a test-suite), the temporary file is created next
-## to the test-suite file.
-##
-## Typically, you're going to want to take the test script defined in your package.json file, and tweak it a
-## little to make it compatible with CodiumAI.
-##
-## You almost always want to start with 'npx' (e.g. 'npx jest', not 'npm jest' or 'yarn test').
-##
-## Note that the test command must be able to run test files that are located in the same directory as the
-## file under test.
-## A common issue is that the test command in the package.json file selects only from
-## a "tests" directory, causing the CodiumAI tests be "not found" - please remove any such restriction from
-## the command / configuration.
-##
-## The placeholder TEST_FILEPATH will be replaced with the actual test file path - this is how we find
-## the file to run.
-##
-## EXAMPLES:
-## Mocha:
-##    npx ts-mocha TEST_FILEPATH --require ./test/mocha/setup.ts
-## Jest:
-##    npx jest --runTestsByPath TEST_FILEPATH
-## 
-## DEBUGGING NOTE:
-## To help debug run-tests issues, you can view run logs in vscode's OUTPUT 
-## (select codium-ai from the dropdown).
-## It's helpful to clear the output (right-click -> clear) and then run the tests again.
-##
-# overrideTestRunScript = "npx jest --runTestsByPath TEST_FILEPATH"
-
-## A multiline string, delimited with triple-quotes ("""),
-## containing import declaration to use in each test file. 
-# overrideImports = """ 
-# import {expect} from 'chai'; """
-overrideImports = """ 
-import { vi, beforeAll, describe, expect, expectTypeOf, test, it } from 'vitest';
-"""
\ No newline at end of file
diff --git a/__tests__/manualREST/ollama.rest b/__tests__/manualREST/ollama.rest
index a3d732a..0dd439f 100644
--- a/__tests__/manualREST/ollama.rest
+++ b/__tests__/manualREST/ollama.rest
@@ -21,16 +21,7 @@
 #################
 # HANDLE LOGIN
 #################
-### login
-# @name login
-POST {{host}}/auth/login
-Accept: application/json
-Content-Type: application/json
 
-{
-  "password": "{{password}}",
-  "email": "{{email}}"
-}
 
 #################
 # HANDLE MODELS
@@ -87,6 +78,17 @@ Content-Type: application/json
 #################
 # CHAT
 #################
+### login
+# @name login
+POST {{host}}/auth/login
+Accept: application/json
+Content-Type: application/json
+
+{
+  "password": "{{password}}",
+  "email": "{{email}}"
+}
+
 ### get chats
 # @name chats
 GET {{host}}/ai/chats
@@ -102,7 +104,7 @@ Accept: application/json
 Content-Type: application/json
 
 {
-  "input": "When does mocking stops feeling like torture?",
+  "input": "Under what path could members of the working group can find the exam git directory?",
   "model": "llama3"
 }
 
@@ -114,7 +116,7 @@ Accept: application/json
 Content-Type: application/json
 
 {
-  "chatId": "1",  
-  "input": "Where did you found this information?",
+  "chatId": "{{chatID}}",  
+  "input": "What else can be found under that path?",
   "model": "llama3"
 }
\ No newline at end of file
diff --git a/controllers/Embeddings.js b/controllers/Embeddings.js
index b7ad482..8a37691 100644
--- a/controllers/Embeddings.js
+++ b/controllers/Embeddings.js
@@ -18,6 +18,7 @@ import fs from 'fs';
 import path from 'path';
 import { PDFLoader } from '@langchain/community/document_loaders/fs/pdf';
 import { MultiFileLoader } from "langchain/document_loaders/fs/multi_file";
+import { ScoreThresholdRetriever } from 'langchain/retrievers/score_threshold';
 
 
 // PROVIDE OLLAMA CONNECTION
@@ -54,6 +55,12 @@ try {
 export default vectorStoreConnection;
 // PROVIDE RETRIEVER
 export const retriever = vectorStoreConnection.asRetriever();
+// export const retriever = vectorStoreConnection.asRetriever(1);
+// export const retriever = ScoreThresholdRetriever.fromVectorStore(vectorStoreConnection, {
+//   minSimilarityScore: 0.1, // Finds results with at least this similarity score
+//   maxK: 100, // The maximum K value to use. Use it based to your chunk size to make sure you don't run out of tokens
+//   kIncrement: 2, // How much to increase K by each time. It'll fetch N results, then N + kIncrement, then N + kIncrement * 2, etc.
+// });
 
 
 
diff --git a/utils/handleAI.js b/utils/handleAI.js
index d6d0eed..b04c72e 100644
--- a/utils/handleAI.js
+++ b/utils/handleAI.js
@@ -156,11 +156,12 @@ export const chat = async (req, res, next) => {
     new MessagesPlaceholder("chat_history"),
     ["user", "{input}"],
   ]);
+
   // create chat chain
   const chatChain = await createStuffDocumentsChain({
     llm,
     prompt: chatPrompt,
-    returnMessages: true
+    returnMessages: false
   });
   performance.mark('createStuffDocumentsChain:end');
 
@@ -171,25 +172,21 @@ export const chat = async (req, res, next) => {
   performance.mark('createConversationalRetrievalChain:start');
   const conversationalRetrievalChain = await createRetrievalChain({
     retriever: historyAwareRetrieverChain,
-    combineDocsChain: chatChain,
+    combineDocsChain: chatChain
   });
+
   performance.mark('createConversationalRetrievalChain:end');
   performance.mark('invokeConversationalRetrievalChain:start');
   // finally ask the question
   const result = await conversationalRetrievalChain.invoke({
     chat_history: req.body.chatHistory ?? [],
-    input: req.body.input,
+    input: req.body.input
   });
   performance.mark('invokeConversationalRetrievalChain:end');
 
-  // get source informations and prepare to store in chat history
-  // Answers from DocumentSource are prefixed with '<DS> '
-  // Answers from pretrained knowledge are prefixed with '<PK> '
-  // BUG: prefixes are not consistent correctly set bet LLM
-  // console.log('Answer: ', result.answer.substring(0, 15), '...');
+  const reliesOnDoc = await isFactual(req.body.model, result.answer, result.context[0].pageContent);
   let sourceLocation;
-  performance.mark('setSourceLocation:start');
-  if (result.answer.startsWith('<DS> ')) {
+  if (reliesOnDoc.content.toLowerCase() === 'true') {
     const file = path.posix.basename(result.context[0].metadata.source);
     const posFrom = result.context[0].metadata.loc.lines.from;
     const posTo = result.context[0].metadata.loc.lines.to;
@@ -198,11 +195,6 @@ export const chat = async (req, res, next) => {
     sourceLocation = 'pretrained';
   }
 
-  result.answer = result.answer.substring(5);
-  performance.mark('setSourceLocation:end');
-
-  console.log(JSON.stringify(result));
-
   // store q/a-pair in chat history
   let chat = await extendChat(req.body.chatId, [
     new HumanMessage(req.body.input),
@@ -210,7 +202,33 @@ export const chat = async (req, res, next) => {
   ]);
   performance.mark('chat:end');
 
-
   // return the answer
   return res.json({ answer: result.answer, chat });
+};
+
+
+
+/** *******************************************************
+ * CREATE AI SUMMARIZED TEXT
+ */
+export const isFactual = async (model, answer, context) => {
+  try {
+    performance.mark('isFactual:start');
+    // define llm
+    const llm = new ChatOllama({
+      baseUrl: process.env['AI_API_URL'],
+      model: model,
+      temperature: Number(process.env['AI_TEMPERATURE'])
+    });
+    // create template
+    const promptTemplate = PromptTemplate.fromTemplate(process.env['AI_FACTUAL_PROMPT']);
+    // create chain combining llm and template
+    const chain = promptTemplate.pipe(llm);
+    // invoke variable text & run chain
+    const factual = await chain.invoke({ answer, context });
+    performance.mark('isFactual:end');
+    return factual;
+  } catch (error) {
+    throw error;
+  }
 };
\ No newline at end of file
diff --git a/utils/handleDB.js b/utils/handleDB.js
index 901421d..6455dd2 100644
--- a/utils/handleDB.js
+++ b/utils/handleDB.js
@@ -154,7 +154,7 @@ export const updateOneRecord = async (newData) => {
 export const findByIdAndUpdate = async (model, id, data) => {
   try {
     performance.mark('findByIdAndUpdate:start');
-    const result = model.findByIdAndUpdate(id, data);
+    const result = model.findByIdAndUpdate(id, data, { returnDocument: 'after' });
     performance.mark('findByIdAndUpdate:end');
     return result;
   } catch (error) {
@@ -205,6 +205,7 @@ export const extendChat = async (chatId, messages) => {
     performance.mark('extendChat:end');
     // save & return chat
     return await findByIdAndUpdate(Chat, chatId, { chatHistory: record.chatHistory });
+
   } catch (error) {
     throw error;
   }
-- 
GitLab