finalized

.env.template

@@ -8,7 +8,8 @@ TARGET=https://localhost:8080
 # TESTING
 ##########
 # API key from https://app.artillery.io/
-# ARTILLERY_CLOUD_API_KEY=a9_7GibberISHw-ApL
+# https://www.artillery.io/docs/get-started/artillery-cloud#configure-the-cli-to-send-data-to-artillery-cloud
+ARTILLERY_CLOUD_API_KEY=a9_7GibberISHw-ApL
 
 # seconds to wait between each http request
 SLEEP=2

README.md

@@ -16,6 +16,8 @@ cd ragchat-artillery
 npm i
 cp ./.env.template ./.env
 # fill envs with production and/or devel values
+# evtl. edit tags on package.json => scripts => start
+npm start
 ```
 
 
@@ -24,5 +26,3 @@ cp ./.env.template ./.env
 - [PM2](https://pm2.keymetrics.io/)
 
 # Roadmap
-- [ ] create a artillery script to test login & fetch jwt
-- [ ] extend script to use JWT to send a prompt

__tests__/prompting.yaml

 config:
   # This is a test server run by team Artillery
   # It's designed to be highly scalable
+  name: "WSL - 4 Cores"
   target: "{{ $processEnvironment.TARGET}}"
   processor: "../helper.js"
   timeout: 300
   phases:
     - duration: 10
       arrivalRate: 1
-      #rampTo: 2
+      rampTo: 2
       name: Warm up phase
-    # - duration: 2
-    #   arrivalRate: 2
-    #   rampTo: 3
-    #   name: Ramp up load
-    # - duration: 1
-    #   arrivalRate: 3
-    #   rampTo: 4
-    #   name: Spike phase
+    - duration: 20
+      arrivalRate: 2
+      rampTo: 3
+      name: Ramp up load
+    - duration: 10
+      arrivalRate: 3
+      rampTo: 4
+      name: Spike phase
   # Load & configure a couple of useful plugins
   # https://docs.art/reference/extensions
   plugins:
@@ -24,36 +25,41 @@ config:
     apdex: {}
     metrics-by-endpoint: {}
     fake-data: {}
+    expect: {}
   apdex:
   # define how many milliseconds is considered satisfactory
-    threshold: 1200
+    threshold: 3000
   ensure:
     thresholds:
-    # p99 of response time must be lower
-      - http.response_time.p99: 1200
-    # p95 of response time must be lower
-      - http.response_time.p95: 1100
+    # 99% of the responses are expected to be faster than this value (in ms)
+      - http.response_time.p99: 5000
+    # 95% of the responses are expected to be faster than this value (in ms)
+      - http.response_time.p95: 4000
 scenarios:
   - flow:
     - function: "createUser"
     - post:
-        url: '/users/signup'
+        url: '/users'
         json:
           name: "{{name}}"
           username: "{{username}}"
           email: "{{email}}"
           password: "{{password}}"
-          passwordConfirm: "{{password}}"
+          confirmPassword: "{{password}}"
+        expect:
+          - statusCode: 201
     - log: "Created user: {{username}}"
     - think : "{{$processEnvironment.SLEEP}}"
     - post:
-       url: '/users/login'
+        url: '/auth/login'
         json:
           email: "{{email}}"
           password: "{{password}}"
         capture:
-         - json: "$.token"
+          - json: "$.accessToken"
             as: "JWT"
+        expect:
+          - statusCode: 200
     - log: "{{username}} fetched JWT {{JWT}}"
     - think : "{{$processEnvironment.SLEEP}}"
     - post:
@@ -66,19 +72,24 @@ scenarios:
         capture:
           - json: "$.chat.id"
             as: "chatId"
-    # - think : "{{$processEnvironment.SLEEP}}"
-    # - post:
-    #    url: '/ai/chat'
-    #    headers:
-    #      Authorization: "Bearer {{ JWT }}"
-    #    json:
-    #      input: "{{$processEnvironment.RAGQ2}}"
-    #      model: "llama3"
-    #      chatId: "{{ chatId }}"
-    #    capture:
-    #      - json: "$.chat.id"
-    #        as: "chatId"
-    #- think : "{{$processEnvironment.SLEEP}}"
-    #- get:
-    #    url: '/users/logout'
-#
+        expect:
+          - statusCode: 200
+    - think : "{{$processEnvironment.SLEEP}}"
+    - post:
+        url: '/ai/chat'
+        headers:
+          Authorization: "Bearer {{ JWT }}"
+        json:
+          input: "{{$processEnvironment.RAGQ2}}"
+          model: "llama3"
+          chatId: "{{ chatId }}"
+        capture:
+          - json: "$.chat.id"
+            as: "chatId"
+        expect:
+          - statusCode: 200
\ No newline at end of file
+    - think : "{{$processEnvironment.SLEEP}}"
+    - delete:
+        url: '/auth'
+        expect:
+          - statusCode: 200        
\ No newline at end of file

__tests__/prompting_intrinsic.yaml

-config:
-  # This is a test server run by team Artillery
-  # It's designed to be highly scalable
-  target: "{{ $processEnvironment.TARGET}}"
-  processor: "../helper.js"
-  timeout: 300
-  phases:
-    - duration: 10
-      arrivalRate: 1
-      #rampTo: 2
-      name: Warm up phase
-    # - duration: 2
-    #   arrivalRate: 2
-    #   rampTo: 3
-    #   name: Ramp up load
-    # - duration: 1
-    #   arrivalRate: 3
-    #   rampTo: 4
-    #   name: Spike phase
-  # Load & configure a couple of useful plugins
-  # https://docs.art/reference/extensions
-  plugins:
-    ensure: {}
-    apdex: {}
-    metrics-by-endpoint: {}
-    fake-data: {}
-  apdex:
-  # define how many milliseconds is considered satisfactory
-    threshold: 1200
-  ensure:
-    thresholds:
-    # p99 of response time must be lower
-      - http.response_time.p99: 1200
-    # p95 of response time must be lower
-      - http.response_time.p95: 1100
-scenarios:
-  - flow:
-    - function: "createUser"
-    - post:
-        url: '/users/signup'
-        json:
-          name: "{{name}}"
-          username: "{{username}}"
-          email: "{{email}}"
-          password: "{{password}}"
-          passwordConfirm: "{{password}}"
-    - log: "Created user: {{username}}"
-    - think : "{{$processEnvironment.SLEEP}}"
-    - post:
-       url: '/users/login'
-       json:
-         email: "{{email}}"
-         password: "{{password}}"
-       capture:
-         - json: "$.token"
-           as: "JWT"
-    - log: "{{username}} fetched JWT {{JWT}}"
-    - think : "{{$processEnvironment.SLEEP}}"
-    - post:
-       url: '/ai/chat'
-       headers:
-         Authorization: "Bearer {{ JWT }}"
-       json:
-         input: "{{$processEnvironment.INTQ1}}"
-         model: "llama3"
-       capture:
-         - json: "$.chat.id"
-           as: "chatId"
-    # - think : "{{$processEnvironment.SLEEP}}"
-    # - post:
-    #    url: '/ai/chat'
-    #    headers:
-    #      Authorization: "Bearer {{ JWT }}"
-    #    json:
-    #      input: "{{$processEnvironment.INTQ2}}"
-    #      model: "llama3"
-    #      chatId: "{{ chatId }}"
-    #    capture:
-    #      - json: "$.chat.id"
-    #        as: "chatId"
-    #- think : "{{$processEnvironment.SLEEP}}"
-    #- get:
-    #    url: '/users/logout'
-#
\ No newline at end of file

helper.js

@@ -6,7 +6,7 @@ module.exports = {
 
 function createUser(userContext, events, done) {
   userContext.vars.name = faker.person.fullName();
-  userContext.vars.username = faker.internet.userName();
+  userContext.vars.username = `artillery_${faker.internet.userName()}`;
   userContext.vars.email = faker.internet.email();
   userContext.vars.password = faker.internet.password({ length: 16, prefix: '#8aZ_' });
   return done();

package.json

@@ -4,9 +4,7 @@
   "description": "performance tests for RAGChat API",
   "main": "server.js",
   "scripts": {
-    "online": "artillery run -k --dotenv ./.env ./__tests__/prompting.yaml --record --key a9_7yTC5bnk1Lgl94FtpIQgepfXzfgw-ApL",
-    "start": "artillery run -k --dotenv ./.env ./__tests__/prompting.yaml --output ./reports/prompting.json ; artillery report --output ./reports/prompting.html ./reports/prompting.json",
-    "intrinsic": "artillery run -k --dotenv ./.env ./__tests__/prompting_intrinsic.yaml --output ./reports/prompting_intrinsic.json ; artillery report --output ./reports/prompting_intrinsic.html ./reports/prompting_intrinsic.json"
+    "start": "artillery run -k --dotenv ./.env ./__tests__/prompting.yaml --tags Cores:8,Host:Node045,Questions:RAG,Storage:SSD,DB:central,Artillery:Node044,OllamaParallel:8 --record"
   },
   "author": "",
   "license": "ISC",