diff --git a/.env.template b/.env.template index 805c8c0cb967a1002ef82b10b16a1bb29d395033..11344d0a99a41c1588edcfc2b225c11dbf78fbb2 100644 --- a/.env.template +++ b/.env.template @@ -1,3 +1,22 @@ +########## +# BACKEND SERVER +########## +# where the backend server is running TARGET=https://localhost:8080 -EMAIL=valid@user.mail -PASSWORD=superSecret123! + +########## +# TESTING +########## +# API key from https://app.artillery.io/ +# ARTILLERY_CLOUD_API_KEY=a9_7GibberISHw-ApL + +# seconds to wait between each http request +SLEEP=2 + +# question and follow up question, which can be answered with RAG only +RAGQ1="Under what path could members of the working group can find the exam git directory?" +RAGQ2="What else can be found under that path?" + +# question and follow up question, which can be answered by the trained model itself +INTQ1="John has five apples. He eats one of them himself. How many apples does John have now?" +INTQ2="Now John gives one apple to his sister. How many apples does John have now?" diff --git a/README.md b/README.md index 8f0bba9d534dbf925e16b818b39e4882b1180ca8..c904f32d27654919b46782b4e214fcad93740959 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,5 @@ cp ./.env.template ./.env - [PM2](https://pm2.keymetrics.io/) # Roadmap -- [ ] fetch existing api account from .env - [ ] create a artillery script to test login & fetch jwt - [ ] extend script to use JWT to send a prompt diff --git a/__tests__/prompting.yaml b/__tests__/prompting.yaml index 1bc74bf5053f7466a3a2fa7c92a9cf7c3400f9ad..a661f956a57906eba7523682ac9abed9cdf60c48 100644 --- a/__tests__/prompting.yaml +++ b/__tests__/prompting.yaml @@ -2,41 +2,83 @@ config: # This is a test server run by team Artillery # It's designed to be highly scalable target: "{{ $processEnvironment.TARGET}}" + processor: "../helper.js" + timeout: 300 phases: - - duration: 1 + - duration: 10 arrivalRate: 1 - rampTo: 2 + #rampTo: 2 name: Warm up phase - - duration: 2 - arrivalRate: 2 - rampTo: 3 - name: Ramp up load - - duration: 1 - arrivalRate: 3 - rampTo: 4 - name: Spike phase + # - duration: 2 + # arrivalRate: 2 + # rampTo: 3 + # name: Ramp up load + # - duration: 1 + # arrivalRate: 3 + # rampTo: 4 + # name: Spike phase # Load & configure a couple of useful plugins # https://docs.art/reference/extensions plugins: ensure: {} apdex: {} metrics-by-endpoint: {} + fake-data: {} apdex: - threshold: 100 + # define how many milliseconds is considered satisfactory + threshold: 1200 ensure: thresholds: - - http.response_time.p99: 100 - - http.response_time.p95: 75 + # p99 of response time must be lower + - http.response_time.p99: 1200 + # p95 of response time must be lower + - http.response_time.p95: 1100 scenarios: - flow: - - loop: - - post: - url: '/users/login' - json: - email: "{{$processEnvironment.EMAIL}}" - password: "{{$processEnvironment.PASSWORD}}" - - get: - url: '/users/logout' - # - get: - # url: '/armadillo' - count: 1 \ No newline at end of file + - function: "createUser" + - post: + url: '/users/signup' + json: + name: "{{name}}" + username: "{{username}}" + email: "{{email}}" + password: "{{password}}" + passwordConfirm: "{{password}}" + - log: "Created user: {{username}}" + - think : "{{$processEnvironment.SLEEP}}" + - post: + url: '/users/login' + json: + email: "{{email}}" + password: "{{password}}" + capture: + - json: "$.token" + as: "JWT" + - log: "{{username}} fetched JWT {{JWT}}" + - think : "{{$processEnvironment.SLEEP}}" + - post: + url: '/ai/chat' + headers: + Authorization: "Bearer {{ JWT }}" + json: + input: "{{$processEnvironment.RAGQ1}}" + model: "llama3" + capture: + - json: "$.chat.id" + as: "chatId" + # - think : "{{$processEnvironment.SLEEP}}" + # - post: + # url: '/ai/chat' + # headers: + # Authorization: "Bearer {{ JWT }}" + # json: + # input: "{{$processEnvironment.RAGQ2}}" + # model: "llama3" + # chatId: "{{ chatId }}" + # capture: + # - json: "$.chat.id" + # as: "chatId" + #- think : "{{$processEnvironment.SLEEP}}" + #- get: + # url: '/users/logout' +# \ No newline at end of file diff --git a/__tests__/prompting_intrinsic.yaml b/__tests__/prompting_intrinsic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e4565a970dbd0b1daa662cde575ef465935ad536 --- /dev/null +++ b/__tests__/prompting_intrinsic.yaml @@ -0,0 +1,84 @@ +config: + # This is a test server run by team Artillery + # It's designed to be highly scalable + target: "{{ $processEnvironment.TARGET}}" + processor: "../helper.js" + timeout: 300 + phases: + - duration: 10 + arrivalRate: 1 + #rampTo: 2 + name: Warm up phase + # - duration: 2 + # arrivalRate: 2 + # rampTo: 3 + # name: Ramp up load + # - duration: 1 + # arrivalRate: 3 + # rampTo: 4 + # name: Spike phase + # Load & configure a couple of useful plugins + # https://docs.art/reference/extensions + plugins: + ensure: {} + apdex: {} + metrics-by-endpoint: {} + fake-data: {} + apdex: + # define how many milliseconds is considered satisfactory + threshold: 1200 + ensure: + thresholds: + # p99 of response time must be lower + - http.response_time.p99: 1200 + # p95 of response time must be lower + - http.response_time.p95: 1100 +scenarios: + - flow: + - function: "createUser" + - post: + url: '/users/signup' + json: + name: "{{name}}" + username: "{{username}}" + email: "{{email}}" + password: "{{password}}" + passwordConfirm: "{{password}}" + - log: "Created user: {{username}}" + - think : "{{$processEnvironment.SLEEP}}" + - post: + url: '/users/login' + json: + email: "{{email}}" + password: "{{password}}" + capture: + - json: "$.token" + as: "JWT" + - log: "{{username}} fetched JWT {{JWT}}" + - think : "{{$processEnvironment.SLEEP}}" + - post: + url: '/ai/chat' + headers: + Authorization: "Bearer {{ JWT }}" + json: + input: "{{$processEnvironment.INTQ1}}" + model: "llama3" + capture: + - json: "$.chat.id" + as: "chatId" + # - think : "{{$processEnvironment.SLEEP}}" + # - post: + # url: '/ai/chat' + # headers: + # Authorization: "Bearer {{ JWT }}" + # json: + # input: "{{$processEnvironment.INTQ2}}" + # model: "llama3" + # chatId: "{{ chatId }}" + # capture: + # - json: "$.chat.id" + # as: "chatId" + #- think : "{{$processEnvironment.SLEEP}}" + #- get: + # url: '/users/logout' +# \ No newline at end of file diff --git a/helper.js b/helper.js new file mode 100644 index 0000000000000000000000000000000000000000..15f08c4471590a73065c46d48d22c36a493d8559 --- /dev/null +++ b/helper.js @@ -0,0 +1,13 @@ +const { faker } = require('@faker-js/faker'); + +module.exports = { + createUser +}; + +function createUser(userContext, events, done) { + userContext.vars.name = faker.person.fullName(); + userContext.vars.username = faker.internet.userName(); + userContext.vars.email = faker.internet.email(); + userContext.vars.password = faker.internet.password({ length: 16, prefix: '#8aZ_' }); + return done(); +}; \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index adbd85ac9ce6bb4987a245b578a7ea80f3d65c93..bec464f727b53e8948b45c3a3398ef9a2a336a38 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,6 +8,9 @@ "name": "ragchat-artillery", "version": "1.0.0", "license": "ISC", + "dependencies": { + "@faker-js/faker": "^8.4.1" + }, "devDependencies": { "artillery": "^2.0.17" } @@ -967,6 +970,22 @@ "node": ">=14" } }, + "node_modules/@faker-js/faker": { + "version": "8.4.1", + "resolved": "https://registry.npmjs.org/@faker-js/faker/-/faker-8.4.1.tgz", + "integrity": "sha512-XQ3cU+Q8Uqmrbf2e0cIC/QN43sTBSC8KF12u29Mb47tWrt2hAgBXSgpZMj4Ao8Uk0iJcU99QsOCaIL8934obCg==", + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/fakerjs" + } + ], + "license": "MIT", + "engines": { + "node": "^14.17.0 || ^16.13.0 || >=18.0.0", + "npm": ">=6.14.13" + } + }, "node_modules/@grpc/grpc-js": { "version": "1.11.0", "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.11.0.tgz", diff --git a/package.json b/package.json index 0ffa3003a1c3254a0f7bad0a9988be57778eb6ba..84bfaaf6533f83e42ff8b4901055ca9906d71f12 100644 --- a/package.json +++ b/package.json @@ -4,11 +4,16 @@ "description": "performance tests for RAGChat API", "main": "server.js", "scripts": { - "start": "artillery run -k --dotenv ./.env ./__tests__/prompting.yaml --output ./reports/prompting.json ; artillery report --output ./reports/prompting.html ./reports/prompting.json" + "online": "artillery run -k --dotenv ./.env ./__tests__/prompting.yaml --record --key a9_7yTC5bnk1Lgl94FtpIQgepfXzfgw-ApL", + "start": "artillery run -k --dotenv ./.env ./__tests__/prompting.yaml --output ./reports/prompting.json ; artillery report --output ./reports/prompting.html ./reports/prompting.json", + "intrinsic": "artillery run -k --dotenv ./.env ./__tests__/prompting_intrinsic.yaml --output ./reports/prompting_intrinsic.json ; artillery report --output ./reports/prompting_intrinsic.html ./reports/prompting_intrinsic.json" }, "author": "", "license": "ISC", "devDependencies": { "artillery": "^2.0.17" + }, + "dependencies": { + "@faker-js/faker": "^8.4.1" } }