diff --git a/.env.template b/.env.template index 11344d0a99a41c1588edcfc2b225c11dbf78fbb2..d50a9c85b248117eab5c0a7fba4c71f749e2677a 100644 --- a/.env.template +++ b/.env.template @@ -8,7 +8,8 @@ TARGET=https://localhost:8080 # TESTING ########## # API key from https://app.artillery.io/ -# ARTILLERY_CLOUD_API_KEY=a9_7GibberISHw-ApL +# https://www.artillery.io/docs/get-started/artillery-cloud#configure-the-cli-to-send-data-to-artillery-cloud +ARTILLERY_CLOUD_API_KEY=a9_7GibberISHw-ApL # seconds to wait between each http request SLEEP=2 diff --git a/README.md b/README.md index c904f32d27654919b46782b4e214fcad93740959..3a3fbe4671dead333fcd327fa0ff62b801282511 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,8 @@ cd ragchat-artillery npm i cp ./.env.template ./.env # fill envs with production and/or devel values +# evtl. edit tags on package.json => scripts => start +npm start ``` @@ -24,5 +26,3 @@ cp ./.env.template ./.env - [PM2](https://pm2.keymetrics.io/) # Roadmap -- [ ] create a artillery script to test login & fetch jwt -- [ ] extend script to use JWT to send a prompt diff --git a/__tests__/prompting.yaml b/__tests__/prompting.yaml index a661f956a57906eba7523682ac9abed9cdf60c48..641077469901289dd53f3d97db114d510213cfbe 100644 --- a/__tests__/prompting.yaml +++ b/__tests__/prompting.yaml @@ -1,22 +1,23 @@ config: # This is a test server run by team Artillery # It's designed to be highly scalable + name: "WSL - 4 Cores" target: "{{ $processEnvironment.TARGET}}" processor: "../helper.js" timeout: 300 phases: - duration: 10 arrivalRate: 1 - #rampTo: 2 + rampTo: 2 name: Warm up phase - # - duration: 2 - # arrivalRate: 2 - # rampTo: 3 - # name: Ramp up load - # - duration: 1 - # arrivalRate: 3 - # rampTo: 4 - # name: Spike phase + - duration: 20 + arrivalRate: 2 + rampTo: 3 + name: Ramp up load + - duration: 10 + arrivalRate: 3 + rampTo: 4 + name: Spike phase # Load & configure a couple of useful plugins # https://docs.art/reference/extensions plugins: @@ -24,61 +25,71 @@ config: apdex: {} metrics-by-endpoint: {} fake-data: {} + expect: {} apdex: # define how many milliseconds is considered satisfactory - threshold: 1200 + threshold: 3000 ensure: thresholds: - # p99 of response time must be lower - - http.response_time.p99: 1200 - # p95 of response time must be lower - - http.response_time.p95: 1100 + # 99% of the responses are expected to be faster than this value (in ms) + - http.response_time.p99: 5000 + # 95% of the responses are expected to be faster than this value (in ms) + - http.response_time.p95: 4000 scenarios: - flow: - function: "createUser" - post: - url: '/users/signup' + url: '/users' json: name: "{{name}}" username: "{{username}}" email: "{{email}}" password: "{{password}}" - passwordConfirm: "{{password}}" + confirmPassword: "{{password}}" + expect: + - statusCode: 201 - log: "Created user: {{username}}" - think : "{{$processEnvironment.SLEEP}}" - post: - url: '/users/login' - json: - email: "{{email}}" - password: "{{password}}" - capture: - - json: "$.token" - as: "JWT" + url: '/auth/login' + json: + email: "{{email}}" + password: "{{password}}" + capture: + - json: "$.accessToken" + as: "JWT" + expect: + - statusCode: 200 - log: "{{username}} fetched JWT {{JWT}}" - think : "{{$processEnvironment.SLEEP}}" - post: - url: '/ai/chat' - headers: - Authorization: "Bearer {{ JWT }}" - json: - input: "{{$processEnvironment.RAGQ1}}" - model: "llama3" - capture: - - json: "$.chat.id" - as: "chatId" - # - think : "{{$processEnvironment.SLEEP}}" - # - post: - # url: '/ai/chat' - # headers: - # Authorization: "Bearer {{ JWT }}" - # json: - # input: "{{$processEnvironment.RAGQ2}}" - # model: "llama3" - # chatId: "{{ chatId }}" - # capture: - # - json: "$.chat.id" - # as: "chatId" - #- think : "{{$processEnvironment.SLEEP}}" - #- get: - # url: '/users/logout' -# \ No newline at end of file + url: '/ai/chat' + headers: + Authorization: "Bearer {{ JWT }}" + json: + input: "{{$processEnvironment.RAGQ1}}" + model: "llama3" + capture: + - json: "$.chat.id" + as: "chatId" + expect: + - statusCode: 200 + - think : "{{$processEnvironment.SLEEP}}" + - post: + url: '/ai/chat' + headers: + Authorization: "Bearer {{ JWT }}" + json: + input: "{{$processEnvironment.RAGQ2}}" + model: "llama3" + chatId: "{{ chatId }}" + capture: + - json: "$.chat.id" + as: "chatId" + expect: + - statusCode: 200 + - think : "{{$processEnvironment.SLEEP}}" + - delete: + url: '/auth' + expect: + - statusCode: 200 \ No newline at end of file diff --git a/__tests__/prompting_intrinsic.yaml b/__tests__/prompting_intrinsic.yaml deleted file mode 100644 index e4565a970dbd0b1daa662cde575ef465935ad536..0000000000000000000000000000000000000000 --- a/__tests__/prompting_intrinsic.yaml +++ /dev/null @@ -1,84 +0,0 @@ -config: - # This is a test server run by team Artillery - # It's designed to be highly scalable - target: "{{ $processEnvironment.TARGET}}" - processor: "../helper.js" - timeout: 300 - phases: - - duration: 10 - arrivalRate: 1 - #rampTo: 2 - name: Warm up phase - # - duration: 2 - # arrivalRate: 2 - # rampTo: 3 - # name: Ramp up load - # - duration: 1 - # arrivalRate: 3 - # rampTo: 4 - # name: Spike phase - # Load & configure a couple of useful plugins - # https://docs.art/reference/extensions - plugins: - ensure: {} - apdex: {} - metrics-by-endpoint: {} - fake-data: {} - apdex: - # define how many milliseconds is considered satisfactory - threshold: 1200 - ensure: - thresholds: - # p99 of response time must be lower - - http.response_time.p99: 1200 - # p95 of response time must be lower - - http.response_time.p95: 1100 -scenarios: - - flow: - - function: "createUser" - - post: - url: '/users/signup' - json: - name: "{{name}}" - username: "{{username}}" - email: "{{email}}" - password: "{{password}}" - passwordConfirm: "{{password}}" - - log: "Created user: {{username}}" - - think : "{{$processEnvironment.SLEEP}}" - - post: - url: '/users/login' - json: - email: "{{email}}" - password: "{{password}}" - capture: - - json: "$.token" - as: "JWT" - - log: "{{username}} fetched JWT {{JWT}}" - - think : "{{$processEnvironment.SLEEP}}" - - post: - url: '/ai/chat' - headers: - Authorization: "Bearer {{ JWT }}" - json: - input: "{{$processEnvironment.INTQ1}}" - model: "llama3" - capture: - - json: "$.chat.id" - as: "chatId" - # - think : "{{$processEnvironment.SLEEP}}" - # - post: - # url: '/ai/chat' - # headers: - # Authorization: "Bearer {{ JWT }}" - # json: - # input: "{{$processEnvironment.INTQ2}}" - # model: "llama3" - # chatId: "{{ chatId }}" - # capture: - # - json: "$.chat.id" - # as: "chatId" - #- think : "{{$processEnvironment.SLEEP}}" - #- get: - # url: '/users/logout' -# \ No newline at end of file diff --git a/helper.js b/helper.js index 15f08c4471590a73065c46d48d22c36a493d8559..a5b4c5319bf72c7e85bde15da278ec8dab5e6a92 100644 --- a/helper.js +++ b/helper.js @@ -6,7 +6,7 @@ module.exports = { function createUser(userContext, events, done) { userContext.vars.name = faker.person.fullName(); - userContext.vars.username = faker.internet.userName(); + userContext.vars.username = `artillery_${faker.internet.userName()}`; userContext.vars.email = faker.internet.email(); userContext.vars.password = faker.internet.password({ length: 16, prefix: '#8aZ_' }); return done(); diff --git a/package.json b/package.json index 84bfaaf6533f83e42ff8b4901055ca9906d71f12..0c7a95f3ffc330ed09d4abff9319bae637963533 100644 --- a/package.json +++ b/package.json @@ -4,9 +4,7 @@ "description": "performance tests for RAGChat API", "main": "server.js", "scripts": { - "online": "artillery run -k --dotenv ./.env ./__tests__/prompting.yaml --record --key a9_7yTC5bnk1Lgl94FtpIQgepfXzfgw-ApL", - "start": "artillery run -k --dotenv ./.env ./__tests__/prompting.yaml --output ./reports/prompting.json ; artillery report --output ./reports/prompting.html ./reports/prompting.json", - "intrinsic": "artillery run -k --dotenv ./.env ./__tests__/prompting_intrinsic.yaml --output ./reports/prompting_intrinsic.json ; artillery report --output ./reports/prompting_intrinsic.html ./reports/prompting_intrinsic.json" + "start": "artillery run -k --dotenv ./.env ./__tests__/prompting.yaml --tags Cores:8,Host:Node045,Questions:RAG,Storage:SSD,DB:central,Artillery:Node044,OllamaParallel:8 --record" }, "author": "", "license": "ISC", diff --git a/reports/.gitkeep b/reports/.gitkeep deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000