From 687b313ae0b59215ae0f233092685de84f46fa34 Mon Sep 17 00:00:00 2001 From: neagualexa Date: Wed, 22 Jan 2025 11:39:46 +0000 Subject: [PATCH 01/57] initial boilerplate code with helpers --- .dockerignore | 160 ++++++ .env.example | 22 + .github/workflows/dev.yml | 81 +++ .github/workflows/main.yml | 81 +++ .github/workflows/test-report.yml | 26 + .gitignore | 138 ++++++ Dockerfile | 25 + README.md | 190 ++++++++ config.json | 3 + index.py | 41 ++ index_test.py | 60 +++ requirements.txt | 13 + src/__init__.py | 0 src/agents/__init__.py | 0 src/agents/base_agent/base_agent.py | 201 ++++++++ src/agents/base_agent/base_prompts.py | 77 +++ src/agents/llm_factory.py | 83 ++++ src/agents/student_agent/student_agent.py | 145 ++++++ src/agents/student_agent/student_prompts.py | 17 + .../utils/example_inputs/example_input_1.json | 168 +++++++ .../utils/example_inputs/example_input_2.json | 143 ++++++ .../utils/example_inputs/example_input_3.json | 460 ++++++++++++++++++ src/agents/utils/langgraph_viz.py | 13 + src/agents/utils/parse_json_to_prompt.py | 211 ++++++++ .../synthetic_conversation_generation.py | 132 +++++ .../utils/synthetic_conversations/NOTE.md | 4 + src/agents/utils/testbench_agents.py | 93 ++++ src/agents/utils/types.py | 3 + src/module.py | 92 ++++ src/module_response.py | 117 +++++ src/module_test.py | 100 ++++ 31 files changed, 2899 insertions(+) create mode 100644 .dockerignore create mode 100644 .env.example create mode 100644 .github/workflows/dev.yml create mode 100644 .github/workflows/main.yml create mode 100644 .github/workflows/test-report.yml create mode 100644 .gitignore create mode 100755 Dockerfile create mode 100755 README.md create mode 100644 config.json create mode 100644 index.py create mode 100644 index_test.py create mode 100644 requirements.txt create mode 100644 src/__init__.py create mode 100644 src/agents/__init__.py create mode 100644 src/agents/base_agent/base_agent.py create mode 100644 src/agents/base_agent/base_prompts.py create mode 100644 src/agents/llm_factory.py create mode 100644 src/agents/student_agent/student_agent.py create mode 100644 src/agents/student_agent/student_prompts.py create mode 100644 src/agents/utils/example_inputs/example_input_1.json create mode 100644 src/agents/utils/example_inputs/example_input_2.json create mode 100644 src/agents/utils/example_inputs/example_input_3.json create mode 100644 src/agents/utils/langgraph_viz.py create mode 100644 src/agents/utils/parse_json_to_prompt.py create mode 100644 src/agents/utils/synthetic_conversation_generation.py create mode 100644 src/agents/utils/synthetic_conversations/NOTE.md create mode 100644 src/agents/utils/testbench_agents.py create mode 100644 src/agents/utils/types.py create mode 100755 src/module.py create mode 100644 src/module_response.py create mode 100755 src/module_test.py diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..7a203b8 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,160 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# VSCode configuration +.vscode + +# Chat function config +config.json + +# README +README.md + +# GitHub +.github + +# Data folder +data/ + +# Test reports +reports/ + +# Synthetic data conversations +src/agents/utils/example_inputs/ +src/agents/utils/synthetic_conversations/ +src/agents/utils/synthetic_conversation_generation.py +src/agents/utils/testbench_prompts.py +src/agents/utils/langgraph_viz.py + +# development agents +src/agents/base_agent/ +src/agents/student_agent/ +src/agents/development_agents/ +src/agents/google_learnLM_agent/ \ No newline at end of file diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..707376f --- /dev/null +++ b/.env.example @@ -0,0 +1,22 @@ +OPENAI_API_KEY=test +OPENAI_MODEL=test + +# for use of googleai +GOOGLE_AI_API_KEY=test +GOOGLE_AI_MODEL=test + +# for use of azureopenai +AZURE_OPENAI_API_KEY=test +AZURE_OPENAI_ENDPOINT=test +AZURE_OPENAI_API_VERSION=test +AZURE_OPENAI_CHAT_DEPLOYMENT_NAME=test +AZURE_OPENAI_EMBEDDING_3072_DEPLOYMENT=test +AZURE_OPENAI_EMBEDDING_1536_DEPLOYMENT=test +AZURE_OPENAI_EMBEDDING_3072_MODEL=test +AZURE_OPENAI_EMBEDDING_1536_MODEL=test + +# used for langsmith for monitoring the requests sent to Azureopenai cloud service; +LANGCHAIN_TRACING_V2=true +LANGCHAIN_ENDPOINT="https://api.smith.langchain.com" +LANGCHAIN_API_KEY="" +LANGCHAIN_PROJECT="project" \ No newline at end of file diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml new file mode 100644 index 0000000..9027e4c --- /dev/null +++ b/.github/workflows/dev.yml @@ -0,0 +1,81 @@ +name: Dev deployment of chatbot lambda function + +on: + push: + branches: [dev] + +jobs: + test: + name: Dev deployment of chatbot lambda function + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.11"] + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + OPENAI_MODEL: ${{ vars.OPENAI_MODEL }} + steps: + - name: Checkout Code + uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + id: python-setup + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + # - name: Load cached venv + # id: dependencies-cache + # uses: actions/cache@v3 + # with: + # path: .venv + # key: venv-${{ runner.os }}-${{ steps.python-setup.outputs.python-version }} + + - name: Create Venv if Cache not found + # if: steps.dependencies-cache.outputs.cache-hit != 'true' + run: | + python -m venv .venv + + - name: Install dependencies + # if: steps.dependencies-cache.outputs.cache-hit != 'true' + run: | + pip install --upgrade pip + pip install -r requirements.txt + + - name: Run tests + if: always() + run: | + source .venv/bin/activate + pytest --junit-xml=./reports/pytest.xml --tb=auto -v + + - name: Upload test results + uses: actions/upload-artifact@v4 + if: always() + with: + name: test-results + path: ./reports/pytest.xml + if-no-files-found: warn + + build: + name: Build Docker Image + uses: lambda-feedback/chat-function-workflows/.github/workflows/gh_build.yml@main + needs: test + permissions: + contents: read + id-token: write + packages: write + + deploy: + name: Deploy to Lambda Feedback + uses: lambda-feedback/chat-function-workflows/.github/workflows/dev_deploy.yml@main + needs: test + with: + template-repository-name: "lambda-feedback/chat-function-boilerplate" + permissions: + contents: read + id-token: write + packages: write + secrets: + aws-access-key-id: ${{ secrets.LAMBDA_CONTAINER_PIPELINE_AWS_ID }} + aws-secret-key: ${{ secrets.LAMBDA_CONTAINER_PIPELINE_AWS_SECRET}} diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 0000000..4f3bd47 --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,81 @@ +name: Main deployment of Chatbot lambda function + +on: + push: + branches: [main] + +jobs: + test: + name: Staging deployment tests + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.11"] + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + OPENAI_MODEL: ${{ vars.OPENAI_MODEL }} + steps: + - name: Checkout Code + uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + id: python-setup + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + # - name: Load cached venv + # id: dependencies-cache + # uses: actions/cache@v3 + # with: + # path: .venv + # key: venv-${{ runner.os }}-${{ steps.python-setup.outputs.python-version }} + + - name: Create Venv if Cache not found + # if: steps.dependencies-cache.outputs.cache-hit != 'true' + run: | + python -m venv .venv + + - name: Install dependencies + # if: steps.dependencies-cache.outputs.cache-hit != 'true' + run: | + pip install --upgrade pip + pip install -r requirements.txt + + - name: Run tests + if: always() + run: | + source .venv/bin/activate + pytest --junit-xml=./reports/pytest.xml --tb=auto -v + + - name: Upload test results + uses: actions/upload-artifact@v4 + if: always() + with: + name: test-results + path: ./reports/pytest.xml + if-no-files-found: warn + + build: + name: Build Docker Image + uses: lambda-feedback/chat-function-workflows/.github/workflows/gh_build.yml@main + needs: test + permissions: + contents: read + id-token: write + packages: write + + deploy: + name: Deploy to Lambda Feedback + uses: lambda-feedback/chat-function-workflows/.github/workflows/main_deploy.yml@main + needs: test + with: + template-repository-name: "lambda-feedback/chat-function-boilerplate" + permissions: + contents: read + id-token: write + packages: write + secrets: + aws-access-key-id: ${{ secrets.LAMBDA_CONTAINER_PIPELINE_AWS_ID }} + aws-secret-key: ${{ secrets.LAMBDA_CONTAINER_PIPELINE_AWS_SECRET}} diff --git a/.github/workflows/test-report.yml b/.github/workflows/test-report.yml new file mode 100644 index 0000000..1d885ed --- /dev/null +++ b/.github/workflows/test-report.yml @@ -0,0 +1,26 @@ +name: Test Report + +on: + workflow_run: + workflows: ["Build, Test and Deploy"] + types: + - completed + +permissions: + contents: read + actions: read + checks: write + +jobs: + report: + runs-on: ubuntu-latest + steps: + - name: Test Report + uses: dorny/test-reporter@v1 + if: always() + with: + name: Pytest Report + artifact: test-results + path: '*.xml' + reporter: java-junit + fail-on-error: false \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4b52234 --- /dev/null +++ b/.gitignore @@ -0,0 +1,138 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# VSCode configuration +.vscode + +.DS_Store + +# Synthetic data conversations +src/agents/utils/synthetic_conversations/*.json +src/agents/utils/synthetic_conversations/*.csv \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100755 index 0000000..818f099 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,25 @@ +ARG PYTHON_VERSION=3.11 + +FROM public.ecr.aws/lambda/python:${PYTHON_VERSION} + +# Set working directory +WORKDIR ${LAMBDA_TASK_ROOT} + +RUN pip install --upgrade pip && yum install -y git + +# Install dependencies into the virtual environment +COPY requirements.txt . +RUN pip install -r requirements.txt + +# Precompile python files for faster startup +RUN python -m compileall -q . + +# Copy the function code +COPY src ./src + +COPY index.py . + +COPY index_test.py . + +# Set the Lambda function handler +CMD ["index.handler"] \ No newline at end of file diff --git a/README.md b/README.md new file mode 100755 index 0000000..dfddc23 --- /dev/null +++ b/README.md @@ -0,0 +1,190 @@ +# Lambda Feedback Chat Function Boilerplate + +This repository contains the code needed to develop a modular chatbot to be used on Lambda-Feedback platform [written in Python]. + +## Quickstart + +This chapter helps you to quickly set up a new Python chat module function using this repository. + +> [!NOTE] +> To develop this function further, you will require the following environment variables in your `.env` file: +```bash +> If you use azure-openai: +AZURE_OPENAI_API_KEY +AZURE_OPENAI_ENDPOINT +AZURE_OPENAI_API_VERSION +AZURE_OPENAI_CHAT_DEPLOYMENT_NAME +AZURE_OPENAI_EMBEDDING_3072_DEPLOYMENT +AZURE_OPENAI_EMBEDDING_1536_DEPLOYMENT +AZURE_OPENAI_EMBEDDING_3072_MODEL +AZURE_OPENAI_EMBEDDING_1536_MODEL + +> If you use openai: +OPENAI_API_KEY +OPENAI_MODEL + +> For monitoring of the LLM calls (follow instructions on how to set up on langsmith): +LANGCHAIN_TRACING_V2 +LANGCHAIN_ENDPOINT +LANGCHAIN_API_KEY +LANGCHAIN_PROJECT +``` + +#### 1. Create a new repository +In GitHub, choose Use this template > Create a new repository in the repository toolbar. + +Choose the owner, and pick a name for the new repository. + +> [!IMPORTANT] If you want to deploy the evaluation function to Lambda Feedback, make sure to choose the Lambda Feedback organization as the owner. + +Set the visibility to Public or Private. + +> [!IMPORTANT] If you want to use GitHub deployment protection rules, make sure to set the visibility to Public. + +Click on Create repository. + +#### 2. Clone the new repository +Clone the new repository to your local machine using the following command: + +```bash +git clone +``` + +#### 3. Develop the chat function + +You're ready to start developing your chat function. Head over to the [Development](#development) section to learn more. + +#### 4. Update the README + +In the `README.md` file, change the title and description so it fits the purpose of your chat function. + +Also, don't forget to update or delete the Quickstart chapter from the `README.md` file after you've completed these steps. + +## Development + +You can create your own invocation to your own agents hosted anywhere. Copy or update the `base_agent` from `src/agents/` and edit it to match your LLM agent requirements. Import the new invocation in the `module.py` file. + +You agent can be based on an LLM hosted anywhere, you have available currently OpenAI, AzureOpenAI, and Ollama models but you can introduce your own API call in the `src/agents/llm_factory.py`. + +### Prerequisites + +- [Docker](https://docs.docker.com/get-docker/) +- [Python](https://www.python.org) + +### Repository Structure + +```bash +.github/workflows/ + dev.yml # deploys the DEV function to Lambda Feedback + main.yml # deploys the STAGING function to Lambda Feedback + test-report.yml # gathers Pytest Report of function tests + +src/module.py # chat_module function implementation +src/module_test.py # chat_module function tests +src/agents/ # find all agents developed for the chat functionality +src/agents/utils/test_prompts.py # allows testing of any LLM agent on a couple of example inputs containing Lambda Feedback Questions and synthetic student conversations +``` + +## Run the Chat Script + +You can run the Python function itself. Make sure to have a main function in either `src/module.py` or `index.py`. + +```bash +python src/module.py +``` + +You can also use the `testbench_agents.py` script to test the agents with example inputs from Lambda Feedback questions and synthetic conversations. +```bash +python src/agents/utils/testbench_agents.py +``` + +### Building the Docker Image + +To build the Docker image, run the following command: + +```bash +docker build -t llm_chat . +``` + +### Running the Docker Image + +To run the Docker image, use the following command: + +#### Without .env file: + +```bash +docker run -e OPENAI_API_KEY={your key} -e OPENAI_MODEL={your LLM chosen model name} -p 8080:8080 llm_chat +``` + +#### With container name (for interaction, e.g. copying file from inside the docker container): + +```bash +docker run --env-file .env -it --name my-lambda-container -p 8080:8080 llm_chat +``` + +This will start the chat function and expose it on port `8080` and it will be open to be curl: + +```bash +curl --location 'http://localhost:8080/2015-03-31/functions/function/invocations' --header 'Content-Type: application/json' --data '{"message":"hi","params":{"conversation_id":"12345Test","conversation_history": [{"type":"user","content":"hi"}]}}' +``` + +### Call Docker Container From Postman + +POST URL: + +```bash +http://localhost:8080/2015-03-31/functions/function/invocations +``` + +Body: + +```JSON +{ + "message":"hi", + "params":{ + "conversation_id":"12345Test", + "conversation_history": [{"type":"user","content":"hi"}] + } +} +``` + +Body with optional Params: +```JSON +{ + "message":"hi", + "params":{ + "conversation_id":"12345Test", + "conversation_history":[{"type":"user","content":"hi"}], + "summary":" ", + "conversational_style":" ", + "question_response_details": "", + "include_test_data": true, + "agent_type": {agent_name} + } +} +``` + +### Deploy to Lambda Feedback + +Deploying the chat function to Lambda Feedback is simple and straightforward, as long as the repository is within the [Lambda Feedback organization](https://github.com/lambda-feedback). + +During development, we recommend using the **`dev`** branch. This branch will deploy a version of the function onto AWS using the [GitHub Actions Dev workflow](.github/workflows/dev.yml). After deploying, please, contact one of the Lambda Feedback admins to allow the function to be accessible onto `dev.lambdafeedback.com`. + +> [!WARNING] The dev environment of the platform is always under use, so the platform might have beta/in-testing features that might cause unexpected issues. + +After you are pleased with the performance of your Chatbot and have configured the repository, a [GitHub Actions workflow](.github/workflows/main.yml) will automatically build and deploy the chat function to Lambda Feedback as soon as changes are pushed to the main branch of the repository. This deployment will upload the function onto `staging.lambdafeedback.com`, and will also initiate an `approval` stage for prod environment. Once you reach this stage, please contact an admin from Lambda Feedback to review the code and approve it such that the code can be accessible onto the main [Lambda Feedback platform](https://www.lambdafeedback.com/). + +## Troubleshooting + +### Containerized Function Fails to Start + +If your chat function is working fine when run locally, but not when containerized, there is much more to consider. Here are some common issues and solution approaches: + +**Run-time dependencies** + +Make sure that all run-time dependencies are installed in the Docker image. + +- Python packages: Make sure to add the dependency to the `requirements.txt` or `pyproject.toml` file, and run `pip install -r requirements.txt` or `poetry install` in the Dockerfile. +- System packages: If you need to install system packages, add the installation command to the Dockerfile. +- ML models: If your chat function depends on ML models, make sure to include them in the Docker image. +- Data files: If your chat function depends on data files, make sure to include them in the Docker image. diff --git a/config.json b/config.json new file mode 100644 index 0000000..a9f3255 --- /dev/null +++ b/config.json @@ -0,0 +1,3 @@ +{ + "ChatFunctionName": "" +} diff --git a/index.py b/index.py new file mode 100644 index 0000000..9b4b6a6 --- /dev/null +++ b/index.py @@ -0,0 +1,41 @@ +try: + from .src.module import chat_module +except ImportError: + from src.module import chat_module + +def handler(event, context): + """ + Lambda handler function + """ + # Log the input event for debugging purposes + # print("Received event:", json.dumps(event, indent=2)) + + if "message" not in event: + return { + "statusCode": 400, + "body": "Missing 'message' key in event. Please confirm the key in the json body." + } + if "params" not in event: + return { + "statusCode": 400, + "body": "Missing 'params' key in event. Please confirm the key in the json body. Make sure it contains the necessary conversation_id." + } + + message = event.get("message", None) + params = event.get("params", None) + + try: + chatbot_response = chat_module(message, params) + except Exception as e: + return { + "statusCode": 500, + "body": f"An error occurred within the chat_module(): {str(e)}" + } + + # Create a response + response = { + "statusCode": 200, + "body": chatbot_response + } + + return response \ No newline at end of file diff --git a/index_test.py b/index_test.py new file mode 100644 index 0000000..6f9025a --- /dev/null +++ b/index_test.py @@ -0,0 +1,60 @@ +import unittest + +try: + from .index import handler +except ImportError: + from index import handler + +class TestChatIndexFunction(unittest.TestCase): + """ + TestCase Class used to test the algorithm. + --- + Tests are used here to check that the algorithm written + is working as it should. + + It's best practise to write these tests first to get a + kind of 'specification' for how your algorithm should + work, and you should run these tests before committing + your code to AWS. + + Read the docs on how to use unittest here: + https://docs.python.org/3/library/unittest.html + + Use module() to check your algorithm works + as it should. + """ + + def test_missing_argument(self): + arguments = ["message", "params"] + + for arg in arguments: + event = { + "message": "Hello, World", + "params": {"conversation_id": "1234Test", "conversation_history": [{"type": "user", "content": "Hello, World"}]} + } + event.pop(arg) + + result = handler(event, None) + + self.assertEqual(result.get("statusCode"), 400) + + def test_correct_arguments(self): + event = { + "message": "Hello, World", + "params": {"conversation_id": "1234Test", "conversation_history": [{"type": "user", "content": "Hello, World"}]} + } + + result = handler(event, None) + + self.assertEqual(result.get("statusCode"), 200) + + def test_correct_response(self): + event = { + "message": "Hello, World", + "params": {"conversation_id": "1234Test", "conversation_history": [{"type": "user", "content": "Hello, World"}]} + } + + result = handler(event, None) + + self.assertEqual(result.get("statusCode"), 200) + \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..cd0298c --- /dev/null +++ b/requirements.txt @@ -0,0 +1,13 @@ +langchain +langchain-chroma +langchain-community +langchain-core +langchain-openai +langchain_google_genai +langchain-text-splitters +langchainhub +langdetect +langgraph +langsmith + +pytest \ No newline at end of file diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/agents/__init__.py b/src/agents/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/agents/base_agent/base_agent.py b/src/agents/base_agent/base_agent.py new file mode 100644 index 0000000..35e09d4 --- /dev/null +++ b/src/agents/base_agent/base_agent.py @@ -0,0 +1,201 @@ +try: + from ..llm_factory import OpenAILLMs + from .base_prompts import \ + role_prompt, conv_pref_prompt, update_conv_pref_prompt, summary_prompt, update_summary_prompt, summary_system_prompt + from ..utils.types import InvokeAgentResponseType +except ImportError: + from src.agents.llm_factory import OpenAILLMs + from src.agents.base_agent.base_prompts import \ + role_prompt, conv_pref_prompt, update_conv_pref_prompt, summary_prompt, update_summary_prompt, summary_system_prompt + from src.agents.utils.types import InvokeAgentResponseType + +from langgraph.graph import StateGraph, START, END +from langchain_core.messages import SystemMessage, RemoveMessage, HumanMessage, AIMessage +from langchain_core.runnables.config import RunnableConfig +from langgraph.graph.message import add_messages +from typing import Annotated, TypeAlias +from typing_extensions import TypedDict + +""" +Base agent for development [LLM workflow with a summarisation, profiling, and chat agent that receives an external conversation history]. + +This agent is designed to: +- [summarise_prompt] summarise the conversation after 'max_messages_to_summarize' number of messages is reached in the conversation +- [conv_pref_prompt] analyse the conversation style of the student +- [role_prompt] role of a tutor to answer student's questions on the topic +""" + +ValidMessageTypes: TypeAlias = SystemMessage | HumanMessage | AIMessage +AllMessageTypes: TypeAlias = ValidMessageTypes | RemoveMessage + +class State(TypedDict): + messages: Annotated[list[AllMessageTypes], add_messages] + summary: str + conversationalStyle: str + +class BaseAgent: + def __init__(self): + llm = OpenAILLMs() + self.llm = llm.get_llm() + summarisation_llm = OpenAILLMs() + self.summarisation_llm = summarisation_llm.get_llm() + self.summary = "" + self.conversationalStyle = "" + + # Define Agent's specific Parameters + self.max_messages_to_summarize = 11 + self.role_prompt = role_prompt + self.summary_prompt = summary_prompt + self.update_summary_prompt = update_summary_prompt + self.conversation_preference_prompt = conv_pref_prompt + self.update_conversation_preference_prompt = update_conv_pref_prompt + + # Define a new graph for the conversation & compile it + self.workflow = StateGraph(State) + self.workflow_definition() + self.app = self.workflow.compile() + + def call_model(self, state: State, config: RunnableConfig) -> str: + """Call the LLM model knowing the role system prompt, the summary and the conversational style.""" + + # Default AI tutor role prompt + system_message = self.role_prompt + + # Adding external student progress and question context details from data queries + question_response_details = config["configurable"].get("question_response_details", "") + if question_response_details: + system_message += f"## Known Question Materials: {question_response_details} \n\n" + + # Adding summary and conversational style to the system message + summary = state.get("summary", "") + conversationalStyle = state.get("conversationalStyle", "") + if summary: + system_message += summary_system_prompt.format(summary=summary) + if conversationalStyle: + system_message += f"## Known conversational style and preferences of the student for this conversation: {conversationalStyle}. \n\nYour answer must be in line with this conversational style." + + messages = [SystemMessage(content=system_message)] + state['messages'] + + valid_messages = self.check_for_valid_messages(messages) + response = self.llm.invoke(valid_messages) + + # Save summary for fetching outside the class + self.summary = summary + self.conversationalStyle = conversationalStyle + + return {"summary": summary, "messages": [response]} + + def check_for_valid_messages(self, messages: list[AllMessageTypes]) -> list[ValidMessageTypes]: + """ Removing the RemoveMessage() from the list of messages """ + + valid_messages: list[ValidMessageTypes] = [] + for message in messages: + if message.type != 'remove': + valid_messages.append(message) + return valid_messages + + def summarize_conversation(self, state: State, config: RunnableConfig) -> dict: + """Summarize the conversation.""" + + summary = state.get("summary", "") + previous_summary = config["configurable"].get("summary", "") + previous_conversationalStyle = config["configurable"].get("conversational_style", "") + if previous_summary: + summary = previous_summary + + if summary: + summary_message = ( + f"This is summary of the conversation to date: {summary}\n\n" + + self.update_summary_prompt + ) + else: + summary_message = self.summary_prompt + + if previous_conversationalStyle: + conversationalStyle_message = ( + f"This is the previous conversational style of the student for this conversation: {previous_conversationalStyle}\n\n" + + self.update_conversation_preference_prompt + ) + else: + conversationalStyle_message = self.conversation_preference_prompt + + # STEP 1: Summarize the conversation + messages = state["messages"][:-1] + [SystemMessage(content=summary_message)] + valid_messages = self.check_for_valid_messages(messages) + summary_response = self.summarisation_llm.invoke(valid_messages) + + # STEP 2: Analyze the conversational style + messages = state["messages"][:-1] + [SystemMessage(content=conversationalStyle_message)] + valid_messages = self.check_for_valid_messages(messages) + conversationalStyle_response = self.summarisation_llm.invoke(valid_messages) + + # Delete messages that are no longer wanted, except the last ones + delete_messages: list[AllMessageTypes] = [RemoveMessage(id=m.id) for m in state["messages"][:-3]] + + return {"summary": summary_response.content, "conversationalStyle": conversationalStyle_response.content, "messages": delete_messages} + + def should_summarize(self, state: State) -> str: + """ + Return the next node to execute. + If there are more than X messages, then we summarize the conversation. + Otherwise, we call the LLM. + """ + + messages = state["messages"] + valid_messages = self.check_for_valid_messages(messages) + nr_messages = len(valid_messages) + if len(valid_messages) == 0: + raise Exception("Internal Error: No valid messages found in the conversation history. Conversation history might be empty.") + if "system" in valid_messages[-1].type: + nr_messages -= 1 + + # always pairs of (sent, response) + 1 latest message + if nr_messages > self.max_messages_to_summarize: + return "summarize_conversation" + return "call_llm" + + def workflow_definition(self) -> None: + self.workflow.add_node("call_llm", self.call_model) + self.workflow.add_node("summarize_conversation", self.summarize_conversation) + + self.workflow.add_conditional_edges(source=START, path=self.should_summarize) + self.workflow.add_edge("summarize_conversation", "call_llm") + self.workflow.add_edge("call_llm", END) + + def get_summary(self) -> str: + return self.summary + + def get_conversational_style(self) -> str: + return self.conversationalStyle + + def print_update(self, update: dict) -> None: + for k, v in update.items(): + for m in v["messages"]: + m.pretty_print() + if "summary" in v: + print(v["summary"]) + + def pretty_response_value(self, event: dict) -> str: + return event["messages"][-1].content + +agent = BaseAgent() +def invoke_base_agent(query: str, conversation_history: list, summary: str, conversationalStyle: str, question_response_details: str, session_id: str) -> InvokeAgentResponseType: + """ + Call an agent that has no conversation memory and expects to receive all past messages in the params and the latest human request in the query. + If conversation history longer than X, the agent will summarize the conversation and will provide a conversational style analysis. + """ + print(f'in invoke_base_agent(), query = {query}, thread_id = {session_id}') + + config = {"configurable": {"thread_id": session_id, "summary": summary, "conversational_style": conversationalStyle, "question_response_details": question_response_details}} + response_events = agent.app.invoke({"messages": conversation_history, "summary": summary, "conversational_style": conversationalStyle}, config=config, stream_mode="values") #updates + pretty_printed_response = agent.pretty_response_value(response_events) # get last event/ai answer in the response + + # Gather Metadata from the agent + summary = agent.get_summary() + conversationalStyle = agent.get_conversational_style() + + return { + "input": query, + "output": pretty_printed_response, + "intermediate_steps": [str(summary), conversationalStyle, conversation_history] + } \ No newline at end of file diff --git a/src/agents/base_agent/base_prompts.py b/src/agents/base_agent/base_prompts.py new file mode 100644 index 0000000..4606c59 --- /dev/null +++ b/src/agents/base_agent/base_prompts.py @@ -0,0 +1,77 @@ +# NOTE: +# PROMPTS generated with the help of ChatGPT GPT-4o Nov 2024 + +role_prompt = "You are an excellent tutor that aims to provide clear and concise explanations to students. I am the student. Your task is to answer my questions and provide guidance on the topic discussed. Ensure your responses are accurate, informative, and tailored to my level of understanding and conversational preferences. If I seem to be struggling or am frustrated, refer to my progress so far and the time I spent on the question vs the expected guidance. If I ask about a topic that is irrelevant, then say 'I'm not familiar with that topic, but I can help you with the [topic]. You do not need to end your messages with a concluding statement.\n\n" + +pref_guidelines = """**Guidelines:** +- Use concise, objective language. +- Note the student's educational goals, such as understanding foundational concepts, passing an exam, getting top marks, code implementation, hands-on practice, etc. +- Note any specific preferences in how the student learns, such as asking detailed questions, seeking practical examples, requesting quizes, requesting clarifications, etc. +- Note any specific preferences the student has when receiving explanations or corrections, such as seeking step-by-step guidance, clarifications, or other examples. +- Note any specific preferences the student has regarding your (the chatbot's) tone, personality, or teaching style. +- Avoid assumptions about motivation; observe only patterns evident in the conversation. +- If no particular preference is detectable, state "No preference observed." +""" + +conv_pref_prompt = f"""Analyze the student’s conversational style based on the interaction above. Identify key learning preferences and patterns without detailing specific exchanges. Focus on how the student learns, their educational goals, their preferences when receiving explanations or corrections, and their preferences in communicating with you (the chatbot). Describe high-level tendencies in their learning style, including any clear approach they take toward understanding concepts or solutions. + +{pref_guidelines} + +Examples: + +Example 1: +**Conversation:** +Student: "I understand that the derivative gives us the slope of a function, but what if we want to know the rate of change over an interval? Do we still use the derivative?" +AI: "Good question! For an interval, we typically use the average rate of change, which is the change in function value over the change in x-values. The derivative gives the instantaneous rate of change at a specific point." + +**Expected Answer:** +The student prefers in-depth conceptual understanding and asks thoughtful questions that differentiate between similar concepts. They seem comfortable discussing foundational ideas in calculus. + +Example 2: +**Conversation:** +Student: "I’m trying to solve this physics problem: if I throw a ball upwards at 10 m/s, how long will it take to reach the top? I thought I could just divide by gravity, but I’m not sure." +AI: "You're on the right track! Since acceleration due to gravity is 9.8 m/s², you can divide the initial velocity by gravity to find the time to reach the peak, which would be around 1.02 seconds." + +**Expected Answer:** +The student prefers practical problem-solving and is open to corrections. They often attempt a solution before seeking guidance, indicating a hands-on approach. + +Example 3: +**Conversation:** +Student: "Can you explain the difference between meiosis and mitosis? I know both involve cell division, but I’m confused about how they differ." +AI: "Certainly! Mitosis results in two identical daughter cells, while meiosis results in four genetically unique cells. Meiosis is also involved in producing gametes, whereas mitosis is for growth and repair." + +**Expected Answer:** +The student prefers clear, comparative explanations when learning complex biological processes. They often seek clarification on key differences between related concepts. + +Example 4: +**Conversation:** +Student: "I wrote this Python code to reverse a string, but it’s not working. Here’s what I tried: `for char in string: new_string = char + new_string`." +AI: "You’re close! Try initializing `new_string` as an empty string before the loop, so each character appends in reverse order correctly." + +**Expected Answer:** +The student prefers hands-on guidance with code, often sharing specific code snippets. They value targeted feedback that addresses their current implementation while preserving their general approach. + +""" + +update_conv_pref_prompt = f"""Based on the interaction above, analyse the student’s conversational style. Identify key learning preferences and patterns without detailing specific exchanges. Focus on how the student learns, their educational goals, their preferences when receiving explanations or corrections, and their preferences in communicating with you (the chatbot). Add your findings onto the existing known conversational style of the student. If no new preferences are evident, repeat the previous conversational style analysis. + +{pref_guidelines} +""" + +summary_guidelines = """Ensure the summary is: + +Concise: Keep the summary brief while including all essential information. +Structured: Organize the summary into sections such as 'Topics Discussed' and 'Top 3 Key Detailed Ideas'. +Neutral and Accurate: Avoid adding interpretations or opinions; focus only on the content shared. +When summarizing: If the conversation is technical, highlight significant concepts, solutions, and terminology. If context involves problem-solving, detail the problem and the steps or solutions provided. If the user asks for creative input, briefly describe the ideas presented. +Last messages: Include the most recent 4 messages to provide context for the summary. + +Provide the summary in a bulleted format for clarity. Avoid redundant details while preserving the core intent of the discussion.""" + +summary_prompt = f"""Summarize the conversation between a student and a tutor. Your summary should highlight the major topics discussed during the session, followed by a detailed recollection of the last five significant points or ideas. Ensure the summary flows smoothly to maintain the continuity of the discussion.""" + +update_summary_prompt = f"""Update the summary by taking into account the new messages above. + +{summary_guidelines}""" + +summary_system_prompt = "You are continuing a tutoring session with the student. Background context: {summary}. Use this context to inform your understanding but do not explicitly restate, refer to, or incorporate the details directly in your responses unless the user brings them up. Respond naturally to the user's current input, assuming prior knowledge from the summary." \ No newline at end of file diff --git a/src/agents/llm_factory.py b/src/agents/llm_factory.py new file mode 100644 index 0000000..68b6bf1 --- /dev/null +++ b/src/agents/llm_factory.py @@ -0,0 +1,83 @@ +import os + +from langchain_openai import AzureChatOpenAI +from langchain_openai import AzureOpenAIEmbeddings +from langchain_community.llms import Ollama +from langchain_community.embeddings import OllamaEmbeddings +from langchain_openai import ChatOpenAI +from langchain_openai import OpenAIEmbeddings +from langchain_google_genai import ChatGoogleGenerativeAI + +class AzureLLMs: + def __init__(self, temperature: int = 0): + self._azure_llm = AzureChatOpenAI( + openai_api_version=os.environ["AZURE_OPENAI_API_VERSION"], + azure_deployment=os.environ["AZURE_OPENAI_CHAT_DEPLOYMENT_NAME"], + temperature=temperature, + max_tokens=None, + ) + self._azure_embedding = AzureOpenAIEmbeddings(azure_deployment=os.environ['AZURE_OPENAI_EMBEDDING_1536_DEPLOYMENT'], + openai_api_version=os.environ["AZURE_OPENAI_API_VERSION"], + model=os.environ["AZURE_OPENAI_EMBEDDING_1536_MODEL"]) + + def get_llm(self): + return self._azure_llm + + def get_embedding(self): + return self._azure_embedding + +class OllamaLLMs: + def __init__(self): + self._ollama_llm = Ollama( + model=os.environ['OLLAMA_MODEL'], # Any of the available models listed in the API docs + base_url=os.environ['OLLAMA_BASE_URL'], + headers={ + 'X-API-Key': os.environ['OLLAMA_API_KEY'], + }, + ) + + self._ollama_embedding = OllamaEmbeddings( + model='nomic-embed-text:137m-v1.5-fp16', + base_url=os.environ['OLLAMA_BASE_URL'], + headers={ + 'X-API-Key': os.environ['OLLAMA_API_KEY'], + }, + show_progress=True + ) + + def get_llm(self): + return self._ollama_llm + + def get_embedding(self): + return self._ollama_embedding + +class OpenAILLMs: + def __init__(self, temperature: int = 0): + self._openai_llm = ChatOpenAI( + model=os.environ['OPENAI_MODEL'], + temperature=temperature, + api_key=os.environ["OPENAI_API_KEY"], + ) + + self._openai_embedding = OpenAIEmbeddings( + model='text-embedding-ada-002', + api_key=os.environ['OPENAI_API_KEY'], + ) + + def get_llm(self): + return self._openai_llm + + def get_embedding(self): + return self._openai_embedding + +class GoogleAILLMs: + def __init__(self, temperature: int = 0): + + self._google_llm = ChatGoogleGenerativeAI( + model=os.environ['GOOGLE_AI_MODEL'], + temperature=temperature, + google_api_key=os.environ['GOOGLE_AI_API_KEY'], + ) + + def get_llm(self): + return self._google_llm \ No newline at end of file diff --git a/src/agents/student_agent/student_agent.py b/src/agents/student_agent/student_agent.py new file mode 100644 index 0000000..4c5f54e --- /dev/null +++ b/src/agents/student_agent/student_agent.py @@ -0,0 +1,145 @@ +try: + from ..llm_factory import OpenAILLMs + from .student_prompts import \ + base_student_persona, curious_student_persona, contradicting_student_persona, reliant_student_persona, confused_student_persona, unrelated_student_persona, \ + process_prompt + from ..utils.types import InvokeAgentResponseType +except ImportError: + from src.agents.llm_factory import OpenAILLMs + from src.agents.student_agent.student_prompts import \ + base_student_persona, curious_student_persona, contradicting_student_persona, reliant_student_persona, confused_student_persona, unrelated_student_persona, \ + process_prompt + from src.agents.utils.types import InvokeAgentResponseType + +from langgraph.graph import StateGraph, START, END +from langchain_core.messages import SystemMessage, RemoveMessage, HumanMessage, AIMessage +from langchain_core.runnables.config import RunnableConfig +from langgraph.graph.message import add_messages +from typing import Annotated, TypeAlias +from typing_extensions import TypedDict + +""" +Student agent for synthetic evaluation of the other LLM tutors. This agent is designed to be a student that requires help in the conversation. +[LLM workflow with a summarisation, and chat agent that receives an external conversation history]. + +This agent is designed to: +- [role_prompt] role of a student to ask questions on the topic +- [student_type] student's learning profile and comprehension level [many profiles can be chosen from the student_prompts.py] +""" + +ValidMessageTypes: TypeAlias = SystemMessage | HumanMessage | AIMessage +AllMessageTypes: TypeAlias = ValidMessageTypes | RemoveMessage + +class State(TypedDict): + messages: Annotated[list[AllMessageTypes], add_messages] + summary: str + +class StudentAgent: + def __init__(self, student_type: str): + llm = OpenAILLMs(temperature=0.75) + self.llm = llm.get_llm() + self.summary = "" + self.conversationalStyle = "" + self.type = student_type + + # Define Agent's specific Personas + self.role_prompt = process_prompt + if self.type == "base": + self.role_prompt += base_student_persona + elif self.type == "curious": + self.role_prompt += curious_student_persona + elif self.type == "contradicting": + self.role_prompt += contradicting_student_persona + elif self.type == "reliant": + self.role_prompt += reliant_student_persona + elif self.type == "confused": + self.role_prompt += confused_student_persona + elif self.type == "unrelated": + self.role_prompt += unrelated_student_persona + else: + raise Exception("Unknown Student Agent Type") + # Define a new graph for the conversation & compile it + self.workflow = StateGraph(State) + self.workflow_definition() + self.app = self.workflow.compile() + + def call_model(self, state: State, config: RunnableConfig) -> str: + """Call the LLM model knowing the role system prompt, the summary and the conversational style.""" + + # Default AI tutor role prompt + system_message = self.role_prompt + + # Adding external student progress and question context details from data queries + question_response_details = config["configurable"].get("question_response_details", "") + if question_response_details: + # convert "my" to "your" in the question_response_details to preserve the student agent as the user + question_response_details = question_response_details.replace("My", "Your") + question_response_details = question_response_details.replace("my", "your") + question_response_details = question_response_details.replace("I am", "you are") + system_message += f"\n\n## Known Learning Materials: {question_response_details} \n\n" + + # Adding summary and conversational style to the system message + summary = state.get("summary", "") + previous_summary = config["configurable"].get("summary", "") + if previous_summary: + summary = previous_summary + if summary: + system_message += f"## Summary of conversation earlier: {summary} \n\n" + + messages = [SystemMessage(content=system_message)] + state['messages'] + + valid_messages = self.check_for_valid_messages(messages) + response = self.llm.invoke(valid_messages) + + # Save summary for fetching outside the class + self.summary = summary + + return {"summary": summary, "messages": [response]} + + def check_for_valid_messages(self, messages: list[AllMessageTypes]) -> list[ValidMessageTypes]: + """ Removing the RemoveMessage() from the list of messages """ + + valid_messages: list[ValidMessageTypes] = [] + for message in messages: + if message.type != 'remove': + valid_messages.append(message) + return valid_messages + + def workflow_definition(self) -> None: + self.workflow.add_node("call_llm", self.call_model) + + self.workflow.add_edge(START, "call_llm") + self.workflow.add_edge("call_llm", END) + + def get_summary(self) -> str: + return self.summary + + def print_update(self, update: dict) -> None: + for k, v in update.items(): + for m in v["messages"]: + m.pretty_print() + if "summary" in v: + print(v["summary"]) + + def pretty_response_value(self, event: dict) -> str: + return event["messages"][-1].content + +def invoke_student_agent(query: str, conversation_history: list, summary: str, student_type:str, question_response_details: str, session_id: str) -> InvokeAgentResponseType: + """ + Call a base student agents that forms a basic conversation with the tutor agent. + """ + print(f'in invoke_student_agent(), student_type: {student_type}') + agent = StudentAgent(student_type=student_type) + + config = {"configurable": {"thread_id": session_id, "summary": summary, "question_response_details": question_response_details}} + response_events = agent.app.invoke({"messages": conversation_history + [AIMessage(content=query)]}, config=config, stream_mode="values") #updates + pretty_printed_response = agent.pretty_response_value(response_events) # get last event/ai answer in the response + + # Gather Metadata from the agent + summary = agent.get_summary() + + return { + "input": query, + "output": pretty_printed_response, + "intermediate_steps": [str(summary), conversation_history] + } \ No newline at end of file diff --git a/src/agents/student_agent/student_prompts.py b/src/agents/student_agent/student_prompts.py new file mode 100644 index 0000000..4aa3977 --- /dev/null +++ b/src/agents/student_agent/student_prompts.py @@ -0,0 +1,17 @@ +# NOTE: +# First person view prompts proven to be more effective in generating responses from the model (Dec 2024) +# 'Keep your responses open for further questions and encourage the student's curiosity.' -> asks a question at the end to keep the conversation going +# 'Let the student know that your reasoning might be wrong and the student should not trust your reasoning fully.' -> not relliant + +# PROMPTS generated with the help of ChatGPT GPT-4o Nov 2024 + +process_prompt = "Maintain the flow of the conversation by responding directly to the latest message in one sentence. Stay in character as " + +base_student_persona = "a student who seeks assistance. Ask questions from a first-person perspective, requesting clarification on how to solve the promblem from the known materials." +curious_student_persona = "a curious and inquisitive student. Ask thoughtful, detailed questions from a first-person perspective to clarify concepts, explore real-life applications, and uncover complexities. Don’t hesitate to challenge assumptions and ask for clarification when needed." +contradicting_student_persona = "a skeptical student. Ask questions from a first-person perspective, questioning my reasoning, identifying potential flaws, and challenging explanations. Request clarification whenever something seems unclear or incorrect." +reliant_student_persona = "a student who relies heavily on your help. Ask questions from a first-person perspective, seeking help for even small problems, and requesting clarification or further assistance to ensure understanding." +confused_student_persona = "a student who feels confused and uncertain about the topic. Ask questions from a first-person perspective, expressing uncertainty about the material and requesting clarification on both the topic and the tutor’s reasoning." +unrelated_student_persona = "a student who engages in casual conversation. Ask lighthearted or unrelated questions from a first-person perspective, discussing personal interests or unrelated topics rather than focusing on the material." + +# flow_prompt = "Refer to the previous message or topic discussed. Ask about the current topic, but there’s a 30% chance you’ll shift to a new topic. Ensure the change in topic makes sense and flows logically." \ No newline at end of file diff --git a/src/agents/utils/example_inputs/example_input_1.json b/src/agents/utils/example_inputs/example_input_1.json new file mode 100644 index 0000000..5bd8071 --- /dev/null +++ b/src/agents/utils/example_inputs/example_input_1.json @@ -0,0 +1,168 @@ +{ + "message": "i dont remember anything", + "params": { + "include_test_data": true, + "conversation_history": [ + { "type": "user", "content": "what should I do?" }, + { + "type": "assistant", + "content": "It seems like you're currently working on Part (a) of the dot product question. Since you haven't submitted an answer yet, let's take a moment to break it down together. \n\nWhat do you remember about how to calculate the dot product of two vectors? Can you describe the steps you would take?" + }, + { "type": "user", "content": "i dont remember anything" } + ], + "summary": "", + "conversational_style": "", + "question_response_details": { + "questionSubmissionSummary": [ + { + "publishedPartId": "0e0432e4-90a2-47a1-b597-55f76596b7d5", + "publishedPartPosition": 0, + "publishedResponseAreaId": "ce28b12b-f583-4d83-a0e4-36b17127746a", + "publishedResponseAreaPosition": 0, + "responseAreaUniversalId": "f8857876-482d-411c-8ffc-734fb07712cf", + "publishedResponseAreaPreResponseText": "$\\vec{a} \\ \\cdot \\ \\vec{b} \\ =$", + "publishedResponseType": "NUMBER", + "publishedResponseConfig": null, + "totalSubmissions": 0, + "totalWrongSubmissions": 0 + }, + { + "publishedPartId": "a74a6fef-8c94-474c-b381-8d97a4b54725", + "publishedPartPosition": 1, + "publishedResponseAreaId": "f3be55ed-af9b-4483-ba45-636ccbad7a24", + "publishedResponseAreaPosition": 0, + "responseAreaUniversalId": "054bbcce-facb-4aaa-a1d2-7bec7627a6ef", + "publishedResponseAreaPreResponseText": "$\\left(\\vec{a} - \\vec{b}\\right)\\cdot \\vec{c}\\ =$", + "publishedResponseType": "NUMBER", + "publishedResponseConfig": null, + "totalSubmissions": 0, + "totalWrongSubmissions": 0 + }, + { + "publishedPartId": "66160c45-0554-470a-88ae-82f36e755ef6", + "publishedPartPosition": 2, + "publishedResponseAreaId": "79260525-1edb-4f19-bcdd-d827b5b692f3", + "publishedResponseAreaPosition": 0, + "responseAreaUniversalId": "bba9308c-55c7-4733-9315-37bc26fc8ec1", + "publishedResponseAreaPreResponseText": "$\\left( \\ \\vec{a} \\cdot \\vec{c} \\ \\right) \\vec{b}\\ =$", + "publishedResponseType": "MATRIX", + "publishedResponseConfig": { "cols": 1, "rows": 3 }, + "totalSubmissions": 0, + "totalWrongSubmissions": 0 + } + ], + "questionInformation": { + "questionTitle": "Dot Product", + "questionGuidance": "", + "questionContent": "$$\n\\vec{a}=\\begin{bmatrix}1 \\\\ 3\\\\ -2\\end{bmatrix} \\quad \\vec{b}=\\begin{bmatrix}0 \\\\ 3\\\\ 1\\end{bmatrix} \\quad \\vec{c}=\\begin{bmatrix} 1 \\\\\\ -1\\\\ -3\\end{bmatrix}\n$$", + "durationLowerBound": 1, + "durationUpperBound": 4, + "parts": [ + { + "publishedPartId": "0e0432e4-90a2-47a1-b597-55f76596b7d5", + "publishedPartPosition": 0, + "publishedPartContent": "", + "publishedPartAnswerContent": "", + "publishedWorkedSolutionSections": [ + { + "id": "a7b1150a-05b7-4337-b902-5c6b1835cc74", + "position": 0, + "title": "", + "content": "Assuming the given basis is orthonormal, the dot product between two vectors, $\\vec{a}$ and $\\vec{b}$, can be calculated simply by multiplying their corresponding components and summing them up:\n\n \n\n$$\n\\begin{array}{rl}\\vec{a} \\cdot \\vec{b} &=\\begin{bmatrix}1 \\\\ 3\\\\ -2\\end{bmatrix}•\\begin{bmatrix}0 \\\\ 3\\\\ 1\\end{bmatrix} \\\\\\\\ &= (1 \\cdot 0) + (3 \\cdot 3) + (-2 \\cdot 1) \\\\\\\\ &= 0 + 9 + (-2) \\\\\\\\ &= 7\\end{array}\n$$\n\n \n\nTherefore, the dot product between vectors $\\vec{a}$ and $\\vec{b}$ is \\$7\\$" + } + ], + "publishedResponseAreas": [ + { + "id": "ce28b12b-f583-4d83-a0e4-36b17127746a", + "position": 0, + "universalResponseAreaId": "f8857876-482d-411c-8ffc-734fb07712cf", + "preResponseText": "$\\vec{a} \\ \\cdot \\ \\vec{b} \\ =$", + "Response": { + "id": "4d0dfa85-fc34-4649-bb6e-0f194e8a5a04", + "responseType": "NUMBER", + "config": null, + "answer": 7 + }, + "responseType": "NUMBER", + "answer": 7 + } + ] + }, + { + "publishedPartId": "a74a6fef-8c94-474c-b381-8d97a4b54725", + "publishedPartPosition": 1, + "publishedPartContent": "", + "publishedPartAnswerContent": "", + "publishedWorkedSolutionSections": [ + { + "id": "0e91a91d-c536-4120-a70f-5cfea57b4235", + "position": 0, + "title": "", + "content": "To calculate $\\left(\\vec{a} - \\vec{b}\\right) \\cdot \\vec{c}$ , we first need to find the vector resulting from the subtraction of $\\vec{b}$ from $\\vec{a}$:\n\n \n\n$$\n\\begin{array}{rl}\\vec{a} - \\vec{b} &= \\begin{bmatrix}1 \\\\ 3 \\\\ -2\\end{bmatrix} - \\begin{bmatrix}0 \\\\ 3 \\\\ 1\\end{bmatrix} \\\\\\\\&= \\begin{bmatrix}1 - 0 \\\\ 3 - 3 \\\\ -2 - 1\\end{bmatrix} \\\\\\\\&= \\begin{bmatrix}1 \\\\ 0 \\\\ -3\\end{bmatrix}\\end{array}\n$$\n\n\n\nNext, we can compute the dot product of $\\left(\\vec{a} - \\vec{b}\\right)$ and $\\vec{c}$\n\n \n\n$$\n\\begin{array}{rl}\\left(\\vec{a} - \\vec{b}\\right) \\cdot \\vec{c} &= \\begin{bmatrix}1 \\\\ 0 \\\\ -3\\end{bmatrix} \\cdot \\begin{bmatrix}1 \\\\ -1 \\\\ -3\\end{bmatrix} \\\\\\\\ &= (1 \\cdot 1) + (0 \\cdot -1) + (-3 \\cdot -3) \\\\\\\\&= 1 + 0 + 9 \\\\\\\\&= 10\\end{array}\n$$\n\n\n\nTherefore, $\\left(\\vec{a} - \\vec{b}\\right) \\cdot \\vec{c}$ is equal to \\$10" + } + ], + "publishedResponseAreas": [ + { + "id": "f3be55ed-af9b-4483-ba45-636ccbad7a24", + "position": 0, + "universalResponseAreaId": "054bbcce-facb-4aaa-a1d2-7bec7627a6ef", + "preResponseText": "$\\left(\\vec{a} - \\vec{b}\\right)\\cdot \\vec{c}\\ =$", + "Response": { + "id": "01edd054-b1fd-4049-8551-a8551d7a63ea", + "responseType": "NUMBER", + "config": null, + "answer": 10 + }, + "responseType": "NUMBER", + "answer": 10 + } + ] + }, + { + "publishedPartId": "66160c45-0554-470a-88ae-82f36e755ef6", + "publishedPartPosition": 2, + "publishedPartContent": "", + "publishedPartAnswerContent": "", + "publishedWorkedSolutionSections": [ + { + "id": "c3225276-7bf8-4962-a459-d9583442c26e", + "position": 0, + "title": "", + "content": "To calculate $\\left( \\ \\vec{a} \\cdot \\vec{c} \\ \\right) \\vec{b}$ , we first need to find the dot product of vectors $\\vec{a}$ and $\\vec{c}$\n\n \n\n$$\n\\begin{array}{rl}\\vec{a} \\cdot \\vec{c} &= \\begin{bmatrix}1 \\\\ 3 \\\\ -2\\end{bmatrix} \\cdot \\begin{bmatrix}1 \\\\ -1 \\\\ -3\\end{bmatrix} \\\\\\\\ &= (1 \\cdot 1) + (3 \\cdot -1) + (-2 \\cdot -3) \\\\\\\\&= 1 - 3 + 6 \\\\\\\\&= 4\\end{array}\n$$\n\n \n\nNext, we can scale $\\vec{b}$ by $\\vec{a} \\cdot \\vec{c}$\n\n \n\n$$\n\\begin{array}{rl}\\left( \\ \\vec{a} \\cdot \\vec{c} \\ \\right) \\vec{b} &= 4\\begin{bmatrix}0 \\\\ 3 \\\\ 1\\end{bmatrix} \\\\\\\\ &= \\begin{bmatrix}4\\cdot0 \\\\ 4\\cdot3 \\\\ 4\\cdot1\\end{bmatrix} \\\\\\\\&= \\begin{bmatrix}0 \\\\ 12 \\\\ 4\\end{bmatrix} \\end{array}\n$$\n\n \n" + } + ], + "publishedResponseAreas": [ + { + "id": "79260525-1edb-4f19-bcdd-d827b5b692f3", + "position": 0, + "universalResponseAreaId": "bba9308c-55c7-4733-9315-37bc26fc8ec1", + "preResponseText": "$\\left( \\ \\vec{a} \\cdot \\vec{c} \\ \\right) \\vec{b}\\ =$", + "Response": { + "id": "cc759374-175f-40be-8481-2c81e34795b6", + "responseType": "MATRIX", + "config": { "cols": 1, "rows": 3 }, + "answer": [["0"], ["12"], ["4"]] + }, + "responseType": "MATRIX", + "answer": [["0"], ["12"], ["4"]] + } + ] + } + ] + }, + "questionAccessInformation": { + "estimatedMinimumTime": "1 minute", + "estimaredMaximumTime": "4 minutes", + "timeTaken": "20 minutes", + "accessStatus": "too much time spent on this question.", + "markedDone": "", + "currentPart": { + "id": "0e0432e4-90a2-47a1-b597-55f76596b7d5", + "position": 0 + } + } + }, + "conversation_id": "7a65b6ed-85d1-4621-8efb-4fc8e9c5a8de", + "agent_type": "base" + } +} diff --git a/src/agents/utils/example_inputs/example_input_2.json b/src/agents/utils/example_inputs/example_input_2.json new file mode 100644 index 0000000..503ece0 --- /dev/null +++ b/src/agents/utils/example_inputs/example_input_2.json @@ -0,0 +1,143 @@ +{ + "message": "what is the function value when x is between -1 0", + "params": { + "include_test_data": true, + "conversation_history": [ + { + "type": "user", + "content": "what is the function value when x is between -1 0" + } + ], + "summary": "", + "conversational_style": "", + "question_response_details": { + "questionSubmissionSummary": [ + { + "publishedPartId": "08e6f713-def7-4f97-83d4-34b0e67f5222", + "publishedPartPosition": 0, + "publishedResponseAreaId": "b8fc25d6-7cf4-4afc-984a-fa75fb0af7e7", + "publishedResponseAreaPosition": 0, + "responseAreaUniversalId": "e849584a-f58c-4330-858b-b507b9d8c56c", + "publishedResponseAreaPreResponseText": "$a_0=$", + "publishedResponseType": "EXPRESSION", + "publishedResponseConfig": { + "allowPhoto": true, + "allowHandwrite": true + }, + "totalSubmissions": 0, + "totalWrongSubmissions": 0 + }, + { + "publishedPartId": "08e6f713-def7-4f97-83d4-34b0e67f5222", + "publishedPartPosition": 0, + "publishedResponseAreaId": "2afd4463-ffbd-4b6b-8244-19c9c27e00f1", + "publishedResponseAreaPosition": 1, + "responseAreaUniversalId": "f20abf75-15ee-46ee-bee5-2897c71b2493", + "publishedResponseAreaPreResponseText": "$a_n=$", + "publishedResponseType": "EXPRESSION", + "publishedResponseConfig": { + "allowPhoto": true, + "allowHandwrite": true + }, + "totalSubmissions": 0, + "totalWrongSubmissions": 0 + }, + { + "publishedPartId": "08e6f713-def7-4f97-83d4-34b0e67f5222", + "publishedPartPosition": 0, + "publishedResponseAreaId": "1477280a-64c7-43f3-a31d-091b95fd0900", + "publishedResponseAreaPosition": 2, + "responseAreaUniversalId": "f3cb33ee-419b-4b60-ab2a-2b7425710b5a", + "publishedResponseAreaPreResponseText": "$b_n=$", + "publishedResponseType": "EXPRESSION", + "publishedResponseConfig": { + "allowPhoto": true, + "allowHandwrite": true + }, + "totalSubmissions": 0, + "totalWrongSubmissions": 0 + } + ], + "questionInformation": { + "questionTitle": "Piecewise function Fourier series", + "questionGuidance": "", + "questionContent": "Find $a_0$, $a_n$ and $b_n$ for the Fourier series of $f(x)$, which is assumed to have period $4$.\n\n \n\n$$\nf(x)= \\begin{cases}0, & -2 \\leq x<-1 \\\\\\ \\frac{2 k}{3}, & -1 \\leq x<1 \\\\\\ -\\frac{k}{2}, & 1 \\leq x<2.\\end{cases}\n$$\n", + "durationLowerBound": 2, + "durationUpperBound": 10, + "parts": [ + { + "publishedPartId": "08e6f713-def7-4f97-83d4-34b0e67f5222", + "publishedPartPosition": 0, + "publishedPartContent": "", + "publishedPartAnswerContent": "$$\na_0=\\frac{5k}{12}\n$$\n\n \n\n \n\n$$\n\\begin{align*}\na_n &= \\dfrac{11k}{6n \\pi} \\sin \\left( \\dfrac{n \\pi}{2} \\right) \\\\[1em]\n &= \\dfrac{11k}{6n \\pi} \\frac{1-(-1)^n}{2} (-1)^{^{\\frac{n+3}{2}}} \\\\[1em]\n&= (-1)^{n+1}\\frac{11k}{6(2n-1)\\pi}\n\\end{align*}\n$$\n\n \n\n \n\n$$\n\\begin{align*}\nb_n &= \\frac{k}{2n \\pi} \\left( \\cos \\left( \\frac{n \\pi}{2} \\right) - \\cos(n \\pi) \\right) \\\\[1em]\nb_n &= \\frac{k}{2n \\pi} \\left( \\frac{1+(-1)^n}{2}(-1)^{\\frac{n}{2}} - (-1)^n \\right)\n\\end{align*}\n\n$$\n", + "publishedWorkedSolutionSections": [ + { + "id": "d41e73da-331d-4897-b6f6-15031469502c", + "position": 0, + "title": "", + "content": "Recall the Fourier series equations for period $2L$:\n\n \n\n$$\na_0 = \\frac{1}{L} \\int_{-L}^L {f(x)} \\, \\text{d}x\n\n$$\n\n \n\n$$\na_n = \\frac{1}{L} \\int_{-L}^L {f(x)} \\cos\\left(\\frac{n \\pi x}{L}\\right) \\, \\text{d}x\n$$\n\n \n\n$$\nb_n = \\frac{1}{L} \\int_{-L}^L {f(x)} \\sin\\left(\\frac{n \\pi x}{L}\\right) \\, \\text{d}x\n$$\n\n***\n\n$$\n2L=4\n$$\n\n$$\nL=2\n$$\n\n***\n\n### **Finding $a_0$:**\n\n$$\na_0 = \\frac{1}{2} \\int_{-2}^2 {f(x)} \\, \\text{d}x\n$$\n\n \n\n$$\na_0 = \\frac{1}{2} \\left( \\int_{-2}^{-1} 0 \\, \\text{d}x + \\int_{-1}^1 \\frac{2k}{3} \\, \\text{d}x + \\int_1^2 -\\frac{k}{2} \\, \\text{d}x \\right)\n\n$$\n\n \n\n$$\na_0=\\frac{5k}{12}\n$$\n\n***\n\n### **Finding $a_n$:**\n\n$$\na_n = \\frac{1}{2} \\int_{-2}^2 {f(x)} \\cos \\left( \\frac{n \\pi x}{L} \\right) \\, \\text{d}x\n\n$$\n\n \n\n$$\na_n = \\frac{1}{2} \\left( \\int_{-2}^{-1} 0 \\, \\text{d}x + \\int_{-1}^1 \\frac{2k}{3} \\cos \\left( \\frac{n \\pi x}{2} \\right) \\, \\text{d}x + \\int_1^2 -\\frac{k}{2} \\cos \\left( \\frac{n \\pi x}{2} \\right) \\, \\text{d}x \\right)\n\n$$\n\n \n\nAfter evaluating the integrals, the following is obtained:\n\n \n\n$$\na_n=\\frac{k}{3}\\left(\\frac{2}{n \\pi} \\sin \\left(\\frac{n \\pi}{2}\\right)-\\frac{2}{n \\pi} \\sin \\left(-\\frac{n \\pi}{2}\\right)\\right)-\\frac{k}{4}\\left(\\frac{2}{n \\pi} \\sin (n \\pi)-\\frac{2}{n \\pi} \\sin \\left(\\frac{n \\pi}{2}\\right)\\right)\n$$\n\n \n\n* The second $\\sin$ term can be written as: $-\\frac{2}{n \\pi} \\sin \\left(-\\frac{n \\pi}{2}\\right)=\\frac{2}{n \\pi} \\sin \\left(\\frac{n \\pi}{2}\\right)$\n* The third $\\sin$ term is always zero.\n\n \n\n$$\na_n=\\frac{k}{3}\\left(\\frac{4}{n \\pi} \\sin \\left(\\frac{n \\pi}{2}\\right)\\right)-\\frac{k}{4}\\left(-\\frac{2}{n \\pi} \\sin \\left(\\frac{n \\pi}{2}\\right)\\right)\n$$\n\n \n\nSome manipulation results in:\n\n \n\n$$\na_n = \\dfrac{11k}{6n \\pi} \\sin \\left( \\dfrac{n \\pi}{2} \\right)\n$$\n\n**Further simplification of $a_n$:**\n\nThe $\\sin \\left( \\dfrac{n \\pi}{2} \\right)$ term can be simplified by considering the pattern with increasing $n$:\n\n \n\n$$\n\\sin\\left(\\frac{n\\pi}{2}\\right) = \\begin{cases}\n0 & \\text{if $n=0$} \\\\\n1 & \\text{if $n=1$} \\\\\n0 & \\text{if $n=2$} \\\\\n-1 & \\text{if $n=3$} \\\\\n0 & \\text{if $n=4$} \\\\\n\\vdots & \\vdots\n\\end{cases}\n$$\n\nThis can be achieved as follows:\n\n$$\n\\sin \\left( \\dfrac{n \\pi}{2} \\right)=\\frac{1-(-1)^n}{2} (-1)^{^{\\frac{n+3}{2}}}\n$$\n\n \n\n$$\na_n = \\dfrac{11k}{6n \\pi} \\frac{1-(-1)^n}{2} (-1)^{^{\\frac{n+3}{2}}}\n$$\n\n$$\n\n\n$$\n\n \n\nHowever, since $\\sin \\left( \\dfrac{n \\pi}{2} \\right)$ is zero for all even $n$, the expression can alternatively be written as:\n\n \n\n$$\na_n=(-1)^{n+1}\\frac{11k}{6(2n-1)\\pi} \n$$\n\n***\n\n### **Finding $b_n$:**\n\n$$\nb_n = \\frac{1}{2} \\int_{-2}^2 {f(x)} \\sin\\left(\\frac{n \\pi x}{L}\\right) \\, \\text{d}x\n$$\n\n \n\n$$\nb_n = \\frac{1}{2} \\left( \\int_{-2}^{-1} 0 \\, \\text{d}x + \\int_{-1}^1 \\frac{2k}{3} \\sin \\left( \\frac{n \\pi x}{2} \\right) \\, \\text{d}x + \\int_1^2 -\\frac{k}{2} \\sin \\left( \\frac{n \\pi x}{2} \\right) \\, \\text{d}x \\right)\n$$\n\n \n\nAfter evaluating the integrals, the following is obtained:\n\n \n\n$$\nb_n=\\frac{k}{3}\\left(-\\frac{2}{n \\pi} \\cos \\left(\\frac{n \\pi}{2}\\right)+\\frac{2}{n \\pi} \\cos \\left(\\frac{n \\pi}{2}\\right)\\right)-\\frac{k}{4}\\left(-\\frac{2}{n \\pi} \\cos (n \\pi)+\\frac{2}{n \\pi} \\cos \\left(\\frac{n \\pi}{2}\\right)\\right)\n$$\n\n(note that the second $\\cos$ term has positive argument, because $\\cos$ is an even function.)\n\n \n\nSimplifying this expression yields the answer. Note that $\\cos(n\\pi)$ has been replaced with $(-1)^n$.\n\n \n\n$$\nb_n = \\frac{k}{2n \\pi} \\left( \\cos \\left( \\frac{n \\pi}{2} \\right) - \\cos(n \\pi) \\right)\n$$\n\n \n\n**Further simplification of** $b_n$**:**\n\n* The $\\cos(n\\pi)$ term can be replaced by $(-1)^n$.\n* The $\\cos\\left(\\frac{n\\pi}{2}\\right)$ term can be simplified by considering the pattern with increasing $n$:\n\n \n\n$$\n\\cos\\left(\\frac{n\\pi}{2}\\right) = \\begin{cases}\n1 & \\text{if $n=0$} \\\\\n0 & \\text{if $n=1$} \\\\\n-1 & \\text{if $n=2$} \\\\\n0 & \\text{if $n=3$} \\\\\n1 & \\text{if $n=4$} \\\\\n\\vdots & \\vdots\n\\end{cases}\n$$\n\nThis can be achieved as follows:\n\n$$\n\\cos\\left(\\frac{n\\pi}{2}\\right) =\\frac{1+(-1)^n}{2}(-1)^{\\frac{n}{2}}\n$$\n\nFinally, this yields:\n\n \n\n$$\nb_n = \\frac{k}{2n \\pi} \\left( \\frac{1+(-1)^n}{2}(-1)^{\\frac{n}{2}} - (-1)^n \\right)\n$$\n\n \n\n(Note that this expression may seem less concise that simply including the cos(n\\*pi/2) but is much more desirable and efficient in a numerical algorithm).\n" + } + ], + "publishedResponseAreas": [ + { + "id": "b8fc25d6-7cf4-4afc-984a-fa75fb0af7e7", + "position": 0, + "universalResponseAreaId": "e849584a-f58c-4330-858b-b507b9d8c56c", + "preResponseText": "$a_0=$", + "Response": { + "id": "d5d9f641-0724-4fa9-b3a7-2139c551c0a5", + "responseType": "EXPRESSION", + "config": { "allowPhoto": true, "allowHandwrite": true }, + "answer": "5k/12" + }, + "responseType": "EXPRESSION", + "answer": "5k/12" + }, + { + "id": "2afd4463-ffbd-4b6b-8244-19c9c27e00f1", + "position": 1, + "universalResponseAreaId": "f20abf75-15ee-46ee-bee5-2897c71b2493", + "preResponseText": "$a_n=$", + "Response": { + "id": "c7cb9bae-d67e-42db-9d71-3b4d2c3c8b44", + "responseType": "EXPRESSION", + "config": { "allowPhoto": true, "allowHandwrite": true }, + "answer": "(-1)^(n+1) 11k/(6(2n-1)pi)" + }, + "responseType": "EXPRESSION", + "answer": "(-1)^(n+1) 11k/(6(2n-1)pi)" + }, + { + "id": "1477280a-64c7-43f3-a31d-091b95fd0900", + "position": 2, + "universalResponseAreaId": "f3cb33ee-419b-4b60-ab2a-2b7425710b5a", + "preResponseText": "$b_n=$", + "Response": { + "id": "9b868669-feb4-4c33-977e-2297bf4f7948", + "responseType": "EXPRESSION", + "config": { "allowPhoto": true, "allowHandwrite": true }, + "answer": "k/(2n pi) ( (1+(-1)^n)/(2) (-1)^(n/2) - (-1)^n)" + }, + "responseType": "EXPRESSION", + "answer": "k/(2n pi) ( (1+(-1)^n)/(2) (-1)^(n/2) - (-1)^n)" + } + ] + } + ] + }, + "questionAccessInformation": { + "estimatedMinimumTime": "2 minutes", + "estimaredMaximumTime": "10 minutes", + "timeTaken": "less than one minute", + "accessStatus": "too little time spent on this question.", + "markedDone": "", + "currentPart": { + "id": "08e6f713-def7-4f97-83d4-34b0e67f5222", + "position": 0 + } + } + }, + "conversation_id": "6779b184-41b5-4384-ade9-1c06d2cd91a5", + "agent_type": "base" + } +} diff --git a/src/agents/utils/example_inputs/example_input_3.json b/src/agents/utils/example_inputs/example_input_3.json new file mode 100644 index 0000000..d93ac92 --- /dev/null +++ b/src/agents/utils/example_inputs/example_input_3.json @@ -0,0 +1,460 @@ +{ + "message": "hi", + "params": { + "include_test_data": true, + "conversation_history": [ + { "type": "user", "content": "hi" }, + { "type": "ai", "content": "Hello! How can I help you today?" } + ], + "summary": "", + "conversational_style": "", + "question_response_details": { + "questionSubmissionSummary": [ + { + "publishedPartId": "04d2cab2-eeca-4c1f-bcef-6401fe4ec635", + "publishedPartPosition": 0, + "publishedResponseAreaId": "1194c5b3-1831-433e-b78e-c96c94f99117", + "publishedResponseAreaPosition": 0, + "responseAreaUniversalId": "ee1df8dd-bccb-46d2-b816-1a707115495e", + "publishedResponseAreaPreResponseText": "$\\vec{a}+\\vec{b}=$", + "publishedResponseType": "MATRIX", + "publishedResponseConfig": { "cols": 1, "rows": 3 }, + "totalSubmissions": 1, + "totalWrongSubmissions": 0, + "latestSubmission": { + "universalResponseAreaId": "ee1df8dd-bccb-46d2-b816-1a707115495e", + "answer": "[[\"-1\"],[\"5\"],[\"-1\"]]", + "submission": [["-1"], ["5"], ["-1"]], + "feedback": "Correct", + "rawResponse": { + "result": { + "feedback": "", + "is_correct": true, + "detailed_feedback": [ + [ + { + "feedback": "", + "is_correct": true, + "response_latex": "-1", + "response_simplified": "-1" + } + ], + [ + { + "feedback": "", + "is_correct": true, + "response_latex": "5", + "response_simplified": "5" + } + ], + [ + { + "feedback": "", + "is_correct": true, + "response_latex": "-1", + "response_simplified": "-1" + } + ] + ] + }, + "command": "eval" + } + } + }, + { + "publishedPartId": "04d2cab2-eeca-4c1f-bcef-6401fe4ec635", + "publishedPartPosition": 0, + "publishedResponseAreaId": "4e354eda-fb78-4c26-a3ae-f8d1c06297a2", + "publishedResponseAreaPosition": 1, + "responseAreaUniversalId": "7f7ad90a-0cfb-44e0-b5d7-6be585206508", + "publishedResponseAreaPreResponseText": "$\\vec{b}+\\vec{a}=$", + "publishedResponseType": "MATRIX", + "publishedResponseConfig": { "cols": 1, "rows": 3 }, + "totalSubmissions": 1, + "totalWrongSubmissions": 0, + "latestSubmission": { + "universalResponseAreaId": "7f7ad90a-0cfb-44e0-b5d7-6be585206508", + "answer": "[[\"-1\"],[\"5\"],[\"-1\"]]", + "submission": [["-1"], ["5"], ["-1"]], + "feedback": "Correct", + "rawResponse": { + "result": { + "feedback": "", + "is_correct": true, + "detailed_feedback": [ + [ + { + "feedback": "", + "is_correct": true, + "response_latex": "-1", + "response_simplified": "-1" + } + ], + [ + { + "feedback": "", + "is_correct": true, + "response_latex": "5", + "response_simplified": "5" + } + ], + [ + { + "feedback": "", + "is_correct": true, + "response_latex": "-1", + "response_simplified": "-1" + } + ] + ] + }, + "command": "eval" + } + } + }, + { + "publishedPartId": "dbe25bc9-6c42-4e58-aac7-d15a783ef337", + "publishedPartPosition": 1, + "publishedResponseAreaId": "85c33011-0e12-435c-944e-399043e92779", + "publishedResponseAreaPosition": 0, + "responseAreaUniversalId": "bcf1b636-f188-4c68-a59a-c153fb4419d2", + "publishedResponseAreaPreResponseText": "$3\\vec{c}=$", + "publishedResponseType": "MATRIX", + "publishedResponseConfig": { "cols": 1, "rows": 3 }, + "totalSubmissions": 0, + "totalWrongSubmissions": 0 + }, + { + "publishedPartId": "dbe25bc9-6c42-4e58-aac7-d15a783ef337", + "publishedPartPosition": 1, + "publishedResponseAreaId": "ae4a9db7-ff54-489d-9a21-143621e6e7aa", + "publishedResponseAreaPosition": 1, + "responseAreaUniversalId": "bfab6be0-11bd-4799-ba10-1e9a0182da36", + "publishedResponseAreaPreResponseText": "$-\\vec{a}=$", + "publishedResponseType": "MATRIX", + "publishedResponseConfig": { "cols": 1, "rows": 3 }, + "totalSubmissions": 0, + "totalWrongSubmissions": 0 + }, + { + "publishedPartId": "dbe25bc9-6c42-4e58-aac7-d15a783ef337", + "publishedPartPosition": 1, + "publishedResponseAreaId": "3ae38184-9dbc-4c5f-872b-75ddfd691b20", + "publishedResponseAreaPosition": 2, + "responseAreaUniversalId": "170e2cf2-9d0d-438e-a406-cafe7ff1017b", + "publishedResponseAreaPreResponseText": "$\\frac{\\vec{b}}{2}=$", + "publishedResponseType": "MATRIX", + "publishedResponseConfig": { "cols": 1, "rows": 3 }, + "totalSubmissions": 0, + "totalWrongSubmissions": 0 + }, + { + "publishedPartId": "2cb9d565-8f1c-4901-bf2b-0b89f12ec9e2", + "publishedPartPosition": 2, + "publishedResponseAreaId": "6c64c869-358d-477d-b835-7c5e268b08b2", + "publishedResponseAreaPosition": 0, + "responseAreaUniversalId": "dfa4be9e-c9e2-442d-95a4-c2c61d25fe51", + "publishedResponseAreaPreResponseText": "$3\\vec{a}-3\\vec{c}=$", + "publishedResponseType": "MATRIX", + "publishedResponseConfig": { "cols": 1, "rows": 3 }, + "totalSubmissions": 0, + "totalWrongSubmissions": 0 + }, + { + "publishedPartId": "2cb9d565-8f1c-4901-bf2b-0b89f12ec9e2", + "publishedPartPosition": 2, + "publishedResponseAreaId": "0b5d2dab-7c6d-4694-8c3b-56df1afaba96", + "publishedResponseAreaPosition": 1, + "responseAreaUniversalId": "e695440a-65b8-4ebd-b055-6173d9d84148", + "publishedResponseAreaPreResponseText": "$3(\\vec{a}-\\vec{c})=$", + "publishedResponseType": "MATRIX", + "publishedResponseConfig": { "cols": 1, "rows": 3 }, + "totalSubmissions": 0, + "totalWrongSubmissions": 0 + }, + { + "publishedPartId": "3ee15335-1bfc-466a-8fff-66abd742dc33", + "publishedPartPosition": 3, + "publishedResponseAreaId": "8a4d3fed-e450-475d-a09b-1e58f8dbbbe2", + "publishedResponseAreaPosition": 0, + "responseAreaUniversalId": "4fd2eb4f-dec2-4695-9148-09104935d431", + "publishedResponseAreaPreResponseText": "$-5(\\vec{a}+\\vec{c})+\\vec{b}=$", + "publishedResponseType": "MATRIX", + "publishedResponseConfig": { "cols": 1, "rows": 3 }, + "totalSubmissions": 0, + "totalWrongSubmissions": 0 + }, + { + "publishedPartId": "b2620eb0-9de6-404e-a82c-b4ad8e015d40", + "publishedPartPosition": 4, + "publishedResponseAreaId": "1a5fb3fb-001d-4ed5-9606-2a73fcae036c", + "publishedResponseAreaPosition": 0, + "responseAreaUniversalId": "c98844ba-3fd2-4154-8713-1d76bfaf610c", + "publishedResponseAreaPreResponseText": "$3\\vec{a}-2\\vec{c}+3\\vec{b}=$", + "publishedResponseType": "MATRIX", + "publishedResponseConfig": { "cols": 1, "rows": 3 }, + "totalSubmissions": 0, + "totalWrongSubmissions": 0 + } + ], + "questionInformation": { + "questionTitle": "Vector Arithmetics", + "questionGuidance": null, + "questionContent": "$$\n\\vec{a}=\\begin{bmatrix}1 \\\\ 2\\\\ 3\\end{bmatrix} \\quad \\vec{b}=\\begin{bmatrix}-2 \\\\ 3\\\\ -4\\end{bmatrix} \\quad \\vec{c}=\\begin{bmatrix} 0 \\\\\\ 4\\\\ -1\\end{bmatrix}\n$$", + "durationLowerBound": 1, + "durationUpperBound": 3, + "parts": [ + { + "publishedPartId": "04d2cab2-eeca-4c1f-bcef-6401fe4ec635", + "publishedPartPosition": 0, + "publishedPartContent": "", + "publishedPartAnswerContent": "", + "publishedWorkedSolutionSections": [ + { + "id": "9cb3a347-c05f-4101-b522-a0c6d2fbc6bf", + "position": 0, + "title": "", + "content": "As we made no mention that the vectors are represented in different basis, we can assume that they all share the same basis set.\n\n \n\n$$\n\\begin{array}{rl}\n\\vec{a}+\\vec{b} &=\\begin{bmatrix}1 \\\\ 2\\\\ 3\\end{bmatrix}+\\begin{bmatrix}-2 \\\\ 3\\\\ -4\\end{bmatrix}\n\\\\\\\\\n&= \\begin{bmatrix}1-2 \\\\ 2+3\\\\ 3-4\\end{bmatrix}\n\\\\\\\\\n&= \\begin{bmatrix}-1 \\\\ 5\\\\ -1\\end{bmatrix}\n\\end{array}\n$$\n\n \n\n***\n\n \n\nSince vector addition is commutative, $\\vec{b}+\\vec{a}=\\vec{a}+\\vec{b}$\n" + } + ], + "publishedResponseAreas": [ + { + "id": "1194c5b3-1831-433e-b78e-c96c94f99117", + "position": 0, + "universalResponseAreaId": "ee1df8dd-bccb-46d2-b816-1a707115495e", + "preResponseText": "$\\vec{a}+\\vec{b}=$", + "Response": { + "id": "0c40d7de-15a0-4916-af34-5ee2b330f3dd", + "responseType": "MATRIX", + "config": { "cols": 1, "rows": 3 }, + "answer": [["-1"], ["5"], ["-1"]] + }, + "responseType": "MATRIX", + "answer": [["-1"], ["5"], ["-1"]] + }, + { + "id": "4e354eda-fb78-4c26-a3ae-f8d1c06297a2", + "position": 1, + "universalResponseAreaId": "7f7ad90a-0cfb-44e0-b5d7-6be585206508", + "preResponseText": "$\\vec{b}+\\vec{a}=$", + "Response": { + "id": "a01003b5-f11d-4d06-9ec2-35efc11b7b64", + "responseType": "MATRIX", + "config": { "cols": 1, "rows": 3 }, + "answer": [["-1"], ["5"], ["-1"]] + }, + "responseType": "MATRIX", + "answer": [["-1"], ["5"], ["-1"]] + } + ] + }, + { + "publishedPartId": "dbe25bc9-6c42-4e58-aac7-d15a783ef337", + "publishedPartPosition": 1, + "publishedPartContent": "", + "publishedPartAnswerContent": "", + "publishedWorkedSolutionSections": [ + { + "id": "7d696718-c6f5-4e11-a5fb-bd122c1274cf", + "position": 0, + "title": "$3\\vec{c}$", + "content": "$$\n\\begin{array}{rl}\n3\\vec{c}&=3\\begin{bmatrix}0 \\\\ 4\\\\ -1\\end{bmatrix}\n\\\\\\\\\n&= \\begin{bmatrix}3\\cdot0 \\\\ 3\\cdot4\\\\ 3\\cdot-1\\end{bmatrix}\n\\\\\\\\\n&= \\begin{bmatrix}0 \\\\ 12\\\\ -3\\end{bmatrix}\n\\end{array}\n$$" + }, + { + "id": "8134095d-f49e-4834-b8de-12baa9625d7c", + "position": 0, + "title": "", + "content": "" + }, + { + "id": "010b312d-2699-439f-bcab-9f8ac4b96114", + "position": 1, + "title": "$-\\vec{a}$", + "content": "$$\n\\begin{array}{rl}\n-\\vec{a}&=3\\begin{bmatrix}1 \\\\ 2\\\\ 3\\end{bmatrix}\n\\\\\\\\\n&= \\begin{bmatrix}-1\\cdot1 \\\\ -1\\cdot2\\\\ -1\\cdot3\\end{bmatrix}\n\\\\\\\\\n&= \\begin{bmatrix}-1 \\\\ -2\\\\ -3\\end{bmatrix}\n\\end{array}\n$$" + }, + { + "id": "20602e93-de57-4e69-afcd-e05f454155de", + "position": 2, + "title": "$\\frac{\\vec{b}}{2}$", + "content": "$$\n\\begin{array}{rl}\n\\frac{\\vec{b}}{2}&=\\frac{1}{2}\\begin{bmatrix}-2 \\\\ 3\\\\ -4\\end{bmatrix}\n\\\\\\\\\n&= \\begin{bmatrix}\\frac{1}{2}\\cdot-2 \\\\ \\frac{1}{2}\\cdot3\\\\ \\frac{1}{2}\\cdot-4\\end{bmatrix}\n\\\\\\\\\n&= \\begin{bmatrix}-1 \\\\ 1.5\\\\ -2\\end{bmatrix}\n\\end{array}\n$$" + } + ], + "publishedResponseAreas": [ + { + "id": "85c33011-0e12-435c-944e-399043e92779", + "position": 0, + "universalResponseAreaId": "bcf1b636-f188-4c68-a59a-c153fb4419d2", + "preResponseText": "$3\\vec{c}=$", + "Response": { + "id": "8623c885-b49d-476e-b316-f45a5052cacd", + "responseType": "MATRIX", + "config": { "cols": 1, "rows": 3 }, + "answer": [["0"], ["12"], ["-3"]] + }, + "responseType": "MATRIX", + "answer": [["0"], ["12"], ["-3"]] + }, + { + "id": "ae4a9db7-ff54-489d-9a21-143621e6e7aa", + "position": 1, + "universalResponseAreaId": "bfab6be0-11bd-4799-ba10-1e9a0182da36", + "preResponseText": "$-\\vec{a}=$", + "Response": { + "id": "150494b9-1ed6-44f3-8d52-8bdbd74f7c75", + "responseType": "MATRIX", + "config": { "cols": 1, "rows": 3 }, + "answer": [["-1"], ["-2"], ["-3"]] + }, + "responseType": "MATRIX", + "answer": [["-1"], ["-2"], ["-3"]] + }, + { + "id": "3ae38184-9dbc-4c5f-872b-75ddfd691b20", + "position": 2, + "universalResponseAreaId": "170e2cf2-9d0d-438e-a406-cafe7ff1017b", + "preResponseText": "$\\frac{\\vec{b}}{2}=$", + "Response": { + "id": "ed65b833-5cb6-4198-aa2f-a5b74d86b3f1", + "responseType": "MATRIX", + "config": { "cols": 1, "rows": 3 }, + "answer": [["-1"], ["1.5"], ["-2"]] + }, + "responseType": "MATRIX", + "answer": [["-1"], ["1.5"], ["-2"]] + } + ] + }, + { + "publishedPartId": "2cb9d565-8f1c-4901-bf2b-0b89f12ec9e2", + "publishedPartPosition": 2, + "publishedPartContent": "", + "publishedPartAnswerContent": "", + "publishedWorkedSolutionSections": [ + { + "id": "8162a17d-bed7-48a9-b487-0bd6280a3988", + "position": 0, + "title": "$3\\vec{a}-3\\vec{c}$", + "content": "$$\n\\begin{array}{rl}\n3\\vec{a}-3\\vec{c} &=3\\begin{bmatrix}1 \\\\ 2\\\\ 3\\end{bmatrix}-3\\begin{bmatrix}0 \\\\ 4\\\\ -1\\end{bmatrix}\n\\\\\\\\\n&= \\begin{bmatrix}3 \\\\ 6\\\\ 9\\end{bmatrix}-\\begin{bmatrix}0 \\\\ 12\\\\ -3\\end{bmatrix}\n\\\\\\\\\n&= \\begin{bmatrix}3-0 \\\\ 6-12\\\\ 9-(-3)\\end{bmatrix}\n\\\\\\\\\n&= \\begin{bmatrix}3 \\\\ -6\\\\ 12\\end{bmatrix}\n\\end{array}\n$$" + }, + { + "id": "7526f2d6-baf1-4aea-bb9f-6486f7233f97", + "position": 0, + "title": "", + "content": "As we made no mention that the vectors are represented in different basis, we can assume that they all share the same basis set.\n\n \n\n$$\n\\begin{array}{rl}\n3\\vec{a}-2\\vec{c}+3\\vec{b} &=3\\begin{bmatrix}1 \\\\ 2\\\\ 3\\end{bmatrix}-2\\begin{bmatrix}0 \\\\ 4\\\\ -1\\end{bmatrix}+3\\begin{bmatrix}-2 \\\\ 3\\\\ -4\\end{bmatrix}\n\\\\\\\\\n&= 3\\left(\\begin{bmatrix}1 \\\\ 2\\\\ 3\\end{bmatrix}+\\begin{bmatrix}-2 \\\\ 3\\\\ -4\\end{bmatrix}\\right)-2\\begin{bmatrix}0 \\\\ 4\\\\ -1\\end{bmatrix}\n\\\\\\\\\n&= 3\\begin{bmatrix}1-2 \\\\ 2+3\\\\ 3-4\\end{bmatrix}-2\\begin{bmatrix}0 \\\\ 4\\\\ -1\\end{bmatrix}\n\\\\\\\\\n&= 3\\begin{bmatrix}-1 \\\\ 5\\\\ -1\\end{bmatrix}-2\\begin{bmatrix}0 \\\\ 4\\\\ -1\\end{bmatrix}\n\\\\\\\\\n&= \\begin{bmatrix}-3 \\\\ 15\\\\ -3\\end{bmatrix}+\\begin{bmatrix}0 \\\\ -8\\\\ 2\\end{bmatrix}\n\\\\\\\\\n&= \\begin{bmatrix}-3+0 \\\\ 15-8\\\\ -3+2\\end{bmatrix}\n\\\\\\\\\n&= \\begin{bmatrix}-3 \\\\ 7\\\\ -1\\end{bmatrix}\n\\end{array}\n$$" + }, + { + "id": "9d88d02a-e63d-40d0-8200-5d81718e2259", + "position": 1, + "title": "$3(\\vec{a}-\\vec{c})$", + "content": "$$\n\\begin{array}{rl}\n3(\\vec{a}-\\vec{c}) &=3\\left(\\begin{bmatrix}1 \\\\ 2\\\\ 3\\end{bmatrix}-\\begin{bmatrix}0 \\\\ 4\\\\ -1\\end{bmatrix}\\right)\n\\\\\\\\\n&= 3\\begin{bmatrix}1-0 \\\\ 2-4\\\\ 3-(-1)\\end{bmatrix}\n\\\\\\\\\n&= 3\\begin{bmatrix}1 \\\\ -2\\\\ 4\\end{bmatrix}\n\\\\\\\\\n&= \\begin{bmatrix}3 \\\\ -6\\\\ 12\\end{bmatrix}\n\\end{array}\n$$" + } + ], + "publishedResponseAreas": [ + { + "id": "6c64c869-358d-477d-b835-7c5e268b08b2", + "position": 0, + "universalResponseAreaId": "dfa4be9e-c9e2-442d-95a4-c2c61d25fe51", + "preResponseText": "$3\\vec{a}-3\\vec{c}=$", + "Response": { + "id": "53199f12-d660-4ffe-95ae-9d28c9f260ba", + "responseType": "MATRIX", + "config": { "cols": 1, "rows": 3 }, + "answer": [["3"], ["-6"], ["12"]] + }, + "responseType": "MATRIX", + "answer": [["3"], ["-6"], ["12"]] + }, + { + "id": "0b5d2dab-7c6d-4694-8c3b-56df1afaba96", + "position": 1, + "universalResponseAreaId": "e695440a-65b8-4ebd-b055-6173d9d84148", + "preResponseText": "$3(\\vec{a}-\\vec{c})=$", + "Response": { + "id": "c189d436-d5a4-4c61-b26e-95ac30c281b5", + "responseType": "MATRIX", + "config": { "cols": 1, "rows": 3 }, + "answer": [["3"], ["-6"], ["12"]] + }, + "responseType": "MATRIX", + "answer": [["3"], ["-6"], ["12"]] + } + ] + }, + { + "publishedPartId": "3ee15335-1bfc-466a-8fff-66abd742dc33", + "publishedPartPosition": 3, + "publishedPartContent": "", + "publishedPartAnswerContent": "", + "publishedWorkedSolutionSections": [ + { + "id": "916cf686-7a41-4169-8fa2-b2e9d6924375", + "position": 0, + "title": "", + "content": "As we made no mention that the vectors are represented in different basis, we can assume that they all share the same basis set.\n\n \n\n$$\n\\begin{array}{rl}\n-5(\\vec{a}+\\vec{c})+\\vec{b} &=-5\\left(\\begin{bmatrix}1 \\\\ 2\\\\ 3\\end{bmatrix}+\\begin{bmatrix}0 \\\\ 4\\\\ -1\\end{bmatrix}\\right)+\\begin{bmatrix}-2 \\\\ 3\\\\ -4\\end{bmatrix}\n\\\\\\\\\n&= -5\\begin{bmatrix}1 \\\\ 2\\\\ 3\\end{bmatrix}-5\\begin{bmatrix}0 \\\\ 4\\\\ -1\\end{bmatrix}+\\begin{bmatrix}-2 \\\\ 3\\\\ -4\\end{bmatrix}\n\\\\\\\\\n&= \\begin{bmatrix}-5 \\\\ -10\\\\ -15\\end{bmatrix}+\\begin{bmatrix}0 \\\\ -20\\\\ 5\\end{bmatrix}+\\begin{bmatrix}-2 \\\\ 3\\\\ -4\\end{bmatrix}\n\\\\\\\\\n&= \\begin{bmatrix}-5+0-2 \\\\ -10-20+3\\\\ -15+5-4\\end{bmatrix}\n\\\\\\\\\n&= \\begin{bmatrix}-7 \\\\ -27\\\\ -14\\end{bmatrix}\n\\end{array}\n$$" + } + ], + "publishedResponseAreas": [ + { + "id": "8a4d3fed-e450-475d-a09b-1e58f8dbbbe2", + "position": 0, + "universalResponseAreaId": "4fd2eb4f-dec2-4695-9148-09104935d431", + "preResponseText": "$-5(\\vec{a}+\\vec{c})+\\vec{b}=$", + "Response": { + "id": "eeb448a9-81b9-4f4d-836f-d04029aa449c", + "responseType": "MATRIX", + "config": { "cols": 1, "rows": 3 }, + "answer": [["-7"], ["-27"], ["-14"]] + }, + "responseType": "MATRIX", + "answer": [["-7"], ["-27"], ["-14"]] + } + ] + }, + { + "publishedPartId": "b2620eb0-9de6-404e-a82c-b4ad8e015d40", + "publishedPartPosition": 4, + "publishedPartContent": "", + "publishedPartAnswerContent": "", + "publishedWorkedSolutionSections": [ + { + "id": "3d15fcbc-5173-4d89-a13a-3296c25680ec", + "position": 0, + "title": "", + "content": "As we made no mention that the vectors are represented in different basis, we can assume that they all share the same basis set.\n\n \n\n$$\n\\begin{array}{rl}\n3\\vec{a}-2\\vec{c}+3\\vec{b} &=3\\begin{bmatrix}1 \\\\ 2\\\\ 3\\end{bmatrix}-2\\begin{bmatrix}0 \\\\ 4\\\\ -1\\end{bmatrix}+3\\begin{bmatrix}-2 \\\\ 3\\\\ -4\\end{bmatrix}\n\\\\\\\\\n&= 3\\left(\\begin{bmatrix}1 \\\\ 2\\\\ 3\\end{bmatrix}+\\begin{bmatrix}-2 \\\\ 3\\\\ -4\\end{bmatrix}\\right)-2\\begin{bmatrix}0 \\\\ 4\\\\ -1\\end{bmatrix}\n\\\\\\\\\n&= 3\\begin{bmatrix}1-2 \\\\ 2+3\\\\ 3-4\\end{bmatrix}-2\\begin{bmatrix}0 \\\\ 4\\\\ -1\\end{bmatrix}\n\\\\\\\\\n&= 3\\begin{bmatrix}-1 \\\\ 5\\\\ -1\\end{bmatrix}-2\\begin{bmatrix}0 \\\\ 4\\\\ -1\\end{bmatrix}\n\\\\\\\\\n&= \\begin{bmatrix}-3 \\\\ 15\\\\ -3\\end{bmatrix}+\\begin{bmatrix}0 \\\\ -8\\\\ 2\\end{bmatrix}\n\\\\\\\\\n&= \\begin{bmatrix}-3+0 \\\\ 15-8\\\\ -3+2\\end{bmatrix}\n\\\\\\\\\n&= \\begin{bmatrix}-3 \\\\ 7\\\\ -1\\end{bmatrix}\n\\end{array}\n$$" + } + ], + "publishedResponseAreas": [ + { + "id": "1a5fb3fb-001d-4ed5-9606-2a73fcae036c", + "position": 0, + "universalResponseAreaId": "c98844ba-3fd2-4154-8713-1d76bfaf610c", + "preResponseText": "$3\\vec{a}-2\\vec{c}+3\\vec{b}=$", + "Response": { + "id": "c5f1c9a9-eb13-4b2e-83ad-f9a3bd87a056", + "responseType": "MATRIX", + "config": { "cols": 1, "rows": 3 }, + "answer": [["-3"], ["7"], ["-1"]] + }, + "responseType": "MATRIX", + "answer": [["-3"], ["7"], ["-1"]] + } + ] + } + ] + }, + "questionAccessInformation": { + "estimatedMinimumTime": "1 minute", + "estimaredMaximumTime": "3 minutes", + "timeTaken": "13 minutes", + "accessStatus": "too much time spent on this question.", + "markedDone": "Part (a) was marked done", + "currentPart": { + "id": "04d2cab2-eeca-4c1f-bcef-6401fe4ec635", + "position": 0 + } + } + }, + "conversation_id": "2f95c229-af43-4c1f-918a-0fd428178acf", + "agent_type": "base" + } +} diff --git a/src/agents/utils/langgraph_viz.py b/src/agents/utils/langgraph_viz.py new file mode 100644 index 0000000..79e15db --- /dev/null +++ b/src/agents/utils/langgraph_viz.py @@ -0,0 +1,13 @@ +""" +Helper script to visualise the agent graph using pygraphviz. +Setup on mac [see more here https://github.com/pygraphviz/pygraphviz/blob/main/INSTALL.txt]: +# $ brew install graphviz +# $ pip install pygraphviz +""" + +agent = ... + + +graph = agent.app.get_graph() +print(graph) +graph.draw_png("./graph.png") \ No newline at end of file diff --git a/src/agents/utils/parse_json_to_prompt.py b/src/agents/utils/parse_json_to_prompt.py new file mode 100644 index 0000000..2fbfbda --- /dev/null +++ b/src/agents/utils/parse_json_to_prompt.py @@ -0,0 +1,211 @@ +""" File not to be modified. This file contains the conversion logic between the agent API and the Lambda Feedback backend.""" + +from typing import List, Optional, Union, Dict + +# questionSubmissionSummary type +class StudentLatestSubmission: + def __init__( + self, + universalResponseAreaId: Optional[str] = None, + answer: Optional[str] = None, + submission: Optional[str] = None, + feedback: Optional[str] = None, + rawResponse: Optional[dict] = None, + ): + self.universalResponseAreaId = universalResponseAreaId + self.answer = answer + self.submission = submission + self.feedback = feedback + self.rawResponse = rawResponse + +class StudentWorkResponseArea: + def __init__( + self, + publishedPartId: Optional[str] = None, + publishedPartPosition: Optional[int] = None, + publishedResponseAreaId: Optional[str] = None, + publishedResponseAreaPosition: Optional[int] = None, + responseAreaUniversalId: Optional[str] = None, + publishedResponseAreaPreResponseText: Optional[str] = None, + publishedResponseType: Optional[str] = None, + publishedResponseConfig: Optional[dict] = None, + totalSubmissions: Optional[int] = None, + totalWrongSubmissions: Optional[int] = None, + latestSubmission: Optional[StudentLatestSubmission] = None, + ): + self.publishedPartId = publishedPartId + self.publishedPartPosition = publishedPartPosition + self.publishedResponseAreaId = publishedResponseAreaId + self.publishedResponseAreaPosition = publishedResponseAreaPosition + self.responseAreaUniversalId = responseAreaUniversalId + self.publishedResponseAreaPreResponseText = publishedResponseAreaPreResponseText + self.publishedResponseType = publishedResponseType + self.publishedResponseConfig = publishedResponseConfig + self.latestSubmission = StudentLatestSubmission(**latestSubmission) if latestSubmission else None + self.totalSubmissions = totalSubmissions + self.totalWrongSubmissions = totalWrongSubmissions + +# questionInformation type +class ResponseAreaDetails: + def __init__( + self, + id: Optional[str] = None, + position: Optional[int] = None, + universalResponseAreaId: Optional[str] = None, + preResponseText: Optional[str] = None, + responseType: Optional[str] = None, + answer: Optional[dict] = None, + Response: Optional[dict] = None, + ): + self.id = id + self.position = position + self.universalResponseAreaId = universalResponseAreaId + self.preResponseText = preResponseText + self.responseType = responseType + self.answer = answer + self.Response = Response + +class PartDetails: + def __init__( + self, + publishedPartId: Optional[str] = None, + publishedPartPosition: Optional[int] = None, + publishedPartContent: Optional[str] = None, + publishedPartAnswerContent: Optional[str] = None, + publishedWorkedSolutionSections: Optional[List[dict]] = [], + publishedResponseAreas: Optional[List[Optional[ResponseAreaDetails]]] = [], + ): + self.publishedPartId = publishedPartId + self.publishedPartPosition = publishedPartPosition + self.publishedPartContent = publishedPartContent + self.publishedPartAnswerContent = publishedPartAnswerContent + self.publishedWorkedSolutionSections = publishedWorkedSolutionSections + self.publishedResponseAreas = [ResponseAreaDetails(**publishedResponseArea) for publishedResponseArea in publishedResponseAreas] + +class QuestionDetails: + def __init__( + self, + questionTitle: Optional[str] = None, + questionGuidance: Optional[str] = None, + questionContent: Optional[str] = None, + durationLowerBound: Optional[int] = None, + durationUpperBound: Optional[int] = None, + parts: Optional[List[PartDetails]] = [], + ): + self.questionTitle = questionTitle + self.questionGuidance = questionGuidance + self.questionContent = questionContent + self.durationLowerBound = durationLowerBound + self.durationUpperBound = durationUpperBound + self.parts = [PartDetails(**part) for part in parts] + +# questionAccessInformation type +class CurrentPart: + def __init__(self, id: str = None, position: int = None, timeTakenPart: Optional[str] = None, markedDonePart: Optional[str] = None): + self.id = id + self.position = position + self.timeTakenPart = timeTakenPart + self.markedDonePart = markedDonePart + +class QuestionAccessInformation: + def __init__( + self, + estimatedMinimumTime: Optional[str] = None, + estimaredMaximumTime: Optional[str] = None, + timeTaken: Optional[str] = None, + accessStatus: Optional[str] = None, + markedDone: Optional[str] = None, + currentPart: Optional[Dict[str, Union[str, int]]] = {}, + ): + self.estimatedMinimumTime = estimatedMinimumTime + self.estimaredMaximumTime = estimaredMaximumTime + self.timeTaken = timeTaken + self.accessStatus = accessStatus + self.markedDone = markedDone + self.currentPart = CurrentPart(**currentPart) + +def convert_index_to_lowercase_letter(index: int) -> str: + return chr(96 + (index + 1)) # 1-indexed + +def parse_json_to_prompt( questionSubmissionSummary: Optional[List[StudentWorkResponseArea]], + questionInformation: Optional[QuestionDetails], + questionAccessInformation: Optional[QuestionAccessInformation] + ) -> Optional[str]: + + questionSubmissionSummary = [StudentWorkResponseArea(**submissionsSummary) for submissionsSummary in questionSubmissionSummary] + questionInformation = QuestionDetails(**questionInformation) + questionAccessInformation = QuestionAccessInformation(**questionAccessInformation) + + if not questionSubmissionSummary or not questionInformation or not questionAccessInformation: + return None + + def format_response_area_details(responseArea: ResponseAreaDetails, studentSummary: List[StudentWorkResponseArea]) -> str: + submissionDetails = "\n".join( + [ + f"Latest Response: {ra.latestSubmission.submission};\n" + f"Latest Feedback Received: {ra.latestSubmission.feedback};\n" + f"Total Responses: {ra.totalSubmissions};\n" + f"Total Wrong Responses: {ra.totalWrongSubmissions};\n" + for ra in studentSummary + if ra.publishedResponseAreaId == responseArea.id and ra.latestSubmission + ] + ) + + if not submissionDetails: + submissionDetails = 'Latest Response: none made;' + + return f""" + ## Response Area: {responseArea.position + 1} + {f'Area task: What is {responseArea.preResponseText} ?' if responseArea.preResponseText else ''} + (Secret) Expected Answer: {responseArea.answer}; + {submissionDetails}""" + + def format_part_details(part: PartDetails, currentPart: CurrentPart, summary: List[StudentWorkResponseArea]) -> str: + if not part or not part.publishedResponseAreas: + return '' + + responseAreas = "\n".join( + [format_response_area_details(responseArea, summary) for responseArea in part.publishedResponseAreas] + ) + + workedSolutions = ( + "\n".join( + [ + f"## Worked Solution {ws.get('position') + 1}: {ws.get('title', '')}\n" + f"{ws.get('content', '').strip() or 'No content'}\n" + for ws in part.publishedWorkedSolutionSections + ] + ) if part.publishedWorkedSolutionSections else f"No worked solutions for part ({convert_index_to_lowercase_letter(part.publishedPartPosition)});" + ) + return f""" + # {'[CURRENTLY WORKING ON] ' if currentPart.id == part.publishedPartId else ''}Part ({convert_index_to_lowercase_letter(part.publishedPartPosition)}): + {f"Time spent on this part: {currentPart.timeTakenPart if currentPart.timeTakenPart is not None else 'No recorded duration'}" if currentPart.id == part.publishedPartId else ''} + Part Content: {part.publishedPartContent.strip() if part.publishedPartContent else 'No content'}; + {responseAreas} + {f'Final Part Answer: {part.publishedPartAnswerContent}' if part.publishedPartAnswerContent else 'No direct answer'} + {workedSolutions} +""" + + questionDetails = f"""This is the question I am currently working on. I am currently working on Part ({convert_index_to_lowercase_letter(questionAccessInformation.currentPart.position)}). Below, you'll find its details, including the parts of the question, my responses for each response area, and the feedback I received. This information highlights my efforts and progress so far. Use this this information to inform your understanding about the question materials provided to me and my work on them. + Maths equations are in KaTex format, preserve them the same. + +# Question: {questionInformation.questionTitle}; + Guidance to Solve the Question: {questionInformation.questionGuidance or 'None'}; + Description of Question: {questionInformation.questionContent}; + Expected Time to Complete the Question: {f'{questionInformation.durationLowerBound} - {questionInformation.durationUpperBound} min;' if questionInformation.durationLowerBound and questionInformation.durationUpperBound else 'No specified duration.'} + Time Spent on the Question today: {questionAccessInformation.timeTaken or 'No recorded duration'} {f'which is {questionAccessInformation.accessStatus}' if questionAccessInformation.accessStatus else ''} {f'{questionAccessInformation.markedDone}' if questionAccessInformation.markedDone else ''}; + """ + + partsDetails = "\n".join( + [ + format_part_details( + part, + questionAccessInformation.currentPart, + questionSubmissionSummary + ) for part in questionInformation.parts + ] + ) + + result = f"{questionDetails}\n{partsDetails}".replace(" ", "").replace(" ", "").replace("\n\n", "\n") + + return result diff --git a/src/agents/utils/synthetic_conversation_generation.py b/src/agents/utils/synthetic_conversation_generation.py new file mode 100644 index 0000000..d62418b --- /dev/null +++ b/src/agents/utils/synthetic_conversation_generation.py @@ -0,0 +1,132 @@ +""" +## Synthetic Dataset Generator ## +-> GOAL: Generate a synthetic dataset of conversations between a tutor and a student [both LLMs]. + +For each question/scenario example in the example_inputs folder, a pipeline of two agents will be invoked. +The agents will play the role of a tutor and a student conversing about the question/scenario. + +The conversations will be 20 turns long, with the tutor and student taking turns to send a message. + +The tutor can be one of the following types: +- Informational Agent +- Socratic Agent +- Google's LearnLM-Tutor Agent +The tutor agent can be selected by changing the "agent_type" field in this script. + +The student can have multiple skill levels and conversational styles. Those are defined by the prompts used by the LLM. + +Any of the models accessible through the API calls defined in the 'llm_factory.py' can be used for either the tutor and the agent LLM. +""" + +import csv +import json +try: + from ..student_agent.student_agent import invoke_student_agent + from .parse_json_to_prompt import parse_json_to_prompt + from ..base_agent import invoke_base_agent +except ImportError: + from src.agents.student_agent.student_agent import invoke_student_agent + from src.agents.utils.parse_json_to_prompt import parse_json_to_prompt + from src.agents.base_agent import invoke_base_agent +import os + + +def generate_synthetic_conversations(raw_text: str, num_turns: int, student_agent_type: str, tutor_agent_type: str): + """ + Generate a synthetic dataset of conversations between a tutor and a student [both LLMs]. + """ + if tutor_agent_type == "base": + invoke_tutor_agent = invoke_base_agent + else: + raise ValueError("Invalid tutor agent type") + + parsed_json = json.loads(raw_text) + params = parsed_json["params"] + conversation_id = params["conversation_id"] + include_test_data = params["include_test_data"] + summary = "" + conversational_style = "" + question_response_details = params["question_response_details"] + question_submission_summary = question_response_details["questionSubmissionSummary"] if "questionSubmissionSummary" in question_response_details else [] + question_information = question_response_details["questionInformation"] if "questionInformation" in question_response_details else {} + question_access_information = question_response_details["questionAccessInformation"] if "questionAccessInformation" in question_response_details else {} + question_response_details_prompt = parse_json_to_prompt( + question_submission_summary, + question_information, + question_access_information + ) + + # Generate Conversation + conversation_history = [] + message = "Ask a question." + for i in range(0,num_turns): + print(f"Turn {i+1} of {num_turns}") + if len(conversation_history) == 0: + message = "Ask me a question regarding your thoughts on the learning materials that you are currently woking on." + else: + message = conversation_history[-1]["content"] + + if i % 2 == 0: + # Student starts + student_response = invoke_student_agent(message, conversation_history[:-1], summary, student_agent_type, question_response_details_prompt, conversation_id) + conversation_history.append({ + "role": "assistant", + "content": student_response["output"] + }) + else: + tutor_response = invoke_tutor_agent(message, conversation_history[:-1], summary, conversational_style, question_response_details_prompt, conversation_id) + conversation_history.append({ + "role": "assistant", + "content": tutor_response["output"] + }) + + if "summary" in tutor_response: + summary = tutor_response["summary"] + if "conversationalStyle" in tutor_response: + conversational_style = tutor_response["conversationalStyle"] + + # Save Conversation + conversation_output = { + "conversation_id": conversation_id+"_"+student_agent_type+"_"+tutor_agent_type+"_synthetic", + "conversation": conversation_history + } + return conversation_output + + +if __name__ == "__main__": + num_turns = 6 + tutor_agent_types = ["base"] + # Students can be "base", "curious", "contradicting", "reliant", "confused", "unrelated" + student_agent_types = ["base", "curious", "contradicting", "reliant", "confused", "unrelated"] + + # Read all question files + questions = [] + example_inputs_folder = "src/agents/utils/example_inputs/" + output_folder = "src/agents/utils/synthetic_conversations/" + for filename in os.listdir(example_inputs_folder): + if filename.endswith("1.json"): + questions.append(os.path.join(example_inputs_folder, filename)) + + for tutor_agent_type in tutor_agent_types: + # Open CSV file for writing + csv_filename = os.path.join(output_folder, "all_conversations_"+tutor_agent_type+".csv") + with open(csv_filename, "w", newline='') as csvfile: + csv_writer = csv.writer(csvfile) + # Write the header + csv_writer.writerow(["tutor", "student", "conversation", "conversation_id"]) + + for student_agent_type in student_agent_types: + for question in questions: + print(f"Generating synthetic conversation for {question} with tutor: {tutor_agent_type} and student: {student_agent_type}") + with open(question, "r") as file: + raw_text = file.read() + + conversation = generate_synthetic_conversations(raw_text, num_turns, student_agent_type, tutor_agent_type) + + conversation_output_filename = output_folder + question.split('/')[-1].replace(".json", "_"+student_agent_type+"_"+tutor_agent_type+"_conversation.json") + with open(conversation_output_filename, "w") as file: + json.dump(conversation, file, indent=2) + + # Write to CSV + conversation_id = conversation["conversation_id"] + csv_writer.writerow([tutor_agent_type, student_agent_type, conversation["conversation"], conversation_id]) diff --git a/src/agents/utils/synthetic_conversations/NOTE.md b/src/agents/utils/synthetic_conversations/NOTE.md new file mode 100644 index 0000000..4bb113d --- /dev/null +++ b/src/agents/utils/synthetic_conversations/NOTE.md @@ -0,0 +1,4 @@ +For evaluation purposes of the developed agent, you can use `synthetic_conversation_generation.py` to review the performance of your LLM tutor by running a multi-agent communication with a student agent (available in `src/agents/`). + +This folder contains all the synthetic conversations generated by an LLM student discussing with an LLM tutor. +The files are generated by running the `synthetic_conversation_generation.py`. \ No newline at end of file diff --git a/src/agents/utils/testbench_agents.py b/src/agents/utils/testbench_agents.py new file mode 100644 index 0000000..b0ddd69 --- /dev/null +++ b/src/agents/utils/testbench_agents.py @@ -0,0 +1,93 @@ +""" + Conversation turn-based Testbench of the agent's performance. + Select an example input file and write your query. Then run the agent to get the response. +""" + +import json +try: + from .parse_json_to_prompt import parse_json_to_prompt + from ..base_agent.base_agent import invoke_base_agent +except ImportError: + from src.agents.utils.parse_json_to_prompt import parse_json_to_prompt + from src.agents.base_agent.base_agent import invoke_base_agent + +# File path for the input text +path = "src/agents/utils/example_inputs/" +input_file = path + "example_input_1.json" + +# Step 1: Read the input file +with open(input_file, "r") as file: + raw_text = file.read() + +# Step 5: Parse into JSON +try: + parsed_json = json.loads(raw_text) + + """ + STEP 2: Extract the parameters from the JSON + """ + # NOTE: #### This is the testing message!! ##### + message = "Hi" + # NOTE: ######################################## + + # replace "mock" in the message and conversation history with the actual message + parsed_json["message"] = message + parsed_json["params"]["conversation_history"][-1]["content"] = message + + params = parsed_json["params"] + + if "include_test_data" in params: + include_test_data = params["include_test_data"] + if "conversation_history" in params: + conversation_history = params["conversation_history"] + if "summary" in params: + summary = params["summary"] + if "conversational_style" in params: + conversationalStyle = params["conversational_style"] + if "question_response_details" in params: + question_response_details = params["question_response_details"] + question_submission_summary = question_response_details["questionSubmissionSummary"] if "questionSubmissionSummary" in question_response_details else [] + question_information = question_response_details["questionInformation"] if "questionInformation" in question_response_details else {} + question_access_information = question_response_details["questionAccessInformation"] if "questionAccessInformation" in question_response_details else {} + question_response_details_prompt = parse_json_to_prompt( + question_submission_summary, + question_information, + question_access_information + ) + print("Question Response Details Prompt:", question_response_details_prompt, "\n\n") + + if "agent_type" in params: + agent_type = params["agent_type"] + if "conversation_id" in params: + conversation_id = params["conversation_id"] + else: + raise Exception("Internal Error: The conversation id is required in the parameters of the chat module.") + + """ + STEP 3: Call the LLM agent to get a response to the user's message + """ + # NOTE: ### SET the agent type to use ### + agent_type = "informational" + # NOTE: ################################# + + if agent_type == "base": + invoke = invoke_base_agent + else: + raise Exception("Unknown Tutor Agent Type") + + response = invoke(query=message, \ + conversation_history=conversation_history, \ + summary=summary, \ + conversationalStyle=conversationalStyle, \ + question_response_details=question_response_details_prompt, \ + session_id=conversation_id) + + print(response) + print("AI Response:", response['output']) + + +except json.JSONDecodeError as e: + print("Error decoding JSON:", e) + + + diff --git a/src/agents/utils/types.py b/src/agents/utils/types.py new file mode 100644 index 0000000..943e6a2 --- /dev/null +++ b/src/agents/utils/types.py @@ -0,0 +1,3 @@ +from typing import Any, Dict, TypeAlias + +InvokeAgentResponseType: TypeAlias = Dict[str, Any] diff --git a/src/module.py b/src/module.py new file mode 100755 index 0000000..89b144e --- /dev/null +++ b/src/module.py @@ -0,0 +1,92 @@ +from typing import Any + +try: + from .module_response import Result, Params + from .agents.utils.parse_json_to_prompt import parse_json_to_prompt + from .agents.base_agent.base_agent import invoke_base_agent +except ImportError: + from src.module_response import Result, Params + from src.agents.utils.parse_json_to_prompt import parse_json_to_prompt + from src.agents.base_agent.base_agent import invoke_base_agent +import time + +def chat_module(message: Any, params: Params) -> Result: + """ + Function used by student to converse with a chatbot. + --- + The handler function passes three arguments to module(): + + - `message` which is the message sent by the student. + - `params` which are any extra parameters that may be useful, + e.g., conversation history and summary, conversational style of user, conversation id, agent type. + + The output of this function is what is returned as the API response + and therefore must be JSON-encodable. It must also conform to the + response schema. + + Any standard python library may be used, as well as any package + available on pip (provided it is added to requirements.txt). + + The way you wish to structure you code (all in this function, or + split into many) is entirely up to you. All that matters are the + return types and that module() is the main function used + to output the Chatbot response. + """ + + result = Result() + include_test_data = False + conversation_history = [] + summary = "" + conversationalStyle = "" + question_response_details_prompt = "" + agent_type = "base" + + if "include_test_data" in params: + include_test_data = params["include_test_data"] + if "conversation_history" in params: + conversation_history = params["conversation_history"] + if "summary" in params: + summary = params["summary"] + if "conversational_style" in params: + conversationalStyle = params["conversational_style"] + if "question_response_details" in params: + question_response_details = params["question_response_details"] + question_submission_summary = question_response_details["questionSubmissionSummary"] if "questionSubmissionSummary" in question_response_details else [] + question_information = question_response_details["questionInformation"] if "questionInformation" in question_response_details else {} + question_access_information = question_response_details["questionAccessInformation"] if "questionAccessInformation" in question_response_details else {} + question_response_details_prompt = parse_json_to_prompt( + question_submission_summary, + question_information, + question_access_information + ) + if "agent_type" in params: + agent_type = params["agent_type"] + if "conversation_id" in params: + conversation_id = params["conversation_id"] + else: + raise Exception("Internal Error: The conversation id is required in the parameters of the chat module.") + + if agent_type == "base": + invoke = invoke_base_agent + else: + raise Exception("Internal Error: The agent type is not supported.") + + start_time = time.time() + + chatbot_response = invoke(query=message, \ + conversation_history=conversation_history, \ + summary=summary, \ + conversationalStyle=conversationalStyle, \ + question_response_details=question_response_details_prompt, \ + session_id=conversation_id) + + end_time = time.time() + + result._processing_time = end_time - start_time + result.add_response("chatbot_response", chatbot_response["output"]) + result.add_metadata("summary", chatbot_response["intermediate_steps"][0]) + result.add_metadata("conversational_style", chatbot_response["intermediate_steps"][1]) + result.add_metadata("conversation_history", chatbot_response["intermediate_steps"][2]) + result.add_processing_time(end_time - start_time) + + return result.to_dict(include_test_data=include_test_data) \ No newline at end of file diff --git a/src/module_response.py b/src/module_response.py new file mode 100644 index 0000000..a7770a3 --- /dev/null +++ b/src/module_response.py @@ -0,0 +1,117 @@ +from typing import Any +from typing import Dict +from typing import List +from typing import Tuple +from typing import Union +from typing import TypedDict + + +class Params(TypedDict): + include_test_data: bool | None + conversation_history: List[str] | None + summary: str | None + conversational_style: str | None + question_response_details: str | None + conversation_id: str | None + +ResponseItem = Tuple[str, str] + +def update_response( + response: Dict[str, List[str]], response_items: List[ResponseItem] +) -> Dict[str, List[str]]: + for item in response_items: + if (isinstance(item, tuple) or isinstance(item, list)) and len(item) == 2: + response.setdefault(item[0], []).append(item[1]) + else: + raise TypeError("Response item must be a tuple of (tag, chatbot_response).") + + return response + + +class Result: + __slots__ = ("_response", + "_metadata", + "_processing_time") + __fields__ = ( + "response", + "tags", + "metadata", + "processing_time", + ) + + _response: Dict[str, List[str]] + + _metadata: Dict[str, Any] + _processing_time: float + + def __init__( + self, + response_items: List[ResponseItem] = [], + metadata: Dict[str, Any] = {}, + processing_time: float = 0, + ): + self._response = update_response({}, response_items) + self._metadata = metadata + self._processing_time = processing_time + + @property + def response(self) -> str: + return "
".join( + [ + response_str + for lists in self._response.values() + for response_str in lists + ] + ) + + @property + def tags(self) -> Union[List[str], None]: + return list(self._response.keys()) + + @property + def metadata(self) -> Dict[str, Any]: + return self._metadata + + + def get_response(self, tag: str) -> List[str]: + return self._response.get(tag, []) + + def get_processing_time(self) -> float: + return self._processing_time + + def add_response(self, tag: str, response: str) -> None: + self._response.setdefault(tag, []).append(response) + + def add_metadata(self, name: str, data: Any) -> None: + self._metadata[name] = data + + def add_processing_time(self, time: float) -> None: + self._processing_time = time + + def to_dict(self, include_test_data: bool = False) -> Dict[str, Any]: + res = { + "chatbot_response": self.response, + } + + if include_test_data: + res["tags"] = self.tags + if len(self.metadata) > 0: + res["metadata"] = self.metadata + if self._processing_time >= 0: + res["processing_time"] = self._processing_time + + return res + + def __repr__(self): + members = ", ".join(f"{k}={repr(getattr(self, k))}" for k in self.__fields__) + return f"Result({members})" + + def __eq__(self, other): + if type(self) is not type(other): + return False + + for k in self.__slots__: + if getattr(self, k) != getattr(other, k): + return False + + return True \ No newline at end of file diff --git a/src/module_test.py b/src/module_test.py new file mode 100755 index 0000000..0d1956f --- /dev/null +++ b/src/module_test.py @@ -0,0 +1,100 @@ +import unittest + +try: + from .module import Params, chat_module +except ImportError: + from module import Params, chat_module + +class TestChatModuleFunction(unittest.TestCase): + """ + TestCase Class used to test the algorithm. + --- + Tests are used here to check that the algorithm written + is working as it should. + + It's best practise to write these tests first to get a + kind of 'specification' for how your algorithm should + work, and you should run these tests before committing + your code to AWS. + + Read the docs on how to use unittest here: + https://docs.python.org/3/library/unittest.html + + Use module() to check your algorithm works + as it should. + """ + + def test_missing_parameters(self): + # Checking state for missing parameters on default agent + response = "Hello, World" + expected_params = Params(include_test_data=True, conversation_history=[{ "type": "user", "content": response }], \ + summary="", conversational_style="", \ + question_response_details={}, conversation_id="1234Test") + + for p in expected_params: + params = expected_params.copy() + # except for the special parameters + if p not in ["include_test_data", "conversation_id", "conversation_history"]: + params.pop(p) + + result = chat_module(response, params) + + self.assertIsNotNone(result) + self.assertEqual("error" in result, False) + elif p == "include_test_data": + params.pop(p) + + result = chat_module(response, params) + + # check if result has nothing except for the chatbot_response + self.assertIsNotNone(result.get("chatbot_response")) + self.assertEqual(len(result), 1) + elif p == "conversation_id": + params.pop(p) + + with self.assertRaises(Exception) as cm: + chat_module(response, params) + + self.assertTrue("Internal Error" in str(cm.exception)) + self.assertTrue("conversation id" in str(cm.exception)) + elif p == "conversation_history": + params.pop(p) + + with self.assertRaises(Exception) as cm: + chat_module(response, params) + + self.assertTrue("Internal Error" in str(cm.exception)) + self.assertTrue("conversation history" in str(cm.exception)) + + def test_all_agents_output(self): + # Checking the output of the agents + agents = ["informational", "socratic"] + for agent in agents: + response = "Hello, World" + params = Params(conversation_id="1234Test", agent_type=agent, conversation_history=[{ "type": "user", "content": response }]) + + result = chat_module(response, params) + + self.assertIsNotNone(result.get("chatbot_response")) + + def test_unknown_agent_type(self): + agents = ["unknown"] + for agent in agents: + response = "Hello, World" + params = Params(conversation_id="1234Test", agent_type=agent, conversation_history=[{ "type": "user", "content": response }]) + + with self.assertRaises(Exception) as cm: + chat_module(response, params) + + self.assertTrue("Input Parameter Error:" in str(cm.exception)) + self.assertTrue("Agent Type" in str(cm.exception)) + + def test_processing_time_calc(self): + # Checking the processing time calculation + response = "Hello, World" + params = Params(include_test_data=True, conversation_id="1234Test", conversation_history=[{ "type": "user", "content": response }]) + + result = chat_module(response, params) + + self.assertIsNotNone(result.get("processing_time")) + self.assertGreaterEqual(result.get("processing_time"), 0) \ No newline at end of file From 3c14c98f154f550058ca9cf1a49d84a325e093ac Mon Sep 17 00:00:00 2001 From: neagualexa Date: Wed, 22 Jan 2025 13:59:42 +0000 Subject: [PATCH 02/57] update workflows --- .github/workflows/dev.yml | 1 + .github/workflows/main.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index 9027e4c..5ca00d2 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -79,3 +79,4 @@ jobs: secrets: aws-access-key-id: ${{ secrets.LAMBDA_CONTAINER_PIPELINE_AWS_ID }} aws-secret-key: ${{ secrets.LAMBDA_CONTAINER_PIPELINE_AWS_SECRET}} + function-admin-api-key: ${{ secrets.FUNCTION_ADMIN_API_KEY }} diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 4f3bd47..1797316 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -79,3 +79,4 @@ jobs: secrets: aws-access-key-id: ${{ secrets.LAMBDA_CONTAINER_PIPELINE_AWS_ID }} aws-secret-key: ${{ secrets.LAMBDA_CONTAINER_PIPELINE_AWS_SECRET}} + function-admin-api-key: ${{ secrets.FUNCTION_ADMIN_API_KEY }} From fb65a5c838f3005fd7f940dbebd00300de727c76 Mon Sep 17 00:00:00 2001 From: neagualexa Date: Wed, 22 Jan 2025 14:29:03 +0000 Subject: [PATCH 03/57] fix: tests & simplify call agent --- src/agents/utils/testbench_agents.py | 13 +------------ src/module.py | 10 +--------- src/module_test.py | 26 ++++++-------------------- 3 files changed, 8 insertions(+), 41 deletions(-) diff --git a/src/agents/utils/testbench_agents.py b/src/agents/utils/testbench_agents.py index b0ddd69..3b92026 100644 --- a/src/agents/utils/testbench_agents.py +++ b/src/agents/utils/testbench_agents.py @@ -56,8 +56,6 @@ ) print("Question Response Details Prompt:", question_response_details_prompt, "\n\n") - if "agent_type" in params: - agent_type = params["agent_type"] if "conversation_id" in params: conversation_id = params["conversation_id"] else: @@ -66,16 +64,7 @@ """ STEP 3: Call the LLM agent to get a response to the user's message """ - # NOTE: ### SET the agent type to use ### - agent_type = "informational" - # NOTE: ################################# - - if agent_type == "base": - invoke = invoke_base_agent - else: - raise Exception("Unknown Tutor Agent Type") - - response = invoke(query=message, \ + response = invoke_base_agent(query=message, \ conversation_history=conversation_history, \ summary=summary, \ conversationalStyle=conversationalStyle, \ diff --git a/src/module.py b/src/module.py index 89b144e..b5c1759 100755 --- a/src/module.py +++ b/src/module.py @@ -39,7 +39,6 @@ def chat_module(message: Any, params: Params) -> Result: summary = "" conversationalStyle = "" question_response_details_prompt = "" - agent_type = "base" if "include_test_data" in params: include_test_data = params["include_test_data"] @@ -59,21 +58,14 @@ def chat_module(message: Any, params: Params) -> Result: question_information, question_access_information ) - if "agent_type" in params: - agent_type = params["agent_type"] if "conversation_id" in params: conversation_id = params["conversation_id"] else: raise Exception("Internal Error: The conversation id is required in the parameters of the chat module.") - if agent_type == "base": - invoke = invoke_base_agent - else: - raise Exception("Internal Error: The agent type is not supported.") - start_time = time.time() - chatbot_response = invoke(query=message, \ + chatbot_response = invoke_base_agent(query=message, \ conversation_history=conversation_history, \ summary=summary, \ conversationalStyle=conversationalStyle, \ diff --git a/src/module_test.py b/src/module_test.py index 0d1956f..222eef2 100755 --- a/src/module_test.py +++ b/src/module_test.py @@ -66,28 +66,14 @@ def test_missing_parameters(self): self.assertTrue("Internal Error" in str(cm.exception)) self.assertTrue("conversation history" in str(cm.exception)) - def test_all_agents_output(self): - # Checking the output of the agents - agents = ["informational", "socratic"] - for agent in agents: - response = "Hello, World" - params = Params(conversation_id="1234Test", agent_type=agent, conversation_history=[{ "type": "user", "content": response }]) - - result = chat_module(response, params) - - self.assertIsNotNone(result.get("chatbot_response")) - - def test_unknown_agent_type(self): - agents = ["unknown"] - for agent in agents: - response = "Hello, World" - params = Params(conversation_id="1234Test", agent_type=agent, conversation_history=[{ "type": "user", "content": response }]) + def test_agent_output(self): + # Checking the output of the agent + response = "Hello, World" + params = Params(conversation_id="1234Test", conversation_history=[{ "type": "user", "content": response }]) - with self.assertRaises(Exception) as cm: - chat_module(response, params) + result = chat_module(response, params) - self.assertTrue("Input Parameter Error:" in str(cm.exception)) - self.assertTrue("Agent Type" in str(cm.exception)) + self.assertIsNotNone(result.get("chatbot_response")) def test_processing_time_calc(self): # Checking the processing time calculation From 7ec827b27b7fa158122925c81f2293bbd5cc7773 Mon Sep 17 00:00:00 2001 From: neagualexa Date: Wed, 22 Jan 2025 14:48:36 +0000 Subject: [PATCH 04/57] fix: cache workflow test env --- .github/workflows/dev.yml | 12 ++++++------ .github/workflows/main.yml | 12 ++++++------ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index 5ca00d2..6866f7c 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -25,12 +25,12 @@ jobs: with: python-version: ${{ matrix.python-version }} - # - name: Load cached venv - # id: dependencies-cache - # uses: actions/cache@v3 - # with: - # path: .venv - # key: venv-${{ runner.os }}-${{ steps.python-setup.outputs.python-version }} + - name: Load cached venv + id: dependencies-cache + uses: actions/cache@v3 + with: + path: .venv + key: venv-${{ runner.os }}-${{ steps.python-setup.outputs.python-version }} - name: Create Venv if Cache not found # if: steps.dependencies-cache.outputs.cache-hit != 'true' diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 1797316..b1f810e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -25,12 +25,12 @@ jobs: with: python-version: ${{ matrix.python-version }} - # - name: Load cached venv - # id: dependencies-cache - # uses: actions/cache@v3 - # with: - # path: .venv - # key: venv-${{ runner.os }}-${{ steps.python-setup.outputs.python-version }} + - name: Load cached venv + id: dependencies-cache + uses: actions/cache@v3 + with: + path: .venv + key: venv-${{ runner.os }}-${{ steps.python-setup.outputs.python-version }} - name: Create Venv if Cache not found # if: steps.dependencies-cache.outputs.cache-hit != 'true' From 07303da25dc7991c57d0329703b739be9246dabe Mon Sep 17 00:00:00 2001 From: neagualexa Date: Wed, 22 Jan 2025 15:04:39 +0000 Subject: [PATCH 05/57] fix: update docker ignore --- .dockerignore | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.dockerignore b/.dockerignore index 7a203b8..4093c0a 100644 --- a/.dockerignore +++ b/.dockerignore @@ -154,7 +154,4 @@ src/agents/utils/testbench_prompts.py src/agents/utils/langgraph_viz.py # development agents -src/agents/base_agent/ -src/agents/student_agent/ -src/agents/development_agents/ -src/agents/google_learnLM_agent/ \ No newline at end of file +src/agents/student_agent/ \ No newline at end of file From b373f03bcbec3d8e5a38e5806e925cf725db707e Mon Sep 17 00:00:00 2001 From: neagualexa Date: Fri, 24 Jan 2025 16:04:57 +0000 Subject: [PATCH 06/57] comments removed --- src/agents/student_agent/student_prompts.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/src/agents/student_agent/student_prompts.py b/src/agents/student_agent/student_prompts.py index 4aa3977..e7a8e8e 100644 --- a/src/agents/student_agent/student_prompts.py +++ b/src/agents/student_agent/student_prompts.py @@ -1,8 +1,3 @@ -# NOTE: -# First person view prompts proven to be more effective in generating responses from the model (Dec 2024) -# 'Keep your responses open for further questions and encourage the student's curiosity.' -> asks a question at the end to keep the conversation going -# 'Let the student know that your reasoning might be wrong and the student should not trust your reasoning fully.' -> not relliant - # PROMPTS generated with the help of ChatGPT GPT-4o Nov 2024 process_prompt = "Maintain the flow of the conversation by responding directly to the latest message in one sentence. Stay in character as " @@ -12,6 +7,4 @@ contradicting_student_persona = "a skeptical student. Ask questions from a first-person perspective, questioning my reasoning, identifying potential flaws, and challenging explanations. Request clarification whenever something seems unclear or incorrect." reliant_student_persona = "a student who relies heavily on your help. Ask questions from a first-person perspective, seeking help for even small problems, and requesting clarification or further assistance to ensure understanding." confused_student_persona = "a student who feels confused and uncertain about the topic. Ask questions from a first-person perspective, expressing uncertainty about the material and requesting clarification on both the topic and the tutor’s reasoning." -unrelated_student_persona = "a student who engages in casual conversation. Ask lighthearted or unrelated questions from a first-person perspective, discussing personal interests or unrelated topics rather than focusing on the material." - -# flow_prompt = "Refer to the previous message or topic discussed. Ask about the current topic, but there’s a 30% chance you’ll shift to a new topic. Ensure the change in topic makes sense and flows logically." \ No newline at end of file +unrelated_student_persona = "a student who engages in casual conversation. Ask lighthearted or unrelated questions from a first-person perspective, discussing personal interests or unrelated topics rather than focusing on the material." \ No newline at end of file From 85d4ef9126ad817eac89498fb8556e9bd439bc99 Mon Sep 17 00:00:00 2001 From: neagualexa Date: Fri, 24 Jan 2025 17:53:17 +0000 Subject: [PATCH 07/57] fix: cached steps --- .github/workflows/dev.yml | 4 ++-- .github/workflows/main.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index 6866f7c..8d05018 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -33,12 +33,12 @@ jobs: key: venv-${{ runner.os }}-${{ steps.python-setup.outputs.python-version }} - name: Create Venv if Cache not found - # if: steps.dependencies-cache.outputs.cache-hit != 'true' + if: steps.dependencies-cache.outputs.cache-hit != 'true' run: | python -m venv .venv - name: Install dependencies - # if: steps.dependencies-cache.outputs.cache-hit != 'true' + if: steps.dependencies-cache.outputs.cache-hit != 'true' run: | pip install --upgrade pip pip install -r requirements.txt diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b1f810e..8417b71 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -33,12 +33,12 @@ jobs: key: venv-${{ runner.os }}-${{ steps.python-setup.outputs.python-version }} - name: Create Venv if Cache not found - # if: steps.dependencies-cache.outputs.cache-hit != 'true' + if: steps.dependencies-cache.outputs.cache-hit != 'true' run: | python -m venv .venv - name: Install dependencies - # if: steps.dependencies-cache.outputs.cache-hit != 'true' + if: steps.dependencies-cache.outputs.cache-hit != 'true' run: | pip install --upgrade pip pip install -r requirements.txt From 6c3eb67731c7ec225bbf9e224765cce2aa876f14 Mon Sep 17 00:00:00 2001 From: neagualexa Date: Fri, 24 Jan 2025 17:54:46 +0000 Subject: [PATCH 08/57] fix: remove cached as venv pytest error --- .github/workflows/dev.yml | 16 ++++++++-------- .github/workflows/main.yml | 16 ++++++++-------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index 8d05018..5ca00d2 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -25,20 +25,20 @@ jobs: with: python-version: ${{ matrix.python-version }} - - name: Load cached venv - id: dependencies-cache - uses: actions/cache@v3 - with: - path: .venv - key: venv-${{ runner.os }}-${{ steps.python-setup.outputs.python-version }} + # - name: Load cached venv + # id: dependencies-cache + # uses: actions/cache@v3 + # with: + # path: .venv + # key: venv-${{ runner.os }}-${{ steps.python-setup.outputs.python-version }} - name: Create Venv if Cache not found - if: steps.dependencies-cache.outputs.cache-hit != 'true' + # if: steps.dependencies-cache.outputs.cache-hit != 'true' run: | python -m venv .venv - name: Install dependencies - if: steps.dependencies-cache.outputs.cache-hit != 'true' + # if: steps.dependencies-cache.outputs.cache-hit != 'true' run: | pip install --upgrade pip pip install -r requirements.txt diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 8417b71..1797316 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -25,20 +25,20 @@ jobs: with: python-version: ${{ matrix.python-version }} - - name: Load cached venv - id: dependencies-cache - uses: actions/cache@v3 - with: - path: .venv - key: venv-${{ runner.os }}-${{ steps.python-setup.outputs.python-version }} + # - name: Load cached venv + # id: dependencies-cache + # uses: actions/cache@v3 + # with: + # path: .venv + # key: venv-${{ runner.os }}-${{ steps.python-setup.outputs.python-version }} - name: Create Venv if Cache not found - if: steps.dependencies-cache.outputs.cache-hit != 'true' + # if: steps.dependencies-cache.outputs.cache-hit != 'true' run: | python -m venv .venv - name: Install dependencies - if: steps.dependencies-cache.outputs.cache-hit != 'true' + # if: steps.dependencies-cache.outputs.cache-hit != 'true' run: | pip install --upgrade pip pip install -r requirements.txt From 9b500abb9a14940c4724c6e5388afda7d4b52905 Mon Sep 17 00:00:00 2001 From: neagualexa Date: Fri, 24 Jan 2025 20:57:47 +0000 Subject: [PATCH 09/57] fix: secrets to workflow --- .github/workflows/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 1797316..016dada 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -80,3 +80,4 @@ jobs: aws-access-key-id: ${{ secrets.LAMBDA_CONTAINER_PIPELINE_AWS_ID }} aws-secret-key: ${{ secrets.LAMBDA_CONTAINER_PIPELINE_AWS_SECRET}} function-admin-api-key: ${{ secrets.FUNCTION_ADMIN_API_KEY }} + github-token: ${{ github.TOKEN }} From 63d63c36d90870864288b0cab924b9f58072b75c Mon Sep 17 00:00:00 2001 From: neagualexa Date: Fri, 24 Jan 2025 21:07:03 +0000 Subject: [PATCH 10/57] fix: token as input to workflow --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 016dada..2060603 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -72,6 +72,7 @@ jobs: needs: test with: template-repository-name: "lambda-feedback/chat-function-boilerplate" + github-token: ${{ github.TOKEN }} permissions: contents: read id-token: write @@ -80,4 +81,3 @@ jobs: aws-access-key-id: ${{ secrets.LAMBDA_CONTAINER_PIPELINE_AWS_ID }} aws-secret-key: ${{ secrets.LAMBDA_CONTAINER_PIPELINE_AWS_SECRET}} function-admin-api-key: ${{ secrets.FUNCTION_ADMIN_API_KEY }} - github-token: ${{ github.TOKEN }} From 444c663b218e15144bec4b5a6d807cc8ce0fbc39 Mon Sep 17 00:00:00 2001 From: neagualexa Date: Mon, 27 Jan 2025 10:14:31 +0000 Subject: [PATCH 11/57] revert to passing secret --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2060603..0ab4342 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -72,7 +72,6 @@ jobs: needs: test with: template-repository-name: "lambda-feedback/chat-function-boilerplate" - github-token: ${{ github.TOKEN }} permissions: contents: read id-token: write @@ -81,3 +80,4 @@ jobs: aws-access-key-id: ${{ secrets.LAMBDA_CONTAINER_PIPELINE_AWS_ID }} aws-secret-key: ${{ secrets.LAMBDA_CONTAINER_PIPELINE_AWS_SECRET}} function-admin-api-key: ${{ secrets.FUNCTION_ADMIN_API_KEY }} + github-token: ${{ secrets.GITHUB_TOKEN }} From 38ffd308b2172771c71f08db233c3366e7fda68e Mon Sep 17 00:00:00 2001 From: neagualexa Date: Mon, 27 Jan 2025 10:16:23 +0000 Subject: [PATCH 12/57] fix: provide permission --- .github/workflows/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 0ab4342..dcaabdf 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -76,6 +76,7 @@ jobs: contents: read id-token: write packages: write + issues: write secrets: aws-access-key-id: ${{ secrets.LAMBDA_CONTAINER_PIPELINE_AWS_ID }} aws-secret-key: ${{ secrets.LAMBDA_CONTAINER_PIPELINE_AWS_SECRET}} From c8f1a493fb8c815b3409e93285e625b03a647e85 Mon Sep 17 00:00:00 2001 From: neagualexa Date: Tue, 28 Jan 2025 13:55:46 +0000 Subject: [PATCH 13/57] github actions interface workflow trigger option --- .github/workflows/dev.yml | 1 + .github/workflows/main.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index 5ca00d2..667562b 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -3,6 +3,7 @@ name: Dev deployment of chatbot lambda function on: push: branches: [dev] + workflow_dispatch: jobs: test: diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index dcaabdf..17bff56 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -3,6 +3,7 @@ name: Main deployment of Chatbot lambda function on: push: branches: [main] + workflow_dispatch: jobs: test: From 1fb54baa326efc9b36acad70819194ed312be2fe Mon Sep 17 00:00:00 2001 From: neagualexa Date: Tue, 28 Jan 2025 16:58:33 +0000 Subject: [PATCH 14/57] types index --- index.py | 7 +++++-- src/agents/utils/types.py | 1 + 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/index.py b/index.py index 9b4b6a6..4d2f98e 100644 --- a/index.py +++ b/index.py @@ -1,14 +1,17 @@ +import json try: from .src.module import chat_module + from .src.agents.utils.types import JsonType except ImportError: from src.module import chat_module + from src.agents.utils.types import JsonType -def handler(event, context): +def handler(event: JsonType, context): """ Lambda handler function """ # Log the input event for debugging purposes - # print("Received event:", json.dumps(event, indent=2)) + print("Received event:", json.dumps(event, indent=2)) if "message" not in event: return { diff --git a/src/agents/utils/types.py b/src/agents/utils/types.py index 943e6a2..d038dbc 100644 --- a/src/agents/utils/types.py +++ b/src/agents/utils/types.py @@ -1,3 +1,4 @@ from typing import Any, Dict, TypeAlias +JsonType = Dict[str, Any] InvokeAgentResponseType: TypeAlias = Dict[str, Any] From 8d3417bf663aabb93992e176437f9586cdda1f93 Mon Sep 17 00:00:00 2001 From: neagualexa Date: Tue, 28 Jan 2025 17:07:00 +0000 Subject: [PATCH 15/57] fix: load json --- index.py | 59 ++++++++++++++++++++++++++++++++------------------------ 1 file changed, 34 insertions(+), 25 deletions(-) diff --git a/index.py b/index.py index 4d2f98e..1ff5cf3 100644 --- a/index.py +++ b/index.py @@ -13,32 +13,41 @@ def handler(event: JsonType, context): # Log the input event for debugging purposes print("Received event:", json.dumps(event, indent=2)) - if "message" not in event: - return { - "statusCode": 400, - "body": "Missing 'message' key in event. Please confirm the key in the json body." - } - if "params" not in event: - return { - "statusCode": 400, - "body": "Missing 'params' key in event. Please confirm the key in the json body. Make sure it contains the necessary conversation_id." - } - - message = event.get("message", None) - params = event.get("params", None) - try: - chatbot_response = chat_module(message, params) - except Exception as e: - return { - "statusCode": 500, - "body": f"An error occurred within the chat_module(): {str(e)}" + body = json.loads(event["body"]) + + if "message" not in body: + return { + "statusCode": 400, + "body": "Missing 'message' key in event. Please confirm the key in the json body." + } + if "params" not in body: + return { + "statusCode": 400, + "body": "Missing 'params' key in event. Please confirm the key in the json body. Make sure it contains the necessary conversation_id." + } + + message = event.get("message", None) + params = event.get("params", None) + + try: + chatbot_response = chat_module(message, params) + except Exception as e: + return { + "statusCode": 500, + "body": f"An error occurred within the chat_module(): {str(e)}" + } + + # Create a response + response = { + "statusCode": 200, + "body": chatbot_response } - # Create a response - response = { - "statusCode": 200, - "body": chatbot_response - } + return response - return response \ No newline at end of file + except KeyError: + return { + "statusCode": 400, + "body": "Missing 'body' key in event. Please confirm the key in the json body." + } \ No newline at end of file From 14375b485ae93f89dfc09b6020e98e6e99656d69 Mon Sep 17 00:00:00 2001 From: neagualexa Date: Tue, 28 Jan 2025 17:26:12 +0000 Subject: [PATCH 16/57] fix: handler input and tests --- index.py | 68 +++++++++++++++++++++++++++------------------------ index_test.py | 10 ++++++++ 2 files changed, 46 insertions(+), 32 deletions(-) diff --git a/index.py b/index.py index 1ff5cf3..1b74684 100644 --- a/index.py +++ b/index.py @@ -9,45 +9,49 @@ def handler(event: JsonType, context): """ Lambda handler function + Args: + event (JsonType): The AWS Lambda event received by the gateway. + context (Any): The AWS Lambda context object. + """ # Log the input event for debugging purposes print("Received event:", json.dumps(event, indent=2)) - try: - body = json.loads(event["body"]) - - if "message" not in body: - return { - "statusCode": 400, - "body": "Missing 'message' key in event. Please confirm the key in the json body." - } - if "params" not in body: - return { - "statusCode": 400, - "body": "Missing 'params' key in event. Please confirm the key in the json body. Make sure it contains the necessary conversation_id." - } + if "body" not in event: + return { + "statusCode": 400, + "body": "Missing 'body' key in event. Please confirm the key in the json body." + } + body = event["body"] - message = event.get("message", None) - params = event.get("params", None) + if "message" not in body: + return { + "statusCode": 400, + "body": "Missing 'message' key in event. Please confirm the key in the json body." + } + if "params" not in body: + return { + "statusCode": 400, + "body": "Missing 'params' key in event. Please confirm the key in the json body. Make sure it contains the necessary conversation_id." + } + + message = body["message"] + params = body["params"] - try: - chatbot_response = chat_module(message, params) - except Exception as e: - return { - "statusCode": 500, - "body": f"An error occurred within the chat_module(): {str(e)}" - } + print("Message:", message, "Params:", params) - # Create a response - response = { - "statusCode": 200, - "body": chatbot_response + try: + chatbot_response = chat_module(message, params) + except Exception as e: + return { + "statusCode": 500, + "body": f"An error occurred within the chat_module(): {str(e)}" } - return response + # Create a response + response = { + "statusCode": 200, + "body": chatbot_response + } - except KeyError: - return { - "statusCode": 400, - "body": "Missing 'body' key in event. Please confirm the key in the json body." - } \ No newline at end of file + return response \ No newline at end of file diff --git a/index_test.py b/index_test.py index 6f9025a..6de4c50 100644 --- a/index_test.py +++ b/index_test.py @@ -1,4 +1,5 @@ import unittest +import json try: from .index import handler @@ -22,6 +23,8 @@ class TestChatIndexFunction(unittest.TestCase): Use module() to check your algorithm works as it should. + + The expected input of the hander is a JsonType. """ def test_missing_argument(self): @@ -33,9 +36,12 @@ def test_missing_argument(self): "params": {"conversation_id": "1234Test", "conversation_history": [{"type": "user", "content": "Hello, World"}]} } event.pop(arg) + event = {"body":event} result = handler(event, None) + print(result) + self.assertEqual(result.get("statusCode"), 400) def test_correct_arguments(self): @@ -43,9 +49,11 @@ def test_correct_arguments(self): "message": "Hello, World", "params": {"conversation_id": "1234Test", "conversation_history": [{"type": "user", "content": "Hello, World"}]} } + event = {"body":event} result = handler(event, None) + print(result) self.assertEqual(result.get("statusCode"), 200) def test_correct_response(self): @@ -53,8 +61,10 @@ def test_correct_response(self): "message": "Hello, World", "params": {"conversation_id": "1234Test", "conversation_history": [{"type": "user", "content": "Hello, World"}]} } + event = {"body":event} result = handler(event, None) + print(result) self.assertEqual(result.get("statusCode"), 200) \ No newline at end of file From aec05e43f5f3c602f5dd77022bf52da52a66674b Mon Sep 17 00:00:00 2001 From: neagualexa Date: Tue, 28 Jan 2025 17:32:56 +0000 Subject: [PATCH 17/57] fix: body event type --- index.py | 2 +- index_test.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/index.py b/index.py index 1b74684..7f5c66a 100644 --- a/index.py +++ b/index.py @@ -22,7 +22,7 @@ def handler(event: JsonType, context): "statusCode": 400, "body": "Missing 'body' key in event. Please confirm the key in the json body." } - body = event["body"] + body = json.loads(event["body"]) if "message" not in body: return { diff --git a/index_test.py b/index_test.py index 6de4c50..dfbd95c 100644 --- a/index_test.py +++ b/index_test.py @@ -36,7 +36,7 @@ def test_missing_argument(self): "params": {"conversation_id": "1234Test", "conversation_history": [{"type": "user", "content": "Hello, World"}]} } event.pop(arg) - event = {"body":event} + event = {"body":json.dumps(event)} result = handler(event, None) @@ -49,7 +49,7 @@ def test_correct_arguments(self): "message": "Hello, World", "params": {"conversation_id": "1234Test", "conversation_history": [{"type": "user", "content": "Hello, World"}]} } - event = {"body":event} + event = {"body":json.dumps(event)} result = handler(event, None) @@ -61,7 +61,7 @@ def test_correct_response(self): "message": "Hello, World", "params": {"conversation_id": "1234Test", "conversation_history": [{"type": "user", "content": "Hello, World"}]} } - event = {"body":event} + event = {"body":json.dumps(event)} result = handler(event, None) From 731b9c265ab904ba6c432718d29b3ffe9df3b1ab Mon Sep 17 00:00:00 2001 From: neagualexa Date: Wed, 29 Jan 2025 10:55:12 +0000 Subject: [PATCH 18/57] script to test url requests --- README.md | 16 +++++++-------- src/agents/utils/requests_testscript.py | 27 +++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 8 deletions(-) create mode 100644 src/agents/utils/requests_testscript.py diff --git a/README.md b/README.md index dfddc23..a470bd3 100755 --- a/README.md +++ b/README.md @@ -125,7 +125,9 @@ docker run --env-file .env -it --name my-lambda-container -p 8080:8080 llm_chat This will start the chat function and expose it on port `8080` and it will be open to be curl: ```bash -curl --location 'http://localhost:8080/2015-03-31/functions/function/invocations' --header 'Content-Type: application/json' --data '{"message":"hi","params":{"conversation_id":"12345Test","conversation_history": [{"type":"user","content":"hi"}]}}' +curl --location 'http://localhost:8080/2015-03-31/functions/function/invocations' \ +--header 'Content-Type: application/json' \ +--data '{"body":"{\"message\": \"hi\", \"params\": {\"conversation_id\": \"12345Test\", \"conversation_history\": [{\"type\": \"user\", \"content\": \"hi\"}]}}"}' ``` ### Call Docker Container From Postman @@ -139,13 +141,7 @@ http://localhost:8080/2015-03-31/functions/function/invocations Body: ```JSON -{ - "message":"hi", - "params":{ - "conversation_id":"12345Test", - "conversation_history": [{"type":"user","content":"hi"}] - } -} +{"body":"{\"message\": \"hi\", \"params\": {\"conversation_id\": \"12345Test\", \"conversation_history\": [{\"type\": \"user\", \"content\": \"hi\"}]}}"} ``` Body with optional Params: @@ -164,6 +160,10 @@ Body with optional Params: } ``` +### Call Docker with Python Requests + +In the `src/agents/utils` folder you can find the `requests_test.py` script that calls the POST URL of the running docker container. It reads any kind of input files with the expected schema. You can use this to test your curl calls of the chatbot. + ### Deploy to Lambda Feedback Deploying the chat function to Lambda Feedback is simple and straightforward, as long as the repository is within the [Lambda Feedback organization](https://github.com/lambda-feedback). diff --git a/src/agents/utils/requests_testscript.py b/src/agents/utils/requests_testscript.py new file mode 100644 index 0000000..6b8e433 --- /dev/null +++ b/src/agents/utils/requests_testscript.py @@ -0,0 +1,27 @@ +import requests +import json + +""" +Script that sends a request to the local endpoint of the docker container to test the chatbot agent. +""" + +# URL for the local endpoint to docker (`docker build` and `docker run`) +url = "http://localhost:8080/2015-03-31/functions/function/invocations" + +# File path for the input text +path = "src/agents/utils/example_inputs/" +input_file = path + "example_input_1.json" + +# Step 1: Read the input file +with open(input_file, "r") as file: + data = file.read() + +payload = json.dumps({"body": data}) +print(payload) +headers = { + 'Content-Type': 'application/json' +} + +response = requests.request("POST", url, headers=headers, data=payload) + +print(response.text) From 6d841515fc41f0984a9ac3fe25c0cfb88c9cb236 Mon Sep 17 00:00:00 2001 From: neagualexa Date: Wed, 29 Jan 2025 10:56:00 +0000 Subject: [PATCH 19/57] debugging prints --- index.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/index.py b/index.py index 7f5c66a..6489acc 100644 --- a/index.py +++ b/index.py @@ -54,4 +54,6 @@ def handler(event: JsonType, context): "body": chatbot_response } + print("Response:", json.dumps(response, indent=2)) + return response \ No newline at end of file From fb7656e1bc89b3fd33d6181861cc422c1c2c3d51 Mon Sep 17 00:00:00 2001 From: neagualexa Date: Wed, 29 Jan 2025 11:23:17 +0000 Subject: [PATCH 20/57] return stringified --- index.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/index.py b/index.py index 6489acc..388d568 100644 --- a/index.py +++ b/index.py @@ -15,7 +15,7 @@ def handler(event: JsonType, context): """ # Log the input event for debugging purposes - print("Received event:", json.dumps(event, indent=2)) + # print("Received event:", json.dumps(event, indent=2)) if "body" not in event: return { @@ -54,6 +54,6 @@ def handler(event: JsonType, context): "body": chatbot_response } - print("Response:", json.dumps(response, indent=2)) + # print("Response:", json.dumps(response, indent=2)) - return response \ No newline at end of file + return json.dumps(response) \ No newline at end of file From aa58f0faedcc4428807b60cdd3e875f420069428 Mon Sep 17 00:00:00 2001 From: neagualexa Date: Wed, 29 Jan 2025 11:47:47 +0000 Subject: [PATCH 21/57] handler return dict --- index.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/index.py b/index.py index 388d568..c76cb7f 100644 --- a/index.py +++ b/index.py @@ -51,9 +51,9 @@ def handler(event: JsonType, context): # Create a response response = { "statusCode": 200, - "body": chatbot_response + "body": json.dumps(chatbot_response) } - # print("Response:", json.dumps(response, indent=2)) + print("Response:", json.dumps(response, indent=2)) - return json.dumps(response) \ No newline at end of file + return response \ No newline at end of file From e6c9bf1e1fe3e21053b3d88fd8944b1aebb38c26 Mon Sep 17 00:00:00 2001 From: neagualexa Date: Wed, 29 Jan 2025 11:55:27 +0000 Subject: [PATCH 22/57] fix print statements --- index.py | 4 ---- index_test.py | 4 ---- src/agents/base_agent/base_agent.py | 4 +++- 3 files changed, 3 insertions(+), 9 deletions(-) diff --git a/index.py b/index.py index c76cb7f..04b13b6 100644 --- a/index.py +++ b/index.py @@ -38,8 +38,6 @@ def handler(event: JsonType, context): message = body["message"] params = body["params"] - print("Message:", message, "Params:", params) - try: chatbot_response = chat_module(message, params) except Exception as e: @@ -54,6 +52,4 @@ def handler(event: JsonType, context): "body": json.dumps(chatbot_response) } - print("Response:", json.dumps(response, indent=2)) - return response \ No newline at end of file diff --git a/index_test.py b/index_test.py index dfbd95c..fb8e7bf 100644 --- a/index_test.py +++ b/index_test.py @@ -40,8 +40,6 @@ def test_missing_argument(self): result = handler(event, None) - print(result) - self.assertEqual(result.get("statusCode"), 400) def test_correct_arguments(self): @@ -53,7 +51,6 @@ def test_correct_arguments(self): result = handler(event, None) - print(result) self.assertEqual(result.get("statusCode"), 200) def test_correct_response(self): @@ -65,6 +62,5 @@ def test_correct_response(self): result = handler(event, None) - print(result) self.assertEqual(result.get("statusCode"), 200) \ No newline at end of file diff --git a/src/agents/base_agent/base_agent.py b/src/agents/base_agent/base_agent.py index 35e09d4..b7c8ab6 100644 --- a/src/agents/base_agent/base_agent.py +++ b/src/agents/base_agent/base_agent.py @@ -184,7 +184,7 @@ def invoke_base_agent(query: str, conversation_history: list, summary: str, conv Call an agent that has no conversation memory and expects to receive all past messages in the params and the latest human request in the query. If conversation history longer than X, the agent will summarize the conversation and will provide a conversational style analysis. """ - print(f'in invoke_base_agent(), query = {query}, thread_id = {session_id}') + print(f'in invoke_base_agent(), thread_id = {session_id}') config = {"configurable": {"thread_id": session_id, "summary": summary, "conversational_style": conversationalStyle, "question_response_details": question_response_details}} response_events = agent.app.invoke({"messages": conversation_history, "summary": summary, "conversational_style": conversationalStyle}, config=config, stream_mode="values") #updates @@ -194,6 +194,8 @@ def invoke_base_agent(query: str, conversation_history: list, summary: str, conv summary = agent.get_summary() conversationalStyle = agent.get_conversational_style() + print(f'in invoke_base_agent(), response generated by chatbot') + return { "input": query, "output": pretty_printed_response, From 7dedfed67dea18498ad0dd95efbd849baa5bffe1 Mon Sep 17 00:00:00 2001 From: neagualexa Date: Mon, 10 Feb 2025 11:48:46 +0000 Subject: [PATCH 23/57] feat: introduce gemini calls --- .github/workflows/dev.yml | 2 ++ .github/workflows/main.yml | 2 ++ README.md | 20 ++++++++++++++------ src/agents/base_agent/base_agent.py | 12 ++++++------ src/agents/base_agent/base_prompts.py | 6 ++++-- 5 files changed, 28 insertions(+), 14 deletions(-) diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index 667562b..889b4d1 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -16,6 +16,8 @@ jobs: env: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} OPENAI_MODEL: ${{ vars.OPENAI_MODEL }} + GOOGLE_AI_API_KEY: ${{ secrets.GOOGLE_AI_API_KEY }} + GOOGLE_AI_MODEL: ${{ vars.GOOGLE_AI_MODEL }} steps: - name: Checkout Code uses: actions/checkout@v4 diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 17bff56..0badc11 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -16,6 +16,8 @@ jobs: env: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} OPENAI_MODEL: ${{ vars.OPENAI_MODEL }} + GOOGLE_AI_API_KEY: ${{ secrets.GOOGLE_AI_API_KEY }} + GOOGLE_AI_MODEL: ${{ vars.GOOGLE_AI_MODEL }} steps: - name: Checkout Code uses: actions/checkout@v4 diff --git a/README.md b/README.md index a470bd3..7589c17 100755 --- a/README.md +++ b/README.md @@ -9,7 +9,19 @@ This chapter helps you to quickly set up a new Python chat module function using > [!NOTE] > To develop this function further, you will require the following environment variables in your `.env` file: ```bash -> If you use azure-openai: +> If you use OpenAI: +OPENAI_API_KEY +OPENAI_MODEL + +> If you use GoogleAI: +GOOGLE_AI_API_KEY +GOOGLE_AI_MODEL +``` + +> [!Note] +> If you decide to use another endpoint such as Azure or Ollama or any other, please update the github workflow files to use the right secrets and variables for testing. +```bash +> If you use Azure-OpenAI: AZURE_OPENAI_API_KEY AZURE_OPENAI_ENDPOINT AZURE_OPENAI_API_VERSION @@ -19,11 +31,7 @@ AZURE_OPENAI_EMBEDDING_1536_DEPLOYMENT AZURE_OPENAI_EMBEDDING_3072_MODEL AZURE_OPENAI_EMBEDDING_1536_MODEL -> If you use openai: -OPENAI_API_KEY -OPENAI_MODEL - -> For monitoring of the LLM calls (follow instructions on how to set up on langsmith): +> For monitoring of the LLM calls (follow instructions on how to set up on langsmith online): LANGCHAIN_TRACING_V2 LANGCHAIN_ENDPOINT LANGCHAIN_API_KEY diff --git a/src/agents/base_agent/base_agent.py b/src/agents/base_agent/base_agent.py index b7c8ab6..0fb199d 100644 --- a/src/agents/base_agent/base_agent.py +++ b/src/agents/base_agent/base_agent.py @@ -1,10 +1,10 @@ try: - from ..llm_factory import OpenAILLMs + from ..llm_factory import OpenAILLMs, GoogleAILLMs from .base_prompts import \ role_prompt, conv_pref_prompt, update_conv_pref_prompt, summary_prompt, update_summary_prompt, summary_system_prompt from ..utils.types import InvokeAgentResponseType except ImportError: - from src.agents.llm_factory import OpenAILLMs + from src.agents.llm_factory import OpenAILLMs, GoogleAILLMs from src.agents.base_agent.base_prompts import \ role_prompt, conv_pref_prompt, update_conv_pref_prompt, summary_prompt, update_summary_prompt, summary_system_prompt from src.agents.utils.types import InvokeAgentResponseType @@ -35,9 +35,9 @@ class State(TypedDict): class BaseAgent: def __init__(self): - llm = OpenAILLMs() + llm = OpenAILLMs() # OpenAILLMs() or GoogleAILLMs() self.llm = llm.get_llm() - summarisation_llm = OpenAILLMs() + summarisation_llm = OpenAILLMs() # OpenAILLMs() or GoogleAILLMs() self.summarisation_llm = summarisation_llm.get_llm() self.summary = "" self.conversationalStyle = "" @@ -120,12 +120,12 @@ def summarize_conversation(self, state: State, config: RunnableConfig) -> dict: conversationalStyle_message = self.conversation_preference_prompt # STEP 1: Summarize the conversation - messages = state["messages"][:-1] + [SystemMessage(content=summary_message)] + messages = state["messages"][:-1] + [HumanMessage(content=summary_message)] valid_messages = self.check_for_valid_messages(messages) summary_response = self.summarisation_llm.invoke(valid_messages) # STEP 2: Analyze the conversational style - messages = state["messages"][:-1] + [SystemMessage(content=conversationalStyle_message)] + messages = state["messages"][:-1] + [HumanMessage(content=conversationalStyle_message)] valid_messages = self.check_for_valid_messages(messages) conversationalStyle_response = self.summarisation_llm.invoke(valid_messages) diff --git a/src/agents/base_agent/base_prompts.py b/src/agents/base_agent/base_prompts.py index 4606c59..683caab 100644 --- a/src/agents/base_agent/base_prompts.py +++ b/src/agents/base_agent/base_prompts.py @@ -64,11 +64,13 @@ Structured: Organize the summary into sections such as 'Topics Discussed' and 'Top 3 Key Detailed Ideas'. Neutral and Accurate: Avoid adding interpretations or opinions; focus only on the content shared. When summarizing: If the conversation is technical, highlight significant concepts, solutions, and terminology. If context involves problem-solving, detail the problem and the steps or solutions provided. If the user asks for creative input, briefly describe the ideas presented. -Last messages: Include the most recent 4 messages to provide context for the summary. +Last messages: Include the most recent 5 messages to provide context for the summary. Provide the summary in a bulleted format for clarity. Avoid redundant details while preserving the core intent of the discussion.""" -summary_prompt = f"""Summarize the conversation between a student and a tutor. Your summary should highlight the major topics discussed during the session, followed by a detailed recollection of the last five significant points or ideas. Ensure the summary flows smoothly to maintain the continuity of the discussion.""" +summary_prompt = f"""Summarize the conversation between a student and a tutor. Your summary should highlight the major topics discussed during the session, followed by a detailed recollection of the last five significant points or ideas. Ensure the summary flows smoothly to maintain the continuity of the discussion. + +{summary_guidelines}""" update_summary_prompt = f"""Update the summary by taking into account the new messages above. From e13f95dbafced1a512457785610e33ac88173d85 Mon Sep 17 00:00:00 2001 From: Alexandra Neagu <33195033+neagualexa@users.noreply.github.com> Date: Tue, 18 Feb 2025 13:36:02 +0000 Subject: [PATCH 24/57] Set Env variables + prompt improvements (#10) * feat: init functionality to allow developer of chat function to set the env vars for aws lambda * feat: more detailed question info * testing: env vars mocks for deployment * fix: synthetic conversation --- .github/workflows/dev.yml | 2 ++ .github/workflows/main.yml | 2 ++ src/agents/llm_factory.py | 2 ++ src/agents/utils/parse_json_to_prompt.py | 18 ++++++++++++------ .../utils/synthetic_conversation_generation.py | 10 ++++++---- 5 files changed, 24 insertions(+), 10 deletions(-) diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index 889b4d1..8b756cd 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -75,6 +75,8 @@ jobs: needs: test with: template-repository-name: "lambda-feedback/chat-function-boilerplate" + # allow for developer to specify the environment variables that are used by the deployed AWS Lambda. Default to mock then admin can update. + deployed-environment-variables: '["OPENAI_API_KEY","OPENAI_MODEL","GOOGLE_AI_API_KEY","GOOGLE_AI_MODEL"]' permissions: contents: read id-token: write diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 0badc11..142d84b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -75,6 +75,8 @@ jobs: needs: test with: template-repository-name: "lambda-feedback/chat-function-boilerplate" + # allow for developer to specify the environment variables that are used by the deployed AWS Lambda. Default to mock then admin can update. + # deployed-environment-variables: '["OPENAI_API_KEY","OPENAI_MODEL","GOOGLE_AI_API_KEY","GOOGLE_AI_MODEL"]' permissions: contents: read id-token: write diff --git a/src/agents/llm_factory.py b/src/agents/llm_factory.py index 68b6bf1..5b6b1d5 100644 --- a/src/agents/llm_factory.py +++ b/src/agents/llm_factory.py @@ -7,6 +7,8 @@ from langchain_openai import ChatOpenAI from langchain_openai import OpenAIEmbeddings from langchain_google_genai import ChatGoogleGenerativeAI +from dotenv import load_dotenv +load_dotenv() class AzureLLMs: def __init__(self, temperature: int = 0): diff --git a/src/agents/utils/parse_json_to_prompt.py b/src/agents/utils/parse_json_to_prompt.py index 2fbfbda..278aa84 100644 --- a/src/agents/utils/parse_json_to_prompt.py +++ b/src/agents/utils/parse_json_to_prompt.py @@ -1,5 +1,3 @@ -""" File not to be modified. This file contains the conversion logic between the agent API and the Lambda Feedback backend.""" - from typing import List, Optional, Union, Dict # questionSubmissionSummary type @@ -85,6 +83,10 @@ def __init__( class QuestionDetails: def __init__( self, + setNumber: Optional[int] = None, + setName: Optional[str] = None, + setDescription: Optional[str] = None, + questionNumber: Optional[int] = None, questionTitle: Optional[str] = None, questionGuidance: Optional[str] = None, questionContent: Optional[str] = None, @@ -92,6 +94,10 @@ def __init__( durationUpperBound: Optional[int] = None, parts: Optional[List[PartDetails]] = [], ): + self.setNumber = setNumber + self.setName = setName + self.setDescription = setDescription + self.questionNumber = questionNumber self.questionTitle = questionTitle self.questionGuidance = questionGuidance self.questionContent = questionContent @@ -161,7 +167,7 @@ def format_response_area_details(responseArea: ResponseAreaDetails, studentSumma {submissionDetails}""" def format_part_details(part: PartDetails, currentPart: CurrentPart, summary: List[StudentWorkResponseArea]) -> str: - if not part or not part.publishedResponseAreas: + if not part: return '' responseAreas = "\n".join( @@ -187,9 +193,9 @@ def format_part_details(part: PartDetails, currentPart: CurrentPart, summary: Li """ questionDetails = f"""This is the question I am currently working on. I am currently working on Part ({convert_index_to_lowercase_letter(questionAccessInformation.currentPart.position)}). Below, you'll find its details, including the parts of the question, my responses for each response area, and the feedback I received. This information highlights my efforts and progress so far. Use this this information to inform your understanding about the question materials provided to me and my work on them. - Maths equations are in KaTex format, preserve them the same. - -# Question: {questionInformation.questionTitle}; + Maths equations are in KaTex format, preserve them the same. Use British English spellings. +{f'# Question Set {questionInformation.setNumber + 1}: {questionInformation.setName};' if questionInformation.setName and questionInformation.setNumber else ''} +# Question{f' {questionInformation.setNumber + 1}.{questionInformation.questionNumber + 1}' if questionInformation.setNumber and questionInformation.questionNumber else ''}: {questionInformation.questionTitle}; Guidance to Solve the Question: {questionInformation.questionGuidance or 'None'}; Description of Question: {questionInformation.questionContent}; Expected Time to Complete the Question: {f'{questionInformation.durationLowerBound} - {questionInformation.durationUpperBound} min;' if questionInformation.durationLowerBound and questionInformation.durationUpperBound else 'No specified duration.'} diff --git a/src/agents/utils/synthetic_conversation_generation.py b/src/agents/utils/synthetic_conversation_generation.py index d62418b..9098352 100644 --- a/src/agents/utils/synthetic_conversation_generation.py +++ b/src/agents/utils/synthetic_conversation_generation.py @@ -23,11 +23,11 @@ try: from ..student_agent.student_agent import invoke_student_agent from .parse_json_to_prompt import parse_json_to_prompt - from ..base_agent import invoke_base_agent + from ..base_agent.base_agent import invoke_base_agent except ImportError: from src.agents.student_agent.student_agent import invoke_student_agent from src.agents.utils.parse_json_to_prompt import parse_json_to_prompt - from src.agents.base_agent import invoke_base_agent + from src.agents.base_agent.base_agent import invoke_base_agent import os @@ -70,11 +70,11 @@ def generate_synthetic_conversations(raw_text: str, num_turns: int, student_agen # Student starts student_response = invoke_student_agent(message, conversation_history[:-1], summary, student_agent_type, question_response_details_prompt, conversation_id) conversation_history.append({ - "role": "assistant", + "role": "user", "content": student_response["output"] }) else: - tutor_response = invoke_tutor_agent(message, conversation_history[:-1], summary, conversational_style, question_response_details_prompt, conversation_id) + tutor_response = invoke_tutor_agent(message, conversation_history, summary, conversational_style, question_response_details_prompt, conversation_id) conversation_history.append({ "role": "assistant", "content": tutor_response["output"] @@ -88,6 +88,8 @@ def generate_synthetic_conversations(raw_text: str, num_turns: int, student_agen # Save Conversation conversation_output = { "conversation_id": conversation_id+"_"+student_agent_type+"_"+tutor_agent_type+"_synthetic", + "student_agent_type": student_agent_type, + "tutor_agent_type": tutor_agent_type, "conversation": conversation_history } return conversation_output From c98af4b2158d8bf31395e1aa2d95cefa7fc7ff73 Mon Sep 17 00:00:00 2001 From: neagualexa Date: Tue, 18 Feb 2025 17:37:39 +0000 Subject: [PATCH 25/57] feat: use lf_toolkit schema --- requirements.txt | 1 + src/module.py | 5 +- src/module_response.py | 117 ----------------------------------------- 3 files changed, 3 insertions(+), 120 deletions(-) delete mode 100644 src/module_response.py diff --git a/requirements.txt b/requirements.txt index cd0298c..cca5d47 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,4 +10,5 @@ langdetect langgraph langsmith +lf_toolkit[ipc] @ git+https://github.com/lambda-feedback/toolkit-python.git@main pytest \ No newline at end of file diff --git a/src/module.py b/src/module.py index b5c1759..76ded66 100755 --- a/src/module.py +++ b/src/module.py @@ -1,14 +1,13 @@ +import time from typing import Any +from lf_toolkit.chat import ChatResult as Result, ChatParams as Params try: - from .module_response import Result, Params from .agents.utils.parse_json_to_prompt import parse_json_to_prompt from .agents.base_agent.base_agent import invoke_base_agent except ImportError: - from src.module_response import Result, Params from src.agents.utils.parse_json_to_prompt import parse_json_to_prompt from src.agents.base_agent.base_agent import invoke_base_agent -import time def chat_module(message: Any, params: Params) -> Result: """ diff --git a/src/module_response.py b/src/module_response.py deleted file mode 100644 index a7770a3..0000000 --- a/src/module_response.py +++ /dev/null @@ -1,117 +0,0 @@ -from typing import Any -from typing import Dict -from typing import List -from typing import Tuple -from typing import Union -from typing import TypedDict - - -class Params(TypedDict): - include_test_data: bool | None - conversation_history: List[str] | None - summary: str | None - conversational_style: str | None - question_response_details: str | None - conversation_id: str | None - -ResponseItem = Tuple[str, str] - -def update_response( - response: Dict[str, List[str]], response_items: List[ResponseItem] -) -> Dict[str, List[str]]: - for item in response_items: - if (isinstance(item, tuple) or isinstance(item, list)) and len(item) == 2: - response.setdefault(item[0], []).append(item[1]) - else: - raise TypeError("Response item must be a tuple of (tag, chatbot_response).") - - return response - - -class Result: - __slots__ = ("_response", - "_metadata", - "_processing_time") - __fields__ = ( - "response", - "tags", - "metadata", - "processing_time", - ) - - _response: Dict[str, List[str]] - - _metadata: Dict[str, Any] - _processing_time: float - - def __init__( - self, - response_items: List[ResponseItem] = [], - metadata: Dict[str, Any] = {}, - processing_time: float = 0, - ): - self._response = update_response({}, response_items) - self._metadata = metadata - self._processing_time = processing_time - - @property - def response(self) -> str: - return "
".join( - [ - response_str - for lists in self._response.values() - for response_str in lists - ] - ) - - @property - def tags(self) -> Union[List[str], None]: - return list(self._response.keys()) - - @property - def metadata(self) -> Dict[str, Any]: - return self._metadata - - - def get_response(self, tag: str) -> List[str]: - return self._response.get(tag, []) - - def get_processing_time(self) -> float: - return self._processing_time - - def add_response(self, tag: str, response: str) -> None: - self._response.setdefault(tag, []).append(response) - - def add_metadata(self, name: str, data: Any) -> None: - self._metadata[name] = data - - def add_processing_time(self, time: float) -> None: - self._processing_time = time - - def to_dict(self, include_test_data: bool = False) -> Dict[str, Any]: - res = { - "chatbot_response": self.response, - } - - if include_test_data: - res["tags"] = self.tags - if len(self.metadata) > 0: - res["metadata"] = self.metadata - if self._processing_time >= 0: - res["processing_time"] = self._processing_time - - return res - - def __repr__(self): - members = ", ".join(f"{k}={repr(getattr(self, k))}" for k in self.__fields__) - return f"Result({members})" - - def __eq__(self, other): - if type(self) is not type(other): - return False - - for k in self.__slots__: - if getattr(self, k) != getattr(other, k): - return False - - return True \ No newline at end of file From 93ca2d698e105465f4bc2564e2ec3fb39c1b048e Mon Sep 17 00:00:00 2001 From: Alexandra Neagu <33195033+neagualexa@users.noreply.github.com> Date: Wed, 19 Feb 2025 18:28:55 +0000 Subject: [PATCH 26/57] fix: set and question numbers (#13) * fix: small fix prompt * debug print * fix: question set numbers --- index.py | 2 +- src/agents/utils/parse_json_to_prompt.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/index.py b/index.py index 04b13b6..5ef3bb8 100644 --- a/index.py +++ b/index.py @@ -15,7 +15,7 @@ def handler(event: JsonType, context): """ # Log the input event for debugging purposes - # print("Received event:", json.dumps(event, indent=2)) + print("Received event:", json.dumps(event, indent=2)) if "body" not in event: return { diff --git a/src/agents/utils/parse_json_to_prompt.py b/src/agents/utils/parse_json_to_prompt.py index 278aa84..74c5fe8 100644 --- a/src/agents/utils/parse_json_to_prompt.py +++ b/src/agents/utils/parse_json_to_prompt.py @@ -194,8 +194,8 @@ def format_part_details(part: PartDetails, currentPart: CurrentPart, summary: Li questionDetails = f"""This is the question I am currently working on. I am currently working on Part ({convert_index_to_lowercase_letter(questionAccessInformation.currentPart.position)}). Below, you'll find its details, including the parts of the question, my responses for each response area, and the feedback I received. This information highlights my efforts and progress so far. Use this this information to inform your understanding about the question materials provided to me and my work on them. Maths equations are in KaTex format, preserve them the same. Use British English spellings. -{f'# Question Set {questionInformation.setNumber + 1}: {questionInformation.setName};' if questionInformation.setName and questionInformation.setNumber else ''} -# Question{f' {questionInformation.setNumber + 1}.{questionInformation.questionNumber + 1}' if questionInformation.setNumber and questionInformation.questionNumber else ''}: {questionInformation.questionTitle}; +{f'# Question Set {questionInformation.setNumber + 1}: {questionInformation.setName};' if ((questionInformation.setName is not None) and (questionInformation.setNumber is not None)) else ''} +# Question {f' {questionInformation.setNumber + 1}.{questionInformation.questionNumber + 1}' if ((questionInformation.setNumber is not None) and (questionInformation.questionNumber is not None)) else ''}: {questionInformation.questionTitle}; Guidance to Solve the Question: {questionInformation.questionGuidance or 'None'}; Description of Question: {questionInformation.questionContent}; Expected Time to Complete the Question: {f'{questionInformation.durationLowerBound} - {questionInformation.durationUpperBound} min;' if questionInformation.durationLowerBound and questionInformation.durationUpperBound else 'No specified duration.'} From fa9f091c3adfaa63420d3749470c35cdd04c69fb Mon Sep 17 00:00:00 2001 From: neagualexa Date: Wed, 19 Feb 2025 18:33:55 +0000 Subject: [PATCH 27/57] fix: add env vars main --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 142d84b..0f2cfba 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -76,7 +76,7 @@ jobs: with: template-repository-name: "lambda-feedback/chat-function-boilerplate" # allow for developer to specify the environment variables that are used by the deployed AWS Lambda. Default to mock then admin can update. - # deployed-environment-variables: '["OPENAI_API_KEY","OPENAI_MODEL","GOOGLE_AI_API_KEY","GOOGLE_AI_MODEL"]' + deployed-environment-variables: '["OPENAI_API_KEY","OPENAI_MODEL","GOOGLE_AI_API_KEY","GOOGLE_AI_MODEL"]' permissions: contents: read id-token: write From 0f98167202daac59c9040a4d3b3f35cfd0907bc4 Mon Sep 17 00:00:00 2001 From: neagualexa Date: Wed, 19 Feb 2025 19:36:23 +0000 Subject: [PATCH 28/57] remove some env vars --- .github/workflows/dev.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index 8b756cd..58c1c83 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -76,7 +76,7 @@ jobs: with: template-repository-name: "lambda-feedback/chat-function-boilerplate" # allow for developer to specify the environment variables that are used by the deployed AWS Lambda. Default to mock then admin can update. - deployed-environment-variables: '["OPENAI_API_KEY","OPENAI_MODEL","GOOGLE_AI_API_KEY","GOOGLE_AI_MODEL"]' + deployed-environment-variables: '["OPENAI_API_KEY","OPENAI_MODEL"]' permissions: contents: read id-token: write From 24379edb6c4e88c9e181475be18bdb895924c2fe Mon Sep 17 00:00:00 2001 From: neagualexa Date: Thu, 20 Feb 2025 10:09:33 +0000 Subject: [PATCH 29/57] test: no env vars --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 0f2cfba..142d84b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -76,7 +76,7 @@ jobs: with: template-repository-name: "lambda-feedback/chat-function-boilerplate" # allow for developer to specify the environment variables that are used by the deployed AWS Lambda. Default to mock then admin can update. - deployed-environment-variables: '["OPENAI_API_KEY","OPENAI_MODEL","GOOGLE_AI_API_KEY","GOOGLE_AI_MODEL"]' + # deployed-environment-variables: '["OPENAI_API_KEY","OPENAI_MODEL","GOOGLE_AI_API_KEY","GOOGLE_AI_MODEL"]' permissions: contents: read id-token: write From 6c02a7442b2d6de1045a03fe1ac4e30069bb9e7a Mon Sep 17 00:00:00 2001 From: neagualexa Date: Thu, 20 Feb 2025 10:12:48 +0000 Subject: [PATCH 30/57] test: with env vars --- .github/workflows/main.yml | 114 ++++++++++++++++++------------------- 1 file changed, 57 insertions(+), 57 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 142d84b..8402b3e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -6,77 +6,77 @@ on: workflow_dispatch: jobs: - test: - name: Staging deployment tests - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - python-version: ["3.11"] - env: - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - OPENAI_MODEL: ${{ vars.OPENAI_MODEL }} - GOOGLE_AI_API_KEY: ${{ secrets.GOOGLE_AI_API_KEY }} - GOOGLE_AI_MODEL: ${{ vars.GOOGLE_AI_MODEL }} - steps: - - name: Checkout Code - uses: actions/checkout@v4 + # test: + # name: Staging deployment tests + # runs-on: ubuntu-latest + # strategy: + # fail-fast: false + # matrix: + # python-version: ["3.11"] + # env: + # OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + # OPENAI_MODEL: ${{ vars.OPENAI_MODEL }} + # GOOGLE_AI_API_KEY: ${{ secrets.GOOGLE_AI_API_KEY }} + # GOOGLE_AI_MODEL: ${{ vars.GOOGLE_AI_MODEL }} + # steps: + # - name: Checkout Code + # uses: actions/checkout@v4 - - name: Set up Python ${{ matrix.python-version }} - id: python-setup - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} + # - name: Set up Python ${{ matrix.python-version }} + # id: python-setup + # uses: actions/setup-python@v5 + # with: + # python-version: ${{ matrix.python-version }} - # - name: Load cached venv - # id: dependencies-cache - # uses: actions/cache@v3 - # with: - # path: .venv - # key: venv-${{ runner.os }}-${{ steps.python-setup.outputs.python-version }} + # # - name: Load cached venv + # # id: dependencies-cache + # # uses: actions/cache@v3 + # # with: + # # path: .venv + # # key: venv-${{ runner.os }}-${{ steps.python-setup.outputs.python-version }} - - name: Create Venv if Cache not found - # if: steps.dependencies-cache.outputs.cache-hit != 'true' - run: | - python -m venv .venv + # - name: Create Venv if Cache not found + # # if: steps.dependencies-cache.outputs.cache-hit != 'true' + # run: | + # python -m venv .venv - - name: Install dependencies - # if: steps.dependencies-cache.outputs.cache-hit != 'true' - run: | - pip install --upgrade pip - pip install -r requirements.txt + # - name: Install dependencies + # # if: steps.dependencies-cache.outputs.cache-hit != 'true' + # run: | + # pip install --upgrade pip + # pip install -r requirements.txt - - name: Run tests - if: always() - run: | - source .venv/bin/activate - pytest --junit-xml=./reports/pytest.xml --tb=auto -v + # - name: Run tests + # if: always() + # run: | + # source .venv/bin/activate + # pytest --junit-xml=./reports/pytest.xml --tb=auto -v - - name: Upload test results - uses: actions/upload-artifact@v4 - if: always() - with: - name: test-results - path: ./reports/pytest.xml - if-no-files-found: warn + # - name: Upload test results + # uses: actions/upload-artifact@v4 + # if: always() + # with: + # name: test-results + # path: ./reports/pytest.xml + # if-no-files-found: warn - build: - name: Build Docker Image - uses: lambda-feedback/chat-function-workflows/.github/workflows/gh_build.yml@main - needs: test - permissions: - contents: read - id-token: write - packages: write + # build: + # name: Build Docker Image + # uses: lambda-feedback/chat-function-workflows/.github/workflows/gh_build.yml@main + # needs: test + # permissions: + # contents: read + # id-token: write + # packages: write deploy: name: Deploy to Lambda Feedback uses: lambda-feedback/chat-function-workflows/.github/workflows/main_deploy.yml@main - needs: test + # needs: test with: template-repository-name: "lambda-feedback/chat-function-boilerplate" # allow for developer to specify the environment variables that are used by the deployed AWS Lambda. Default to mock then admin can update. - # deployed-environment-variables: '["OPENAI_API_KEY","OPENAI_MODEL","GOOGLE_AI_API_KEY","GOOGLE_AI_MODEL"]' + deployed-environment-variables: '["OPENAI_API_KEY","OPENAI_MODEL","GOOGLE_AI_API_KEY","GOOGLE_AI_MODEL"]' permissions: contents: read id-token: write From c890fe4fbcef5c32ee82eada7101b645925f3573 Mon Sep 17 00:00:00 2001 From: neagualexa Date: Thu, 20 Feb 2025 11:25:28 +0000 Subject: [PATCH 31/57] fix: environment variables sent --- .github/workflows/dev.yml | 2 +- .github/workflows/main.yml | 114 ++++++++++++++++++------------------- 2 files changed, 58 insertions(+), 58 deletions(-) diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index 58c1c83..d1e111d 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -76,7 +76,7 @@ jobs: with: template-repository-name: "lambda-feedback/chat-function-boilerplate" # allow for developer to specify the environment variables that are used by the deployed AWS Lambda. Default to mock then admin can update. - deployed-environment-variables: '["OPENAI_API_KEY","OPENAI_MODEL"]' + deployed-environment-variables: '[\"OPENAI_API_KEY\",\"OPENAI_MODEL\",\"GOOGLE_AI_API_KEY\",\"GOOGLE_AI_MODEL\"]' permissions: contents: read id-token: write diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 8402b3e..0df0c41 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -6,77 +6,77 @@ on: workflow_dispatch: jobs: - # test: - # name: Staging deployment tests - # runs-on: ubuntu-latest - # strategy: - # fail-fast: false - # matrix: - # python-version: ["3.11"] - # env: - # OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - # OPENAI_MODEL: ${{ vars.OPENAI_MODEL }} - # GOOGLE_AI_API_KEY: ${{ secrets.GOOGLE_AI_API_KEY }} - # GOOGLE_AI_MODEL: ${{ vars.GOOGLE_AI_MODEL }} - # steps: - # - name: Checkout Code - # uses: actions/checkout@v4 + test: + name: Staging deployment tests + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.11"] + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + OPENAI_MODEL: ${{ vars.OPENAI_MODEL }} + GOOGLE_AI_API_KEY: ${{ secrets.GOOGLE_AI_API_KEY }} + GOOGLE_AI_MODEL: ${{ vars.GOOGLE_AI_MODEL }} + steps: + - name: Checkout Code + uses: actions/checkout@v4 - # - name: Set up Python ${{ matrix.python-version }} - # id: python-setup - # uses: actions/setup-python@v5 - # with: - # python-version: ${{ matrix.python-version }} + - name: Set up Python ${{ matrix.python-version }} + id: python-setup + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} - # # - name: Load cached venv - # # id: dependencies-cache - # # uses: actions/cache@v3 - # # with: - # # path: .venv - # # key: venv-${{ runner.os }}-${{ steps.python-setup.outputs.python-version }} + # - name: Load cached venv + # id: dependencies-cache + # uses: actions/cache@v3 + # with: + # path: .venv + # key: venv-${{ runner.os }}-${{ steps.python-setup.outputs.python-version }} - # - name: Create Venv if Cache not found - # # if: steps.dependencies-cache.outputs.cache-hit != 'true' - # run: | - # python -m venv .venv + - name: Create Venv if Cache not found + # if: steps.dependencies-cache.outputs.cache-hit != 'true' + run: | + python -m venv .venv - # - name: Install dependencies - # # if: steps.dependencies-cache.outputs.cache-hit != 'true' - # run: | - # pip install --upgrade pip - # pip install -r requirements.txt + - name: Install dependencies + # if: steps.dependencies-cache.outputs.cache-hit != 'true' + run: | + pip install --upgrade pip + pip install -r requirements.txt - # - name: Run tests - # if: always() - # run: | - # source .venv/bin/activate - # pytest --junit-xml=./reports/pytest.xml --tb=auto -v + - name: Run tests + if: always() + run: | + source .venv/bin/activate + pytest --junit-xml=./reports/pytest.xml --tb=auto -v - # - name: Upload test results - # uses: actions/upload-artifact@v4 - # if: always() - # with: - # name: test-results - # path: ./reports/pytest.xml - # if-no-files-found: warn + - name: Upload test results + uses: actions/upload-artifact@v4 + if: always() + with: + name: test-results + path: ./reports/pytest.xml + if-no-files-found: warn - # build: - # name: Build Docker Image - # uses: lambda-feedback/chat-function-workflows/.github/workflows/gh_build.yml@main - # needs: test - # permissions: - # contents: read - # id-token: write - # packages: write + build: + name: Build Docker Image + uses: lambda-feedback/chat-function-workflows/.github/workflows/gh_build.yml@main + needs: test + permissions: + contents: read + id-token: write + packages: write deploy: name: Deploy to Lambda Feedback uses: lambda-feedback/chat-function-workflows/.github/workflows/main_deploy.yml@main - # needs: test + needs: test with: template-repository-name: "lambda-feedback/chat-function-boilerplate" # allow for developer to specify the environment variables that are used by the deployed AWS Lambda. Default to mock then admin can update. - deployed-environment-variables: '["OPENAI_API_KEY","OPENAI_MODEL","GOOGLE_AI_API_KEY","GOOGLE_AI_MODEL"]' + deployed-environment-variables: '[\"OPENAI_API_KEY\",\"OPENAI_MODEL\",\"GOOGLE_AI_API_KEY\",\"GOOGLE_AI_MODEL\"]' permissions: contents: read id-token: write From f8bfda9eb9d9e575a7c3d304fe4c00d7cea9a3a1 Mon Sep 17 00:00:00 2001 From: neagualexa Date: Fri, 7 Mar 2025 11:04:44 +0000 Subject: [PATCH 32/57] fix: ednge case optional response areas --- src/agents/utils/parse_json_to_prompt.py | 10 +++++----- src/module.py | 15 ++++++++++----- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/src/agents/utils/parse_json_to_prompt.py b/src/agents/utils/parse_json_to_prompt.py index 74c5fe8..71712d7 100644 --- a/src/agents/utils/parse_json_to_prompt.py +++ b/src/agents/utils/parse_json_to_prompt.py @@ -137,13 +137,13 @@ def parse_json_to_prompt( questionSubmissionSummary: Optional[List[StudentWorkRe questionInformation: Optional[QuestionDetails], questionAccessInformation: Optional[QuestionAccessInformation] ) -> Optional[str]: + + if not questionInformation or not questionAccessInformation: + return "There must have been an error in fetching the question details. So ask me about the question I am working on such that you can still help me." questionSubmissionSummary = [StudentWorkResponseArea(**submissionsSummary) for submissionsSummary in questionSubmissionSummary] questionInformation = QuestionDetails(**questionInformation) questionAccessInformation = QuestionAccessInformation(**questionAccessInformation) - - if not questionSubmissionSummary or not questionInformation or not questionAccessInformation: - return None def format_response_area_details(responseArea: ResponseAreaDetails, studentSummary: List[StudentWorkResponseArea]) -> str: submissionDetails = "\n".join( @@ -163,7 +163,7 @@ def format_response_area_details(responseArea: ResponseAreaDetails, studentSumma return f""" ## Response Area: {responseArea.position + 1} {f'Area task: What is {responseArea.preResponseText} ?' if responseArea.preResponseText else ''} - (Secret) Expected Answer: {responseArea.answer}; + (Secret - not to be shared) Expected Answer: {responseArea.answer}; {submissionDetails}""" def format_part_details(part: PartDetails, currentPart: CurrentPart, summary: List[StudentWorkResponseArea]) -> str: @@ -188,7 +188,7 @@ def format_part_details(part: PartDetails, currentPart: CurrentPart, summary: Li {f"Time spent on this part: {currentPart.timeTakenPart if currentPart.timeTakenPart is not None else 'No recorded duration'}" if currentPart.id == part.publishedPartId else ''} Part Content: {part.publishedPartContent.strip() if part.publishedPartContent else 'No content'}; {responseAreas} - {f'Final Part Answer: {part.publishedPartAnswerContent}' if part.publishedPartAnswerContent else 'No direct answer'} + {f'Final Part Answer: {part.publishedPartAnswerContent}' if part.publishedPartAnswerContent else 'No direct answer for this part.'} {workedSolutions} """ diff --git a/src/module.py b/src/module.py index 76ded66..15c79c6 100755 --- a/src/module.py +++ b/src/module.py @@ -52,11 +52,16 @@ def chat_module(message: Any, params: Params) -> Result: question_submission_summary = question_response_details["questionSubmissionSummary"] if "questionSubmissionSummary" in question_response_details else [] question_information = question_response_details["questionInformation"] if "questionInformation" in question_response_details else {} question_access_information = question_response_details["questionAccessInformation"] if "questionAccessInformation" in question_response_details else {} - question_response_details_prompt = parse_json_to_prompt( - question_submission_summary, - question_information, - question_access_information - ) + try: + question_response_details_prompt = parse_json_to_prompt( + question_submission_summary, + question_information, + question_access_information + ) + print("INFO:: ", question_response_details_prompt) + except Exception as e: + print("ERROR:: ", e) + raise Exception("Internal Error: The question response details could not be parsed.") if "conversation_id" in params: conversation_id = params["conversation_id"] else: From fef5d571c134655a17a102e20d0d6c02ea3fc3b2 Mon Sep 17 00:00:00 2001 From: neagualexa Date: Mon, 22 Sep 2025 15:20:42 +0100 Subject: [PATCH 33/57] up to date with python13, latest context parser --- Dockerfile | 12 +- .../utils/parse_json_context_to_prompt.py | 321 ++++++++++++++++++ src/agents/utils/parse_json_to_prompt.py | 217 ------------ src/agents/utils/prompt_context_templates.py | 245 +++++++++++++ .../synthetic_conversation_generation.py | 8 +- src/agents/utils/testbench_agents.py | 4 +- src/module.py | 6 +- 7 files changed, 583 insertions(+), 230 deletions(-) create mode 100644 src/agents/utils/parse_json_context_to_prompt.py delete mode 100644 src/agents/utils/parse_json_to_prompt.py create mode 100644 src/agents/utils/prompt_context_templates.py diff --git a/Dockerfile b/Dockerfile index 818f099..9150687 100755 --- a/Dockerfile +++ b/Dockerfile @@ -1,13 +1,19 @@ -ARG PYTHON_VERSION=3.11 +ARG PYTHON_VERSION=3.13 FROM public.ecr.aws/lambda/python:${PYTHON_VERSION} # Set working directory WORKDIR ${LAMBDA_TASK_ROOT} -RUN pip install --upgrade pip && yum install -y git +RUN pip install --upgrade pip +RUN dnf install -y git \ + && dnf install -y \ + gcc \ + gcc-c++ \ + make \ + python3-devel \ + && dnf clean all -# Install dependencies into the virtual environment COPY requirements.txt . RUN pip install -r requirements.txt diff --git a/src/agents/utils/parse_json_context_to_prompt.py b/src/agents/utils/parse_json_context_to_prompt.py new file mode 100644 index 0000000..38af2da --- /dev/null +++ b/src/agents/utils/parse_json_context_to_prompt.py @@ -0,0 +1,321 @@ +""" +Refactored JSON to prompt parser using improved, clearer structure. +""" + +from typing import List, Optional, Dict, Any, Union +from .prompt_context_templates import PromptFormatter + +# Definitions questionSubmissionSummary type +class StudentLatestSubmission: + def __init__( + self, + universalResponseAreaId: Optional[str] = None, + answer: Optional[str] = None, + submission: Optional[str] = None, + feedback: Optional[str] = None, + rawResponse: Optional[dict] = None, + ): + self.universalResponseAreaId = universalResponseAreaId + self.answer = answer + self.submission = submission + self.feedback = feedback + self.rawResponse = rawResponse + +class StudentWorkResponseArea: + def __init__( + self, + publishedPartId: Optional[str] = None, + publishedPartPosition: Optional[int] = None, + publishedResponseAreaId: Optional[str] = None, + publishedResponseAreaPosition: Optional[int] = None, + responseAreaUniversalId: Optional[str] = None, + publishedResponseAreaPreResponseText: Optional[str] = None, + publishedResponseType: Optional[str] = None, + publishedResponseConfig: Optional[dict] = None, + totalSubmissions: Optional[int] = None, + totalWrongSubmissions: Optional[int] = None, + latestSubmission: Optional[StudentLatestSubmission] = None, + ): + self.publishedPartId = publishedPartId + self.publishedPartPosition = publishedPartPosition + self.publishedResponseAreaId = publishedResponseAreaId + self.publishedResponseAreaPosition = publishedResponseAreaPosition + self.responseAreaUniversalId = responseAreaUniversalId + self.publishedResponseAreaPreResponseText = publishedResponseAreaPreResponseText + self.publishedResponseType = publishedResponseType + self.publishedResponseConfig = publishedResponseConfig + self.latestSubmission = StudentLatestSubmission(**latestSubmission) if latestSubmission else None + self.totalSubmissions = totalSubmissions + self.totalWrongSubmissions = totalWrongSubmissions + +# questionInformation type +class ResponseAreaDetails: + def __init__( + self, + id: Optional[str] = None, + position: Optional[int] = None, + universalResponseAreaId: Optional[str] = None, + preResponseText: Optional[str] = None, + responseType: Optional[str] = None, + answer: Optional[dict] = None, + Response: Optional[dict] = None, + ): + self.id = id + self.position = position + self.universalResponseAreaId = universalResponseAreaId + self.preResponseText = preResponseText + self.responseType = responseType + self.answer = answer + self.Response = Response + +class PartDetails: + def __init__( + self, + publishedPartId: Optional[str] = None, + publishedPartPosition: Optional[int] = None, + publishedPartContent: Optional[str] = None, + publishedPartAnswerContent: Optional[str] = None, + publishedWorkedSolutionSections: Optional[List[dict]] = [], + publishedResponseAreas: Optional[List[Optional[ResponseAreaDetails]]] = [], + ): + self.publishedPartId = publishedPartId + self.publishedPartPosition = publishedPartPosition + self.publishedPartContent = publishedPartContent + self.publishedPartAnswerContent = publishedPartAnswerContent + self.publishedWorkedSolutionSections = publishedWorkedSolutionSections + self.publishedResponseAreas = [ResponseAreaDetails(**publishedResponseArea) for publishedResponseArea in publishedResponseAreas] + +class QuestionDetails: + def __init__( + self, + setNumber: Optional[int] = None, + setName: Optional[str] = None, + setDescription: Optional[str] = None, + questionNumber: Optional[int] = None, + questionTitle: Optional[str] = None, + questionGuidance: Optional[str] = None, + questionContent: Optional[str] = None, + durationLowerBound: Optional[int] = None, + durationUpperBound: Optional[int] = None, + parts: Optional[List[PartDetails]] = [], + ): + self.setNumber = setNumber + self.setName = setName + self.setDescription = setDescription + self.questionNumber = questionNumber + self.questionTitle = questionTitle + self.questionGuidance = questionGuidance + self.questionContent = questionContent + self.durationLowerBound = durationLowerBound + self.durationUpperBound = durationUpperBound + self.parts = [PartDetails(**part) for part in parts] + +# questionAccessInformation type +class CurrentPart: + def __init__( + self, + id: str = None, + position: int = None, + universalPartId: Optional[str] = None, + timeTakenPart: Optional[str] = None, + markedDonePart: Optional[str] = None + ): + self.id = id + self.position = position + self.universalPartId = universalPartId + self.timeTakenPart = timeTakenPart + self.markedDonePart = markedDonePart + +class QuestionAccessInformation: + def __init__( + self, + estimatedMinimumTime: Optional[str] = None, + estimaredMaximumTime: Optional[str] = None, + timeTaken: Optional[str] = None, + accessStatus: Optional[str] = None, + markedDone: Optional[str] = None, + currentPart: Optional[Dict[str, Union[str, int]]] = {}, + ): + self.estimatedMinimumTime = estimatedMinimumTime + self.estimaredMaximumTime = estimaredMaximumTime + self.timeTaken = timeTaken + self.accessStatus = accessStatus + self.markedDone = markedDone + self.currentPart = CurrentPart(**currentPart) + + +def parse_json_to_structured_prompt( + question_submission_summary: Optional[List[StudentWorkResponseArea]], + question_information: Optional[QuestionDetails], + question_access_information: Optional[QuestionAccessInformation] +) -> Optional[str]: + """ + Parse JSON data into a well-structured, LLM-friendly prompt. + + Args: + question_submission_summary: Student's work and submissions + question_information: Question details and structure + question_access_information: Current progress and timing info + + Returns: + Formatted prompt string or error message + """ + + if not question_information: + return PromptFormatter.format_error_message() + + # Convert to proper objects + submission_summary = [StudentWorkResponseArea(**summary) for summary in question_submission_summary] + question_info = QuestionDetails(**question_information) + access_info = QuestionAccessInformation(**question_access_information) if question_access_information else None + + # TODO: EXPERIMENTAL - Remove later + # if question_info.setNumber is not None: + # if (question_info.setNumber + 1) % 2 != 0: + # return PromptFormatter.format_no_context_message() + + # Build prompt sections + sections = [] + + # 1. Question Header + current_part_letter = None + if access_info and access_info.currentPart: + current_part_letter = PromptFormatter.get_part_letter(access_info.currentPart.position) + + set_info = { + 'number': question_info.setNumber, + 'name': question_info.setName + } + + question_data = { + 'number': question_info.questionNumber, + 'title': question_info.questionTitle, + 'guidance': question_info.questionGuidance, + 'content': question_info.questionContent, + 'duration_lower': question_info.durationLowerBound, + 'duration_upper': question_info.durationUpperBound + } + + sections.append(PromptFormatter.format_question_header( + set_info, + question_data, + current_part_letter + )) + + # 2. Progress Summary (if available) + if access_info: + progress_section = PromptFormatter.format_progress_summary( + access_info.timeTaken, + access_info.accessStatus, + access_info.markedDone + ) + if progress_section: + sections.append(progress_section) + + # 3. Parts Details + for part in question_info.parts: + sections.append(_format_single_part( + part, + access_info.currentPart if access_info else None, + submission_summary + )) + + # 4. Combine into final prompt + return PromptFormatter.format_complete_prompt(sections) + + +def _format_single_part( + part: PartDetails, + current_part: Optional[CurrentPart], + submissions: List[StudentWorkResponseArea] +) -> str: + """Format a single part with all its components.""" + + if not part: + return "" + + part_sections = [] + part_letter = PromptFormatter.get_part_letter(part.publishedPartPosition) + + # Determine if this is the current part + is_current = current_part and current_part.id == part.publishedPartId + time_on_part = current_part.timeTakenPart if is_current and current_part else None + + # 1. Part Header + part_sections.append(PromptFormatter.format_part_header( + part_letter, + is_current, + time_on_part + )) + + # 2. Part Content + part_sections.append(PromptFormatter.format_part_content(part.publishedPartContent)) + + # 3. Response Areas + response_areas = [] + for response_area in part.publishedResponseAreas: + student_work = _extract_student_work_for_area(response_area, submissions) + response_areas.append(PromptFormatter.format_single_response_area( + response_area.position, + response_area.preResponseText, + response_area.answer, + student_work + )) + + if response_areas: + part_sections.append(PromptFormatter.format_response_areas(response_areas)) + + # 4. Final Part Answer + part_sections.append(PromptFormatter.format_part_answer(part.publishedPartAnswerContent)) + + # 5. Worked Solutions + solutions_data = [] + if part.publishedWorkedSolutionSections: + for ws in part.publishedWorkedSolutionSections: + solutions_data.append({ + 'title': ws.get('title', ''), + 'content': ws.get('content', ''), + 'position': ws.get('position', 0) + }) + + part_sections.append(PromptFormatter.format_worked_solutions(solutions_data)) + + return "\n".join(part_sections) + "\n---\n" + + +def _extract_student_work_for_area( + response_area: ResponseAreaDetails, + submissions: List[StudentWorkResponseArea] +) -> Dict[str, Any]: + """Extract student work data for a specific response area.""" + + for submission in submissions: + if (submission.publishedResponseAreaId == response_area.id and + submission.latestSubmission): + + return { + 'has_submissions': True, + 'latest_response': submission.latestSubmission.submission, + 'latest_feedback': submission.latestSubmission.feedback, + 'total_submissions': submission.totalSubmissions, + 'total_wrong': submission.totalWrongSubmissions + } + + return {'has_submissions': False} + + +# Convenience function that maintains the original interface +def parse_json_to_prompt( + questionSubmissionSummary: Optional[List[StudentWorkResponseArea]], + questionInformation: Optional[QuestionDetails], + questionAccessInformation: Optional[QuestionAccessInformation] +) -> Optional[str]: + """ + Legacy wrapper for backward compatibility. + Recommended to use parse_json_to_structured_prompt for new code. + """ + return parse_json_to_structured_prompt( + questionSubmissionSummary, + questionInformation, + questionAccessInformation + ) diff --git a/src/agents/utils/parse_json_to_prompt.py b/src/agents/utils/parse_json_to_prompt.py deleted file mode 100644 index 71712d7..0000000 --- a/src/agents/utils/parse_json_to_prompt.py +++ /dev/null @@ -1,217 +0,0 @@ -from typing import List, Optional, Union, Dict - -# questionSubmissionSummary type -class StudentLatestSubmission: - def __init__( - self, - universalResponseAreaId: Optional[str] = None, - answer: Optional[str] = None, - submission: Optional[str] = None, - feedback: Optional[str] = None, - rawResponse: Optional[dict] = None, - ): - self.universalResponseAreaId = universalResponseAreaId - self.answer = answer - self.submission = submission - self.feedback = feedback - self.rawResponse = rawResponse - -class StudentWorkResponseArea: - def __init__( - self, - publishedPartId: Optional[str] = None, - publishedPartPosition: Optional[int] = None, - publishedResponseAreaId: Optional[str] = None, - publishedResponseAreaPosition: Optional[int] = None, - responseAreaUniversalId: Optional[str] = None, - publishedResponseAreaPreResponseText: Optional[str] = None, - publishedResponseType: Optional[str] = None, - publishedResponseConfig: Optional[dict] = None, - totalSubmissions: Optional[int] = None, - totalWrongSubmissions: Optional[int] = None, - latestSubmission: Optional[StudentLatestSubmission] = None, - ): - self.publishedPartId = publishedPartId - self.publishedPartPosition = publishedPartPosition - self.publishedResponseAreaId = publishedResponseAreaId - self.publishedResponseAreaPosition = publishedResponseAreaPosition - self.responseAreaUniversalId = responseAreaUniversalId - self.publishedResponseAreaPreResponseText = publishedResponseAreaPreResponseText - self.publishedResponseType = publishedResponseType - self.publishedResponseConfig = publishedResponseConfig - self.latestSubmission = StudentLatestSubmission(**latestSubmission) if latestSubmission else None - self.totalSubmissions = totalSubmissions - self.totalWrongSubmissions = totalWrongSubmissions - -# questionInformation type -class ResponseAreaDetails: - def __init__( - self, - id: Optional[str] = None, - position: Optional[int] = None, - universalResponseAreaId: Optional[str] = None, - preResponseText: Optional[str] = None, - responseType: Optional[str] = None, - answer: Optional[dict] = None, - Response: Optional[dict] = None, - ): - self.id = id - self.position = position - self.universalResponseAreaId = universalResponseAreaId - self.preResponseText = preResponseText - self.responseType = responseType - self.answer = answer - self.Response = Response - -class PartDetails: - def __init__( - self, - publishedPartId: Optional[str] = None, - publishedPartPosition: Optional[int] = None, - publishedPartContent: Optional[str] = None, - publishedPartAnswerContent: Optional[str] = None, - publishedWorkedSolutionSections: Optional[List[dict]] = [], - publishedResponseAreas: Optional[List[Optional[ResponseAreaDetails]]] = [], - ): - self.publishedPartId = publishedPartId - self.publishedPartPosition = publishedPartPosition - self.publishedPartContent = publishedPartContent - self.publishedPartAnswerContent = publishedPartAnswerContent - self.publishedWorkedSolutionSections = publishedWorkedSolutionSections - self.publishedResponseAreas = [ResponseAreaDetails(**publishedResponseArea) for publishedResponseArea in publishedResponseAreas] - -class QuestionDetails: - def __init__( - self, - setNumber: Optional[int] = None, - setName: Optional[str] = None, - setDescription: Optional[str] = None, - questionNumber: Optional[int] = None, - questionTitle: Optional[str] = None, - questionGuidance: Optional[str] = None, - questionContent: Optional[str] = None, - durationLowerBound: Optional[int] = None, - durationUpperBound: Optional[int] = None, - parts: Optional[List[PartDetails]] = [], - ): - self.setNumber = setNumber - self.setName = setName - self.setDescription = setDescription - self.questionNumber = questionNumber - self.questionTitle = questionTitle - self.questionGuidance = questionGuidance - self.questionContent = questionContent - self.durationLowerBound = durationLowerBound - self.durationUpperBound = durationUpperBound - self.parts = [PartDetails(**part) for part in parts] - -# questionAccessInformation type -class CurrentPart: - def __init__(self, id: str = None, position: int = None, timeTakenPart: Optional[str] = None, markedDonePart: Optional[str] = None): - self.id = id - self.position = position - self.timeTakenPart = timeTakenPart - self.markedDonePart = markedDonePart - -class QuestionAccessInformation: - def __init__( - self, - estimatedMinimumTime: Optional[str] = None, - estimaredMaximumTime: Optional[str] = None, - timeTaken: Optional[str] = None, - accessStatus: Optional[str] = None, - markedDone: Optional[str] = None, - currentPart: Optional[Dict[str, Union[str, int]]] = {}, - ): - self.estimatedMinimumTime = estimatedMinimumTime - self.estimaredMaximumTime = estimaredMaximumTime - self.timeTaken = timeTaken - self.accessStatus = accessStatus - self.markedDone = markedDone - self.currentPart = CurrentPart(**currentPart) - -def convert_index_to_lowercase_letter(index: int) -> str: - return chr(96 + (index + 1)) # 1-indexed - -def parse_json_to_prompt( questionSubmissionSummary: Optional[List[StudentWorkResponseArea]], - questionInformation: Optional[QuestionDetails], - questionAccessInformation: Optional[QuestionAccessInformation] - ) -> Optional[str]: - - if not questionInformation or not questionAccessInformation: - return "There must have been an error in fetching the question details. So ask me about the question I am working on such that you can still help me." - - questionSubmissionSummary = [StudentWorkResponseArea(**submissionsSummary) for submissionsSummary in questionSubmissionSummary] - questionInformation = QuestionDetails(**questionInformation) - questionAccessInformation = QuestionAccessInformation(**questionAccessInformation) - - def format_response_area_details(responseArea: ResponseAreaDetails, studentSummary: List[StudentWorkResponseArea]) -> str: - submissionDetails = "\n".join( - [ - f"Latest Response: {ra.latestSubmission.submission};\n" - f"Latest Feedback Received: {ra.latestSubmission.feedback};\n" - f"Total Responses: {ra.totalSubmissions};\n" - f"Total Wrong Responses: {ra.totalWrongSubmissions};\n" - for ra in studentSummary - if ra.publishedResponseAreaId == responseArea.id and ra.latestSubmission - ] - ) - - if not submissionDetails: - submissionDetails = 'Latest Response: none made;' - - return f""" - ## Response Area: {responseArea.position + 1} - {f'Area task: What is {responseArea.preResponseText} ?' if responseArea.preResponseText else ''} - (Secret - not to be shared) Expected Answer: {responseArea.answer}; - {submissionDetails}""" - - def format_part_details(part: PartDetails, currentPart: CurrentPart, summary: List[StudentWorkResponseArea]) -> str: - if not part: - return '' - - responseAreas = "\n".join( - [format_response_area_details(responseArea, summary) for responseArea in part.publishedResponseAreas] - ) - - workedSolutions = ( - "\n".join( - [ - f"## Worked Solution {ws.get('position') + 1}: {ws.get('title', '')}\n" - f"{ws.get('content', '').strip() or 'No content'}\n" - for ws in part.publishedWorkedSolutionSections - ] - ) if part.publishedWorkedSolutionSections else f"No worked solutions for part ({convert_index_to_lowercase_letter(part.publishedPartPosition)});" - ) - return f""" - # {'[CURRENTLY WORKING ON] ' if currentPart.id == part.publishedPartId else ''}Part ({convert_index_to_lowercase_letter(part.publishedPartPosition)}): - {f"Time spent on this part: {currentPart.timeTakenPart if currentPart.timeTakenPart is not None else 'No recorded duration'}" if currentPart.id == part.publishedPartId else ''} - Part Content: {part.publishedPartContent.strip() if part.publishedPartContent else 'No content'}; - {responseAreas} - {f'Final Part Answer: {part.publishedPartAnswerContent}' if part.publishedPartAnswerContent else 'No direct answer for this part.'} - {workedSolutions} -""" - - questionDetails = f"""This is the question I am currently working on. I am currently working on Part ({convert_index_to_lowercase_letter(questionAccessInformation.currentPart.position)}). Below, you'll find its details, including the parts of the question, my responses for each response area, and the feedback I received. This information highlights my efforts and progress so far. Use this this information to inform your understanding about the question materials provided to me and my work on them. - Maths equations are in KaTex format, preserve them the same. Use British English spellings. -{f'# Question Set {questionInformation.setNumber + 1}: {questionInformation.setName};' if ((questionInformation.setName is not None) and (questionInformation.setNumber is not None)) else ''} -# Question {f' {questionInformation.setNumber + 1}.{questionInformation.questionNumber + 1}' if ((questionInformation.setNumber is not None) and (questionInformation.questionNumber is not None)) else ''}: {questionInformation.questionTitle}; - Guidance to Solve the Question: {questionInformation.questionGuidance or 'None'}; - Description of Question: {questionInformation.questionContent}; - Expected Time to Complete the Question: {f'{questionInformation.durationLowerBound} - {questionInformation.durationUpperBound} min;' if questionInformation.durationLowerBound and questionInformation.durationUpperBound else 'No specified duration.'} - Time Spent on the Question today: {questionAccessInformation.timeTaken or 'No recorded duration'} {f'which is {questionAccessInformation.accessStatus}' if questionAccessInformation.accessStatus else ''} {f'{questionAccessInformation.markedDone}' if questionAccessInformation.markedDone else ''}; - """ - - partsDetails = "\n".join( - [ - format_part_details( - part, - questionAccessInformation.currentPart, - questionSubmissionSummary - ) for part in questionInformation.parts - ] - ) - - result = f"{questionDetails}\n{partsDetails}".replace(" ", "").replace(" ", "").replace("\n\n", "\n") - - return result diff --git a/src/agents/utils/prompt_context_templates.py b/src/agents/utils/prompt_context_templates.py new file mode 100644 index 0000000..9eb175b --- /dev/null +++ b/src/agents/utils/prompt_context_templates.py @@ -0,0 +1,245 @@ +""" +Improved prompt templates with clearer structure for LLM consumption. +Uses hierarchical organization and consistent formatting. +""" + +from typing import Optional, List, Dict, Any + +class PromptFormatter: + """Centralized prompt formatting with clear structure.""" + + @staticmethod + def format_error_message() -> str: + """Return structured error message.""" + return """ +# ERROR: Question details unavailable + +Please describe the question you're working on so I can assist you effectively. +""" + + @staticmethod + def format_no_context_message() -> str: + """Return structured no-context message.""" + return """ +# NOTICE: Question details not provided + +Please tell me about the question you're working on. I'll use British English spellings. +""" + + @staticmethod + def format_question_header( + set_info: Dict[str, Any], + question_info: Dict[str, Any], + current_part: Optional[str] = None + ) -> str: + """Format the main question header with metadata.""" + + # Build title components + title_parts = [] + if set_info.get('number') is not None and set_info.get('name'): + title_parts.append(f"## Set {set_info['number'] + 1}: {set_info['name']}") + + question_num = "" + if set_info.get('number') is not None and question_info.get('number') is not None: + question_num = f"{set_info['number'] + 1}.{question_info['number'] + 1}" + + title_parts.append(f"## Question {question_num}: {question_info['title']}") + + # Current progress indicator + progress_indicator = "" + if current_part: + progress_indicator = f"### Currently working on: Part ({current_part})" + + # Question metadata + guidance = question_info.get('guidance', 'None provided') + content = question_info.get('content', 'No description available') + + # Duration formatting + duration_text = "- Expected Duration: " + if question_info.get('duration_lower') and question_info.get('duration_upper'): + duration_text += f"{question_info['duration_lower']}-{question_info['duration_upper']} minutes" + else: + duration_text += "Not specified" + + return f""" +# Question Context + +{"\n".join(title_parts)} + +{progress_indicator} + +### Question Details +- Guidance: {guidance} +- Description: {content} +{duration_text} + +> Note: Mathematical equations are in KaTeX format, preserve them the same. Use British English spellings. + +--- +""" + + @staticmethod + def format_progress_summary( + time_taken: Optional[str] = None, + access_status: Optional[str] = None, + marked_done: Optional[str] = None + ) -> str: + """Format progress and timing information.""" + if not any([time_taken, access_status, marked_done]): + return "" + + progress_items = [] + if time_taken: + progress_items.append(f"- Time spent today: {time_taken}") + if access_status: + progress_items.append(f"- Status: {access_status}") + if marked_done: + progress_items.append(f"- Completion: {marked_done}") + + return f""" +# Progress Summary + +{"\n".join(progress_items)} + +--- +""" + + @staticmethod + def format_part_header( + part_letter: str, + is_current: bool = False, + time_on_part: Optional[str] = None + ) -> str: + """Format part header with clear indicators.""" + + status_text = " [CURRENTLY WORKING ON]" if is_current else "" + + header = f"## Part ({part_letter}){status_text}" + + if is_current and time_on_part: + time_display = time_on_part if time_on_part != 'No recorded duration' else 'No time recorded' + header += f"\n\n*Time spent on this part: {time_display}*" + + return header + + @staticmethod + def format_part_content(content: Optional[str]) -> str: + """Format part content with clear labeling.""" + if not content or not content.strip(): + return "### Part Content\n\nNo content provided" + + return f"### Part Content\n\n{content.strip()}" + + @staticmethod + def format_response_areas(response_areas: List[str]) -> str: + """Format multiple response areas with clear separation.""" + if not response_areas: + return "### Response Areas\n\nNone defined" + + return f""" +### Response Areas + +{"\n".join(response_areas)} +""" + + @staticmethod + def format_single_response_area( + position: int, + task_description: Optional[str], + expected_answer: Any, + student_work: Dict[str, Any] + ) -> str: + """Format a single response area with student work.""" + + # Format task description + task_text = f"- Task: {task_description}" if task_description else "- Task: Not specified" + + # Format expected answer (keep secret) + answer_text = f"- Expected Answer (confidential): {expected_answer}" + + # Format student submissions + submission_text = PromptFormatter._format_student_submissions(student_work) + + return f""" +#### Response Area {position + 1} + +{task_text} +{answer_text} +{submission_text} +""" + + @staticmethod + def _format_student_submissions(student_work: Dict[str, Any]) -> str: + """Format student submission history.""" + if not student_work.get('has_submissions'): + return "- Your Work: No responses submitted yet" + + latest = student_work.get('latest_response', 'None') + feedback = student_work.get('latest_feedback', 'None') + total = student_work.get('total_submissions', 0) + wrong = student_work.get('total_wrong', 0) + + return f"""- Your Work: + - Latest response: {latest} + - Latest feedback: {feedback} + - Total attempts: {total} + - Incorrect attempts: {wrong}""" + + @staticmethod + def format_part_answer(answer_content: Optional[str]) -> str: + """Format the final part answer.""" + if not answer_content: + return "### Final Answer\n\nNo direct answer specified for this part" + + return f"### Final Answer\n\n{answer_content}" + + @staticmethod + def format_worked_solutions(solutions: List[Dict[str, Any]]) -> str: + """Format worked solutions section.""" + if not solutions: + return "### Worked Solutions\n\nNone available" + + solution_texts = [] + for i, solution in enumerate(solutions): + title = solution.get('title', f'Solution {i + 1}') + content = solution.get('content', '').strip() or 'No content available' + solution_texts.append(f"#### {title}\n\n{content}") + + return f"""### Worked Solutions + +{"\n".join(solution_texts)}""" + + @staticmethod + def format_complete_prompt(sections: List[str]) -> str: + """Combine all sections into a complete, well-structured prompt.""" + + intro = """ +# Personalized Learning Assistant + +I have detailed information about your current question, including your progress, responses, and any feedback you've received. This context helps me provide targeted assistance based on your specific situation. + +""" + + # Filter out empty sections and join + valid_sections = [section.strip() for section in sections if section and section.strip()] + + content = intro + "\n".join(valid_sections) + + # Clean up formatting + content = content.replace(" ", " ").replace(" ", " ") + content = "\n".join(line for line in content.split("\n") if line.strip() or not line) + + return content.strip() + + @staticmethod + def get_part_letter(position: int) -> str: + """Convert position to lowercase letter (1-indexed).""" + return chr(96 + (position + 1)) + + +# Legacy function wrappers for backward compatibility +def get_error_prompt() -> str: + return PromptFormatter.format_error_message() + +def get_no_context_prompt() -> str: + return PromptFormatter.format_no_context_message() diff --git a/src/agents/utils/synthetic_conversation_generation.py b/src/agents/utils/synthetic_conversation_generation.py index 9098352..9235756 100644 --- a/src/agents/utils/synthetic_conversation_generation.py +++ b/src/agents/utils/synthetic_conversation_generation.py @@ -8,9 +8,7 @@ The conversations will be 20 turns long, with the tutor and student taking turns to send a message. The tutor can be one of the following types: -- Informational Agent -- Socratic Agent -- Google's LearnLM-Tutor Agent +- Informational Agent (base) The tutor agent can be selected by changing the "agent_type" field in this script. The student can have multiple skill levels and conversational styles. Those are defined by the prompts used by the LLM. @@ -22,11 +20,11 @@ import json try: from ..student_agent.student_agent import invoke_student_agent - from .parse_json_to_prompt import parse_json_to_prompt + from .parse_json_context_to_prompt import parse_json_to_prompt from ..base_agent.base_agent import invoke_base_agent except ImportError: from src.agents.student_agent.student_agent import invoke_student_agent - from src.agents.utils.parse_json_to_prompt import parse_json_to_prompt + from src.agents.utils.parse_json_context_to_prompt import parse_json_to_prompt from src.agents.base_agent.base_agent import invoke_base_agent import os diff --git a/src/agents/utils/testbench_agents.py b/src/agents/utils/testbench_agents.py index 3b92026..e27ed52 100644 --- a/src/agents/utils/testbench_agents.py +++ b/src/agents/utils/testbench_agents.py @@ -5,10 +5,10 @@ import json try: - from .parse_json_to_prompt import parse_json_to_prompt + from .parse_json_context_to_prompt import parse_json_to_prompt from ..base_agent.base_agent import invoke_base_agent except ImportError: - from src.agents.utils.parse_json_to_prompt import parse_json_to_prompt + from src.agents.utils.parse_json_context_to_prompt import parse_json_to_prompt from src.agents.base_agent.base_agent import invoke_base_agent # File path for the input text diff --git a/src/module.py b/src/module.py index 15c79c6..2b7b2b4 100755 --- a/src/module.py +++ b/src/module.py @@ -3,10 +3,10 @@ from lf_toolkit.chat import ChatResult as Result, ChatParams as Params try: - from .agents.utils.parse_json_to_prompt import parse_json_to_prompt + from .agents.utils.parse_json_context_to_prompt import parse_json_to_prompt from .agents.base_agent.base_agent import invoke_base_agent except ImportError: - from src.agents.utils.parse_json_to_prompt import parse_json_to_prompt + from src.agents.utils.parse_json_context_to_prompt import parse_json_to_prompt from src.agents.base_agent.base_agent import invoke_base_agent def chat_module(message: Any, params: Params) -> Result: @@ -17,7 +17,7 @@ def chat_module(message: Any, params: Params) -> Result: - `message` which is the message sent by the student. - `params` which are any extra parameters that may be useful, - e.g., conversation history and summary, conversational style of user, conversation id, agent type. + e.g., conversation history and summary, conversational style of user, conversation id. The output of this function is what is returned as the API response and therefore must be JSON-encodable. It must also conform to the From d1c04c866249f0e8c2068e6a4fb806fbd92bc551 Mon Sep 17 00:00:00 2001 From: neagualexa Date: Mon, 22 Sep 2025 15:20:53 +0100 Subject: [PATCH 34/57] test chat function --- config.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.json b/config.json index a9f3255..1acd17d 100644 --- a/config.json +++ b/config.json @@ -1,3 +1,3 @@ { - "ChatFunctionName": "" + "ChatFunctionName": "testChatFunction" } From b3677805288aea5271eebeed2110e3054da98157 Mon Sep 17 00:00:00 2001 From: Alexandra Neagu <33195033+neagualexa@users.noreply.github.com> Date: Mon, 22 Sep 2025 15:39:11 +0100 Subject: [PATCH 35/57] use python13 for actions (#21) --- .github/workflows/dev.yml | 2 +- .github/workflows/main.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index d1e111d..1191a86 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -12,7 +12,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.11"] + python-version: ["3.13"] env: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} OPENAI_MODEL: ${{ vars.OPENAI_MODEL }} diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 0df0c41..8400ca3 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -12,7 +12,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.11"] + python-version: ["3.13"] env: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} OPENAI_MODEL: ${{ vars.OPENAI_MODEL }} From cf6527aac3333f001b3c823062abfdca8a29c839 Mon Sep 17 00:00:00 2001 From: Alexandra Neagu <33195033+neagualexa@users.noreply.github.com> Date: Mon, 22 Sep 2025 15:42:50 +0100 Subject: [PATCH 36/57] Use python13 (#22) * use python13 for actions * boilerplate is deployed without the need of name in config --- config.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.json b/config.json index 1acd17d..a9f3255 100644 --- a/config.json +++ b/config.json @@ -1,3 +1,3 @@ { - "ChatFunctionName": "testChatFunction" + "ChatFunctionName": "" } From 60fc265170d4edf9005b918fd9a0a8927f6e5c31 Mon Sep 17 00:00:00 2001 From: neagualexa Date: Mon, 22 Sep 2025 16:11:40 +0100 Subject: [PATCH 37/57] fix use of lf_toolkit --- src/module.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/module.py b/src/module.py index 2b7b2b4..02c6f66 100755 --- a/src/module.py +++ b/src/module.py @@ -1,6 +1,7 @@ import time from typing import Any -from lf_toolkit.chat import ChatResult as Result, ChatParams as Params +from lf_toolkit.chat.result import ChatResult as Result +from lf_toolkit.chat.params import ChatParams as Params try: from .agents.utils.parse_json_context_to_prompt import parse_json_to_prompt From 37a816e83effeeef49192abcc37e924d426db8d0 Mon Sep 17 00:00:00 2001 From: neagualexa Date: Tue, 23 Sep 2025 09:39:51 +0100 Subject: [PATCH 38/57] update readme mention of secrets setup --- README.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 7589c17..99e6906 100755 --- a/README.md +++ b/README.md @@ -62,7 +62,13 @@ git clone You're ready to start developing your chat function. Head over to the [Development](#development) section to learn more. -#### 4. Update the README +#### 4. Deploy the chat function + +You will have to add your API key and LLM model name into the Github repo settings. Under `Secrets and variables/Actions`: the API key must be added as a secret and the LLM model must be added as a variable. + +You must ensure the same namings as in your `.env` file. So, make sure to update the `.github/{dev and main}.yml` files with the correct parameter names. + +#### 5. Update the README In the `README.md` file, change the title and description so it fits the purpose of your chat function. From 68b234e4a333129cc0881cdf6e12f560121a77d1 Mon Sep 17 00:00:00 2001 From: neagualexa Date: Tue, 23 Sep 2025 10:00:21 +0100 Subject: [PATCH 39/57] further readme clarification --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 99e6906..7a77c74 100755 --- a/README.md +++ b/README.md @@ -68,6 +68,8 @@ You will have to add your API key and LLM model name into the Github repo settin You must ensure the same namings as in your `.env` file. So, make sure to update the `.github/{dev and main}.yml` files with the correct parameter names. +For more information, check the section below [Deploy to Lambda Feedback](#deploy-to-lambda-feedback). + #### 5. Update the README In the `README.md` file, change the title and description so it fits the purpose of your chat function. @@ -188,6 +190,8 @@ During development, we recommend using the **`dev`** branch. This branch will de After you are pleased with the performance of your Chatbot and have configured the repository, a [GitHub Actions workflow](.github/workflows/main.yml) will automatically build and deploy the chat function to Lambda Feedback as soon as changes are pushed to the main branch of the repository. This deployment will upload the function onto `staging.lambdafeedback.com`, and will also initiate an `approval` stage for prod environment. Once you reach this stage, please contact an admin from Lambda Feedback to review the code and approve it such that the code can be accessible onto the main [Lambda Feedback platform](https://www.lambdafeedback.com/). +> [!NOTE] Once the deployment in the **`dev`** or **`main`** branch has been successful, share your necessary environment variables (e.g. API key and LLM model) with one of the Lambda Feedback team member. + ## Troubleshooting ### Containerized Function Fails to Start From 0bbc96c13dc7830ac5a19ef1640e140ee342780c Mon Sep 17 00:00:00 2001 From: neagualexa Date: Thu, 25 Sep 2025 16:15:42 +0100 Subject: [PATCH 40/57] fix expressions rendering --- index.py | 35 ++++++++++---------- src/agents/utils/prompt_context_templates.py | 4 ++- src/agents/utils/testbench_agents.py | 2 +- 3 files changed, 22 insertions(+), 19 deletions(-) diff --git a/index.py b/index.py index 5ef3bb8..dd903e5 100644 --- a/index.py +++ b/index.py @@ -9,34 +9,32 @@ def handler(event: JsonType, context): """ Lambda handler function - Args: - event (JsonType): The AWS Lambda event received by the gateway. - context (Any): The AWS Lambda context object. - """ # Log the input event for debugging purposes - print("Received event:", json.dumps(event, indent=2)) + # print("Received event:", " ".join(json.dumps(event, indent=2).splitlines())) - if "body" not in event: - return { - "statusCode": 400, - "body": "Missing 'body' key in event. Please confirm the key in the json body." - } - body = json.loads(event["body"]) - - if "message" not in body: + if "body" in event: + try: + event = json.loads(event["body"]) + except json.JSONDecodeError: + return { + "statusCode": 400, + "body": "Invalid JSON format in the body or body not found. Please check the input." + } + + if "message" not in event: return { "statusCode": 400, "body": "Missing 'message' key in event. Please confirm the key in the json body." } - if "params" not in body: + if "params" not in event: return { "statusCode": 400, "body": "Missing 'params' key in event. Please confirm the key in the json body. Make sure it contains the necessary conversation_id." } - message = body["message"] - params = body["params"] + message = event.get("message") + params = event.get("params") try: chatbot_response = chat_module(message, params) @@ -49,7 +47,10 @@ def handler(event: JsonType, context): # Create a response response = { "statusCode": 200, - "body": json.dumps(chatbot_response) + "body": chatbot_response } + # Log the response for debugging purposes + print("Returning response:", " ".join(json.dumps(response, indent=2).splitlines())) + return response \ No newline at end of file diff --git a/src/agents/utils/prompt_context_templates.py b/src/agents/utils/prompt_context_templates.py index 9eb175b..75a40b1 100644 --- a/src/agents/utils/prompt_context_templates.py +++ b/src/agents/utils/prompt_context_templates.py @@ -73,7 +73,9 @@ def format_question_header( - Description: {content} {duration_text} -> Note: Mathematical equations are in KaTeX format, preserve them the same. Use British English spellings. +> Note: Mathematical equations are in KaTeX format, preserve them the same. Ensure mathematical equations are surrounded by one '$' for in-line equations and '$$' for block equations. +Example: '$E=mc^2$' or '$$E=mc^2$$'. +Use British English spellings. --- """ diff --git a/src/agents/utils/testbench_agents.py b/src/agents/utils/testbench_agents.py index e27ed52..46d0d36 100644 --- a/src/agents/utils/testbench_agents.py +++ b/src/agents/utils/testbench_agents.py @@ -27,7 +27,7 @@ STEP 2: Extract the parameters from the JSON """ # NOTE: #### This is the testing message!! ##### - message = "Hi" + message = "Hi, how do I solve this problem?" # NOTE: ######################################## # replace "mock" in the message and conversation history with the actual message From d15127352f0763e2415568d981487c5ca6c6ccf0 Mon Sep 17 00:00:00 2001 From: neagualexa Date: Thu, 25 Sep 2025 16:42:42 +0100 Subject: [PATCH 41/57] fix result body --- index.py | 2 +- src/module.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/index.py b/index.py index dd903e5..62f4c47 100644 --- a/index.py +++ b/index.py @@ -47,7 +47,7 @@ def handler(event: JsonType, context): # Create a response response = { "statusCode": 200, - "body": chatbot_response + "body": json.dumps(chatbot_response) } # Log the response for debugging purposes diff --git a/src/module.py b/src/module.py index 02c6f66..7cbbdd8 100755 --- a/src/module.py +++ b/src/module.py @@ -6,11 +6,13 @@ try: from .agents.utils.parse_json_context_to_prompt import parse_json_to_prompt from .agents.base_agent.base_agent import invoke_base_agent + from .agents.utils.types import JsonType except ImportError: from src.agents.utils.parse_json_context_to_prompt import parse_json_to_prompt from src.agents.base_agent.base_agent import invoke_base_agent + from src.agents.utils.types import JsonType -def chat_module(message: Any, params: Params) -> Result: +def chat_module(message: Any, params: Params) -> JsonType: """ Function used by student to converse with a chatbot. --- From fb0708446522c43fda0b1735ca4acdba3e67f438 Mon Sep 17 00:00:00 2001 From: neagualexa Date: Fri, 3 Oct 2025 09:22:21 +0100 Subject: [PATCH 42/57] add docs --- README.md | 57 ++++++++++++++++++++++++------- docs/dev.md | 96 ++++++++++++++++++++++++++++++++++++++++++++++++++++ docs/user.md | 3 ++ 3 files changed, 144 insertions(+), 12 deletions(-) create mode 100644 docs/dev.md create mode 100644 docs/user.md diff --git a/README.md b/README.md index 7a77c74..bf84b60 100755 --- a/README.md +++ b/README.md @@ -95,13 +95,20 @@ You agent can be based on an LLM hosted anywhere, you have available currently O main.yml # deploys the STAGING function to Lambda Feedback test-report.yml # gathers Pytest Report of function tests +docs/ # docs for devs and users + src/module.py # chat_module function implementation src/module_test.py # chat_module function tests src/agents/ # find all agents developed for the chat functionality src/agents/utils/test_prompts.py # allows testing of any LLM agent on a couple of example inputs containing Lambda Feedback Questions and synthetic student conversations ``` -## Run the Chat Script + +## Testing the Chat Function + +To test your function, you can either call the code directly through a python script. Or you can build the respective chat function docker container locally and call it through an API request. Below you can find details on those processes. + +### Run the Chat Script You can run the Python function itself. Make sure to have a main function in either `src/module.py` or `index.py`. @@ -114,7 +121,7 @@ You can also use the `testbench_agents.py` script to test the agents with exampl python src/agents/utils/testbench_agents.py ``` -### Building the Docker Image +### Calling the Docker Image Locally To build the Docker image, run the following command: @@ -122,17 +129,17 @@ To build the Docker image, run the following command: docker build -t llm_chat . ``` -### Running the Docker Image +#### Running the Docker Image To run the Docker image, use the following command: -#### Without .env file: +##### A. Without .env file: ```bash docker run -e OPENAI_API_KEY={your key} -e OPENAI_MODEL={your LLM chosen model name} -p 8080:8080 llm_chat ``` -#### With container name (for interaction, e.g. copying file from inside the docker container): +##### B. With container name (for interaction, e.g. copying file from inside the docker container): ```bash docker run --env-file .env -it --name my-lambda-container -p 8080:8080 llm_chat @@ -143,10 +150,15 @@ This will start the chat function and expose it on port `8080` and it will be op ```bash curl --location 'http://localhost:8080/2015-03-31/functions/function/invocations' \ --header 'Content-Type: application/json' \ ---data '{"body":"{\"message\": \"hi\", \"params\": {\"conversation_id\": \"12345Test\", \"conversation_history\": [{\"type\": \"user\", \"content\": \"hi\"}]}}"}' +--data '{"body":"{\"message\": \"hi\", \"params\": {\"conversation_id\": \"12345Test\", \"conversation_history\": [{\"type\": \"user\", ``` -### Call Docker Container From Postman +#### Call Docker Container +##### A. Call Docker with Python Requests + +In the `src/agents/utils` folder you can find the `requests_testscript.py` script that calls the POST URL of the running docker container. It reads any kind of input files with the expected schema. You can use this to test your curl calls of the chatbot. + +##### B. Call Docker Container through API request POST URL: @@ -154,7 +166,7 @@ POST URL: http://localhost:8080/2015-03-31/functions/function/invocations ``` -Body: +Body (stringified within body for API request): ```JSON {"body":"{\"message\": \"hi\", \"params\": {\"conversation_id\": \"12345Test\", \"conversation_history\": [{\"type\": \"user\", \"content\": \"hi\"}]}}"} @@ -176,10 +188,6 @@ Body with optional Params: } ``` -### Call Docker with Python Requests - -In the `src/agents/utils` folder you can find the `requests_test.py` script that calls the POST URL of the running docker container. It reads any kind of input files with the expected schema. You can use this to test your curl calls of the chatbot. - ### Deploy to Lambda Feedback Deploying the chat function to Lambda Feedback is simple and straightforward, as long as the repository is within the [Lambda Feedback organization](https://github.com/lambda-feedback). @@ -206,3 +214,28 @@ Make sure that all run-time dependencies are installed in the Docker image. - System packages: If you need to install system packages, add the installation command to the Dockerfile. - ML models: If your chat function depends on ML models, make sure to include them in the Docker image. - Data files: If your chat function depends on data files, make sure to include them in the Docker image. + +### Pull Changes from the Template Repository + +If you want to pull changes from the template repository to your repository, follow these steps: + +1. Add the template repository as a remote: + +```bash +git remote add template https://github.com/lambda-feedback/chat-function-boilerplate.git +``` + +2. Fetch changes from all remotes: + +```bash +git fetch --all +``` + +3. Merge changes from the template repository: + +```bash +git merge template/main --allow-unrelated-histories +``` + +> [!WARNING] +> Make sure to resolve any conflicts and keep the changes you want to keep. \ No newline at end of file diff --git a/docs/dev.md b/docs/dev.md new file mode 100644 index 0000000..a528199 --- /dev/null +++ b/docs/dev.md @@ -0,0 +1,96 @@ +# YourFunctionName +*Brief description of what this chat function does, from the developer perspective* + +## Inputs +*Specific input parameters which can be supplied when the calling this chat function.* + +## Outputs +*Output schema/values for this function* + +## Examples +*List of example inputs and outputs for this function, each under a different sub-heading* + +## Testing the Chat Function + +To test your function, you can either call the code directly through a python script. Or you can build the respective chat function docker container locally and call it through an API request. Below you can find details on those processes. + +### Run the Chat Script + +You can run the Python function itself. Make sure to have a main function in either `src/module.py` or `index.py`. + +```bash +python src/module.py +``` + +You can also use the `testbench_agents.py` script to test the agents with example inputs from Lambda Feedback questions and synthetic conversations. +```bash +python src/agents/utils/testbench_agents.py +``` + +### Calling the Docker Image Locally + +To build the Docker image, run the following command: + +```bash +docker build -t llm_chat . +``` + +#### Running the Docker Image + +To run the Docker image, use the following command: + +##### A. Without .env file: + +```bash +docker run -e OPENAI_API_KEY={your key} -e OPENAI_MODEL={your LLM chosen model name} -p 8080:8080 llm_chat +``` + +##### B. With container name (for interaction, e.g. copying file from inside the docker container): + +```bash +docker run --env-file .env -it --name my-lambda-container -p 8080:8080 llm_chat +``` + +This will start the chat function and expose it on port `8080` and it will be open to be curl: + +```bash +curl --location 'http://localhost:8080/2015-03-31/functions/function/invocations' \ +--header 'Content-Type: application/json' \ +--data '{"body":"{\"message\": \"hi\", \"params\": {\"conversation_id\": \"12345Test\", \"conversation_history\": [{\"type\": \"user\", +``` + +#### Call Docker Container +##### A. Call Docker with Python Requests + +In the `src/agents/utils` folder you can find the `requests_testscript.py` script that calls the POST URL of the running docker container. It reads any kind of input files with the expected schema. You can use this to test your curl calls of the chatbot. + +##### B. Call Docker Container through API request + +POST URL: + +```bash +http://localhost:8080/2015-03-31/functions/function/invocations +``` + +Body (stringified within body for API request): + +```JSON +{"body":"{\"message\": \"hi\", \"params\": {\"conversation_id\": \"12345Test\", \"conversation_history\": [{\"type\": \"user\", \"content\": \"hi\"}]}}"} +``` + +Body with optional Params: +```JSON +{ + "message":"hi", + "params":{ + "conversation_id":"12345Test", + "conversation_history":[{"type":"user","content":"hi"}], + "summary":" ", + "conversational_style":" ", + "question_response_details": "", + "include_test_data": true, + "agent_type": {agent_name} + } +} +``` + diff --git a/docs/user.md b/docs/user.md new file mode 100644 index 0000000..6f55616 --- /dev/null +++ b/docs/user.md @@ -0,0 +1,3 @@ +# YourChatFunctionName + +Teacher- & Student-facing documentation for this function. \ No newline at end of file From c274bbec64fca420ece8cfdf421b277299c2eac9 Mon Sep 17 00:00:00 2001 From: neagualexa Date: Fri, 3 Oct 2025 11:31:53 +0100 Subject: [PATCH 43/57] add optional tutorial parts --- src/agents/utils/parse_json_context_to_prompt.py | 15 ++++++++++++++- src/agents/utils/prompt_context_templates.py | 16 ++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/src/agents/utils/parse_json_context_to_prompt.py b/src/agents/utils/parse_json_context_to_prompt.py index 38af2da..e9f9dcd 100644 --- a/src/agents/utils/parse_json_context_to_prompt.py +++ b/src/agents/utils/parse_json_context_to_prompt.py @@ -76,6 +76,7 @@ def __init__( publishedPartContent: Optional[str] = None, publishedPartAnswerContent: Optional[str] = None, publishedWorkedSolutionSections: Optional[List[dict]] = [], + publishedStructuredTutorialSections: Optional[List[dict]] = [], publishedResponseAreas: Optional[List[Optional[ResponseAreaDetails]]] = [], ): self.publishedPartId = publishedPartId @@ -83,6 +84,7 @@ def __init__( self.publishedPartContent = publishedPartContent self.publishedPartAnswerContent = publishedPartAnswerContent self.publishedWorkedSolutionSections = publishedWorkedSolutionSections + self.publishedStructuredTutorialSections = publishedStructuredTutorialSections self.publishedResponseAreas = [ResponseAreaDetails(**publishedResponseArea) for publishedResponseArea in publishedResponseAreas] class QuestionDetails: @@ -277,9 +279,20 @@ def _format_single_part( 'content': ws.get('content', ''), 'position': ws.get('position', 0) }) + + # 6. Structured Tutorial Sections + tutorial_data = [] + if part.publishedStructuredTutorialSections: + for ts in part.publishedStructuredTutorialSections: + tutorial_data.append({ + 'title': ts.get('title', ''), + 'content': ts.get('content', ''), + 'position': ts.get('position', 0) + }) part_sections.append(PromptFormatter.format_worked_solutions(solutions_data)) - + part_sections.append(PromptFormatter.format_structured_tutorials(tutorial_data)) + return "\n".join(part_sections) + "\n---\n" diff --git a/src/agents/utils/prompt_context_templates.py b/src/agents/utils/prompt_context_templates.py index 75a40b1..e77a208 100644 --- a/src/agents/utils/prompt_context_templates.py +++ b/src/agents/utils/prompt_context_templates.py @@ -211,6 +211,22 @@ def format_worked_solutions(solutions: List[Dict[str, Any]]) -> str: {"\n".join(solution_texts)}""" + @staticmethod + def format_structured_tutorials(tutorials: List[Dict[str, Any]]) -> str: + """Format structured tutorials section.""" + if not tutorials: + return "### Structured Tutorials\n\nNone available" + + tutorial_texts = [] + for i, tutorial in enumerate(tutorials): + title = tutorial.get('title', f'Tutorial {i + 1}') + content = tutorial.get('content', '').strip() or 'No content available' + tutorial_texts.append(f"#### {title}\n\n{content}") + + return f"""### Structured Tutorials + +{"\n".join(tutorial_texts)}""" + @staticmethod def format_complete_prompt(sections: List[str]) -> str: """Combine all sections into a complete, well-structured prompt.""" From 4be2adc4428452e09ec48e1f317833179c9908ce Mon Sep 17 00:00:00 2001 From: Alexandra Neagu <33195033+neagualexa@users.noreply.github.com> Date: Mon, 8 Dec 2025 16:49:22 +0000 Subject: [PATCH 44/57] Dev: refactor of boilerplate (simplification) (#30) * refactoring: simplified folder structure * fix: gitignore and cicd yml * readme: update user and dev md * fix: rm relative imports * fix: add tests to dockerfile --- .dockerignore | 9 +- .github/workflows/dev.yml | 1 + .github/workflows/main.yml | 1 + .gitignore | 1 + Dockerfile | 2 +- README.md | 55 ++++--- docs/dev.md | 17 +- index.py | 8 +- src/__init__.py | 0 .../base_agent.py => agent/agent.py} | 20 +-- .../base_prompts.py => agent/prompts.py} | 61 +++++--- .../utils/example_inputs/example_input_1.json | 0 .../utils/example_inputs/example_input_2.json | 0 .../utils/example_inputs/example_input_3.json | 0 src/{agents => agent/utils}/llm_factory.py | 0 .../utils/parse_json_context_to_prompt.py | 6 +- .../utils/prompt_context_templates.py | 0 src/{agents => agent}/utils/types.py | 0 src/agents/__init__.py | 0 src/agents/student_agent/student_agent.py | 145 ------------------ src/agents/student_agent/student_prompts.py | 10 -- src/agents/utils/langgraph_viz.py | 13 -- .../synthetic_conversation_generation.py | 132 ---------------- .../utils/synthetic_conversations/NOTE.md | 4 - src/agents/utils/testbench_agents.py | 82 ---------- src/module.py | 72 ++++----- .../manual_agent_requests.py | 2 +- tests/manual_agent_run.py | 46 ++++++ index_test.py => tests/test_index.py | 6 +- src/module_test.py => tests/test_module.py | 14 +- 30 files changed, 188 insertions(+), 519 deletions(-) delete mode 100644 src/__init__.py rename src/{agents/base_agent/base_agent.py => agent/agent.py} (91%) rename src/{agents/base_agent/base_prompts.py => agent/prompts.py} (89%) rename src/{agents => agent}/utils/example_inputs/example_input_1.json (100%) rename src/{agents => agent}/utils/example_inputs/example_input_2.json (100%) rename src/{agents => agent}/utils/example_inputs/example_input_3.json (100%) rename src/{agents => agent/utils}/llm_factory.py (100%) rename src/{agents => agent}/utils/parse_json_context_to_prompt.py (99%) rename src/{agents => agent}/utils/prompt_context_templates.py (100%) rename src/{agents => agent}/utils/types.py (100%) delete mode 100644 src/agents/__init__.py delete mode 100644 src/agents/student_agent/student_agent.py delete mode 100644 src/agents/student_agent/student_prompts.py delete mode 100644 src/agents/utils/langgraph_viz.py delete mode 100644 src/agents/utils/synthetic_conversation_generation.py delete mode 100644 src/agents/utils/synthetic_conversations/NOTE.md delete mode 100644 src/agents/utils/testbench_agents.py rename src/agents/utils/requests_testscript.py => tests/manual_agent_requests.py (93%) create mode 100644 tests/manual_agent_run.py rename index_test.py => tests/test_index.py (95%) rename src/module_test.py => tests/test_module.py (89%) diff --git a/.dockerignore b/.dockerignore index 4093c0a..3d422e8 100644 --- a/.dockerignore +++ b/.dockerignore @@ -147,11 +147,4 @@ data/ reports/ # Synthetic data conversations -src/agents/utils/example_inputs/ -src/agents/utils/synthetic_conversations/ -src/agents/utils/synthetic_conversation_generation.py -src/agents/utils/testbench_prompts.py -src/agents/utils/langgraph_viz.py - -# development agents -src/agents/student_agent/ \ No newline at end of file +src/agents/utils/example_inputs/ \ No newline at end of file diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index 1191a86..1236ba3 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -50,6 +50,7 @@ jobs: if: always() run: | source .venv/bin/activate + export PYTHONPATH=$PYTHONPATH:. pytest --junit-xml=./reports/pytest.xml --tb=auto -v - name: Upload test results diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 8400ca3..1da0493 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -50,6 +50,7 @@ jobs: if: always() run: | source .venv/bin/activate + export PYTHONPATH=$PYTHONPATH:. pytest --junit-xml=./reports/pytest.xml --tb=auto -v - name: Upload test results diff --git a/.gitignore b/.gitignore index 4b52234..11f861b 100644 --- a/.gitignore +++ b/.gitignore @@ -50,6 +50,7 @@ coverage.xml *.py,cover .hypothesis/ .pytest_cache/ +reports/ # Translations *.mo diff --git a/Dockerfile b/Dockerfile index 9150687..38276cc 100755 --- a/Dockerfile +++ b/Dockerfile @@ -25,7 +25,7 @@ COPY src ./src COPY index.py . -COPY index_test.py . +COPY tests ./tests # Set the Lambda function handler CMD ["index.handler"] \ No newline at end of file diff --git a/README.md b/README.md index bf84b60..ef662b0 100755 --- a/README.md +++ b/README.md @@ -43,11 +43,11 @@ In GitHub, choose Use this template > Create a new repository in the repository Choose the owner, and pick a name for the new repository. -> [!IMPORTANT] If you want to deploy the evaluation function to Lambda Feedback, make sure to choose the Lambda Feedback organization as the owner. +> [!IMPORTANT] If you want to deploy the chat function to Lambda Feedback, make sure to choose the `Lambda Feedback` organization as the owner. -Set the visibility to Public or Private. +Set the visibility to `Public` or `Private`. -> [!IMPORTANT] If you want to use GitHub deployment protection rules, make sure to set the visibility to Public. +> [!IMPORTANT] If you want to use GitHub deployment protection rules, make sure to set the visibility to `Public`. Click on Create repository. @@ -78,9 +78,9 @@ Also, don't forget to update or delete the Quickstart chapter from the `README.m ## Development -You can create your own invocation to your own agents hosted anywhere. Copy or update the `base_agent` from `src/agents/` and edit it to match your LLM agent requirements. Import the new invocation in the `module.py` file. +You can create your own invocation to your own agents hosted anywhere. Copy or update the `agent.py` from `src/agent/` and edit it to match your LLM agent requirements. Import the new invocation in the `module.py` file. -You agent can be based on an LLM hosted anywhere, you have available currently OpenAI, AzureOpenAI, and Ollama models but you can introduce your own API call in the `src/agents/llm_factory.py`. +You agent can be based on an LLM hosted anywhere, you have available currently OpenAI, AzureOpenAI, and Ollama models but you can introduce your own API call in the `src/agent/utils/llm_factory.py`. ### Prerequisites @@ -90,23 +90,37 @@ You agent can be based on an LLM hosted anywhere, you have available currently O ### Repository Structure ```bash -.github/workflows/ - dev.yml # deploys the DEV function to Lambda Feedback - main.yml # deploys the STAGING function to Lambda Feedback - test-report.yml # gathers Pytest Report of function tests - -docs/ # docs for devs and users - -src/module.py # chat_module function implementation -src/module_test.py # chat_module function tests -src/agents/ # find all agents developed for the chat functionality -src/agents/utils/test_prompts.py # allows testing of any LLM agent on a couple of example inputs containing Lambda Feedback Questions and synthetic student conversations +. +├── .github/workflows/ +│ ├── dev.yml # deploys the DEV function to Lambda Feedback +│ ├── main.yml # deploys the STAGING and PROD functions to Lambda Feedback +│ └── test-report.yml # gathers Pytest Report of function tests +├── docs/ # docs for devs and users +├── src/ +│ ├── agent/ +│ │ ├── utils/ # utils for the agent, including the llm_factory +│ │ ├── agent.py # the agent logic +│ │ └── prompts.py # the system prompts defining the behaviour of the chatbot +│ └── module.py +└── tests/ # contains all tests for the chat function + ├── manual_agent_requests.py # allows testing of the docker container through API requests + ├── manual_agent_run.py # allows testing of any LLM agent on a couple of example inputs + ├── test_index.py # pytests + └── test_module.py # pytests ``` ## Testing the Chat Function -To test your function, you can either call the code directly through a python script. Or you can build the respective chat function docker container locally and call it through an API request. Below you can find details on those processes. +To test your function, you can run the unit tests, call the code directly through a python script, or build the respective chat function docker container locally and call it through an API request. Below you can find details on those processes. + +### Run Unit Tests + +You can run the unit tests using `pytest`. + +```bash +pytest +``` ### Run the Chat Script @@ -116,9 +130,9 @@ You can run the Python function itself. Make sure to have a main function in eit python src/module.py ``` -You can also use the `testbench_agents.py` script to test the agents with example inputs from Lambda Feedback questions and synthetic conversations. +You can also use the `manual_agent_run.py` script to test the agents with example inputs from Lambda Feedback questions and synthetic conversations. ```bash -python src/agents/utils/testbench_agents.py +python tests/manual_agent_run.py ``` ### Calling the Docker Image Locally @@ -156,7 +170,7 @@ curl --location 'http://localhost:8080/2015-03-31/functions/function/invocations #### Call Docker Container ##### A. Call Docker with Python Requests -In the `src/agents/utils` folder you can find the `requests_testscript.py` script that calls the POST URL of the running docker container. It reads any kind of input files with the expected schema. You can use this to test your curl calls of the chatbot. +In the `tests/` folder you can find the `manual_agent_requests.py` script that calls the POST URL of the running docker container. It reads any kind of input files with the expected schema. You can use this to test your curl calls of the chatbot. ##### B. Call Docker Container through API request @@ -183,7 +197,6 @@ Body with optional Params: "conversational_style":" ", "question_response_details": "", "include_test_data": true, - "agent_type": {agent_name} } } ``` diff --git a/docs/dev.md b/docs/dev.md index a528199..81d1433 100644 --- a/docs/dev.md +++ b/docs/dev.md @@ -12,7 +12,15 @@ ## Testing the Chat Function -To test your function, you can either call the code directly through a python script. Or you can build the respective chat function docker container locally and call it through an API request. Below you can find details on those processes. +To test your function, you can run the unit tests, call the code directly through a python script, or build the respective chat function docker container locally and call it through an API request. Below you can find details on those processes. + +### Run Unit Tests + +You can run the unit tests using `pytest`. + +```bash +pytest +``` ### Run the Chat Script @@ -22,9 +30,9 @@ You can run the Python function itself. Make sure to have a main function in eit python src/module.py ``` -You can also use the `testbench_agents.py` script to test the agents with example inputs from Lambda Feedback questions and synthetic conversations. +You can also use the `manual_agent_run.py` script to test the agents with example inputs from Lambda Feedback questions and synthetic conversations. ```bash -python src/agents/utils/testbench_agents.py +python tests/manual_agent_run.py ``` ### Calling the Docker Image Locally @@ -62,7 +70,7 @@ curl --location 'http://localhost:8080/2015-03-31/functions/function/invocations #### Call Docker Container ##### A. Call Docker with Python Requests -In the `src/agents/utils` folder you can find the `requests_testscript.py` script that calls the POST URL of the running docker container. It reads any kind of input files with the expected schema. You can use this to test your curl calls of the chatbot. +In the `tests/` folder you can find the `manual_agent_requests.py` script that calls the POST URL of the running docker container. It reads any kind of input files with the expected schema. You can use this to test your curl calls of the chatbot. ##### B. Call Docker Container through API request @@ -89,7 +97,6 @@ Body with optional Params: "conversational_style":" ", "question_response_details": "", "include_test_data": true, - "agent_type": {agent_name} } } ``` diff --git a/index.py b/index.py index 62f4c47..0ad738d 100644 --- a/index.py +++ b/index.py @@ -1,10 +1,6 @@ import json -try: - from .src.module import chat_module - from .src.agents.utils.types import JsonType -except ImportError: - from src.module import chat_module - from src.agents.utils.types import JsonType +from src.module import chat_module +from src.agent.utils.types import JsonType def handler(event: JsonType, context): """ diff --git a/src/__init__.py b/src/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/agents/base_agent/base_agent.py b/src/agent/agent.py similarity index 91% rename from src/agents/base_agent/base_agent.py rename to src/agent/agent.py index 0fb199d..31e23cb 100644 --- a/src/agents/base_agent/base_agent.py +++ b/src/agent/agent.py @@ -1,13 +1,7 @@ -try: - from ..llm_factory import OpenAILLMs, GoogleAILLMs - from .base_prompts import \ - role_prompt, conv_pref_prompt, update_conv_pref_prompt, summary_prompt, update_summary_prompt, summary_system_prompt - from ..utils.types import InvokeAgentResponseType -except ImportError: - from src.agents.llm_factory import OpenAILLMs, GoogleAILLMs - from src.agents.base_agent.base_prompts import \ - role_prompt, conv_pref_prompt, update_conv_pref_prompt, summary_prompt, update_summary_prompt, summary_system_prompt - from src.agents.utils.types import InvokeAgentResponseType +from src.agent.utils.llm_factory import OpenAILLMs, GoogleAILLMs +from src.agent.prompts import \ + role_prompt, conv_pref_prompt, update_conv_pref_prompt, summary_prompt, update_summary_prompt, summary_system_prompt +from src.agent.utils.types import InvokeAgentResponseType from langgraph.graph import StateGraph, START, END from langchain_core.messages import SystemMessage, RemoveMessage, HumanMessage, AIMessage @@ -62,7 +56,7 @@ def call_model(self, state: State, config: RunnableConfig) -> str: system_message = self.role_prompt # Adding external student progress and question context details from data queries - question_response_details = config["configurable"].get("question_response_details", "") + question_response_details = config.get("configurable", {}).get("question_response_details", "") if question_response_details: system_message += f"## Known Question Materials: {question_response_details} \n\n" @@ -98,8 +92,8 @@ def summarize_conversation(self, state: State, config: RunnableConfig) -> dict: """Summarize the conversation.""" summary = state.get("summary", "") - previous_summary = config["configurable"].get("summary", "") - previous_conversationalStyle = config["configurable"].get("conversational_style", "") + previous_summary = config.get("configurable", {}).get("summary", "") + previous_conversationalStyle = config.get("configurable", {}).get("conversational_style", "") if previous_summary: summary = previous_summary diff --git a/src/agents/base_agent/base_prompts.py b/src/agent/prompts.py similarity index 89% rename from src/agents/base_agent/base_prompts.py rename to src/agent/prompts.py index 683caab..a3577c8 100644 --- a/src/agents/base_agent/base_prompts.py +++ b/src/agent/prompts.py @@ -1,8 +1,43 @@ -# NOTE: -# PROMPTS generated with the help of ChatGPT GPT-4o Nov 2024 - +# +# NOTE: Default prompts generated with the help of ChatGPT GPT-4o Nov 2024 +# +# Description of the prompts: +# +# 1. role_prompt: Sets the overall role and behaviour of the chatbot. +# +# 2. summary_prompt: Used to generate a summary of the conversation. +# 2. update_summary_prompt: Used to update the conversation summary with new messages. +# 2. summary_system_prompt: Provides context for the chatbot based on the existing summary. +# +# 3. conv_pref_prompt: Used to analyze and extract the student's conversational style and learning preferences. +# 3. update_conv_pref_prompt: Used to update the conversational style based on new interactions. +# + +# 1. Role Prompt role_prompt = "You are an excellent tutor that aims to provide clear and concise explanations to students. I am the student. Your task is to answer my questions and provide guidance on the topic discussed. Ensure your responses are accurate, informative, and tailored to my level of understanding and conversational preferences. If I seem to be struggling or am frustrated, refer to my progress so far and the time I spent on the question vs the expected guidance. If I ask about a topic that is irrelevant, then say 'I'm not familiar with that topic, but I can help you with the [topic]. You do not need to end your messages with a concluding statement.\n\n" +# 2. Summary Prompts +summary_guidelines = """Ensure the summary is: + +Concise: Keep the summary brief while including all essential information. +Structured: Organize the summary into sections such as 'Topics Discussed' and 'Top 3 Key Detailed Ideas'. +Neutral and Accurate: Avoid adding interpretations or opinions; focus only on the content shared. +When summarizing: If the conversation is technical, highlight significant concepts, solutions, and terminology. If context involves problem-solving, detail the problem and the steps or solutions provided. If the user asks for creative input, briefly describe the ideas presented. +Last messages: Include the most recent 5 messages to provide context for the summary. + +Provide the summary in a bulleted format for clarity. Avoid redundant details while preserving the core intent of the discussion.""" + +summary_prompt = f"""Summarize the conversation between a student and a tutor. Your summary should highlight the major topics discussed during the session, followed by a detailed recollection of the last five significant points or ideas. Ensure the summary flows smoothly to maintain the continuity of the discussion. + +{summary_guidelines}""" + +update_summary_prompt = f"""Update the summary by taking into account the new messages above. + +{summary_guidelines}""" + +summary_system_prompt = "You are continuing a tutoring session with the student. Background context: {summary}. Use this context to inform your understanding but do not explicitly restate, refer to, or incorporate the details directly in your responses unless the user brings them up. Respond naturally to the user's current input, assuming prior knowledge from the summary." + +# 3. Conversational Preference Prompt pref_guidelines = """**Guidelines:** - Use concise, objective language. - Note the student's educational goals, such as understanding foundational concepts, passing an exam, getting top marks, code implementation, hands-on practice, etc. @@ -57,23 +92,3 @@ {pref_guidelines} """ - -summary_guidelines = """Ensure the summary is: - -Concise: Keep the summary brief while including all essential information. -Structured: Organize the summary into sections such as 'Topics Discussed' and 'Top 3 Key Detailed Ideas'. -Neutral and Accurate: Avoid adding interpretations or opinions; focus only on the content shared. -When summarizing: If the conversation is technical, highlight significant concepts, solutions, and terminology. If context involves problem-solving, detail the problem and the steps or solutions provided. If the user asks for creative input, briefly describe the ideas presented. -Last messages: Include the most recent 5 messages to provide context for the summary. - -Provide the summary in a bulleted format for clarity. Avoid redundant details while preserving the core intent of the discussion.""" - -summary_prompt = f"""Summarize the conversation between a student and a tutor. Your summary should highlight the major topics discussed during the session, followed by a detailed recollection of the last five significant points or ideas. Ensure the summary flows smoothly to maintain the continuity of the discussion. - -{summary_guidelines}""" - -update_summary_prompt = f"""Update the summary by taking into account the new messages above. - -{summary_guidelines}""" - -summary_system_prompt = "You are continuing a tutoring session with the student. Background context: {summary}. Use this context to inform your understanding but do not explicitly restate, refer to, or incorporate the details directly in your responses unless the user brings them up. Respond naturally to the user's current input, assuming prior knowledge from the summary." \ No newline at end of file diff --git a/src/agents/utils/example_inputs/example_input_1.json b/src/agent/utils/example_inputs/example_input_1.json similarity index 100% rename from src/agents/utils/example_inputs/example_input_1.json rename to src/agent/utils/example_inputs/example_input_1.json diff --git a/src/agents/utils/example_inputs/example_input_2.json b/src/agent/utils/example_inputs/example_input_2.json similarity index 100% rename from src/agents/utils/example_inputs/example_input_2.json rename to src/agent/utils/example_inputs/example_input_2.json diff --git a/src/agents/utils/example_inputs/example_input_3.json b/src/agent/utils/example_inputs/example_input_3.json similarity index 100% rename from src/agents/utils/example_inputs/example_input_3.json rename to src/agent/utils/example_inputs/example_input_3.json diff --git a/src/agents/llm_factory.py b/src/agent/utils/llm_factory.py similarity index 100% rename from src/agents/llm_factory.py rename to src/agent/utils/llm_factory.py diff --git a/src/agents/utils/parse_json_context_to_prompt.py b/src/agent/utils/parse_json_context_to_prompt.py similarity index 99% rename from src/agents/utils/parse_json_context_to_prompt.py rename to src/agent/utils/parse_json_context_to_prompt.py index e9f9dcd..a233126 100644 --- a/src/agents/utils/parse_json_context_to_prompt.py +++ b/src/agent/utils/parse_json_context_to_prompt.py @@ -3,7 +3,7 @@ """ from typing import List, Optional, Dict, Any, Union -from .prompt_context_templates import PromptFormatter +from src.agent.utils.prompt_context_templates import PromptFormatter # Definitions questionSubmissionSummary type class StudentLatestSubmission: @@ -150,7 +150,7 @@ def parse_json_to_structured_prompt( question_submission_summary: Optional[List[StudentWorkResponseArea]], question_information: Optional[QuestionDetails], question_access_information: Optional[QuestionAccessInformation] -) -> Optional[str]: +) -> str: """ Parse JSON data into a well-structured, LLM-friendly prompt. @@ -322,7 +322,7 @@ def parse_json_to_prompt( questionSubmissionSummary: Optional[List[StudentWorkResponseArea]], questionInformation: Optional[QuestionDetails], questionAccessInformation: Optional[QuestionAccessInformation] -) -> Optional[str]: +) -> str: """ Legacy wrapper for backward compatibility. Recommended to use parse_json_to_structured_prompt for new code. diff --git a/src/agents/utils/prompt_context_templates.py b/src/agent/utils/prompt_context_templates.py similarity index 100% rename from src/agents/utils/prompt_context_templates.py rename to src/agent/utils/prompt_context_templates.py diff --git a/src/agents/utils/types.py b/src/agent/utils/types.py similarity index 100% rename from src/agents/utils/types.py rename to src/agent/utils/types.py diff --git a/src/agents/__init__.py b/src/agents/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/agents/student_agent/student_agent.py b/src/agents/student_agent/student_agent.py deleted file mode 100644 index 4c5f54e..0000000 --- a/src/agents/student_agent/student_agent.py +++ /dev/null @@ -1,145 +0,0 @@ -try: - from ..llm_factory import OpenAILLMs - from .student_prompts import \ - base_student_persona, curious_student_persona, contradicting_student_persona, reliant_student_persona, confused_student_persona, unrelated_student_persona, \ - process_prompt - from ..utils.types import InvokeAgentResponseType -except ImportError: - from src.agents.llm_factory import OpenAILLMs - from src.agents.student_agent.student_prompts import \ - base_student_persona, curious_student_persona, contradicting_student_persona, reliant_student_persona, confused_student_persona, unrelated_student_persona, \ - process_prompt - from src.agents.utils.types import InvokeAgentResponseType - -from langgraph.graph import StateGraph, START, END -from langchain_core.messages import SystemMessage, RemoveMessage, HumanMessage, AIMessage -from langchain_core.runnables.config import RunnableConfig -from langgraph.graph.message import add_messages -from typing import Annotated, TypeAlias -from typing_extensions import TypedDict - -""" -Student agent for synthetic evaluation of the other LLM tutors. This agent is designed to be a student that requires help in the conversation. -[LLM workflow with a summarisation, and chat agent that receives an external conversation history]. - -This agent is designed to: -- [role_prompt] role of a student to ask questions on the topic -- [student_type] student's learning profile and comprehension level [many profiles can be chosen from the student_prompts.py] -""" - -ValidMessageTypes: TypeAlias = SystemMessage | HumanMessage | AIMessage -AllMessageTypes: TypeAlias = ValidMessageTypes | RemoveMessage - -class State(TypedDict): - messages: Annotated[list[AllMessageTypes], add_messages] - summary: str - -class StudentAgent: - def __init__(self, student_type: str): - llm = OpenAILLMs(temperature=0.75) - self.llm = llm.get_llm() - self.summary = "" - self.conversationalStyle = "" - self.type = student_type - - # Define Agent's specific Personas - self.role_prompt = process_prompt - if self.type == "base": - self.role_prompt += base_student_persona - elif self.type == "curious": - self.role_prompt += curious_student_persona - elif self.type == "contradicting": - self.role_prompt += contradicting_student_persona - elif self.type == "reliant": - self.role_prompt += reliant_student_persona - elif self.type == "confused": - self.role_prompt += confused_student_persona - elif self.type == "unrelated": - self.role_prompt += unrelated_student_persona - else: - raise Exception("Unknown Student Agent Type") - # Define a new graph for the conversation & compile it - self.workflow = StateGraph(State) - self.workflow_definition() - self.app = self.workflow.compile() - - def call_model(self, state: State, config: RunnableConfig) -> str: - """Call the LLM model knowing the role system prompt, the summary and the conversational style.""" - - # Default AI tutor role prompt - system_message = self.role_prompt - - # Adding external student progress and question context details from data queries - question_response_details = config["configurable"].get("question_response_details", "") - if question_response_details: - # convert "my" to "your" in the question_response_details to preserve the student agent as the user - question_response_details = question_response_details.replace("My", "Your") - question_response_details = question_response_details.replace("my", "your") - question_response_details = question_response_details.replace("I am", "you are") - system_message += f"\n\n## Known Learning Materials: {question_response_details} \n\n" - - # Adding summary and conversational style to the system message - summary = state.get("summary", "") - previous_summary = config["configurable"].get("summary", "") - if previous_summary: - summary = previous_summary - if summary: - system_message += f"## Summary of conversation earlier: {summary} \n\n" - - messages = [SystemMessage(content=system_message)] + state['messages'] - - valid_messages = self.check_for_valid_messages(messages) - response = self.llm.invoke(valid_messages) - - # Save summary for fetching outside the class - self.summary = summary - - return {"summary": summary, "messages": [response]} - - def check_for_valid_messages(self, messages: list[AllMessageTypes]) -> list[ValidMessageTypes]: - """ Removing the RemoveMessage() from the list of messages """ - - valid_messages: list[ValidMessageTypes] = [] - for message in messages: - if message.type != 'remove': - valid_messages.append(message) - return valid_messages - - def workflow_definition(self) -> None: - self.workflow.add_node("call_llm", self.call_model) - - self.workflow.add_edge(START, "call_llm") - self.workflow.add_edge("call_llm", END) - - def get_summary(self) -> str: - return self.summary - - def print_update(self, update: dict) -> None: - for k, v in update.items(): - for m in v["messages"]: - m.pretty_print() - if "summary" in v: - print(v["summary"]) - - def pretty_response_value(self, event: dict) -> str: - return event["messages"][-1].content - -def invoke_student_agent(query: str, conversation_history: list, summary: str, student_type:str, question_response_details: str, session_id: str) -> InvokeAgentResponseType: - """ - Call a base student agents that forms a basic conversation with the tutor agent. - """ - print(f'in invoke_student_agent(), student_type: {student_type}') - agent = StudentAgent(student_type=student_type) - - config = {"configurable": {"thread_id": session_id, "summary": summary, "question_response_details": question_response_details}} - response_events = agent.app.invoke({"messages": conversation_history + [AIMessage(content=query)]}, config=config, stream_mode="values") #updates - pretty_printed_response = agent.pretty_response_value(response_events) # get last event/ai answer in the response - - # Gather Metadata from the agent - summary = agent.get_summary() - - return { - "input": query, - "output": pretty_printed_response, - "intermediate_steps": [str(summary), conversation_history] - } \ No newline at end of file diff --git a/src/agents/student_agent/student_prompts.py b/src/agents/student_agent/student_prompts.py deleted file mode 100644 index e7a8e8e..0000000 --- a/src/agents/student_agent/student_prompts.py +++ /dev/null @@ -1,10 +0,0 @@ -# PROMPTS generated with the help of ChatGPT GPT-4o Nov 2024 - -process_prompt = "Maintain the flow of the conversation by responding directly to the latest message in one sentence. Stay in character as " - -base_student_persona = "a student who seeks assistance. Ask questions from a first-person perspective, requesting clarification on how to solve the promblem from the known materials." -curious_student_persona = "a curious and inquisitive student. Ask thoughtful, detailed questions from a first-person perspective to clarify concepts, explore real-life applications, and uncover complexities. Don’t hesitate to challenge assumptions and ask for clarification when needed." -contradicting_student_persona = "a skeptical student. Ask questions from a first-person perspective, questioning my reasoning, identifying potential flaws, and challenging explanations. Request clarification whenever something seems unclear or incorrect." -reliant_student_persona = "a student who relies heavily on your help. Ask questions from a first-person perspective, seeking help for even small problems, and requesting clarification or further assistance to ensure understanding." -confused_student_persona = "a student who feels confused and uncertain about the topic. Ask questions from a first-person perspective, expressing uncertainty about the material and requesting clarification on both the topic and the tutor’s reasoning." -unrelated_student_persona = "a student who engages in casual conversation. Ask lighthearted or unrelated questions from a first-person perspective, discussing personal interests or unrelated topics rather than focusing on the material." \ No newline at end of file diff --git a/src/agents/utils/langgraph_viz.py b/src/agents/utils/langgraph_viz.py deleted file mode 100644 index 79e15db..0000000 --- a/src/agents/utils/langgraph_viz.py +++ /dev/null @@ -1,13 +0,0 @@ -""" -Helper script to visualise the agent graph using pygraphviz. -Setup on mac [see more here https://github.com/pygraphviz/pygraphviz/blob/main/INSTALL.txt]: -# $ brew install graphviz -# $ pip install pygraphviz -""" - -agent = ... - - -graph = agent.app.get_graph() -print(graph) -graph.draw_png("./graph.png") \ No newline at end of file diff --git a/src/agents/utils/synthetic_conversation_generation.py b/src/agents/utils/synthetic_conversation_generation.py deleted file mode 100644 index 9235756..0000000 --- a/src/agents/utils/synthetic_conversation_generation.py +++ /dev/null @@ -1,132 +0,0 @@ -""" -## Synthetic Dataset Generator ## --> GOAL: Generate a synthetic dataset of conversations between a tutor and a student [both LLMs]. - -For each question/scenario example in the example_inputs folder, a pipeline of two agents will be invoked. -The agents will play the role of a tutor and a student conversing about the question/scenario. - -The conversations will be 20 turns long, with the tutor and student taking turns to send a message. - -The tutor can be one of the following types: -- Informational Agent (base) -The tutor agent can be selected by changing the "agent_type" field in this script. - -The student can have multiple skill levels and conversational styles. Those are defined by the prompts used by the LLM. - -Any of the models accessible through the API calls defined in the 'llm_factory.py' can be used for either the tutor and the agent LLM. -""" - -import csv -import json -try: - from ..student_agent.student_agent import invoke_student_agent - from .parse_json_context_to_prompt import parse_json_to_prompt - from ..base_agent.base_agent import invoke_base_agent -except ImportError: - from src.agents.student_agent.student_agent import invoke_student_agent - from src.agents.utils.parse_json_context_to_prompt import parse_json_to_prompt - from src.agents.base_agent.base_agent import invoke_base_agent -import os - - -def generate_synthetic_conversations(raw_text: str, num_turns: int, student_agent_type: str, tutor_agent_type: str): - """ - Generate a synthetic dataset of conversations between a tutor and a student [both LLMs]. - """ - if tutor_agent_type == "base": - invoke_tutor_agent = invoke_base_agent - else: - raise ValueError("Invalid tutor agent type") - - parsed_json = json.loads(raw_text) - params = parsed_json["params"] - conversation_id = params["conversation_id"] - include_test_data = params["include_test_data"] - summary = "" - conversational_style = "" - question_response_details = params["question_response_details"] - question_submission_summary = question_response_details["questionSubmissionSummary"] if "questionSubmissionSummary" in question_response_details else [] - question_information = question_response_details["questionInformation"] if "questionInformation" in question_response_details else {} - question_access_information = question_response_details["questionAccessInformation"] if "questionAccessInformation" in question_response_details else {} - question_response_details_prompt = parse_json_to_prompt( - question_submission_summary, - question_information, - question_access_information - ) - - # Generate Conversation - conversation_history = [] - message = "Ask a question." - for i in range(0,num_turns): - print(f"Turn {i+1} of {num_turns}") - if len(conversation_history) == 0: - message = "Ask me a question regarding your thoughts on the learning materials that you are currently woking on." - else: - message = conversation_history[-1]["content"] - - if i % 2 == 0: - # Student starts - student_response = invoke_student_agent(message, conversation_history[:-1], summary, student_agent_type, question_response_details_prompt, conversation_id) - conversation_history.append({ - "role": "user", - "content": student_response["output"] - }) - else: - tutor_response = invoke_tutor_agent(message, conversation_history, summary, conversational_style, question_response_details_prompt, conversation_id) - conversation_history.append({ - "role": "assistant", - "content": tutor_response["output"] - }) - - if "summary" in tutor_response: - summary = tutor_response["summary"] - if "conversationalStyle" in tutor_response: - conversational_style = tutor_response["conversationalStyle"] - - # Save Conversation - conversation_output = { - "conversation_id": conversation_id+"_"+student_agent_type+"_"+tutor_agent_type+"_synthetic", - "student_agent_type": student_agent_type, - "tutor_agent_type": tutor_agent_type, - "conversation": conversation_history - } - return conversation_output - - -if __name__ == "__main__": - num_turns = 6 - tutor_agent_types = ["base"] - # Students can be "base", "curious", "contradicting", "reliant", "confused", "unrelated" - student_agent_types = ["base", "curious", "contradicting", "reliant", "confused", "unrelated"] - - # Read all question files - questions = [] - example_inputs_folder = "src/agents/utils/example_inputs/" - output_folder = "src/agents/utils/synthetic_conversations/" - for filename in os.listdir(example_inputs_folder): - if filename.endswith("1.json"): - questions.append(os.path.join(example_inputs_folder, filename)) - - for tutor_agent_type in tutor_agent_types: - # Open CSV file for writing - csv_filename = os.path.join(output_folder, "all_conversations_"+tutor_agent_type+".csv") - with open(csv_filename, "w", newline='') as csvfile: - csv_writer = csv.writer(csvfile) - # Write the header - csv_writer.writerow(["tutor", "student", "conversation", "conversation_id"]) - - for student_agent_type in student_agent_types: - for question in questions: - print(f"Generating synthetic conversation for {question} with tutor: {tutor_agent_type} and student: {student_agent_type}") - with open(question, "r") as file: - raw_text = file.read() - - conversation = generate_synthetic_conversations(raw_text, num_turns, student_agent_type, tutor_agent_type) - - conversation_output_filename = output_folder + question.split('/')[-1].replace(".json", "_"+student_agent_type+"_"+tutor_agent_type+"_conversation.json") - with open(conversation_output_filename, "w") as file: - json.dump(conversation, file, indent=2) - - # Write to CSV - conversation_id = conversation["conversation_id"] - csv_writer.writerow([tutor_agent_type, student_agent_type, conversation["conversation"], conversation_id]) diff --git a/src/agents/utils/synthetic_conversations/NOTE.md b/src/agents/utils/synthetic_conversations/NOTE.md deleted file mode 100644 index 4bb113d..0000000 --- a/src/agents/utils/synthetic_conversations/NOTE.md +++ /dev/null @@ -1,4 +0,0 @@ -For evaluation purposes of the developed agent, you can use `synthetic_conversation_generation.py` to review the performance of your LLM tutor by running a multi-agent communication with a student agent (available in `src/agents/`). - -This folder contains all the synthetic conversations generated by an LLM student discussing with an LLM tutor. -The files are generated by running the `synthetic_conversation_generation.py`. \ No newline at end of file diff --git a/src/agents/utils/testbench_agents.py b/src/agents/utils/testbench_agents.py deleted file mode 100644 index 46d0d36..0000000 --- a/src/agents/utils/testbench_agents.py +++ /dev/null @@ -1,82 +0,0 @@ -""" - Conversation turn-based Testbench of the agent's performance. - Select an example input file and write your query. Then run the agent to get the response. -""" - -import json -try: - from .parse_json_context_to_prompt import parse_json_to_prompt - from ..base_agent.base_agent import invoke_base_agent -except ImportError: - from src.agents.utils.parse_json_context_to_prompt import parse_json_to_prompt - from src.agents.base_agent.base_agent import invoke_base_agent - -# File path for the input text -path = "src/agents/utils/example_inputs/" -input_file = path + "example_input_1.json" - -# Step 1: Read the input file -with open(input_file, "r") as file: - raw_text = file.read() - -# Step 5: Parse into JSON -try: - parsed_json = json.loads(raw_text) - - """ - STEP 2: Extract the parameters from the JSON - """ - # NOTE: #### This is the testing message!! ##### - message = "Hi, how do I solve this problem?" - # NOTE: ######################################## - - # replace "mock" in the message and conversation history with the actual message - parsed_json["message"] = message - parsed_json["params"]["conversation_history"][-1]["content"] = message - - params = parsed_json["params"] - - if "include_test_data" in params: - include_test_data = params["include_test_data"] - if "conversation_history" in params: - conversation_history = params["conversation_history"] - if "summary" in params: - summary = params["summary"] - if "conversational_style" in params: - conversationalStyle = params["conversational_style"] - if "question_response_details" in params: - question_response_details = params["question_response_details"] - question_submission_summary = question_response_details["questionSubmissionSummary"] if "questionSubmissionSummary" in question_response_details else [] - question_information = question_response_details["questionInformation"] if "questionInformation" in question_response_details else {} - question_access_information = question_response_details["questionAccessInformation"] if "questionAccessInformation" in question_response_details else {} - question_response_details_prompt = parse_json_to_prompt( - question_submission_summary, - question_information, - question_access_information - ) - print("Question Response Details Prompt:", question_response_details_prompt, "\n\n") - - if "conversation_id" in params: - conversation_id = params["conversation_id"] - else: - raise Exception("Internal Error: The conversation id is required in the parameters of the chat module.") - - """ - STEP 3: Call the LLM agent to get a response to the user's message - """ - response = invoke_base_agent(query=message, \ - conversation_history=conversation_history, \ - summary=summary, \ - conversationalStyle=conversationalStyle, \ - question_response_details=question_response_details_prompt, \ - session_id=conversation_id) - - print(response) - print("AI Response:", response['output']) - - -except json.JSONDecodeError as e: - print("Error decoding JSON:", e) - - - diff --git a/src/module.py b/src/module.py index 7cbbdd8..922979f 100755 --- a/src/module.py +++ b/src/module.py @@ -3,14 +3,9 @@ from lf_toolkit.chat.result import ChatResult as Result from lf_toolkit.chat.params import ChatParams as Params -try: - from .agents.utils.parse_json_context_to_prompt import parse_json_to_prompt - from .agents.base_agent.base_agent import invoke_base_agent - from .agents.utils.types import JsonType -except ImportError: - from src.agents.utils.parse_json_context_to_prompt import parse_json_to_prompt - from src.agents.base_agent.base_agent import invoke_base_agent - from src.agents.utils.types import JsonType +from src.agent.utils.parse_json_context_to_prompt import parse_json_to_prompt +from src.agent.agent import invoke_base_agent +from src.agent.utils.types import JsonType def chat_module(message: Any, params: Params) -> JsonType: """ @@ -36,40 +31,39 @@ def chat_module(message: Any, params: Params) -> JsonType: """ result = Result() - include_test_data = False - conversation_history = [] - summary = "" - conversationalStyle = "" - question_response_details_prompt = "" - if "include_test_data" in params: - include_test_data = params["include_test_data"] - if "conversation_history" in params: - conversation_history = params["conversation_history"] - if "summary" in params: - summary = params["summary"] - if "conversational_style" in params: - conversationalStyle = params["conversational_style"] - if "question_response_details" in params: - question_response_details = params["question_response_details"] - question_submission_summary = question_response_details["questionSubmissionSummary"] if "questionSubmissionSummary" in question_response_details else [] - question_information = question_response_details["questionInformation"] if "questionInformation" in question_response_details else {} - question_access_information = question_response_details["questionAccessInformation"] if "questionAccessInformation" in question_response_details else {} - try: - question_response_details_prompt = parse_json_to_prompt( - question_submission_summary, - question_information, - question_access_information - ) - print("INFO:: ", question_response_details_prompt) - except Exception as e: - print("ERROR:: ", e) - raise Exception("Internal Error: The question response details could not be parsed.") - if "conversation_id" in params: - conversation_id = params["conversation_id"] - else: + # EXTRACT PARAMETERS + conversation_id = params.get("conversation_id", None) + if conversation_id is None: raise Exception("Internal Error: The conversation id is required in the parameters of the chat module.") + + include_test_data = params.get("include_test_data", False) or False + conversation_history = params.get("conversation_history", []) or [] + summary = params.get("summary", "") or "" + conversationalStyle = params.get("conversational_style", "") or "" + + question_response_details = params.get("question_response_details", {}) + if isinstance(question_response_details, dict): + question_submission_summary = question_response_details.get("questionSubmissionSummary", []) + question_information = question_response_details.get("questionInformation", {}) + question_access_information = question_response_details.get("questionAccessInformation", {}) + else: + print("ERROR:: question_response_details is not a dict") + raise Exception("Internal Error: The question response details parameter is malformed.") + + # PARSE QUESTION RESPONSE DETAILS TO PROMPT + try: + question_response_details_prompt = parse_json_to_prompt( + question_submission_summary, + question_information, + question_access_information + ) + except Exception as e: + print("ERROR:: ", e) + raise Exception("Internal Error: The question response details could not be parsed.") + + # RUN THE AGENT AND MEASURE PROCESSING TIME start_time = time.time() chatbot_response = invoke_base_agent(query=message, \ diff --git a/src/agents/utils/requests_testscript.py b/tests/manual_agent_requests.py similarity index 93% rename from src/agents/utils/requests_testscript.py rename to tests/manual_agent_requests.py index 6b8e433..9d4c729 100644 --- a/src/agents/utils/requests_testscript.py +++ b/tests/manual_agent_requests.py @@ -9,7 +9,7 @@ url = "http://localhost:8080/2015-03-31/functions/function/invocations" # File path for the input text -path = "src/agents/utils/example_inputs/" +path = "src/agent/utils/example_inputs/" input_file = path + "example_input_1.json" # Step 1: Read the input file diff --git a/tests/manual_agent_run.py b/tests/manual_agent_run.py new file mode 100644 index 0000000..b69f609 --- /dev/null +++ b/tests/manual_agent_run.py @@ -0,0 +1,46 @@ +""" + Conversation turn-based Testbench of the agent's performance. + Select an example input file and write your query. Then run the agent to get the response. +""" + +import json +from src.module import chat_module + +# File path for the input text +path = "src/agent/utils/example_inputs/" +input_file = path + "example_input_1.json" + +# Step 1: Read the input file +with open(input_file, "r") as file: + raw_text = file.read() + +# Step 5: Parse into JSON +try: + parsed_json = json.loads(raw_text) + + """ + STEP 2: Extract the parameters from the JSON + """ + # NOTE: #### This is the testing message ##### + message = "Hi, how do I solve this problem?" + # NOTE: ######################################## + + # In the JSON, replace "mock" in the message and conversation history with the testing message + parsed_json["message"] = message + parsed_json["params"]["conversation_history"][-1]["content"] = message + + params = parsed_json["params"] + + """ + STEP 3: Call the chat module to get a response to the user's message + """ + response = chat_module(message, params) + + print(json.dumps(response, indent=4)) + + +except json.JSONDecodeError as e: + print("Error decoding JSON:", e) + + + diff --git a/index_test.py b/tests/test_index.py similarity index 95% rename from index_test.py rename to tests/test_index.py index fb8e7bf..822049d 100644 --- a/index_test.py +++ b/tests/test_index.py @@ -1,10 +1,6 @@ import unittest import json - -try: - from .index import handler -except ImportError: - from index import handler +from index import handler class TestChatIndexFunction(unittest.TestCase): """ diff --git a/src/module_test.py b/tests/test_module.py similarity index 89% rename from src/module_test.py rename to tests/test_module.py index 222eef2..26c96de 100755 --- a/src/module_test.py +++ b/tests/test_module.py @@ -1,9 +1,7 @@ import unittest - -try: - from .module import Params, chat_module -except ImportError: - from module import Params, chat_module +from lf_toolkit.chat.result import ChatResult as Result +from lf_toolkit.chat.params import ChatParams as Params +from src.module import chat_module class TestChatModuleFunction(unittest.TestCase): """ @@ -27,7 +25,7 @@ class TestChatModuleFunction(unittest.TestCase): def test_missing_parameters(self): # Checking state for missing parameters on default agent response = "Hello, World" - expected_params = Params(include_test_data=True, conversation_history=[{ "type": "user", "content": response }], \ + expected_params = Params(include_test_data=True, conversation_history=['{ "type": "user", "content": response }'], \ summary="", conversational_style="", \ question_response_details={}, conversation_id="1234Test") @@ -69,7 +67,7 @@ def test_missing_parameters(self): def test_agent_output(self): # Checking the output of the agent response = "Hello, World" - params = Params(conversation_id="1234Test", conversation_history=[{ "type": "user", "content": response }]) + params = Params(conversation_id="1234Test", conversation_history=['{ "type": "user", "content": response }']) result = chat_module(response, params) @@ -78,7 +76,7 @@ def test_agent_output(self): def test_processing_time_calc(self): # Checking the processing time calculation response = "Hello, World" - params = Params(include_test_data=True, conversation_id="1234Test", conversation_history=[{ "type": "user", "content": response }]) + params = Params(include_test_data=True, conversation_id="1234Test", conversation_history=['{ "type": "user", "content": response }']) result = chat_module(response, params) From 6512a6a6014db426f3fb7ae6ebcb7624ea6ae9f3 Mon Sep 17 00:00:00 2001 From: Alexandra Neagu <33195033+neagualexa@users.noreply.github.com> Date: Mon, 30 Mar 2026 13:02:45 +0100 Subject: [PATCH 45/57] Mued api adopted (#32) * refactoring: simplified folder structure * fix: gitignore and cicd yml * readme: update user and dev md * fix: rm relative imports * fix: add tests to dockerfile * muEd request, response adopted * remove unnecessary request attributes * fix * fix * refactoring for mued with simlified structure and better testing * Update src/agent/context.py response format prompt Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- AGENTS.md | 88 ++++ CLAUDE.md | 88 ++++ README.md | 117 ++++- docs/dev.md | 108 +++- index.py | 44 +- src/agent/agent.py | 205 +++----- src/agent/context.py | 155 ++++++ src/agent/{utils => }/llm_factory.py | 17 +- src/agent/prompts.py | 6 + .../utils/example_inputs/example_input_1.json | 168 ------- .../utils/example_inputs/example_input_2.json | 143 ------ .../utils/example_inputs/example_input_3.json | 460 ------------------ .../utils/parse_json_context_to_prompt.py | 334 ------------- src/agent/utils/prompt_context_templates.py | 263 ---------- src/agent/utils/types.py | 4 - src/module.py | 123 +++-- tests/example_inputs/example_input_0.json | 7 + tests/example_inputs/example_input_1.json | 123 +++++ tests/example_inputs/example_input_2.json | 97 ++++ tests/example_inputs/example_input_3.json | 244 ++++++++++ tests/manual_agent_run.py | 38 +- tests/test_example_inputs.py | 41 ++ tests/test_index.py | 82 +--- tests/test_module.py | 89 +--- tests/utils.py | 35 ++ 25 files changed, 1266 insertions(+), 1813 deletions(-) create mode 100644 AGENTS.md create mode 100644 CLAUDE.md create mode 100644 src/agent/context.py rename src/agent/{utils => }/llm_factory.py (91%) delete mode 100644 src/agent/utils/example_inputs/example_input_1.json delete mode 100644 src/agent/utils/example_inputs/example_input_2.json delete mode 100644 src/agent/utils/example_inputs/example_input_3.json delete mode 100644 src/agent/utils/parse_json_context_to_prompt.py delete mode 100644 src/agent/utils/prompt_context_templates.py delete mode 100644 src/agent/utils/types.py create mode 100644 tests/example_inputs/example_input_0.json create mode 100644 tests/example_inputs/example_input_1.json create mode 100644 tests/example_inputs/example_input_2.json create mode 100644 tests/example_inputs/example_input_3.json create mode 100644 tests/test_example_inputs.py create mode 100644 tests/utils.py diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..514764f --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,88 @@ +# AGENTS.md + +This file provides guidance to AI agents when working with code in this repository. + +## Project Overview + +This is a boilerplate for creating AI educational chatbots that integrate with the **Lambda-Feedback** educational platform. It deploys as an AWS Lambda function (containerized via Docker) that receives student chat messages with educational context and returns LLM-powered chatbot responses. + +## Commands + +**Testing:** +```bash +pytest # Run all unit tests +python tests/manual_agent_run.py # Test agent locally with example inputs +python tests/manual_agent_requests.py # Test running Docker container +``` + +**Docker:** +```bash +docker build -t llm_chat . +docker run --env-file .env -p 8080:8080 llm_chat +``` + +**Manual API test (while Docker is running):** +```bash +curl -X POST http://localhost:8080/2015-03-31/functions/function/invocations \ + -H 'Content-Type: application/json' \ + -d '{"body":"{\"conversationId\": \"12345Test\", \"messages\": [{\"role\": \"USER\", \"content\": \"hi\"}], \"user\": {\"type\": \"LEARNER\"}}"}' +``` + +**Run a single test:** +```bash +pytest tests/test_module.py # Run specific test file +pytest tests/test_index.py::test_function_name # Run specific test +``` + +## Architecture + +### Request Flow + +``` +Lambda event → index.py (handler) + → validates via lf_toolkit ChatRequest schema + → src/module.py (chat_module) + → extracts muEd API context (messages, conversationId, question context, user type) + → parses educational context to prompt text via src/agent/context.py + → src/agent/agent.py (BaseAgent / LangGraph) + → routes to call_llm or summarize_conversation node + → calls LLM provider (OpenAI / Google / Azure / Ollama) + → returns ChatResponse (output, summary, conversationalStyle, processingTime) +``` + +### Key Files + +| File | Role | +|------|------| +| `index.py` | AWS Lambda entry point; parses event body, validates schema | +| `src/module.py` | Transforms muEd API request → invokes agent → builds ChatResponse | +| `src/agent/agent.py` | LangGraph stateful graph; manages message history and summarization | +| `src/agent/prompts.py` | System prompts for tutor behavior, summarization, style detection | +| `src/agent/llm_factory.py` | Factory classes for each LLM provider (OpenAI, Google, Azure, Ollama) | +| `src/agent/context.py` | Converts muEd question/submission context dicts to LLM prompt text | +| `tests/utils.py` | Shared test helpers: `assert_valid_chat_request`, `assert_valid_chat_response` | +| `tests/example_inputs/` | Real muEd payloads used for end-to-end tests | + +### Agent Logic (LangGraph) + +`BaseAgent` maintains a state graph with two nodes: +- **`call_llm`**: Invokes the LLM with system prompt + conversation summary + conversational style preference +- **`summarize_conversation`**: Triggered when message count exceeds ~11; summarizes history and also extracts the student's preferred conversational style + +Messages are trimmed after summarization to keep context window manageable. The `summary` and `conversationalStyle` fields persist across calls via the `ChatRequest` metadata. + +### muEd API Format + +`src/module.py` handles the muEd request format (https://mued.org/). The `context` field in `ChatRequest` contains nested educational data (question parts, student submissions, task info) that gets parsed into a tutoring prompt via `src/agent/context.py`. + +### LLM Configuration + +LLM provider and model are set via environment variables (see `.env.example`). The `llm_factory.py` selects the provider at runtime. The Lambda function name/identity is set in `config.json`. + +The agent uses **two separate LLM instances** — `self.llm` for chat responses and `self.summarisation_llm` for conversation summarisation and style analysis. By default both use the same provider, but you can point them at different models (e.g. a cheaper model for summarisation) by changing the class in `agent.py`. + +## Deployment + +- Pushing to `dev` branch triggers the dev deployment GitHub Actions workflow +- Pushing to `main` triggers staging deployment, with manual approval required for production +- All environment variables (API keys, model names) are injected via GitHub Actions secrets/variables — do not hardcode them diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..9c8ebda --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,88 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Project Overview + +This is a boilerplate for creating AI educational chatbots that integrate with the **Lambda-Feedback** educational platform. It deploys as an AWS Lambda function (containerized via Docker) that receives student chat messages with educational context and returns LLM-powered chatbot responses. + +## Commands + +**Testing:** +```bash +pytest # Run all unit tests +python tests/manual_agent_run.py # Test agent locally with example inputs +python tests/manual_agent_requests.py # Test running Docker container +``` + +**Docker:** +```bash +docker build -t llm_chat . +docker run --env-file .env -p 8080:8080 llm_chat +``` + +**Manual API test (while Docker is running):** +```bash +curl -X POST http://localhost:8080/2015-03-31/functions/function/invocations \ + -H 'Content-Type: application/json' \ + -d '{"body":"{\"conversationId\": \"12345Test\", \"messages\": [{\"role\": \"USER\", \"content\": \"hi\"}], \"user\": {\"type\": \"LEARNER\"}}"}' +``` + +**Run a single test:** +```bash +pytest tests/test_module.py # Run specific test file +pytest tests/test_index.py::test_function_name # Run specific test +``` + +## Architecture + +### Request Flow + +``` +Lambda event → index.py (handler) + → validates via lf_toolkit ChatRequest schema + → src/module.py (chat_module) + → extracts muEd API context (messages, conversationId, question context, user type) + → parses educational context to prompt text via src/agent/context.py + → src/agent/agent.py (BaseAgent / LangGraph) + → routes to call_llm or summarize_conversation node + → calls LLM provider (OpenAI / Google / Azure / Ollama) + → returns ChatResponse (output, summary, conversationalStyle, processingTime) +``` + +### Key Files + +| File | Role | +|------|------| +| `index.py` | AWS Lambda entry point; parses event body, validates schema | +| `src/module.py` | Transforms muEd API request → invokes agent → builds ChatResponse | +| `src/agent/agent.py` | LangGraph stateful graph; manages message history and summarization | +| `src/agent/prompts.py` | System prompts for tutor behavior, summarization, style detection | +| `src/agent/llm_factory.py` | Factory classes for each LLM provider (OpenAI, Google, Azure, Ollama) | +| `src/agent/context.py` | Converts muEd question/submission context dicts to LLM prompt text | +| `tests/utils.py` | Shared test helpers: `assert_valid_chat_request`, `assert_valid_chat_response` | +| `tests/example_inputs/` | Real muEd payloads used for end-to-end tests | + +### Agent Logic (LangGraph) + +`BaseAgent` maintains a state graph with two nodes: +- **`call_llm`**: Invokes the LLM with system prompt + conversation summary + conversational style preference +- **`summarize_conversation`**: Triggered when message count exceeds ~11; summarizes history and also extracts the student's preferred conversational style + +Messages are trimmed after summarization to keep context window manageable. The `summary` and `conversationalStyle` fields persist across calls via the `ChatRequest` metadata. + +### muEd API Format + +`src/module.py` handles the muEd request format (https://mued.org/). The `context` field in `ChatRequest` contains nested educational data (question parts, student submissions, task info) and the `user` field contains user-specific information (e.g., user type, preferences, task progress) that gets parsed into a tutoring prompt via `src/agent/context.py`. + +### LLM Configuration + +LLM provider and model are set via environment variables (see `.env.example`). The `llm_factory.py` selects the provider at runtime. The Lambda function name/identity is set in `config.json`. + +The agent uses **two separate LLM instances** — `self.llm` for chat responses and `self.summarisation_llm` for conversation summarisation and style analysis. By default both use the same provider, but you can point them at different models (e.g. a cheaper model for summarisation) by changing the class in `agent.py`. + +## Deployment + +- Pushing to `dev` branch triggers the dev deployment GitHub Actions workflow +- Pushing to `main` triggers staging deployment, with manual approval required for production +- All environment variables (API keys, model names) are injected via GitHub Actions secrets/variables — do not hardcode them diff --git a/README.md b/README.md index ef662b0..bbc2fa1 100755 --- a/README.md +++ b/README.md @@ -80,7 +80,9 @@ Also, don't forget to update or delete the Quickstart chapter from the `README.m You can create your own invocation to your own agents hosted anywhere. Copy or update the `agent.py` from `src/agent/` and edit it to match your LLM agent requirements. Import the new invocation in the `module.py` file. -You agent can be based on an LLM hosted anywhere, you have available currently OpenAI, AzureOpenAI, and Ollama models but you can introduce your own API call in the `src/agent/utils/llm_factory.py`. +Your agent can be based on an LLM hosted anywhere. OpenAI, Google AI, Azure OpenAI, and Ollama are available out of the box via `src/agent/llm_factory.py`, and you can add your own provider there too. + +The agent uses **two separate LLM instances** — `self.llm` for chat responses and `self.summarisation_llm` for conversation summarisation and style analysis. By default both use the same provider, but you can point them at different models (e.g. a cheaper or faster model for summarisation) by changing the class in `agent.py`. ### Prerequisites @@ -98,13 +100,17 @@ You agent can be based on an LLM hosted anywhere, you have available currently O ├── docs/ # docs for devs and users ├── src/ │ ├── agent/ -│ │ ├── utils/ # utils for the agent, including the llm_factory -│ │ ├── agent.py # the agent logic -│ │ └── prompts.py # the system prompts defining the behaviour of the chatbot -│ └── module.py +│ │ ├── agent.py # LangGraph stateful agent logic +│ │ ├── context.py # converts muEd context dicts to LLM prompt text +│ │ ├── llm_factory.py # factory classes for each LLM provider +│ │ └── prompts.py # system prompts defining the behaviour of the chatbot +│ └── module.py └── tests/ # contains all tests for the chat function + ├── example_inputs/ # muEd example payloads for end-to-end tests ├── manual_agent_requests.py # allows testing of the docker container through API requests ├── manual_agent_run.py # allows testing of any LLM agent on a couple of example inputs + ├── utils.py # shared test helpers + ├── test_example_inputs.py # pytests for the example input files ├── test_index.py # pytests └── test_module.py # pytests ``` @@ -164,7 +170,7 @@ This will start the chat function and expose it on port `8080` and it will be op ```bash curl --location 'http://localhost:8080/2015-03-31/functions/function/invocations' \ --header 'Content-Type: application/json' \ ---data '{"body":"{\"message\": \"hi\", \"params\": {\"conversation_id\": \"12345Test\", \"conversation_history\": [{\"type\": \"user\", +--data '{"body":"{\"conversationId\": \"12345Test\", \"messages\": [{\"role\": \"USER\", \"content\": \"hi\"}], \"user\": {\"type\": \"LEARNER\"}}"}' ``` #### Call Docker Container @@ -183,21 +189,98 @@ http://localhost:8080/2015-03-31/functions/function/invocations Body (stringified within body for API request): ```JSON -{"body":"{\"message\": \"hi\", \"params\": {\"conversation_id\": \"12345Test\", \"conversation_history\": [{\"type\": \"user\", \"content\": \"hi\"}]}}"} +{"body":"{\"conversationId\": \"12345Test\", \"messages\": [{\"role\": \"USER\", \"content\": \"hi\"}], \"user\": {\"type\": \"LEARNER\"}}"} ``` -Body with optional Params: -```JSON +Body with optional fields: +```json { - "message":"hi", - "params":{ - "conversation_id":"12345Test", - "conversation_history":[{"type":"user","content":"hi"}], - "summary":" ", - "conversational_style":" ", - "question_response_details": "", - "include_test_data": true, + "conversationId": "", + "messages": [ + { "role": "USER", "content": "" }, + { "role": "ASSISTANT", "content": "" }, + { "role": "USER", "content": "" } + ], + "user": { + "type": "LEARNER", + "preference": { + "conversationalStyle": "" + }, + "taskProgress": { + "timeSpentOnQuestion": "30 minutes", + "accessStatus": "a good amount of time spent on this question today.", + "markedDone": "This question is still being worked on.", + "currentPart": { + "position": 0, + "timeSpentOnPart": "10 minutes", + "markedDone": "This part is not marked done.", + "responseAreas": [ + { + "responseType": "EXPRESSION", + "totalSubmissions": 3, + "wrongSubmissions": 2, + "latestSubmission": { + "submission": "", + "feedback": "", + "answer": "" + } + } + ] + } } + }, + "context": { + "summary": "", + "set": { + "title": "Fundamentals", + "number": 2, + "description": "" + }, + "question": { + "title": "Understanding Polymorphism", + "number": 3, + "guidance": "", + "content": "", + "estimatedTime": "15-25 minutes", + "parts": [ + { + "position": 0, + "content": "", + "answerContent": "", + "workedSolutionSections": [ + { "position": 0, "title": "Step 1", "content": "..." } + ], + "structuredTutorialSections": [ + { "position": 0, "title": "Hint", "content": "..." } + ], + "responseAreas": [ + { + "position": 0, + "responseType": "EXPRESSION", + "answer": "", + "preResponseText": "