From 7e495eaa4cbfa96dcc51e2495a6543554dc5607a Mon Sep 17 00:00:00 2001 From: Pari Work Temp Date: Fri, 21 Mar 2025 13:36:52 -0700 Subject: [PATCH 01/26] updated ollama code --- .config.example | 4 +- README.md | 16 +++ justfile | 68 +++++++++++-- llm/__pycache__/pydantic.cpython-312.pyc | Bin 0 -> 150 bytes llm/__pycache__/pydantic.cpython-313.pyc | Bin 0 -> 150 bytes llm/ollama_client.py | 124 +++++++++++++++++++++++ 6 files changed, 201 insertions(+), 11 deletions(-) create mode 100644 llm/__pycache__/pydantic.cpython-312.pyc create mode 100644 llm/__pycache__/pydantic.cpython-313.pyc create mode 100644 llm/ollama_client.py diff --git a/.config.example b/.config.example index 961af7e..3ab5728 100644 --- a/.config.example +++ b/.config.example @@ -1 +1,3 @@ -PROJECTS=project_1,project_2,project_3 \ No newline at end of file +PROJECTS=project_1,project_2,project_3 +OLLAMA_API_URL="http://localhost:11434" +OLLAMA_MODEL="llama3.3" diff --git a/README.md b/README.md index 5dfd0c8..f85740d 100644 --- a/README.md +++ b/README.md @@ -140,6 +140,22 @@ Note: For AWS helpers, the AWS CLI is required for these commands to work. - Use force=true to skip confirmation prompt - Example: `tn heroku-delete-pipeline my-project true` +### Ollama as a local LLM +Use `tn update-config` to set a basic config the default for the local ollama is http://localhost:11434. You should be setting + +``` +OLLAMA_API_URL="https://pari-ollama.ngrok.io" +OLLAMA_MODEL="llama3.3" +``` + +- `tn ollama-serve`: serve a local llm server this is needed to run any commands +- `tn ollama-pull`: When working locally if you want to try a model you have to pull it +- `tn ollama-run`: Run an interactive chat with the local ollama server +- `tn ollama-codegen message`: Use the code gen specific cli (WIP) + optional args: `--api_url=""` use a custom url other than the default (eg ngrok) + `--model=""` use a custom model the default is qwen2.5-coder:7b + + ## Contributing New Commands - aka "Recipes" It's easy to contribute new commands to tn-cli, simply add some commands to the `justfile` and open a Pull Request. diff --git a/justfile b/justfile index bbfff6d..0632aab 100644 --- a/justfile +++ b/justfile @@ -10,7 +10,6 @@ os-info: [group('general')] install-uv: curl -LsSf https://astral.sh/uv/install.sh | sh - # # Re-clone and reinstall tn-cli # @@ -23,6 +22,50 @@ update: git clone git@github.com:thinknimble/tn-cli.git ~/.tn/cli fi +# +# Set up the config file +# +[group('config')] +init-config: + #!/usr/bin/env bash + ls -a ~/.tn + if [ -d ~/.tn/.config ]; then + echo "Config file already exists." + else + if [ -f ~/.tn/cli/.config.example ]; then + echo "Source config file exists: ~/.tn/cli/.config.example" + mkdir -p ~/.tn + cp ~/.tn/cli/.config.example ~/.tn/.config + if [ $? -eq 0 ]; then + echo "Config file copied successfully." + else + echo "Failed to copy config file." + fi + else + echo "Source config file does not exist: ~/.tn/cli/.config.example" + fi + fi +# +# Update the config +# + +[group('config')] +update-config var_name var_value: + #!/usr/bin/env bash + if [ ! -d ~/.tn/.config ]; then + echo "No config file found. Run 'tn config-init' first." + else + # read the file, if the variable exists update it otherwise add it + if grep -q "^$var_name=" ~/.tn/.config; then + sed -i "s/^$var_name=.*/$var_name=$var_value/" ~/.tn/.config + echo "Updated $var_name in config file." + else + echo "$var_name=$var_value" >> ~/.tn/.config + echo "Added $var_name to config file." + fi + fi + + # # Bootstrap new projects # @@ -205,8 +248,6 @@ gh-all-prs: echo "" done - # Ollama CLI - # # Ollama CLI # @@ -231,16 +272,23 @@ ollama-serve: ollama serve [group('ollama')] -ollama-gen: - ollama generate +ollama-pull model: + ollama pull {{model}} [group('ollama')] -ollama-codegen: - ollama - +ollama-run model: + ollama run {{model}} + + [group('ollama')] -ollama-customgen: - ollama +ollama-codegen message api_url='' model='qwen2.5-coder:7b': + #!/usr/bin/env bash + echo "Running Ollama codegen... using base model {{model}}" + echo "To clear the chat history, type 'clear chat history' and press enter." + CLEAN_API_URL=$(echo "{{api_url}}" | sed 's/^--api_url=//') + CLEAN_MODEL=$(echo "{{model}}" | sed 's/^--model=//') + + uvx uv run ./llm/ollama_client.py "{{message}}" --api_url="$CLEAN_API_URL" --model="$CLEAN_MODEL" # repo should be like: `owner/repo_name` [group('github')] diff --git a/llm/__pycache__/pydantic.cpython-312.pyc b/llm/__pycache__/pydantic.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0d18d5251123a5b580fb7c2a4399ec2c314dd56f GIT binary patch literal 150 zcmX@j%ge<81oQUYO=kqsk3k%C@RXR6n#hwWwIX zAh9U3JijPg-z7h}G&eP`q*x!!Ey>eO&dJoz$;s6(s7y)BE6GgOE2zB1VUwGmQks)$ XSHuc51!PMxi1Cq`k&&^88OQ The user has provided input + output_messages.append(f'=== UserPromptNode: {node.user_prompt} ===') + print(node.user_prompt) + print("\n") + elif Agent.is_model_request_node(node): + # A model request node => We can stream tokens from the model's request + output_messages.append( + '=== ModelRequestNode: streaming partial request tokens ===' + ) + async with node.stream(run.ctx) as request_stream: + async for event in request_stream: + if isinstance(event, PartStartEvent): + output_messages.append( + f'[Request] Starting part {event.index}: {event.part!r}' + ) + print(event.part.content) + + elif isinstance(event, PartDeltaEvent): + if isinstance(event.delta, TextPartDelta): + output_messages.append( + f'[Request] Part {event.index} text delta: {event.delta.content_delta!r}' + ) + print(event.delta.content_delta) + elif isinstance(event.delta, ToolCallPartDelta): + output_messages.append( + f'[Request] Part {event.index} args_delta={event.delta.args_delta}' + ) + + elif isinstance(event, FinalResultEvent): + output_messages.append( + f'[Result] The model produced a final result (tool_name={event.tool_name})' + ) + + elif Agent.is_call_tools_node(node): + # A handle-response node => The model returned some data, potentially calls a tool + output_messages.append( + '=== CallToolsNode: streaming partial response & tool usage ===' + ) + async with node.stream(run.ctx) as handle_stream: + async for event in handle_stream: + if isinstance(event, FunctionToolCallEvent): + output_messages.append( + f'[Tools] The LLM calls tool={event.part.tool_name!r} with args={event.part.args} (tool_call_id={event.part.tool_call_id!r})' + ) + elif isinstance(event, FunctionToolResultEvent): + output_messages.append( + f'[Tools] Tool call {event.tool_call_id!r} returned => {event.result.content}' + ) + elif Agent.is_end_node(node): + assert run.result.data == node.data.data + # Once an End node is reached, the agent run is complete + output_messages.append(f'=== Final Agent Output: {run.result.data} ===') + + +if __name__ == "__main__": + asyncio.run(main()) + # print(output_messages) \ No newline at end of file From d9e4705a253c582b09ebcbb418fe908c0a216ed8 Mon Sep 17 00:00:00 2001 From: Pari Work Temp Date: Fri, 21 Mar 2025 13:39:27 -0700 Subject: [PATCH 02/26] updated ollama code --- .gitignore | 5 ++++- llm/__pycache__/pydantic.cpython-312.pyc | Bin 150 -> 0 bytes llm/__pycache__/pydantic.cpython-313.pyc | Bin 150 -> 0 bytes 3 files changed, 4 insertions(+), 1 deletion(-) delete mode 100644 llm/__pycache__/pydantic.cpython-312.pyc delete mode 100644 llm/__pycache__/pydantic.cpython-313.pyc diff --git a/.gitignore b/.gitignore index 2ec2572..9bb3b4f 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,4 @@ -.config \ No newline at end of file +.config +*/.ruff_cache/* +*/.venv/* +*/__pycache__/* \ No newline at end of file diff --git a/llm/__pycache__/pydantic.cpython-312.pyc b/llm/__pycache__/pydantic.cpython-312.pyc deleted file mode 100644 index 0d18d5251123a5b580fb7c2a4399ec2c314dd56f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 150 zcmX@j%ge<81oQUYO=kqsk3k%C@RXR6n#hwWwIX zAh9U3JijPg-z7h}G&eP`q*x!!Ey>eO&dJoz$;s6(s7y)BE6GgOE2zB1VUwGmQks)$ XSHuc51!PMxi1Cq`k&&^88OQ Date: Fri, 21 Mar 2025 13:39:40 -0700 Subject: [PATCH 03/26] updated ollama code --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f85740d..cf2d5b1 100644 --- a/README.md +++ b/README.md @@ -144,7 +144,7 @@ Note: For AWS helpers, the AWS CLI is required for these commands to work. Use `tn update-config` to set a basic config the default for the local ollama is http://localhost:11434. You should be setting ``` -OLLAMA_API_URL="https://pari-ollama.ngrok.io" +OLLAMA_API_URL="http://localhost:11434" OLLAMA_MODEL="llama3.3" ``` From 6e85206d9f48ea0c2323c8f87775db29eb7c68e6 Mon Sep 17 00:00:00 2001 From: Pari Work Temp Date: Sat, 22 Mar 2025 11:37:53 -0700 Subject: [PATCH 04/26] remove excess line endings --- llm/ollama_client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/llm/ollama_client.py b/llm/ollama_client.py index 7e8d227..0b9d8ec 100644 --- a/llm/ollama_client.py +++ b/llm/ollama_client.py @@ -80,14 +80,14 @@ async def main(): output_messages.append( f'[Request] Starting part {event.index}: {event.part!r}' ) - print(event.part.content) + print(event.part.content, end='', flush=True) elif isinstance(event, PartDeltaEvent): if isinstance(event.delta, TextPartDelta): output_messages.append( f'[Request] Part {event.index} text delta: {event.delta.content_delta!r}' ) - print(event.delta.content_delta) + print(event.delta.content_delta, end='', flush=True) elif isinstance(event.delta, ToolCallPartDelta): output_messages.append( f'[Request] Part {event.index} args_delta={event.delta.args_delta}' From a0d281aa957b6bf77092792a3d3aa1d1c218b4b3 Mon Sep 17 00:00:00 2001 From: Pari Work Temp Date: Mon, 20 Jul 2026 20:13:36 -0500 Subject: [PATCH 05/26] Add spec: aws-terraform-recipes Declare 3 assertions for extracting generic AWS infrastructure scripts from tn-spa-bootstrapper into tn-cli justfile recipes: VPC recipe (aws-setup-vpc), 6 script recipes (ecs-exec, stream-logs, tf-setup-backend, tf-init-backend, setup-oidc, setup-secrets), and recipe group listing under [aws-terraform]. Cross-repo dependency from bootstrapper's aws-fargate-infra-refactor spec. Co-Authored-By: Claude Opus 4.6 --- .../assertions/cli-recipe-group-listing.md | 28 ++++++++++++++ .../assertions/cli-script-recipes.md | 36 ++++++++++++++++++ .../assertions/cli-vpc-recipe.md | 30 +++++++++++++++ .../aws-terraform-recipes.md | 38 +++++++++++++++++++ 4 files changed, 132 insertions(+) create mode 100644 specs/aws-terraform-recipes/assertions/cli-recipe-group-listing.md create mode 100644 specs/aws-terraform-recipes/assertions/cli-script-recipes.md create mode 100644 specs/aws-terraform-recipes/assertions/cli-vpc-recipe.md create mode 100644 specs/aws-terraform-recipes/aws-terraform-recipes.md diff --git a/specs/aws-terraform-recipes/assertions/cli-recipe-group-listing.md b/specs/aws-terraform-recipes/assertions/cli-recipe-group-listing.md new file mode 100644 index 0000000..01369f9 --- /dev/null +++ b/specs/aws-terraform-recipes/assertions/cli-recipe-group-listing.md @@ -0,0 +1,28 @@ +--- +id: cli-recipe-group-listing +parent: aws-terraform-recipes +created: 2026-07-20T23:00:00Z +priority: 1 +status: not_started +depends-on: cli-script-recipes +--- + +# Recipe Group: All 7 Recipes Appear Under `[aws-terraform]` in `tn --list` + +## What Must Be True + +All seven AWS Terraform recipes (the VPC recipe plus the six extracted script recipes) are grouped under an `[aws-terraform]` section header in the justfile, and appear as a cohesive group in `tn --list` output. + +## Success Criteria + +- `tn --list` output includes an `[aws-terraform]` group header +- The following 7 recipes appear under that group: + - `aws-setup-vpc` + - `aws-ecs-exec` + - `aws-stream-logs` + - `aws-tf-setup-backend` + - `aws-tf-init-backend` + - `aws-setup-oidc` + - `aws-setup-secrets` +- No AWS Terraform recipes appear outside the `[aws-terraform]` group +- Group ordering is logical: VPC first, then backend setup, then OIDC/secrets, then operational tools (ecs-exec, stream-logs) diff --git a/specs/aws-terraform-recipes/assertions/cli-script-recipes.md b/specs/aws-terraform-recipes/assertions/cli-script-recipes.md new file mode 100644 index 0000000..5c4a791 --- /dev/null +++ b/specs/aws-terraform-recipes/assertions/cli-script-recipes.md @@ -0,0 +1,36 @@ +--- +id: cli-script-recipes +parent: aws-terraform-recipes +created: 2026-07-20T23:00:00Z +priority: 1 +status: not_started +--- + +# Script Recipes: Six Extracted Operations Exist as tn-cli Recipes + +## What Must Be True + +The `tn-cli` justfile contains six recipes extracted from the bootstrapper template's scripts. Each recipe accepts parameters instead of relying on cookiecutter template variables, and contains no jinja template syntax. + +## Recipes + +| Recipe Name | Extracted From | +|---|---| +| `aws-ecs-exec` | `terraform/scripts/ecs-exec.sh` | +| `aws-stream-logs` | `terraform/scripts/stream-logs.sh` | +| `aws-tf-setup-backend` | `terraform/scripts/setup_backend.sh` | +| `aws-tf-init-backend` | `terraform/scripts/init_backend.sh` | +| `aws-setup-oidc` | `terraform/scripts/setup-github-oidc-role.sh` | +| `aws-setup-secrets` | `.github/scripts/setup-secrets-bucket.sh` | + +## Success Criteria + +- All 6 recipes exist in the justfile +- No `{% raw %}` / `{% endraw %}` jinja guards remain in any recipe code +- No cookiecutter template variables (`{{cookiecutter.*}}`) appear in any recipe +- Each recipe accepts project-specific values as parameters (e.g., project name, environment, region, profile) rather than hardcoded or template-derived values +- Each recipe is idempotent (safe to run multiple times without side effects) + +## Cross-Repo Dependency + +The bootstrapper repo's `template-extracted-scripts-removed` assertion depends on these recipes existing before the corresponding scripts are removed from the template. diff --git a/specs/aws-terraform-recipes/assertions/cli-vpc-recipe.md b/specs/aws-terraform-recipes/assertions/cli-vpc-recipe.md new file mode 100644 index 0000000..6e7afa2 --- /dev/null +++ b/specs/aws-terraform-recipes/assertions/cli-vpc-recipe.md @@ -0,0 +1,30 @@ +--- +id: cli-vpc-recipe +parent: aws-terraform-recipes +created: 2026-07-20T23:00:00Z +priority: 1 +status: not_started +--- + +# VPC Recipe: `aws-setup-vpc` Idempotently Creates a Tagged Shared VPC + +## What Must Be True + +The `tn-cli` justfile contains an `aws-setup-vpc` recipe that creates (or confirms existence of) a shared VPC with all required networking resources, tagged with a naming convention that Terraform data sources can filter on deterministically. + +## Success Criteria + +- Recipe exists in the justfile with parameters for `vpc_name` (default: `tn-shared`), `environment` (default: `dev`), `profile` (default: `default`), and `region` (default: `us-east-1`) +- Running the recipe twice produces no changes on the second run (idempotent) +- Created resources and their tags: + - VPC: Name tag `{vpc_name}-{environment}` (e.g., `tn-shared-dev`) + - Internet Gateway: Name tag `{vpc_name}-{environment}-igw` + - Route Table: Name tag `{vpc_name}-{environment}-rt` + - Subnets: at least 2 across different AZs, Name tags include `{vpc_name}-{environment}` +- No `{% raw %}` / `{% endraw %}` jinja guards in the recipe code +- No cookiecutter template variables (`{{cookiecutter.*}}`) in the recipe code +- Tag naming convention matches the filters used by `data "aws_vpc"`, `data "aws_internet_gateway"`, and `data "aws_route_table"` data sources in the bootstrapper's Terraform + +## Cross-Repo Dependency + +The bootstrapper repo's `tf-mandatory-vpc-data-sources` assertion depends on this recipe's tag naming convention. The data source filters in `main.tf` must match the tags this recipe creates. diff --git a/specs/aws-terraform-recipes/aws-terraform-recipes.md b/specs/aws-terraform-recipes/aws-terraform-recipes.md new file mode 100644 index 0000000..e24bc4a --- /dev/null +++ b/specs/aws-terraform-recipes/aws-terraform-recipes.md @@ -0,0 +1,38 @@ +--- +id: aws-terraform-recipes +created: 2026-07-20T23:00:00Z +priority: 1 +--- + +# AWS Terraform Recipes + +## Problem + +The tn-spa-bootstrapper template bundles one-time setup scripts (VPC creation, OIDC setup, backend initialization, secrets bucket, etc.) inside each generated project. These scripts contain no project-specific logic but are trapped inside the cookiecutter template, wrapped in `{% raw %}` / `{% endraw %}` jinja guards, and duplicated across every project that uses the template. + +Worse, the VPC creation logic lives inside per-project Terraform state, causing a race condition: multiple projects sharing a VPC can collide during concurrent `terraform apply` runs because each project believes it owns the VPC lifecycle. + +## Solution + +Extract all generic AWS infrastructure scripts into `tn-cli` as reusable justfile recipes under an `[aws-terraform]` group. Each recipe accepts parameters (project name, environment, region, profile) instead of relying on cookiecutter template variables. + +The key architectural change: VPC creation becomes a one-time `tn aws-setup-vpc` command run before any project deploys, rather than a conditional resource inside each project's Terraform. This eliminates the shared-state race condition entirely. + +## Recipes + +Seven recipes total: + +1. **`aws-setup-vpc`** -- Idempotent shared VPC creation with tagged resources +2. **`aws-ecs-exec`** -- Interactive ECS container exec session +3. **`aws-stream-logs`** -- CloudWatch log streaming +4. **`aws-tf-setup-backend`** -- S3 + DynamoDB backend creation +5. **`aws-tf-init-backend`** -- Terraform backend initialization +6. **`aws-setup-oidc`** -- GitHub OIDC role provisioning +7. **`aws-setup-secrets`** -- S3 secrets bucket creation + +## Constraints + +- Recipes must be pure shell -- no cookiecutter, no jinja +- All recipes must be idempotent (safe to run multiple times) +- Tag naming conventions must match the Terraform data source filters in the bootstrapper template +- Recipes appear under an `[aws-terraform]` group in `tn --list` output From 82e53c95fc66f8689dbc334b7bd0f0b43599c9b2 Mon Sep 17 00:00:00 2001 From: Pari Work Temp Date: Mon, 20 Jul 2026 20:25:37 -0500 Subject: [PATCH 06/26] Complete cli-script-recipes: add 6 AWS Terraform recipes to justfile Extract 6 scripts from tn-spa-bootstrapper template into reusable justfile recipes under [aws-terraform] group. All recipes accept parameters instead of cookiecutter template variables and contain no jinja syntax. Recipes added: - aws-ecs-exec: Interactive ECS container exec session - aws-stream-logs: CloudWatch log streaming - aws-tf-setup-backend: S3 + DynamoDB backend creation - aws-tf-init-backend: Terraform backend initialization - aws-setup-oidc: GitHub OIDC role provisioning - aws-setup-secrets: S3 secrets bucket creation Co-Authored-By: Claude Opus 4.6 --- justfile | 765 ++++++++++++++++++ .../assertions/cli-script-recipes.md | 3 +- 2 files changed, 767 insertions(+), 1 deletion(-) diff --git a/justfile b/justfile index 0632aab..919d4be 100644 --- a/justfile +++ b/justfile @@ -135,6 +135,771 @@ aws-make-s3-bucket project_name profile='default' region='us-east-1': aws-enable-bedrock project_name profile='default' region='us-east-1' model='*': aws cloudformation create-stack --stack-name {{project_name}}-bedrock-stack --template-url 'https://tn-s3-cloud-formation.s3.amazonaws.com/bedrock-user-permissions.yaml' --region {{region}} --parameters ParameterKey=ProjectName,ParameterValue={{project_name}} ParameterKey=AllowedModels,ParameterValue={{model}} --capabilities CAPABILITY_NAMED_IAM --profile={{profile}} +# +# AWS Terraform Recipes +# +# Extracted from tn-spa-bootstrapper template scripts. +# Each recipe accepts parameters instead of relying on cookiecutter template variables. +# + +# Connect to a running ECS task via ECS Exec +[group('aws-terraform')] +aws-ecs-exec service environment='development' profile='default' region='us-east-1' command='bash': + #!/usr/bin/env bash + set -e + + SERVICE="{{service}}" + ENVIRONMENT="{{environment}}" + AWS_PROFILE="{{profile}}" + AWS_REGION="{{region}}" + COMMAND_CHOICE="{{command}}" + + PROFILE_FLAG="" + if [[ "$AWS_PROFILE" != "default" ]]; then + PROFILE_FLAG="--profile $AWS_PROFILE" + fi + REGION_FLAG="--region $AWS_REGION" + + CLUSTER_NAME="cluster-${SERVICE}-${ENVIRONMENT}" + + echo "ECS Exec - Connect to running tasks" + echo "=======================================" + echo " Service: $SERVICE" + echo " Environment: $ENVIRONMENT" + echo " Cluster: $CLUSTER_NAME" + echo " AWS Profile: $AWS_PROFILE" + echo " AWS Region: $AWS_REGION" + + # Check if cluster exists + if ! aws ecs describe-clusters --clusters "$CLUSTER_NAME" $PROFILE_FLAG $REGION_FLAG &>/dev/null; then + echo "Error: Cluster '$CLUSTER_NAME' not found" + exit 1 + fi + + # List available services + echo "" + echo "Available services:" + SERVICES=$(aws ecs list-services --cluster "$CLUSTER_NAME" $PROFILE_FLAG $REGION_FLAG --query 'serviceArns[*]' --output text) + + if [[ -z "$SERVICES" ]]; then + echo "Error: No services found in cluster '$CLUSTER_NAME'" + exit 1 + fi + + SERVICE_NAMES=() + i=1 + for service_arn in $SERVICES; do + service_name=$(basename "$service_arn") + SERVICE_NAMES+=("$service_name") + echo " $i) $service_name" + ((i++)) + done + + echo "" + read -p "Select service number (1): " SERVICE_CHOICE + SERVICE_CHOICE=${SERVICE_CHOICE:-1} + + if [[ "$SERVICE_CHOICE" -lt 1 || "$SERVICE_CHOICE" -gt ${#SERVICE_NAMES[@]} ]]; then + echo "Error: Invalid service selection" + exit 1 + fi + + SELECTED_SERVICE=${SERVICE_NAMES[$((SERVICE_CHOICE-1))]} + echo "Selected: $SELECTED_SERVICE" + + # Get running tasks + TASKS=$(aws ecs list-tasks --cluster "$CLUSTER_NAME" --service-name "$SELECTED_SERVICE" $PROFILE_FLAG $REGION_FLAG --desired-status RUNNING --query 'taskArns[*]' --output text) + + if [[ -z "$TASKS" ]]; then + echo "Error: No running tasks found for service '$SELECTED_SERVICE'" + exit 1 + fi + + TASK_ARNS=($TASKS) + if [[ ${#TASK_ARNS[@]} -gt 1 ]]; then + echo "" + echo "Multiple tasks found:" + for i in "${!TASK_ARNS[@]}"; do + task_id=$(basename "${TASK_ARNS[$i]}") + echo " $((i+1))) $task_id" + done + read -p "Select task number (1): " TASK_CHOICE + TASK_CHOICE=${TASK_CHOICE:-1} + SELECTED_TASK=${TASK_ARNS[$((TASK_CHOICE-1))]} + else + SELECTED_TASK=${TASK_ARNS[0]} + fi + + TASK_ID=$(basename "$SELECTED_TASK") + echo "Selected task: $TASK_ID" + + # Get container name + TASK_DEF=$(aws ecs describe-tasks --cluster "$CLUSTER_NAME" --tasks "$SELECTED_TASK" $PROFILE_FLAG $REGION_FLAG --query 'tasks[0].taskDefinitionArn' --output text) + CONTAINER_NAME=$(aws ecs describe-task-definition --task-definition "$TASK_DEF" $PROFILE_FLAG $REGION_FLAG --query 'taskDefinition.containerDefinitions[0].name' --output text) + echo "Container: $CONTAINER_NAME" + + # Parse command choice + case $COMMAND_CHOICE in + bash) COMMAND="/bin/bash" ;; + sh) COMMAND="/bin/sh" ;; + django) COMMAND="python manage.py shell" ;; + dbshell) COMMAND="python manage.py dbshell" ;; + *) COMMAND="$COMMAND_CHOICE" ;; + esac + + echo "" + echo "Connecting... (command: $COMMAND)" + echo "Type 'exit' to disconnect" + echo "====================" + + aws ecs execute-command \ + --cluster "$CLUSTER_NAME" \ + --task "$TASK_ID" \ + --container "$CONTAINER_NAME" \ + --interactive \ + --command "$COMMAND" \ + $PROFILE_FLAG $REGION_FLAG + +# Stream CloudWatch logs from ECS services +[group('aws-terraform')] +aws-stream-logs service environment='development' profile='default' region='us-east-1' stream_type='a' filter='' duration='5m': + #!/usr/bin/env bash + set -e + + SERVICE="{{service}}" + ENVIRONMENT="{{environment}}" + AWS_PROFILE="{{profile}}" + AWS_REGION="{{region}}" + STREAM_TYPE="{{stream_type}}" + FILTER_PATTERN="{{filter}}" + START_TIME="{{duration}}" + + PROFILE_FLAG="" + if [[ "$AWS_PROFILE" != "default" ]]; then + PROFILE_FLAG="--profile $AWS_PROFILE" + fi + REGION_FLAG="--region $AWS_REGION" + + LOG_GROUP="/ecs/${SERVICE}/${ENVIRONMENT}" + + echo "ECS Logs Streaming" + echo "=============================================" + echo " Service: $SERVICE" + echo " Environment: $ENVIRONMENT" + echo " Log Group: $LOG_GROUP" + echo " AWS Profile: $AWS_PROFILE" + echo " AWS Region: $AWS_REGION" + + # Check if log group exists + if ! aws logs describe-log-groups --log-group-name-prefix "$LOG_GROUP" $PROFILE_FLAG $REGION_FLAG --query 'logGroups[?logGroupName==`'"$LOG_GROUP"'`]' --output text | grep -q "$LOG_GROUP"; then + echo "Error: Log group '$LOG_GROUP' not found" + echo "Tip: Make sure your service is deployed and running" + exit 1 + fi + + # Get available log streams + STREAMS=$(aws logs describe-log-streams \ + --log-group-name "$LOG_GROUP" \ + --order-by LastEventTime \ + --descending \ + --max-items 20 \ + $PROFILE_FLAG $REGION_FLAG \ + --query 'logStreams[*].logStreamName' \ + --output text) + + if [[ -z "$STREAMS" ]]; then + echo "Error: No log streams found in '$LOG_GROUP'" + exit 1 + fi + + # Categorize streams + SERVER_STREAMS=() + WORKER_STREAMS=() + OTHER_STREAMS=() + + i=1 + for stream in $STREAMS; do + if [[ "$stream" =~ server- ]]; then + SERVER_STREAMS+=("$stream") + echo " $i) [SERVER] $stream" + elif [[ "$stream" =~ worker- ]]; then + WORKER_STREAMS+=("$stream") + echo " $i) [WORKER] $stream" + else + OTHER_STREAMS+=("$stream") + echo " $i) [OTHER] $stream" + fi + ((i++)) + done + + ALL_STREAMS=("${SERVER_STREAMS[@]}" "${WORKER_STREAMS[@]}" "${OTHER_STREAMS[@]}") + + SELECTED_STREAMS=() + case $STREAM_TYPE in + a|A) SELECTED_STREAMS=("${SERVER_STREAMS[@]}"); echo "Streaming all server logs" ;; + w|W) SELECTED_STREAMS=("${WORKER_STREAMS[@]}"); echo "Streaming all worker logs" ;; + '*') SELECTED_STREAMS=("${ALL_STREAMS[@]}"); echo "Streaming all logs" ;; + *) + if [[ "$STREAM_TYPE" =~ ^[0-9]+$ ]] && [[ "$STREAM_TYPE" -ge 1 ]] && [[ "$STREAM_TYPE" -le ${#ALL_STREAMS[@]} ]]; then + SELECTED_STREAMS=("${ALL_STREAMS[$((STREAM_TYPE-1))]}") + echo "Streaming: ${ALL_STREAMS[$((STREAM_TYPE-1))]}" + else + echo "Error: Invalid stream selection" + exit 1 + fi + ;; + esac + + if [[ ${#SELECTED_STREAMS[@]} -eq 0 ]]; then + echo "Error: No streams selected" + exit 1 + fi + + # Validate duration format + if [[ ! "$START_TIME" =~ ^[0-9]+[mh]$ ]]; then + echo "Error: Invalid duration format. Use '30m' or '2h'" + exit 1 + fi + + # Build filter command + FILTER_CMD="aws logs filter-log-events --log-group-name \"$LOG_GROUP\" --start-time \$(date -v-${START_TIME} +%s)000 $PROFILE_FLAG $REGION_FLAG" + + if [[ -n "$FILTER_PATTERN" ]]; then + FILTER_CMD="$FILTER_CMD --filter-pattern \"$FILTER_PATTERN\"" + fi + + if [[ ${#SELECTED_STREAMS[@]} -lt ${#ALL_STREAMS[@]} ]]; then + STREAM_NAMES=$(IFS=' '; echo "${SELECTED_STREAMS[*]}") + FILTER_CMD="$FILTER_CMD --log-stream-names $STREAM_NAMES" + fi + + echo "" + echo "Log Group: $LOG_GROUP" + echo "Streams: ${#SELECTED_STREAMS[@]} selected" + echo "Time Range: Last $START_TIME" + if [[ -n "$FILTER_PATTERN" ]]; then + echo "Filter: $FILTER_PATTERN" + fi + echo "" + echo "Press Ctrl+C to stop streaming" + echo "====================" + + # Stream logs with continuous updates + LAST_SEEN="" + while true; do + CMD="$FILTER_CMD --output json" + if [[ -n "$LAST_SEEN" ]]; then + CMD="$CMD --next-token $LAST_SEEN" + fi + + RESPONSE=$(eval "$CMD" 2>/dev/null || echo '{"events":[],"nextToken":null}') + EVENTS=$(echo "$RESPONSE" | jq -c '.events[]?' 2>/dev/null) + NEXT_TOKEN=$(echo "$RESPONSE" | jq -r '.nextToken // empty' 2>/dev/null) + + if [[ -n "$EVENTS" ]]; then + while IFS= read -r event; do + if [[ -n "$event" ]]; then + timestamp=$(echo "$event" | jq -r '.timestamp // empty') + message=$(echo "$event" | jq -r '.message // empty') + stream=$(echo "$event" | jq -r '.logStreamName // empty') + if [[ -n "$timestamp" && -n "$message" ]]; then + formatted_time=$(date -r "$((timestamp/1000))" '+%Y-%m-%d %H:%M:%S' 2>/dev/null || echo "$timestamp") + stream_short=$(basename "$stream") + echo "[$formatted_time] [$stream_short] $message" + fi + fi + done <<< "$EVENTS" + fi + + if [[ -n "$NEXT_TOKEN" && "$NEXT_TOKEN" != "null" ]]; then + LAST_SEEN="$NEXT_TOKEN" + fi + + sleep 2 + done + +# Create S3 bucket and DynamoDB table for Terraform remote state backend +[group('aws-terraform')] +aws-tf-setup-backend service profile='default': + #!/usr/bin/env bash + set -e + + SERVICE="{{service}}" + AWS_PROFILE="{{profile}}" + + PROFILE_FLAG="" + if [[ "$AWS_PROFILE" != "default" ]]; then + PROFILE_FLAG="--profile $AWS_PROFILE" + fi + + # Check AWS CLI + if ! command -v aws &> /dev/null; then + echo "Error: AWS CLI not found. Please install AWS CLI first." + exit 1 + fi + + if ! aws sts get-caller-identity $PROFILE_FLAG &> /dev/null; then + echo "Error: AWS CLI not configured for profile '$AWS_PROFILE'. Run 'aws configure --profile $AWS_PROFILE' first." + exit 1 + fi + + echo "AWS CLI configured for profile: $AWS_PROFILE" + + # Get AWS account info + AWS_ACCOUNT_ID=$(aws sts get-caller-identity $PROFILE_FLAG --query Account --output text) + AWS_REGION=$(aws configure get region $PROFILE_FLAG 2>/dev/null || echo "us-east-1") + + echo "" + echo "Terraform S3 Backend Setup" + echo "==========================" + echo " Service: $SERVICE" + echo " Profile: $AWS_PROFILE" + echo " Account ID: $AWS_ACCOUNT_ID" + echo " Region: $AWS_REGION" + + # Generate standard names + BUCKET_NAME="${AWS_ACCOUNT_ID}-${SERVICE}-terraform-state" + TABLE_NAME="${SERVICE}-terraform-state-lock" + + echo "" + echo "Resources to create:" + echo " S3 Bucket: $BUCKET_NAME" + echo " DynamoDB Table: $TABLE_NAME" + echo "" + + echo -n "Proceed? (y/N): " + read confirm + if [[ ! "$confirm" =~ ^[Yy]$ ]]; then + echo "Cancelled" + exit 0 + fi + + # Create S3 bucket (idempotent) + echo "" + echo "Creating S3 bucket: $BUCKET_NAME" + if aws s3api head-bucket --bucket "$BUCKET_NAME" --region "$AWS_REGION" $PROFILE_FLAG 2>/dev/null; then + echo "S3 bucket '$BUCKET_NAME' already exists" + else + if [[ "$AWS_REGION" == "us-east-1" ]]; then + aws s3api create-bucket --bucket "$BUCKET_NAME" --region "$AWS_REGION" $PROFILE_FLAG + else + aws s3api create-bucket --bucket "$BUCKET_NAME" --region "$AWS_REGION" \ + --create-bucket-configuration LocationConstraint="$AWS_REGION" $PROFILE_FLAG + fi + + aws s3api put-bucket-versioning --bucket "$BUCKET_NAME" \ + --versioning-configuration Status=Enabled $PROFILE_FLAG + + aws s3api put-bucket-encryption --bucket "$BUCKET_NAME" \ + --server-side-encryption-configuration '{ + "Rules": [{"ApplyServerSideEncryptionByDefault":{"SSEAlgorithm":"AES256"},"BucketKeyEnabled":true}] + }' $PROFILE_FLAG + + aws s3api put-public-access-block --bucket "$BUCKET_NAME" \ + --public-access-block-configuration \ + BlockPublicAcls=true,IgnorePublicAcls=true,BlockPublicPolicy=true,RestrictPublicBuckets=true \ + $PROFILE_FLAG + + echo "S3 bucket created with versioning, encryption, and public access blocked" + fi + + # Create DynamoDB table (idempotent) + echo "" + echo "Creating DynamoDB table: $TABLE_NAME" + if aws dynamodb describe-table --table-name "$TABLE_NAME" --region "$AWS_REGION" $PROFILE_FLAG 2>/dev/null; then + echo "DynamoDB table '$TABLE_NAME' already exists" + else + aws dynamodb create-table \ + --table-name "$TABLE_NAME" \ + --attribute-definitions AttributeName=LockID,AttributeType=S \ + --key-schema AttributeName=LockID,KeyType=HASH \ + --billing-mode PAY_PER_REQUEST \ + --region "$AWS_REGION" $PROFILE_FLAG + + echo "Waiting for DynamoDB table to be active..." + aws dynamodb wait table-exists --table-name "$TABLE_NAME" --region "$AWS_REGION" $PROFILE_FLAG + echo "DynamoDB table created" + fi + + echo "" + echo "Backend setup complete!" + echo " S3 Bucket: $BUCKET_NAME" + echo " DynamoDB Table: $TABLE_NAME" + echo " Region: $AWS_REGION" + echo "" + echo "Next: run 'tn aws-tf-init-backend $SERVICE ' to initialize Terraform" + +# Initialize Terraform with the correct backend configuration for an environment +[group('aws-terraform')] +aws-tf-init-backend service environment='development' profile='default' region='us-east-1' force='false': + #!/usr/bin/env bash + set -e + + SERVICE="{{service}}" + ENVIRONMENT="{{environment}}" + AWS_PROFILE="{{profile}}" + AWS_REGION="{{region}}" + FORCE="{{force}}" + + PROFILE_FLAG="" + if [[ "$AWS_PROFILE" != "default" ]]; then + PROFILE_FLAG="--profile $AWS_PROFILE" + fi + + # Get AWS account ID + AWS_ACCOUNT_ID=$(aws sts get-caller-identity $PROFILE_FLAG --query Account --output text) + + # Derive backend resource names (must match setup-backend convention) + BUCKET="${AWS_ACCOUNT_ID}-${SERVICE}-terraform-state" + TABLE="${SERVICE}-terraform-state-lock" + STATE_KEY="${ENVIRONMENT}/terraform.tfstate" + + echo "Terraform Backend Initialization" + echo "==================================" + echo " Service: $SERVICE" + echo " Environment: $ENVIRONMENT" + echo " AWS Profile: $AWS_PROFILE" + echo " State Key: $STATE_KEY" + echo " S3 Bucket: $BUCKET" + echo " DynamoDB Table: $TABLE" + echo " Region: $AWS_REGION" + echo "" + + # Test AWS access + echo "Testing AWS access..." + AWS_IDENTITY=$(aws sts get-caller-identity $PROFILE_FLAG 2>/dev/null || echo "FAILED") + if [[ "$AWS_IDENTITY" == "FAILED" ]]; then + echo "Error: Failed to get AWS caller identity" + exit 1 + fi + echo "AWS Identity verified: $(echo "$AWS_IDENTITY" | jq -r '.Arn')" + + # Test DynamoDB table access + echo "Testing DynamoDB table access..." + if aws dynamodb describe-table --table-name "$TABLE" --region "$AWS_REGION" $PROFILE_FLAG &>/dev/null; then + echo "DynamoDB table accessible: $TABLE" + else + echo "Error: DynamoDB table not accessible: $TABLE" + echo "Tip: Run 'tn aws-tf-setup-backend $SERVICE' first" + exit 1 + fi + + # Build backend config args + BACKEND_ARGS="-backend-config=\"bucket=${BUCKET}\"" + BACKEND_ARGS+=" -backend-config=\"key=${STATE_KEY}\"" + BACKEND_ARGS+=" -backend-config=\"region=${AWS_REGION}\"" + BACKEND_ARGS+=" -backend-config=\"dynamodb_table=${TABLE}\"" + BACKEND_ARGS+=" -backend-config=\"encrypt=true\"" + + if [[ "$AWS_PROFILE" != "default" ]]; then + BACKEND_ARGS+=" -backend-config=\"profile=${AWS_PROFILE}\"" + fi + + if [[ "$FORCE" == "true" ]]; then + BACKEND_ARGS+=" -migrate-state" + fi + + echo "" + echo "Running terraform init..." + eval "terraform init ${BACKEND_ARGS}" + + echo "" + echo "Backend initialized!" + echo " State Location: s3://${BUCKET}/${STATE_KEY}" + echo " Lock Table: ${TABLE}" + echo " Environment: ${ENVIRONMENT}" + +# Create GitHub Actions OIDC IAM role for a given org and environment +[group('aws-terraform')] +aws-setup-oidc github_org environment='development' secrets_bucket='' profile='default': + #!/usr/bin/env bash + set -e + + GITHUB_ORG="{{github_org}}" + ENVIRONMENT="{{environment}}" + SECRETS_BUCKET="{{secrets_bucket}}" + AWS_PROFILE="{{profile}}" + + if [[ -z "$GITHUB_ORG" ]]; then + echo "Error: github_org is required" + exit 1 + fi + + # Set up AWS command helper + run_aws() { + if [[ "$AWS_PROFILE" != "default" && -n "$AWS_PROFILE" ]]; then + aws --profile "$AWS_PROFILE" "$@" + else + aws "$@" + fi + } + + ACCOUNT_ID=$(run_aws sts get-caller-identity --query Account --output text) + ROLE_NAME="github-actions-${ENVIRONMENT}" + + echo "GitHub Actions OIDC Setup" + echo "=========================" + echo " GitHub Org: $GITHUB_ORG" + echo " Environment: $ENVIRONMENT" + echo " AWS Account: $ACCOUNT_ID" + echo " Role Name: $ROLE_NAME" + echo " AWS Profile: $AWS_PROFILE" + if [[ -n "$SECRETS_BUCKET" ]]; then + echo " Secrets Bucket: $SECRETS_BUCKET" + fi + echo "" + + # 1. Create OIDC Identity Provider (idempotent) + echo "Creating OIDC Identity Provider..." + if run_aws iam get-open-id-connect-provider \ + --open-id-connect-provider-arn "arn:aws:iam::${ACCOUNT_ID}:oidc-provider/token.actions.githubusercontent.com" &>/dev/null; then + echo "OIDC provider already exists" + else + run_aws iam create-open-id-connect-provider \ + --url https://token.actions.githubusercontent.com \ + --client-id-list sts.amazonaws.com \ + --thumbprint-list 1c58a3a8518e8759bf075b76b750d4f2df264fcd + echo "OIDC provider created" + fi + + # 2. Create or update IAM Role (idempotent) + echo "" + echo "Processing IAM Role: $ROLE_NAME" + if run_aws iam get-role --role-name "$ROLE_NAME" &>/dev/null; then + echo "Role $ROLE_NAME already exists" + # Check if GitHub org already in trust policy + TRUST_POLICY=$(run_aws iam get-role --role-name "$ROLE_NAME" --query 'Role.AssumeRolePolicyDocument' --output json) + if echo "$TRUST_POLICY" | grep -q "repo:${GITHUB_ORG}/"; then + echo "GitHub org '$GITHUB_ORG' already has access" + else + echo "Adding GitHub org '$GITHUB_ORG' to trust policy..." + echo "$TRUST_POLICY" | jq --arg org "$GITHUB_ORG" ' + (.Statement[0].Condition.StringLike["token.actions.githubusercontent.com:sub"]) |= + if type == "string" then [., "repo:\($org)/*:*"] + elif type == "array" then . + ["repo:\($org)/*:*"] + else "repo:\($org)/*:*" + end + ' > /tmp/oidc-trust-policy.json + run_aws iam update-assume-role-policy --role-name "$ROLE_NAME" \ + --policy-document file:///tmp/oidc-trust-policy.json + rm -f /tmp/oidc-trust-policy.json + echo "Trust policy updated" + fi + else + jq -n --arg account "$ACCOUNT_ID" --arg org "$GITHUB_ORG" '{ + Version: "2012-10-17", + Statement: [{ + Effect: "Allow", + Principal: { Federated: "arn:aws:iam::\($account):oidc-provider/token.actions.githubusercontent.com" }, + Action: "sts:AssumeRoleWithWebIdentity", + Condition: { + StringEquals: { "token.actions.githubusercontent.com:aud": "sts.amazonaws.com" }, + StringLike: { "token.actions.githubusercontent.com:sub": "repo:\($org)/*:*" } + } + }] + }' > /tmp/oidc-trust-policy.json + run_aws iam create-role --role-name "$ROLE_NAME" \ + --assume-role-policy-document file:///tmp/oidc-trust-policy.json + rm -f /tmp/oidc-trust-policy.json + echo "IAM role created: $ROLE_NAME" + fi + + # 3. Create and attach deployment policy (idempotent) + echo "" + echo "Creating deployment policy..." + POLICY_NAME="${ROLE_NAME}-deployment-policy" + POLICY_ARN="arn:aws:iam::${ACCOUNT_ID}:policy/${POLICY_NAME}" + + jq -n '{ + Version: "2012-10-17", + Statement: [ + {Sid: "ECRFullAccess", Effect: "Allow", Action: ["ecr:*"], Resource: "*"}, + {Sid: "ECSFullAccess", Effect: "Allow", Action: ["ecs:*"], Resource: "*"}, + {Sid: "VPCAccess", Effect: "Allow", Action: ["ec2:Describe*","ec2:CreateVpc","ec2:DeleteVpc","ec2:ModifyVpcAttribute","ec2:CreateSubnet","ec2:DeleteSubnet","ec2:ModifySubnetAttribute","ec2:CreateInternetGateway","ec2:DeleteInternetGateway","ec2:AttachInternetGateway","ec2:DetachInternetGateway","ec2:CreateRouteTable","ec2:DeleteRouteTable","ec2:CreateRoute","ec2:DeleteRoute","ec2:AssociateRouteTable","ec2:DisassociateRouteTable","ec2:CreateSecurityGroup","ec2:DeleteSecurityGroup","ec2:AuthorizeSecurityGroupIngress","ec2:AuthorizeSecurityGroupEgress","ec2:RevokeSecurityGroupIngress","ec2:RevokeSecurityGroupEgress","ec2:CreateTags","ec2:DeleteTags"], Resource: "*"}, + {Sid: "RDSAccess", Effect: "Allow", Action: ["rds:*"], Resource: "*"}, + {Sid: "ACMAccess", Effect: "Allow", Action: ["acm:*"], Resource: "*"}, + {Sid: "IAMAccess", Effect: "Allow", Action: ["iam:*"], Resource: "*"}, + {Sid: "SecretsManagerAccess", Effect: "Allow", Action: ["secretsmanager:*"], Resource: "*"}, + {Sid: "CloudWatchLogsAccess", Effect: "Allow", Action: ["logs:*"], Resource: "*"}, + {Sid: "S3FullAccess", Effect: "Allow", Action: ["s3:*"], Resource: "*"}, + {Sid: "DynamoDBAccess", Effect: "Allow", Action: ["dynamodb:*"], Resource: "*"}, + {Sid: "ELBAccess", Effect: "Allow", Action: ["elasticloadbalancing:*"], Resource: "*"}, + {Sid: "ElastiCacheAccess", Effect: "Allow", Action: ["elasticache:*"], Resource: "*"}, + {Sid: "Route53Access", Effect: "Allow", Action: ["route53:*"], Resource: "*"}, + {Sid: "EventBridgeAccess", Effect: "Allow", Action: ["events:*"], Resource: "*"}, + {Sid: "STSAccess", Effect: "Allow", Action: ["sts:GetCallerIdentity"], Resource: "*"} + ] + }' > /tmp/oidc-deploy-policy.json + + # Delete existing policy if it exists (idempotent) + if run_aws iam get-policy --policy-arn "$POLICY_ARN" &>/dev/null; then + run_aws iam detach-role-policy --role-name "$ROLE_NAME" --policy-arn "$POLICY_ARN" 2>/dev/null || true + run_aws iam list-policy-versions --policy-arn "$POLICY_ARN" \ + --query 'Versions[?!IsDefaultVersion].[VersionId]' --output text | while read version; do + run_aws iam delete-policy-version --policy-arn "$POLICY_ARN" --version-id "$version" 2>/dev/null || true + done + run_aws iam delete-policy --policy-arn "$POLICY_ARN" 2>/dev/null || true + fi + + run_aws iam create-policy --policy-name "$POLICY_NAME" \ + --policy-document file:///tmp/oidc-deploy-policy.json + run_aws iam attach-role-policy --role-name "$ROLE_NAME" --policy-arn "$POLICY_ARN" + rm -f /tmp/oidc-deploy-policy.json + echo "Deployment policy attached" + + # 4. Create secrets policy if secrets_bucket provided (idempotent) + if [[ -n "$SECRETS_BUCKET" ]]; then + echo "" + echo "Creating S3 secrets policy..." + SECRETS_POLICY_NAME="${ROLE_NAME}-secrets-access" + SECRETS_POLICY_ARN="arn:aws:iam::${ACCOUNT_ID}:policy/${SECRETS_POLICY_NAME}" + + jq -n --arg bucket "$SECRETS_BUCKET" --arg env "$ENVIRONMENT" '{ + Version: "2012-10-17", + Statement: [ + {Sid: "SecretsS3Access", Effect: "Allow", Action: ["s3:GetObject","s3:PutObject","s3:DeleteObject","s3:GetObjectVersion"], Resource: ["arn:aws:s3:::\($bucket)/\($env)/*"]}, + {Sid: "AllowListBucketForEnv", Effect: "Allow", Action: "s3:ListBucket", Resource: "arn:aws:s3:::\($bucket)", Condition: {StringLike: {"s3:prefix": "\($env)/*"}}}, + {Sid: "AllowListBuckets", Effect: "Allow", Action: "s3:ListAllMyBuckets", Resource: "*"} + ] + }' > /tmp/oidc-secrets-policy.json + + if run_aws iam get-policy --policy-arn "$SECRETS_POLICY_ARN" &>/dev/null; then + run_aws iam detach-role-policy --role-name "$ROLE_NAME" --policy-arn "$SECRETS_POLICY_ARN" 2>/dev/null || true + run_aws iam list-policy-versions --policy-arn "$SECRETS_POLICY_ARN" \ + --query 'Versions[?!IsDefaultVersion].[VersionId]' --output text | while read version; do + run_aws iam delete-policy-version --policy-arn "$SECRETS_POLICY_ARN" --version-id "$version" 2>/dev/null || true + done + run_aws iam delete-policy --policy-arn "$SECRETS_POLICY_ARN" 2>/dev/null || true + fi + + run_aws iam create-policy --policy-name "$SECRETS_POLICY_NAME" \ + --policy-document file:///tmp/oidc-secrets-policy.json + run_aws iam attach-role-policy --role-name "$ROLE_NAME" --policy-arn "$SECRETS_POLICY_ARN" + rm -f /tmp/oidc-secrets-policy.json + echo "Secrets policy attached" + fi + + # Summary + ROLE_ARN=$(run_aws iam get-role --role-name "$ROLE_NAME" --query Role.Arn --output text) + echo "" + echo "Setup complete!" + echo " Role ARN: $ROLE_ARN" + echo "" + echo "Next steps:" + echo " 1. In GitHub repo Settings > Secrets and variables > Actions > Variables" + echo " 2. Add: $(echo $ENVIRONMENT | tr '[:lower:]' '[:upper:]')_AWS_ROLE_ARN = $ROLE_ARN" + +# Create S3 bucket for secrets storage with proper security +[group('aws-terraform')] +aws-setup-secrets service environment profile='default' region='us-east-1': + #!/usr/bin/env bash + set -e + + SERVICE="{{service}}" + ENVIRONMENT="{{environment}}" + AWS_PROFILE="{{profile}}" + AWS_REGION="{{region}}" + + PROFILE_FLAG="" + if [[ "$AWS_PROFILE" != "default" ]]; then + PROFILE_FLAG="--profile $AWS_PROFILE" + fi + + AWS_ACCOUNT_ID=$(aws sts get-caller-identity $PROFILE_FLAG --query Account --output text) + SECRETS_BUCKET="${SERVICE}-terraform-secrets" + + echo "S3 Secrets Bucket Setup" + echo "========================" + echo " Service: $SERVICE" + echo " Environment: $ENVIRONMENT" + echo " Account ID: $AWS_ACCOUNT_ID" + echo " Region: $AWS_REGION" + echo " Bucket: $SECRETS_BUCKET" + echo "" + + # Create bucket (idempotent) + if aws s3api head-bucket --bucket "$SECRETS_BUCKET" $PROFILE_FLAG 2>/dev/null; then + echo "Bucket '$SECRETS_BUCKET' already exists" + else + echo "Creating S3 bucket: $SECRETS_BUCKET" + if [[ "$AWS_REGION" == "us-east-1" ]]; then + aws s3api create-bucket --bucket "$SECRETS_BUCKET" $PROFILE_FLAG + else + aws s3api create-bucket --bucket "$SECRETS_BUCKET" --region "$AWS_REGION" \ + --create-bucket-configuration LocationConstraint="$AWS_REGION" $PROFILE_FLAG + fi + + # Enable versioning + aws s3api put-bucket-versioning --bucket "$SECRETS_BUCKET" \ + --versioning-configuration Status=Enabled $PROFILE_FLAG + + # Enable encryption + aws s3api put-bucket-encryption --bucket "$SECRETS_BUCKET" \ + --server-side-encryption-configuration '{ + "Rules": [{"ApplyServerSideEncryptionByDefault":{"SSEAlgorithm":"AES256"},"BucketKeyEnabled":true}] + }' $PROFILE_FLAG + + # Block public access + aws s3api put-public-access-block --bucket "$SECRETS_BUCKET" \ + --public-access-block-configuration \ + BlockPublicAcls=true,IgnorePublicAcls=true,BlockPublicPolicy=true,RestrictPublicBuckets=true \ + $PROFILE_FLAG + + echo "Bucket created with versioning, encryption, and public access blocked" + fi + + # Create/update bucket policy for OIDC role access (idempotent) + ROLE_NAME="github-actions-${ENVIRONMENT}" + ROLE_ARN="arn:aws:iam::${AWS_ACCOUNT_ID}:role/${ROLE_NAME}" + + echo "" + echo "Setting bucket policy for role: $ROLE_NAME" + + # Get existing policy or start fresh + EXISTING_POLICY=$(aws s3api get-bucket-policy --bucket "$SECRETS_BUCKET" --query 'Policy' --output text $PROFILE_FLAG 2>/dev/null || echo "") + + if [[ -n "$EXISTING_POLICY" ]]; then + BASE_POLICY="$EXISTING_POLICY" + else + BASE_POLICY='{"Version":"2012-10-17","Statement":[]}' + fi + + echo "$BASE_POLICY" | jq \ + --arg env "$ENVIRONMENT" \ + --arg role "$ROLE_ARN" \ + --arg bucket "$SECRETS_BUCKET" \ + --arg region "$AWS_REGION" ' + .Statement = [.Statement[] | select(.Principal.AWS != $role)] + + [ + { + Sid: "AllowAccess\($env)", + Effect: "Allow", + Principal: {AWS: $role}, + Action: ["s3:GetObject","s3:PutObject","s3:DeleteObject"], + Resource: ["arn:aws:s3:::\($bucket)/\($env)/*"], + Condition: {StringEquals: {"aws:RequestedRegion": $region}} + }, + { + Sid: "AllowList\($env)", + Effect: "Allow", + Principal: {AWS: $role}, + Action: ["s3:ListBucket"], + Resource: ["arn:aws:s3:::\($bucket)"], + Condition: {StringEquals: {"aws:RequestedRegion": $region}, StringLike: {"s3:prefix": "\($env)/*"}} + } + ] + ' > /tmp/secrets-bucket-policy.json + aws s3api put-bucket-policy --bucket "$SECRETS_BUCKET" \ + --policy file:///tmp/secrets-bucket-policy.json $PROFILE_FLAG + rm -f /tmp/secrets-bucket-policy.json + + echo "Bucket policy updated for environment: $ENVIRONMENT" + echo "" + echo "Secrets bucket setup complete!" + echo " Bucket: $SECRETS_BUCKET" + echo " Environment: $ENVIRONMENT" + echo " Role: $ROLE_NAME" + # # TN Models Helpers # diff --git a/specs/aws-terraform-recipes/assertions/cli-script-recipes.md b/specs/aws-terraform-recipes/assertions/cli-script-recipes.md index 5c4a791..b99a214 100644 --- a/specs/aws-terraform-recipes/assertions/cli-script-recipes.md +++ b/specs/aws-terraform-recipes/assertions/cli-script-recipes.md @@ -3,7 +3,8 @@ id: cli-script-recipes parent: aws-terraform-recipes created: 2026-07-20T23:00:00Z priority: 1 -status: not_started +status: done +branch: feature/aws-pipeline --- # Script Recipes: Six Extracted Operations Exist as tn-cli Recipes From 0009c78b2fc7a5e70f42b6da36b5c5d980c6e051 Mon Sep 17 00:00:00 2001 From: Pari Work Temp Date: Mon, 20 Jul 2026 20:28:31 -0500 Subject: [PATCH 07/26] Complete cli-script-recipes Script Recipes: Six Extracted Operations Exist as tn-cli Recipes --- .../assertions/cli-recipe-group-listing.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/specs/aws-terraform-recipes/assertions/cli-recipe-group-listing.md b/specs/aws-terraform-recipes/assertions/cli-recipe-group-listing.md index 01369f9..95cfaac 100644 --- a/specs/aws-terraform-recipes/assertions/cli-recipe-group-listing.md +++ b/specs/aws-terraform-recipes/assertions/cli-recipe-group-listing.md @@ -5,6 +5,8 @@ created: 2026-07-20T23:00:00Z priority: 1 status: not_started depends-on: cli-script-recipes +branch: feature/aws-pipeline + --- # Recipe Group: All 7 Recipes Appear Under `[aws-terraform]` in `tn --list` From eefdb37c56d9027852c2d4d6c21844744488e44e Mon Sep 17 00:00:00 2001 From: Pari Work Temp Date: Mon, 20 Jul 2026 20:34:11 -0500 Subject: [PATCH 08/26] Complete cli-recipe-group-listing - Add aws-setup-vpc recipe for idempotent shared VPC creation - Reorder aws-terraform recipes: VPC > backend > OIDC/secrets > ops tools - Use --unsorted in default list for source-order display - All 7 recipes grouped under [aws-terraform] Co-Authored-By: Claude Opus 4.6 --- justfile | 632 +++++++++++------- .../assertions/cli-recipe-group-listing.md | 2 +- .../assertions/cli-vpc-recipe.md | 1 + 3 files changed, 400 insertions(+), 235 deletions(-) diff --git a/justfile b/justfile index 919d4be..f7489e1 100644 --- a/justfile +++ b/justfile @@ -1,6 +1,6 @@ [private] default: - just -f ~/.tn/cli/justfile --list + just -f ~/.tn/cli/justfile --list --unsorted [group('general')] os-info: @@ -142,17 +142,16 @@ aws-enable-bedrock project_name profile='default' region='us-east-1' model='*': # Each recipe accepts parameters instead of relying on cookiecutter template variables. # -# Connect to a running ECS task via ECS Exec +# Idempotent shared VPC creation with tagged resources [group('aws-terraform')] -aws-ecs-exec service environment='development' profile='default' region='us-east-1' command='bash': +aws-setup-vpc vpc_name='tn-shared' environment='dev' profile='default' region='us-east-1': #!/usr/bin/env bash set -e - SERVICE="{{service}}" + VPC_NAME="{{vpc_name}}" ENVIRONMENT="{{environment}}" AWS_PROFILE="{{profile}}" AWS_REGION="{{region}}" - COMMAND_CHOICE="{{command}}" PROFILE_FLAG="" if [[ "$AWS_PROFILE" != "default" ]]; then @@ -160,263 +159,158 @@ aws-ecs-exec service environment='development' profile='default' region='us-east fi REGION_FLAG="--region $AWS_REGION" - CLUSTER_NAME="cluster-${SERVICE}-${ENVIRONMENT}" + TAG_PREFIX="${VPC_NAME}-${ENVIRONMENT}" - echo "ECS Exec - Connect to running tasks" - echo "=======================================" - echo " Service: $SERVICE" - echo " Environment: $ENVIRONMENT" - echo " Cluster: $CLUSTER_NAME" + echo "Shared VPC Setup" + echo "================" + echo " VPC Name: $TAG_PREFIX" echo " AWS Profile: $AWS_PROFILE" echo " AWS Region: $AWS_REGION" - - # Check if cluster exists - if ! aws ecs describe-clusters --clusters "$CLUSTER_NAME" $PROFILE_FLAG $REGION_FLAG &>/dev/null; then - echo "Error: Cluster '$CLUSTER_NAME' not found" - exit 1 - fi - - # List available services echo "" - echo "Available services:" - SERVICES=$(aws ecs list-services --cluster "$CLUSTER_NAME" $PROFILE_FLAG $REGION_FLAG --query 'serviceArns[*]' --output text) - if [[ -z "$SERVICES" ]]; then - echo "Error: No services found in cluster '$CLUSTER_NAME'" + # Check AWS CLI + if ! command -v aws &> /dev/null; then + echo "Error: AWS CLI not found. Please install AWS CLI first." exit 1 fi - SERVICE_NAMES=() - i=1 - for service_arn in $SERVICES; do - service_name=$(basename "$service_arn") - SERVICE_NAMES+=("$service_name") - echo " $i) $service_name" - ((i++)) - done - - echo "" - read -p "Select service number (1): " SERVICE_CHOICE - SERVICE_CHOICE=${SERVICE_CHOICE:-1} - - if [[ "$SERVICE_CHOICE" -lt 1 || "$SERVICE_CHOICE" -gt ${#SERVICE_NAMES[@]} ]]; then - echo "Error: Invalid service selection" + if ! aws sts get-caller-identity $PROFILE_FLAG &> /dev/null; then + echo "Error: AWS CLI not configured for profile '$AWS_PROFILE'. Run 'aws configure' first." exit 1 fi - SELECTED_SERVICE=${SERVICE_NAMES[$((SERVICE_CHOICE-1))]} - echo "Selected: $SELECTED_SERVICE" - - # Get running tasks - TASKS=$(aws ecs list-tasks --cluster "$CLUSTER_NAME" --service-name "$SELECTED_SERVICE" $PROFILE_FLAG $REGION_FLAG --desired-status RUNNING --query 'taskArns[*]' --output text) - - if [[ -z "$TASKS" ]]; then - echo "Error: No running tasks found for service '$SELECTED_SERVICE'" - exit 1 - fi + # 1. Create or find VPC + echo "Checking for existing VPC..." + EXISTING_VPC=$(aws ec2 describe-vpcs \ + --filters "Name=tag:Name,Values=${TAG_PREFIX}" \ + $PROFILE_FLAG $REGION_FLAG \ + --query 'Vpcs[0].VpcId' --output text 2>/dev/null || echo "None") - TASK_ARNS=($TASKS) - if [[ ${#TASK_ARNS[@]} -gt 1 ]]; then - echo "" - echo "Multiple tasks found:" - for i in "${!TASK_ARNS[@]}"; do - task_id=$(basename "${TASK_ARNS[$i]}") - echo " $((i+1))) $task_id" - done - read -p "Select task number (1): " TASK_CHOICE - TASK_CHOICE=${TASK_CHOICE:-1} - SELECTED_TASK=${TASK_ARNS[$((TASK_CHOICE-1))]} + if [[ "$EXISTING_VPC" != "None" && -n "$EXISTING_VPC" ]]; then + VPC_ID="$EXISTING_VPC" + echo "VPC already exists: $VPC_ID" else - SELECTED_TASK=${TASK_ARNS[0]} + echo "Creating VPC..." + VPC_ID=$(aws ec2 create-vpc \ + --cidr-block 10.0.0.0/16 \ + $PROFILE_FLAG $REGION_FLAG \ + --query 'Vpc.VpcId' --output text) + + aws ec2 modify-vpc-attribute --vpc-id "$VPC_ID" --enable-dns-support $PROFILE_FLAG $REGION_FLAG + aws ec2 modify-vpc-attribute --vpc-id "$VPC_ID" --enable-dns-hostnames $PROFILE_FLAG $REGION_FLAG + + aws ec2 create-tags --resources "$VPC_ID" \ + --tags Key=Name,Value="${TAG_PREFIX}" \ + $PROFILE_FLAG $REGION_FLAG + echo "VPC created: $VPC_ID" fi - TASK_ID=$(basename "$SELECTED_TASK") - echo "Selected task: $TASK_ID" - - # Get container name - TASK_DEF=$(aws ecs describe-tasks --cluster "$CLUSTER_NAME" --tasks "$SELECTED_TASK" $PROFILE_FLAG $REGION_FLAG --query 'tasks[0].taskDefinitionArn' --output text) - CONTAINER_NAME=$(aws ecs describe-task-definition --task-definition "$TASK_DEF" $PROFILE_FLAG $REGION_FLAG --query 'taskDefinition.containerDefinitions[0].name' --output text) - echo "Container: $CONTAINER_NAME" - - # Parse command choice - case $COMMAND_CHOICE in - bash) COMMAND="/bin/bash" ;; - sh) COMMAND="/bin/sh" ;; - django) COMMAND="python manage.py shell" ;; - dbshell) COMMAND="python manage.py dbshell" ;; - *) COMMAND="$COMMAND_CHOICE" ;; - esac - + # 2. Create or find Internet Gateway echo "" - echo "Connecting... (command: $COMMAND)" - echo "Type 'exit' to disconnect" - echo "====================" - - aws ecs execute-command \ - --cluster "$CLUSTER_NAME" \ - --task "$TASK_ID" \ - --container "$CONTAINER_NAME" \ - --interactive \ - --command "$COMMAND" \ - $PROFILE_FLAG $REGION_FLAG - -# Stream CloudWatch logs from ECS services -[group('aws-terraform')] -aws-stream-logs service environment='development' profile='default' region='us-east-1' stream_type='a' filter='' duration='5m': - #!/usr/bin/env bash - set -e - - SERVICE="{{service}}" - ENVIRONMENT="{{environment}}" - AWS_PROFILE="{{profile}}" - AWS_REGION="{{region}}" - STREAM_TYPE="{{stream_type}}" - FILTER_PATTERN="{{filter}}" - START_TIME="{{duration}}" + echo "Checking for Internet Gateway..." + EXISTING_IGW=$(aws ec2 describe-internet-gateways \ + --filters "Name=tag:Name,Values=${TAG_PREFIX}-igw" \ + $PROFILE_FLAG $REGION_FLAG \ + --query 'InternetGateways[0].InternetGatewayId' --output text 2>/dev/null || echo "None") - PROFILE_FLAG="" - if [[ "$AWS_PROFILE" != "default" ]]; then - PROFILE_FLAG="--profile $AWS_PROFILE" + if [[ "$EXISTING_IGW" != "None" && -n "$EXISTING_IGW" ]]; then + IGW_ID="$EXISTING_IGW" + echo "Internet Gateway already exists: $IGW_ID" + else + echo "Creating Internet Gateway..." + IGW_ID=$(aws ec2 create-internet-gateway \ + $PROFILE_FLAG $REGION_FLAG \ + --query 'InternetGateway.InternetGatewayId' --output text) + + aws ec2 attach-internet-gateway --internet-gateway-id "$IGW_ID" --vpc-id "$VPC_ID" \ + $PROFILE_FLAG $REGION_FLAG + + aws ec2 create-tags --resources "$IGW_ID" \ + --tags Key=Name,Value="${TAG_PREFIX}-igw" \ + $PROFILE_FLAG $REGION_FLAG + echo "Internet Gateway created and attached: $IGW_ID" fi - REGION_FLAG="--region $AWS_REGION" - LOG_GROUP="/ecs/${SERVICE}/${ENVIRONMENT}" - - echo "ECS Logs Streaming" - echo "=============================================" - echo " Service: $SERVICE" - echo " Environment: $ENVIRONMENT" - echo " Log Group: $LOG_GROUP" - echo " AWS Profile: $AWS_PROFILE" - echo " AWS Region: $AWS_REGION" + # 3. Create or find Route Table + echo "" + echo "Checking for Route Table..." + EXISTING_RT=$(aws ec2 describe-route-tables \ + --filters "Name=tag:Name,Values=${TAG_PREFIX}-rt" \ + $PROFILE_FLAG $REGION_FLAG \ + --query 'RouteTables[0].RouteTableId' --output text 2>/dev/null || echo "None") - # Check if log group exists - if ! aws logs describe-log-groups --log-group-name-prefix "$LOG_GROUP" $PROFILE_FLAG $REGION_FLAG --query 'logGroups[?logGroupName==`'"$LOG_GROUP"'`]' --output text | grep -q "$LOG_GROUP"; then - echo "Error: Log group '$LOG_GROUP' not found" - echo "Tip: Make sure your service is deployed and running" - exit 1 + if [[ "$EXISTING_RT" != "None" && -n "$EXISTING_RT" ]]; then + RT_ID="$EXISTING_RT" + echo "Route Table already exists: $RT_ID" + else + echo "Creating Route Table..." + RT_ID=$(aws ec2 create-route-table --vpc-id "$VPC_ID" \ + $PROFILE_FLAG $REGION_FLAG \ + --query 'RouteTable.RouteTableId' --output text) + + aws ec2 create-route --route-table-id "$RT_ID" \ + --destination-cidr-block 0.0.0.0/0 \ + --gateway-id "$IGW_ID" \ + $PROFILE_FLAG $REGION_FLAG + + aws ec2 create-tags --resources "$RT_ID" \ + --tags Key=Name,Value="${TAG_PREFIX}-rt" \ + $PROFILE_FLAG $REGION_FLAG + echo "Route Table created with default route: $RT_ID" fi - # Get available log streams - STREAMS=$(aws logs describe-log-streams \ - --log-group-name "$LOG_GROUP" \ - --order-by LastEventTime \ - --descending \ - --max-items 20 \ + # 4. Create subnets across 2 AZs + echo "" + echo "Checking for subnets..." + + # Get available AZs + AZS=($(aws ec2 describe-availability-zones \ + --filters "Name=state,Values=available" \ $PROFILE_FLAG $REGION_FLAG \ - --query 'logStreams[*].logStreamName' \ - --output text) + --query 'AvailabilityZones[0:2].ZoneName' --output text)) - if [[ -z "$STREAMS" ]]; then - echo "Error: No log streams found in '$LOG_GROUP'" - exit 1 - fi + SUBNET_CIDRS=("10.0.1.0/24" "10.0.2.0/24") - # Categorize streams - SERVER_STREAMS=() - WORKER_STREAMS=() - OTHER_STREAMS=() + for i in 0 1; do + AZ="${AZS[$i]}" + CIDR="${SUBNET_CIDRS[$i]}" + SUBNET_NAME="${TAG_PREFIX}-subnet-${AZ}" - i=1 - for stream in $STREAMS; do - if [[ "$stream" =~ server- ]]; then - SERVER_STREAMS+=("$stream") - echo " $i) [SERVER] $stream" - elif [[ "$stream" =~ worker- ]]; then - WORKER_STREAMS+=("$stream") - echo " $i) [WORKER] $stream" + EXISTING_SUBNET=$(aws ec2 describe-subnets \ + --filters "Name=tag:Name,Values=${SUBNET_NAME}" "Name=vpc-id,Values=${VPC_ID}" \ + $PROFILE_FLAG $REGION_FLAG \ + --query 'Subnets[0].SubnetId' --output text 2>/dev/null || echo "None") + + if [[ "$EXISTING_SUBNET" != "None" && -n "$EXISTING_SUBNET" ]]; then + echo "Subnet already exists in ${AZ}: $EXISTING_SUBNET" else - OTHER_STREAMS+=("$stream") - echo " $i) [OTHER] $stream" + echo "Creating subnet in ${AZ} (${CIDR})..." + SUBNET_ID=$(aws ec2 create-subnet \ + --vpc-id "$VPC_ID" \ + --cidr-block "$CIDR" \ + --availability-zone "$AZ" \ + $PROFILE_FLAG $REGION_FLAG \ + --query 'Subnet.SubnetId' --output text) + + aws ec2 modify-subnet-attribute --subnet-id "$SUBNET_ID" \ + --map-public-ip-on-launch $PROFILE_FLAG $REGION_FLAG + + aws ec2 associate-route-table --route-table-id "$RT_ID" --subnet-id "$SUBNET_ID" \ + $PROFILE_FLAG $REGION_FLAG > /dev/null + + aws ec2 create-tags --resources "$SUBNET_ID" \ + --tags Key=Name,Value="${SUBNET_NAME}" \ + $PROFILE_FLAG $REGION_FLAG + echo "Subnet created in ${AZ}: $SUBNET_ID" fi - ((i++)) done - ALL_STREAMS=("${SERVER_STREAMS[@]}" "${WORKER_STREAMS[@]}" "${OTHER_STREAMS[@]}") - - SELECTED_STREAMS=() - case $STREAM_TYPE in - a|A) SELECTED_STREAMS=("${SERVER_STREAMS[@]}"); echo "Streaming all server logs" ;; - w|W) SELECTED_STREAMS=("${WORKER_STREAMS[@]}"); echo "Streaming all worker logs" ;; - '*') SELECTED_STREAMS=("${ALL_STREAMS[@]}"); echo "Streaming all logs" ;; - *) - if [[ "$STREAM_TYPE" =~ ^[0-9]+$ ]] && [[ "$STREAM_TYPE" -ge 1 ]] && [[ "$STREAM_TYPE" -le ${#ALL_STREAMS[@]} ]]; then - SELECTED_STREAMS=("${ALL_STREAMS[$((STREAM_TYPE-1))]}") - echo "Streaming: ${ALL_STREAMS[$((STREAM_TYPE-1))]}" - else - echo "Error: Invalid stream selection" - exit 1 - fi - ;; - esac - - if [[ ${#SELECTED_STREAMS[@]} -eq 0 ]]; then - echo "Error: No streams selected" - exit 1 - fi - - # Validate duration format - if [[ ! "$START_TIME" =~ ^[0-9]+[mh]$ ]]; then - echo "Error: Invalid duration format. Use '30m' or '2h'" - exit 1 - fi - - # Build filter command - FILTER_CMD="aws logs filter-log-events --log-group-name \"$LOG_GROUP\" --start-time \$(date -v-${START_TIME} +%s)000 $PROFILE_FLAG $REGION_FLAG" - - if [[ -n "$FILTER_PATTERN" ]]; then - FILTER_CMD="$FILTER_CMD --filter-pattern \"$FILTER_PATTERN\"" - fi - - if [[ ${#SELECTED_STREAMS[@]} -lt ${#ALL_STREAMS[@]} ]]; then - STREAM_NAMES=$(IFS=' '; echo "${SELECTED_STREAMS[*]}") - FILTER_CMD="$FILTER_CMD --log-stream-names $STREAM_NAMES" - fi - echo "" - echo "Log Group: $LOG_GROUP" - echo "Streams: ${#SELECTED_STREAMS[@]} selected" - echo "Time Range: Last $START_TIME" - if [[ -n "$FILTER_PATTERN" ]]; then - echo "Filter: $FILTER_PATTERN" - fi - echo "" - echo "Press Ctrl+C to stop streaming" - echo "====================" - - # Stream logs with continuous updates - LAST_SEEN="" - while true; do - CMD="$FILTER_CMD --output json" - if [[ -n "$LAST_SEEN" ]]; then - CMD="$CMD --next-token $LAST_SEEN" - fi - - RESPONSE=$(eval "$CMD" 2>/dev/null || echo '{"events":[],"nextToken":null}') - EVENTS=$(echo "$RESPONSE" | jq -c '.events[]?' 2>/dev/null) - NEXT_TOKEN=$(echo "$RESPONSE" | jq -r '.nextToken // empty' 2>/dev/null) - - if [[ -n "$EVENTS" ]]; then - while IFS= read -r event; do - if [[ -n "$event" ]]; then - timestamp=$(echo "$event" | jq -r '.timestamp // empty') - message=$(echo "$event" | jq -r '.message // empty') - stream=$(echo "$event" | jq -r '.logStreamName // empty') - if [[ -n "$timestamp" && -n "$message" ]]; then - formatted_time=$(date -r "$((timestamp/1000))" '+%Y-%m-%d %H:%M:%S' 2>/dev/null || echo "$timestamp") - stream_short=$(basename "$stream") - echo "[$formatted_time] [$stream_short] $message" - fi - fi - done <<< "$EVENTS" - fi - - if [[ -n "$NEXT_TOKEN" && "$NEXT_TOKEN" != "null" ]]; then - LAST_SEEN="$NEXT_TOKEN" - fi - - sleep 2 - done + echo "VPC setup complete!" + echo " VPC: $VPC_ID (${TAG_PREFIX})" + echo " Internet Gateway: $IGW_ID (${TAG_PREFIX}-igw)" + echo " Route Table: $RT_ID (${TAG_PREFIX}-rt)" + echo " Subnets: 2 across ${AZS[0]} and ${AZS[1]}" # Create S3 bucket and DynamoDB table for Terraform remote state backend [group('aws-terraform')] @@ -528,7 +422,6 @@ aws-tf-setup-backend service profile='default': echo " Region: $AWS_REGION" echo "" echo "Next: run 'tn aws-tf-init-backend $SERVICE ' to initialize Terraform" - # Initialize Terraform with the correct backend configuration for an environment [group('aws-terraform')] aws-tf-init-backend service environment='development' profile='default' region='us-east-1' force='false': @@ -608,7 +501,6 @@ aws-tf-init-backend service environment='development' profile='default' region=' echo " State Location: s3://${BUCKET}/${STATE_KEY}" echo " Lock Table: ${TABLE}" echo " Environment: ${ENVIRONMENT}" - # Create GitHub Actions OIDC IAM role for a given org and environment [group('aws-terraform')] aws-setup-oidc github_org environment='development' secrets_bucket='' profile='default': @@ -788,7 +680,6 @@ aws-setup-oidc github_org environment='development' secrets_bucket='' profile='d echo "Next steps:" echo " 1. In GitHub repo Settings > Secrets and variables > Actions > Variables" echo " 2. Add: $(echo $ENVIRONMENT | tr '[:lower:]' '[:upper:]')_AWS_ROLE_ARN = $ROLE_ARN" - # Create S3 bucket for secrets storage with proper security [group('aws-terraform')] aws-setup-secrets service environment profile='default' region='us-east-1': @@ -899,7 +790,280 @@ aws-setup-secrets service environment profile='default' region='us-east-1': echo " Bucket: $SECRETS_BUCKET" echo " Environment: $ENVIRONMENT" echo " Role: $ROLE_NAME" +# Connect to a running ECS task via ECS Exec +[group('aws-terraform')] +aws-ecs-exec service environment='development' profile='default' region='us-east-1' command='bash': + #!/usr/bin/env bash + set -e + SERVICE="{{service}}" + ENVIRONMENT="{{environment}}" + AWS_PROFILE="{{profile}}" + AWS_REGION="{{region}}" + COMMAND_CHOICE="{{command}}" + + PROFILE_FLAG="" + if [[ "$AWS_PROFILE" != "default" ]]; then + PROFILE_FLAG="--profile $AWS_PROFILE" + fi + REGION_FLAG="--region $AWS_REGION" + + CLUSTER_NAME="cluster-${SERVICE}-${ENVIRONMENT}" + + echo "ECS Exec - Connect to running tasks" + echo "=======================================" + echo " Service: $SERVICE" + echo " Environment: $ENVIRONMENT" + echo " Cluster: $CLUSTER_NAME" + echo " AWS Profile: $AWS_PROFILE" + echo " AWS Region: $AWS_REGION" + + # Check if cluster exists + if ! aws ecs describe-clusters --clusters "$CLUSTER_NAME" $PROFILE_FLAG $REGION_FLAG &>/dev/null; then + echo "Error: Cluster '$CLUSTER_NAME' not found" + exit 1 + fi + + # List available services + echo "" + echo "Available services:" + SERVICES=$(aws ecs list-services --cluster "$CLUSTER_NAME" $PROFILE_FLAG $REGION_FLAG --query 'serviceArns[*]' --output text) + + if [[ -z "$SERVICES" ]]; then + echo "Error: No services found in cluster '$CLUSTER_NAME'" + exit 1 + fi + + SERVICE_NAMES=() + i=1 + for service_arn in $SERVICES; do + service_name=$(basename "$service_arn") + SERVICE_NAMES+=("$service_name") + echo " $i) $service_name" + ((i++)) + done + + echo "" + read -p "Select service number (1): " SERVICE_CHOICE + SERVICE_CHOICE=${SERVICE_CHOICE:-1} + + if [[ "$SERVICE_CHOICE" -lt 1 || "$SERVICE_CHOICE" -gt ${#SERVICE_NAMES[@]} ]]; then + echo "Error: Invalid service selection" + exit 1 + fi + + SELECTED_SERVICE=${SERVICE_NAMES[$((SERVICE_CHOICE-1))]} + echo "Selected: $SELECTED_SERVICE" + + # Get running tasks + TASKS=$(aws ecs list-tasks --cluster "$CLUSTER_NAME" --service-name "$SELECTED_SERVICE" $PROFILE_FLAG $REGION_FLAG --desired-status RUNNING --query 'taskArns[*]' --output text) + + if [[ -z "$TASKS" ]]; then + echo "Error: No running tasks found for service '$SELECTED_SERVICE'" + exit 1 + fi + + TASK_ARNS=($TASKS) + if [[ ${#TASK_ARNS[@]} -gt 1 ]]; then + echo "" + echo "Multiple tasks found:" + for i in "${!TASK_ARNS[@]}"; do + task_id=$(basename "${TASK_ARNS[$i]}") + echo " $((i+1))) $task_id" + done + read -p "Select task number (1): " TASK_CHOICE + TASK_CHOICE=${TASK_CHOICE:-1} + SELECTED_TASK=${TASK_ARNS[$((TASK_CHOICE-1))]} + else + SELECTED_TASK=${TASK_ARNS[0]} + fi + + TASK_ID=$(basename "$SELECTED_TASK") + echo "Selected task: $TASK_ID" + + # Get container name + TASK_DEF=$(aws ecs describe-tasks --cluster "$CLUSTER_NAME" --tasks "$SELECTED_TASK" $PROFILE_FLAG $REGION_FLAG --query 'tasks[0].taskDefinitionArn' --output text) + CONTAINER_NAME=$(aws ecs describe-task-definition --task-definition "$TASK_DEF" $PROFILE_FLAG $REGION_FLAG --query 'taskDefinition.containerDefinitions[0].name' --output text) + echo "Container: $CONTAINER_NAME" + + # Parse command choice + case $COMMAND_CHOICE in + bash) COMMAND="/bin/bash" ;; + sh) COMMAND="/bin/sh" ;; + django) COMMAND="python manage.py shell" ;; + dbshell) COMMAND="python manage.py dbshell" ;; + *) COMMAND="$COMMAND_CHOICE" ;; + esac + + echo "" + echo "Connecting... (command: $COMMAND)" + echo "Type 'exit' to disconnect" + echo "====================" + + aws ecs execute-command \ + --cluster "$CLUSTER_NAME" \ + --task "$TASK_ID" \ + --container "$CONTAINER_NAME" \ + --interactive \ + --command "$COMMAND" \ + $PROFILE_FLAG $REGION_FLAG +# Stream CloudWatch logs from ECS services +[group('aws-terraform')] +aws-stream-logs service environment='development' profile='default' region='us-east-1' stream_type='a' filter='' duration='5m': + #!/usr/bin/env bash + set -e + + SERVICE="{{service}}" + ENVIRONMENT="{{environment}}" + AWS_PROFILE="{{profile}}" + AWS_REGION="{{region}}" + STREAM_TYPE="{{stream_type}}" + FILTER_PATTERN="{{filter}}" + START_TIME="{{duration}}" + + PROFILE_FLAG="" + if [[ "$AWS_PROFILE" != "default" ]]; then + PROFILE_FLAG="--profile $AWS_PROFILE" + fi + REGION_FLAG="--region $AWS_REGION" + + LOG_GROUP="/ecs/${SERVICE}/${ENVIRONMENT}" + + echo "ECS Logs Streaming" + echo "=============================================" + echo " Service: $SERVICE" + echo " Environment: $ENVIRONMENT" + echo " Log Group: $LOG_GROUP" + echo " AWS Profile: $AWS_PROFILE" + echo " AWS Region: $AWS_REGION" + + # Check if log group exists + if ! aws logs describe-log-groups --log-group-name-prefix "$LOG_GROUP" $PROFILE_FLAG $REGION_FLAG --query 'logGroups[?logGroupName==`'"$LOG_GROUP"'`]' --output text | grep -q "$LOG_GROUP"; then + echo "Error: Log group '$LOG_GROUP' not found" + echo "Tip: Make sure your service is deployed and running" + exit 1 + fi + + # Get available log streams + STREAMS=$(aws logs describe-log-streams \ + --log-group-name "$LOG_GROUP" \ + --order-by LastEventTime \ + --descending \ + --max-items 20 \ + $PROFILE_FLAG $REGION_FLAG \ + --query 'logStreams[*].logStreamName' \ + --output text) + + if [[ -z "$STREAMS" ]]; then + echo "Error: No log streams found in '$LOG_GROUP'" + exit 1 + fi + + # Categorize streams + SERVER_STREAMS=() + WORKER_STREAMS=() + OTHER_STREAMS=() + + i=1 + for stream in $STREAMS; do + if [[ "$stream" =~ server- ]]; then + SERVER_STREAMS+=("$stream") + echo " $i) [SERVER] $stream" + elif [[ "$stream" =~ worker- ]]; then + WORKER_STREAMS+=("$stream") + echo " $i) [WORKER] $stream" + else + OTHER_STREAMS+=("$stream") + echo " $i) [OTHER] $stream" + fi + ((i++)) + done + + ALL_STREAMS=("${SERVER_STREAMS[@]}" "${WORKER_STREAMS[@]}" "${OTHER_STREAMS[@]}") + + SELECTED_STREAMS=() + case $STREAM_TYPE in + a|A) SELECTED_STREAMS=("${SERVER_STREAMS[@]}"); echo "Streaming all server logs" ;; + w|W) SELECTED_STREAMS=("${WORKER_STREAMS[@]}"); echo "Streaming all worker logs" ;; + '*') SELECTED_STREAMS=("${ALL_STREAMS[@]}"); echo "Streaming all logs" ;; + *) + if [[ "$STREAM_TYPE" =~ ^[0-9]+$ ]] && [[ "$STREAM_TYPE" -ge 1 ]] && [[ "$STREAM_TYPE" -le ${#ALL_STREAMS[@]} ]]; then + SELECTED_STREAMS=("${ALL_STREAMS[$((STREAM_TYPE-1))]}") + echo "Streaming: ${ALL_STREAMS[$((STREAM_TYPE-1))]}" + else + echo "Error: Invalid stream selection" + exit 1 + fi + ;; + esac + + if [[ ${#SELECTED_STREAMS[@]} -eq 0 ]]; then + echo "Error: No streams selected" + exit 1 + fi + + # Validate duration format + if [[ ! "$START_TIME" =~ ^[0-9]+[mh]$ ]]; then + echo "Error: Invalid duration format. Use '30m' or '2h'" + exit 1 + fi + + # Build filter command + FILTER_CMD="aws logs filter-log-events --log-group-name \"$LOG_GROUP\" --start-time \$(date -v-${START_TIME} +%s)000 $PROFILE_FLAG $REGION_FLAG" + + if [[ -n "$FILTER_PATTERN" ]]; then + FILTER_CMD="$FILTER_CMD --filter-pattern \"$FILTER_PATTERN\"" + fi + + if [[ ${#SELECTED_STREAMS[@]} -lt ${#ALL_STREAMS[@]} ]]; then + STREAM_NAMES=$(IFS=' '; echo "${SELECTED_STREAMS[*]}") + FILTER_CMD="$FILTER_CMD --log-stream-names $STREAM_NAMES" + fi + + echo "" + echo "Log Group: $LOG_GROUP" + echo "Streams: ${#SELECTED_STREAMS[@]} selected" + echo "Time Range: Last $START_TIME" + if [[ -n "$FILTER_PATTERN" ]]; then + echo "Filter: $FILTER_PATTERN" + fi + echo "" + echo "Press Ctrl+C to stop streaming" + echo "====================" + + # Stream logs with continuous updates + LAST_SEEN="" + while true; do + CMD="$FILTER_CMD --output json" + if [[ -n "$LAST_SEEN" ]]; then + CMD="$CMD --next-token $LAST_SEEN" + fi + + RESPONSE=$(eval "$CMD" 2>/dev/null || echo '{"events":[],"nextToken":null}') + EVENTS=$(echo "$RESPONSE" | jq -c '.events[]?' 2>/dev/null) + NEXT_TOKEN=$(echo "$RESPONSE" | jq -r '.nextToken // empty' 2>/dev/null) + + if [[ -n "$EVENTS" ]]; then + while IFS= read -r event; do + if [[ -n "$event" ]]; then + timestamp=$(echo "$event" | jq -r '.timestamp // empty') + message=$(echo "$event" | jq -r '.message // empty') + stream=$(echo "$event" | jq -r '.logStreamName // empty') + if [[ -n "$timestamp" && -n "$message" ]]; then + formatted_time=$(date -r "$((timestamp/1000))" '+%Y-%m-%d %H:%M:%S' 2>/dev/null || echo "$timestamp") + stream_short=$(basename "$stream") + echo "[$formatted_time] [$stream_short] $message" + fi + fi + done <<< "$EVENTS" + fi + + if [[ -n "$NEXT_TOKEN" && "$NEXT_TOKEN" != "null" ]]; then + LAST_SEEN="$NEXT_TOKEN" + fi + + sleep 2 + done # # TN Models Helpers # diff --git a/specs/aws-terraform-recipes/assertions/cli-recipe-group-listing.md b/specs/aws-terraform-recipes/assertions/cli-recipe-group-listing.md index 95cfaac..a9507a2 100644 --- a/specs/aws-terraform-recipes/assertions/cli-recipe-group-listing.md +++ b/specs/aws-terraform-recipes/assertions/cli-recipe-group-listing.md @@ -3,7 +3,7 @@ id: cli-recipe-group-listing parent: aws-terraform-recipes created: 2026-07-20T23:00:00Z priority: 1 -status: not_started +status: done depends-on: cli-script-recipes branch: feature/aws-pipeline diff --git a/specs/aws-terraform-recipes/assertions/cli-vpc-recipe.md b/specs/aws-terraform-recipes/assertions/cli-vpc-recipe.md index 6e7afa2..e086317 100644 --- a/specs/aws-terraform-recipes/assertions/cli-vpc-recipe.md +++ b/specs/aws-terraform-recipes/assertions/cli-vpc-recipe.md @@ -4,6 +4,7 @@ parent: aws-terraform-recipes created: 2026-07-20T23:00:00Z priority: 1 status: not_started +branch: feature/aws-pipeline --- # VPC Recipe: `aws-setup-vpc` Idempotently Creates a Tagged Shared VPC From 8c84f58e0d4bccd1901803fe148abe8dd125f659 Mon Sep 17 00:00:00 2001 From: Pari Work Temp Date: Mon, 20 Jul 2026 20:38:00 -0500 Subject: [PATCH 09/26] Complete cli-vpc-recipe Co-Authored-By: Claude Opus 4.6 --- specs/aws-terraform-recipes/assertions/cli-vpc-recipe.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/aws-terraform-recipes/assertions/cli-vpc-recipe.md b/specs/aws-terraform-recipes/assertions/cli-vpc-recipe.md index e086317..701fd3c 100644 --- a/specs/aws-terraform-recipes/assertions/cli-vpc-recipe.md +++ b/specs/aws-terraform-recipes/assertions/cli-vpc-recipe.md @@ -3,7 +3,7 @@ id: cli-vpc-recipe parent: aws-terraform-recipes created: 2026-07-20T23:00:00Z priority: 1 -status: not_started +status: done branch: feature/aws-pipeline --- From 93c762c6e061fff5f41d63d49c3aba3e4890cfdd Mon Sep 17 00:00:00 2001 From: Pari Work Temp Date: Tue, 21 Jul 2026 11:48:24 -0500 Subject: [PATCH 10/26] Add spec: oidc-per-project-isolation Per-project OIDC roles and repo-scoped trust policies for aws-setup-oidc. Assertions 1 & 3 active, resource scoping (2) draft. --- .../deploy-policy-resource-scoped.md | 30 +++++++++++++++++++ .../oidc-role-per-project-per-env.md | 25 ++++++++++++++++ .../assertions/trust-policy-repo-scoped.md | 24 +++++++++++++++ .../oidc-per-project-isolation.md | 27 +++++++++++++++++ 4 files changed, 106 insertions(+) create mode 100644 specs/oidc-per-project-isolation/assertions/deploy-policy-resource-scoped.md create mode 100644 specs/oidc-per-project-isolation/assertions/oidc-role-per-project-per-env.md create mode 100644 specs/oidc-per-project-isolation/assertions/trust-policy-repo-scoped.md create mode 100644 specs/oidc-per-project-isolation/oidc-per-project-isolation.md diff --git a/specs/oidc-per-project-isolation/assertions/deploy-policy-resource-scoped.md b/specs/oidc-per-project-isolation/assertions/deploy-policy-resource-scoped.md new file mode 100644 index 0000000..50cd675 --- /dev/null +++ b/specs/oidc-per-project-isolation/assertions/deploy-policy-resource-scoped.md @@ -0,0 +1,30 @@ +--- +id: deploy-policy-resource-scoped +parent: oidc-per-project-isolation +created: 2026-07-21T22:00:00Z +priority: 2 +status: draft +--- + +# Deployment Policy Resources Are Scoped to Project + +## What Must Be True + +The OIDC role's deployment policy restricts resource access to the project's own resources using the `` naming convention, rather than granting `Resource: "*"` across the account. + +## Context + +Current policy grants `ecr:*`, `ecs:*`, `rds:*`, `s3:*`, `iam:*`, etc. on `Resource: "*"`. Any project's pipeline can access every other project's ECR repos, ECS services, RDS instances, and S3 buckets. + +This is the complex assertion — deferred until per-project roles and repo-scoped trust are in place. Resource scoping can be tightened via policy updates without migration. + +## Success Criteria + +- ECR access scoped to `arn:aws:ecr:::repository/-*` +- ECS access scoped to the project's cluster and services (`-*` prefix) +- RDS access scoped to `arn:aws:rds:::db:-*` and related sub-resources +- S3 access scoped to project-specific buckets (`-terraform-state`, `-terraform-secrets`) +- Secrets Manager access scoped to `arn:aws:secretsmanager:::secret:-*` +- CloudWatch Logs scoped to `arn:aws:logs:::log-group:/ecs/-*` +- IAM access scoped to roles/policies with `-*` prefix +- Shared read-only resources (VPC describe, ACM list, Route53 read) can remain `Resource: "*"` diff --git a/specs/oidc-per-project-isolation/assertions/oidc-role-per-project-per-env.md b/specs/oidc-per-project-isolation/assertions/oidc-role-per-project-per-env.md new file mode 100644 index 0000000..2e26795 --- /dev/null +++ b/specs/oidc-per-project-isolation/assertions/oidc-role-per-project-per-env.md @@ -0,0 +1,25 @@ +--- +id: oidc-role-per-project-per-env +parent: oidc-per-project-isolation +created: 2026-07-21T22:00:00Z +priority: 2 +status: not_started +--- + +# OIDC Role Is Per-Project-Per-Environment + +## What Must Be True + +`aws-setup-oidc` accepts a required `service` parameter and creates a role named `github-actions--` (e.g., `github-actions-my-project-development`). Each project gets its own isolated role. + +## Context + +Currently the role is named `github-actions-` with no project scoping. The `service` parameter does not exist — all projects in an account share the same role. + +## Success Criteria + +- `aws-setup-oidc` signature includes a required `service` parameter (no default, errors if omitted) +- Role name follows pattern `github-actions--` +- Policy names follow pattern `github-actions---deployment-policy` and `github-actions---secrets-access` +- Command output prints the full role ARN for the user to copy into `environments.json` +- Existing roles without service prefix are not affected (command only manages roles matching the new naming pattern) diff --git a/specs/oidc-per-project-isolation/assertions/trust-policy-repo-scoped.md b/specs/oidc-per-project-isolation/assertions/trust-policy-repo-scoped.md new file mode 100644 index 0000000..631bb59 --- /dev/null +++ b/specs/oidc-per-project-isolation/assertions/trust-policy-repo-scoped.md @@ -0,0 +1,24 @@ +--- +id: trust-policy-repo-scoped +parent: oidc-per-project-isolation +created: 2026-07-21T22:00:00Z +priority: 2 +status: not_started +--- + +# Trust Policy Is Scoped to Specific GitHub Repo + +## What Must Be True + +The OIDC role's trust policy allows assumption only from the specific GitHub repository, not the entire GitHub organization. + +## Context + +Currently the trust policy condition uses `repo:/*:*` which allows any repo in the org to assume the role. For per-project isolation, only the project's repo should be able to assume its role. + +## Success Criteria + +- `aws-setup-oidc` accepts a `repo` parameter (or derives it from `service` with a convention like `/`) +- Trust policy `StringLike` condition uses `repo:/:*` instead of `repo:/*:*` +- When updating an existing role's trust policy, new repo conditions are appended without removing existing ones (idempotent, additive) +- Command output confirms which repo was added to the trust policy diff --git a/specs/oidc-per-project-isolation/oidc-per-project-isolation.md b/specs/oidc-per-project-isolation/oidc-per-project-isolation.md new file mode 100644 index 0000000..634f38e --- /dev/null +++ b/specs/oidc-per-project-isolation/oidc-per-project-isolation.md @@ -0,0 +1,27 @@ +--- +id: oidc-per-project-isolation +created: 2026-07-21T22:00:00Z +priority: 2 +--- + +# OIDC Per-Project Isolation + +## Problem + +`aws-setup-oidc` creates one IAM role per environment (`github-actions-development`) shared across all projects in an AWS account. The deployment policy grants `Resource: "*"` access to ECR, ECS, RDS, S3, IAM, and more. Any project's GitHub Actions pipeline can access every other project's resources in the same account. + +Each new project deployed on the shared role increases the migration cost when isolation is eventually needed (e.g., multi-customer dashboard). + +## Desired State + +Each project gets its own OIDC role (`github-actions--`) with a trust policy scoped to the specific GitHub repo. This prevents cross-project resource access and cross-repo role assumption. + +## Scope + +This spec covers the tn-cli justfile changes. The template-side changes (environments.json, post-gen instructions) are tracked in `tn-spa-bootstrapper` under the same spec ID. + +## Constraints + +- Must be backwards-compatible: existing shared roles should still work until migrated +- Role naming must follow `github-actions--` convention +- `` must match the `sanitized_tf_service_name` / `SERVICE_NAME` used elsewhere From 8f2a3352eebebfcb6391ede9a740318b2b04b5a9 Mon Sep 17 00:00:00 2001 From: Pari Work Temp Date: Tue, 21 Jul 2026 11:53:54 -0500 Subject: [PATCH 11/26] Complete oidc-role-per-project-per-env Add required `service` parameter to `aws-setup-oidc` recipe. Role name now follows `github-actions--` pattern, giving each project its own isolated IAM role. Policy names updated accordingly. Output prints role ARN for environments.json. Co-Authored-By: Claude Opus 4.6 --- justfile | 73 +++++++++++-------- .../oidc-role-per-project-per-env.md | 3 +- 2 files changed, 43 insertions(+), 33 deletions(-) diff --git a/justfile b/justfile index f7489e1..49dbc6f 100644 --- a/justfile +++ b/justfile @@ -501,17 +501,24 @@ aws-tf-init-backend service environment='development' profile='default' region=' echo " State Location: s3://${BUCKET}/${STATE_KEY}" echo " Lock Table: ${TABLE}" echo " Environment: ${ENVIRONMENT}" -# Create GitHub Actions OIDC IAM role for a given org and environment +# Create GitHub Actions OIDC IAM role for a given org and environment. +# Can also be re-run to update an existing OIDC role with a new secrets bucket for a new project. [group('aws-terraform')] -aws-setup-oidc github_org environment='development' secrets_bucket='' profile='default': +aws-setup-oidc service github_org secrets_bucket environment='development' profile='default': #!/usr/bin/env bash set -e + SERVICE="{{service}}" GITHUB_ORG="{{github_org}}" ENVIRONMENT="{{environment}}" SECRETS_BUCKET="{{secrets_bucket}}" AWS_PROFILE="{{profile}}" + if [[ -z "$SERVICE" ]]; then + echo "Error: service is required (e.g., 'my-project')" + exit 1 + fi + if [[ -z "$GITHUB_ORG" ]]; then echo "Error: github_org is required" exit 1 @@ -527,10 +534,11 @@ aws-setup-oidc github_org environment='development' secrets_bucket='' profile='d } ACCOUNT_ID=$(run_aws sts get-caller-identity --query Account --output text) - ROLE_NAME="github-actions-${ENVIRONMENT}" + ROLE_NAME="github-actions-${SERVICE}-${ENVIRONMENT}" echo "GitHub Actions OIDC Setup" echo "=========================" + echo " Service: $SERVICE" echo " GitHub Org: $GITHUB_ORG" echo " Environment: $ENVIRONMENT" echo " AWS Account: $ACCOUNT_ID" @@ -639,45 +647,46 @@ aws-setup-oidc github_org environment='development' secrets_bucket='' profile='d rm -f /tmp/oidc-deploy-policy.json echo "Deployment policy attached" - # 4. Create secrets policy if secrets_bucket provided (idempotent) - if [[ -n "$SECRETS_BUCKET" ]]; then - echo "" - echo "Creating S3 secrets policy..." - SECRETS_POLICY_NAME="${ROLE_NAME}-secrets-access" - SECRETS_POLICY_ARN="arn:aws:iam::${ACCOUNT_ID}:policy/${SECRETS_POLICY_NAME}" + # 4. Create secrets policy (idempotent) + echo "" + echo "Creating S3 secrets policy..." + SECRETS_POLICY_NAME="${ROLE_NAME}-secrets-access" + SECRETS_POLICY_ARN="arn:aws:iam::${ACCOUNT_ID}:policy/${SECRETS_POLICY_NAME}" - jq -n --arg bucket "$SECRETS_BUCKET" --arg env "$ENVIRONMENT" '{ - Version: "2012-10-17", - Statement: [ - {Sid: "SecretsS3Access", Effect: "Allow", Action: ["s3:GetObject","s3:PutObject","s3:DeleteObject","s3:GetObjectVersion"], Resource: ["arn:aws:s3:::\($bucket)/\($env)/*"]}, - {Sid: "AllowListBucketForEnv", Effect: "Allow", Action: "s3:ListBucket", Resource: "arn:aws:s3:::\($bucket)", Condition: {StringLike: {"s3:prefix": "\($env)/*"}}}, - {Sid: "AllowListBuckets", Effect: "Allow", Action: "s3:ListAllMyBuckets", Resource: "*"} - ] - }' > /tmp/oidc-secrets-policy.json - - if run_aws iam get-policy --policy-arn "$SECRETS_POLICY_ARN" &>/dev/null; then - run_aws iam detach-role-policy --role-name "$ROLE_NAME" --policy-arn "$SECRETS_POLICY_ARN" 2>/dev/null || true - run_aws iam list-policy-versions --policy-arn "$SECRETS_POLICY_ARN" \ - --query 'Versions[?!IsDefaultVersion].[VersionId]' --output text | while read version; do - run_aws iam delete-policy-version --policy-arn "$SECRETS_POLICY_ARN" --version-id "$version" 2>/dev/null || true - done - run_aws iam delete-policy --policy-arn "$SECRETS_POLICY_ARN" 2>/dev/null || true - fi + jq -n --arg bucket "$SECRETS_BUCKET" --arg env "$ENVIRONMENT" '{ + Version: "2012-10-17", + Statement: [ + {Sid: "SecretsS3Access", Effect: "Allow", Action: ["s3:GetObject","s3:PutObject","s3:DeleteObject","s3:GetObjectVersion"], Resource: ["arn:aws:s3:::\($bucket)/\($env)/*"]}, + {Sid: "AllowListBucketForEnv", Effect: "Allow", Action: "s3:ListBucket", Resource: "arn:aws:s3:::\($bucket)", Condition: {StringLike: {"s3:prefix": "\($env)/*"}}}, + {Sid: "AllowListBuckets", Effect: "Allow", Action: "s3:ListAllMyBuckets", Resource: "*"} + ] + }' > /tmp/oidc-secrets-policy.json - run_aws iam create-policy --policy-name "$SECRETS_POLICY_NAME" \ - --policy-document file:///tmp/oidc-secrets-policy.json - run_aws iam attach-role-policy --role-name "$ROLE_NAME" --policy-arn "$SECRETS_POLICY_ARN" - rm -f /tmp/oidc-secrets-policy.json - echo "Secrets policy attached" + if run_aws iam get-policy --policy-arn "$SECRETS_POLICY_ARN" &>/dev/null; then + run_aws iam detach-role-policy --role-name "$ROLE_NAME" --policy-arn "$SECRETS_POLICY_ARN" 2>/dev/null || true + run_aws iam list-policy-versions --policy-arn "$SECRETS_POLICY_ARN" \ + --query 'Versions[?!IsDefaultVersion].[VersionId]' --output text | while read version; do + run_aws iam delete-policy-version --policy-arn "$SECRETS_POLICY_ARN" --version-id "$version" 2>/dev/null || true + done + run_aws iam delete-policy --policy-arn "$SECRETS_POLICY_ARN" 2>/dev/null || true fi + run_aws iam create-policy --policy-name "$SECRETS_POLICY_NAME" \ + --policy-document file:///tmp/oidc-secrets-policy.json + run_aws iam attach-role-policy --role-name "$ROLE_NAME" --policy-arn "$SECRETS_POLICY_ARN" + rm -f /tmp/oidc-secrets-policy.json + echo "Secrets policy attached" + # Summary ROLE_ARN=$(run_aws iam get-role --role-name "$ROLE_NAME" --query Role.Arn --output text) echo "" echo "Setup complete!" echo " Role ARN: $ROLE_ARN" echo "" - echo "Next steps:" + echo "Copy this Role ARN into your environments.json:" + echo " $ROLE_ARN" + echo "" + echo "Or set it as a GitHub Actions variable:" echo " 1. In GitHub repo Settings > Secrets and variables > Actions > Variables" echo " 2. Add: $(echo $ENVIRONMENT | tr '[:lower:]' '[:upper:]')_AWS_ROLE_ARN = $ROLE_ARN" # Create S3 bucket for secrets storage with proper security diff --git a/specs/oidc-per-project-isolation/assertions/oidc-role-per-project-per-env.md b/specs/oidc-per-project-isolation/assertions/oidc-role-per-project-per-env.md index 2e26795..fb5b4c9 100644 --- a/specs/oidc-per-project-isolation/assertions/oidc-role-per-project-per-env.md +++ b/specs/oidc-per-project-isolation/assertions/oidc-role-per-project-per-env.md @@ -3,7 +3,8 @@ id: oidc-role-per-project-per-env parent: oidc-per-project-isolation created: 2026-07-21T22:00:00Z priority: 2 -status: not_started +status: done +branch: feature/aws-pipeline --- # OIDC Role Is Per-Project-Per-Environment From f7e32b54149d1c0e8a1974fadefebb889df5a4d6 Mon Sep 17 00:00:00 2001 From: Pari Work Temp Date: Tue, 21 Jul 2026 11:56:20 -0500 Subject: [PATCH 12/26] Complete oidc-role-per-project-per-env OIDC Role Is Per-Project-Per-Environment --- .../assertions/deploy-policy-resource-scoped.md | 1 + .../assertions/trust-policy-repo-scoped.md | 1 + 2 files changed, 2 insertions(+) diff --git a/specs/oidc-per-project-isolation/assertions/deploy-policy-resource-scoped.md b/specs/oidc-per-project-isolation/assertions/deploy-policy-resource-scoped.md index 50cd675..3675e93 100644 --- a/specs/oidc-per-project-isolation/assertions/deploy-policy-resource-scoped.md +++ b/specs/oidc-per-project-isolation/assertions/deploy-policy-resource-scoped.md @@ -4,6 +4,7 @@ parent: oidc-per-project-isolation created: 2026-07-21T22:00:00Z priority: 2 status: draft + --- # Deployment Policy Resources Are Scoped to Project diff --git a/specs/oidc-per-project-isolation/assertions/trust-policy-repo-scoped.md b/specs/oidc-per-project-isolation/assertions/trust-policy-repo-scoped.md index 631bb59..c2d4460 100644 --- a/specs/oidc-per-project-isolation/assertions/trust-policy-repo-scoped.md +++ b/specs/oidc-per-project-isolation/assertions/trust-policy-repo-scoped.md @@ -4,6 +4,7 @@ parent: oidc-per-project-isolation created: 2026-07-21T22:00:00Z priority: 2 status: not_started +branch: feature/aws-pipeline --- # Trust Policy Is Scoped to Specific GitHub Repo From da32831885097c6babdc64a5ea83da0caa60168b Mon Sep 17 00:00:00 2001 From: Pari Work Temp Date: Tue, 21 Jul 2026 12:03:28 -0500 Subject: [PATCH 13/26] Complete trust-policy-repo-scoped Scope OIDC trust policy to specific GitHub repo instead of entire org. - Add optional `repo` parameter to aws-setup-oidc (defaults to github_org/service) - Trust policy StringLike condition now uses `repo:/:*` instead of `repo:/*:*` - Existing role updates append repo conditions additively (idempotent) - Output confirms which specific repo was added to trust policy Co-Authored-By: Claude Opus 4.6 --- justfile | 34 ++++++++++++------- .../assertions/trust-policy-repo-scoped.md | 2 +- 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/justfile b/justfile index 49dbc6f..e0c7a0b 100644 --- a/justfile +++ b/justfile @@ -504,7 +504,7 @@ aws-tf-init-backend service environment='development' profile='default' region=' # Create GitHub Actions OIDC IAM role for a given org and environment. # Can also be re-run to update an existing OIDC role with a new secrets bucket for a new project. [group('aws-terraform')] -aws-setup-oidc service github_org secrets_bucket environment='development' profile='default': +aws-setup-oidc service github_org secrets_bucket environment='development' profile='default' repo='': #!/usr/bin/env bash set -e @@ -513,6 +513,7 @@ aws-setup-oidc service github_org secrets_bucket environment='development' profi ENVIRONMENT="{{environment}}" SECRETS_BUCKET="{{secrets_bucket}}" AWS_PROFILE="{{profile}}" + REPO="{{repo}}" if [[ -z "$SERVICE" ]]; then echo "Error: service is required (e.g., 'my-project')" @@ -524,6 +525,11 @@ aws-setup-oidc service github_org secrets_bucket environment='development' profi exit 1 fi + # Default repo to github_org/service if not provided + if [[ -z "$REPO" ]]; then + REPO="${GITHUB_ORG}/${SERVICE}" + fi + # Set up AWS command helper run_aws() { if [[ "$AWS_PROFILE" != "default" && -n "$AWS_PROFILE" ]]; then @@ -540,6 +546,7 @@ aws-setup-oidc service github_org secrets_bucket environment='development' profi echo "=========================" echo " Service: $SERVICE" echo " GitHub Org: $GITHUB_ORG" + echo " GitHub Repo: $REPO" echo " Environment: $ENVIRONMENT" echo " AWS Account: $ACCOUNT_ID" echo " Role Name: $ROLE_NAME" @@ -565,28 +572,29 @@ aws-setup-oidc service github_org secrets_bucket environment='development' profi # 2. Create or update IAM Role (idempotent) echo "" echo "Processing IAM Role: $ROLE_NAME" + REPO_CONDITION="repo:${REPO}:*" if run_aws iam get-role --role-name "$ROLE_NAME" &>/dev/null; then echo "Role $ROLE_NAME already exists" - # Check if GitHub org already in trust policy + # Check if this specific repo already in trust policy TRUST_POLICY=$(run_aws iam get-role --role-name "$ROLE_NAME" --query 'Role.AssumeRolePolicyDocument' --output json) - if echo "$TRUST_POLICY" | grep -q "repo:${GITHUB_ORG}/"; then - echo "GitHub org '$GITHUB_ORG' already has access" + if echo "$TRUST_POLICY" | grep -q "repo:${REPO}:"; then + echo "Repo '$REPO' already has access in trust policy" else - echo "Adding GitHub org '$GITHUB_ORG' to trust policy..." - echo "$TRUST_POLICY" | jq --arg org "$GITHUB_ORG" ' + echo "Adding repo '$REPO' to trust policy..." + echo "$TRUST_POLICY" | jq --arg repo_cond "$REPO_CONDITION" ' (.Statement[0].Condition.StringLike["token.actions.githubusercontent.com:sub"]) |= - if type == "string" then [., "repo:\($org)/*:*"] - elif type == "array" then . + ["repo:\($org)/*:*"] - else "repo:\($org)/*:*" + if type == "string" then [., $repo_cond] + elif type == "array" then . + [$repo_cond] + else $repo_cond end ' > /tmp/oidc-trust-policy.json run_aws iam update-assume-role-policy --role-name "$ROLE_NAME" \ --policy-document file:///tmp/oidc-trust-policy.json rm -f /tmp/oidc-trust-policy.json - echo "Trust policy updated" + echo "Trust policy updated — added repo: $REPO" fi else - jq -n --arg account "$ACCOUNT_ID" --arg org "$GITHUB_ORG" '{ + jq -n --arg account "$ACCOUNT_ID" --arg repo_cond "$REPO_CONDITION" '{ Version: "2012-10-17", Statement: [{ Effect: "Allow", @@ -594,14 +602,14 @@ aws-setup-oidc service github_org secrets_bucket environment='development' profi Action: "sts:AssumeRoleWithWebIdentity", Condition: { StringEquals: { "token.actions.githubusercontent.com:aud": "sts.amazonaws.com" }, - StringLike: { "token.actions.githubusercontent.com:sub": "repo:\($org)/*:*" } + StringLike: { "token.actions.githubusercontent.com:sub": $repo_cond } } }] }' > /tmp/oidc-trust-policy.json run_aws iam create-role --role-name "$ROLE_NAME" \ --assume-role-policy-document file:///tmp/oidc-trust-policy.json rm -f /tmp/oidc-trust-policy.json - echo "IAM role created: $ROLE_NAME" + echo "IAM role created: $ROLE_NAME (trust scoped to repo: $REPO)" fi # 3. Create and attach deployment policy (idempotent) diff --git a/specs/oidc-per-project-isolation/assertions/trust-policy-repo-scoped.md b/specs/oidc-per-project-isolation/assertions/trust-policy-repo-scoped.md index c2d4460..cccc4e2 100644 --- a/specs/oidc-per-project-isolation/assertions/trust-policy-repo-scoped.md +++ b/specs/oidc-per-project-isolation/assertions/trust-policy-repo-scoped.md @@ -3,7 +3,7 @@ id: trust-policy-repo-scoped parent: oidc-per-project-isolation created: 2026-07-21T22:00:00Z priority: 2 -status: not_started +status: done branch: feature/aws-pipeline --- From a77ff54b375310645d8d9156185442ac8e597e2b Mon Sep 17 00:00:00 2001 From: Pari Work Temp Date: Tue, 21 Jul 2026 14:53:28 -0500 Subject: [PATCH 14/26] Fix role name mismatch in aws-setup-secrets aws-setup-secrets used `github-actions-${ENVIRONMENT}` but aws-setup-oidc creates `github-actions-${SERVICE}-${ENVIRONMENT}`. This caused "Invalid principal" when PutBucketPolicy tried to reference a non-existent role ARN. Co-Authored-By: Claude Opus 4.6 --- justfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/justfile b/justfile index e0c7a0b..76c0741 100644 --- a/justfile +++ b/justfile @@ -757,7 +757,7 @@ aws-setup-secrets service environment profile='default' region='us-east-1': fi # Create/update bucket policy for OIDC role access (idempotent) - ROLE_NAME="github-actions-${ENVIRONMENT}" + ROLE_NAME="github-actions-${SERVICE}-${ENVIRONMENT}" ROLE_ARN="arn:aws:iam::${AWS_ACCOUNT_ID}:role/${ROLE_NAME}" echo "" From 7b75b4b8ee368a5d72e30723f2a20447a16230fc Mon Sep 17 00:00:00 2001 From: Pari Work Temp Date: Tue, 21 Jul 2026 15:25:20 -0500 Subject: [PATCH 15/26] Remove stale ROLE_ARN GitHub variable suggestion from OIDC output Role ARNs go in .github/environments.json, not as separate per-environment GitHub Actions variables. The old pattern (DEVELOPMENT_AWS_ROLE_ARN, etc.) is no longer used by the deploy workflow. Co-Authored-By: Claude Opus 4.6 --- justfile | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/justfile b/justfile index 76c0741..6fce7c9 100644 --- a/justfile +++ b/justfile @@ -691,12 +691,10 @@ aws-setup-oidc service github_org secrets_bucket environment='development' profi echo "Setup complete!" echo " Role ARN: $ROLE_ARN" echo "" - echo "Copy this Role ARN into your environments.json:" - echo " $ROLE_ARN" + echo "Copy this Role ARN into your .github/environments.json:" + echo " \"role_arn\": \"$ROLE_ARN\"" echo "" - echo "Or set it as a GitHub Actions variable:" - echo " 1. In GitHub repo Settings > Secrets and variables > Actions > Variables" - echo " 2. Add: $(echo $ENVIRONMENT | tr '[:lower:]' '[:upper:]')_AWS_ROLE_ARN = $ROLE_ARN" + echo "Set it under the '$ENVIRONMENT' key alongside account_id, secrets_bucket, and region." # Create S3 bucket for secrets storage with proper security [group('aws-terraform')] aws-setup-secrets service environment profile='default' region='us-east-1': From ab709d1d9a355753aaf7c9f1cee2b3647cbda25b Mon Sep 17 00:00:00 2001 From: Pari Work Temp Date: Tue, 21 Jul 2026 17:39:46 -0500 Subject: [PATCH 16/26] Fix OIDC trust policy for GitHub's new sub claim format GitHub OIDC tokens now include numeric IDs in the sub claim: repo:org@12345/repo@67890:ref:refs/heads/main Use wildcards (org*/repo*) to match both old and new formats. Co-Authored-By: Claude Opus 4.6 --- justfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/justfile b/justfile index 6fce7c9..26db58d 100644 --- a/justfile +++ b/justfile @@ -572,7 +572,9 @@ aws-setup-oidc service github_org secrets_bucket environment='development' profi # 2. Create or update IAM Role (idempotent) echo "" echo "Processing IAM Role: $ROLE_NAME" - REPO_CONDITION="repo:${REPO}:*" + # GitHub OIDC sub claims now include numeric IDs: org@ID/repo@ID + # Use wildcards after org and repo names to match both old and new formats + REPO_CONDITION="repo:${GITHUB_ORG}*/${SERVICE}*:*" if run_aws iam get-role --role-name "$ROLE_NAME" &>/dev/null; then echo "Role $ROLE_NAME already exists" # Check if this specific repo already in trust policy From 3ebbda09eadb43bff3026b09aec21d2ac5498b44 Mon Sep 17 00:00:00 2001 From: Pari Work Temp Date: Wed, 22 Jul 2026 11:02:38 -0500 Subject: [PATCH 17/26] Fix heredoc indentation in aws-tf-init-backend recipe Replace heredoc with printf to avoid inconsistent leading whitespace that breaks just's recipe parser. Co-Authored-By: Claude Opus 4.6 --- justfile | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/justfile b/justfile index 26db58d..a9d3ca5 100644 --- a/justfile +++ b/justfile @@ -477,6 +477,17 @@ aws-tf-init-backend service environment='development' profile='default' region=' exit 1 fi + # Generate backend.hcl for CI (idempotent — same values for all environments) + BACKEND_HCL="terraform/backend.hcl" + if [[ ! -f "$BACKEND_HCL" ]]; then + echo "Generating $BACKEND_HCL for CI..." + printf 'bucket = "%s"\nregion = "%s"\ndynamodb_table = "%s"\nencrypt = true\n' \ + "$BUCKET" "$AWS_REGION" "$TABLE" > "$BACKEND_HCL" + echo "Created $BACKEND_HCL — commit this file to your repo" + else + echo "Backend config already exists: $BACKEND_HCL" + fi + # Build backend config args BACKEND_ARGS="-backend-config=\"bucket=${BUCKET}\"" BACKEND_ARGS+=" -backend-config=\"key=${STATE_KEY}\"" From 7a5389ff126f95b0103a0f1513c88ddf0a0dcdac Mon Sep 17 00:00:00 2001 From: Pari Work Temp Date: Thu, 30 Jul 2026 16:56:13 -0500 Subject: [PATCH 18/26] update to add log viewer --- justfile | 113 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 113 insertions(+) diff --git a/justfile b/justfile index a9d3ca5..0285ae0 100644 --- a/justfile +++ b/justfile @@ -708,6 +708,119 @@ aws-setup-oidc service github_org secrets_bucket environment='development' profi echo " \"role_arn\": \"$ROLE_ARN\"" echo "" echo "Set it under the '$ENVIRONMENT' key alongside account_id, secrets_bucket, and region." +# Create IAM group + read-only CloudWatch Logs policy for a project. +# Lets team members view logs without full AWS admin access. +[group('aws-terraform')] +aws-setup-log-viewer service profile='default' region='us-east-1': + #!/usr/bin/env bash + set -e + + SERVICE="{{service}}" + AWS_PROFILE="{{profile}}" + AWS_REGION="{{region}}" + + if [[ -z "$SERVICE" ]]; then + echo "Error: service is required (e.g., 'my-project')" + exit 1 + fi + + # Set up AWS command helper + run_aws() { + if [[ "$AWS_PROFILE" != "default" && -n "$AWS_PROFILE" ]]; then + aws --profile "$AWS_PROFILE" --region "$AWS_REGION" "$@" + else + aws --region "$AWS_REGION" "$@" + fi + } + + ACCOUNT_ID=$(run_aws sts get-caller-identity --query Account --output text) + POLICY_NAME="${SERVICE}-log-viewer" + POLICY_ARN="arn:aws:iam::${ACCOUNT_ID}:policy/${POLICY_NAME}" + GROUP_NAME="${SERVICE}-log-viewers" + + echo "CloudWatch Log Viewer Setup" + echo "===========================" + echo " Service: $SERVICE" + echo " AWS Account: $ACCOUNT_ID" + echo " AWS Region: $AWS_REGION" + echo " Policy: $POLICY_NAME" + echo " Group: $GROUP_NAME" + echo " Log Scope: /ecs/${SERVICE}/*" + echo "" + + # 1. Create or update IAM policy (idempotent — delete + recreate) + echo "Creating log viewer policy..." + LOG_GROUP_ARN="arn:aws:logs:${AWS_REGION}:${ACCOUNT_ID}:log-group:/ecs/${SERVICE}/*" + + jq -n --arg lg_arn "$LOG_GROUP_ARN" '{ + Version: "2012-10-17", + Statement: [ + { + Sid: "DescribeLogGroups", + Effect: "Allow", + Action: ["logs:DescribeLogGroups"], + Resource: "*" + }, + { + Sid: "ReadLogStreams", + Effect: "Allow", + Action: [ + "logs:DescribeLogStreams", + "logs:GetLogEvents", + "logs:FilterLogEvents", + "logs:StartQuery", + "logs:GetQueryResults", + "logs:StopQuery" + ], + Resource: $lg_arn + } + ] + }' > /tmp/log-viewer-policy.json + + if run_aws iam get-policy --policy-arn "$POLICY_ARN" &>/dev/null; then + echo "Policy already exists, updating..." + # Detach from group before deleting + run_aws iam detach-group-policy --group-name "$GROUP_NAME" --policy-arn "$POLICY_ARN" 2>/dev/null || true + # Clean up non-default versions + run_aws iam list-policy-versions --policy-arn "$POLICY_ARN" \ + --query 'Versions[?!IsDefaultVersion].[VersionId]' --output text | while read version; do + run_aws iam delete-policy-version --policy-arn "$POLICY_ARN" --version-id "$version" 2>/dev/null || true + done + run_aws iam delete-policy --policy-arn "$POLICY_ARN" 2>/dev/null || true + fi + + run_aws iam create-policy --policy-name "$POLICY_NAME" \ + --policy-document file:///tmp/log-viewer-policy.json --output text --query 'Policy.Arn' + rm -f /tmp/log-viewer-policy.json + echo "Policy created: $POLICY_NAME" + + # 2. Create IAM group (idempotent) + echo "" + echo "Creating log viewer group..." + if run_aws iam get-group --group-name "$GROUP_NAME" &>/dev/null; then + echo "Group already exists: $GROUP_NAME" + else + run_aws iam create-group --group-name "$GROUP_NAME" + echo "Group created: $GROUP_NAME" + fi + + # 3. Attach policy to group + run_aws iam attach-group-policy --group-name "$GROUP_NAME" --policy-arn "$POLICY_ARN" + echo "Policy attached to group" + + # Summary + echo "" + echo "Setup complete!" + echo "" + echo "Add a user to the group:" + echo " aws iam add-user-to-group --group-name $GROUP_NAME --user-name " + echo "" + echo "Remove a user from the group:" + echo " aws iam remove-user-from-group --group-name $GROUP_NAME --user-name " + echo "" + echo "Users in this group can view CloudWatch logs under /ecs/${SERVICE}/*" + echo "using the AWS Console or the tn CLI:" + echo " tn aws-stream-logs $SERVICE " # Create S3 bucket for secrets storage with proper security [group('aws-terraform')] aws-setup-secrets service environment profile='default' region='us-east-1': From 255460700b86a2b7596955a0720973529d646cba Mon Sep 17 00:00:00 2001 From: Pari Work Temp Date: Fri, 31 Jul 2026 17:51:11 -0500 Subject: [PATCH 19/26] fix cd --- justfile | 1 + 1 file changed, 1 insertion(+) diff --git a/justfile b/justfile index 0285ae0..ab9dda3 100644 --- a/justfile +++ b/justfile @@ -505,6 +505,7 @@ aws-tf-init-backend service environment='development' profile='default' region=' echo "" echo "Running terraform init..." + cd terraform eval "terraform init ${BACKEND_ARGS}" echo "" From b7a7ccd45f84c976924619c60c37d5ce3e0b539b Mon Sep 17 00:00:00 2001 From: Pari Work Temp Date: Fri, 31 Jul 2026 18:07:53 -0500 Subject: [PATCH 20/26] Fix state key duplication and add aws-tf-cleanup recipe - Use static key in aws-tf-init-backend (workspaces handle isolation) - Add aws-tf-cleanup: dry-run-first recipe to delete orphaned AWS resources from failed Terraform applies by naming convention Co-Authored-By: Claude Opus 4.6 --- justfile | 181 ++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 180 insertions(+), 1 deletion(-) diff --git a/justfile b/justfile index ab9dda3..a46b6fb 100644 --- a/justfile +++ b/justfile @@ -445,7 +445,9 @@ aws-tf-init-backend service environment='development' profile='default' region=' # Derive backend resource names (must match setup-backend convention) BUCKET="${AWS_ACCOUNT_ID}-${SERVICE}-terraform-state" TABLE="${SERVICE}-terraform-state-lock" - STATE_KEY="${ENVIRONMENT}/terraform.tfstate" + # Use a static key — workspaces handle environment isolation + # (Terraform stores state at env://terraform.tfstate) + STATE_KEY="terraform.tfstate" echo "Terraform Backend Initialization" echo "==================================" @@ -513,6 +515,183 @@ aws-tf-init-backend service environment='development' profile='default' region=' echo " State Location: s3://${BUCKET}/${STATE_KEY}" echo " Lock Table: ${TABLE}" echo " Environment: ${ENVIRONMENT}" +# Delete orphaned AWS resources from a failed Terraform apply. +# Finds resources by naming convention ({service}-{environment}) and deletes them. +# Safe for fresh environments with no user data. +[group('aws-terraform')] +aws-tf-cleanup service environment profile='default' region='us-east-1' dry_run='true': + #!/usr/bin/env bash + set -e + + SERVICE="{{service}}" + ENVIRONMENT="{{environment}}" + AWS_PROFILE="{{profile}}" + AWS_REGION="{{region}}" + DRY_RUN="{{dry_run}}" + + run_aws() { + if [[ "$AWS_PROFILE" != "default" && -n "$AWS_PROFILE" ]]; then + aws --profile "$AWS_PROFILE" --region "$AWS_REGION" "$@" + else + aws --region "$AWS_REGION" "$@" + fi + } + + if [[ "$DRY_RUN" == "true" ]]; then + echo "🔍 DRY RUN — showing what would be deleted (pass dry_run=false to delete)" + else + echo "⚠️ DESTRUCTIVE — deleting orphaned resources for ${SERVICE}-${ENVIRONMENT}" + fi + echo "" + + FOUND=0 + + # 1. ECS services and cluster + CLUSTER="cluster-${SERVICE}-${ENVIRONMENT}" + if run_aws ecs describe-clusters --clusters "$CLUSTER" --query 'clusters[?status==`ACTIVE`].clusterName' --output text 2>/dev/null | grep -q "$CLUSTER"; then + FOUND=$((FOUND+1)) + echo "📦 ECS cluster: $CLUSTER" + # Stop services first + for SVC in $(run_aws ecs list-services --cluster "$CLUSTER" --query 'serviceArns[]' --output text 2>/dev/null); do + SVC_NAME=$(basename "$SVC") + echo " Service: $SVC_NAME" + if [[ "$DRY_RUN" == "false" ]]; then + run_aws ecs update-service --cluster "$CLUSTER" --service "$SVC_NAME" --desired-count 0 >/dev/null 2>&1 || true + run_aws ecs delete-service --cluster "$CLUSTER" --service "$SVC_NAME" --force >/dev/null 2>&1 || true + echo " ✅ Deleted service: $SVC_NAME" + fi + done + if [[ "$DRY_RUN" == "false" ]]; then + run_aws ecs delete-cluster --cluster "$CLUSTER" >/dev/null 2>&1 || true + echo " ✅ Deleted cluster: $CLUSTER" + fi + fi + + # 2. ALB and listeners + ALB_ARN=$(run_aws elbv2 describe-load-balancers --query "LoadBalancers[?starts_with(LoadBalancerName, '${SERVICE}-${ENVIRONMENT}')].LoadBalancerArn" --output text 2>/dev/null || echo "") + if [[ -n "$ALB_ARN" && "$ALB_ARN" != "None" ]]; then + FOUND=$((FOUND+1)) + ALB_NAME=$(run_aws elbv2 describe-load-balancers --load-balancer-arns "$ALB_ARN" --query 'LoadBalancers[0].LoadBalancerName' --output text) + echo "⚖️ ALB: $ALB_NAME" + if [[ "$DRY_RUN" == "false" ]]; then + # Delete listeners first + for LISTENER in $(run_aws elbv2 describe-listeners --load-balancer-arn "$ALB_ARN" --query 'Listeners[].ListenerArn' --output text 2>/dev/null); do + run_aws elbv2 delete-listener --listener-arn "$LISTENER" >/dev/null 2>&1 || true + done + run_aws elbv2 delete-load-balancer --load-balancer-arn "$ALB_ARN" >/dev/null 2>&1 || true + echo " ✅ Deleted ALB: $ALB_NAME" + fi + fi + + # 3. Target groups + for TG_ARN in $(run_aws elbv2 describe-target-groups --query "TargetGroups[?starts_with(TargetGroupName, 'http-${SERVICE}-${ENVIRONMENT}')].TargetGroupArn" --output text 2>/dev/null); do + if [[ -n "$TG_ARN" && "$TG_ARN" != "None" ]]; then + FOUND=$((FOUND+1)) + TG_NAME=$(run_aws elbv2 describe-target-groups --target-group-arns "$TG_ARN" --query 'TargetGroups[0].TargetGroupName' --output text) + echo "🎯 Target group: $TG_NAME" + if [[ "$DRY_RUN" == "false" ]]; then + run_aws elbv2 delete-target-group --target-group-arn "$TG_ARN" >/dev/null 2>&1 || true + echo " ✅ Deleted target group: $TG_NAME" + fi + fi + done + + # 4. RDS instance + DB_ID="db-${SERVICE}-${ENVIRONMENT}" + if run_aws rds describe-db-instances --db-instance-identifier "$DB_ID" >/dev/null 2>&1; then + FOUND=$((FOUND+1)) + echo "🗄️ RDS instance: $DB_ID" + if [[ "$DRY_RUN" == "false" ]]; then + run_aws rds delete-db-instance --db-instance-identifier "$DB_ID" --skip-final-snapshot --delete-automated-backups >/dev/null 2>&1 || true + echo " ⏳ Deleting RDS (takes several minutes)..." + fi + fi + + # 5. ElastiCache cluster + REDIS_ID="redis-${SERVICE}-${ENVIRONMENT}" + if run_aws elasticache describe-cache-clusters --cache-cluster-id "$REDIS_ID" >/dev/null 2>&1; then + FOUND=$((FOUND+1)) + echo "🔴 ElastiCache cluster: $REDIS_ID" + if [[ "$DRY_RUN" == "false" ]]; then + run_aws elasticache delete-cache-cluster --cache-cluster-id "$REDIS_ID" >/dev/null 2>&1 || true + echo " ⏳ Deleting ElastiCache (takes several minutes)..." + fi + fi + + # 6. Security groups (delete last — other resources reference them) + VPC_ID=$(run_aws ec2 describe-vpcs --filters "Name=tag:Name,Values=*shared*dev*" --query 'Vpcs[0].VpcId' --output text 2>/dev/null || echo "") + if [[ -n "$VPC_ID" && "$VPC_ID" != "None" ]]; then + SG_PATTERNS=("ecs-lb-sg-${SERVICE}-${ENVIRONMENT}" "ecs-app-${SERVICE}-${ENVIRONMENT}" "rds-sg-${SERVICE}-${ENVIRONMENT}" "redis-sg-${SERVICE}-${ENVIRONMENT}") + for SG_NAME in "${SG_PATTERNS[@]}"; do + SG_ID=$(run_aws ec2 describe-security-groups --filters "Name=group-name,Values=${SG_NAME}" "Name=vpc-id,Values=${VPC_ID}" --query 'SecurityGroups[0].GroupId' --output text 2>/dev/null || echo "") + if [[ -n "$SG_ID" && "$SG_ID" != "None" ]]; then + FOUND=$((FOUND+1)) + echo "🔒 Security group: $SG_NAME ($SG_ID)" + if [[ "$DRY_RUN" == "false" ]]; then + run_aws ec2 delete-security-group --group-id "$SG_ID" >/dev/null 2>&1 || echo " ⚠️ Could not delete $SG_NAME (may have dependencies — retry after RDS/ElastiCache finish deleting)" + echo " ✅ Deleted security group: $SG_NAME" + fi + fi + done + fi + + # 7. DB subnet group + DB_SUBNET="db-subnet-${SERVICE}-${ENVIRONMENT}" + if run_aws rds describe-db-subnet-groups --db-subnet-group-name "$DB_SUBNET" >/dev/null 2>&1; then + FOUND=$((FOUND+1)) + echo "🌐 DB subnet group: $DB_SUBNET" + if [[ "$DRY_RUN" == "false" ]]; then + run_aws rds delete-db-subnet-group --db-subnet-group-name "$DB_SUBNET" >/dev/null 2>&1 || echo " ⚠️ Could not delete (RDS may still be deleting)" + fi + fi + + # 8. ElastiCache subnet group + REDIS_SUBNET="redis-subnet-${SERVICE}-${ENVIRONMENT}" + if run_aws elasticache describe-cache-subnet-groups --cache-subnet-group-name "$REDIS_SUBNET" >/dev/null 2>&1; then + FOUND=$((FOUND+1)) + echo "🌐 ElastiCache subnet group: $REDIS_SUBNET" + if [[ "$DRY_RUN" == "false" ]]; then + run_aws elasticache delete-cache-subnet-group --cache-subnet-group-name "$REDIS_SUBNET" >/dev/null 2>&1 || echo " ⚠️ Could not delete (ElastiCache may still be deleting)" + fi + fi + + # 9. CloudWatch log group + LOG_GROUP="/ecs/${SERVICE}/${ENVIRONMENT}" + if run_aws logs describe-log-groups --log-group-name-prefix "$LOG_GROUP" --query "logGroups[?logGroupName=='${LOG_GROUP}'].logGroupName" --output text 2>/dev/null | grep -q "$LOG_GROUP"; then + FOUND=$((FOUND+1)) + echo "📝 Log group: $LOG_GROUP" + if [[ "$DRY_RUN" == "false" ]]; then + run_aws logs delete-log-group --log-group-name "$LOG_GROUP" >/dev/null 2>&1 || true + echo " ✅ Deleted log group" + fi + fi + + # 10. Secrets Manager secrets + for SECRET_NAME in $(run_aws secretsmanager list-secrets --filters "Key=name,Values=${SERVICE}-${ENVIRONMENT}" --query 'SecretList[].Name' --output text 2>/dev/null); do + if [[ -n "$SECRET_NAME" && "$SECRET_NAME" != "None" ]]; then + FOUND=$((FOUND+1)) + echo "🔑 Secret: $SECRET_NAME" + if [[ "$DRY_RUN" == "false" ]]; then + run_aws secretsmanager delete-secret --secret-id "$SECRET_NAME" --force-delete-without-recovery >/dev/null 2>&1 || true + echo " ✅ Deleted secret: $SECRET_NAME" + fi + fi + done + + echo "" + if [[ $FOUND -eq 0 ]]; then + echo "✅ No orphaned resources found for ${SERVICE}-${ENVIRONMENT}" + elif [[ "$DRY_RUN" == "true" ]]; then + echo "Found $FOUND orphaned resource(s). Run with dry_run=false to delete:" + echo " tn aws-tf-cleanup ${SERVICE} ${ENVIRONMENT} dry_run=false" + echo "" + echo "Note: Security groups may fail on first attempt if RDS/ElastiCache" + echo "are still deleting. Re-run after a few minutes to clean those up." + else + echo "🧹 Cleanup initiated for $FOUND resource(s)" + echo "RDS and ElastiCache take several minutes to fully delete." + echo "Re-run to clean up security groups and subnet groups after they finish." + fi # Create GitHub Actions OIDC IAM role for a given org and environment. # Can also be re-run to update an existing OIDC role with a new secrets bucket for a new project. [group('aws-terraform')] From cfd273718f99bfd6cbf43084b36e7f249ca53566 Mon Sep 17 00:00:00 2001 From: Pari Work Temp Date: Fri, 31 Jul 2026 20:08:44 -0500 Subject: [PATCH 21/26] Add SNS permissions to OIDC deployment policy Required for CloudWatch alarm SNS topics created by the alarms module. Co-Authored-By: Claude Opus 4.6 --- justfile | 1 + 1 file changed, 1 insertion(+) diff --git a/justfile b/justfile index a46b6fb..72d7967 100644 --- a/justfile +++ b/justfile @@ -828,6 +828,7 @@ aws-setup-oidc service github_org secrets_bucket environment='development' profi {Sid: "ElastiCacheAccess", Effect: "Allow", Action: ["elasticache:*"], Resource: "*"}, {Sid: "Route53Access", Effect: "Allow", Action: ["route53:*"], Resource: "*"}, {Sid: "EventBridgeAccess", Effect: "Allow", Action: ["events:*"], Resource: "*"}, + {Sid: "SNSAccess", Effect: "Allow", Action: ["sns:*"], Resource: "*"}, {Sid: "STSAccess", Effect: "Allow", Action: ["sts:GetCallerIdentity"], Resource: "*"} ] }' > /tmp/oidc-deploy-policy.json From b0857f3b3268e0dd0dd108533a65c1e253cef147 Mon Sep 17 00:00:00 2001 From: Pari Work Temp Date: Fri, 31 Jul 2026 20:22:29 -0500 Subject: [PATCH 22/26] Add cloudwatch:* and simplify EC2 policy to ec2:* wildcard The OIDC deployment policy was missing cloudwatch:* (needed for aws_cloudwatch_metric_alarm resources), and the EC2 policy was using a long action list that missed newer VPC security group rule API actions. Switching both to wildcards matches the pattern used by all other service policies. Co-Authored-By: Claude Opus 4.6 --- justfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/justfile b/justfile index 72d7967..29cd2ed 100644 --- a/justfile +++ b/justfile @@ -816,7 +816,7 @@ aws-setup-oidc service github_org secrets_bucket environment='development' profi Statement: [ {Sid: "ECRFullAccess", Effect: "Allow", Action: ["ecr:*"], Resource: "*"}, {Sid: "ECSFullAccess", Effect: "Allow", Action: ["ecs:*"], Resource: "*"}, - {Sid: "VPCAccess", Effect: "Allow", Action: ["ec2:Describe*","ec2:CreateVpc","ec2:DeleteVpc","ec2:ModifyVpcAttribute","ec2:CreateSubnet","ec2:DeleteSubnet","ec2:ModifySubnetAttribute","ec2:CreateInternetGateway","ec2:DeleteInternetGateway","ec2:AttachInternetGateway","ec2:DetachInternetGateway","ec2:CreateRouteTable","ec2:DeleteRouteTable","ec2:CreateRoute","ec2:DeleteRoute","ec2:AssociateRouteTable","ec2:DisassociateRouteTable","ec2:CreateSecurityGroup","ec2:DeleteSecurityGroup","ec2:AuthorizeSecurityGroupIngress","ec2:AuthorizeSecurityGroupEgress","ec2:RevokeSecurityGroupIngress","ec2:RevokeSecurityGroupEgress","ec2:CreateTags","ec2:DeleteTags"], Resource: "*"}, + {Sid: "EC2Access", Effect: "Allow", Action: ["ec2:*"], Resource: "*"}, {Sid: "RDSAccess", Effect: "Allow", Action: ["rds:*"], Resource: "*"}, {Sid: "ACMAccess", Effect: "Allow", Action: ["acm:*"], Resource: "*"}, {Sid: "IAMAccess", Effect: "Allow", Action: ["iam:*"], Resource: "*"}, @@ -829,6 +829,7 @@ aws-setup-oidc service github_org secrets_bucket environment='development' profi {Sid: "Route53Access", Effect: "Allow", Action: ["route53:*"], Resource: "*"}, {Sid: "EventBridgeAccess", Effect: "Allow", Action: ["events:*"], Resource: "*"}, {Sid: "SNSAccess", Effect: "Allow", Action: ["sns:*"], Resource: "*"}, + {Sid: "CloudWatchAccess", Effect: "Allow", Action: ["cloudwatch:*"], Resource: "*"}, {Sid: "STSAccess", Effect: "Allow", Action: ["sts:GetCallerIdentity"], Resource: "*"} ] }' > /tmp/oidc-deploy-policy.json From 699d363f44e6fe7271221fa559b1095422462e3c Mon Sep 17 00:00:00 2001 From: Pari Work Temp Date: Wed, 5 Aug 2026 20:59:33 -0500 Subject: [PATCH 23/26] update hte changes to the justfile --- justfile | 2 +- tn-spa-bootstrapper.code-workspace | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 tn-spa-bootstrapper.code-workspace diff --git a/justfile b/justfile index 29cd2ed..f48dc3e 100644 --- a/justfile +++ b/justfile @@ -1289,7 +1289,7 @@ aws-stream-logs service environment='development' profile='default' region='us-e i=1 for stream in $STREAMS; do - if [[ "$stream" =~ server- ]]; then + if [[ "$stream" =~ (server|app)- ]]; then SERVER_STREAMS+=("$stream") echo " $i) [SERVER] $stream" elif [[ "$stream" =~ worker- ]]; then diff --git a/tn-spa-bootstrapper.code-workspace b/tn-spa-bootstrapper.code-workspace new file mode 100644 index 0000000..b484c22 --- /dev/null +++ b/tn-spa-bootstrapper.code-workspace @@ -0,0 +1,11 @@ +{ + "folders": [ + { + "path": "../tn-spa-bootstrapper" + }, + { + "path": "." + } + ], + "settings": {} +} \ No newline at end of file From 102028717af82c774e602b1136cb18619deb77a2 Mon Sep 17 00:00:00 2001 From: Pari Work Temp Date: Thu, 6 Aug 2026 12:58:20 -0500 Subject: [PATCH 24/26] Add aws-ecs-events diagnostics and aws-add-log-viewer commands aws-ecs-events: shows service events, status, stopped task reasons, and running task details in one command for debugging deployment issues. aws-add-log-viewer: creates IAM user, adds to log-viewer group, and generates access keys. Also fix stream categorization to match both server- and app- prefixes. Co-Authored-By: Claude Opus 4.6 --- justfile | 170 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 170 insertions(+) diff --git a/justfile b/justfile index f48dc3e..9688ed9 100644 --- a/justfile +++ b/justfile @@ -1003,6 +1003,80 @@ aws-setup-log-viewer service profile='default' region='us-east-1': echo "Users in this group can view CloudWatch logs under /ecs/${SERVICE}/*" echo "using the AWS Console or the tn CLI:" echo " tn aws-stream-logs $SERVICE " +# Add an IAM user to a project's log-viewer group (creates the user if needed) +[group('aws-terraform')] +aws-add-log-viewer service username profile='default' region='us-east-1': + #!/usr/bin/env bash + set -e + + SERVICE="{{service}}" + USERNAME="{{username}}" + AWS_PROFILE="{{profile}}" + AWS_REGION="{{region}}" + + if [[ -z "$SERVICE" || -z "$USERNAME" ]]; then + echo "Error: service and username are required" + echo "Usage: tn aws-add-log-viewer " + exit 1 + fi + + run_aws() { + if [[ "$AWS_PROFILE" != "default" && -n "$AWS_PROFILE" ]]; then + aws --profile "$AWS_PROFILE" --region "$AWS_REGION" "$@" + else + aws --region "$AWS_REGION" "$@" + fi + } + + GROUP_NAME="${SERVICE}-log-viewers" + + # Verify the group exists + if ! run_aws iam get-group --group-name "$GROUP_NAME" &>/dev/null; then + echo "Error: Group '$GROUP_NAME' does not exist." + echo "Run 'tn aws-setup-log-viewer $SERVICE' first." + exit 1 + fi + + # Create IAM user if it doesn't exist + if run_aws iam get-user --user-name "$USERNAME" &>/dev/null; then + echo "User '$USERNAME' already exists" + else + echo "Creating IAM user '$USERNAME'..." + run_aws iam create-user --user-name "$USERNAME" --output text --query 'User.Arn' + echo "✅ User created" + fi + + # Add to group + echo "Adding '$USERNAME' to group '$GROUP_NAME'..." + run_aws iam add-user-to-group --group-name "$GROUP_NAME" --user-name "$USERNAME" + echo "✅ Added to group" + + # Generate access keys + echo "" + echo "Generating access keys..." + KEYS=$(run_aws iam create-access-key --user-name "$USERNAME" --output json) + + ACCESS_KEY=$(echo "$KEYS" | jq -r '.AccessKey.AccessKeyId') + SECRET_KEY=$(echo "$KEYS" | jq -r '.AccessKey.SecretAccessKey') + + echo "" + echo "=============================================" + echo " Credentials for $USERNAME" + echo "=============================================" + echo "" + echo "Add this to ~/.aws/credentials:" + echo "" + echo " [$SERVICE]" + echo " aws_access_key_id = $ACCESS_KEY" + echo " aws_secret_access_key = $SECRET_KEY" + echo "" + echo "Then use:" + echo " tn aws-stream-logs $SERVICE $SERVICE $AWS_REGION" + echo " tn aws-ecs-events $SERVICE $SERVICE $AWS_REGION" + echo "" + echo "⚠️ Save these credentials now — the secret key cannot be retrieved again." + echo "=============================================" + # Create S3 bucket for secrets storage with proper security [group('aws-terraform')] aws-setup-secrets service environment profile='default' region='us-east-1': @@ -1387,6 +1461,102 @@ aws-stream-logs service environment='development' profile='default' region='us-e sleep 2 done +# Show ECS service events and stopped task diagnostics +[group('aws-terraform')] +aws-ecs-events service environment='development' profile='default' region='us-east-1' count='10': + #!/usr/bin/env bash + set -e + + SERVICE="{{service}}" + ENVIRONMENT="{{environment}}" + AWS_PROFILE="{{profile}}" + AWS_REGION="{{region}}" + COUNT="{{count}}" + + CLUSTER="cluster-${SERVICE}-${ENVIRONMENT}" + ECS_SERVICE="service-app-${SERVICE}-${ENVIRONMENT}" + + PROFILE_FLAG="" + if [[ "$AWS_PROFILE" != "default" ]]; then + PROFILE_FLAG="--profile $AWS_PROFILE" + fi + REGION_FLAG="--region $AWS_REGION" + + echo "" + echo "ECS Diagnostics" + echo "=============================================" + echo " Cluster: $CLUSTER" + echo " Service: $ECS_SERVICE" + echo "=============================================" + + # --- Service Events --- + echo "" + echo "📋 Recent Service Events (last $COUNT):" + echo "---------------------------------------------" + aws ecs describe-services \ + --cluster "$CLUSTER" \ + --services "$ECS_SERVICE" \ + --query "services[0].events[:${COUNT}].[createdAt,message]" \ + --output table \ + $PROFILE_FLAG $REGION_FLAG 2>/dev/null || echo " Could not fetch service events" + + # --- Service Status --- + echo "" + echo "📊 Service Status:" + echo "---------------------------------------------" + aws ecs describe-services \ + --cluster "$CLUSTER" \ + --services "$ECS_SERVICE" \ + --query 'services[0].{desiredCount:desiredCount,runningCount:runningCount,pendingCount:pendingCount,status:status,rolloutState:deployments[0].rolloutState,rolloutReason:deployments[0].rolloutStateReason}' \ + --output table \ + $PROFILE_FLAG $REGION_FLAG 2>/dev/null || echo " Could not fetch service status" + + # --- Stopped Tasks --- + echo "" + echo "🛑 Recently Stopped Tasks:" + echo "---------------------------------------------" + STOPPED_TASKS=$(aws ecs list-tasks \ + --cluster "$CLUSTER" \ + --service-name "$ECS_SERVICE" \ + --desired-status STOPPED \ + --query 'taskArns' \ + --output json \ + $PROFILE_FLAG $REGION_FLAG 2>/dev/null) + + if [[ "$STOPPED_TASKS" == "[]" || -z "$STOPPED_TASKS" ]]; then + echo " No recently stopped tasks" + else + aws ecs describe-tasks \ + --cluster "$CLUSTER" \ + --tasks $(echo "$STOPPED_TASKS" | jq -r '.[]') \ + --query 'tasks[].{taskArn:taskArn,stoppedReason:stoppedReason,stopCode:stopCode,lastStatus:lastStatus,exitCode:containers[0].exitCode,containerReason:containers[0].reason,createdAt:createdAt,stoppedAt:stoppedAt}' \ + --output table \ + $PROFILE_FLAG $REGION_FLAG 2>/dev/null || echo " Could not fetch stopped task details" + fi + + # --- Running Tasks --- + echo "" + echo "✅ Running Tasks:" + echo "---------------------------------------------" + RUNNING_TASKS=$(aws ecs list-tasks \ + --cluster "$CLUSTER" \ + --service-name "$ECS_SERVICE" \ + --desired-status RUNNING \ + --query 'taskArns' \ + --output json \ + $PROFILE_FLAG $REGION_FLAG 2>/dev/null) + + if [[ "$RUNNING_TASKS" == "[]" || -z "$RUNNING_TASKS" ]]; then + echo " No running tasks" + else + aws ecs describe-tasks \ + --cluster "$CLUSTER" \ + --tasks $(echo "$RUNNING_TASKS" | jq -r '.[]') \ + --query 'tasks[].{taskArn:taskArn,lastStatus:lastStatus,healthStatus:healthStatus,taskDefinition:taskDefinitionArn,createdAt:createdAt,ip:containers[0].networkInterfaces[0].privateIpv4Address}' \ + --output table \ + $PROFILE_FLAG $REGION_FLAG 2>/dev/null || echo " Could not fetch running task details" + fi + # # TN Models Helpers # From e8c2740cc5af4b138d2906c56dee04d84cbe3781 Mon Sep 17 00:00:00 2001 From: Pari Work Temp Date: Wed, 12 Aug 2026 09:58:41 -0500 Subject: [PATCH 25/26] Add aws-setup-env-access and aws-add-env-user recipes Environment-scoped access management: one IAM user per person with a single set of keys, permissions controlled via group membership. Creates log-viewers, ecs-operators, and secrets-readers groups per service+environment. Co-Authored-By: Claude Opus 4.6 --- justfile | 351 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 351 insertions(+) diff --git a/justfile b/justfile index 9688ed9..1742e68 100644 --- a/justfile +++ b/justfile @@ -1077,6 +1077,357 @@ aws-add-log-viewer service username profile='default' region='us-east-1': echo "⚠️ Save these credentials now — the secret key cannot be retrieved again." echo "=============================================" +# Set up environment-scoped access groups (logs, exec, secrets) for a service+environment +[group('aws-terraform')] +aws-setup-env-access service environment profile='default' region='us-east-1': + #!/usr/bin/env bash + set -e + + SERVICE="{{service}}" + ENVIRONMENT="{{environment}}" + AWS_PROFILE="{{profile}}" + AWS_REGION="{{region}}" + + if [[ -z "$SERVICE" || -z "$ENVIRONMENT" ]]; then + echo "Error: service and environment are required" + echo "Usage: tn aws-setup-env-access " + exit 1 + fi + + run_aws() { + if [[ "$AWS_PROFILE" != "default" && -n "$AWS_PROFILE" ]]; then + aws --profile "$AWS_PROFILE" --region "$AWS_REGION" "$@" + else + aws --region "$AWS_REGION" "$@" + fi + } + + ACCOUNT_ID=$(run_aws sts get-caller-identity --query Account --output text) + PREFIX="${SERVICE}-${ENVIRONMENT}" + + echo "Environment Access Setup" + echo "===========================" + echo " Service: $SERVICE" + echo " Environment: $ENVIRONMENT" + echo " AWS Account: $ACCOUNT_ID" + echo " AWS Region: $AWS_REGION" + echo "" + + # Helper: create or replace an IAM policy + upsert_policy() { + local POLICY_NAME="$1" + local POLICY_FILE="$2" + local POLICY_ARN="arn:aws:iam::${ACCOUNT_ID}:policy/${POLICY_NAME}" + + if run_aws iam get-policy --policy-arn "$POLICY_ARN" &>/dev/null; then + echo " Policy exists, updating..." + # Detach from all groups before deleting + ATTACHED_GROUPS=$(run_aws iam list-entities-for-policy --policy-arn "$POLICY_ARN" \ + --query 'PolicyGroups[].GroupName' --output text 2>/dev/null || echo "") + for grp in $ATTACHED_GROUPS; do + run_aws iam detach-group-policy --group-name "$grp" --policy-arn "$POLICY_ARN" 2>/dev/null || true + done + run_aws iam list-policy-versions --policy-arn "$POLICY_ARN" \ + --query 'Versions[?!IsDefaultVersion].[VersionId]' --output text | while read version; do + run_aws iam delete-policy-version --policy-arn "$POLICY_ARN" --version-id "$version" 2>/dev/null || true + done + run_aws iam delete-policy --policy-arn "$POLICY_ARN" 2>/dev/null || true + fi + + run_aws iam create-policy --policy-name "$POLICY_NAME" \ + --policy-document "file://${POLICY_FILE}" --output text --query 'Policy.Arn' + echo " Policy created: $POLICY_NAME" + } + + # Helper: create group and attach policy + setup_group() { + local GROUP_NAME="$1" + local POLICY_ARN="$2" + + if run_aws iam get-group --group-name "$GROUP_NAME" &>/dev/null; then + echo " Group already exists: $GROUP_NAME" + else + run_aws iam create-group --group-name "$GROUP_NAME" + echo " Group created: $GROUP_NAME" + fi + run_aws iam attach-group-policy --group-name "$GROUP_NAME" --policy-arn "$POLICY_ARN" + echo " Policy attached to $GROUP_NAME" + } + + # ------------------------------------------------------- + # 1. LOG VIEWERS — scoped to /ecs/{SERVICE}/{ENVIRONMENT} + # ------------------------------------------------------- + echo "--- Log Viewer Access ---" + LOG_POLICY_NAME="${PREFIX}-log-viewer" + LOG_GROUP_ARN="arn:aws:logs:${AWS_REGION}:${ACCOUNT_ID}:log-group:/ecs/${SERVICE}/${ENVIRONMENT}:*" + + jq -n --arg lg_arn "$LOG_GROUP_ARN" '{ + Version: "2012-10-17", + Statement: [ + { + Sid: "DescribeLogGroups", + Effect: "Allow", + Action: ["logs:DescribeLogGroups"], + Resource: "*" + }, + { + Sid: "ReadLogStreams", + Effect: "Allow", + Action: [ + "logs:DescribeLogStreams", + "logs:GetLogEvents", + "logs:FilterLogEvents", + "logs:StartQuery", + "logs:GetQueryResults", + "logs:StopQuery" + ], + Resource: $lg_arn + } + ] + }' > /tmp/${PREFIX}-log-policy.json + + upsert_policy "$LOG_POLICY_NAME" "/tmp/${PREFIX}-log-policy.json" + setup_group "${PREFIX}-log-viewers" "arn:aws:iam::${ACCOUNT_ID}:policy/${LOG_POLICY_NAME}" + rm -f /tmp/${PREFIX}-log-policy.json + echo "" + + # ------------------------------------------------------- + # 2. ECS OPERATORS — exec + events for cluster-{SERVICE}-{ENVIRONMENT} + # ------------------------------------------------------- + echo "--- ECS Exec Access ---" + ECS_POLICY_NAME="${PREFIX}-ecs-operator" + CLUSTER_ARN="arn:aws:ecs:${AWS_REGION}:${ACCOUNT_ID}:cluster/cluster-${SERVICE}-${ENVIRONMENT}" + SERVICE_ARN="arn:aws:ecs:${AWS_REGION}:${ACCOUNT_ID}:service/cluster-${SERVICE}-${ENVIRONMENT}/*" + TASK_ARN="arn:aws:ecs:${AWS_REGION}:${ACCOUNT_ID}:task/cluster-${SERVICE}-${ENVIRONMENT}/*" + + jq -n \ + --arg cluster "$CLUSTER_ARN" \ + --arg service "$SERVICE_ARN" \ + --arg task "$TASK_ARN" '{ + Version: "2012-10-17", + Statement: [ + { + Sid: "ECSDescribe", + Effect: "Allow", + Action: [ + "ecs:DescribeClusters", + "ecs:ListServices", + "ecs:DescribeServices", + "ecs:ListTasks", + "ecs:DescribeTasks" + ], + Resource: [$cluster, $service, $task] + }, + { + Sid: "ECSExec", + Effect: "Allow", + Action: ["ecs:ExecuteCommand"], + Resource: $task + }, + { + Sid: "ECSTaskDefinition", + Effect: "Allow", + Action: ["ecs:DescribeTaskDefinition"], + Resource: "*" + }, + { + Sid: "SSMSession", + Effect: "Allow", + Action: ["ssm:StartSession"], + Resource: "*", + Condition: { + StringEquals: { "aws:ResourceTag/aws:ecs:clusterName": ("cluster-" + ($cluster | split("/")[-1] | split("cluster-")[-1])) } + } + } + ] + }' > /tmp/${PREFIX}-ecs-policy.json + + # Fix the SSM condition — just use the cluster name directly + CLUSTER_NAME="cluster-${SERVICE}-${ENVIRONMENT}" + jq -n \ + --arg cluster "$CLUSTER_ARN" \ + --arg service "$SERVICE_ARN" \ + --arg task "$TASK_ARN" \ + --arg cluster_name "$CLUSTER_NAME" '{ + Version: "2012-10-17", + Statement: [ + { + Sid: "ECSDescribe", + Effect: "Allow", + Action: [ + "ecs:DescribeClusters", + "ecs:ListServices", + "ecs:DescribeServices", + "ecs:ListTasks", + "ecs:DescribeTasks" + ], + Resource: [$cluster, $service, $task] + }, + { + Sid: "ECSExec", + Effect: "Allow", + Action: ["ecs:ExecuteCommand"], + Resource: $task + }, + { + Sid: "ECSTaskDefinition", + Effect: "Allow", + Action: ["ecs:DescribeTaskDefinition"], + Resource: "*" + }, + { + Sid: "SSMSession", + Effect: "Allow", + Action: ["ssm:StartSession"], + Resource: "*" + } + ] + }' > /tmp/${PREFIX}-ecs-policy.json + + upsert_policy "$ECS_POLICY_NAME" "/tmp/${PREFIX}-ecs-policy.json" + setup_group "${PREFIX}-ecs-operators" "arn:aws:iam::${ACCOUNT_ID}:policy/${ECS_POLICY_NAME}" + rm -f /tmp/${PREFIX}-ecs-policy.json + echo "" + + # ------------------------------------------------------- + # 3. SECRETS READERS — S3 read for {SERVICE}-terraform-secrets/{ENVIRONMENT}/* + # ------------------------------------------------------- + echo "--- Secrets Read Access ---" + SECRETS_POLICY_NAME="${PREFIX}-secrets-reader" + SECRETS_BUCKET="${SERVICE}-terraform-secrets" + + jq -n \ + --arg bucket "$SECRETS_BUCKET" \ + --arg env "$ENVIRONMENT" '{ + Version: "2012-10-17", + Statement: [ + { + Sid: "SecretsRead", + Effect: "Allow", + Action: ["s3:GetObject"], + Resource: ("arn:aws:s3:::" + $bucket + "/" + $env + "/*") + }, + { + Sid: "SecretsList", + Effect: "Allow", + Action: ["s3:ListBucket"], + Resource: ("arn:aws:s3:::" + $bucket), + Condition: { StringLike: { "s3:prefix": ($env + "/*") } } + } + ] + }' > /tmp/${PREFIX}-secrets-policy.json + + upsert_policy "$SECRETS_POLICY_NAME" "/tmp/${PREFIX}-secrets-policy.json" + setup_group "${PREFIX}-secrets-readers" "arn:aws:iam::${ACCOUNT_ID}:policy/${SECRETS_POLICY_NAME}" + rm -f /tmp/${PREFIX}-secrets-policy.json + echo "" + + # Summary + echo "===========================" + echo "Setup complete for $SERVICE / $ENVIRONMENT" + echo "" + echo "Groups created:" + echo " ${PREFIX}-log-viewers → CloudWatch logs (/ecs/${SERVICE}/${ENVIRONMENT})" + echo " ${PREFIX}-ecs-operators → ECS exec & diagnostics (cluster-${SERVICE}-${ENVIRONMENT})" + echo " ${PREFIX}-secrets-readers → S3 secrets read (${SECRETS_BUCKET}/${ENVIRONMENT}/*)" + echo "" + echo "Next: add users with" + echo " tn aws-add-env-user $SERVICE $ENVIRONMENT " + +# Add an IAM user to a service+environment's access groups (creates the user + keys if needed) +[group('aws-terraform')] +aws-add-env-user service environment username profile='default' region='us-east-1': + #!/usr/bin/env bash + set -e + + SERVICE="{{service}}" + ENVIRONMENT="{{environment}}" + USERNAME="{{username}}" + AWS_PROFILE="{{profile}}" + AWS_REGION="{{region}}" + + if [[ -z "$SERVICE" || -z "$ENVIRONMENT" || -z "$USERNAME" ]]; then + echo "Error: service, environment, and username are required" + echo "Usage: tn aws-add-env-user " + exit 1 + fi + + run_aws() { + if [[ "$AWS_PROFILE" != "default" && -n "$AWS_PROFILE" ]]; then + aws --profile "$AWS_PROFILE" --region "$AWS_REGION" "$@" + else + aws --region "$AWS_REGION" "$@" + fi + } + + PREFIX="${SERVICE}-${ENVIRONMENT}" + GRP_LOGS="${PREFIX}-log-viewers" + GRP_ECS="${PREFIX}-ecs-operators" + GRP_SECRETS="${PREFIX}-secrets-readers" + + # Verify all groups exist + for grp in "$GRP_LOGS" "$GRP_ECS" "$GRP_SECRETS"; do + if ! run_aws iam get-group --group-name "$grp" --max-items 0 >/dev/null 2>&1; then + echo "Error: Group '$grp' does not exist." + echo "Run 'tn aws-setup-env-access $SERVICE $ENVIRONMENT' first." + exit 1 + fi + done + + # Create IAM user if it doesn't exist + CREATED_USER=false + if run_aws iam get-user --user-name "$USERNAME" >/dev/null 2>&1; then + echo "User '$USERNAME' already exists" + else + echo "Creating IAM user '$USERNAME'..." + run_aws iam create-user --user-name "$USERNAME" --output text --query 'User.Arn' + CREATED_USER=true + echo "User created" + fi + + # Add to all groups + for grp in "$GRP_LOGS" "$GRP_ECS" "$GRP_SECRETS"; do + echo "Adding '$USERNAME' to group '$grp'..." + run_aws iam add-user-to-group --group-name "$grp" --user-name "$USERNAME" + done + echo "Added to all ${ENVIRONMENT} groups" + + # Generate access keys only if user was just created + if [[ "$CREATED_USER" == "true" ]]; then + echo "" + echo "Generating access keys..." + KEYS=$(run_aws iam create-access-key --user-name "$USERNAME" --output json) + + ACCESS_KEY=$(echo "$KEYS" | jq -r '.AccessKey.AccessKeyId') + SECRET_KEY=$(echo "$KEYS" | jq -r '.AccessKey.SecretAccessKey') + + echo "" + echo "=============================================" + echo " Credentials for $USERNAME" + echo "=============================================" + echo "" + echo "Add this to ~/.aws/credentials:" + echo "" + echo " [$SERVICE]" + echo " aws_access_key_id = $ACCESS_KEY" + echo " aws_secret_access_key = $SECRET_KEY" + echo "" + echo "Save these credentials now — the secret key" + echo "cannot be retrieved again." + echo "=============================================" + else + echo "" + echo "User already existed — no new keys generated." + echo "Existing credentials for '$USERNAME' now have" + echo "${ENVIRONMENT} access via group membership." + fi + + echo "" + echo "User '$USERNAME' now has ${ENVIRONMENT} access:" + echo " Logs: tn aws-stream-logs $SERVICE $ENVIRONMENT $SERVICE $AWS_REGION" + echo " Events: tn aws-ecs-events $SERVICE $ENVIRONMENT $SERVICE $AWS_REGION" + echo " Exec: tn aws-ecs-exec $SERVICE $ENVIRONMENT $SERVICE $AWS_REGION" + # Create S3 bucket for secrets storage with proper security [group('aws-terraform')] aws-setup-secrets service environment profile='default' region='us-east-1': From aa91fc089394eb9ab3768129da91140dc3d89dc3 Mon Sep 17 00:00:00 2001 From: Pari Work Temp Date: Sat, 15 Aug 2026 13:27:35 -0500 Subject: [PATCH 26/26] Fix aws-stream-logs time filter and reduce log noise Compute start time upfront as epoch milliseconds instead of deferring date calculation to eval, which wasn't evaluating properly. Reorder params so duration comes before stream_type/filter. Filter out DisallowedHost/SuspiciousOperation Django noise from log output. Co-Authored-By: Claude Opus 4.6 --- justfile | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/justfile b/justfile index 1742e68..f222bdf 100644 --- a/justfile +++ b/justfile @@ -1657,7 +1657,7 @@ aws-ecs-exec service environment='development' profile='default' region='us-east $PROFILE_FLAG $REGION_FLAG # Stream CloudWatch logs from ECS services [group('aws-terraform')] -aws-stream-logs service environment='development' profile='default' region='us-east-1' stream_type='a' filter='' duration='5m': +aws-stream-logs service environment='development' profile='default' region='us-east-1' duration='5m' stream_type='a' filter='': #!/usr/bin/env bash set -e @@ -1750,14 +1750,19 @@ aws-stream-logs service environment='development' profile='default' region='us-e exit 1 fi - # Validate duration format - if [[ ! "$START_TIME" =~ ^[0-9]+[mh]$ ]]; then + # Validate duration format and compute start time in epoch milliseconds + if [[ "$START_TIME" =~ ^([0-9]+)m$ ]]; then + OFFSET_SECS=$(( ${BASH_REMATCH[1]} * 60 )) + elif [[ "$START_TIME" =~ ^([0-9]+)h$ ]]; then + OFFSET_SECS=$(( ${BASH_REMATCH[1]} * 3600 )) + else echo "Error: Invalid duration format. Use '30m' or '2h'" exit 1 fi + START_MS=$(( ($(date +%s) - OFFSET_SECS) * 1000 )) # Build filter command - FILTER_CMD="aws logs filter-log-events --log-group-name \"$LOG_GROUP\" --start-time \$(date -v-${START_TIME} +%s)000 $PROFILE_FLAG $REGION_FLAG" + FILTER_CMD="aws logs filter-log-events --log-group-name \"$LOG_GROUP\" --start-time $START_MS $PROFILE_FLAG $REGION_FLAG" if [[ -n "$FILTER_PATTERN" ]]; then FILTER_CMD="$FILTER_CMD --filter-pattern \"$FILTER_PATTERN\"" @@ -1798,6 +1803,10 @@ aws-stream-logs service environment='development' profile='default' region='us-e message=$(echo "$event" | jq -r '.message // empty') stream=$(echo "$event" | jq -r '.logStreamName // empty') if [[ -n "$timestamp" && -n "$message" ]]; then + # Skip common Django noise (DisallowedHost, SuspiciousOperation, phishing probes) + if echo "$message" | grep -qiE 'DisallowedHost|Invalid HTTP_HOST header|SuspiciousOperation'; then + continue + fi formatted_time=$(date -r "$((timestamp/1000))" '+%Y-%m-%d %H:%M:%S' 2>/dev/null || echo "$timestamp") stream_short=$(basename "$stream") echo "[$formatted_time] [$stream_short] $message"