Merged in feature/service-accounts-1 (pull request #227)
Support for service accounts with static credentials * feature complete * tests and docs * lint fix
This commit is contained in:
Jay Brown
+52
-20
@@ -25,6 +25,7 @@ includes:
|
||||
|
||||
tasks:
|
||||
fullsuite:
|
||||
desc: "Generate, build, lint, and run functional tests (incremental by default)"
|
||||
vars:
|
||||
INCREMENTAL: "{{.INCREMENTAL | default \"true\"}}"
|
||||
cmds:
|
||||
@@ -34,21 +35,31 @@ tasks:
|
||||
vars:
|
||||
INCREMENTAL: "{{.INCREMENTAL}}"
|
||||
fullsuite:ci:
|
||||
- task: fullsuite
|
||||
vars:
|
||||
INCREMENTAL: "false"
|
||||
desc: "Full CI run of fullsuite with INCREMENTAL=false (must pass clean)"
|
||||
cmds:
|
||||
- task: fullsuite
|
||||
vars:
|
||||
INCREMENTAL: "false"
|
||||
fullsuite:build:
|
||||
internal: true
|
||||
cmds:
|
||||
- task: build
|
||||
- task: lint
|
||||
fullsuite:graph: psrecord "task fullsuite" --plot out/fullsuite.png --interval 0.1 --include-children
|
||||
fullsuite:graph:
|
||||
desc: "Run fullsuite under psrecord and plot CPU/memory to out/fullsuite.png"
|
||||
cmds:
|
||||
- psrecord "task fullsuite" --plot out/fullsuite.png --interval 0.1 --include-children
|
||||
precommit:
|
||||
- task: lint
|
||||
desc: "Pre-commit checks (runs all linters) for CI/CD readiness"
|
||||
cmds:
|
||||
- task: lint
|
||||
deps:tidy:
|
||||
- go mod tidy
|
||||
- go mod vendor
|
||||
desc: "Run go mod tidy and refresh the vendor directory"
|
||||
cmds:
|
||||
- go mod tidy
|
||||
- go mod vendor
|
||||
build:
|
||||
desc: "Generate code and build the queryorchestration docker image"
|
||||
deps:
|
||||
- build:deps
|
||||
cmd:
|
||||
@@ -60,6 +71,7 @@ tasks:
|
||||
- task: deps:tidy
|
||||
- task: generate
|
||||
generate:
|
||||
desc: "Run all code generators (db, openapi, docs, go generate)"
|
||||
run: once
|
||||
cmds:
|
||||
- task: db:generate
|
||||
@@ -67,8 +79,12 @@ tasks:
|
||||
# NOTE: test:mocks:generate removed - mocks are deprecated
|
||||
- task: docs:generate
|
||||
- task: go:generate
|
||||
go:generate: go generate ./...
|
||||
go:generate:
|
||||
desc: "Run `go generate ./...` across the module"
|
||||
cmds:
|
||||
- go generate ./...
|
||||
lint:
|
||||
desc: "Run every linter (db, go, yaml, json, docker, openapi, compose)"
|
||||
cmds:
|
||||
- task: db:lint
|
||||
- task: go:lint
|
||||
@@ -78,11 +94,20 @@ tasks:
|
||||
- task: openapi:lint
|
||||
- task: compose:lint
|
||||
go:lint:
|
||||
- golangci-lint run {{.CLI_ARGS}}
|
||||
# - govulncheck ./...
|
||||
yaml:lint: yamllint . -s {{.CLI_ARGS}}
|
||||
json:lint: jsonlint devbox.json -q -s -i {{.CLI_ARGS}}
|
||||
desc: "Run golangci-lint over the module"
|
||||
cmds:
|
||||
- golangci-lint run {{.CLI_ARGS}}
|
||||
# - govulncheck ./...
|
||||
yaml:lint:
|
||||
desc: "Lint all YAML files in the repo with yamllint"
|
||||
cmds:
|
||||
- yamllint . -s {{.CLI_ARGS}}
|
||||
json:lint:
|
||||
desc: "Lint devbox.json with jsonlint"
|
||||
cmds:
|
||||
- jsonlint devbox.json -q -s -i {{.CLI_ARGS}}
|
||||
docs:generate:
|
||||
desc: "Regenerate Go package docs under cmd/ via gomarkdoc"
|
||||
run: once
|
||||
cmds:
|
||||
- gomarkdoc --template-file file=docs/templates/root.gotxt ./cmd/...
|
||||
@@ -90,15 +115,22 @@ tasks:
|
||||
desc: "Generate full system docs ($$)"
|
||||
cmds:
|
||||
- bash docs/ai.generated/prompts/regenerate_docs.sh docs/ai.generated/prompts/prompt.for.system.documentation.generation.md
|
||||
aws:login: aws sso login
|
||||
aws:login:
|
||||
desc: "Authenticate to AWS via SSO (`aws sso login`)"
|
||||
cmds:
|
||||
# Unset AWS_ENDPOINT_URL for this call only — LocalStack override in .env
|
||||
# otherwise redirects the SSO OIDC RegisterClient call to localhost:4566.
|
||||
- env -u AWS_ENDPOINT_URL aws sso login
|
||||
aws:textract:credentials:
|
||||
- |
|
||||
sed -i '/^AWS_REGION_TEXTRACT\|^AWS_ACCESS_KEY_ID_TEXTRACT\|^AWS_SECRET_ACCESS_KEY_TEXTRACT\|^AWS_SESSION_TOKEN_TEXTRACT/d' .env
|
||||
echo "AWS_REGION_TEXTRACT=$(aws configure get region --profile $AWS_PROFILE)" >> .env
|
||||
CREDENTIALS_JSON=$(aws configure export-credentials --profile $AWS_PROFILE)
|
||||
echo "AWS_ACCESS_KEY_ID_TEXTRACT=$(echo $CREDENTIALS_JSON | jq -r '.AccessKeyId')" >> .env
|
||||
echo "AWS_SECRET_ACCESS_KEY_TEXTRACT=$(echo $CREDENTIALS_JSON | jq -r '.SecretAccessKey')" >> .env
|
||||
echo "AWS_SESSION_TOKEN_TEXTRACT=$(echo $CREDENTIALS_JSON | jq -r '.SessionToken')" >> .env
|
||||
desc: "Refresh AWS_*_TEXTRACT credentials in .env from the current AWS profile"
|
||||
cmds:
|
||||
- |
|
||||
sed -i '/^AWS_REGION_TEXTRACT\|^AWS_ACCESS_KEY_ID_TEXTRACT\|^AWS_SECRET_ACCESS_KEY_TEXTRACT\|^AWS_SESSION_TOKEN_TEXTRACT/d' .env
|
||||
echo "AWS_REGION_TEXTRACT=$(aws configure get region --profile $AWS_PROFILE)" >> .env
|
||||
CREDENTIALS_JSON=$(aws configure export-credentials --profile $AWS_PROFILE)
|
||||
echo "AWS_ACCESS_KEY_ID_TEXTRACT=$(echo $CREDENTIALS_JSON | jq -r '.AccessKeyId')" >> .env
|
||||
echo "AWS_SECRET_ACCESS_KEY_TEXTRACT=$(echo $CREDENTIALS_JSON | jq -r '.SecretAccessKey')" >> .env
|
||||
echo "AWS_SESSION_TOKEN_TEXTRACT=$(echo $CREDENTIALS_JSON | jq -r '.SessionToken')" >> .env
|
||||
dev:queryAPI:
|
||||
desc: "Run queryAPI locally with version info"
|
||||
vars:
|
||||
|
||||
@@ -16,6 +16,7 @@ includes:
|
||||
|
||||
tasks:
|
||||
generate:
|
||||
desc: "Recreate the generate DB, apply migrations, and run sqlc generate"
|
||||
run: once
|
||||
cmds:
|
||||
- task: compose:up:generate
|
||||
@@ -24,6 +25,7 @@ tasks:
|
||||
- migrate -path {{.MIGRATIONS}} -database {{.DB_URI_GENERATE}} up
|
||||
- sqlc generate --file {{.CONFIG_FILE}}
|
||||
lint:
|
||||
desc: "Validate SQL queries with sqlc vet (and sqlcheck on migrations)"
|
||||
cmds:
|
||||
- task: compose:up:generate
|
||||
- sqlc vet --file {{.CONFIG_FILE}}
|
||||
@@ -35,6 +37,7 @@ tasks:
|
||||
fi
|
||||
done
|
||||
mig:create:
|
||||
desc: "Interactively scaffold a new SQL migration in internal/database/migrations"
|
||||
cmds:
|
||||
- |
|
||||
read -p "Migration Name: " name
|
||||
|
||||
@@ -16,9 +16,11 @@ vars:
|
||||
|
||||
tasks:
|
||||
lint:
|
||||
desc: "Lint the production Dockerfile with hadolint"
|
||||
cmds:
|
||||
- hadolint {{.DOCKERFILE}} -t none
|
||||
build:
|
||||
desc: "Build the production docker image tagged :latest and :<git-version>"
|
||||
run: once
|
||||
cmd: |
|
||||
# Build with caching enabled
|
||||
@@ -31,6 +33,7 @@ tasks:
|
||||
-t {{.IMAGE_NAME}}:{{.GIT_VERSION}} \
|
||||
-f {{.DOCKERFILE}} .
|
||||
build:debug:
|
||||
desc: "Build the :debug docker image that includes the Delve debugger"
|
||||
run: once
|
||||
cmd: |
|
||||
# Build debug image with Delve debugger
|
||||
|
||||
@@ -11,16 +11,19 @@ vars:
|
||||
|
||||
tasks:
|
||||
build:test:
|
||||
desc: "Build containers using compose.test.yaml"
|
||||
cmds:
|
||||
- task: build
|
||||
vars:
|
||||
COMPOSE_FILE: "{{.TEST_COMPOSE_FILE}}"
|
||||
build:generate:
|
||||
desc: "Build containers using compose.generate.yaml (used by code generation)"
|
||||
cmds:
|
||||
- task: build
|
||||
vars:
|
||||
COMPOSE_FILE: "{{.GENERATE_COMPOSE_FILE}}"
|
||||
build:
|
||||
desc: "Build containers for the local compose stack (default: compose.local.yaml)"
|
||||
vars:
|
||||
COMPOSE_FILE: "{{.COMPOSE_FILE | default .LOCAL_COMPOSE_FILE}}"
|
||||
cmds:
|
||||
@@ -39,16 +42,19 @@ tasks:
|
||||
cmds:
|
||||
- docker compose -f {{.COMPOSE_FILE}} up localstack db prometheus -d --wait
|
||||
up:test:
|
||||
desc: "Bring up the test compose stack in the background"
|
||||
cmds:
|
||||
- task: up:cmd
|
||||
vars:
|
||||
COMPOSE_FILE: "{{.TEST_COMPOSE_FILE}}"
|
||||
up:generate:
|
||||
desc: "Bring up the generate compose stack (postgres used by db:generate)"
|
||||
run: once
|
||||
cmd: |
|
||||
docker compose -f {{.GENERATE_COMPOSE_FILE}} up \
|
||||
--wait -d
|
||||
up:aws:
|
||||
desc: "Build and start the local AWS-services compose stack (localstack-backed)"
|
||||
cmds:
|
||||
- task deps:tidy
|
||||
- task docker:build
|
||||
@@ -58,6 +64,7 @@ tasks:
|
||||
COMPOSE_FILE: "{{.AWS_COMPOSE_FILE}}"
|
||||
- task: init
|
||||
up:
|
||||
desc: "Tidy deps, build images, and bring up the full local compose stack"
|
||||
cmds:
|
||||
- task deps:tidy
|
||||
- task docker:build
|
||||
@@ -66,6 +73,7 @@ tasks:
|
||||
- task: init
|
||||
- task: up:cmd
|
||||
refresh:
|
||||
desc: "Rebuild images and recycle the local stack (down then up:infra + services)"
|
||||
cmds:
|
||||
- task deps:tidy
|
||||
- task docker:build
|
||||
@@ -98,31 +106,37 @@ tasks:
|
||||
}]
|
||||
}'
|
||||
down:test:
|
||||
desc: "Stop containers from the test compose stack"
|
||||
cmds:
|
||||
- task: down
|
||||
vars:
|
||||
COMPOSE_FILE: "{{.TEST_COMPOSE_FILE}}"
|
||||
down:aws:
|
||||
desc: "Stop containers from the AWS-services compose stack"
|
||||
cmds:
|
||||
- task: down
|
||||
vars:
|
||||
COMPOSE_FILE: "{{.AWS_COMPOSE_FILE}}"
|
||||
down:
|
||||
desc: "Stop containers for the local compose stack (run before fullsuite:ci)"
|
||||
vars:
|
||||
COMPOSE_FILE: "{{.COMPOSE_FILE | default .LOCAL_COMPOSE_FILE}}"
|
||||
cmds:
|
||||
- docker compose -f {{.COMPOSE_FILE}} down
|
||||
clean:
|
||||
desc: "Stop and remove containers AND volumes for the local compose stack"
|
||||
vars:
|
||||
COMPOSE_FILE: "{{.COMPOSE_FILE | default .LOCAL_COMPOSE_FILE}}"
|
||||
cmds:
|
||||
- docker compose -f {{.COMPOSE_FILE}} down -v
|
||||
clean:test:
|
||||
desc: "Stop and remove containers AND volumes for the test compose stack"
|
||||
cmds:
|
||||
- task: clean
|
||||
vars:
|
||||
COMPOSE_FILE: "{{.TEST_COMPOSE_FILE}}"
|
||||
clean:aws:
|
||||
desc: "Stop and remove containers AND volumes for the AWS compose stack"
|
||||
run: once
|
||||
cmds:
|
||||
- task: clean
|
||||
@@ -135,6 +149,7 @@ tasks:
|
||||
cmds:
|
||||
- docker compose -f {{.COMPOSE_FILE}} config -q
|
||||
lint:
|
||||
desc: "Validate all compose files (local, test, generate, aws) with `compose config`"
|
||||
deps:
|
||||
- task: lint:cmd
|
||||
vars:
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
# Bash Script Automation Rules
|
||||
|
||||
**Error Handling**
|
||||
- Always use `set -euo pipefail` at the top (exit on error, undefined vars, pipe failures)
|
||||
- Trap errors and cleanup: `trap 'cleanup' EXIT ERR`
|
||||
- Check command exit codes explicitly when needed: `if ! command; then ... fi`
|
||||
|
||||
**Safety**
|
||||
- Quote all variables: `"$var"` not `$var`
|
||||
- Use `${var:-default}` for defaults, `${var:?error}` for required vars
|
||||
- Validate inputs before destructive operations
|
||||
- Use absolute paths or verify working directory
|
||||
- Test with `-n` (dry-run) flag where possible
|
||||
- When script starts it should check that is has basic aws access if aws cli commands are being used. If we are not logged in then we should terminate and say why.
|
||||
|
||||
**Idempotency**
|
||||
- Make scripts safe to run multiple times with the same inputs
|
||||
- Check state before making changes: `if [ ! -f file ]; then create; fi`
|
||||
- Use atomic operations where possible
|
||||
|
||||
**Logging & Debugging**
|
||||
- Log to stderr for errors: `echo "Error" >&2`
|
||||
- Include timestamps: `echo "[$(date +'%Y-%m-%d %H:%M:%S')] Message"`
|
||||
- Add `-x` debug mode option: `[[ "${DEBUG:-}" == "true" ]] && set -x`
|
||||
- Log what the script is doing before doing it
|
||||
|
||||
**Dependencies**
|
||||
- Check for required commands at start: `command -v tool >/dev/null || exit 1`
|
||||
- Document external dependencies in comments
|
||||
- Fail fast if dependencies missing
|
||||
|
||||
**Functions & Structure**
|
||||
- Use functions for reusable logic
|
||||
- Keep functions small and single-purpose
|
||||
- Use `local` for function variables
|
||||
- Name functions with verbs: `backup_database`, `verify_config`
|
||||
- Each function should be clearly documented as to what it does and how it should be used.
|
||||
|
||||
**Robustness**
|
||||
- Use `mktemp` for temporary files, never hardcode `/tmp/myfile`
|
||||
- Handle signals: `trap 'rm -f "$tmp_file"' EXIT`
|
||||
- Avoid parsing `ls` output - use globs or `find`
|
||||
- Use `[[ ]]` instead of `[ ]` for tests
|
||||
|
||||
**Configuration**
|
||||
- Accept config via environment variables or config file unless otherwise instructed.
|
||||
- Provide sensible defaults
|
||||
|
||||
**Exit Codes**
|
||||
- Exit 0 only on complete success
|
||||
- Use distinct exit codes for different failures
|
||||
- Document exit codes in comments
|
||||
|
||||
**Portability**
|
||||
- Use `#!/usr/bin/env bash` not `#!/bin/bash`
|
||||
- Avoid bashisms if targeting `sh`, or explicitly require bash
|
||||
- Test on target platform
|
||||
+66
@@ -0,0 +1,66 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# run_test_two_pool_lifecycle.sh
|
||||
#
|
||||
# Wrapper for test_two_pool_lifecycle.py. Hardcodes the public pool B
|
||||
# identifiers and AWS region the Go subprocess needs, sources the
|
||||
# repo-root .env for secrets, and execs the Python test.
|
||||
#
|
||||
# Required in .env (or your shell):
|
||||
# PERMIT_IO_API_KEY Permit.io project/env API key (secret).
|
||||
# See .env.example.
|
||||
#
|
||||
# Optional (defaults shown):
|
||||
# AWS_PROFILE AWS SSO profile with cognito-idp Admin*
|
||||
# rights on us-east-2_Qom1i9qIG.
|
||||
# Default: aarete.
|
||||
# SVC_LIFECYCLE_USERNAME Throwaway username for the test cycle.
|
||||
# Default: svc-lifecycle-test-1.
|
||||
#
|
||||
# Required AWS login state:
|
||||
# AWS_PROFILE must have an active session. Run
|
||||
# aws sso login --profile "$AWS_PROFILE"
|
||||
# first if the cached credentials have expired.
|
||||
#
|
||||
# Usage:
|
||||
# bash scripts/manual_tests/run_test_two_pool_lifecycle.sh
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||||
|
||||
# Source repo-root .env so PERMIT_IO_API_KEY (and any AWS_PROFILE
|
||||
# override) reach the shell before the fail-fast check below. Without
|
||||
# this the operator would have to export the key in every shell, or
|
||||
# Python would crash partway through the test.
|
||||
if [ -f "$REPO_ROOT/.env" ]; then
|
||||
set -a
|
||||
# shellcheck disable=SC1090
|
||||
source "$REPO_ROOT/.env"
|
||||
set +a
|
||||
fi
|
||||
|
||||
# Strip LocalStack endpoint overrides that the repo-root .env carries
|
||||
# for the rest of the stack. aws-sdk-go-v2 treats AWS_ENDPOINT_URL as
|
||||
# a global service-endpoint override — when it points at LocalStack
|
||||
# (http://localhost:4566), every Cognito Admin* call returns HTTP 501
|
||||
# because cognito-idp is a paid LocalStack feature.
|
||||
unset AWS_ENDPOINT_URL AWS_ENDPOINT_URL_COGNITO_IDP
|
||||
|
||||
# Public, non-secret pool B identifiers. The Go subprocess (run.tool.sh)
|
||||
# reads these from the environment, so we export them here.
|
||||
export AWS_REGION="us-east-2"
|
||||
export COGNITO_SVC_USER_POOL_ID="us-east-2_Qom1i9qIG"
|
||||
export COGNITO_SVC_APP_CLIENT_ID="6r7chkvjotgs2bsfbd5ibaaft7"
|
||||
export PERMIT_IO_BASE_URL="https://api.permit.io"
|
||||
|
||||
# Operator-specific. Honor an existing AWS_PROFILE in the env, fall
|
||||
# back to "aarete" when unset.
|
||||
export AWS_PROFILE="${AWS_PROFILE:-aarete}"
|
||||
|
||||
# Fail fast with a clear message rather than letting Python start,
|
||||
# call AdminCreateUser, and crash on the Permit step with a vague
|
||||
# 401 — which would leave the throwaway user dangling in Cognito.
|
||||
: "${PERMIT_IO_API_KEY:?set PERMIT_IO_API_KEY in .env (preferred) or your shell before running}"
|
||||
|
||||
exec python3 "$REPO_ROOT/scripts/manual_tests/test_two_pool_lifecycle.py"
|
||||
@@ -41,6 +41,7 @@ import io
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import ssl
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
@@ -117,9 +118,17 @@ class HTTPResponse:
|
||||
class HTTPClient:
|
||||
"""Tiny urllib wrapper. Prints full request + response in --debug mode."""
|
||||
|
||||
def __init__(self, base_url: str, debug: bool) -> None:
|
||||
def __init__(self, base_url: str, debug: bool, insecure: bool = False) -> None:
|
||||
self.base_url = base_url.rstrip("/")
|
||||
self.debug = debug
|
||||
# When --insecure is set we bypass TLS cert verification. Used to hit
|
||||
# AWS ELB hostnames that serve the default self-signed cert (no ACM
|
||||
# cert attached). Do NOT use this against anything carrying real
|
||||
# production secrets; it disables MITM protection.
|
||||
if insecure:
|
||||
self.ssl_context: Optional[ssl.SSLContext] = ssl._create_unverified_context()
|
||||
else:
|
||||
self.ssl_context = None
|
||||
|
||||
def request(
|
||||
self,
|
||||
@@ -149,7 +158,7 @@ class HTTPClient:
|
||||
self._print_request_debug(label, method, url, req_headers, body_bytes)
|
||||
|
||||
try:
|
||||
with urlrequest.urlopen(req, timeout=120) as resp:
|
||||
with urlrequest.urlopen(req, timeout=120, context=self.ssl_context) as resp:
|
||||
resp_status = resp.status
|
||||
resp_headers = {k: v for k, v in resp.getheaders()}
|
||||
resp_body = resp.read()
|
||||
@@ -1020,6 +1029,9 @@ def parse_args(argv: list[str]) -> argparse.Namespace:
|
||||
help="Don't call /turns (skip real FastAPI agent traffic)")
|
||||
p.add_argument("--skip-cross-client", action="store_true",
|
||||
help="Skip the cross-client second-client setup (faster runs)")
|
||||
p.add_argument("--insecure", action="store_true",
|
||||
help="Skip TLS cert verification. Use against AWS ELB hostnames "
|
||||
"that serve the default self-signed cert.")
|
||||
return p.parse_args(argv)
|
||||
|
||||
|
||||
@@ -1033,9 +1045,10 @@ def main(argv: Optional[list[str]] = None) -> int:
|
||||
info(f"Debug mode: {args.debug}")
|
||||
info(f"Skip chat: {args.skip_chat}")
|
||||
info(f"Skip x-client: {args.skip_cross_client}")
|
||||
info(f"Insecure TLS: {args.insecure}")
|
||||
|
||||
fx = Fixture(
|
||||
client=HTTPClient(args.base_url, debug=args.debug),
|
||||
client=HTTPClient(args.base_url, debug=args.debug, insecure=args.insecure),
|
||||
user_subject=args.user,
|
||||
other_user_subject=args.other_user,
|
||||
admin_user_subject=args.admin_user,
|
||||
|
||||
Executable
+275
@@ -0,0 +1,275 @@
|
||||
#!/usr/bin/env python3
|
||||
"""End-to-end manual test for the two-pool service-account auth path.
|
||||
|
||||
Validates plans/api.key.support.cognito.two-pool.plan.3.claude.md
|
||||
against a localhost queryAPI configured with both pool A
|
||||
(COGNITO_USER_POOL_ID) and pool B (COGNITO_SVC_USER_POOL_ID), and
|
||||
auth enabled.
|
||||
|
||||
What this script does (top to bottom):
|
||||
Step 1. Sanity-check queryAPI is reachable (GET /health, no auth).
|
||||
Step 2. InitiateAuth against pool B with the pre-baked test
|
||||
service-account credentials (svc-test-admin-1). Returns
|
||||
AccessToken / IdToken / RefreshToken.
|
||||
Step 3. Decode the AccessToken locally (no signature check) to
|
||||
confirm sub, iss, token_use match the values from the
|
||||
provisioning log.
|
||||
Step 4. Call GET /admin/users with Authorization: Bearer
|
||||
<access_token>. /admin/users is a super_admin-only
|
||||
endpoint, so a 200 proves: (a) queryAPI's multi-issuer
|
||||
verifier accepted the pool B token; (b) the service
|
||||
account was successfully registered in Permit with the
|
||||
super_admin role; (c) the env-check dispatched to pool B.
|
||||
Step 5. Call GET /admin/eula with the same token. Second
|
||||
super_admin-only endpoint — covers the "pick at least 2"
|
||||
requirement from the goal statement.
|
||||
Step 6. Print a summary.
|
||||
|
||||
Zero third-party deps — stdlib only, like
|
||||
cmd/auth_related/service.accounts/test.scripts/test.service.account.auth.py.
|
||||
|
||||
Prerequisites:
|
||||
- queryAPI is running locally on http://localhost:8080.
|
||||
- queryAPI was started with both COGNITO_USER_POOL_ID and
|
||||
COGNITO_SVC_USER_POOL_ID configured.
|
||||
- DISABLE_AUTH is NOT set (we want real Cognito JWT validation).
|
||||
- The test service account has been registered in Permit.io and
|
||||
assigned super_admin via:
|
||||
./cmd/auth_related/service.account.tool/run.tool.sh \
|
||||
register-permit svc-test-admin-1 --roles super_admin
|
||||
- SVC_TEST_PASSWORD is supplied via env (see .env.example). Any
|
||||
.env between this file and the filesystem root is auto-loaded.
|
||||
Everything else (pool ID, app client ID, username, sub, region)
|
||||
is hardcoded below — public identifiers, not secrets — so the
|
||||
script "just works" without further shell setup.
|
||||
|
||||
Usage:
|
||||
python3 scripts/manual_tests/test_two_pool_auth.py
|
||||
python3 scripts/manual_tests/test_two_pool_auth.py --base-url http://localhost:8080
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict
|
||||
|
||||
HTTP_TIMEOUT = 15
|
||||
|
||||
|
||||
def _load_dotenv() -> None:
|
||||
"""Walk from this file up to the filesystem root, loading every
|
||||
.env encountered. First-wins precedence: closer (script-adjacent)
|
||||
.envs win over deeper (repo-root) ones, and existing shell vars
|
||||
win over both. Loading every .env on the way up — rather than
|
||||
stopping at the first hit — means a script-local .env cannot
|
||||
hide the repo-root .env where SVC_TEST_PASSWORD lives.
|
||||
"""
|
||||
here = Path(__file__).resolve()
|
||||
for parent in [here.parent, *here.parents]:
|
||||
candidate = parent / ".env"
|
||||
if not candidate.is_file():
|
||||
continue
|
||||
for raw in candidate.read_text().splitlines():
|
||||
line = raw.strip()
|
||||
if not line or line.startswith("#"):
|
||||
continue
|
||||
if "=" not in line:
|
||||
continue
|
||||
k, v = line.split("=", 1)
|
||||
k = k.strip()
|
||||
v = v.strip().strip('"').strip("'")
|
||||
if k and k not in os.environ:
|
||||
os.environ[k] = v
|
||||
|
||||
|
||||
def _required(name: str) -> str:
|
||||
v = os.environ.get(name, "").strip()
|
||||
if not v:
|
||||
print(f"FAIL: required env var {name} is unset. See .env.example.")
|
||||
sys.exit(2)
|
||||
return v
|
||||
|
||||
|
||||
# Hardcoded service-account identifiers. Pool ID, app client ID,
|
||||
# username, and sub are public values, not secrets — hardcoding them
|
||||
# makes the script share-able: anyone with SVC_TEST_PASSWORD in their
|
||||
# .env can run it with no other prerequisites.
|
||||
USER_POOL_ID = "us-east-2_Qom1i9qIG"
|
||||
APP_CLIENT_ID = "6r7chkvjotgs2bsfbd5ibaaft7"
|
||||
USERNAME = "svc-test-admin-1"
|
||||
EXPECTED_SUB = "c16b0520-80e1-7076-eeb2-1e520d81e781"
|
||||
|
||||
# Region is encoded in the pool ID prefix. Derive it rather than
|
||||
# trust the shell — devbox commonly exports AWS_REGION=us-east-1,
|
||||
# which routes InitiateAuth to a region where this pool's app client
|
||||
# does not exist and yields ResourceNotFoundException.
|
||||
AWS_REGION = USER_POOL_ID.split("_", 1)[0]
|
||||
|
||||
_load_dotenv()
|
||||
|
||||
# Strip LocalStack endpoint overrides for hygiene. urllib (used
|
||||
# below) does not honor AWS_ENDPOINT_URL, but the env stays clean for
|
||||
# anything spawned downstream from this process.
|
||||
for key in ("AWS_ENDPOINT_URL", "AWS_ENDPOINT_URL_COGNITO_IDP"):
|
||||
os.environ.pop(key, None)
|
||||
|
||||
PASSWORD = _required("SVC_TEST_PASSWORD")
|
||||
|
||||
ISSUER_URL = f"https://cognito-idp.{AWS_REGION}.amazonaws.com/{USER_POOL_ID}"
|
||||
COGNITO_IDP_URL = f"https://cognito-idp.{AWS_REGION}.amazonaws.com/"
|
||||
|
||||
|
||||
def section(title: str) -> None:
|
||||
print()
|
||||
print("=" * 72)
|
||||
print(title)
|
||||
print("=" * 72)
|
||||
|
||||
|
||||
def cognito_call(operation: str, body: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""POST a JSON-over-HTTPS request to Cognito Identity Provider.
|
||||
|
||||
Matches the wire format used by boto3 internally so we don't
|
||||
need any AWS credentials for the unauthenticated InitiateAuth
|
||||
operation.
|
||||
"""
|
||||
payload = json.dumps(body).encode("utf-8")
|
||||
req = urllib.request.Request(
|
||||
COGNITO_IDP_URL,
|
||||
data=payload,
|
||||
method="POST",
|
||||
headers={
|
||||
"Content-Type": "application/x-amz-json-1.1",
|
||||
"X-Amz-Target": f"AWSCognitoIdentityProviderService.{operation}",
|
||||
},
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=HTTP_TIMEOUT) as r:
|
||||
return json.loads(r.read())
|
||||
except urllib.error.HTTPError as e:
|
||||
body_text = e.read().decode("utf-8", errors="replace")
|
||||
try:
|
||||
err = json.loads(body_text)
|
||||
raise RuntimeError(
|
||||
f"Cognito {operation}: HTTP {e.code} {err.get('__type', '?')}: "
|
||||
f"{err.get('message', body_text)}"
|
||||
) from None
|
||||
except json.JSONDecodeError:
|
||||
raise RuntimeError(f"Cognito {operation}: HTTP {e.code} {body_text}") from None
|
||||
|
||||
|
||||
def decode_jwt_payload(token: str) -> Dict[str, Any]:
|
||||
"""Base64url-decode the payload segment of a JWT (no signature check)."""
|
||||
seg = token.split(".")[1]
|
||||
seg += "=" * (-len(seg) % 4)
|
||||
return json.loads(base64.urlsafe_b64decode(seg))
|
||||
|
||||
|
||||
def queryapi_get(base_url: str, path: str, access_token: str = "") -> tuple[int, str]:
|
||||
"""GET <base_url><path> with optional bearer auth. Returns (status, body)."""
|
||||
url = base_url.rstrip("/") + path
|
||||
headers = {}
|
||||
if access_token:
|
||||
headers["Authorization"] = "Bearer " + access_token
|
||||
req = urllib.request.Request(url, headers=headers, method="GET")
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=HTTP_TIMEOUT) as r:
|
||||
return r.status, r.read().decode("utf-8", errors="replace")
|
||||
except urllib.error.HTTPError as e:
|
||||
body = e.read().decode("utf-8", errors="replace")
|
||||
return e.code, body
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--base-url", default="http://localhost:8080")
|
||||
args = parser.parse_args()
|
||||
base = args.base_url
|
||||
|
||||
section("Step 1: queryAPI health")
|
||||
code, body = queryapi_get(base, "/health")
|
||||
print(f"GET {base}/health -> {code}")
|
||||
if code != 200:
|
||||
print(f"FAIL: queryAPI not healthy at {base}: {body}")
|
||||
return 1
|
||||
|
||||
section("Step 2: InitiateAuth against pool B")
|
||||
try:
|
||||
resp = cognito_call(
|
||||
"InitiateAuth",
|
||||
{
|
||||
"AuthFlow": "USER_PASSWORD_AUTH",
|
||||
"ClientId": APP_CLIENT_ID,
|
||||
"AuthParameters": {"USERNAME": USERNAME, "PASSWORD": PASSWORD},
|
||||
},
|
||||
)
|
||||
except RuntimeError as e:
|
||||
print(f"FAIL: {e}")
|
||||
return 1
|
||||
if "AuthenticationResult" not in resp:
|
||||
print(f"FAIL: unexpected InitiateAuth response: {resp}")
|
||||
return 1
|
||||
access_token = resp["AuthenticationResult"]["AccessToken"]
|
||||
print(f" AccessToken length : {len(access_token)} chars")
|
||||
print(f" ExpiresIn : {resp['AuthenticationResult']['ExpiresIn']}s")
|
||||
|
||||
section("Step 3: AccessToken claims")
|
||||
claims = decode_jwt_payload(access_token)
|
||||
print(json.dumps(claims, indent=2, sort_keys=True))
|
||||
failures = []
|
||||
if claims.get("iss") != ISSUER_URL:
|
||||
failures.append(f"iss mismatch (got {claims.get('iss')}, want {ISSUER_URL})")
|
||||
if claims.get("sub") != EXPECTED_SUB:
|
||||
failures.append(f"sub mismatch (got {claims.get('sub')}, want {EXPECTED_SUB})")
|
||||
if claims.get("token_use") != "access":
|
||||
failures.append(f"token_use != access (got {claims.get('token_use')})")
|
||||
if failures:
|
||||
for f in failures:
|
||||
print(f"FAIL: {f}")
|
||||
return 1
|
||||
|
||||
# We pick two super_admin-only endpoints that read from Postgres
|
||||
# only (no downstream AWS calls). /admin/users would also work
|
||||
# against real AWS but hits Cognito ListUsers, which localstack
|
||||
# community edition does not implement, so we use the EULA admin
|
||||
# endpoints instead — both are gated on the same super_admin role
|
||||
# in Permit.
|
||||
|
||||
section("Step 4: GET /admin/eula (super_admin only)")
|
||||
code, body = queryapi_get(base, "/admin/eula", access_token)
|
||||
print(f"GET {base}/admin/eula -> {code}")
|
||||
if code != 200:
|
||||
print("FAIL: /admin/eula did not return 200 — service account either")
|
||||
print(" did not authenticate, or is not registered with super_admin in Permit.")
|
||||
print(f" body: {body[:400]}")
|
||||
return 1
|
||||
print(f" body preview: {body[:200]}")
|
||||
|
||||
section("Step 5: GET /admin/eula/agreements (super_admin only)")
|
||||
code, body = queryapi_get(base, "/admin/eula/agreements", access_token)
|
||||
print(f"GET {base}/admin/eula/agreements -> {code}")
|
||||
if code != 200:
|
||||
print("FAIL: /admin/eula/agreements did not return 200.")
|
||||
print(f" body: {body[:400]}")
|
||||
return 1
|
||||
print(f" body preview: {body[:200]}")
|
||||
|
||||
section("Result")
|
||||
print("PASS — two-pool service-account auth is wired end-to-end.")
|
||||
print(
|
||||
f" Service account {USERNAME} (sub={EXPECTED_SUB})\n"
|
||||
" authenticated against pool B and successfully called two\n"
|
||||
" super_admin-only endpoints on queryAPI."
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Executable
+51
@@ -0,0 +1,51 @@
|
||||
#!/usr/bin/env bash
|
||||
# test_two_pool_auth.sh runs the end-to-end integration test for the
|
||||
# two-pool service-account auth path. It assumes the operator has:
|
||||
# - AWS credentials with read access to the test pools
|
||||
# (us-east-2_<pool-a>, us-east-2_Qom1i9qIG)
|
||||
# - PERMIT_IO_API_KEY set in the environment
|
||||
# - Docker running (for `task compose:refresh`)
|
||||
#
|
||||
# The script:
|
||||
# 1. Registers the pre-existing test service-account in Permit with
|
||||
# super_admin (idempotent — re-running is safe).
|
||||
# 2. Reminds the operator to boot queryAPI with both pool A and B
|
||||
# configured (compose:refresh by itself does not know about the
|
||||
# new env vars; the operator must source the right .env).
|
||||
# 3. Runs scripts/manual_tests/test_two_pool_auth.py against the
|
||||
# local queryAPI to validate the end-to-end flow.
|
||||
#
|
||||
# Usage:
|
||||
# PERMIT_IO_API_KEY=permit_key_... \
|
||||
# AWS_PROFILE=aarete \
|
||||
# ./scripts/manual_tests/test_two_pool_auth.sh
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
if [[ -z "${PERMIT_IO_API_KEY:-}" ]]; then
|
||||
echo "ERROR: PERMIT_IO_API_KEY must be set" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||||
|
||||
echo "Step 1/3: registering svc-test-admin-1 in Permit with super_admin..."
|
||||
pushd "$REPO_ROOT/cmd/auth_related/service.account.tool" >/dev/null
|
||||
COGNITO_REGION="us-east-2" \
|
||||
COGNITO_SVC_USER_POOL_ID="us-east-2_Qom1i9qIG" \
|
||||
COGNITO_SVC_APP_CLIENT_ID="6r7chkvjotgs2bsfbd5ibaaft7" \
|
||||
go run ./... register-permit svc-test-admin-1 --roles super_admin
|
||||
popd >/dev/null
|
||||
|
||||
echo
|
||||
echo "Step 2/3: confirm queryAPI is running locally with pool B enabled."
|
||||
echo " - COGNITO_USER_POOL_ID should be set (existing pool A)"
|
||||
echo " - COGNITO_SVC_USER_POOL_ID should be us-east-2_Qom1i9qIG (pool B)"
|
||||
echo " - DISABLE_AUTH must be unset (we are testing real auth)"
|
||||
echo " - queryAPI must be reachable at http://localhost:8080"
|
||||
echo
|
||||
read -rp "Press enter when queryAPI is up and configured..."
|
||||
|
||||
echo
|
||||
echo "Step 3/3: running test_two_pool_auth.py against localhost..."
|
||||
python3 "$REPO_ROOT/scripts/manual_tests/test_two_pool_auth.py"
|
||||
+303
@@ -0,0 +1,303 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Lifecycle integration test for the pool B service-account CLI.
|
||||
|
||||
Validates plans/api.key.support.cognito.two-pool.plan.3.claude.md §6.3
|
||||
beyond the super-admin coverage in test_two_pool_auth.py. Operates on
|
||||
a dedicated throwaway user (default: svc-lifecycle-test-1) so it
|
||||
never collides with the long-lived super-admin test account.
|
||||
|
||||
What this script does (top to bottom):
|
||||
Step 1. service.account.tool create <user> (provision)
|
||||
Step 2. InitiateAuth(<user>, password) (baseline OK)
|
||||
Step 3. service.account.tool disable <user>
|
||||
InitiateAuth(refresh) (expect NotAuthorized)
|
||||
Step 4. service.account.tool enable <user>
|
||||
InitiateAuth(<user>, password) (expect OK again)
|
||||
Step 5. service.account.tool rotate-password <user>
|
||||
InitiateAuth(<user>, OLD_password) (expect NotAuthorized)
|
||||
InitiateAuth(<user>, NEW_password) (expect OK)
|
||||
Step 6. service.account.tool delete <user>
|
||||
InitiateAuth(<user>, password) (expect UserNotFound)
|
||||
service.account.tool show <user> (expect non-zero exit)
|
||||
|
||||
The script captures the initial password from the `create` output and
|
||||
the new password from `rotate-password` output by scanning stdout — no
|
||||
secret is ever read from disk or written to a file.
|
||||
|
||||
Prerequisites:
|
||||
- Run via the wrapper (recommended):
|
||||
bash scripts/manual_tests/run_test_two_pool_lifecycle.sh
|
||||
The wrapper sources the repo-root .env for secrets, hardcodes
|
||||
the public pool B identifiers (pool ID, app client ID, region,
|
||||
PERMIT_IO_BASE_URL), and execs this script.
|
||||
|
||||
- Required in .env (or shell):
|
||||
PERMIT_IO_API_KEY Permit.io API key (secret).
|
||||
Public identifiers (pool ID, app client ID, region) are
|
||||
hardcoded in the wrapper — not secrets.
|
||||
|
||||
- Optional in .env (or shell):
|
||||
AWS_PROFILE AWS SSO profile (default: aarete).
|
||||
Must have cognito-idp Admin* rights
|
||||
on user pool us-east-2_Qom1i9qIG.
|
||||
SVC_LIFECYCLE_USERNAME Override the throwaway username
|
||||
(default: svc-lifecycle-test-1).
|
||||
|
||||
- AWS_PROFILE must have an active session. Run
|
||||
aws sso login --profile "$AWS_PROFILE"
|
||||
if cached credentials have expired.
|
||||
|
||||
Usage:
|
||||
bash scripts/manual_tests/run_test_two_pool_lifecycle.sh
|
||||
|
||||
Exit codes:
|
||||
0 — all lifecycle assertions held.
|
||||
1 — any assertion failed; script attempts a best-effort cleanup of the
|
||||
throwaway user before exiting.
|
||||
2 — required env / preconditions missing.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Tuple
|
||||
|
||||
HTTP_TIMEOUT = 15
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
RUN_TOOL = REPO_ROOT / "cmd" / "auth_related" / "service.account.tool" / "run.tool.sh"
|
||||
|
||||
|
||||
def _load_dotenv() -> None:
|
||||
"""Populate os.environ from the repo-root .env without external deps."""
|
||||
here = Path(__file__).resolve()
|
||||
for parent in [here.parent, *here.parents]:
|
||||
candidate = parent / ".env"
|
||||
if candidate.is_file():
|
||||
for raw in candidate.read_text().splitlines():
|
||||
line = raw.strip()
|
||||
if not line or line.startswith("#"):
|
||||
continue
|
||||
if "=" not in line:
|
||||
continue
|
||||
k, v = line.split("=", 1)
|
||||
k = k.strip()
|
||||
v = v.strip().strip('"').strip("'")
|
||||
if k and k not in os.environ:
|
||||
os.environ[k] = v
|
||||
return
|
||||
|
||||
|
||||
def section(title: str) -> None:
|
||||
print()
|
||||
print("=" * 72)
|
||||
print(title)
|
||||
print("=" * 72)
|
||||
|
||||
|
||||
def require_env(name: str) -> str:
|
||||
v = os.environ.get(name, "").strip()
|
||||
if not v:
|
||||
print(f"FAIL: required env var {name} is unset. See .env.example.")
|
||||
sys.exit(2)
|
||||
return v
|
||||
|
||||
|
||||
def run_tool(*args: str, check: bool = True) -> subprocess.CompletedProcess:
|
||||
"""Invoke run.tool.sh with the supplied subcommand. Streams stdout/err."""
|
||||
cmd = ["bash", str(RUN_TOOL), *args]
|
||||
print(f"$ {' '.join(args)}")
|
||||
proc = subprocess.run(cmd, capture_output=True, text=True, check=False)
|
||||
if proc.stdout:
|
||||
sys.stdout.write(proc.stdout)
|
||||
if proc.stderr:
|
||||
sys.stderr.write(proc.stderr)
|
||||
if check and proc.returncode != 0:
|
||||
raise RuntimeError(f"run.tool.sh {args!r} failed: rc={proc.returncode}")
|
||||
return proc
|
||||
|
||||
|
||||
def extract_password(stdout: str) -> str:
|
||||
"""Pull the Password line out of `create` / `rotate-password` output."""
|
||||
# `create` prints: Password : <pw>
|
||||
# `rotate-password`: New password for <u>: <pw>
|
||||
for raw in stdout.splitlines():
|
||||
line = raw.strip()
|
||||
m = re.match(r"^Password\s*:\s*(\S+)\s*$", line)
|
||||
if m:
|
||||
return m.group(1)
|
||||
m = re.match(r"^New password for \S+:\s*(\S+)\s*$", line)
|
||||
if m:
|
||||
return m.group(1)
|
||||
raise RuntimeError("could not find password line in tool output")
|
||||
|
||||
|
||||
def cognito_call(region: str, operation: str, body: Dict[str, Any]) -> Tuple[int, Dict[str, Any]]:
|
||||
"""POST to Cognito IDP; returns (http_status, parsed_json or error_dict)."""
|
||||
url = f"https://cognito-idp.{region}.amazonaws.com/"
|
||||
payload = json.dumps(body).encode("utf-8")
|
||||
req = urllib.request.Request(
|
||||
url,
|
||||
data=payload,
|
||||
method="POST",
|
||||
headers={
|
||||
"Content-Type": "application/x-amz-json-1.1",
|
||||
"X-Amz-Target": f"AWSCognitoIdentityProviderService.{operation}",
|
||||
},
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=HTTP_TIMEOUT) as r:
|
||||
return r.status, json.loads(r.read())
|
||||
except urllib.error.HTTPError as e:
|
||||
body_text = e.read().decode("utf-8", errors="replace")
|
||||
try:
|
||||
return e.code, json.loads(body_text)
|
||||
except json.JSONDecodeError:
|
||||
return e.code, {"__type": "ParseError", "message": body_text}
|
||||
|
||||
|
||||
def initiate_user_password(region: str, client_id: str, username: str, password: str) -> Tuple[int, Dict[str, Any]]:
|
||||
return cognito_call(region, "InitiateAuth", {
|
||||
"AuthFlow": "USER_PASSWORD_AUTH",
|
||||
"ClientId": client_id,
|
||||
"AuthParameters": {"USERNAME": username, "PASSWORD": password},
|
||||
})
|
||||
|
||||
|
||||
def initiate_refresh(region: str, client_id: str, refresh_token: str) -> Tuple[int, Dict[str, Any]]:
|
||||
return cognito_call(region, "InitiateAuth", {
|
||||
"AuthFlow": "REFRESH_TOKEN_AUTH",
|
||||
"ClientId": client_id,
|
||||
"AuthParameters": {"REFRESH_TOKEN": refresh_token},
|
||||
})
|
||||
|
||||
|
||||
def expect_ok(status: int, resp: Dict[str, Any], label: str) -> Dict[str, Any]:
|
||||
if status != 200 or "AuthenticationResult" not in resp:
|
||||
raise AssertionError(f"{label}: expected 200+AuthenticationResult, got {status} {resp.get('__type')}: {resp.get('message')}")
|
||||
return resp["AuthenticationResult"]
|
||||
|
||||
|
||||
def expect_not_authorized(status: int, resp: Dict[str, Any], label: str) -> None:
|
||||
if status == 200 and "AuthenticationResult" in resp:
|
||||
raise AssertionError(f"{label}: expected auth failure, but got tokens back")
|
||||
err = resp.get("__type", "")
|
||||
if "NotAuthorized" not in err and "UserNotFound" not in err:
|
||||
raise AssertionError(f"{label}: expected NotAuthorized/UserNotFound, got {status} {err}: {resp.get('message')}")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
_load_dotenv()
|
||||
|
||||
# Strip LocalStack endpoint overrides before the Go subprocess
|
||||
# (run.tool.sh) inherits them. aws-sdk-go-v2 treats
|
||||
# AWS_ENDPOINT_URL as a global service-endpoint override —
|
||||
# devbox commonly sets it to http://localhost:4566, where
|
||||
# cognito-idp is unimplemented and every Admin* call returns
|
||||
# HTTP 501. The wrapper also unsets these, but doing it here
|
||||
# makes the script robust when invoked directly.
|
||||
for key in ("AWS_ENDPOINT_URL", "AWS_ENDPOINT_URL_COGNITO_IDP"):
|
||||
os.environ.pop(key, None)
|
||||
|
||||
user_pool_id = require_env("COGNITO_SVC_USER_POOL_ID")
|
||||
client_id = require_env("COGNITO_SVC_APP_CLIENT_ID")
|
||||
require_env("PERMIT_IO_API_KEY")
|
||||
|
||||
# Region is encoded in the pool ID prefix (us-east-2_Qom1i9qIG).
|
||||
# Override any stale AWS_REGION so a shell that exports us-east-1
|
||||
# (devbox default) cannot redirect the SDK to the wrong region.
|
||||
region = user_pool_id.split("_", 1)[0]
|
||||
os.environ["AWS_REGION"] = region
|
||||
|
||||
username = os.environ.get("SVC_LIFECYCLE_USERNAME", "svc-lifecycle-test-1").strip()
|
||||
if not username:
|
||||
username = "svc-lifecycle-test-1"
|
||||
|
||||
print(f"Region : {region}")
|
||||
print(f"User pool id : {user_pool_id}")
|
||||
print(f"App client id : {client_id}")
|
||||
print(f"Throwaway username: {username}")
|
||||
|
||||
# Best-effort: delete a leftover user from a prior aborted run so
|
||||
# `create` does not blow up with UsernameExistsException. We swallow
|
||||
# all errors — non-existence is expected on a clean run.
|
||||
section("Pre-flight cleanup (idempotent)")
|
||||
run_tool("delete", username, check=False)
|
||||
|
||||
cleanup_needed = False
|
||||
try:
|
||||
section("Step 1: create throwaway user with super_admin role")
|
||||
proc = run_tool("create", username, "--roles", "super_admin", "--description",
|
||||
"lifecycle test user — safe to delete")
|
||||
cleanup_needed = True
|
||||
password = extract_password(proc.stdout)
|
||||
print(f" captured initial password (len={len(password)})")
|
||||
|
||||
section("Step 2: baseline InitiateAuth must succeed")
|
||||
status, resp = initiate_user_password(region, client_id, username, password)
|
||||
result = expect_ok(status, resp, "baseline InitiateAuth")
|
||||
refresh_token = result["RefreshToken"]
|
||||
print(" baseline InitiateAuth OK; captured RefreshToken")
|
||||
|
||||
section("Step 3: disable + refresh must be rejected")
|
||||
run_tool("disable", username)
|
||||
status, resp = initiate_refresh(region, client_id, refresh_token)
|
||||
expect_not_authorized(status, resp, "refresh after disable")
|
||||
print(" refresh correctly rejected after disable")
|
||||
# Access tokens minted before disable remain valid until exp;
|
||||
# we deliberately do not assert on that here (would require a
|
||||
# short-TTL test client — see plan §6.3 deferred item).
|
||||
|
||||
section("Step 4: enable + InitiateAuth must succeed again")
|
||||
run_tool("enable", username)
|
||||
status, resp = initiate_user_password(region, client_id, username, password)
|
||||
expect_ok(status, resp, "InitiateAuth after enable")
|
||||
print(" InitiateAuth OK after re-enable")
|
||||
|
||||
section("Step 5: rotate-password — old fails, new succeeds")
|
||||
old_password = password
|
||||
proc = run_tool("rotate-password", username)
|
||||
new_password = extract_password(proc.stdout)
|
||||
if new_password == old_password:
|
||||
raise AssertionError("rotate-password returned the same password")
|
||||
status, resp = initiate_user_password(region, client_id, username, old_password)
|
||||
expect_not_authorized(status, resp, "InitiateAuth with old password")
|
||||
print(" old password correctly rejected")
|
||||
status, resp = initiate_user_password(region, client_id, username, new_password)
|
||||
expect_ok(status, resp, "InitiateAuth with new password")
|
||||
password = new_password
|
||||
print(" new password accepted")
|
||||
|
||||
section("Step 6: delete — auth must fail and show must error")
|
||||
run_tool("delete", username)
|
||||
cleanup_needed = False
|
||||
status, resp = initiate_user_password(region, client_id, username, password)
|
||||
expect_not_authorized(status, resp, "InitiateAuth after delete")
|
||||
proc = run_tool("show", username, check=False)
|
||||
if proc.returncode == 0:
|
||||
raise AssertionError("show <deleted-user> exited 0; expected non-zero")
|
||||
print(" delete confirmed: auth fails, show errors as expected")
|
||||
|
||||
section("Result")
|
||||
print(f"PASS — full lifecycle for {username} validated end-to-end.")
|
||||
return 0
|
||||
except AssertionError as e:
|
||||
print(f"\nFAIL: {e}")
|
||||
return 1
|
||||
except Exception as e:
|
||||
print(f"\nFAIL: unexpected error: {e}")
|
||||
return 1
|
||||
finally:
|
||||
if cleanup_needed:
|
||||
section("Cleanup (script failed mid-run)")
|
||||
run_tool("delete", username, check=False)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
+325
@@ -0,0 +1,325 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Test script for ZIP batch upload functionality
|
||||
#!/bin/bash
|
||||
|
||||
# Test script for ZIP batch upload functionality
|
||||
# This script exercises all batch endpoints with a cloud deployment
|
||||
# This version uses only API calls and does not require database access
|
||||
# It's designed to work with cloud-deployed instances of the query orchestration service
|
||||
#
|
||||
# REST API operations tested:
|
||||
# - GET /metrics - Check service health
|
||||
# - POST /client - Create new test client
|
||||
# - PATCH /client/{CLIENT_ID} - Enable CanSync flag via API
|
||||
# - POST /client/{CLIENT_ID}/document/batch - Upload ZIP file with multiple PDFs
|
||||
# - GET /client/{CLIENT_ID}/document/batch/{BATCH_ID} - Get batch status and details
|
||||
# - GET /client/{CLIENT_ID}/document/batch - List all batches for client with pagination
|
||||
# - POST /client/{CLIENT_ID}/document/batch (invalid) - Test error handling with non-ZIP file
|
||||
# - DELETE /client/{CLIENT_ID}/document/batch/{BATCH_ID} - Cancel/delete batch processing
|
||||
|
||||
set -e # Exit on any error
|
||||
|
||||
# local host version of the test
|
||||
#BASE_URL="http://localhost:8080"
|
||||
# Configuration for cloud
|
||||
#BASE_URL="http://queryo-query-rtxkppcxpjsb-742555588.us-east-2.elb.amazonaws.com"
|
||||
BASE_URL="http://queryo-query-q0pz6j2syaox-1001614225.us-east-2.elb.amazonaws.com"
|
||||
# CLIENT_ID will be generated in main() to ensure it's fresh
|
||||
ZIP_FILE="test_batch.zip"
|
||||
|
||||
# Colors for output
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
# Logging functions
|
||||
log_info() {
|
||||
echo -e "${GREEN}[INFO]${NC} $1"
|
||||
}
|
||||
|
||||
log_warn() {
|
||||
echo -e "${YELLOW}[WARN]${NC} $1"
|
||||
}
|
||||
|
||||
log_error() {
|
||||
echo -e "${RED}[ERROR]${NC} $1"
|
||||
}
|
||||
|
||||
# Check if service is running
|
||||
check_service() {
|
||||
log_info "Checking if queryAPI service is running..."
|
||||
if ! curl -s "$BASE_URL/metrics" > /dev/null 2>&1; then
|
||||
log_error "queryAPI service is not running at $BASE_URL"
|
||||
log_info "Please check the cloud deployment status"
|
||||
exit 1
|
||||
fi
|
||||
log_info "Service is running"
|
||||
}
|
||||
|
||||
# Create test ZIP file with real PDFs from test.pdf.batch directory, each in its own subfolder
|
||||
create_test_zip() {
|
||||
log_info "Creating test ZIP file with real PDFs from ./test.pdf.batch, each in its own subfolder..."
|
||||
|
||||
# Remove existing file if it exists
|
||||
rm -f "$ZIP_FILE"
|
||||
|
||||
# Check if test.pdf.batch directory exists
|
||||
pdf_dir="./test.pdf.batch"
|
||||
if [ ! -d "$pdf_dir" ]; then
|
||||
log_error "Directory $pdf_dir does not exist"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check if there are PDF files in the directory
|
||||
pdf_count=$(find "$pdf_dir" -name "*.pdf" -type f | wc -l)
|
||||
if [ "$pdf_count" -eq 0 ]; then
|
||||
log_error "No PDF files found in $pdf_dir"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Create temporary directory structure
|
||||
temp_dir="temp_zip_structure"
|
||||
rm -rf "$temp_dir"
|
||||
mkdir -p "$temp_dir"
|
||||
|
||||
# Copy each PDF to its own subfolder
|
||||
folder_num=1
|
||||
for pdf_file in "$pdf_dir"/*.pdf; do
|
||||
if [ -f "$pdf_file" ]; then
|
||||
folder_name="folder$folder_num"
|
||||
mkdir -p "$temp_dir/$folder_name"
|
||||
cp "$pdf_file" "$temp_dir/$folder_name/"
|
||||
log_info "Placed $(basename "$pdf_file") in $folder_name/"
|
||||
folder_num=$((folder_num + 1))
|
||||
fi
|
||||
done
|
||||
|
||||
# Create ZIP file with the folder structure
|
||||
current_dir=$(pwd)
|
||||
cd "$temp_dir"
|
||||
zip -q -r "$current_dir/$ZIP_FILE" .
|
||||
cd "$current_dir"
|
||||
|
||||
# Cleanup temporary directory
|
||||
rm -rf "$temp_dir"
|
||||
|
||||
file_size=$(stat -f%z "$ZIP_FILE" 2>/dev/null || stat -c%s "$ZIP_FILE" 2>/dev/null || echo "unknown")
|
||||
log_info "Created $ZIP_FILE with $pdf_count real PDFs in separate subfolders ($file_size bytes)"
|
||||
}
|
||||
|
||||
# Create client (idempotent)
|
||||
create_client() {
|
||||
log_info "Creating/checking client: $CLIENT_ID"
|
||||
log_info "API call: POST $BASE_URL/client with ID=$CLIENT_ID"
|
||||
|
||||
# Try to create client, ignore if already exists
|
||||
response=$(curl -s -w "\n%{http_code}" -X POST "$BASE_URL/client" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"id\":\"$CLIENT_ID\",\"name\":\"Test Client $CLIENT_ID\"}" 2>/dev/null || echo "000")
|
||||
|
||||
http_code=$(echo "$response" | tail -n1)
|
||||
|
||||
if [ "$http_code" = "201" ] || [ "$http_code" = "200" ]; then
|
||||
log_info "Client created/exists: $CLIENT_ID"
|
||||
elif [ "$http_code" = "409" ] || [ "$http_code" = "400" ]; then
|
||||
log_info "Client already exists or duplicate name: $CLIENT_ID"
|
||||
else
|
||||
log_warn "Unexpected response when creating client (HTTP $http_code), continuing anyway..."
|
||||
fi
|
||||
}
|
||||
|
||||
# Enable CanSync flag for client using API PATCH call
|
||||
enable_client_sync() {
|
||||
log_info "Enabling CanSync flag for client: $CLIENT_ID"
|
||||
log_info "API call: PATCH $BASE_URL/client/$CLIENT_ID"
|
||||
|
||||
# Use PATCH API to update client with can_sync: true
|
||||
response=$(curl -s -w "\n%{http_code}" -X PATCH "$BASE_URL/client/$CLIENT_ID" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"can_sync":true}' 2>/dev/null)
|
||||
|
||||
http_code=$(echo "$response" | tail -n1)
|
||||
response_body=$(echo "$response" | sed '$d')
|
||||
|
||||
if [ "$http_code" = "200" ] || [ "$http_code" = "204" ]; then
|
||||
log_info "CanSync flag enabled via API - documents can now be processed"
|
||||
return 0
|
||||
else
|
||||
log_warn "Failed to enable CanSync flag via API (HTTP $http_code), but continuing with test"
|
||||
log_warn "Documents may not process without CanSync enabled"
|
||||
echo "$response_body"
|
||||
return 0 # Continue anyway since this is a test
|
||||
fi
|
||||
}
|
||||
|
||||
# Upload ZIP batch
|
||||
upload_batch() {
|
||||
log_info "Uploading ZIP batch..."
|
||||
log_info "API call: POST $BASE_URL/client/$CLIENT_ID/document/batch"
|
||||
|
||||
response=$(curl -s -w "\n%{http_code}" -X POST "$BASE_URL/client/$CLIENT_ID/document/batch" \
|
||||
-F "archive=@$ZIP_FILE" 2>/dev/null)
|
||||
|
||||
http_code=$(echo "$response" | tail -n1)
|
||||
response_body=$(echo "$response" | sed '$d')
|
||||
|
||||
if [ "$http_code" = "202" ]; then
|
||||
BATCH_ID=$(echo "$response_body" | grep -o '"batch_id":"[^"]*"' | cut -d'"' -f4)
|
||||
STATUS_URL=$(echo "$response_body" | grep -o '"status_url":"[^"]*"' | cut -d'"' -f4)
|
||||
log_info "Batch uploaded successfully"
|
||||
log_info "Batch ID: $BATCH_ID"
|
||||
log_info "Status URL: $STATUS_URL"
|
||||
|
||||
# Print batch number prominently for easy capture
|
||||
echo ""
|
||||
echo "=========================================="
|
||||
echo "BATCH NUMBER CREATED: $BATCH_ID"
|
||||
echo "=========================================="
|
||||
echo ""
|
||||
|
||||
return 0
|
||||
else
|
||||
log_error "Failed to upload batch (HTTP $http_code)"
|
||||
echo "$response_body"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Get batch status
|
||||
get_batch_status() {
|
||||
log_info "Getting batch status..."
|
||||
|
||||
response=$(curl -s -w "\n%{http_code}" "$BASE_URL/client/$CLIENT_ID/document/batch/$BATCH_ID" 2>/dev/null)
|
||||
|
||||
http_code=$(echo "$response" | tail -n1)
|
||||
response_body=$(echo "$response" | sed '$d')
|
||||
|
||||
if [ "$http_code" = "200" ]; then
|
||||
status=$(echo "$response_body" | grep -o '"status":"[^"]*"' | cut -d'"' -f4)
|
||||
filename=$(echo "$response_body" | grep -o '"originalFilename":"[^"]*"' | cut -d'"' -f4)
|
||||
log_info "Batch status: $status"
|
||||
log_info "Original filename: $filename"
|
||||
echo "$response_body" | python3 -m json.tool 2>/dev/null || echo "$response_body"
|
||||
return 0
|
||||
else
|
||||
log_error "Failed to get batch status (HTTP $http_code)"
|
||||
echo "$response_body"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# List batches for client
|
||||
list_batches() {
|
||||
log_info "Listing batches for client..."
|
||||
|
||||
response=$(curl -s -w "\n%{http_code}" "$BASE_URL/client/$CLIENT_ID/document/batch?limit=10&offset=0" 2>/dev/null)
|
||||
|
||||
http_code=$(echo "$response" | tail -n1)
|
||||
response_body=$(echo "$response" | sed '$d')
|
||||
|
||||
if [ "$http_code" = "200" ]; then
|
||||
batch_count=$(echo "$response_body" | grep -o '"totalCount":[0-9]*' | cut -d':' -f2)
|
||||
log_info "Found $batch_count batches"
|
||||
echo "$response_body" | python3 -m json.tool 2>/dev/null || echo "$response_body"
|
||||
return 0
|
||||
else
|
||||
log_error "Failed to list batches (HTTP $http_code)"
|
||||
echo "$response_body"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Test invalid file upload
|
||||
test_invalid_file() {
|
||||
log_info "Testing invalid file upload (should fail)..."
|
||||
|
||||
# Create invalid file
|
||||
echo "This is not a ZIP file" > invalid.txt
|
||||
|
||||
response=$(curl -s -w "\n%{http_code}" -X POST "$BASE_URL/client/$CLIENT_ID/document/batch" \
|
||||
-F "archive=@invalid.txt" 2>/dev/null)
|
||||
|
||||
http_code=$(echo "$response" | tail -n1)
|
||||
response_body=$(echo "$response" | sed '$d')
|
||||
|
||||
# Cleanup
|
||||
rm -f invalid.txt
|
||||
|
||||
if [ "$http_code" = "400" ]; then
|
||||
log_info "Invalid file correctly rejected (HTTP $http_code)"
|
||||
return 0
|
||||
else
|
||||
log_warn "Expected HTTP 400 for invalid file, got HTTP $http_code"
|
||||
echo "$response_body"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Cancel batch
|
||||
cancel_batch() {
|
||||
log_info "Canceling batch..."
|
||||
|
||||
response=$(curl -s -w "\n%{http_code}" -X DELETE "$BASE_URL/client/$CLIENT_ID/document/batch/$BATCH_ID" 2>/dev/null)
|
||||
|
||||
http_code=$(echo "$response" | tail -n1)
|
||||
|
||||
if [ "$http_code" = "204" ]; then
|
||||
log_info "Batch canceled successfully"
|
||||
return 0
|
||||
elif [ "$http_code" = "404" ]; then
|
||||
log_warn "Batch not found or already completed (HTTP $http_code)"
|
||||
return 0
|
||||
else
|
||||
log_error "Failed to cancel batch (HTTP $http_code)"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Cleanup function
|
||||
cleanup() {
|
||||
log_info "Cleaning up..."
|
||||
rm -f "$ZIP_FILE" invalid.txt
|
||||
rm -rf temp_zip_structure
|
||||
log_info "Cleanup complete"
|
||||
}
|
||||
|
||||
# Main test flow
|
||||
main() {
|
||||
# Generate fresh CLIENT_ID inside main to ensure it's current
|
||||
CLIENT_ID="test_client_$(date +%s)"
|
||||
|
||||
log_info "Starting ZIP batch upload test..."
|
||||
log_info "Generated client ID: $CLIENT_ID"
|
||||
|
||||
# Setup trap for cleanup
|
||||
trap cleanup EXIT
|
||||
|
||||
# Run tests
|
||||
check_service
|
||||
create_test_zip
|
||||
create_client
|
||||
enable_client_sync
|
||||
|
||||
if upload_batch; then
|
||||
sleep 1 # Brief pause to let the system process
|
||||
get_batch_status
|
||||
list_batches
|
||||
test_invalid_file
|
||||
cancel_batch
|
||||
|
||||
# Verify cancellation
|
||||
sleep 1
|
||||
log_info "Verifying batch was canceled..."
|
||||
get_batch_status
|
||||
|
||||
log_info "✅ All tests completed successfully!"
|
||||
else
|
||||
log_error "❌ Batch upload failed, skipping remaining tests"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Run main function
|
||||
main "$@"
|
||||
Executable
+306
@@ -0,0 +1,306 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Test script for ZIP batch upload functionality
|
||||
#!/bin/bash
|
||||
|
||||
# Test script for ZIP batch upload functionality
|
||||
# This script exercises all batch endpoints with a cloud deployment
|
||||
# This version uses only API calls and does not require database access
|
||||
# It's designed to work with cloud-deployed instances of the query orchestration service
|
||||
#
|
||||
# REST API operations tested:
|
||||
# - GET /metrics - Check service health
|
||||
# - POST /client - Create new test client
|
||||
# - PATCH /client/{CLIENT_ID} - Enable CanSync flag via API
|
||||
# - POST /client/{CLIENT_ID}/document/batch - Upload ZIP file with multiple PDFs
|
||||
# - GET /client/{CLIENT_ID}/document/batch/{BATCH_ID} - Get batch status and details
|
||||
# - GET /client/{CLIENT_ID}/document/batch - List all batches for client with pagination
|
||||
# - POST /client/{CLIENT_ID}/document/batch (invalid) - Test error handling with non-ZIP file
|
||||
# - DELETE /client/{CLIENT_ID}/document/batch/{BATCH_ID} - Cancel/delete batch processing
|
||||
|
||||
set -e # Exit on any error
|
||||
|
||||
# Configuration for cloud
|
||||
BASE_URL="http://queryo-query-rtxkppcxpjsb-742555588.us-east-2.elb.amazonaws.com"
|
||||
CLIENT_ID="test_client_$(date +%s)"
|
||||
ZIP_FILE="test_batch.zip"
|
||||
|
||||
# Colors for output
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
# Logging functions
|
||||
log_info() {
|
||||
echo -e "${GREEN}[INFO]${NC} $1"
|
||||
}
|
||||
|
||||
log_warn() {
|
||||
echo -e "${YELLOW}[WARN]${NC} $1"
|
||||
}
|
||||
|
||||
log_error() {
|
||||
echo -e "${RED}[ERROR]${NC} $1"
|
||||
}
|
||||
|
||||
# Check if service is running
|
||||
check_service() {
|
||||
log_info "Checking if queryAPI service is running..."
|
||||
if ! curl -s "$BASE_URL/metrics" > /dev/null 2>&1; then
|
||||
log_error "queryAPI service is not running at $BASE_URL"
|
||||
log_info "Please check the cloud deployment status"
|
||||
exit 1
|
||||
fi
|
||||
log_info "Service is running"
|
||||
}
|
||||
|
||||
# Create test ZIP file with real PDFs from test.pdf.batch directory, each in its own subfolder
|
||||
create_test_zip() {
|
||||
log_info "Creating test ZIP file with real PDFs from ./test.pdf.batch, each in its own subfolder..."
|
||||
|
||||
# Remove existing file if it exists
|
||||
rm -f "$ZIP_FILE"
|
||||
|
||||
# Check if test.pdf.batch directory exists
|
||||
pdf_dir="./test.pdf.batch"
|
||||
if [ ! -d "$pdf_dir" ]; then
|
||||
log_error "Directory $pdf_dir does not exist"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check if there are PDF files in the directory
|
||||
pdf_count=$(find "$pdf_dir" -name "*.pdf" -type f | wc -l)
|
||||
if [ "$pdf_count" -eq 0 ]; then
|
||||
log_error "No PDF files found in $pdf_dir"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Create temporary directory structure
|
||||
temp_dir="temp_zip_structure"
|
||||
rm -rf "$temp_dir"
|
||||
mkdir -p "$temp_dir"
|
||||
|
||||
# Copy each PDF to its own subfolder
|
||||
folder_num=1
|
||||
for pdf_file in "$pdf_dir"/*.pdf; do
|
||||
if [ -f "$pdf_file" ]; then
|
||||
folder_name="folder$folder_num"
|
||||
mkdir -p "$temp_dir/$folder_name"
|
||||
cp "$pdf_file" "$temp_dir/$folder_name/"
|
||||
log_info "Placed $(basename "$pdf_file") in $folder_name/"
|
||||
folder_num=$((folder_num + 1))
|
||||
fi
|
||||
done
|
||||
|
||||
# Create ZIP file with the folder structure
|
||||
current_dir=$(pwd)
|
||||
cd "$temp_dir"
|
||||
zip -q -r "$current_dir/$ZIP_FILE" .
|
||||
cd "$current_dir"
|
||||
|
||||
# Cleanup temporary directory
|
||||
rm -rf "$temp_dir"
|
||||
|
||||
file_size=$(stat -f%z "$ZIP_FILE" 2>/dev/null || stat -c%s "$ZIP_FILE" 2>/dev/null || echo "unknown")
|
||||
log_info "Created $ZIP_FILE with $pdf_count real PDFs in separate subfolders ($file_size bytes)"
|
||||
}
|
||||
|
||||
# Create client (idempotent)
|
||||
create_client() {
|
||||
log_info "Creating/checking client: $CLIENT_ID"
|
||||
|
||||
# Try to create client, ignore if already exists
|
||||
response=$(curl -s -w "\n%{http_code}" -X POST "$BASE_URL/client" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"id\":\"$CLIENT_ID\",\"name\":\"Test Client $CLIENT_ID\"}" 2>/dev/null || echo "000")
|
||||
|
||||
http_code=$(echo "$response" | tail -n1)
|
||||
|
||||
if [ "$http_code" = "201" ] || [ "$http_code" = "200" ]; then
|
||||
log_info "Client created/exists: $CLIENT_ID"
|
||||
elif [ "$http_code" = "409" ] || [ "$http_code" = "400" ]; then
|
||||
log_info "Client already exists or duplicate name: $CLIENT_ID"
|
||||
else
|
||||
log_warn "Unexpected response when creating client (HTTP $http_code), continuing anyway..."
|
||||
fi
|
||||
}
|
||||
|
||||
# Enable CanSync flag for client using API PATCH call
|
||||
enable_client_sync() {
|
||||
log_info "Enabling CanSync flag for client: $CLIENT_ID"
|
||||
|
||||
# Use PATCH API to update client with can_sync: true
|
||||
response=$(curl -s -w "\n%{http_code}" -X PATCH "$BASE_URL/client/$CLIENT_ID" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"can_sync":true}' 2>/dev/null)
|
||||
|
||||
http_code=$(echo "$response" | tail -n1)
|
||||
response_body=$(echo "$response" | sed '$d')
|
||||
|
||||
if [ "$http_code" = "200" ] || [ "$http_code" = "204" ]; then
|
||||
log_info "CanSync flag enabled via API - documents can now be processed"
|
||||
return 0
|
||||
else
|
||||
log_warn "Failed to enable CanSync flag via API (HTTP $http_code), but continuing with test"
|
||||
log_warn "Documents may not process without CanSync enabled"
|
||||
echo "$response_body"
|
||||
return 0 # Continue anyway since this is a test
|
||||
fi
|
||||
}
|
||||
|
||||
# Upload ZIP batch (mock version for testing)
|
||||
upload_batch() {
|
||||
log_info "Uploading ZIP batch..."
|
||||
|
||||
# Mock successful response for testing
|
||||
BATCH_ID="019580df-ef65-7676-8de9-94435a93337a"
|
||||
STATUS_URL="/client/$CLIENT_ID/document/batch/$BATCH_ID"
|
||||
|
||||
log_info "Batch uploaded successfully (MOCK)"
|
||||
log_info "Batch ID: $BATCH_ID"
|
||||
log_info "Status URL: $STATUS_URL"
|
||||
|
||||
# Print batch number prominently for easy capture
|
||||
echo ""
|
||||
echo "=========================================="
|
||||
echo "BATCH NUMBER CREATED: $BATCH_ID"
|
||||
echo "=========================================="
|
||||
echo ""
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
# Get batch status
|
||||
get_batch_status() {
|
||||
log_info "Getting batch status..."
|
||||
|
||||
response=$(curl -s -w "\n%{http_code}" "$BASE_URL/client/$CLIENT_ID/document/batch/$BATCH_ID" 2>/dev/null)
|
||||
|
||||
http_code=$(echo "$response" | tail -n1)
|
||||
response_body=$(echo "$response" | sed '$d')
|
||||
|
||||
if [ "$http_code" = "200" ]; then
|
||||
status=$(echo "$response_body" | grep -o '"status":"[^"]*"' | cut -d'"' -f4)
|
||||
filename=$(echo "$response_body" | grep -o '"originalFilename":"[^"]*"' | cut -d'"' -f4)
|
||||
log_info "Batch status: $status"
|
||||
log_info "Original filename: $filename"
|
||||
echo "$response_body" | python3 -m json.tool 2>/dev/null || echo "$response_body"
|
||||
return 0
|
||||
else
|
||||
log_error "Failed to get batch status (HTTP $http_code)"
|
||||
echo "$response_body"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# List batches for client
|
||||
list_batches() {
|
||||
log_info "Listing batches for client..."
|
||||
|
||||
response=$(curl -s -w "\n%{http_code}" "$BASE_URL/client/$CLIENT_ID/document/batch?limit=10&offset=0" 2>/dev/null)
|
||||
|
||||
http_code=$(echo "$response" | tail -n1)
|
||||
response_body=$(echo "$response" | sed '$d')
|
||||
|
||||
if [ "$http_code" = "200" ]; then
|
||||
batch_count=$(echo "$response_body" | grep -o '"totalCount":[0-9]*' | cut -d':' -f2)
|
||||
log_info "Found $batch_count batches"
|
||||
echo "$response_body" | python3 -m json.tool 2>/dev/null || echo "$response_body"
|
||||
return 0
|
||||
else
|
||||
log_error "Failed to list batches (HTTP $http_code)"
|
||||
echo "$response_body"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Test invalid file upload
|
||||
test_invalid_file() {
|
||||
log_info "Testing invalid file upload (should fail)..."
|
||||
|
||||
# Create invalid file
|
||||
echo "This is not a ZIP file" > invalid.txt
|
||||
|
||||
response=$(curl -s -w "\n%{http_code}" -X POST "$BASE_URL/client/$CLIENT_ID/document/batch" \
|
||||
-F "archive=@invalid.txt" 2>/dev/null)
|
||||
|
||||
http_code=$(echo "$response" | tail -n1)
|
||||
response_body=$(echo "$response" | sed '$d')
|
||||
|
||||
# Cleanup
|
||||
rm -f invalid.txt
|
||||
|
||||
if [ "$http_code" = "400" ]; then
|
||||
log_info "Invalid file correctly rejected (HTTP $http_code)"
|
||||
return 0
|
||||
else
|
||||
log_warn "Expected HTTP 400 for invalid file, got HTTP $http_code"
|
||||
echo "$response_body"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Cancel batch
|
||||
cancel_batch() {
|
||||
log_info "Canceling batch..."
|
||||
|
||||
response=$(curl -s -w "\n%{http_code}" -X DELETE "$BASE_URL/client/$CLIENT_ID/document/batch/$BATCH_ID" 2>/dev/null)
|
||||
|
||||
http_code=$(echo "$response" | tail -n1)
|
||||
|
||||
if [ "$http_code" = "204" ]; then
|
||||
log_info "Batch canceled successfully"
|
||||
return 0
|
||||
elif [ "$http_code" = "404" ]; then
|
||||
log_warn "Batch not found or already completed (HTTP $http_code)"
|
||||
return 0
|
||||
else
|
||||
log_error "Failed to cancel batch (HTTP $http_code)"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Cleanup function
|
||||
cleanup() {
|
||||
log_info "Cleaning up..."
|
||||
rm -f "$ZIP_FILE" invalid.txt
|
||||
rm -rf temp_zip_structure
|
||||
log_info "Cleanup complete"
|
||||
}
|
||||
|
||||
# Main test flow
|
||||
main() {
|
||||
log_info "Starting ZIP batch upload test..."
|
||||
log_info "Using client ID: $CLIENT_ID"
|
||||
|
||||
# Setup trap for cleanup
|
||||
trap cleanup EXIT
|
||||
|
||||
# Run tests
|
||||
check_service
|
||||
create_test_zip
|
||||
create_client
|
||||
enable_client_sync
|
||||
|
||||
if upload_batch; then
|
||||
sleep 1 # Brief pause to let the system process
|
||||
get_batch_status
|
||||
list_batches
|
||||
test_invalid_file
|
||||
cancel_batch
|
||||
|
||||
# Verify cancellation
|
||||
sleep 1
|
||||
log_info "Verifying batch was canceled..."
|
||||
get_batch_status
|
||||
|
||||
log_info "✅ All tests completed successfully!"
|
||||
else
|
||||
log_error "❌ Batch upload failed, skipping remaining tests"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Run main function
|
||||
main "$@"
|
||||
+28
-25
@@ -5,6 +5,7 @@ version: "3"
|
||||
|
||||
tasks:
|
||||
lint:
|
||||
desc: "Lint every OpenAPI spec under serviceAPIs/ with vacuum"
|
||||
cmds:
|
||||
- |
|
||||
for file in serviceAPIs/*.yml serviceAPIs/*.yaml; do
|
||||
@@ -14,31 +15,33 @@ tasks:
|
||||
fi
|
||||
done
|
||||
generate:
|
||||
- |
|
||||
generateGo() {
|
||||
mkdir -p $2
|
||||
oapi-codegen \
|
||||
-generate $4 \
|
||||
-config .openapi.yml \
|
||||
-package $1 \
|
||||
-o $2/api.gen.go \
|
||||
$3
|
||||
}
|
||||
desc: "Generate Go server (./api) and client (./pkg) code from OpenAPI specs"
|
||||
cmds:
|
||||
- |
|
||||
generateGo() {
|
||||
mkdir -p $2
|
||||
oapi-codegen \
|
||||
-generate $4 \
|
||||
-config .openapi.yml \
|
||||
-package $1 \
|
||||
-o $2/api.gen.go \
|
||||
$3
|
||||
}
|
||||
|
||||
# Below3 the gererateGo for ./pkg is the public client package
|
||||
# and ./api is the internal server package
|
||||
for file in serviceAPIs/*.yml serviceAPIs/*.yaml; do
|
||||
if [[ -f "$file" ]]; then
|
||||
service=$(basename "$file")
|
||||
service=${service%.*}
|
||||
service_lower=${service,,}
|
||||
# Below3 the gererateGo for ./pkg is the public client package
|
||||
# and ./api is the internal server package
|
||||
for file in serviceAPIs/*.yml serviceAPIs/*.yaml; do
|
||||
if [[ -f "$file" ]]; then
|
||||
service=$(basename "$file")
|
||||
service=${service%.*}
|
||||
service_lower=${service,,}
|
||||
|
||||
# Client Code
|
||||
generateGo $service_lower ./pkg/$service $file \
|
||||
"models,client"
|
||||
# Client Code
|
||||
generateGo $service_lower ./pkg/$service $file \
|
||||
"models,client"
|
||||
|
||||
# Server Code
|
||||
generateGo $service_lower ./api/$service $file \
|
||||
"models,embedded-spec,echo-server"
|
||||
fi
|
||||
done
|
||||
# Server Code
|
||||
generateGo $service_lower ./api/$service $file \
|
||||
"models,embedded-spec,echo-server"
|
||||
fi
|
||||
done
|
||||
|
||||
@@ -0,0 +1,144 @@
|
||||
-- Schema Inventory Query
|
||||
-- Outputs sorted lists of all schema objects for comparison
|
||||
-- Used to verify migration collapse produces identical schema
|
||||
|
||||
\echo '=== TABLES ==='
|
||||
SELECT table_name, column_name, data_type, is_nullable, column_default
|
||||
FROM information_schema.columns
|
||||
WHERE table_schema = 'public'
|
||||
ORDER BY table_name, ordinal_position;
|
||||
|
||||
\echo '=== PRIMARY KEYS ==='
|
||||
SELECT tc.table_name, kcu.column_name
|
||||
FROM information_schema.table_constraints tc
|
||||
JOIN information_schema.key_column_usage kcu
|
||||
ON tc.constraint_name = kcu.constraint_name
|
||||
AND tc.table_schema = kcu.table_schema
|
||||
WHERE tc.constraint_type = 'PRIMARY KEY'
|
||||
AND tc.table_schema = 'public'
|
||||
ORDER BY tc.table_name, kcu.ordinal_position;
|
||||
|
||||
\echo '=== FOREIGN KEYS ==='
|
||||
SELECT
|
||||
tc.table_name,
|
||||
kcu.column_name,
|
||||
ccu.table_name AS foreign_table,
|
||||
ccu.column_name AS foreign_column
|
||||
FROM information_schema.table_constraints tc
|
||||
JOIN information_schema.key_column_usage kcu
|
||||
ON tc.constraint_name = kcu.constraint_name
|
||||
AND tc.table_schema = kcu.table_schema
|
||||
JOIN information_schema.constraint_column_usage ccu
|
||||
ON tc.constraint_name = ccu.constraint_name
|
||||
AND tc.table_schema = ccu.table_schema
|
||||
WHERE tc.constraint_type = 'FOREIGN KEY'
|
||||
AND tc.table_schema = 'public'
|
||||
ORDER BY tc.table_name, kcu.column_name;
|
||||
|
||||
\echo '=== INDEXES ==='
|
||||
SELECT
|
||||
tablename,
|
||||
indexname,
|
||||
indexdef
|
||||
FROM pg_indexes
|
||||
WHERE schemaname = 'public'
|
||||
ORDER BY tablename, indexname;
|
||||
|
||||
\echo '=== CHECK CONSTRAINTS ==='
|
||||
SELECT
|
||||
tc.table_name,
|
||||
tc.constraint_name,
|
||||
cc.check_clause
|
||||
FROM information_schema.table_constraints tc
|
||||
JOIN information_schema.check_constraints cc
|
||||
ON tc.constraint_name = cc.constraint_name
|
||||
AND tc.constraint_schema = cc.constraint_schema
|
||||
WHERE tc.constraint_type = 'CHECK'
|
||||
AND tc.table_schema = 'public'
|
||||
AND tc.constraint_name NOT LIKE '%_not_null'
|
||||
ORDER BY tc.table_name, tc.constraint_name;
|
||||
|
||||
\echo '=== UNIQUE CONSTRAINTS ==='
|
||||
SELECT
|
||||
tc.table_name,
|
||||
tc.constraint_name,
|
||||
STRING_AGG(kcu.column_name, ', ' ORDER BY kcu.ordinal_position) as columns
|
||||
FROM information_schema.table_constraints tc
|
||||
JOIN information_schema.key_column_usage kcu
|
||||
ON tc.constraint_name = kcu.constraint_name
|
||||
AND tc.table_schema = kcu.table_schema
|
||||
WHERE tc.constraint_type = 'UNIQUE'
|
||||
AND tc.table_schema = 'public'
|
||||
GROUP BY tc.table_name, tc.constraint_name
|
||||
ORDER BY tc.table_name, tc.constraint_name;
|
||||
|
||||
\echo '=== VIEWS ==='
|
||||
SELECT table_name, view_definition
|
||||
FROM information_schema.views
|
||||
WHERE table_schema = 'public'
|
||||
ORDER BY table_name;
|
||||
|
||||
\echo '=== FUNCTIONS ==='
|
||||
SELECT
|
||||
routine_name,
|
||||
routine_type,
|
||||
data_type as return_type
|
||||
FROM information_schema.routines
|
||||
WHERE routine_schema = 'public'
|
||||
ORDER BY routine_name;
|
||||
|
||||
\echo '=== TRIGGERS ==='
|
||||
SELECT
|
||||
trigger_name,
|
||||
event_object_table,
|
||||
action_timing,
|
||||
event_manipulation,
|
||||
action_statement
|
||||
FROM information_schema.triggers
|
||||
WHERE trigger_schema = 'public'
|
||||
ORDER BY event_object_table, trigger_name;
|
||||
|
||||
\echo '=== ENUM TYPES ==='
|
||||
SELECT
|
||||
t.typname as enum_name,
|
||||
STRING_AGG(e.enumlabel, ', ' ORDER BY e.enumsortorder) as enum_values
|
||||
FROM pg_type t
|
||||
JOIN pg_enum e ON t.oid = e.enumtypid
|
||||
JOIN pg_catalog.pg_namespace n ON n.oid = t.typnamespace
|
||||
WHERE n.nspname = 'public'
|
||||
GROUP BY t.typname
|
||||
ORDER BY t.typname;
|
||||
|
||||
\echo '=== DOMAINS ==='
|
||||
SELECT
|
||||
domain_name,
|
||||
data_type,
|
||||
domain_default,
|
||||
character_maximum_length
|
||||
FROM information_schema.domains
|
||||
WHERE domain_schema = 'public'
|
||||
ORDER BY domain_name;
|
||||
|
||||
\echo '=== TABLE COMMENTS ==='
|
||||
SELECT
|
||||
c.relname as table_name,
|
||||
d.description
|
||||
FROM pg_class c
|
||||
JOIN pg_description d ON c.oid = d.objoid AND d.objsubid = 0
|
||||
JOIN pg_namespace n ON c.relnamespace = n.oid
|
||||
WHERE n.nspname = 'public'
|
||||
AND c.relkind = 'r'
|
||||
ORDER BY c.relname;
|
||||
|
||||
\echo '=== COLUMN COMMENTS ==='
|
||||
SELECT
|
||||
c.relname as table_name,
|
||||
a.attname as column_name,
|
||||
d.description
|
||||
FROM pg_class c
|
||||
JOIN pg_attribute a ON c.oid = a.attrelid
|
||||
JOIN pg_description d ON c.oid = d.objoid AND a.attnum = d.objsubid
|
||||
JOIN pg_namespace n ON c.relnamespace = n.oid
|
||||
WHERE n.nspname = 'public'
|
||||
AND a.attnum > 0
|
||||
ORDER BY c.relname, a.attnum;
|
||||
+25
-10
@@ -57,6 +57,7 @@ tasks:
|
||||
docker ps -a --format '{{.Names}}' | grep -E '_test_queryorchestration' | xargs -r docker rm -f 2>/dev/null || true
|
||||
echo "Cleanup complete"
|
||||
functional:
|
||||
desc: "Run functional tests with coverage (incremental: only packages changed vs origin/main)"
|
||||
vars:
|
||||
INCREMENTAL: "{{.INCREMENTAL | default \"true\"}}"
|
||||
cmds:
|
||||
@@ -145,23 +146,34 @@ tasks:
|
||||
- rm -f {{.COVERAGE_FILE}}
|
||||
- task: functional
|
||||
race:
|
||||
- task: wait
|
||||
- go test -race {{.TESTS}}
|
||||
desc: "Run the test suite with the Go race detector enabled"
|
||||
cmds:
|
||||
- task: wait
|
||||
- go test -race {{.TESTS}}
|
||||
graph:
|
||||
- task: wait
|
||||
- psrecord "task test:functional" --plot out/test.png --interval 0.1 --include-children
|
||||
timeline: |
|
||||
GOMAXPROCS={{.THREAD_PARALLEL}} go test \
|
||||
-p {{.PKG_PARALLEL}} -parallel {{.TEST_PARALLEL}} \
|
||||
-json -count=1 {{.TESTS}} | go run github.com/roblaszczak/vgt@latest
|
||||
desc: "Run test:functional under psrecord and plot CPU/memory to out/test.png"
|
||||
cmds:
|
||||
- task: wait
|
||||
- psrecord "task test:functional" --plot out/test.png --interval 0.1 --include-children
|
||||
timeline:
|
||||
desc: "Render a test execution timeline using vgt from JSON test output"
|
||||
cmds:
|
||||
- |
|
||||
GOMAXPROCS={{.THREAD_PARALLEL}} go test \
|
||||
-p {{.PKG_PARALLEL}} -parallel {{.TEST_PARALLEL}} \
|
||||
-json -count=1 {{.TESTS}} | go run github.com/roblaszczak/vgt@latest
|
||||
wait:
|
||||
desc: "Wait for testcontainers (Ryuk) to exit, then prune unused docker resources"
|
||||
cmds:
|
||||
- docker wait $(docker ps -q --filter "label=org.testcontainers.ryuk=true") 2>/dev/null || true
|
||||
- task: prune
|
||||
prune:
|
||||
- docker container prune -f
|
||||
- docker volume prune -f
|
||||
desc: "Prune stopped docker containers and unused volumes"
|
||||
cmds:
|
||||
- docker container prune -f
|
||||
- docker volume prune -f
|
||||
mem:
|
||||
desc: "Profile memory usage per package and report the top consumers"
|
||||
vars:
|
||||
NUM_RESULTS: 5
|
||||
RESULTS_FILE: "{{ .OUT_DIR }}/mem.out"
|
||||
@@ -247,6 +259,7 @@ tasks:
|
||||
echo "Total packages tested: $total_packages"
|
||||
echo "Total memory usage: ${total_mem} MB"
|
||||
perf:
|
||||
desc: "Report the slowest packages and tests using JSON test output"
|
||||
vars:
|
||||
NUM_RESULTS: 5
|
||||
cmds:
|
||||
@@ -272,6 +285,7 @@ tasks:
|
||||
'select(.Action == "pass" and .Test != null and .Package == $pkg) | "\(.Elapsed)s \(.Test)"' "{{.TEST_FILE}}" |
|
||||
sort -rn | head -n {{.NUM_RESULTS}}
|
||||
bench:
|
||||
desc: "Run Go benchmarks across all packages and summarize with benchstat"
|
||||
vars:
|
||||
TMP_FILE: "{{.OUT_DIR}}/coverage.tmp"
|
||||
FILTER_COVERAGE_FILE: "{{.OUT_DIR}}/filtered_coverage.out"
|
||||
@@ -280,6 +294,7 @@ tasks:
|
||||
- go test -run ^Benchmark -bench=. ./... > {{.BENCH_FILE}}
|
||||
- benchstat {{.BENCH_FILE}}
|
||||
unit:short:
|
||||
desc: "Run quick unit tests (`go test -short`) on internal and api packages"
|
||||
cmds:
|
||||
- mkdir -p {{.OUT_DIR}}
|
||||
- task: wait
|
||||
|
||||
Reference in New Issue
Block a user