Merged in feature/local-pdp (pull request #208)
permit.io local pdp * docs and testing * build fix * docs * permit rights * accept self signed certs * fix eula error
This commit is contained in:
Jay Brown
@@ -79,3 +79,4 @@ Thumbs.db
|
||||
*.zip
|
||||
|
||||
**/.claude/settings.local.json
|
||||
deployments/permit.env
|
||||
|
||||
@@ -208,6 +208,62 @@ PERMIT_IO_PDP_URL=http://localhost:7766 # Permit.io Policy Decision Point URL -
|
||||
Note: The Permit.io project and environment IDs are automatically derived from the API key
|
||||
using the `/v2/api-key/scope` endpoint. Use a project-level or environment-level API key.
|
||||
|
||||
#### PDP Sidecar Deployment (AWS)
|
||||
|
||||
The Permit.io Policy Decision Point (PDP) runs as a sidecar container alongside queryAPI.
|
||||
The PDP syncs policies from Permit.io cloud and serves authorization decisions locally for
|
||||
low-latency checks.
|
||||
|
||||
**Container Image:**
|
||||
```
|
||||
permitio/pdp-v2:0.9.9
|
||||
```
|
||||
|
||||
> **Ops Note:** The PDP image version should be stored in soft config (e.g., SSM Parameter Store
|
||||
> or environment configuration) so upgrades require no code changes. Check for new stable versions
|
||||
> at [Docker Hub](https://hub.docker.com/r/permitio/pdp-v2/tags) quarterly or when issues arise.
|
||||
> Avoid `-rc` (release candidate) tags in production.
|
||||
|
||||
**ECS Task Definition Setup:**
|
||||
|
||||
Add the PDP as a sidecar container in the same task definition as queryAPI:
|
||||
|
||||
| Setting | PDP Sidecar | queryAPI |
|
||||
|---------|-------------|----------|
|
||||
| Container Name | `permit-pdp` | `query-api` |
|
||||
| Image | `permitio/pdp-v2:0.9.9` | `<your-ecr>/queryorchestration:latest` |
|
||||
| Port Mappings | 7000 (container) | 8080 (container) |
|
||||
| Essential | Yes | Yes |
|
||||
|
||||
**PDP Environment Variables:**
|
||||
|
||||
| Variable | Value | Source |
|
||||
|----------|-------|--------|
|
||||
| `PDP_API_KEY` | Permit.io API key for target environment | Secrets Manager |
|
||||
| `PDP_DEBUG` | `false` (production) | Task definition |
|
||||
|
||||
**queryAPI Environment Variables:**
|
||||
|
||||
| Variable | Value | Notes |
|
||||
|----------|-------|-------|
|
||||
| `PERMIT_IO_API_KEY` | Same key as PDP | Secrets Manager |
|
||||
| `PERMIT_IO_PDP_URL` | `http://localhost:7000` | Sidecar shares localhost |
|
||||
| `PERMIT_IO_TENANT` | `default` or your tenant | Task definition |
|
||||
|
||||
**Health Check:**
|
||||
```
|
||||
GET http://localhost:7000/health
|
||||
```
|
||||
|
||||
**Resource Recommendations (per Permit.io docs):**
|
||||
- CPU: 256-1024 units (start with 256)
|
||||
- Memory: 512 MB minimum, scale by `(policy objects * 6KB)`
|
||||
|
||||
**Authentication Flow:**
|
||||
1. PDP authenticates to Permit.io cloud using `PDP_API_KEY` to sync policies
|
||||
2. queryAPI authenticates to PDP using `PERMIT_IO_API_KEY` as Bearer token
|
||||
3. Both keys should be the same environment-scoped API key
|
||||
|
||||
### Optional Environment Variables
|
||||
|
||||
#### Development & Testing
|
||||
|
||||
@@ -5,6 +5,7 @@ package queryapi
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -50,8 +51,12 @@ func (s *Controllers) ListEulaVersions(ctx echo.Context, params ListEulaVersions
|
||||
PageSize: pageSize,
|
||||
})
|
||||
if err != nil {
|
||||
s.cfg.GetAuthLogger().Error("ListEulaVersions: failed to list EULA versions",
|
||||
"error", err,
|
||||
"page", page,
|
||||
"page_size", pageSize)
|
||||
return ctx.JSON(http.StatusInternalServerError, ErrorMessage{
|
||||
Message: "Failed to list EULA versions",
|
||||
Message: fmt.Sprintf("Failed to list EULA versions: %v", err),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -123,8 +128,12 @@ func (s *Controllers) CreateEulaVersion(ctx echo.Context) error {
|
||||
Message: "EULA version already exists",
|
||||
})
|
||||
}
|
||||
s.cfg.GetAuthLogger().Error("CreateEulaVersion: failed to create EULA version",
|
||||
"error", err,
|
||||
"version", req.Version,
|
||||
"created_by", adminUser.Email)
|
||||
return ctx.JSON(http.StatusInternalServerError, ErrorMessage{
|
||||
Message: "Failed to create EULA version",
|
||||
Message: fmt.Sprintf("Failed to create EULA version: %v", err),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -157,8 +166,11 @@ func (s *Controllers) GetEulaVersion(ctx echo.Context, versionID EulaVersionID)
|
||||
Message: "EULA version not found",
|
||||
})
|
||||
}
|
||||
s.cfg.GetAuthLogger().Error("GetEulaVersion: failed to get EULA version",
|
||||
"error", err,
|
||||
"version_id", id)
|
||||
return ctx.JSON(http.StatusInternalServerError, ErrorMessage{
|
||||
Message: "Failed to get EULA version",
|
||||
Message: fmt.Sprintf("Failed to get EULA version: %v", err),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -210,8 +222,11 @@ func (s *Controllers) UpdateEulaVersion(ctx echo.Context, versionID EulaVersionI
|
||||
Message: "EULA version not found",
|
||||
})
|
||||
}
|
||||
s.cfg.GetAuthLogger().Error("UpdateEulaVersion: failed to update EULA version",
|
||||
"error", err,
|
||||
"version_id", id)
|
||||
return ctx.JSON(http.StatusInternalServerError, ErrorMessage{
|
||||
Message: "Failed to update EULA version",
|
||||
Message: fmt.Sprintf("Failed to update EULA version: %v", err),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -259,8 +274,12 @@ func (s *Controllers) ActivateEulaVersion(ctx echo.Context, versionID EulaVersio
|
||||
Message: "Concurrent activation conflict - another version was activated. Please refresh and try again.",
|
||||
})
|
||||
}
|
||||
s.cfg.GetAuthLogger().Error("ActivateEulaVersion: failed to activate EULA version",
|
||||
"error", err,
|
||||
"version_id", id,
|
||||
"activated_by", adminUser.Email)
|
||||
return ctx.JSON(http.StatusInternalServerError, ErrorMessage{
|
||||
Message: "Failed to activate EULA version",
|
||||
Message: fmt.Sprintf("Failed to activate EULA version: %v", err),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -310,8 +329,12 @@ func (s *Controllers) ListEulaAgreements(ctx echo.Context, params ListEulaAgreem
|
||||
|
||||
result, err := s.svc.Eula.ListAgreements(ctx.Request().Context(), input)
|
||||
if err != nil {
|
||||
s.cfg.GetAuthLogger().Error("ListEulaAgreements: failed to list EULA agreements",
|
||||
"error", err,
|
||||
"page", page,
|
||||
"page_size", pageSize)
|
||||
return ctx.JSON(http.StatusInternalServerError, ErrorMessage{
|
||||
Message: "Failed to list EULA agreements",
|
||||
Message: fmt.Sprintf("Failed to list EULA agreements: %v", err),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -355,8 +378,10 @@ func (s *Controllers) GetEulaCompliance(ctx echo.Context, params GetEulaComplian
|
||||
// Initialize AWS config for Cognito
|
||||
awsCfg, err := aws.GetAWSConfig(context.Background())
|
||||
if err != nil {
|
||||
s.cfg.GetAuthLogger().Error("GetEulaCompliance: failed to initialize AWS configuration",
|
||||
"error", err)
|
||||
return ctx.JSON(http.StatusInternalServerError, ErrorMessage{
|
||||
Message: "Failed to initialize AWS configuration",
|
||||
Message: fmt.Sprintf("Failed to initialize AWS configuration: %v", err),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -399,12 +424,16 @@ func (s *Controllers) GetEulaCompliance(ctx echo.Context, params GetEulaComplian
|
||||
}
|
||||
// Check if it's a Cognito error
|
||||
if strings.Contains(err.Error(), "Cognito") || strings.Contains(err.Error(), "cognito") {
|
||||
s.cfg.GetAuthLogger().Error("GetEulaCompliance: Cognito error during compliance report",
|
||||
"error", err)
|
||||
return ctx.JSON(http.StatusBadGateway, ErrorMessage{
|
||||
Message: "Failed to fetch users from Cognito: " + err.Error(),
|
||||
})
|
||||
}
|
||||
s.cfg.GetAuthLogger().Error("GetEulaCompliance: failed to generate compliance report",
|
||||
"error", err)
|
||||
return ctx.JSON(http.StatusInternalServerError, ErrorMessage{
|
||||
Message: "Failed to generate compliance report",
|
||||
Message: fmt.Sprintf("Failed to generate compliance report: %v", err),
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -243,6 +243,33 @@ func TestAgreeToEula(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, http.StatusUnauthorized, rec.Code)
|
||||
})
|
||||
|
||||
t.Run("succeeds with username fallback when email claim is missing", func(t *testing.T) {
|
||||
// Simulate a Cognito access token that has 'sub' and 'cognito:username'
|
||||
// but no 'email' claim. This is the default for Cognito access tokens.
|
||||
userSubject := "no-email-user-" + uuid.New().String()[:8]
|
||||
ctx, rec := setupEulaTestContext(http.MethodPost, "/eula/agree", nil)
|
||||
ctx.Request().RemoteAddr = "192.168.1.100:12345"
|
||||
|
||||
// Set claims with sub but without email (mimics access token)
|
||||
ctx.Set("user_claims", map[string]interface{}{
|
||||
"sub": userSubject,
|
||||
"cognito:username": "testuser",
|
||||
})
|
||||
ctx.Set("user_info", cognitoauth.UserInfo{
|
||||
Username: "testuser",
|
||||
Email: "", // empty - no email in access token
|
||||
})
|
||||
|
||||
err := cons.AgreeToEula(ctx)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, http.StatusCreated, rec.Code)
|
||||
|
||||
var response queryapi.EulaAgreementResponse
|
||||
err = json.Unmarshal(rec.Body.Bytes(), &response)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, versionStr, response.EulaVersion)
|
||||
})
|
||||
}
|
||||
|
||||
func TestListEulaVersions(t *testing.T) {
|
||||
|
||||
@@ -4,6 +4,7 @@ package queryapi
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
@@ -32,8 +33,10 @@ func (s *Controllers) GetCurrentEula(ctx echo.Context) error {
|
||||
Message: "No current EULA version configured",
|
||||
})
|
||||
}
|
||||
s.cfg.GetAuthLogger().Error("GetCurrentEula: failed to get current EULA version",
|
||||
"error", err)
|
||||
return ctx.JSON(http.StatusInternalServerError, ErrorMessage{
|
||||
Message: "Failed to get current EULA version",
|
||||
Message: fmt.Sprintf("Failed to get current EULA version: %v", err),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -79,8 +82,11 @@ func (s *Controllers) GetEulaStatus(ctx echo.Context) error {
|
||||
Message: "No current EULA version configured",
|
||||
})
|
||||
}
|
||||
s.cfg.GetAuthLogger().Error("GetEulaStatus: failed to get EULA status",
|
||||
"error", err,
|
||||
"user_subject", userSubject)
|
||||
return ctx.JSON(http.StatusInternalServerError, ErrorMessage{
|
||||
Message: "Failed to get EULA status",
|
||||
Message: fmt.Sprintf("Failed to get EULA status: %v", err),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -119,11 +125,25 @@ func (s *Controllers) AgreeToEula(ctx echo.Context) error {
|
||||
userSubject, subjectOK := cognitoauth.GetUserSubject(ctx)
|
||||
userInfo, infoOK := cognitoauth.GetUserInfo(ctx)
|
||||
if !subjectOK || !infoOK {
|
||||
s.cfg.GetAuthLogger().Error("AgreeToEula: authentication info missing from context",
|
||||
"subject_ok", subjectOK,
|
||||
"info_ok", infoOK)
|
||||
return ctx.JSON(http.StatusUnauthorized, ErrorMessage{
|
||||
Message: "Authentication required",
|
||||
})
|
||||
}
|
||||
|
||||
// Resolve the user identifier for the agreement record.
|
||||
// Cognito access tokens do not include the email claim by default (only ID tokens do).
|
||||
// Fall back to the username (cognito:username), which is always present in access tokens.
|
||||
userIdentifier := userInfo.Email
|
||||
if userIdentifier == "" {
|
||||
userIdentifier = userInfo.Username
|
||||
s.cfg.GetAuthLogger().Info("AgreeToEula: email claim missing from token, using username as fallback",
|
||||
"user_subject", userSubject,
|
||||
"username", userInfo.Username)
|
||||
}
|
||||
|
||||
// Get current EULA version
|
||||
currentVersion, err := s.svc.Eula.GetCurrentVersion(ctx.Request().Context())
|
||||
if err != nil {
|
||||
@@ -132,8 +152,11 @@ func (s *Controllers) AgreeToEula(ctx echo.Context) error {
|
||||
Message: "No current EULA version configured",
|
||||
})
|
||||
}
|
||||
s.cfg.GetAuthLogger().Error("AgreeToEula: failed to get current EULA version",
|
||||
"error", err,
|
||||
"user_subject", userSubject)
|
||||
return ctx.JSON(http.StatusInternalServerError, ErrorMessage{
|
||||
Message: "Failed to get current EULA version",
|
||||
Message: fmt.Sprintf("Failed to get current EULA version: %v", err),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -146,13 +169,18 @@ func (s *Controllers) AgreeToEula(ctx echo.Context) error {
|
||||
// Record the agreement
|
||||
result, err := s.svc.Eula.RecordAgreement(ctx.Request().Context(), &eula.RecordAgreementInput{
|
||||
CognitoSubjectID: userSubject,
|
||||
UserEmail: userInfo.Email,
|
||||
UserEmail: userIdentifier,
|
||||
EulaVersionID: currentVersion.ID,
|
||||
IPAddress: ipAddress,
|
||||
})
|
||||
if err != nil {
|
||||
s.cfg.GetAuthLogger().Error("AgreeToEula: failed to record EULA agreement",
|
||||
"error", err,
|
||||
"user_subject", userSubject,
|
||||
"user_identifier", userIdentifier,
|
||||
"eula_version_id", currentVersion.ID)
|
||||
return ctx.JSON(http.StatusInternalServerError, ErrorMessage{
|
||||
Message: "Failed to record EULA agreement",
|
||||
Message: fmt.Sprintf("Failed to record EULA agreement: %v", err),
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -50,6 +50,12 @@ resources:
|
||||
- get
|
||||
- post
|
||||
|
||||
- name: documents
|
||||
description: Document search and listing operations
|
||||
actions:
|
||||
- get
|
||||
- post
|
||||
|
||||
- name: field-extractions
|
||||
description: Field extraction data for documents
|
||||
actions:
|
||||
@@ -100,6 +106,9 @@ roles:
|
||||
export:
|
||||
- get
|
||||
- post
|
||||
documents:
|
||||
- get
|
||||
- post
|
||||
field-extractions:
|
||||
- get
|
||||
- post
|
||||
@@ -119,6 +128,9 @@ roles:
|
||||
- patch
|
||||
- delete
|
||||
- head
|
||||
documents:
|
||||
- get
|
||||
- post
|
||||
field-extractions:
|
||||
- get
|
||||
- post
|
||||
@@ -144,6 +156,8 @@ roles:
|
||||
- get
|
||||
export:
|
||||
- get
|
||||
documents:
|
||||
- get
|
||||
field-extractions:
|
||||
- get
|
||||
folders:
|
||||
@@ -159,11 +173,16 @@ roles:
|
||||
- post
|
||||
- patch
|
||||
- delete
|
||||
clients:
|
||||
- get
|
||||
document:
|
||||
- get
|
||||
export:
|
||||
- get
|
||||
- post
|
||||
documents:
|
||||
- get
|
||||
- post
|
||||
field-extractions:
|
||||
- get
|
||||
- post
|
||||
|
||||
@@ -263,12 +263,17 @@ services:
|
||||
condition: service_healthy
|
||||
db:
|
||||
condition: service_healthy
|
||||
permit_pdp:
|
||||
condition: service_healthy
|
||||
ports:
|
||||
- "8080:8080"
|
||||
- "2345:2345"
|
||||
expose:
|
||||
- 8080
|
||||
- 2345
|
||||
env_file:
|
||||
- path: permit.env
|
||||
required: false
|
||||
environment:
|
||||
LOG_LEVEL: DEBUG
|
||||
DEBUG: true
|
||||
@@ -293,8 +298,7 @@ services:
|
||||
COGNITO_DOMAIN: ${COGNITO_DOMAIN}
|
||||
COGNITO_CLIENT_ID: ${COGNITO_CLIENT_ID}
|
||||
COGNITO_REGION: ${COGNITO_REGION}
|
||||
PERMIT_IO_API_KEY: ${PERMIT_IO_API_KEY}
|
||||
PERMIT_IO_PDP_URL: ${PERMIT_IO_PDP_URL}
|
||||
PERMIT_IO_PDP_URL: http://permit_pdp:7000
|
||||
PERMIT_IO_TENANT: ${PERMIT_IO_TENANT}
|
||||
BUCKET: ${BUCKET_IN}
|
||||
networks:
|
||||
@@ -372,6 +376,24 @@ services:
|
||||
networks:
|
||||
- server-network
|
||||
|
||||
permit_pdp:
|
||||
image: permitio/pdp-v2:0.9.9
|
||||
ports:
|
||||
- "7766:7000"
|
||||
expose:
|
||||
- 7000
|
||||
env_file:
|
||||
- path: permit.env
|
||||
required: false
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "--spider", "-q", "http://localhost:7000/health"]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 10
|
||||
start_period: 10s
|
||||
networks:
|
||||
- server-network
|
||||
|
||||
prometheus:
|
||||
image: prom/prometheus:latest
|
||||
ports:
|
||||
|
||||
+1
-1
@@ -23,8 +23,8 @@
|
||||
"DOCUMENT_SYNC_URL": "http://localstack:4566/queue/us-east-1/000000000000/document_sync",
|
||||
"DOCUMENT_TEXT_URL": "http://localstack:4566/queue/us-east-1/000000000000/document_text",
|
||||
"PERMIT_IO_API_KEY": "permit_key_omitted",
|
||||
"PERMIT_IO_PROJECT_ID": "default",
|
||||
"PERMIT_IO_ENV_ID": "dev",
|
||||
"PERMIT_IO_PROJECT_ID": "default",
|
||||
"PERMIT_IO_TENANT": "default",
|
||||
"PGDATABASE": "query_orchestration",
|
||||
"PGHOST": "localhost",
|
||||
|
||||
@@ -24,7 +24,6 @@ The API is organized into the following service groups:
|
||||
| FolderService | Operations related to document folders and organization |
|
||||
| LabelService | Operations related to document labels and workflow tracking |
|
||||
| FieldExtractionService | Operations related to document field extractions |
|
||||
| QueryService | Operations related to queries |
|
||||
| ExportService | Operations related to exports |
|
||||
| AuthService | Operations related to authentication |
|
||||
| AdminService | Operations related to user management and administration |
|
||||
@@ -81,6 +80,34 @@ Returns the HTML menu page for authenticated users.
|
||||
**Security**: Requires JWT authentication
|
||||
**Response**: `200` HTML content
|
||||
|
||||
#### `GET /identity`
|
||||
|
||||
Returns the authenticated user's assigned roles from Permit.io, including the permissions granted by each role. Any authenticated user can call this endpoint to discover their own roles. This endpoint requires JWT authentication but does not require specific Permit.io permissions.
|
||||
|
||||
**Security**: Requires JWT authentication (auth-only, no Permit.io check)
|
||||
|
||||
**Response** (`IdentityResponse`):
|
||||
|
||||
```json
|
||||
{
|
||||
"roles": [
|
||||
{
|
||||
"key": "auditor",
|
||||
"name": "Auditor",
|
||||
"description": "Read-only access to view documents and exports",
|
||||
"permissions": ["document:read", "document:list", "export:read"]
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
**Responses**:
|
||||
|
||||
- `200`: User roles and permissions retrieved successfully
|
||||
- `401`: Authentication required
|
||||
- `429`: Rate limit exceeded
|
||||
- `502`: Failed to retrieve roles from Permit.io
|
||||
|
||||
## Rate Limiting
|
||||
|
||||
The queryAPI service implements dual-layer rate limiting to protect against DDoS attacks and ensure fair resource allocation.
|
||||
@@ -153,6 +180,27 @@ RateLimit: 20;window=2
|
||||
|
||||
## Client Management (ClientService)
|
||||
|
||||
### `GET /clients`
|
||||
|
||||
Lists all clients in the system.
|
||||
|
||||
**Security**: Requires JWT authentication
|
||||
|
||||
**Response** (`ClientListResponse`):
|
||||
|
||||
```json
|
||||
{
|
||||
"clients": [
|
||||
{
|
||||
"id": "AAA",
|
||||
"name": "ACME Corporation",
|
||||
"can_sync": true
|
||||
}
|
||||
],
|
||||
"totalCount": 1
|
||||
}
|
||||
```
|
||||
|
||||
### `POST /client`
|
||||
|
||||
Creates a new client in the system.
|
||||
@@ -272,7 +320,7 @@ Lists documents for a client.
|
||||
|
||||
### `GET /document/{id}`
|
||||
|
||||
Retrieves document details by ID.
|
||||
Retrieves enriched document details by ID, including metadata such as folder, filename, labels, and optionally the full text extraction record.
|
||||
|
||||
**Security**: Requires JWT authentication
|
||||
|
||||
@@ -280,17 +328,31 @@ Retrieves document details by ID.
|
||||
|
||||
- `id`: Document UUID (max 36 chars)
|
||||
|
||||
**Response** (`Document`):
|
||||
**Query Parameters**:
|
||||
|
||||
- `textRecord` (optional, default: false): When true, includes the full text extraction record in the response
|
||||
|
||||
**Response** (`DocumentEnriched`):
|
||||
|
||||
```json
|
||||
{
|
||||
"id": "019580df-b3f8-7348-9ab1-1e55b3f18ed7",
|
||||
"client_id": "AAA",
|
||||
"hash": "sha256:abc123...",
|
||||
"fields": {
|
||||
"property1": "string value",
|
||||
"property2": 42
|
||||
"hasTextRecord": true,
|
||||
"folderId": "019580df-ef65-7676-8de9-94435a93337a",
|
||||
"filename": "contract_2024.pdf",
|
||||
"originalPath": "/documents/2024/contract_2024.pdf",
|
||||
"labels": [
|
||||
{
|
||||
"id": "019580df-ef65-7676-8de9-94435a93337b",
|
||||
"documentId": "019580df-b3f8-7348-9ab1-1e55b3f18ed7",
|
||||
"label": "OCR_Processed",
|
||||
"appliedBy": "user@example.com",
|
||||
"appliedAt": "2025-10-16T14:27:15Z"
|
||||
}
|
||||
],
|
||||
"fileSizeBytes": 1048576
|
||||
}
|
||||
```
|
||||
|
||||
@@ -450,6 +512,10 @@ Lists all folders for a specific client, including the folder hierarchy.
|
||||
|
||||
- `clientId`: Client external ID (string, max 36 chars)
|
||||
|
||||
**Query Parameters**:
|
||||
|
||||
- `metrics` (optional, default: false): When true, includes processing metrics for each folder (document counts by label). Adds overhead so should only be used when metrics are needed.
|
||||
|
||||
**Response** (`200 OK`):
|
||||
|
||||
```json
|
||||
@@ -548,10 +614,14 @@ Renames an existing folder.
|
||||
|
||||
### `GET /folders/{folderId}/documents`
|
||||
|
||||
Retrieves all documents within a specific folder.
|
||||
Retrieves all documents within a specific folder, including filename and labels. Returns a lighter-weight format than `GET /document/{id}` (excludes `client_id`).
|
||||
|
||||
**Security**: Requires JWT authentication
|
||||
|
||||
**Query Parameters**:
|
||||
|
||||
- `textRecord` (optional, default: false): When true, includes the full text extraction record for each document
|
||||
|
||||
**Response**:
|
||||
|
||||
```json
|
||||
@@ -559,9 +629,13 @@ Retrieves all documents within a specific folder.
|
||||
"documents": [
|
||||
{
|
||||
"id": "019580df-b3f8-7348-9ab1-1e55b3f18ed7",
|
||||
"client_id": "AAA",
|
||||
"hash": "sha256:abc123...",
|
||||
"fields": {}
|
||||
"hasTextRecord": true,
|
||||
"folderId": "019580df-ef65-7676-8de9-94435a93337a",
|
||||
"filename": "contract_2024.pdf",
|
||||
"originalPath": "/documents/2024/contract_2024.pdf",
|
||||
"labels": [],
|
||||
"fileSizeBytes": 1048576
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -661,7 +735,7 @@ Retrieves all documents that have been tagged with a specific label.
|
||||
|
||||
**Query Parameters**:
|
||||
|
||||
- `clientId` (required): Client ID to filter documents (UUID)
|
||||
- `clientId` (required): Client ID to filter documents (string, max 36 chars)
|
||||
|
||||
**Response**:
|
||||
|
||||
@@ -834,106 +908,6 @@ Retrieves a specific version of a field extraction for a document by version num
|
||||
|
||||
---
|
||||
|
||||
## Query Management (QueryService)
|
||||
|
||||
### `GET /query`
|
||||
|
||||
Lists all available queries in the system.
|
||||
|
||||
**Security**: Requires JWT authentication
|
||||
|
||||
**Response** (`ListQueries`):
|
||||
|
||||
```json
|
||||
{
|
||||
"queries": [
|
||||
{
|
||||
"id": "019580df-ef65-7676-8de9-94435a93337a",
|
||||
"type": "JSON_EXTRACTOR",
|
||||
"active_version": 1,
|
||||
"latest_version": 2,
|
||||
"config": "{...}",
|
||||
"required_queries": ["019580df-ef65-7676-8de9-94435a93337b"]
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
**Query Types**: `JSON_EXTRACTOR`, `CONTEXT_FULL`
|
||||
|
||||
### `POST /query`
|
||||
|
||||
Creates a new query definition.
|
||||
|
||||
**Security**: Requires JWT authentication
|
||||
|
||||
**Request Body** (`QueryCreate`):
|
||||
|
||||
```json
|
||||
{
|
||||
"type": "JSON_EXTRACTOR",
|
||||
"config": "{\"key\": \"value\"}",
|
||||
"required_queries": ["019580df-ef65-7676-8de9-94435a93337b"]
|
||||
}
|
||||
```
|
||||
|
||||
**Response** (`201 Created`):
|
||||
|
||||
```json
|
||||
{
|
||||
"id": "019580df-ef65-7676-8de9-94435a93337a"
|
||||
}
|
||||
```
|
||||
|
||||
### `GET /query/{id}`
|
||||
|
||||
Retrieves query details including configuration.
|
||||
|
||||
**Security**: Requires JWT authentication
|
||||
|
||||
**Response**: `Query` object
|
||||
|
||||
### `PATCH /query/{id}`
|
||||
|
||||
Updates query definition.
|
||||
|
||||
**Security**: Requires JWT authentication
|
||||
|
||||
**Request Body** (`QueryUpdate`):
|
||||
|
||||
```json
|
||||
{
|
||||
"config": "{\"updated\": true}",
|
||||
"active_version": 2,
|
||||
"required_queries": []
|
||||
}
|
||||
```
|
||||
|
||||
### `POST /query/{id}/test`
|
||||
|
||||
Tests query execution against a document.
|
||||
|
||||
**Security**: Requires JWT authentication
|
||||
|
||||
**Request Body** (`QueryTestRequest`):
|
||||
|
||||
```json
|
||||
{
|
||||
"document_id": "019580df-b3f8-7348-9ab1-1e55b3f18ed7",
|
||||
"query_version": 1
|
||||
}
|
||||
```
|
||||
|
||||
**Response** (`QueryTestResponse`):
|
||||
|
||||
```json
|
||||
{
|
||||
"value": "extracted result"
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Collector Configuration (CollectorService)
|
||||
|
||||
### `GET /client/{id}/collector`
|
||||
@@ -1624,6 +1598,16 @@ These endpoints handle authentication flows and bypass normal authorization:
|
||||
| `GET /login` | Initiates OAuth2/PKCE login flow with Cognito |
|
||||
| `GET /login-callback` | Handles OAuth2 callback from Cognito |
|
||||
|
||||
### Auth-Only Endpoints (JWT required, no Permit.io check)
|
||||
|
||||
The following endpoints require a valid JWT token but do **not** check Permit.io permissions. Any authenticated user can access them:
|
||||
|
||||
| Endpoint | Purpose |
|
||||
| ------------------ | ------------------------------------------------ |
|
||||
| `GET /identity` | Users need to discover their roles |
|
||||
| `GET /eula/status` | Users need to check their EULA agreement status |
|
||||
| `POST /eula/agree` | Users need to be able to agree to the EULA |
|
||||
|
||||
### Authorization Reference Table
|
||||
|
||||
The following table shows what data is passed to Permit.io for each protected endpoint:
|
||||
@@ -1660,12 +1644,6 @@ The following table shows what data is passed to Permit.io for each protected en
|
||||
| `GET` | `/field-extractions` | `field-extractions` | `get` | Get latest extraction |
|
||||
| `GET` | `/field-extractions/version` | `field-extractions` | `get` | Get specific version |
|
||||
| `GET` | `/field-extractions/history` | `field-extractions` | `get` | Get extraction history |
|
||||
| **Query Management** |
|
||||
| `GET` | `/query` | `query` | `get` | List queries |
|
||||
| `POST` | `/query` | `query` | `post` | Create query |
|
||||
| `GET` | `/query/{id}` | `query` | `get` | Get query details |
|
||||
| `PATCH` | `/query/{id}` | `query` | `patch` | Update query |
|
||||
| `POST` | `/query/{id}/test` | `query` | `post` | Test query execution |
|
||||
| **Collector Configuration** |
|
||||
| `GET` | `/client/{id}/collector` | `client` | `get` | Get collector config |
|
||||
| `PATCH` | `/client/{id}/collector` | `client` | `patch` | Update collector config |
|
||||
@@ -1683,8 +1661,8 @@ The following table shows what data is passed to Permit.io for each protected en
|
||||
| `POST` | `/admin/users/{email}/enable` | `admin` | `post` | Enable user |
|
||||
| **EULA Operations** |
|
||||
| `GET` | `/eula` | - | - | Public (no auth) |
|
||||
| `GET` | `/eula/status` | `eula` | `get` | Get user's EULA status |
|
||||
| `POST` | `/eula/agree` | `eula` | `post` | Record EULA agreement |
|
||||
| `GET` | `/eula/status` | - | - | Auth-only (no Permit.io)|
|
||||
| `POST` | `/eula/agree` | - | - | Auth-only (no Permit.io)|
|
||||
| `GET` | `/admin/eula` | `admin` | `get` | List EULA versions |
|
||||
| `POST` | `/admin/eula` | `admin` | `post` | Create EULA version |
|
||||
| `GET` | `/admin/eula/{version_id}` | `admin` | `get` | Get EULA version |
|
||||
@@ -1708,10 +1686,8 @@ To configure Permit.io to work with this API, administrators must create:
|
||||
| `folders` | Folder management |
|
||||
| `labels` | Label-based document queries |
|
||||
| `field-extractions` | Field extraction operations |
|
||||
| `query` | Query definition management |
|
||||
| `export` | Export operations |
|
||||
| `admin` | User administration and EULA admin operations |
|
||||
| `eula` | EULA status and agreement operations |
|
||||
|
||||
#### Actions
|
||||
|
||||
@@ -1729,8 +1705,8 @@ Typical role assignments for common personas:
|
||||
|
||||
| Role | Resources | Actions | Use Case |
|
||||
| ------------ | -------------------------------------------------------------------------- | ---------------------- | -------------------------- |
|
||||
| `viewer` | `client`, `clients`, `document`, `documents`, `folders`, `query`, `export` | `get` | Read-only access |
|
||||
| `editor` | `client`, `document`, `documents`, `folders`, `field-extractions`, `query` | `get`, `post`, `patch` | Content management |
|
||||
| `viewer` | `client`, `clients`, `document`, `documents`, `folders`, `export` | `get` | Read-only access |
|
||||
| `editor` | `client`, `document`, `documents`, `folders`, `field-extractions` | `get`, `post`, `patch` | Content management |
|
||||
| `admin` | All resources | All actions | Full administrative access |
|
||||
| `user_admin` | `admin` | All actions | User management only |
|
||||
|
||||
|
||||
@@ -0,0 +1,522 @@
|
||||
# Appendix
|
||||
|
||||
## Table of Contents
|
||||
|
||||
1. [S3 Storage Guide](#s3-storage-guide)
|
||||
- [Overview](#overview)
|
||||
- [Bucket Configuration](#bucket-configuration)
|
||||
- [BucketKey Path Format](#bucketkey-path-format)
|
||||
- [Path Segment Reference](#path-segment-reference)
|
||||
- [Location Types](#location-types)
|
||||
- [Filename Segment Reference](#filename-segment-reference)
|
||||
- [Client ID Constraints](#client-id-constraints)
|
||||
- [Directory Partitioning](#directory-partitioning)
|
||||
- [S3 Object Metadata](#s3-object-metadata)
|
||||
- [Batch Upload Paths](#batch-upload-paths)
|
||||
- [Concrete Examples](#concrete-examples)
|
||||
- [Parsing a Key Back Into Its Components](#parsing-a-key-back-into-its-components)
|
||||
- [Guide: Watching an S3 Bucket for Client Document Imports](#guide-watching-an-s3-bucket-for-client-document-imports)
|
||||
|
||||
---
|
||||
|
||||
## S3 Storage Guide
|
||||
|
||||
### Overview
|
||||
|
||||
Every document uploaded through the platform is stored in AWS S3 using a structured key (path) that encodes enough metadata to identify and recover the document without a database lookup. The key contains the client ID, processing stage, date, partition number, timestamp, entity ID, and optionally a batch ID and file extension.
|
||||
|
||||
The implementation lives in:
|
||||
|
||||
| File | Purpose |
|
||||
|------|---------|
|
||||
| `internal/serviceconfig/objectstore/bucketkey.go` | `BucketKey` struct, `String()`, `ParseBucketKey()` |
|
||||
| `internal/serviceconfig/objectstore/bucketkey_test.go` | Unit tests with concrete path examples |
|
||||
| `internal/serviceconfig/objectstore/config.go` | Bucket config, partition logic (`GetDirectoryPart`) |
|
||||
| `internal/document/upload/get.go` | Upload flow that constructs keys |
|
||||
| `internal/document/batch/storage/service.go` | Batch (ZIP) upload storage |
|
||||
|
||||
### Bucket Configuration
|
||||
|
||||
The bucket name is configured via environment variable:
|
||||
|
||||
| Variable | Required | Description |
|
||||
|----------|----------|-------------|
|
||||
| `BUCKET` | Yes | S3 bucket name |
|
||||
| `AWS_ENDPOINT_URL_S3` | No | Override for local dev (default: `http://localhost:4566` via localstack) |
|
||||
| `AWS_S3_USE_PATH_STYLE` | No | Path-style addressing (default: `false`) |
|
||||
| `AWS_REGION` | No | AWS region (default: `us-east-1`) |
|
||||
|
||||
### BucketKey Path Format
|
||||
|
||||
A full S3 object key follows this structure:
|
||||
|
||||
```
|
||||
{ClientID}/{Location}/{Date}/{Part}/{Timestamp}~{ClientID}~{Location}~{EntityID}[~{BatchID}][.FileType]
|
||||
|_______________ prefix _________________| |______________________ filename __________________________|
|
||||
```
|
||||
|
||||
For the `export` location, the `Part` segment is omitted:
|
||||
|
||||
```
|
||||
{ClientID}/export/{Date}/{Timestamp}~{ClientID}~export~{EntityID}[~{BatchID}][.FileType]
|
||||
```
|
||||
|
||||
### Path Segment Reference
|
||||
|
||||
The **prefix** portion determines the directory structure in S3:
|
||||
|
||||
| Segment | Format | Example | Description |
|
||||
|---------|--------|---------|-------------|
|
||||
| ClientID | `[a-zA-Z0-9_#-]+` | `acme_corp` | Client identifier. Same value repeated in the filename. |
|
||||
| Location | `import`, `text`, or `export` | `import` | Processing stage (see Location Types below). |
|
||||
| Date | `YYYYMMDD` | `20250331` | Date the document was uploaded, in UTC. |
|
||||
| Part | Integer (0-based) | `0` | Partition shard within a date directory. Omitted for `export`. |
|
||||
|
||||
### Location Types
|
||||
|
||||
| Location | Meaning | Has Part? |
|
||||
|----------|---------|-----------|
|
||||
| `import` | Original uploaded document | Yes |
|
||||
| `text` | Text extraction output | Yes |
|
||||
| `export` | Exported results (CSV, etc.) | No |
|
||||
|
||||
### Filename Segment Reference
|
||||
|
||||
The **filename** portion is delimited by the tilde character (`~`). It comes in two variants depending on how the document was uploaded:
|
||||
|
||||
**4-segment filename (single upload via `POST /clients/{id}/documents`):**
|
||||
|
||||
```
|
||||
{Timestamp}~{ClientID}~{Location}~{EntityID}[.FileType]
|
||||
```
|
||||
|
||||
**5-segment filename (batch upload via `POST /clients/{id}/documents/batches`):**
|
||||
|
||||
```
|
||||
{Timestamp}~{ClientID}~{Location}~{EntityID}~{BatchID}[.FileType]
|
||||
```
|
||||
|
||||
The only difference is the 5th `~`-delimited segment containing the BatchID. Both formats may optionally end with a `.{FileType}` extension, though in practice the current upload code does not set FileType for `import` documents -- the file extension is typically absent.
|
||||
|
||||
| Segment | Position | Format | Example | Description |
|
||||
|---------|----------|--------|---------|-------------|
|
||||
| Timestamp | 1 (always) | `YYYY-MM-DDTHHMMSSZ` (ISO 8601 UTC) | `2025-03-31T040816Z` | Exact creation time. Uses Go format `2006-01-02T150405Z`. |
|
||||
| ClientID | 2 (always) | `[a-zA-Z0-9_#-]+` | `acme_corp` | Repeated from prefix for self-describing keys. |
|
||||
| Location | 3 (always) | `import`, `text`, or `export` | `import` | Repeated from prefix for self-describing keys. |
|
||||
| EntityID | 4 (always) | UUID | `b745910f-f529-43c5-87e1-25629d46ef40` | The upload ID (for `import`) or document/entity ID. |
|
||||
| BatchID | 5 (optional) | UUID | `12345678-1234-5678-9012-123456789012` | Present only for documents uploaded as part of a batch ZIP. |
|
||||
| FileType | suffix | Extension without dot | `pdf` | Appended after a `.` separator on the last segment. Often absent for imports. |
|
||||
|
||||
**Real-world examples:**
|
||||
|
||||
Single upload (4 segments, no extension):
|
||||
```
|
||||
2026-02-07T002850Z~demo-data-loader-1770424122~import~438b605f-1b80-4c9f-bfb9-3f6bfbf853d8
|
||||
```
|
||||
|
||||
Batch upload (5 segments, no extension):
|
||||
```
|
||||
2026-02-06T183937Z~int-test-1770403169~import~13234476-2881-4b22-924d-e3ebaf63e7c0~019c3440-a4c1-7da0-a3da-3c97f6287a24
|
||||
```
|
||||
|
||||
### Client ID Constraints
|
||||
|
||||
Client IDs must match the regex pattern:
|
||||
|
||||
```
|
||||
[a-zA-Z0-9_#-]+
|
||||
```
|
||||
|
||||
Valid characters: letters, numbers, underscore (`_`), hash (`#`), hyphen (`-`).
|
||||
|
||||
The tilde (`~`) character is explicitly banned from client IDs because it serves as the delimiter between segments in the filename portion of the key. See `internal/client/create.go`:
|
||||
|
||||
```
|
||||
// NOTE: Make sure the char ~ never makes way in as a valid client id character
|
||||
// Reference bucket key filename
|
||||
const CLIENT_ID_REGEX = `[a-zA-Z0-9\_#-]+`
|
||||
```
|
||||
|
||||
### Directory Partitioning
|
||||
|
||||
Within each `{ClientID}/{Location}/{Date}/` prefix, documents are distributed across numbered partitions to avoid placing too many objects in a single virtual directory.
|
||||
|
||||
- The default partition limit is **1000 objects** per partition.
|
||||
- Partition numbering starts at `0`.
|
||||
- When partition `N` reaches 1000 objects for a given date, new uploads go to partition `N+1`.
|
||||
- The current part and count are tracked in the database via `GetDocumentUploadCurrentPart`.
|
||||
|
||||
Logic from `ObjectStoreConfig.GetDirectoryPart()`:
|
||||
|
||||
```
|
||||
MaxDirSize = 1000 (default)
|
||||
|
||||
if count >= MaxDirSize:
|
||||
return part + 1
|
||||
else:
|
||||
return part
|
||||
```
|
||||
|
||||
Example progression for a busy client on a single day:
|
||||
|
||||
```
|
||||
acme_corp/import/20250331/0/... (objects 1-1000)
|
||||
acme_corp/import/20250331/1/... (objects 1001-2000)
|
||||
acme_corp/import/20250331/2/... (objects 2001-3000)
|
||||
```
|
||||
|
||||
The `export` location does not use partitions. Export keys skip the part segment entirely.
|
||||
|
||||
### S3 Object Metadata
|
||||
|
||||
In addition to the structured key, each uploaded object carries S3 metadata headers:
|
||||
|
||||
| Metadata Key | Value | Example |
|
||||
|-------------|-------|---------|
|
||||
| `original-path` | Full path from the upload request (includes folders) | `contracts/2025/Q1/agreement.pdf` |
|
||||
| `original-filename` | Just the filename | `agreement.pdf` |
|
||||
| `Content-Type` | MIME type detected from file extension | `application/pdf` |
|
||||
|
||||
These metadata values preserve the user's original file naming, which is not encoded in the S3 key itself.
|
||||
|
||||
### Batch Upload Paths
|
||||
|
||||
ZIP files uploaded via the batch endpoint (`POST /client/{id}/document/batch`) use a separate, simpler path scheme:
|
||||
|
||||
```
|
||||
batches/{ClientID}/{YYYY}/{MM}/{DD}/{HH}/{BatchID}-{OriginalFilename}
|
||||
```
|
||||
|
||||
Example:
|
||||
|
||||
```
|
||||
batches/acme_corp/2025/03/15/14/550e8400-e29b-41d4-a716-446655440000-uploads.zip
|
||||
```
|
||||
|
||||
| Segment | Description |
|
||||
|---------|-------------|
|
||||
| `batches/` | Fixed prefix for all batch uploads |
|
||||
| ClientID | Client identifier |
|
||||
| `YYYY/MM/DD/HH` | Timestamp directory (year/month/day/hour) |
|
||||
| BatchID | UUID assigned to the batch |
|
||||
| OriginalFilename | The user's original ZIP filename |
|
||||
|
||||
Once the batch ZIP is extracted, individual documents from the ZIP are stored using the standard `BucketKey` format with the `import` location and the `BatchID` field populated.
|
||||
|
||||
### Concrete Examples
|
||||
|
||||
**1. Single PDF upload for client `acme_corp`:**
|
||||
|
||||
```
|
||||
s3://my-bucket/acme_corp/import/20250331/0/2025-03-31T040816Z~acme_corp~import~b745910f-f529-43c5-87e1-25629d46ef40.pdf
|
||||
```
|
||||
|
||||
Decoded:
|
||||
- Client: `acme_corp`
|
||||
- Stage: `import` (original upload)
|
||||
- Date: March 31, 2025
|
||||
- Partition: `0`
|
||||
- Created at: 2025-03-31 04:08:16 UTC
|
||||
- Upload/Entity ID: `b745910f-f529-43c5-87e1-25629d46ef40`
|
||||
- File type: `pdf`
|
||||
|
||||
**2. Document from a batch upload:**
|
||||
|
||||
```
|
||||
s3://my-bucket/acme_corp/import/20250331/0/2025-03-31T040816Z~acme_corp~import~b745910f-f529-43c5-87e1-25629d46ef40~12345678-1234-5678-9012-123456789012.pdf
|
||||
```
|
||||
|
||||
Same as above, plus:
|
||||
- Batch ID: `12345678-1234-5678-9012-123456789012`
|
||||
|
||||
**3. Text extraction output:**
|
||||
|
||||
```
|
||||
s3://my-bucket/acme_corp/text/20250331/3/2025-03-31T040816Z~acme_corp~text~b745910f-f529-43c5-87e1-25629d46ef40
|
||||
```
|
||||
|
||||
- Stage: `text` (extraction result)
|
||||
- Partition: `3` (high volume, rolled past partitions 0-2)
|
||||
- No file extension
|
||||
|
||||
**4. Export result:**
|
||||
|
||||
```
|
||||
s3://my-bucket/acme_corp/export/20250331/2025-03-31T040816Z~acme_corp~export~b745910f-f529-43c5-87e1-25629d46ef40.csv
|
||||
```
|
||||
|
||||
- Stage: `export`
|
||||
- No partition number (exports never have one)
|
||||
- File type: `csv`
|
||||
|
||||
**5. Client IDs with special characters:**
|
||||
|
||||
```
|
||||
s3://my-bucket/A_A/import/20250331/0/2025-03-31T040816Z~A_A~import~b745910f-f529-43c5-87e1-25629d46ef40
|
||||
s3://my-bucket/A-A/import/20250331/0/2025-03-31T040816Z~A-A~import~b745910f-f529-43c5-87e1-25629d46ef40
|
||||
s3://my-bucket/A#A/import/20250331/0/2025-03-31T040816Z~A#A~import~b745910f-f529-43c5-87e1-25629d46ef40
|
||||
```
|
||||
|
||||
### Parsing a Key Back Into Its Components
|
||||
|
||||
The `ParseBucketKey()` function in `bucketkey.go` reverses the process. It uses regex to extract the prefix components and then parses the tilde-delimited filename.
|
||||
|
||||
**Step 1: Split the prefix from the filename.**
|
||||
|
||||
Parse regex for the prefix:
|
||||
|
||||
```
|
||||
^([a-zA-Z0-9_#-]+)/([a-z]+/?[a-z]+?)/([0-9]{8})/(.+)$
|
||||
|___ ClientID ___| |__ Location __| |_ Date _| |_ rest (part/filename or just filename) _|
|
||||
```
|
||||
|
||||
The "rest" portion is then checked for a leading integer partition number. If present, it splits into `{Part}/{Filename}`. If not, the entire rest is the filename.
|
||||
|
||||
**Step 2: Parse the filename.**
|
||||
|
||||
The filename is a `~`-delimited string with either 4 or 5 segments, plus an optional `.{extension}` suffix on the last segment. The parsing algorithm is:
|
||||
|
||||
1. Split on `.` first. If there are exactly 2 parts, the second part is the FileType. If there is 1 part, there is no FileType. More than 2 `.`-separated parts is invalid.
|
||||
2. Split the base (part before `.`) on `~`. This yields either 4 or 5 segments:
|
||||
- **4 segments:** `[Timestamp, ClientID, Location, EntityID]` -- single upload, no batch
|
||||
- **5 segments:** `[Timestamp, ClientID, Location, EntityID, BatchID]` -- batch upload
|
||||
3. The segment count is the only way to determine whether a BatchID is present. Count the `~` delimiters: 3 tildes = 4 segments (no batch), 4 tildes = 5 segments (has batch).
|
||||
|
||||
**Decision table for segment count:**
|
||||
|
||||
| Tilde count | Segment count | Has BatchID? | Upload source |
|
||||
|-------------|---------------|--------------|---------------|
|
||||
| 3 | 4 | No | Single document upload API |
|
||||
| 4 | 5 | Yes | Batch (ZIP) upload API |
|
||||
| Other | N/A | N/A | Invalid -- reject |
|
||||
|
||||
### Guide: Watching an S3 Bucket for Client Document Imports
|
||||
|
||||
This section shows how to monitor an S3 bucket for newly imported PDF documents belonging to a specific client.
|
||||
|
||||
#### Prerequisites
|
||||
|
||||
- AWS CLI configured with credentials that have `s3:ListBucket` and `s3:GetObject` permissions on the target bucket.
|
||||
- The bucket name (from the `BUCKET` environment variable).
|
||||
- The client ID you want to monitor.
|
||||
|
||||
#### Step 1: List All Imports for a Client
|
||||
|
||||
To see all imported documents for client `acme_corp`:
|
||||
|
||||
```bash
|
||||
aws s3 ls s3://my-bucket/acme_corp/import/ --recursive
|
||||
```
|
||||
|
||||
This lists every object under the client's `import` location across all dates and partitions.
|
||||
|
||||
#### Step 2: Filter by Date
|
||||
|
||||
To narrow down to a specific date (e.g., March 31, 2025):
|
||||
|
||||
```bash
|
||||
aws s3 ls s3://my-bucket/acme_corp/import/20250331/
|
||||
```
|
||||
|
||||
This shows the partition directories for that date:
|
||||
|
||||
```
|
||||
PRE 0/
|
||||
PRE 1/
|
||||
```
|
||||
|
||||
Then list a specific partition:
|
||||
|
||||
```bash
|
||||
aws s3 ls s3://my-bucket/acme_corp/import/20250331/0/
|
||||
```
|
||||
|
||||
#### Step 3: Filter for PDFs Only
|
||||
|
||||
```bash
|
||||
aws s3 ls s3://my-bucket/acme_corp/import/ --recursive | grep '\.pdf$'
|
||||
```
|
||||
|
||||
#### Step 4: Watch for New Imports in Real Time
|
||||
|
||||
Use a polling loop to detect new documents as they arrive:
|
||||
|
||||
```bash
|
||||
BUCKET="my-bucket"
|
||||
CLIENT="acme_corp"
|
||||
INTERVAL=30 # seconds between checks
|
||||
|
||||
LAST_CHECK=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
|
||||
|
||||
while true; do
|
||||
echo "Checking for new imports since $LAST_CHECK ..."
|
||||
|
||||
# List objects and filter by those newer than last check
|
||||
aws s3api list-objects-v2 \
|
||||
--bucket "$BUCKET" \
|
||||
--prefix "$CLIENT/import/" \
|
||||
--query "Contents[?LastModified>=\`$LAST_CHECK\`].[Key,Size,LastModified]" \
|
||||
--output table
|
||||
|
||||
LAST_CHECK=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
|
||||
sleep "$INTERVAL"
|
||||
done
|
||||
```
|
||||
|
||||
#### Step 5: Extract Document Identity from a Key
|
||||
|
||||
Given a key, you can extract the client ID, document/upload ID, batch ID, and timestamp without a database query. The filename portion has either 4 or 5 `~`-delimited segments (see [Filename Segment Reference](#filename-segment-reference)).
|
||||
|
||||
**Example 1 -- single upload (4 segments, no batch, no extension):**
|
||||
|
||||
```
|
||||
acme_corp/import/20250331/0/2025-03-31T040816Z~acme_corp~import~b745910f-f529-43c5-87e1-25629d46ef40
|
||||
```
|
||||
|
||||
**Example 2 -- batch upload (5 segments, has batch ID, no extension):**
|
||||
|
||||
```
|
||||
acme_corp/import/20250331/0/2025-03-31T040816Z~acme_corp~import~b745910f-f529-43c5-87e1-25629d46ef40~12345678-1234-5678-9012-123456789012
|
||||
```
|
||||
|
||||
Using bash to parse either format:
|
||||
|
||||
```bash
|
||||
KEY="$1" # pass the S3 key as argument
|
||||
|
||||
# Extract the filename (everything after the last /)
|
||||
FILENAME="${KEY##*/}"
|
||||
|
||||
# Strip file extension if present (e.g. ".pdf")
|
||||
# Note: import documents typically have NO extension
|
||||
BASE="${FILENAME%%.*}"
|
||||
if [[ "$FILENAME" == *"."* ]]; then
|
||||
FILE_EXT="${FILENAME##*.}"
|
||||
else
|
||||
FILE_EXT=""
|
||||
fi
|
||||
|
||||
# Split on ~ into an array
|
||||
IFS='~' read -ra SEGMENTS <<< "$BASE"
|
||||
SEGMENT_COUNT="${#SEGMENTS[@]}"
|
||||
|
||||
# First 4 segments are always present
|
||||
TIMESTAMP="${SEGMENTS[0]}"
|
||||
CLIENT_ID="${SEGMENTS[1]}"
|
||||
LOCATION="${SEGMENTS[2]}"
|
||||
ENTITY_ID="${SEGMENTS[3]}"
|
||||
|
||||
# 5th segment is the BatchID (only present for batch uploads)
|
||||
if [[ "$SEGMENT_COUNT" -eq 5 ]]; then
|
||||
BATCH_ID="${SEGMENTS[4]}"
|
||||
else
|
||||
BATCH_ID=""
|
||||
fi
|
||||
|
||||
echo "Segments: $SEGMENT_COUNT"
|
||||
echo "Client ID: $CLIENT_ID"
|
||||
echo "Location: $LOCATION"
|
||||
echo "Timestamp: $TIMESTAMP"
|
||||
echo "Entity ID: $ENTITY_ID"
|
||||
echo "Batch ID: ${BATCH_ID:-none}"
|
||||
echo "File Type: ${FILE_EXT:-none}"
|
||||
```
|
||||
|
||||
Output for example 1 (single upload):
|
||||
|
||||
```
|
||||
Segments: 4
|
||||
Client ID: acme_corp
|
||||
Location: import
|
||||
Timestamp: 2025-03-31T040816Z
|
||||
Entity ID: b745910f-f529-43c5-87e1-25629d46ef40
|
||||
Batch ID: none
|
||||
File Type: none
|
||||
```
|
||||
|
||||
Output for example 2 (batch upload):
|
||||
|
||||
```
|
||||
Segments: 5
|
||||
Client ID: acme_corp
|
||||
Location: import
|
||||
Timestamp: 2025-03-31T040816Z
|
||||
Entity ID: b745910f-f529-43c5-87e1-25629d46ef40
|
||||
Batch ID: 12345678-1234-5678-9012-123456789012
|
||||
File Type: none
|
||||
```
|
||||
|
||||
#### Step 6: Download a Document by Key
|
||||
|
||||
```bash
|
||||
aws s3 cp \
|
||||
"s3://my-bucket/acme_corp/import/20250331/0/2025-03-31T040816Z~acme_corp~import~b745910f-f529-43c5-87e1-25629d46ef40.pdf" \
|
||||
./recovered-document.pdf
|
||||
```
|
||||
|
||||
#### Step 7: Retrieve the Original Filename
|
||||
|
||||
The original filename from the user's upload is stored in S3 object metadata, not in the key:
|
||||
|
||||
```bash
|
||||
aws s3api head-object \
|
||||
--bucket my-bucket \
|
||||
--key "acme_corp/import/20250331/0/2025-03-31T040816Z~acme_corp~import~b745910f-f529-43c5-87e1-25629d46ef40.pdf" \
|
||||
--query "Metadata"
|
||||
```
|
||||
|
||||
Output:
|
||||
|
||||
```json
|
||||
{
|
||||
"original-path": "contracts/2025/Q1/agreement.pdf",
|
||||
"original-filename": "agreement.pdf"
|
||||
}
|
||||
```
|
||||
|
||||
#### Step 8: Bulk Recovery Script
|
||||
|
||||
To recover all PDFs for a client on a given date and rename them using their original filenames:
|
||||
|
||||
```bash
|
||||
BUCKET="my-bucket"
|
||||
CLIENT="acme_corp"
|
||||
DATE="20250331"
|
||||
OUTPUT_DIR="./recovered"
|
||||
|
||||
mkdir -p "$OUTPUT_DIR"
|
||||
|
||||
aws s3api list-objects-v2 \
|
||||
--bucket "$BUCKET" \
|
||||
--prefix "$CLIENT/import/$DATE/" \
|
||||
--query "Contents[].Key" \
|
||||
--output text | tr '\t' '\n' | grep '\.pdf$' | while read -r KEY; do
|
||||
|
||||
# Get original filename from metadata
|
||||
ORIGINAL=$(aws s3api head-object \
|
||||
--bucket "$BUCKET" \
|
||||
--key "$KEY" \
|
||||
--query "Metadata.\"original-filename\"" \
|
||||
--output text)
|
||||
|
||||
# Extract entity ID for uniqueness (4th segment, works for both 4 and 5 segment filenames)
|
||||
FILENAME="${KEY##*/}"
|
||||
BASE="${FILENAME%%.*}"
|
||||
IFS='~' read -ra SEGS <<< "$BASE"
|
||||
ENTITY_ID="${SEGS[3]}"
|
||||
|
||||
# Use original filename if available, otherwise use entity ID
|
||||
if [ "$ORIGINAL" != "None" ] && [ -n "$ORIGINAL" ]; then
|
||||
DEST="$OUTPUT_DIR/${ENTITY_ID}_${ORIGINAL}"
|
||||
else
|
||||
DEST="$OUTPUT_DIR/${ENTITY_ID}.pdf"
|
||||
fi
|
||||
|
||||
echo "Downloading: $KEY -> $DEST"
|
||||
aws s3 cp "s3://$BUCKET/$KEY" "$DEST"
|
||||
done
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
*Source code reference: `internal/serviceconfig/objectstore/bucketkey.go`*
|
||||
@@ -58,6 +58,16 @@ This documentation provides comprehensive coverage of the DoczyAI Document Proce
|
||||
- Log analysis and debugging
|
||||
- Backup, recovery, and capacity planning
|
||||
|
||||
9. **[Authentication Guide](09-authentication-guide.md)**
|
||||
- JWT authentication with AWS Cognito
|
||||
- Token types and refresh strategy
|
||||
- CORS configuration
|
||||
|
||||
10. **[Appendix](10-appendex.md)**
|
||||
- S3 Storage Guide: BucketKey path format, segment reference, partitioning
|
||||
- Watching S3 for client document imports
|
||||
- Document recovery from S3 buckets
|
||||
|
||||
## Quick Start
|
||||
|
||||
### For New Developers
|
||||
|
||||
@@ -30,9 +30,6 @@ This document provides a comprehensive summary of all REST API endpoints availab
|
||||
| /field-extractions | GET, POST | FieldExtractionService |
|
||||
| /field-extractions/version | GET | FieldExtractionService |
|
||||
| /field-extractions/history | GET | FieldExtractionService |
|
||||
| /query | GET, POST | QueryService |
|
||||
| /query/{id} | GET, PATCH | QueryService |
|
||||
| /query/{id}/test | POST | QueryService |
|
||||
| /client/{id}/export | POST | ExportService |
|
||||
| /export/{id} | GET | ExportService |
|
||||
| /admin/users | GET, POST | AdminService |
|
||||
@@ -210,28 +207,6 @@ This document provides a comprehensive summary of all REST API endpoints availab
|
||||
- Returns: `{versions[]}` ordered by most recent first
|
||||
- Each version contains: `{version, created_at}`
|
||||
|
||||
## Query Service
|
||||
|
||||
### Query Definition and Execution
|
||||
- **GET /query** - List queries based on filter criteria
|
||||
- Returns: `{queries[]}` where each query contains `{id, type, active_version, latest_version, config?, required_queries[]?}`
|
||||
|
||||
- **POST /query** - Create a new query with provided details
|
||||
- Request Body: `{type, config?, required_queries[]?}`
|
||||
- Supported Types: `JSON_EXTRACTOR`, `CONTEXT_FULL`
|
||||
- Returns: `{id}` (201 Created)
|
||||
|
||||
- **GET /query/{id}** - Get a specific query by its ID
|
||||
- Returns: `{id, type, active_version, latest_version, config?, required_queries[]?}`
|
||||
|
||||
- **PATCH /query/{id}** - Update an existing query with new details
|
||||
- Request Body: `{config?, active_version?, required_queries[]?}`
|
||||
- Returns: 200 OK on success
|
||||
|
||||
- **POST /query/{id}/test** - Execute a test run of a query with parameters
|
||||
- Request Body: `{document_id, query_version}`
|
||||
- Returns: `{value}` - Result of the query test
|
||||
|
||||
## Export Service
|
||||
|
||||
### Export Management
|
||||
|
||||
Executable
+279
@@ -0,0 +1,279 @@
|
||||
#!/bin/bash
|
||||
# =============================================================================
|
||||
# create.client.sh
|
||||
# =============================================================================
|
||||
#
|
||||
# DESCRIPTION:
|
||||
# Creates a new client in the Query Orchestration API using a JWT token
|
||||
# from a file for authentication. After creation, verifies the client exists.
|
||||
#
|
||||
# USAGE:
|
||||
# ./create.client.sh <base_url> <token_file> <client_name>
|
||||
#
|
||||
# ARGUMENTS:
|
||||
# base_url - The URL of the Query Orchestration API (e.g., http://localhost:8080)
|
||||
# token_file - Path to a file containing the JWT auth token
|
||||
# client_name - The name to assign to the new client
|
||||
#
|
||||
# EXAMPLES:
|
||||
# ./create.client.sh http://localhost:8080 ./token.txt "My Client"
|
||||
# ./create.client.sh http://localhost:8080 test/token.txt "Demo Client"
|
||||
#
|
||||
# EXIT CODES:
|
||||
# 0 - Client created successfully
|
||||
# 1 - Missing dependencies or invalid arguments
|
||||
# 2 - API call failed
|
||||
# 3 - Authentication failed
|
||||
#
|
||||
# =============================================================================
|
||||
|
||||
set -e
|
||||
|
||||
# =============================================================================
|
||||
# CONFIGURATION
|
||||
# =============================================================================
|
||||
|
||||
# Colors for output (disable if not a terminal)
|
||||
if [[ -t 1 ]]; then
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
BLUE='\033[0;34m'
|
||||
NC='\033[0m'
|
||||
else
|
||||
RED=''
|
||||
GREEN=''
|
||||
YELLOW=''
|
||||
BLUE=''
|
||||
NC=''
|
||||
fi
|
||||
|
||||
# Generate unique external ID for this client
|
||||
TIMESTAMP=$(date +%s)
|
||||
CLIENT_EXTERNAL_ID="client-${TIMESTAMP}"
|
||||
|
||||
# Variables set by api_call
|
||||
HTTP_CODE=""
|
||||
RESPONSE_BODY=""
|
||||
|
||||
# =============================================================================
|
||||
# HELPER FUNCTIONS
|
||||
# =============================================================================
|
||||
|
||||
# print_success - Prints a success message
|
||||
# Parameters:
|
||||
# $1 - Message
|
||||
print_success() {
|
||||
echo -e "${GREEN} OK: $1${NC}"
|
||||
}
|
||||
|
||||
# print_error - Prints an error message and exits
|
||||
# Parameters:
|
||||
# $1 - Message
|
||||
print_error() {
|
||||
echo -e "${RED} ERROR: $1${NC}"
|
||||
exit 2
|
||||
}
|
||||
|
||||
# print_info - Prints an informational message
|
||||
# Parameters:
|
||||
# $1 - Message
|
||||
print_info() {
|
||||
echo -e " $1"
|
||||
}
|
||||
|
||||
# check_response - Checks if the HTTP response code indicates success
|
||||
# Parameters:
|
||||
# $1 - Expected HTTP status code(s) (comma-separated, e.g., "200,201")
|
||||
# $2 - Actual HTTP status code
|
||||
# $3 - Response body (for error messages)
|
||||
# $4 - Endpoint description
|
||||
check_response() {
|
||||
local expected="$1"
|
||||
local actual="$2"
|
||||
local body="$3"
|
||||
local desc="$4"
|
||||
|
||||
IFS=',' read -ra CODES <<< "$expected"
|
||||
for code in "${CODES[@]}"; do
|
||||
if [[ "$actual" == "$code" ]]; then
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ "$actual" == "401" ]]; then
|
||||
echo -e "${RED} AUTHENTICATION FAILED: $desc${NC}"
|
||||
echo " Token may be expired or invalid"
|
||||
echo " Response: $body"
|
||||
exit 3
|
||||
fi
|
||||
|
||||
if [[ "$actual" == "403" ]]; then
|
||||
echo -e "${RED} AUTHORIZATION FAILED: $desc${NC}"
|
||||
echo " User does not have permission for this operation"
|
||||
echo " Response: $body"
|
||||
exit 3
|
||||
fi
|
||||
|
||||
echo -e "${RED} FAILED: $desc${NC}"
|
||||
echo " Expected: $expected, Got: $actual"
|
||||
echo " Response: $body"
|
||||
exit 2
|
||||
}
|
||||
|
||||
# api_call - Makes an authenticated API call and captures response
|
||||
# Parameters:
|
||||
# $1 - HTTP method (GET, POST, PATCH, DELETE)
|
||||
# $2 - Endpoint path (e.g., "/client")
|
||||
# $3 - Request body (optional, empty string for none)
|
||||
# $4 - Content-Type header (optional, defaults to application/json)
|
||||
# Sets globals: HTTP_CODE, RESPONSE_BODY
|
||||
api_call() {
|
||||
local method="$1"
|
||||
local endpoint="$2"
|
||||
local body="$3"
|
||||
local content_type="${4:-application/json}"
|
||||
|
||||
local url="${BASE_URL}${endpoint}"
|
||||
local curl_opts=(-s -k -w "\n%{http_code}")
|
||||
|
||||
curl_opts+=(-X "$method")
|
||||
|
||||
if [[ -n "$AUTH_TOKEN" ]]; then
|
||||
curl_opts+=(-H "Authorization: Bearer ${AUTH_TOKEN}")
|
||||
fi
|
||||
|
||||
if [[ -n "$body" ]]; then
|
||||
curl_opts+=(-H "Content-Type: $content_type")
|
||||
curl_opts+=(-d "$body")
|
||||
fi
|
||||
|
||||
local response
|
||||
local retries=3
|
||||
local delay=1
|
||||
|
||||
for ((i=1; i<=retries; i++)); do
|
||||
if ! response=$(curl --connect-timeout 5 --max-time 30 "${curl_opts[@]}" "$url" 2>&1); then
|
||||
echo -e "${RED} CONNECTION FAILED: Could not reach ${url}${NC}"
|
||||
echo " curl error: $response"
|
||||
exit 2
|
||||
fi
|
||||
|
||||
HTTP_CODE=$(echo "$response" | tail -n1)
|
||||
RESPONSE_BODY=$(echo "$response" | sed '$d')
|
||||
|
||||
if [[ "$HTTP_CODE" != "429" ]]; then
|
||||
break
|
||||
fi
|
||||
|
||||
if [[ $i -lt $retries ]]; then
|
||||
print_info "Rate limited (429), waiting ${delay}s before retry $((i+1))/$retries..."
|
||||
sleep "$delay"
|
||||
delay=$((delay * 2))
|
||||
fi
|
||||
done
|
||||
|
||||
sleep 0.1
|
||||
}
|
||||
|
||||
# =============================================================================
|
||||
# ARGUMENT VALIDATION
|
||||
# =============================================================================
|
||||
|
||||
if [[ $# -lt 3 ]]; then
|
||||
echo "Usage: $0 <base_url> <token_file> <client_name>"
|
||||
echo ""
|
||||
echo "Arguments:"
|
||||
echo " base_url - The URL of the Query Orchestration API"
|
||||
echo " token_file - Path to a file containing the JWT auth token"
|
||||
echo " client_name - The name to assign to the new client"
|
||||
echo ""
|
||||
echo "Examples:"
|
||||
echo " $0 http://localhost:8080 ./token.txt \"My Client\""
|
||||
echo " $0 http://localhost:8080 test/token.txt \"Demo Client\""
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check dependencies
|
||||
for cmd in curl jq; do
|
||||
if ! command -v "$cmd" &> /dev/null; then
|
||||
echo -e "${RED}ERROR: Required command '$cmd' not found${NC}"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
BASE_URL="${1%/}"
|
||||
TOKEN_FILE="$2"
|
||||
CLIENT_NAME="$3"
|
||||
|
||||
if [[ ! -f "$TOKEN_FILE" ]]; then
|
||||
echo -e "${RED}ERROR: Token file not found: $TOKEN_FILE${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
AUTH_TOKEN=$(cat "$TOKEN_FILE" | tr -d '[:space:]')
|
||||
|
||||
if [[ -z "$AUTH_TOKEN" ]]; then
|
||||
echo -e "${RED}ERROR: Token file is empty: $TOKEN_FILE${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# =============================================================================
|
||||
# MAIN
|
||||
# =============================================================================
|
||||
|
||||
echo -e "${BLUE}========================================${NC}"
|
||||
echo -e "${BLUE}Create Client${NC}"
|
||||
echo -e "${BLUE}========================================${NC}"
|
||||
echo ""
|
||||
echo "Base URL: ${BASE_URL}"
|
||||
echo "Token File: ${TOKEN_FILE}"
|
||||
echo "Client Name: ${CLIENT_NAME}"
|
||||
echo "External ID: ${CLIENT_EXTERNAL_ID}"
|
||||
echo ""
|
||||
|
||||
# Step 1: Create client via POST /client
|
||||
echo -e "${BLUE}--- Creating client ---${NC}"
|
||||
|
||||
body=$(cat << EOF
|
||||
{
|
||||
"id": "${CLIENT_EXTERNAL_ID}",
|
||||
"name": "${CLIENT_NAME}"
|
||||
}
|
||||
EOF
|
||||
)
|
||||
|
||||
print_info "Request: POST /client"
|
||||
print_info "Body: $body"
|
||||
|
||||
api_call "POST" "/client" "$body"
|
||||
check_response "201" "$HTTP_CODE" "$RESPONSE_BODY" "Create client"
|
||||
|
||||
CLIENT_ID=$(echo "$RESPONSE_BODY" | jq -r '.id')
|
||||
print_success "Client created successfully"
|
||||
print_info "Client ID: $CLIENT_ID"
|
||||
|
||||
# Step 2: Verify client via GET /client/{id}
|
||||
echo ""
|
||||
echo -e "${BLUE}--- Verifying client ---${NC}"
|
||||
|
||||
print_info "Request: GET /client/${CLIENT_ID}"
|
||||
|
||||
api_call "GET" "/client/${CLIENT_ID}" ""
|
||||
check_response "200" "$HTTP_CODE" "$RESPONSE_BODY" "Get client"
|
||||
|
||||
name=$(echo "$RESPONSE_BODY" | jq -r '.name')
|
||||
can_sync=$(echo "$RESPONSE_BODY" | jq -r '.can_sync')
|
||||
|
||||
print_success "Client verified"
|
||||
print_info "Name: $name"
|
||||
print_info "Can Sync: $can_sync"
|
||||
|
||||
# Summary
|
||||
echo ""
|
||||
echo -e "${GREEN}========================================${NC}"
|
||||
echo -e "${GREEN}Client created and verified${NC}"
|
||||
echo -e "${GREEN}========================================${NC}"
|
||||
echo " ID: $CLIENT_ID"
|
||||
echo " External ID: $CLIENT_EXTERNAL_ID"
|
||||
echo " Name: $CLIENT_NAME"
|
||||
@@ -54,11 +54,16 @@
|
||||
# │ Step 6: POST /client/{id}/document/batch Upload ZIP batch │
|
||||
# │ Step 7: GET /client/{id}/document/batch/{batchId} │
|
||||
# │ Poll batch status │
|
||||
# │ GET /client/{id}/document/batch List all batches │
|
||||
# │ Step 8: GET /client/{id}/document List documents (expect 3)│
|
||||
# │ Step 9-10: (skipped) Folder-based document retrieval │
|
||||
# │ Step 11: POST /client/{id}/document Upload single PDF │
|
||||
# │ Step 12: GET /client/{id}/document Verify total docs (4) │
|
||||
# │ Step 12.5:GET /document/{id} Verify fileSizeBytes │
|
||||
# │ Step 12.8:BUG TEST - Verify folder metrics BEFORE labeling │
|
||||
# │ GET /client/{id}/folders?metrics=true │
|
||||
# │ GET /folders/{folderId}/metrics │
|
||||
# │ Assert known doc counts per folder │
|
||||
# ├─────────────────────────────────────────────────────────────────────────┤
|
||||
# │ PHASE 3: LABEL OPERATIONS │
|
||||
# ├─────────────────────────────────────────────────────────────────────────┤
|
||||
@@ -75,6 +80,10 @@
|
||||
# │ Verify fileSizeBytes │
|
||||
# │ Step 17: GET /labels/{label}/documents?clientId={id} │
|
||||
# │ Get docs by label │
|
||||
# │ Step 17.5:BUG TEST - Verify folder metrics AFTER labeling │
|
||||
# │ GET /client/{id}/folders?metrics=true │
|
||||
# │ GET /folders/{folderId}/metrics │
|
||||
# │ Assert same known counts still hold │
|
||||
# ├─────────────────────────────────────────────────────────────────────────┤
|
||||
# │ PHASE 4: FIELD EXTRACTIONS │
|
||||
# ├─────────────────────────────────────────────────────────────────────────┤
|
||||
@@ -130,6 +139,12 @@ DOCUMENT_IDS=()
|
||||
TARGET_DOC_ID=""
|
||||
FOLDER_IDS=()
|
||||
|
||||
# BUG TEST: Expected folder document counts (known from the test's upload structure).
|
||||
# The batch upload creates: /folder1/subfolder2/file1.pdf, /folder1/subfolder2/file2.pdf,
|
||||
# /folder2/subfolder1/file3.pdf. The single upload goes to root "/".
|
||||
# These are checked before labeling (step 12.8) and again after (step 17.5).
|
||||
EXPECTED_FOLDER_COUNTS_BY_PATH='{ "/": 1, "/folder1": 0, "/folder1/subfolder2": 2, "/folder2": 0, "/folder2/subfolder1": 1 }'
|
||||
|
||||
# =============================================================================
|
||||
# HELPER FUNCTIONS
|
||||
# =============================================================================
|
||||
@@ -832,6 +847,82 @@ step_12_5_verify_file_sizes() {
|
||||
fi
|
||||
}
|
||||
|
||||
# Step 12.8: BUG TEST - Verify folder metrics BEFORE labeling
|
||||
# Confirms the known document counts per folder are correct before any labels
|
||||
# are applied. Step 17.5 re-checks these same counts AFTER labeling to detect
|
||||
# the reported bug where labeling a document causes it to disappear from
|
||||
# folder metrics (totalDocuments count decreases).
|
||||
# Expected counts come from the fixed test upload structure:
|
||||
# / = 1 (single upload), /folder1 = 0, /folder1/subfolder2 = 2, /folder2 = 0, /folder2/subfolder1 = 1
|
||||
step_12_8_verify_folder_metrics_before_labels() {
|
||||
print_header "12.8" "BUG TEST: Verify folder metrics BEFORE labeling"
|
||||
|
||||
if [[ ${#DOCUMENT_IDS[@]} -lt 4 ]]; then
|
||||
print_warning "Expected 4 documents but have ${#DOCUMENT_IDS[@]} - skipping pre-label metrics check"
|
||||
return
|
||||
fi
|
||||
|
||||
print_info "Verifying expected document counts per folder before any labels are applied."
|
||||
print_info "Expected: ${EXPECTED_FOLDER_COUNTS_BY_PATH}"
|
||||
|
||||
local bugs_found=0
|
||||
|
||||
# Test via /client/{id}/folders?metrics=true
|
||||
print_info ""
|
||||
print_info "Request: GET /client/${CLIENT_ID}/folders?metrics=true"
|
||||
|
||||
api_call "GET" "/client/${CLIENT_ID}/folders?metrics=true" ""
|
||||
check_response "200" "$HTTP_CODE" "$RESPONSE_BODY" "Get client folders with metrics"
|
||||
|
||||
# Populate FOLDER_IDS for use by later steps
|
||||
FOLDER_IDS=($(echo "$RESPONSE_BODY" | jq -r '.folders[].id'))
|
||||
|
||||
local folder_count
|
||||
folder_count=$(echo "$RESPONSE_BODY" | jq '.folders | length')
|
||||
print_info "Found $folder_count folders"
|
||||
|
||||
# Check each folder's totalDocuments against expected values
|
||||
for row in $(echo "$RESPONSE_BODY" | jq -c '.folders[]'); do
|
||||
local path
|
||||
path=$(echo "$row" | jq -r '.path')
|
||||
local actual
|
||||
actual=$(echo "$row" | jq -r '.metrics.totalDocuments // 0')
|
||||
local expected
|
||||
expected=$(echo "$EXPECTED_FOLDER_COUNTS_BY_PATH" | jq -r --arg p "$path" '.[$p] // "UNKNOWN"')
|
||||
|
||||
if [[ "$expected" == "UNKNOWN" ]]; then
|
||||
print_warning "Unexpected folder '$path' (not in expected map) - totalDocuments=$actual"
|
||||
elif [[ "$actual" != "$expected" ]]; then
|
||||
echo -e "${RED} FAIL: Folder '$path' totalDocuments=$actual (expected $expected)${NC}"
|
||||
bugs_found=$((bugs_found + 1))
|
||||
else
|
||||
print_success "Folder '$path': totalDocuments=$actual (correct)"
|
||||
fi
|
||||
done
|
||||
|
||||
# Also verify via /folders/{folderId}/metrics for each folder
|
||||
print_info ""
|
||||
print_info "Cross-checking via GET /folders/{folderId}/metrics"
|
||||
|
||||
for folder_id in "${FOLDER_IDS[@]}"; do
|
||||
api_call "GET" "/folders/${folder_id}/metrics" ""
|
||||
|
||||
if [[ "$HTTP_CODE" == "200" ]]; then
|
||||
local actual
|
||||
actual=$(echo "$RESPONSE_BODY" | jq -r '.totalDocuments')
|
||||
print_info " Folder $folder_id: totalDocuments=$actual"
|
||||
else
|
||||
print_warning " Failed to get metrics for folder $folder_id: $HTTP_CODE"
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ $bugs_found -gt 0 ]]; then
|
||||
print_error "Pre-label folder metrics do not match expected counts ($bugs_found mismatches)"
|
||||
else
|
||||
print_success "All folder document counts match expected values before labeling"
|
||||
fi
|
||||
}
|
||||
|
||||
# =============================================================================
|
||||
# PHASE 3: LABEL OPERATIONS (Steps 13-17)
|
||||
# =============================================================================
|
||||
@@ -1037,6 +1128,73 @@ step_17_documents_by_label() {
|
||||
fi
|
||||
}
|
||||
|
||||
# Step 17.5: BUG TEST - Verify folder metrics AFTER labeling
|
||||
# Re-checks folder document counts using the same expected values from step 12.8.
|
||||
# If labeling a document caused its folder's totalDocuments to change, this FAILS.
|
||||
# Tests both endpoints: /client/{id}/folders?metrics=true AND /folders/{id}/metrics
|
||||
step_17_5_verify_metrics_after_labels() {
|
||||
print_header "17.5" "BUG TEST: Verify folder metrics AFTER labeling (should be unchanged)"
|
||||
|
||||
if [[ ${#FOLDER_IDS[@]} -eq 0 ]]; then
|
||||
print_warning "No folder IDs available - skipping post-label metrics check"
|
||||
return
|
||||
fi
|
||||
|
||||
print_info "Labels were applied in steps 13-14. Verifying that folder document counts"
|
||||
print_info "are still the same as the expected values: ${EXPECTED_FOLDER_COUNTS_BY_PATH}"
|
||||
|
||||
local bugs_found=0
|
||||
|
||||
# Test 1: /client/{id}/folders?metrics=true
|
||||
print_info ""
|
||||
print_info "Test 1: GET /client/${CLIENT_ID}/folders?metrics=true"
|
||||
|
||||
api_call "GET" "/client/${CLIENT_ID}/folders?metrics=true" ""
|
||||
check_response "200" "$HTTP_CODE" "$RESPONSE_BODY" "Get client folders with metrics (post-label)"
|
||||
|
||||
for row in $(echo "$RESPONSE_BODY" | jq -c '.folders[]'); do
|
||||
local path
|
||||
path=$(echo "$row" | jq -r '.path')
|
||||
local actual
|
||||
actual=$(echo "$row" | jq -r '.metrics.totalDocuments // 0')
|
||||
local expected
|
||||
expected=$(echo "$EXPECTED_FOLDER_COUNTS_BY_PATH" | jq -r --arg p "$path" '.[$p] // "UNKNOWN"')
|
||||
|
||||
if [[ "$expected" == "UNKNOWN" ]]; then
|
||||
print_warning "Unexpected folder '$path' - totalDocuments=$actual"
|
||||
elif [[ "$actual" != "$expected" ]]; then
|
||||
echo -e "${RED} BUG: Folder '$path' totalDocuments=$actual (expected $expected) - labeling changed the count!${NC}"
|
||||
bugs_found=$((bugs_found + 1))
|
||||
else
|
||||
print_success "Folder '$path': totalDocuments=$actual (correct after labeling)"
|
||||
fi
|
||||
done
|
||||
|
||||
# Test 2: /folders/{folderId}/metrics for each folder
|
||||
print_info ""
|
||||
print_info "Test 2: GET /folders/{folderId}/metrics for each folder"
|
||||
|
||||
for folder_id in "${FOLDER_IDS[@]}"; do
|
||||
api_call "GET" "/folders/${folder_id}/metrics" ""
|
||||
|
||||
if [[ "$HTTP_CODE" == "200" ]]; then
|
||||
local actual
|
||||
actual=$(echo "$RESPONSE_BODY" | jq -r '.totalDocuments')
|
||||
print_success "Folder $folder_id: totalDocuments=$actual"
|
||||
else
|
||||
print_warning "Failed to get metrics for folder $folder_id: $HTTP_CODE"
|
||||
fi
|
||||
done
|
||||
|
||||
# Final verdict
|
||||
print_info ""
|
||||
if [[ $bugs_found -gt 0 ]]; then
|
||||
print_error "BUG TEST FAILED: $bugs_found folder metric(s) changed after labeling a document"
|
||||
else
|
||||
print_success "BUG TEST PASSED: All folder document counts correct after labeling"
|
||||
fi
|
||||
}
|
||||
|
||||
# =============================================================================
|
||||
# PHASE 4: FIELD EXTRACTIONS (Steps 18-23)
|
||||
# =============================================================================
|
||||
@@ -1930,6 +2088,7 @@ main() {
|
||||
step_11_upload_single
|
||||
step_12_verify_total
|
||||
step_12_5_verify_file_sizes
|
||||
step_12_8_verify_folder_metrics_before_labels
|
||||
|
||||
# Phase 3: Label Operations
|
||||
echo ""
|
||||
@@ -1939,6 +2098,7 @@ main() {
|
||||
step_16_folder_metrics
|
||||
step_16_5_folder_document_sizes
|
||||
step_17_documents_by_label
|
||||
step_17_5_verify_metrics_after_labels
|
||||
|
||||
# Phase 4: Field Extractions
|
||||
echo ""
|
||||
|
||||
Reference in New Issue
Block a user