Merged in feature/mutable-metadata1 (pull request #221)

M1, M2 and M3 complete

* M1, M2 and M3 complete

* review changes

* docs

* docs

cmd/auth_related/permit.setup/list_roles.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+# List all roles in the production environment
+
+export PERMIT_API_KEY="permit_key_UiT5HGXS2kwsWJbZOnzEI86Y2ut4tGqHQcTJoMP6XaqlKmlLFWMVLPwlx0faz35ABigea2htQ6RpGjRDLkLIkT"
+
+PROJECT_ID="default"
+ENV_ID="production"
+
+echo "Fetching roles from Production environment..."
+curl -s -X GET "https://api.permit.io/v2/schema/${PROJECT_ID}/${ENV_ID}/roles" \
+  -H "Authorization: Bearer ${PERMIT_API_KEY}" \
+  -H "Content-Type: application/json" | jq '.'

cmd/auth_related/permit.setup/permit_policies.yaml

@@ -68,6 +68,25 @@ resources:
     actions:
       - get
 
+  # v4 (mutable metadata) — super-admin resource covers every /super-admin/*
+  # path. No "put" action: the v4 "create a new schema version" route uses
+  # POST /super-admin/custom-schemas/{schemaId}/versions, not PUT.
+  - name: super-admin
+    description: Super-admin-only surface (schema CRUD, schema assignment, reset-metadata, bulk folder assignment)
+    actions:
+      - get
+      - post
+      - patch
+      - delete
+
+  # v4 (mutable metadata) — custom-metadata resource covers the user-facing
+  # /custom-metadata/* routes that read and write validated custom metadata.
+  - name: custom-metadata
+    description: User-facing custom metadata CRUD on documents (read + write)
+    actions:
+      - get
+      - post
+
 # Roles to be created in Permit.io
 roles:
   - name: super_admin
@@ -109,6 +128,18 @@ roles:
         - delete
       labels:
         - get
+      # v4 (mutable metadata) — full super-admin surface.
+      super-admin:
+        - get
+        - post
+        - patch
+        - delete
+      # v4 (mutable metadata) — super_admin is also allowed to read and
+      # write custom metadata; the role is a strict superset of user_admin
+      # and client_user on this resource.
+      custom-metadata:
+        - get
+        - post
 
   - name: user_admin
     description: User management - primarily for creating new client users
@@ -128,6 +159,14 @@ roles:
       folders:
         - get
         - post
+      # v4 (mutable metadata) — user_admin may read and write custom
+      # metadata on documents it manages, but deliberately does NOT get any
+      # super-admin action. Schema CRUD, assignment, reset, and bulk folder
+      # assignment stay super_admin-only. Verify no `super-admin:` entry
+      # appears in this block.
+      custom-metadata:
+        - get
+        - post
 
   - name: auditor
     description: Full read-only access across the entire system for auditing and compliance
@@ -151,6 +190,13 @@ roles:
         - get
       labels:
         - get
+      # v4 (mutable metadata) — auditor gets read on both resources so
+      # compliance reviewers can inspect schema CRUD history and read the
+      # custom metadata on any document. No write actions.
+      super-admin:
+        - get
+      custom-metadata:
+        - get
 
   - name: client_user
     description: Client-specific access for external users (requires tenant filtering)
@@ -178,6 +224,13 @@ roles:
         - patch
       labels:
         - get
+      # v4 (mutable metadata) — client_user may read and write custom
+      # metadata on its own documents (tenant isolation still a TODO at
+      # the application layer; see notes at the bottom of this file).
+      # No super-admin access.
+      custom-metadata:
+        - get
+        - post
     notes: |
       This role requires tenant-level filtering by client_id.
       Currently, client_user will have access to all clients (no tenant isolation).
@@ -296,6 +349,44 @@ policy_mappings:
       methods:
         GET: folders:get
 
+  # v4 (mutable metadata) — every /super-admin/* path maps to the
+  # super-admin resource. Documentation only; the dynamic first-segment
+  # resource extractor derives the same mapping at request time.
+  super_admin_endpoints:
+    - path: /super-admin/custom-schemas
+      methods:
+        GET: super-admin:get
+        POST: super-admin:post
+    - path: /super-admin/custom-schemas/{schemaId}
+      methods:
+        GET: super-admin:get
+        DELETE: super-admin:delete
+    - path: /super-admin/custom-schemas/{schemaId}/versions
+      methods:
+        POST: super-admin:post
+    - path: /super-admin/documents/{id}/schema
+      methods:
+        PATCH: super-admin:patch
+    - path: /super-admin/documents/{id}/reset-metadata
+      methods:
+        POST: super-admin:post
+    - path: /super-admin/folders/{folderId}/assign-schema
+      methods:
+        POST: super-admin:post
+
+  # v4 (mutable metadata) — user-facing custom metadata CRUD.
+  custom_metadata_endpoints:
+    - path: /custom-metadata
+      methods:
+        GET: custom-metadata:get
+        POST: custom-metadata:post
+    - path: /custom-metadata/version
+      methods:
+        GET: custom-metadata:get
+    - path: /custom-metadata/history
+      methods:
+        GET: custom-metadata:get
+
   eula_endpoints:
     - path: /eula
       public: true

cmd/auth_related/permit.setup/run.tool.dev.sh

@@ -0,0 +1,18 @@
+# dev environment key
+export PERMIT_API_KEY="permit_key_nwcB5ebXeeZyyt684otN2h70P7EVYWvFjp88QLObZK0uBFLArgPTidmkMxXGGoMh0aSbLTZ8lp9Qg72bYFde1B"
+
+# production key
+# export PERMIT_API_KEY="permit_key_UiT5HGXS2kwsWJbZOnzEI86Y2ut4tGqHQcTJoMP6XaqlKmlLFWMVLPwlx0faz35ABigea2htQ6RpGjRDLkLIkT"
+
+
+# this tool can be run locally since it does not need aws enviroment 
+
+# permit uat key
+#export PERMIT_API_KEY="permit_key_VjXBwoTFVhWWNecOt49BcAjyWWBaM8vzO9IvqNfJnqrLotz6BiSfr3hHV9v5zWqSQjYerqrpMlhz0WKYaLWMOg"
+go run setup_permit.go -list
+# for dev setup (this works)
+go run setup_permit.go -config permit_policies.yaml -project default -env 0551401b0e9e4075b14957506f3d6752
+
+# for production (this does not work)
+#go run setup_permit.go -config permit_policies.yaml -project default -env production
+

cmd/auth_related/permit.setup/run.tool.prod.sh

@@ -0,0 +1,15 @@
+# dev environment key
+# export PERMIT_API_KEY="permit_key_nwcB5ebXeeZyyt684otN2h70P7EVYWvFjp88QLObZK0uBFLArgPTidmkMxXGGoMh0aSbLTZ8lp9Qg72bYFde1B"
+# production key
+# export PERMIT_API_KEY="permit_key_UiT5HGXS2kwsWJbZOnzEI86Y2ut4tGqHQcTJoMP6XaqlKmlLFWMVLPwlx0faz35ABigea2htQ6RpGjRDLkLIkT"
+# permit uat key
+
+# this tool can be run locally since it does not need aws enviroment 
+export PERMIT_API_KEY="permit_key_UiT5HGXS2kwsWJbZOnzEI86Y2ut4tGqHQcTJoMP6XaqlKmlLFWMVLPwlx0faz35ABigea2htQ6RpGjRDLkLIkT"
+go run setup_permit.go -list
+# production 
+go run setup_permit.go -config permit_policies.yaml -project default -env ba1a6110b24f44c1b00d8d7a00b8c7b7
+
+# for production (this does not work)
+#go run setup_permit.go -config permit_policies.yaml -project default -env production
+

cmd/auth_related/permit.setup/run.tool.uat.sh

@@ -0,0 +1,17 @@
+# dev environment key
+# export PERMIT_API_KEY="permit_key_nwcB5ebXeeZyyt684otN2h70P7EVYWvFjp88QLObZK0uBFLArgPTidmkMxXGGoMh0aSbLTZ8lp9Qg72bYFde1B"
+# production key
+# export PERMIT_API_KEY="permit_key_UiT5HGXS2kwsWJbZOnzEI86Y2ut4tGqHQcTJoMP6XaqlKmlLFWMVLPwlx0faz35ABigea2htQ6RpGjRDLkLIkT"
+
+
+# this tool can be run locally since it does not need aws enviroment 
+
+# permit uat key
+export PERMIT_API_KEY="permit_key_VjXBwoTFVhWWNecOt49BcAjyWWBaM8vzO9IvqNfJnqrLotz6BiSfr3hHV9v5zWqSQjYerqrpMlhz0WKYaLWMOg"
+go run setup_permit.go -list
+# for dev setup (this works)
+go run setup_permit.go -config permit_policies.yaml -project default -env uat
+
+# for production (this does not work)
+#go run setup_permit.go -config permit_policies.yaml -project default -env production
+

cmd/queryAPI/main.go

@@ -26,6 +26,7 @@ import (
 	"queryorchestration/internal/cognitoauth"
 	"queryorchestration/internal/collector"
 	collectorset "queryorchestration/internal/collector/set"
+	"queryorchestration/internal/customschema"
 	"queryorchestration/internal/document"
 	documentbatch "queryorchestration/internal/document/batch"
 	documentdownload "queryorchestration/internal/document/download"
@@ -83,6 +84,7 @@ func main() {
 		lbl := label.New(cfg)
 		eul := eula.New(cfg)
 		uiSvc := uisettings.New(cfg)
+		customSchema := customschema.New(cfg, &customschema.SchemaValidator{}, customschema.NewSlogAuditSink(cfg.GetLogger()))
 
 		services := &queryapi.Services{
 			Export:           exp,
@@ -96,6 +98,7 @@ func main() {
 			DocumentBatch:    docbatch,
 			UISettings:       uiSvc,
 			FieldExtraction:  fieldext,
+			CustomSchema:     customSchema,
 			Folder:           fld,
 			Label:            lbl,
 			Eula:             eul,