Merged in feature/remove-mocks (pull request #202)

Feature/remove mocks * remove mocks * cleanup db migrations
2026-01-15 20:39:32 +00:00
parent 0ddae4f91e
commit 35d72fccbe
244 changed files with 2573 additions and 50860 deletions
@@ -1,113 +0,0 @@
-package clientsync
-
-import (
-	"fmt"
-	"testing"
-
-	"queryorchestration/internal/database/repository"
-	"queryorchestration/internal/document"
-	"queryorchestration/internal/serviceconfig"
-	"queryorchestration/internal/serviceconfig/queue/documentsync"
-	queuemock "queryorchestration/mocks/queue"
-
-	"github.com/aws/aws-sdk-go-v2/service/sqs"
-	"github.com/google/uuid"
-	"github.com/pashagolub/pgxmock/v3"
-	"github.com/stretchr/testify/mock"
-	"github.com/stretchr/testify/require"
-)
-
-type ClientSyncConfig struct {
-	serviceconfig.BaseConfig
-	documentsync.DocSyncConfig
-}
-
-func TestTrigger(t *testing.T) {
-	t.Run("two batches", func(t *testing.T) {
-		ctx := t.Context()
-
-		pool, err := pgxmock.NewPool()
-		require.NoError(t, err)
-
-		cfg := &ClientSyncConfig{}
-		cfg.DBPool = pool
-		cfg.DBQueries = repository.New(pool)
-		mockSQS := queuemock.NewMockSQSClient(t)
-		cfg.QueueClient = mockSQS
-		cfg.DocumentSyncURL = "/i/am/here"
-
-		svc := New(cfg)
-
-		clientId := "huuh"
-		docs := []document.DocumentSummary{
-			{
-				ID:       uuid.New(),
-				ClientID: clientId,
-			},
-			{
-				ID:       uuid.New(),
-				ClientID: clientId,
-			},
-		}
-
-		svc.batchSize = 1
-		total := int64(2)
-		pool.ExpectQuery("name: ListDocumentIDsBatch :many").WithArgs(clientId, svc.batchSize, int32(0)).
-			WillReturnRows(
-				pgxmock.NewRows([]string{"id", "totalCount"}).
-					AddRow(&docs[0].ID, &total),
-			)
-		mockSQS.EXPECT().
-			SendMessage(
-				mock.Anything,
-				mock.MatchedBy(func(in *sqs.SendMessageInput) bool {
-					return *in.QueueUrl == cfg.DocumentSyncURL && *in.MessageBody == fmt.Sprintf("{\"id\":\"%s\"}", docs[0].ID.String())
-				}),
-				mock.Anything,
-			).
-			Return(&sqs.SendMessageOutput{}, nil)
-		pool.ExpectQuery("name: ListDocumentIDsBatch :many").WithArgs(clientId, svc.batchSize, int32(1)).
-			WillReturnRows(
-				pgxmock.NewRows([]string{"id", "totalCount"}).
-					AddRow(&docs[1].ID, &total),
-			)
-		mockSQS.EXPECT().
-			SendMessage(
-				mock.Anything,
-				mock.MatchedBy(func(in *sqs.SendMessageInput) bool {
-					return *in.QueueUrl == cfg.DocumentSyncURL && *in.MessageBody == fmt.Sprintf("{\"id\":\"%s\"}", docs[1].ID.String())
-				}),
-				mock.Anything,
-			).
-			Return(&sqs.SendMessageOutput{}, nil)
-
-		err = svc.Sync(ctx, clientId)
-		require.NoError(t, err)
-	})
-	t.Run("empty batch", func(t *testing.T) {
-		ctx := t.Context()
-
-		pool, err := pgxmock.NewPool()
-		require.NoError(t, err)
-
-		cfg := &ClientSyncConfig{}
-		cfg.DBPool = pool
-		cfg.DBQueries = repository.New(pool)
-		mockSQS := queuemock.NewMockSQSClient(t)
-		cfg.QueueClient = mockSQS
-		cfg.DocumentSyncURL = "/i/am/here"
-
-		svc := New(cfg)
-
-		clientId := "hola"
-
-		svc.batchSize = 1
-		pool.ExpectQuery("name: ListDocumentIDsBatch :many").WithArgs(clientId, svc.batchSize, int32(0)).
-			WillReturnRows(
-				pgxmock.NewRows([]string{"id", "totalCount"}),
-			)
-
-		err = svc.Sync(ctx, clientId)
-		require.NoError(t, err)
-	})
-}
@@ -1,7 +1,8 @@
+// Package clientupdate tests for client update functionality.
+// Note: Mock-based tests have been removed. See remove_texttract_and_mocks_plan.md.
 package clientupdate

 import (
-	"fmt"
 	"testing"

 	"queryorchestration/internal/client"
@@ -9,12 +10,8 @@ import (
 	"queryorchestration/internal/serviceconfig"
 	"queryorchestration/internal/serviceconfig/queue/clientsync"

-	queuemock "queryorchestration/mocks/queue"
-
-	"github.com/aws/aws-sdk-go-v2/service/sqs"
 	"github.com/pashagolub/pgxmock/v3"
 	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/mock"
 	"github.com/stretchr/testify/require"
 )

@@ -165,48 +162,5 @@ func TestSubmitUpdate(t *testing.T) {
 	require.NoError(t, err)
 }

-func TestInformUpdate(t *testing.T) {
-	ctx := t.Context()
-
-	cfg := &Config{}
-	mockSQS := queuemock.NewMockSQSClient(t)
-	cfg.QueueClient = mockSQS
-
-	svc := New(cfg, &Services{})
-
-	id := "HIELO"
-	t.Run("no cansync", func(t *testing.T) {
-		update := Update{}
-
-		err := svc.informUpdate(ctx, id, &update)
-		require.NoError(t, err)
-	})
-	t.Run("false cansync", func(t *testing.T) {
-		cansync := false
-		update := Update{
-			CanSync: &cansync,
-		}
-
-		err := svc.informUpdate(ctx, id, &update)
-		require.NoError(t, err)
-	})
-	t.Run("true cansync", func(t *testing.T) {
-		cansync := true
-		update := Update{
-			CanSync: &cansync,
-		}
-
-		mockSQS.EXPECT().
-			SendMessage(
-				mock.Anything,
-				mock.MatchedBy(func(in *sqs.SendMessageInput) bool {
-					return *in.QueueUrl == cfg.ClientSyncURL && *in.MessageBody == fmt.Sprintf("{\"id\":\"%s\"}", id)
-				}),
-				mock.Anything,
-			).
-			Return(&sqs.SendMessageOutput{}, nil)
-
-		err := svc.informUpdate(ctx, id, &update)
-		require.NoError(t, err)
-	})
-}
+// Note: TestInformUpdate was removed as it used SQS mocks.
+// Real SQS tests should be added using LocalStack via testcontainers.
@@ -1,3 +1,5 @@
+// Package collector_test contains tests for the collector package.
+// Note: MinTextVersion has been removed since text extraction is deprecated.
 package collector_test

 import (
@@ -24,17 +26,15 @@ func TestGetByClientID(t *testing.T) {
 	svc := collector.New(cfg)

 	minCleanV := int64(2)
-	minTextV := int64(4)
 	ogc := collector.Collector{
 		ClientID:        "hi",
 		MinCleanVersion: minCleanV,
-		MinTextVersion:  minTextV,
 	}

 	pool.ExpectQuery("name: GetCollectorByClientID :one").WithArgs(ogc.ClientID).
 		WillReturnRows(
-			pgxmock.NewRows([]string{"clientId", "minCleanVersion", "minTextVersion", "activeVersion", "latestVersion"}).
-				AddRow(ogc.ClientID, ogc.MinCleanVersion, ogc.MinTextVersion, ogc.ActiveVersion, ogc.LatestVersion),
+			pgxmock.NewRows([]string{"clientId", "minCleanVersion", "activeVersion", "latestVersion"}).
+				AddRow(ogc.ClientID, ogc.MinCleanVersion, ogc.ActiveVersion, ogc.LatestVersion),
 		)

 	coll, err := svc.GetByClientID(ctx, ogc.ClientID)
@@ -5,7 +5,7 @@ import (
 )

 // parseDBCollector converts a database collector record to a Collector struct.
-// Query fields have been removed - only document processing metadata is returned.
+// Note: MinTextVersion has been removed since text extraction is deprecated.
 func parseDBCollector(c *repository.Fullactivecollector) (*Collector, error) {
 	if c == nil {
 		return nil, nil
@@ -14,7 +14,6 @@ func parseDBCollector(c *repository.Fullactivecollector) (*Collector, error) {
 	return &Collector{
 		ClientID:        c.Clientid,
 		MinCleanVersion: c.Mincleanversion,
-		MinTextVersion:  c.Mintextversion,
 		ActiveVersion:   c.Activeversion,
 		LatestVersion:   c.Latestversion,
 	}, nil
@@ -1,3 +1,5 @@
+// Package collector tests.
+// Note: MinTextVersion has been removed since text extraction is deprecated.
 package collector

 import (
@@ -15,22 +17,18 @@ func TestParseDBCollector(t *testing.T) {
 	assert.Nil(t, c)

 	minCleanV := int64(1)
-	minTextV := int64(2)
 	ogc := Collector{
 		ClientID:        "hi",
 		MinCleanVersion: minCleanV,
-		MinTextVersion:  minTextV,
 	}
 	c, err = parseDBCollector(&repository.Fullactivecollector{
 		Clientid:        ogc.ClientID,
 		Mincleanversion: ogc.MinCleanVersion,
-		Mintextversion:  ogc.MinTextVersion,
 	})
 	require.NoError(t, err)
 	assert.EqualExportedValues(t, ogc, *c)

 	ogc.MinCleanVersion = 0
-	ogc.MinTextVersion = 0
 	c, err = parseDBCollector(&repository.Fullactivecollector{
 		Clientid: ogc.ClientID,
 	})
@@ -1,3 +1,6 @@
+// Package collector provides collector configuration functionality.
+// Note: Text extraction has been removed. MinTextVersion is no longer supported.
+// See remove_texttract_and_mocks_plan.md for details.
 package collector

 import (
@@ -5,19 +8,20 @@ import (
 )

 // Collector represents a client's document collection configuration.
-// Query processing has been removed - only document processing metadata remains.
+// Note: MinTextVersion has been removed since text extraction is deprecated.
 type Collector struct {
 	ClientID        string
 	MinCleanVersion int64
-	MinTextVersion  int64
 	ActiveVersion   int32
 	LatestVersion   int32
 }

+// Service provides collector operations.
 type Service struct {
 	cfg serviceconfig.ConfigProvider
 }

+// New creates a new collector service.
 func New(cfg serviceconfig.ConfigProvider) *Service {
 	return &Service{
 		cfg,
@@ -1,3 +1,6 @@
+// Package collectorset provides functionality for updating collector configurations.
+// Note: Text extraction has been removed. MinTextVersion support has been removed.
+// See remove_texttract_and_mocks_plan.md for details.
 package collectorset

 import (
@@ -14,11 +17,10 @@ import (
 )

 // SetParams defines parameters for updating a collector's configuration.
-// Query fields have been removed - only document processing settings remain.
+// Note: MinTextVersion has been removed since text extraction is deprecated.
 type SetParams struct {
 	ActiveVersion   *int32
 	MinCleanVersion *int64
-	MinTextVersion  *int64
 }

 func (s *Service) SetByClientId(ctx context.Context, id string, params *SetParams) error {
@@ -62,7 +64,6 @@ type dbSetParams struct {
 	ClientID        string
 	ActiveVersion   *int32
 	MinCleanVersion *int64
-	MinTextVersion  *int64
 }

 func (s *Service) getSetParams(id string, current *collector.Collector, params *SetParams) (*dbSetParams, error) {
@@ -90,7 +91,6 @@ func (s *Service) getSetParams(id string, current *collector.Collector, params *
 		ClientID:        id,
 		ActiveVersion:   params.ActiveVersion,
 		MinCleanVersion: params.MinCleanVersion,
-		MinTextVersion:  params.MinTextVersion,
 	}, nil
 }

@@ -114,17 +114,6 @@ func (s *Service) normalizeCodeVersions(current *collector.Collector, params *Se
 		}
 	}

-	if params.MinTextVersion != nil && *params.MinTextVersion == current.MinTextVersion {
-		params.MinTextVersion = nil
-	}
-
-	if params.MinTextVersion != nil {
-		err := build.IsValidUnixVersion(*params.MinTextVersion)
-		if err != nil {
-			return err
-		}
-	}
-
 	return nil
 }

@@ -182,17 +171,6 @@ func (s *Service) submitSet(ctx context.Context, params *dbSetParams) error {
 			}
 		}

-		if params.MinTextVersion != nil {
-			err = qtx.SetCollectorTextVersion(ctx, &repository.SetCollectorTextVersionParams{
-				Clientid:     params.ClientID,
-				Versionid:    *params.MinTextVersion,
-				Addedversion: latestVersion,
-			})
-			if err != nil {
-				return err
-			}
-		}
-
 		slog.Debug("client collector updated", "update", *params)

 		return nil
@@ -1,126 +0,0 @@
-package collectorset_test
-
-import (
-	"fmt"
-	"testing"
-
-	"queryorchestration/internal/collector"
-	collectorset "queryorchestration/internal/collector/set"
-	"queryorchestration/internal/database/repository"
-	"queryorchestration/internal/serviceconfig"
-	"queryorchestration/internal/serviceconfig/queue/clientsync"
-	queuemock "queryorchestration/mocks/queue"
-
-	"github.com/aws/aws-sdk-go-v2/service/sqs"
-	"github.com/jackc/pgx/v5"
-	"github.com/pashagolub/pgxmock/v3"
-	"github.com/stretchr/testify/mock"
-	"github.com/stretchr/testify/require"
-)
-
-type CollectorSetConfig struct {
-	serviceconfig.BaseConfig
-	clientsync.ClientSyncConfig
-}
-
-func TestSet(t *testing.T) {
-	ctx := t.Context()
-
-	pool, err := pgxmock.NewPool()
-	require.NoError(t, err)
-	cfg := &CollectorSetConfig{}
-	cfg.DBPool = pool
-	cfg.DBQueries = repository.New(pool)
-	mockSQS := queuemock.NewMockSQSClient(t)
-	cfg.QueueClient = mockSQS
-	cfg.ClientSyncURL = "here"
-
-	svc := collectorset.New(cfg, &collectorset.Services{
-		Collector: collector.New(cfg),
-	})
-
-	t.Run("valid", func(t *testing.T) {
-		current := collector.Collector{
-			ClientID:      "hi",
-			ActiveVersion: 1,
-			LatestVersion: 4,
-		}
-		av := int32(2)
-		mv := int64(1)
-		update := collectorset.SetParams{
-			ActiveVersion:   &av,
-			MinCleanVersion: &mv,
-		}
-
-		pool.ExpectQuery("name: GetCollectorByClientID :one").WithArgs(current.ClientID).
-			WillReturnRows(
-				pgxmock.NewRows([]string{"clientId", "minCleanVersion", "minTextVersion", "activeVersion", "latestVersion"}).
-					AddRow(current.ClientID, current.MinCleanVersion, current.MinTextVersion, current.ActiveVersion, current.LatestVersion),
-			)
-		pool.ExpectBeginTx(pgx.TxOptions{})
-		rv := current.LatestVersion + 1
-		pool.ExpectQuery("name: AddLatestCollectorVersion :one").WithArgs(current.ClientID).WillReturnRows(
-			pgxmock.NewRows([]string{"version"}).
-				AddRow(rv),
-		)
-		pool.ExpectExec("name: SetActiveCollectorVersion :exec").WithArgs(current.ClientID, int32(2)).
-			WillReturnResult(pgxmock.NewResult("", 1))
-		pool.ExpectExec("name: SetCollectorCleanVersion :exec").WithArgs(current.ClientID, rv, *update.MinCleanVersion).
-			WillReturnResult(pgxmock.NewResult("", 1))
-		pool.ExpectCommit()
-
-		mockSQS.EXPECT().
-			SendMessage(
-				mock.Anything,
-				mock.MatchedBy(func(in *sqs.SendMessageInput) bool {
-					return *in.QueueUrl == cfg.ClientSyncURL && *in.MessageBody == fmt.Sprintf("{\"id\":\"%s\"}", current.ClientID)
-				}),
-				mock.Anything,
-			).
-			Return(&sqs.SendMessageOutput{}, nil)
-
-		err = svc.SetByClientId(ctx, current.ClientID, &update)
-		require.NoError(t, err)
-	})
-	t.Run("no collector", func(t *testing.T) {
-		current := collector.Collector{
-			ClientID: "string",
-		}
-		av := int32(1)
-		mv := int64(1)
-		update := collectorset.SetParams{
-			ActiveVersion:   &av,
-			MinCleanVersion: &mv,
-		}
-
-		pool.ExpectQuery("name: GetCollectorByClientID :one").WithArgs(current.ClientID).
-			WillReturnRows(
-				pgxmock.NewRows([]string{"clientId", "minCleanVersion", "minTextVersion", "activeVersion", "latestVersion"}).
-					AddRow(current.ClientID, current.MinCleanVersion, current.MinTextVersion, current.ActiveVersion, current.LatestVersion),
-			)
-		pool.ExpectBeginTx(pgx.TxOptions{})
-		rv := current.LatestVersion + 1
-		pool.ExpectQuery("name: AddLatestCollectorVersion :one").WithArgs(current.ClientID).WillReturnRows(
-			pgxmock.NewRows([]string{"version"}).
-				AddRow(rv),
-		)
-		pool.ExpectExec("name: SetActiveCollectorVersion :exec").WithArgs(current.ClientID, int32(1)).
-			WillReturnResult(pgxmock.NewResult("", 1))
-		pool.ExpectExec("name: SetCollectorCleanVersion :exec").WithArgs(current.ClientID, rv, *update.MinCleanVersion).
-			WillReturnResult(pgxmock.NewResult("", 1))
-		pool.ExpectCommit()
-
-		mockSQS.EXPECT().
-			SendMessage(
-				mock.Anything,
-				mock.MatchedBy(func(in *sqs.SendMessageInput) bool {
-					return *in.QueueUrl == cfg.ClientSyncURL && *in.MessageBody == fmt.Sprintf("{\"id\":\"%s\"}", current.ClientID)
-				}),
-				mock.Anything,
-			).
-			Return(&sqs.SendMessageOutput{}, nil)
-
-		err = svc.SetByClientId(ctx, current.ClientID, &update)
-		require.NoError(t, err)
-	})
-}
@@ -1,197 +1,17 @@
+// Package collectorset private tests.
+// Note: MinTextVersion has been removed since text extraction is deprecated.
+// Note: Mock-based tests have been removed. See remove_texttract_and_mocks_plan.md.
 package collectorset

 import (
-	"fmt"
 	"testing"

 	"queryorchestration/internal/collector"
-	"queryorchestration/internal/database/repository"
-	"queryorchestration/internal/serviceconfig"
-	"queryorchestration/internal/serviceconfig/queue/clientsync"
-	queuemock "queryorchestration/mocks/queue"

-	"github.com/aws/aws-sdk-go-v2/service/sqs"
-	"github.com/jackc/pgx/v5"
-	"github.com/pashagolub/pgxmock/v3"
 	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/mock"
 	"github.com/stretchr/testify/require"
 )

-type CollectorSetConfig struct {
-	serviceconfig.BaseConfig
-	clientsync.ClientSyncConfig
-}
-
-func TestGetSetParams(t *testing.T) {
-	t.Run("all params", func(t *testing.T) {
-		pool, err := pgxmock.NewPool()
-		require.NoError(t, err)
-		cfg := &CollectorSetConfig{}
-		cfg.DBPool = pool
-		cfg.DBQueries = repository.New(pool)
-
-		svc := Service{
-			cfg: cfg,
-			svc: &Services{
-				Collector: collector.New(cfg),
-			},
-		}
-
-		minCleanV := int64(1)
-		minTextV := int64(1)
-		aV := int32(3)
-		current := collector.Collector{
-			ActiveVersion: 1,
-			LatestVersion: 10,
-		}
-		params := SetParams{
-			ActiveVersion:   &aV,
-			MinCleanVersion: &minCleanV,
-			MinTextVersion:  &minTextV,
-		}
-
-		clientId := "string"
-
-		dbparams, err := svc.getSetParams(clientId, &current, &params)
-		require.NoError(t, err)
-		assert.EqualExportedValues(t, &dbSetParams{
-			ClientID:        clientId,
-			ActiveVersion:   &aV,
-			MinCleanVersion: &minCleanV,
-			MinTextVersion:  &minTextV,
-		}, dbparams)
-	})
-
-	t.Run("no params", func(t *testing.T) {
-		pool, err := pgxmock.NewPool()
-		require.NoError(t, err)
-		cfg := &CollectorSetConfig{}
-		cfg.DBPool = pool
-		cfg.DBQueries = repository.New(pool)
-
-		svc := Service{
-			cfg: cfg,
-			svc: &Services{},
-		}
-
-		current := collector.Collector{}
-		params := SetParams{}
-		_, err = svc.getSetParams(current.ClientID, &current, &params)
-		assert.EqualError(t, err, "no changes")
-	})
-
-	t.Run("ONLY active version to next latest", func(t *testing.T) {
-		pool, err := pgxmock.NewPool()
-		require.NoError(t, err)
-		cfg := &CollectorSetConfig{}
-		cfg.DBPool = pool
-		cfg.DBQueries = repository.New(pool)
-
-		svc := Service{
-			cfg: cfg,
-			svc: &Services{},
-		}
-
-		current := collector.Collector{
-			ActiveVersion: 2,
-			LatestVersion: 4,
-		}
-		version := int32(5)
-		params := SetParams{
-			ActiveVersion: &version,
-		}
-		_, err = svc.getSetParams(current.ClientID, &current, &params)
-		assert.EqualError(t, err, "no changes")
-	})
-}
-
-func TestSubmitSet(t *testing.T) {
-	t.Run("all fields", func(t *testing.T) {
-		ctx := t.Context()
-
-		pool, err := pgxmock.NewPool()
-		require.NoError(t, err)
-		cfg := &CollectorSetConfig{}
-		cfg.DBPool = pool
-		cfg.DBQueries = repository.New(pool)
-
-		svc := Service{
-			cfg: cfg,
-			svc: &Services{},
-		}
-
-		minCleanV := int64(2)
-		minTextV := int64(4)
-		current := collector.Collector{
-			ClientID:        "hello",
-			ActiveVersion:   1,
-			LatestVersion:   1,
-			MinCleanVersion: 1,
-			MinTextVersion:  2,
-		}
-		aV := int32(2)
-		params := dbSetParams{
-			ClientID:        current.ClientID,
-			ActiveVersion:   &aV,
-			MinCleanVersion: &minCleanV,
-			MinTextVersion:  &minTextV,
-		}
-
-		pool.ExpectBeginTx(pgx.TxOptions{})
-		rv := int32(2)
-		pool.ExpectQuery("name: AddLatestCollectorVersion :one").WithArgs(current.ClientID).WillReturnRows(
-			pgxmock.NewRows([]string{"version"}).
-				AddRow(rv),
-		)
-		pool.ExpectExec("name: SetActiveCollectorVersion :exec").WithArgs(current.ClientID, int32(2)).
-			WillReturnResult(pgxmock.NewResult("", 1))
-		pool.ExpectExec("name: SetCollectorCleanVersion :exec").WithArgs(current.ClientID, rv, *params.MinCleanVersion).
-			WillReturnResult(pgxmock.NewResult("", 1))
-		pool.ExpectExec("name: SetCollectorTextVersion :exec").WithArgs(current.ClientID, rv, *params.MinTextVersion).
-			WillReturnResult(pgxmock.NewResult("", 1))
-		pool.ExpectCommit()
-
-		err = svc.submitSet(ctx, &params)
-		require.NoError(t, err)
-	})
-	t.Run("only active version", func(t *testing.T) {
-		ctx := t.Context()
-
-		pool, err := pgxmock.NewPool()
-		require.NoError(t, err)
-		cfg := &CollectorSetConfig{}
-		cfg.DBPool = pool
-		cfg.DBQueries = repository.New(pool)
-
-		svc := Service{
-			cfg: cfg,
-			svc: &Services{},
-		}
-
-		current := collector.Collector{
-			ClientID:        "hell",
-			ActiveVersion:   1,
-			LatestVersion:   1,
-			MinCleanVersion: 1,
-			MinTextVersion:  2,
-		}
-		av := int32(2)
-		params := dbSetParams{
-			ClientID:      current.ClientID,
-			ActiveVersion: &av,
-		}
-
-		pool.ExpectBeginTx(pgx.TxOptions{})
-		pool.ExpectExec("name: SetActiveCollectorVersion :exec").WithArgs(current.ClientID, int32(2)).
-			WillReturnResult(pgxmock.NewResult("", 1))
-		pool.ExpectCommit()
-
-		err = svc.submitSet(ctx, &params)
-		require.NoError(t, err)
-	})
-}
-
 func TestNormalizeActiveVersion(t *testing.T) {
 	svc := Service{}

@@ -284,7 +104,6 @@ func TestNormalizeCodeVersions(t *testing.T) {
 		err := svc.normalizeCodeVersions(&current, &update)
 		require.NoError(t, err)
 		assert.Nil(t, update.MinCleanVersion)
-		assert.Nil(t, update.MinTextVersion)
 	})

 	t.Run("valid clean", func(t *testing.T) {
@@ -296,31 +115,6 @@ func TestNormalizeCodeVersions(t *testing.T) {
 		err := svc.normalizeCodeVersions(&current, &update)
 		require.NoError(t, err)
 		assert.Equal(t, cv, *update.MinCleanVersion)
-		assert.Nil(t, update.MinTextVersion)
-	})
-
-	t.Run("valid text", func(t *testing.T) {
-		current := collector.Collector{}
-		update := SetParams{}
-
-		tv := int64(1)
-		update.MinTextVersion = &tv
-		err := svc.normalizeCodeVersions(&current, &update)
-		require.NoError(t, err)
-		assert.Nil(t, update.MinCleanVersion)
-		assert.Equal(t, tv, *update.MinTextVersion)
-	})
-
-	t.Run("valid clean and text", func(t *testing.T) {
-		current := collector.Collector{}
-		update := SetParams{}
-
-		current.MinCleanVersion = 1
-		current.MinTextVersion = 1
-		err := svc.normalizeCodeVersions(&current, &update)
-		require.NoError(t, err)
-		assert.Nil(t, update.MinCleanVersion)
-		assert.Nil(t, update.MinTextVersion)
 	})

 	t.Run("current clean", func(t *testing.T) {
@@ -333,91 +127,5 @@ func TestNormalizeCodeVersions(t *testing.T) {
 		err := svc.normalizeCodeVersions(&current, &update)
 		require.NoError(t, err)
 		assert.Nil(t, update.MinCleanVersion)
-		assert.Nil(t, update.MinTextVersion)
-	})
-
-	t.Run("current text", func(t *testing.T) {
-		current := collector.Collector{
-			MinTextVersion: 1,
-		}
-		update := SetParams{}
-
-		update.MinTextVersion = &current.MinTextVersion
-		err := svc.normalizeCodeVersions(&current, &update)
-		require.NoError(t, err)
-		assert.Nil(t, update.MinCleanVersion)
-		assert.Nil(t, update.MinTextVersion)
-	})
-
-	t.Run("current clean and text", func(t *testing.T) {
-		current := collector.Collector{
-			MinCleanVersion: 1,
-			MinTextVersion:  1,
-		}
-		update := SetParams{}
-
-		update.MinCleanVersion = &current.MinCleanVersion
-		update.MinTextVersion = &current.MinTextVersion
-		err := svc.normalizeCodeVersions(&current, &update)
-		require.NoError(t, err)
-		assert.Nil(t, update.MinCleanVersion)
-		assert.Nil(t, update.MinTextVersion)
-	})
-}
-
-func TestInformSet(t *testing.T) {
-	t.Run("with version update", func(t *testing.T) {
-		ctx := t.Context()
-
-		pool, err := pgxmock.NewPool()
-		require.NoError(t, err)
-		cfg := &CollectorSetConfig{}
-		cfg.DBPool = pool
-		cfg.DBQueries = repository.New(pool)
-		mockSQS := queuemock.NewMockSQSClient(t)
-		cfg.QueueClient = mockSQS
-		cfg.ClientSyncURL = "here"
-
-		svc := New(cfg, &Services{})
-
-		av := int32(2)
-		update := dbSetParams{
-			ClientID:      "HI",
-			ActiveVersion: &av,
-		}
-
-		mockSQS.EXPECT().
-			SendMessage(
-				mock.Anything,
-				mock.MatchedBy(func(in *sqs.SendMessageInput) bool {
-					return *in.QueueUrl == cfg.ClientSyncURL && *in.MessageBody == fmt.Sprintf("{\"id\":\"%s\"}", update.ClientID)
-				}),
-				mock.Anything,
-			).
-			Return(&sqs.SendMessageOutput{}, nil)
-
-		err = svc.informSet(ctx, &update)
-		require.NoError(t, err)
-	})
-	t.Run("with no update", func(t *testing.T) {
-		ctx := t.Context()
-
-		pool, err := pgxmock.NewPool()
-		require.NoError(t, err)
-		cfg := &CollectorSetConfig{}
-		cfg.DBPool = pool
-		cfg.DBQueries = repository.New(pool)
-		mockSQS := queuemock.NewMockSQSClient(t)
-		cfg.QueueClient = mockSQS
-		cfg.ClientSyncURL = "here"
-
-		svc := New(cfg, &Services{})
-
-		update := dbSetParams{
-			ClientID: "hello",
-		}
-
-		err = svc.informSet(ctx, &update)
-		require.NoError(t, err)
 	})
 }
@@ -1 +1,7 @@
-DROP EXTENSION "pgcrypto";
+-- Migration 001: Extensions and Core Functions (DOWN)
+-- Drops in reverse order of creation
+
+DROP DOMAIN IF EXISTS unsignedsmallint;
+DROP FUNCTION IF EXISTS isInVersion;
+DROP FUNCTION IF EXISTS uuid_generate_v7;
+DROP EXTENSION IF EXISTS "pgcrypto";
@@ -1,9 +1,13 @@
+-- Migration 001: Extensions and Core Functions
+-- Creates PostgreSQL extensions and utility functions used throughout the schema
+
 CREATE EXTENSION IF NOT EXISTS "pgcrypto";

-create or replace function uuid_generate_v7()
-returns uuid
-as $$
-select encode(
+-- UUID v7 generator function (time-ordered UUIDs)
+CREATE OR REPLACE FUNCTION uuid_generate_v7()
+RETURNS uuid
+AS $$
+SELECT encode(
    set_bit(
      set_bit(
        overlay(uuid_send(gen_random_uuid())
@@ -16,22 +20,24 @@ select encode(
    ),
    'hex')::uuid;
 $$
-language SQL
-volatile;
+LANGUAGE SQL
+VOLATILE;

+-- Version checking function for temporal data patterns
 CREATE FUNCTION isInVersion(
    _version int,
    _addedVersion int,
    _removedVersion int
 )
-RETURNS boolean as $$
+RETURNS boolean AS $$
 BEGIN
-    RETURN _version is null OR (
+    RETURN _version IS NULL OR (
        _version >= _addedVersion AND
-        (_removedVersion is null OR _version < _removedVersion)
+        (_removedVersion IS NULL OR _version < _removedVersion)
    );
 END;
 $$ LANGUAGE plpgsql;

+-- Domain for unsigned small integers
 CREATE DOMAIN unsignedsmallint AS SMALLINT
  CHECK (VALUE >= 0);
@@ -0,0 +1,4 @@
+-- Migration 002: Clients (DOWN)
+
+DROP TABLE IF EXISTS clientCanSync;
+DROP TABLE IF EXISTS clients;
@@ -0,0 +1,15 @@
+-- Migration 002: Clients
+-- Core client tables for customer configuration
+
+CREATE TABLE clients (
+    clientId varchar(255) PRIMARY KEY,
+    name TEXT NOT NULL,
+    UNIQUE (name)
+);
+
+CREATE TABLE clientCanSync (
+    syncId uuid PRIMARY KEY DEFAULT uuid_generate_v7(),
+    clientId varchar(255) NOT NULL,
+    canSync boolean NOT NULL,
+    FOREIGN KEY (clientId) REFERENCES clients(clientId)
+);
@@ -0,0 +1,9 @@
+-- Migration 003: Collectors (DOWN)
+
+DROP TRIGGER IF EXISTS removeMinCleanVersionTrigger ON collectorMinCleanVersions;
+DROP FUNCTION IF EXISTS removeMinCleanVersion;
+DROP TABLE IF EXISTS collectorMinCleanVersions;
+DROP TABLE IF EXISTS collectorActiveVersions;
+DROP TRIGGER IF EXISTS setCollectorVersionNumberTrigger ON collectorVersions;
+DROP FUNCTION IF EXISTS setCollectorVersionNumber;
+DROP TABLE IF EXISTS collectorVersions;
@@ -0,0 +1,62 @@
+-- Migration 003: Collectors (Simplified)
+-- Collector versioning and configuration
+-- NOTE: collectorMinTextVersions and collectorQueries removed (no longer used)
+
+CREATE TABLE collectorVersions (
+    clientId varchar(255) NOT NULL,
+    id int NOT NULL,
+    addedAt timestamp NOT NULL DEFAULT current_timestamp,
+    FOREIGN KEY (clientId) REFERENCES clients(clientId),
+    PRIMARY KEY (id, clientId)
+);
+
+CREATE OR REPLACE FUNCTION setCollectorVersionNumber()
+RETURNS TRIGGER AS $$
+BEGIN
+    SELECT COALESCE(MAX(id), 0) + 1
+    INTO NEW.id
+    FROM collectorVersions
+    WHERE clientId = NEW.clientId;
+
+    RETURN NEW;
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE TRIGGER setCollectorVersionNumberTrigger
+BEFORE INSERT ON collectorVersions
+FOR EACH ROW
+EXECUTE FUNCTION setCollectorVersionNumber();
+
+CREATE TABLE collectorActiveVersions (
+    id uuid PRIMARY KEY DEFAULT uuid_generate_v7(),
+    clientId varchar(255) NOT NULL,
+    versionId int NOT NULL,
+    FOREIGN KEY (clientId, versionId) REFERENCES collectorVersions(clientId, id)
+);
+
+CREATE TABLE collectorMinCleanVersions (
+    id uuid PRIMARY KEY DEFAULT uuid_generate_v7(),
+    clientId varchar(255) NOT NULL,
+    versionId bigint NOT NULL,
+    addedVersion int NOT NULL,
+    removedVersion int,
+    FOREIGN KEY (clientId, addedVersion) REFERENCES collectorVersions(clientId, id),
+    FOREIGN KEY (clientId, removedVersion) REFERENCES collectorVersions(clientId, id),
+    FOREIGN KEY (clientId) REFERENCES clients(clientId)
+);
+
+CREATE OR REPLACE FUNCTION removeMinCleanVersion()
+RETURNS TRIGGER AS $$
+BEGIN
+    UPDATE collectorMinCleanVersions
+        SET removedVersion = NEW.addedVersion
+        WHERE clientId = NEW.clientId AND removedVersion IS NULL;
+
+    RETURN NEW;
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE TRIGGER removeMinCleanVersionTrigger
+BEFORE INSERT ON collectorMinCleanVersions
+FOR EACH ROW
+EXECUTE FUNCTION removeMinCleanVersion();
@@ -0,0 +1,6 @@
+-- Migration 004: Batch Uploads (DOWN)
+
+DROP INDEX IF EXISTS idx_batch_uploads_status;
+DROP INDEX IF EXISTS idx_batch_uploads_client_id;
+DROP TABLE IF EXISTS batch_uploads;
+DROP TYPE IF EXISTS batch_status;
@@ -0,0 +1,52 @@
+-- Migration 004: Batch Uploads
+-- Batch upload tracking for ZIP file uploads
+-- Merges original migrations 103 + 104
+
+-- Create batch status ENUM type
+CREATE TYPE batch_status AS ENUM (
+    'processing',       -- Currently processing
+    'completed',        -- All documents processed (may have individual failures)
+    'failed',           -- Batch-level failure (ZIP corrupt, etc.)
+    'cancelled'         -- Cancelled by user
+);
+
+-- Create batch uploads table with all columns
+CREATE TABLE batch_uploads (
+    id uuid PRIMARY KEY DEFAULT uuid_generate_v7(),
+    client_id varchar(255) NOT NULL,
+    original_filename text NOT NULL,
+
+    -- Essential batch metrics
+    total_documents integer NOT NULL,
+    processed_documents integer NOT NULL DEFAULT 0,
+    failed_documents integer NOT NULL DEFAULT 0,
+    invalid_type_documents integer NOT NULL DEFAULT 0,
+
+    -- Status and progress
+    status batch_status NOT NULL DEFAULT 'processing',
+    progress_percent integer NOT NULL DEFAULT 0,
+
+    -- Failed document filenames (JSON array)
+    failed_filenames jsonb DEFAULT '[]'::jsonb,
+
+    -- Timestamps
+    created_at timestamp NOT NULL DEFAULT NOW(),
+    completed_at timestamp,
+
+    -- Storage metadata (from migration 104)
+    archive_bucket text,
+    archive_key text,
+    file_size_bytes bigint,
+
+    FOREIGN KEY (client_id) REFERENCES clients(clientId),
+
+    -- Constraint: storage metadata consistency
+    CONSTRAINT batch_storage_consistent CHECK (
+        (archive_bucket IS NULL AND archive_key IS NULL AND file_size_bytes IS NULL) OR
+        (archive_bucket IS NOT NULL AND archive_key IS NOT NULL)
+    )
+);
+
+-- Create indexes for efficient queries
+CREATE INDEX idx_batch_uploads_client_id ON batch_uploads(client_id);
+CREATE INDEX idx_batch_uploads_status ON batch_uploads(status);
@@ -0,0 +1,6 @@
+-- Migration 005: Folders (DOWN)
+
+DROP INDEX IF EXISTS idx_folders_path;
+DROP INDEX IF EXISTS idx_folders_clientid;
+DROP INDEX IF EXISTS idx_folders_parentid;
+DROP TABLE IF EXISTS folders;
@@ -0,0 +1,26 @@
+-- Migration 005: Folders
+-- Virtual folder hierarchy for document organization
+
+CREATE TABLE folders (
+    id uuid PRIMARY KEY DEFAULT uuid_generate_v7(),
+    path text NOT NULL,
+    parentId uuid,
+    clientId varchar(255) NOT NULL,
+    createdAt timestamp NOT NULL DEFAULT NOW(),
+    createdBy varchar(255) NOT NULL,
+    FOREIGN KEY (parentId) REFERENCES folders(id),
+    FOREIGN KEY (clientId) REFERENCES clients(clientId),
+    UNIQUE(clientId, path),
+    CHECK (id != parentId)
+);
+
+-- Indexes for efficient folder queries
+CREATE INDEX idx_folders_parentid ON folders(parentId);
+CREATE INDEX idx_folders_clientid ON folders(clientId);
+CREATE INDEX idx_folders_path ON folders(path);
+
+-- Add comment to clarify that folders are virtual (database-only, no S3 relationship)
+COMMENT ON TABLE folders IS
+    'Virtual folder hierarchy for document organization. Database-only - has no direct relationship to S3 storage locations.';
+COMMENT ON COLUMN folders.path IS
+    'Virtual folder path (e.g., "/contracts/2025"). Can be renamed without affecting S3 storage.';
@@ -0,0 +1,31 @@
+-- Migration 006: Documents (DOWN)
+
+-- Drop document clean entries first (depends on documentCleans)
+DROP TABLE IF EXISTS documentCleanEntries;
+
+-- Drop document cleans
+DROP TABLE IF EXISTS documentCleans;
+
+-- Drop document entries
+DROP TABLE IF EXISTS documentEntries;
+
+-- Drop documentUploads indexes
+DROP INDEX IF EXISTS idx_documentuploads_folder_id;
+DROP INDEX IF EXISTS idx_documentuploads_clientid;
+DROP INDEX IF EXISTS idx_documentuploads_batch_id;
+DROP INDEX IF EXISTS idx_documentuploads_filename;
+
+-- Drop documentUploads table
+DROP TABLE IF EXISTS documentUploads;
+
+-- Drop documents indexes
+DROP INDEX IF EXISTS idx_documents_folderid;
+DROP INDEX IF EXISTS idx_documents_filename;
+DROP INDEX IF EXISTS idx_documents_batch_id;
+
+-- Drop documents table
+DROP TABLE IF EXISTS documents;
+
+-- Drop enums
+DROP TYPE IF EXISTS cleanMimeType;
+DROP TYPE IF EXISTS cleanFailType;
@@ -0,0 +1,125 @@
+-- Migration 006: Documents (Consolidated)
+-- Document storage and processing tables
+-- Merges: 005_documents + 103 + 105 + 106 + 107 + 108 + 111 + 116 + 118
+-- NOTE: documentTextExtractions and documentTextExtractionEntries removed (no longer used)
+
+-- Enum for document clean failure types
+CREATE TYPE cleanFailType AS ENUM (
+    'invalid_mimetype',
+    'invalid_read',
+    'invalid_read_pages',
+    'zero_page_count',
+    'large_file',
+    'small_dimensions',
+    'large_dimensions',
+    'small_dpi',
+    'large_dpi'
+);
+
+-- Enum for clean document mime types
+CREATE TYPE cleanMimeType AS ENUM ('application/pdf');
+
+-- Main documents table with all columns
+CREATE TABLE documents (
+    id uuid PRIMARY KEY DEFAULT uuid_generate_v7(),
+    clientId varchar(255) NOT NULL,
+    hash text NOT NULL,
+
+    -- Batch reference (from migration 103)
+    batch_id uuid,
+
+    -- Filename (from migration 105)
+    filename TEXT,
+
+    -- Folder organization (from migration 111)
+    folderId uuid,
+    originalPath text,
+
+    -- File size (from migration 118)
+    file_size_bytes BIGINT,
+
+    FOREIGN KEY (clientId) REFERENCES clients(clientId),
+    FOREIGN KEY (batch_id) REFERENCES batch_uploads(id),
+    FOREIGN KEY (folderId) REFERENCES folders(id),
+    UNIQUE(clientId, hash)
+);
+
+-- Indexes for documents table
+CREATE INDEX idx_documents_batch_id ON documents(batch_id);
+CREATE INDEX idx_documents_filename ON documents(filename);
+CREATE INDEX idx_documents_folderid ON documents(folderId);
+
+-- Comments for immutable fields (from migration 111)
+COMMENT ON COLUMN documents.originalPath IS
+    'Original path provided during upload. IMMUTABLE after creation - never modify this value.';
+
+-- Document uploads table (staging area for incoming documents)
+CREATE TABLE documentUploads (
+    id uuid PRIMARY KEY DEFAULT uuid_generate_v7(),
+    bucket text NOT NULL,
+    key text NOT NULL,
+    clientId varchar(255) NOT NULL,
+    part unsignedsmallint NOT NULL,
+    createdAt timestamp NOT NULL,
+
+    -- Filename (from migration 106)
+    filename TEXT,
+
+    -- Batch reference (from migration 107)
+    batch_id uuid,
+
+    -- Folder reference (from migration 116)
+    folder_id uuid,
+
+    FOREIGN KEY (clientId) REFERENCES clients(clientId),
+    FOREIGN KEY (folder_id) REFERENCES folders(id)
+);
+
+-- Indexes for documentUploads
+CREATE INDEX idx_documentuploads_filename ON documentUploads(filename);
+CREATE INDEX idx_documentuploads_batch_id ON documentUploads(batch_id);
+CREATE INDEX idx_documentuploads_clientid ON documentUploads(clientId);
+CREATE INDEX idx_documentuploads_folder_id ON documentUploads(folder_id);
+
+-- Document entries (S3 storage records)
+CREATE TABLE documentEntries (
+    id uuid PRIMARY KEY DEFAULT uuid_generate_v7(),
+    documentId uuid NOT NULL,
+    bucket text NOT NULL,
+    key text NOT NULL,
+    FOREIGN KEY (documentId) REFERENCES documents(id)
+);
+
+-- Comments for immutable fields (from migration 111)
+COMMENT ON COLUMN documentEntries.key IS
+    'S3 object key. IMMUTABLE after creation - never modify this value.';
+COMMENT ON COLUMN documentEntries.bucket IS
+    'S3 bucket name. IMMUTABLE after creation - never modify this value.';
+
+-- Document cleans (processed/cleaned documents)
+CREATE TABLE documentCleans (
+    id uuid PRIMARY KEY DEFAULT uuid_generate_v7(),
+    documentId uuid NOT NULL,
+    bucket text,
+    key text,
+    hash text,
+    mimetype cleanMimeType,
+    fail cleanFailType,
+    FOREIGN KEY (documentId) REFERENCES documents(id),
+    CONSTRAINT bucket_and_key_together CHECK (
+        (bucket IS NULL) = (key IS NULL) AND
+        (bucket IS NULL) = (mimetype IS NULL) AND
+        (bucket IS NULL) = (hash IS NULL)
+    ),
+    CONSTRAINT location_xor_fail CHECK (
+        (bucket IS NOT NULL) != (fail IS NOT NULL)
+    )
+);
+
+-- Document clean entries (version tracking for cleans)
+CREATE TABLE documentCleanEntries (
+    id uuid PRIMARY KEY DEFAULT uuid_generate_v7(),
+    cleanId uuid NOT NULL,
+    version bigint NOT NULL,
+    FOREIGN KEY (cleanId) REFERENCES documentCleans(id)
+);
@@ -0,0 +1,8 @@
+-- Migration 007: Labels (DOWN)
+
+DROP INDEX IF EXISTS idx_documentlabels_document_label_time;
+DROP INDEX IF EXISTS idx_documentlabels_appliedat;
+DROP INDEX IF EXISTS idx_documentlabels_label;
+DROP INDEX IF EXISTS idx_documentlabels_documentid;
+DROP TABLE IF EXISTS documentLabels;
+DROP TABLE IF EXISTS labels;
@@ -0,0 +1,34 @@
+-- Migration 007: Labels
+-- Document labeling system for tracking processing status
+
+-- Create labels lookup table
+CREATE TABLE labels (
+    label text PRIMARY KEY,
+    description text NOT NULL
+);
+
+-- Seed initial label values
+INSERT INTO labels (label, description) VALUES
+    ('Ingested', 'Document has been ingested into the system'),
+    ('OCR_Processed', 'OCR text extraction completed'),
+    ('GenAI_Processed', 'GenAI processing completed'),
+    ('Doczy_AI_Completed', 'Doczy.AI processing completed'),
+    ('Dashboard_Ready', 'Document ready for dashboard display');
+
+-- Create documentLabels junction table
+CREATE TABLE documentLabels (
+    id uuid PRIMARY KEY DEFAULT uuid_generate_v7(),
+    documentId uuid NOT NULL,
+    label text NOT NULL,
+    appliedAt timestamp NOT NULL DEFAULT NOW(),
+    appliedBy varchar(255) NOT NULL,
+    FOREIGN KEY (documentId) REFERENCES documents(id) ON DELETE CASCADE,
+    FOREIGN KEY (label) REFERENCES labels(label)
+);
+
+-- Indexes for efficient label queries
+CREATE INDEX idx_documentlabels_documentid ON documentLabels(documentId);
+CREATE INDEX idx_documentlabels_label ON documentLabels(label);
+CREATE INDEX idx_documentlabels_appliedat ON documentLabels(appliedAt);
+-- Composite index for finding most recent label application
+CREATE INDEX idx_documentlabels_document_label_time ON documentLabels(documentId, label, appliedAt DESC);
@@ -0,0 +1,19 @@
+-- Migration 008: Field Extractions (DOWN)
+
+-- Drop array fields table and indexes
+DROP INDEX IF EXISTS idx_documentfieldextractionarrayfields_id_index;
+DROP INDEX IF EXISTS idx_documentfieldextractionarrayfields_fieldextractionid;
+DROP TABLE IF EXISTS documentFieldExtractionArrayFields;
+
+-- Drop versions table and indexes
+DROP INDEX IF EXISTS idx_unique_version_per_document;
+DROP INDEX IF EXISTS idx_documentfieldextractionversions_documentid;
+DROP INDEX IF EXISTS idx_documentfieldextractionversions_version;
+DROP INDEX IF EXISTS idx_documentfieldextractionversions_fieldextractionid;
+DROP TABLE IF EXISTS documentFieldExtractionVersions;
+
+-- Drop main field extractions table and indexes
+DROP INDEX IF EXISTS idx_documentfieldextractions_filename;
+DROP INDEX IF EXISTS idx_documentfieldextractions_createdby;
+DROP INDEX IF EXISTS idx_documentfieldextractions_documentid;
+DROP TABLE IF EXISTS documentFieldExtractions;
@@ -0,0 +1,254 @@
+-- Migration 008: Field Extractions (Consolidated)
+-- Document field extraction storage for GenAI-extracted data
+-- Merges: 112 + 113 + 114 + 117
+
+-- Create documentFieldExtractions table for single-value fields
+CREATE TABLE documentFieldExtractions (
+    id uuid PRIMARY KEY DEFAULT uuid_generate_v7(),
+    documentId uuid NOT NULL,
+
+    -- Single-value fields (19 fields total - 1:1 relationship)
+    -- Document identification
+    fileName text,
+    contractTitle text,
+    aareteDerivedAmendmentNum int,
+
+    -- Party information
+    clientName text,
+    payerName text,
+    payerState text,
+    providerState text,
+
+    -- Tax identification numbers (stored as text to preserve leading zeros)
+    filenameTin text,
+    provGroupTin text,
+    provGroupNpi text,
+    provGroupNameFull text,
+    provOtherTin text,
+    provOtherNpi text,
+    provOtherNameFull text,
+
+    -- Contract dates
+    aareteDerivedEffectiveDt date,
+    aareteDerivedTerminationDt date,
+
+    -- Renewal information
+    autoRenewalInd boolean,
+    autoRenewalTerm text,
+
+    -- Metadata
+    createdAt timestamp NOT NULL DEFAULT NOW(),
+    createdBy varchar(255) NOT NULL,
+
+    FOREIGN KEY (documentId) REFERENCES documents(id)
+);
+
+-- Indexes for efficient queries
+CREATE INDEX idx_documentfieldextractions_documentid
+    ON documentFieldExtractions(documentId);
+CREATE INDEX idx_documentfieldextractions_createdby
+    ON documentFieldExtractions(createdBy);
+CREATE INDEX idx_documentfieldextractions_filename
+    ON documentFieldExtractions(fileName);
+
+-- Create documentFieldExtractionVersions table for version tracking
+-- Includes documentId column from migration 117 for unique constraint
+CREATE TABLE documentFieldExtractionVersions (
+    id uuid PRIMARY KEY DEFAULT uuid_generate_v7(),
+    fieldExtractionId uuid NOT NULL,
+    version bigint NOT NULL,
+    createdBy varchar(255) NOT NULL,
+    createdAt timestamp NOT NULL DEFAULT NOW(),
+
+    -- documentId for unique version constraint (from migration 117)
+    documentId uuid NOT NULL,
+
+    FOREIGN KEY (fieldExtractionId) REFERENCES documentFieldExtractions(id),
+    FOREIGN KEY (documentId) REFERENCES documents(id)
+);
+
+-- Indexes for version queries
+CREATE INDEX idx_documentfieldextractionversions_fieldextractionid
+    ON documentFieldExtractionVersions(fieldExtractionId);
+CREATE INDEX idx_documentfieldextractionversions_version
+    ON documentFieldExtractionVersions(version);
+CREATE INDEX idx_documentfieldextractionversions_documentid
+    ON documentFieldExtractionVersions(documentId);
+
+-- Unique constraint to prevent duplicate versions per document (from migration 117)
+CREATE UNIQUE INDEX idx_unique_version_per_document
+    ON documentFieldExtractionVersions(documentId, version);
+
+-- Create documentFieldExtractionArrayFields table for 1:N array fields
+CREATE TABLE documentFieldExtractionArrayFields (
+    id uuid PRIMARY KEY DEFAULT uuid_generate_v7(),
+    fieldExtractionId uuid NOT NULL,
+    arrayIndex smallint NOT NULL,
+
+    -- Array-value fields (112 fields total - 1:N relationship)
+    -- All fields at same arrayIndex form one "row" of the array
+
+    -- Exhibit information
+    exhibitTitle text,
+    exhibitPage text,
+
+    -- Reimbursement provider information
+    reimbProvTin text,
+    reimbProvNpi text,
+    reimbProvName text,
+    reimbEffectiveDt date,
+    reimbTerminationDt date,
+
+    -- Claim and product codes
+    aareteDerivedClaimTypeCd text,
+    aareteDerivedProduct text,
+    aareteDerivedLob text,
+    aareteDerivedProgram text,
+    aareteDerivedNetwork text,
+    aareteDerivedProvType text,
+
+    -- Provider taxonomy and specialty
+    provTaxonomyCd text,
+    provTaxonomyCdDesc text,
+    provSpecialtyCd text,
+    provSpecialtyCdDesc text,
+
+    -- Service location
+    placeOfServiceCd text,
+    placeOfServiceCdDesc text,
+    billTypeCd text,
+    billTypeCdDesc text,
+
+    -- Patient demographics
+    patientAgeMin text,
+    patientAgeMax text,
+
+    -- Reimbursement terms
+    reimbTerm text,
+    lobProgramRelationship text,
+    lobProductRelationship text,
+
+    -- Carveout and payment logic
+    carveoutInd boolean,
+    carveoutCd text,
+    lesserOfInd boolean,
+    greaterOfInd boolean,
+    aareteDerivedReimbMethod text,
+    unitOfMeasure text,
+
+    -- Reimbursement rates
+    reimbPctRate numeric(10,4),
+    reimbFeeRate numeric(12,2),
+    reimbConversionFactor numeric(12,4),
+    triggerCapThresholdAmt numeric(12,2),
+    triggerBaseThreshold numeric(12,2),
+
+    -- Default and addition
+    defaultInd boolean,
+    additionDesc text,
+    additionMaxFeeRateInc numeric(12,2),
+    additionMaxPctRateInc numeric(10,4),
+    aareteDerivedAdditionRateChangeTimeline text,
+
+    -- Fee schedule
+    aareteDerivedFeeSchedule text,
+    aareteDerivedFeeScheduleVersion text,
+
+    -- Service codes
+    serviceTerm text,
+    cpt4ProcCd text,
+    cpt4ProcCdDesc text,
+    cpt4ProcMod text,
+    cpt4ProcModDesc text,
+    revenueCd text,
+    revenueCdDesc text,
+    diagCd text,
+    diagCdDesc text,
+    ndcCd text,
+    ndcCdDesc text,
+
+    -- Claim admit and status
+    claimAdmitTypeCd text,
+    authAdmitTypeDesc text,
+    claimStatusCd text,
+    claimStatusCdDesc text,
+
+    -- Grouper information
+    grouperType text,
+    grouperCd text,
+    grouperCdDesc text,
+    grouperPctRate numeric(10,4),
+    grouperBaseRate numeric(12,2),
+    aareteDerivedGrouperVersion text,
+    grouperAlternativeLevelOfCare text,
+    grouperSeverityInd boolean,
+    grouperSeverity text,
+    grouperRiskOfMortalitySubclass text,
+    grouperTransferInd boolean,
+    grouperReadmissionsInd boolean,
+    grouperHacInd boolean,
+
+    -- Outlier terms
+    outlierTerm text,
+    outlierFirstDollarInd boolean,
+    rangeNbrDays text,
+    outlierFixedLossNbrDaysThreshold numeric(10,2),
+    outlierFixedLossThreshold numeric(12,2),
+    outlierMaximum numeric(12,2),
+    outlierMaximumFrequency numeric(10,2),
+    outlierPctRate numeric(10,4),
+    outlierExclusionCd text,
+    outlierExclusionCdDesc text,
+
+    -- Facility adjustments
+    facilityAdjustmentTerm text,
+    dshInd boolean,
+    dshPctRate numeric(10,4),
+    dshFeeRate numeric(12,2),
+    imeInd boolean,
+    imePctRate numeric(10,4),
+    imeFeeRate numeric(12,2),
+    ntapInd boolean,
+    ntapPctRate numeric(10,4),
+    ntapFeeRate numeric(12,2),
+    ucInd boolean,
+    ucPctRate numeric(10,4),
+    ucFeeRate numeric(12,2),
+    gmeInd boolean,
+    gmePctRate numeric(10,4),
+    gmeFeeRate numeric(12,2),
+
+    -- Rate escalator
+    rateEscalatorInd boolean,
+    rateEscalatorDesc text,
+    rateEscalatorMaxRateIncPct numeric(10,4),
+    rateEscalatorRateChangeTimeline numeric(10,2),
+
+    -- Stop loss
+    stopLossTerm text,
+    stopLossFirstDollarInd boolean,
+    stopLossRangeNbrDays numeric(10,2),
+    stopLossFixedLossThreshold numeric(12,2),
+    stopLossMaximum numeric(12,2),
+    stopLossMaximumFrequency numeric(10,2),
+    stopLossDailyMaxRate numeric(12,2),
+    stopLossPctRateOnExcessCharges numeric(10,4),
+    stopLossExclusionCd text,
+    stopLossExclusionDesc text,
+
+    FOREIGN KEY (fieldExtractionId) REFERENCES documentFieldExtractions(id),
+
+    -- Ensure unique index per extraction
+    UNIQUE (fieldExtractionId, arrayIndex),
+
+    -- Ensure arrayIndex is non-negative
+    CHECK (arrayIndex >= 0)
+);
+
+-- Indexes for efficient array field queries
+CREATE INDEX idx_documentfieldextractionarrayfields_fieldextractionid
+    ON documentFieldExtractionArrayFields(fieldExtractionId);
+
+-- Composite index for efficient ordered retrieval
+CREATE INDEX idx_documentfieldextractionarrayfields_id_index
+    ON documentFieldExtractionArrayFields(fieldExtractionId, arrayIndex);
@@ -0,0 +1,6 @@
+-- Migration 010: Collector Views (DOWN)
+
+DROP VIEW IF EXISTS fullActiveCollectors;
+DROP VIEW IF EXISTS currentCollectorMinCleanVersions;
+DROP VIEW IF EXISTS collectorLatestVersions;
+DROP VIEW IF EXISTS collectorCurrentActiveVersions;
@@ -0,0 +1,43 @@
+-- Migration 010: Collector Views (Simplified)
+-- Views for collector configuration
+-- NOTE: Query-related views and text version views removed (no longer used)
+
+CREATE VIEW collectorCurrentActiveVersions AS
+SELECT
+    c.clientId,
+    COALESCE(v.versionId, 0)::int AS activeVersion
+FROM
+    clients c
+LEFT JOIN (
+    SELECT DISTINCT ON (clientId)
+        clientId,
+        versionId
+    FROM
+        collectorActiveVersions
+    ORDER BY clientId, id DESC
+) v ON c.clientId = v.clientId;
+
+CREATE VIEW collectorLatestVersions AS
+    SELECT
+        j.clientId,
+        COALESCE(MAX(v.id), 0)::int AS latestVersion
+    FROM clients AS j
+    LEFT JOIN collectorVersions AS v ON v.clientId = j.clientId
+    GROUP BY j.clientId;
+
+CREATE VIEW currentCollectorMinCleanVersions AS
+SELECT DISTINCT
+        av.clientId,
+        COALESCE(ctv.versionId, 0) AS minCleanVersion
+    FROM collectorCurrentActiveVersions AS av
+    LEFT JOIN collectorMinCleanVersions AS ctv ON av.clientId = ctv.clientId
+        AND isInVersion(av.activeVersion, ctv.addedVersion, ctv.removedVersion);
+
+-- Final fullActiveCollectors view (simplified without minTextVersion or fields)
+CREATE VIEW fullActiveCollectors AS
+    SELECT DISTINCT av.clientId,
+    ccv.minCleanVersion,
+    av.activeVersion, lv.latestVersion
+    FROM collectorCurrentActiveVersions AS av
+    JOIN collectorLatestVersions AS lv ON lv.clientId = av.clientId
+    JOIN currentCollectorMinCleanVersions AS ccv ON av.clientId = ccv.clientId;
@@ -0,0 +1,7 @@
+-- Migration 011: Document Views (DOWN)
+
+DROP FUNCTION IF EXISTS listValidDocumentResults;
+DROP VIEW IF EXISTS fullClients;
+DROP VIEW IF EXISTS currentClientCanSync;
+DROP VIEW IF EXISTS currentCleanEntries;
+DROP FUNCTION IF EXISTS listDocumentIDs;
@@ -0,0 +1,190 @@
+-- Migration 011: Document Views (Simplified)
+-- Views and functions for document queries
+-- NOTE: currentTextEntries and listValidDocumentResults removed (no longer used)
+
+CREATE OR REPLACE FUNCTION listDocumentIDs(
+    _clientId varchar(255),
+    _batchSize INTEGER,
+    _offset INTEGER
+)
+RETURNS TABLE (id uuid, totalCount BIGINT) AS $$
+DECLARE
+    totalCount BIGINT := 0;
+BEGIN
+    SELECT COUNT(*)::BIGINT INTO totalCount FROM documents WHERE clientId = _clientId;
+
+    RETURN QUERY
+        SELECT d.id, totalCount
+        FROM documents AS d
+        WHERE d.clientId = _clientId
+        ORDER BY d.id
+        LIMIT _batchSize
+        OFFSET _offset;
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE VIEW currentCleanEntries AS
+WITH RankedExtractions AS (
+    SELECT
+        dc.id,
+        dc.documentId,
+        dc.bucket,
+        dc.key,
+        dc.mimetype,
+        dc.fail,
+        dc.hash,
+        dce.version,
+        d.clientId,
+        ROW_NUMBER() OVER (PARTITION BY dc.documentId ORDER BY dc.id DESC, dce.id DESC) AS row_num
+    FROM documents d
+    JOIN currentCollectorMinCleanVersions ccv ON d.clientId = ccv.clientId
+    JOIN documentCleans dc ON dc.documentId = d.id
+    JOIN documentCleanEntries dce ON dc.id = dce.cleanId
+        AND dce.version >= ccv.minCleanVersion
+)
+SELECT
+    re.id,
+    re.documentId,
+    re.bucket,
+    re.key,
+    re.version,
+    re.hash,
+    re.mimetype,
+    re.fail,
+    re.clientId
+FROM RankedExtractions re
+WHERE row_num = 1;
+
+CREATE VIEW currentClientCanSync AS
+WITH RankedExtractions AS (
+    SELECT
+        ccs.clientId,
+        ccs.canSync,
+        ROW_NUMBER() OVER (PARTITION BY ccs.clientId ORDER BY ccs.syncId DESC) AS row_num
+    FROM clients c
+    JOIN clientCanSync ccs ON c.clientId = ccs.clientId
+)
+SELECT
+    c.clientId,
+    COALESCE(r.canSync, false) AS canSync
+FROM clients c
+LEFT JOIN RankedExtractions r ON r.clientId = c.clientId AND r.row_num = 1;
+
+CREATE VIEW fullClients AS
+SELECT c.clientId, c.name, cs.canSync
+    FROM clients AS c
+    JOIN currentClientCanSync AS cs ON cs.clientId = c.clientId;
+
+-- NOTE: This function exists for schema compatibility with original migrations.
+-- It references tables/functions that no longer exist (results, resultDependencies,
+-- collectorQueryDependencyTreeByClient, currentTextEntries) and will fail if called.
+-- It was created in original migration 102 but never explicitly dropped.
+CREATE OR REPLACE FUNCTION listValidDocumentResults(doc_id uuid)
+RETURNS TABLE (documentId uuid, queryId uuid, resultId uuid, value text) AS $$
+DECLARE
+ count_before INT;
+ count_after INT;
+ iterations INT := 0;
+BEGIN
+    CREATE TEMPORARY TABLE allResults AS
+    WITH
+        docs AS (
+            SELECT id as documentId, clientId
+            FROM documents
+            WHERE id = doc_id
+    ),
+    tree as (
+        select q.queryId, q.requiredIds, q.clientId, q.queryVersion
+            from docs,
+                LATERAL collectorQueryDependencyTreeByClient(docs.clientId) AS q
+    ),
+    query_dependency_tree AS (
+        WITH RECURSIVE query_deps AS (
+            SELECT
+            q.queryId,
+            unnest(q.requiredIds) as requiredId
+            FROM tree AS q
+            WHERE array_length(q.requiredIds, 1) > 0
+
+            UNION ALL
+
+            SELECT
+            qd.queryId,
+            unnest(q.requiredIds) as requiredId
+            FROM query_deps qd
+            JOIN tree q ON qd.requiredId = q.queryId
+            WHERE array_length(q.requiredIds, 1) > 0
+        )
+        SELECT
+        query_deps.queryId,
+        array_agg(DISTINCT requiredId) AS all_required_ids
+        FROM query_deps
+        GROUP BY query_deps.queryId
+    ),
+    docResults AS (
+    SELECT
+        d.documentId,
+        q.queryId,
+        r.id AS resultId,
+        r.value,
+        q.requiredIds as directRequiredIds,
+        COALESCE(qdt.all_required_ids, ARRAY[]::uuid[]) AS allRequiredIds
+        FROM docs AS d
+        JOIN tree AS q ON q.clientId = d.clientId
+        LEFT JOIN query_dependency_tree qdt ON q.queryId = qdt.queryId
+        LEFT JOIN currentTextEntries AS tte ON tte.documentId = d.documentId
+        LEFT JOIN results AS r
+            ON q.queryId = r.queryId
+            AND r.queryVersion = q.queryVersion
+            AND tte.id = r.textEntryId
+        WHERE r.value is not null
+    )
+    SELECT * FROM docResults dr;
+
+    CREATE TEMPORARY TABLE finalResults AS
+    SELECT * from allResults where array_length(directRequiredIds, 1) is null;
+
+    SELECT COUNT(*) INTO count_before FROM finalResults;
+    RAISE NOTICE 'Starting with % rows', count_before;
+
+    LOOP
+        iterations := iterations + 1;
+
+        INSERT INTO finalResults
+            SELECT dr.*
+            FROM allResults dr
+            WHERE
+                dr.directRequiredIds IS NOT NULL
+                AND NOT EXISTS (
+                    SELECT 1
+                    FROM unnest(dr.directRequiredIds) AS req_id(queryId)
+                    WHERE NOT EXISTS (
+                        SELECT 1
+                        FROM allResults req_dr
+                        WHERE req_dr.queryId = req_id.queryId
+                        AND req_dr.resultId IN (SELECT fr.resultId FROM finalResults fr)
+                        AND exists (
+                            select 1 from resultDependencies rrd
+                            where dr.resultId = rrd.resultId
+                            and req_dr.resultId = rrd.requiredResultId
+                        )
+                    )
+                )
+                and dr.resultId NOT IN (SELECT fr.resultId FROM finalResults fr);
+
+        GET DIAGNOSTICS count_after = ROW_COUNT;
+        RAISE NOTICE 'Iteration %: Additional Entries %',
+        iterations, count_after;
+
+        IF count_after = 0 THEN
+            EXIT;
+        END IF;
+    END LOOP;
+
+    RETURN QUERY SELECT tr.documentId, tr.queryId, tr.resultId, tr.value
+        FROM finalResults tr;
+
+    DROP TABLE finalResults;
+    DROP TABLE allResults;
+END;
+$$ LANGUAGE plpgsql;
@@ -0,0 +1,4 @@
+-- Migration 012: Field Extraction Views (DOWN)
+
+DROP VIEW IF EXISTS currentFieldExtractionsWithArrayCount;
+DROP VIEW IF EXISTS currentFieldExtractions;
@@ -0,0 +1,47 @@
+-- Migration 012: Field Extraction Views
+-- Views for querying current field extractions
+
+-- Create view for current (newest) field extraction per document
+CREATE VIEW currentFieldExtractions AS
+SELECT DISTINCT ON (dfe.documentId)
+    dfe.id,
+    dfe.documentId,
+    dfe.fileName,
+    dfe.contractTitle,
+    dfe.aareteDerivedAmendmentNum,
+    dfe.clientName,
+    dfe.payerName,
+    dfe.payerState,
+    dfe.providerState,
+    dfe.filenameTin,
+    dfe.provGroupTin,
+    dfe.provGroupNpi,
+    dfe.provGroupNameFull,
+    dfe.provOtherTin,
+    dfe.provOtherNpi,
+    dfe.provOtherNameFull,
+    dfe.aareteDerivedEffectiveDt,
+    dfe.aareteDerivedTerminationDt,
+    dfe.autoRenewalInd,
+    dfe.autoRenewalTerm,
+    dfev.version,
+    dfev.createdBy,
+    dfev.createdAt
+FROM documentFieldExtractions dfe
+JOIN documentFieldExtractionVersions dfev
+    ON dfev.fieldExtractionId = dfe.id
+ORDER BY dfe.documentId, dfev.id DESC;
+
+-- Create extended view with array field count
+CREATE VIEW currentFieldExtractionsWithArrayCount AS
+SELECT
+    cfe.*,
+    COALESCE(array_counts.arraySize, 0) AS arraySize
+FROM currentFieldExtractions cfe
+LEFT JOIN (
+    SELECT
+        fieldExtractionId,
+        COUNT(*) AS arraySize
+    FROM documentFieldExtractionArrayFields
+    GROUP BY fieldExtractionId
+) array_counts ON array_counts.fieldExtractionId = cfe.id;
@@ -0,0 +1 @@
+DROP EXTENSION "pgcrypto";
@@ -0,0 +1,37 @@
+CREATE EXTENSION IF NOT EXISTS "pgcrypto";
+
+create or replace function uuid_generate_v7()
+returns uuid
+as $$
+select encode(
+    set_bit(
+      set_bit(
+        overlay(uuid_send(gen_random_uuid())
+                placing substring(int8send(floor(extract(epoch from clock_timestamp()) * 1000)::bigint) from 3)
+                from 1 for 6
+        ),
+        52, 1
+      ),
+      53, 1
+    ),
+    'hex')::uuid;
+$$
+language SQL
+volatile;
+
+CREATE FUNCTION isInVersion(
+    _version int,
+    _addedVersion int,
+    _removedVersion int
+)
+RETURNS boolean as $$
+BEGIN
+    RETURN _version is null OR (
+        _version >= _addedVersion AND
+        (_removedVersion is null OR _version < _removedVersion)
+    );
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE DOMAIN unsignedsmallint AS SMALLINT
+  CHECK (VALUE >= 0);
@@ -0,0 +1,104 @@
+-- Rollback migration to restore text extraction functionality
+-- NOTE: This recreates text extraction tables and views
+
+-- First drop the simplified fullActiveCollectors view
+DROP VIEW IF EXISTS fullActiveCollectors;
+
+-- Recreate documentTextExtractions table
+CREATE TABLE documentTextExtractions (
+    id uuid primary key DEFAULT uuid_generate_v7(),
+    cleanId uuid not null,
+    bucket text not null,
+    key text not null,
+    hash text not null,
+    createdAt timestamp not null,
+    part unsignedsmallint not null,
+    foreign key (cleanId) references documentCleans(id)
+);
+
+-- Recreate documentTextExtractionEntries table
+CREATE TABLE documentTextExtractionEntries (
+    id uuid primary key DEFAULT uuid_generate_v7(),
+    textId uuid not null,
+    version bigint not null,
+    foreign key (textId) references documentTextExtractions(id)
+);
+
+-- Recreate collectorMinTextVersions table
+CREATE TABLE collectorMinTextVersions (
+    id uuid primary key DEFAULT uuid_generate_v7(),
+    clientId varchar(255) not null,
+    versionId bigint not null,
+    addedVersion int not null,
+    removedVersion int,
+    foreign key (clientId, addedVersion) references collectorVersions(clientId, id),
+    foreign key (clientId, removedVersion) references collectorVersions(clientId, id),
+    foreign key (clientId) references clients(clientId)
+);
+
+-- Recreate removeMinTextVersion function
+CREATE OR REPLACE FUNCTION removeMinTextVersion()
+RETURNS TRIGGER AS $$
+BEGIN
+    UPDATE collectorMinTextVersions
+        SET removedVersion = NEW.addedVersion
+        WHERE clientId = NEW.clientId and removedVersion is null;
+
+    RETURN NEW;
+END;
+$$ LANGUAGE plpgsql;
+
+-- Recreate trigger for collectorMinTextVersions
+CREATE TRIGGER removeMinTextVersionTrigger
+BEFORE INSERT ON collectorMinTextVersions
+FOR EACH ROW
+EXECUTE FUNCTION removeMinTextVersion();
+
+-- Recreate currentCollectorMinTextVersions view
+CREATE VIEW currentCollectorMinTextVersions as
+SELECT DISTINCT
+        av.clientId,
+        coalesce(ctv.versionId, 0) as minTextVersion
+    FROM collectorCurrentActiveVersions as av
+    LEFT JOIN collectorMinTextVersions AS ctv ON av.clientId = ctv.clientId
+        and isInVersion(av.activeVersion, ctv.addedVersion, ctv.removedVersion);
+
+-- Recreate fullActiveCollectors view with minTextVersion
+CREATE VIEW fullActiveCollectors AS
+    SELECT DISTINCT av.clientId,
+    ccv.minCleanVersion, ctv.minTextVersion,
+    av.activeVersion, lv.latestVersion
+    FROM collectorCurrentActiveVersions as av
+    JOIN collectorLatestVersions as lv on lv.clientId = av.clientId
+    JOIN currentCollectorMinCleanVersions AS ccv ON av.clientId = ccv.clientId
+    JOIN currentCollectorMinTextVersions AS ctv ON av.clientId = ctv.clientId;
+
+-- Recreate currentTextEntries view
+CREATE VIEW currentTextEntries as
+WITH RankedExtractions AS (
+    SELECT
+        extract.id,
+        cc.documentId,
+        extract.bucket,
+        extract.key,
+        extract.hash,
+        extract.cleanId,
+        dtee.version as extractionVersion,
+        ROW_NUMBER() OVER (PARTITION BY cc.documentId ORDER BY dtee.id DESC) as row_num
+    FROM documents d
+    JOIN currentCleanEntries cc on cc.documentId = d.id
+    JOIN currentCollectorMinTextVersions ctv ON ctv.clientId = d.clientId
+    JOIN documentTextExtractions extract ON extract.cleanId = cc.id
+    JOIN documentTextExtractionEntries dtee on dtee.textId = extract.id
+        AND dtee.version >= ctv.minTextVersion
+)
+SELECT
+    id,
+    documentId,
+    bucket,
+    key,
+    hash,
+    cleanId,
+    extractionVersion
+FROM RankedExtractions
+WHERE row_num = 1;
@@ -0,0 +1,32 @@
+-- Migration to remove text extraction functionality from the database
+-- This drops all text extraction related tables, views, and functions
+-- NOTE: Field extractions are separate and NOT affected by this migration
+
+-- First drop the view that depends on text extraction tables
+DROP VIEW IF EXISTS currentTextEntries;
+
+-- Drop fullActiveCollectors view that depends on currentCollectorMinTextVersions
+DROP VIEW IF EXISTS fullActiveCollectors;
+
+-- Drop the currentCollectorMinTextVersions view
+DROP VIEW IF EXISTS currentCollectorMinTextVersions;
+
+-- Drop the trigger and function for collectorMinTextVersions
+DROP TRIGGER IF EXISTS removeMinTextVersionTrigger ON collectorMinTextVersions;
+DROP FUNCTION IF EXISTS removeMinTextVersion;
+
+-- Drop the collectorMinTextVersions table
+DROP TABLE IF EXISTS collectorMinTextVersions;
+
+-- Drop the text extraction tables (child first for FK constraints)
+DROP TABLE IF EXISTS documentTextExtractionEntries;
+DROP TABLE IF EXISTS documentTextExtractions;
+
+-- Recreate fullActiveCollectors view WITHOUT minTextVersion
+CREATE VIEW fullActiveCollectors AS
+    SELECT DISTINCT av.clientId,
+    ccv.minCleanVersion,
+    av.activeVersion, lv.latestVersion
+    FROM collectorCurrentActiveVersions as av
+    JOIN collectorLatestVersions as lv on lv.clientId = av.clientId
+    JOIN currentCollectorMinCleanVersions AS ccv ON av.clientId = ccv.clientId;
@@ -0,0 +1,7 @@
+-- Migration 001: Extensions and Core Functions (DOWN)
+-- Drops in reverse order of creation
+
+DROP DOMAIN IF EXISTS unsignedsmallint;
+DROP FUNCTION IF EXISTS isInVersion;
+DROP FUNCTION IF EXISTS uuid_generate_v7;
+DROP EXTENSION IF EXISTS "pgcrypto";
@@ -0,0 +1,43 @@
+-- Migration 001: Extensions and Core Functions
+-- Creates PostgreSQL extensions and utility functions used throughout the schema
+
+CREATE EXTENSION IF NOT EXISTS "pgcrypto";
+
+-- UUID v7 generator function (time-ordered UUIDs)
+CREATE OR REPLACE FUNCTION uuid_generate_v7()
+RETURNS uuid
+AS $$
+SELECT encode(
+    set_bit(
+      set_bit(
+        overlay(uuid_send(gen_random_uuid())
+                placing substring(int8send(floor(extract(epoch from clock_timestamp()) * 1000)::bigint) from 3)
+                from 1 for 6
+        ),
+        52, 1
+      ),
+      53, 1
+    ),
+    'hex')::uuid;
+$$
+LANGUAGE SQL
+VOLATILE;
+
+-- Version checking function for temporal data patterns
+CREATE FUNCTION isInVersion(
+    _version int,
+    _addedVersion int,
+    _removedVersion int
+)
+RETURNS boolean AS $$
+BEGIN
+    RETURN _version IS NULL OR (
+        _version >= _addedVersion AND
+        (_removedVersion IS NULL OR _version < _removedVersion)
+    );
+END;
+$$ LANGUAGE plpgsql;
+
+-- Domain for unsigned small integers
+CREATE DOMAIN unsignedsmallint AS SMALLINT
+  CHECK (VALUE >= 0);
@@ -0,0 +1,4 @@
+-- Migration 002: Clients (DOWN)
+
+DROP TABLE IF EXISTS clientCanSync;
+DROP TABLE IF EXISTS clients;
@@ -0,0 +1,15 @@
+-- Migration 002: Clients
+-- Core client tables for customer configuration
+
+CREATE TABLE clients (
+    clientId varchar(255) PRIMARY KEY,
+    name TEXT NOT NULL,
+    UNIQUE (name)
+);
+
+CREATE TABLE clientCanSync (
+    syncId uuid PRIMARY KEY DEFAULT uuid_generate_v7(),
+    clientId varchar(255) NOT NULL,
+    canSync boolean NOT NULL,
+    FOREIGN KEY (clientId) REFERENCES clients(clientId)
+);
@@ -0,0 +1,9 @@
+-- Migration 003: Collectors (DOWN)
+
+DROP TRIGGER IF EXISTS removeMinCleanVersionTrigger ON collectorMinCleanVersions;
+DROP FUNCTION IF EXISTS removeMinCleanVersion;
+DROP TABLE IF EXISTS collectorMinCleanVersions;
+DROP TABLE IF EXISTS collectorActiveVersions;
+DROP TRIGGER IF EXISTS setCollectorVersionNumberTrigger ON collectorVersions;
+DROP FUNCTION IF EXISTS setCollectorVersionNumber;
+DROP TABLE IF EXISTS collectorVersions;
@@ -0,0 +1,62 @@
+-- Migration 003: Collectors (Simplified)
+-- Collector versioning and configuration
+-- NOTE: collectorMinTextVersions and collectorQueries removed (no longer used)
+
+CREATE TABLE collectorVersions (
+    clientId varchar(255) NOT NULL,
+    id int NOT NULL,
+    addedAt timestamp NOT NULL DEFAULT current_timestamp,
+    FOREIGN KEY (clientId) REFERENCES clients(clientId),
+    PRIMARY KEY (id, clientId)
+);
+
+CREATE OR REPLACE FUNCTION setCollectorVersionNumber()
+RETURNS TRIGGER AS $$
+BEGIN
+    SELECT COALESCE(MAX(id), 0) + 1
+    INTO NEW.id
+    FROM collectorVersions
+    WHERE clientId = NEW.clientId;
+
+    RETURN NEW;
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE TRIGGER setCollectorVersionNumberTrigger
+BEFORE INSERT ON collectorVersions
+FOR EACH ROW
+EXECUTE FUNCTION setCollectorVersionNumber();
+
+CREATE TABLE collectorActiveVersions (
+    id uuid PRIMARY KEY DEFAULT uuid_generate_v7(),
+    clientId varchar(255) NOT NULL,
+    versionId int NOT NULL,
+    FOREIGN KEY (clientId, versionId) REFERENCES collectorVersions(clientId, id)
+);
+
+CREATE TABLE collectorMinCleanVersions (
+    id uuid PRIMARY KEY DEFAULT uuid_generate_v7(),
+    clientId varchar(255) NOT NULL,
+    versionId bigint NOT NULL,
+    addedVersion int NOT NULL,
+    removedVersion int,
+    FOREIGN KEY (clientId, addedVersion) REFERENCES collectorVersions(clientId, id),
+    FOREIGN KEY (clientId, removedVersion) REFERENCES collectorVersions(clientId, id),
+    FOREIGN KEY (clientId) REFERENCES clients(clientId)
+);
+
+CREATE OR REPLACE FUNCTION removeMinCleanVersion()
+RETURNS TRIGGER AS $$
+BEGIN
+    UPDATE collectorMinCleanVersions
+        SET removedVersion = NEW.addedVersion
+        WHERE clientId = NEW.clientId AND removedVersion IS NULL;
+
+    RETURN NEW;
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE TRIGGER removeMinCleanVersionTrigger
+BEFORE INSERT ON collectorMinCleanVersions
+FOR EACH ROW
+EXECUTE FUNCTION removeMinCleanVersion();
@@ -0,0 +1,6 @@
+-- Migration 004: Batch Uploads (DOWN)
+
+DROP INDEX IF EXISTS idx_batch_uploads_status;
+DROP INDEX IF EXISTS idx_batch_uploads_client_id;
+DROP TABLE IF EXISTS batch_uploads;
+DROP TYPE IF EXISTS batch_status;
@@ -0,0 +1,52 @@
+-- Migration 004: Batch Uploads
+-- Batch upload tracking for ZIP file uploads
+-- Merges original migrations 103 + 104
+
+-- Create batch status ENUM type
+CREATE TYPE batch_status AS ENUM (
+    'processing',       -- Currently processing
+    'completed',        -- All documents processed (may have individual failures)
+    'failed',           -- Batch-level failure (ZIP corrupt, etc.)
+    'cancelled'         -- Cancelled by user
+);
+
+-- Create batch uploads table with all columns
+CREATE TABLE batch_uploads (
+    id uuid PRIMARY KEY DEFAULT uuid_generate_v7(),
+    client_id varchar(255) NOT NULL,
+    original_filename text NOT NULL,
+
+    -- Essential batch metrics
+    total_documents integer NOT NULL,
+    processed_documents integer NOT NULL DEFAULT 0,
+    failed_documents integer NOT NULL DEFAULT 0,
+    invalid_type_documents integer NOT NULL DEFAULT 0,
+
+    -- Status and progress
+    status batch_status NOT NULL DEFAULT 'processing',
+    progress_percent integer NOT NULL DEFAULT 0,
+
+    -- Failed document filenames (JSON array)
+    failed_filenames jsonb DEFAULT '[]'::jsonb,
+
+    -- Timestamps
+    created_at timestamp NOT NULL DEFAULT NOW(),
+    completed_at timestamp,
+
+    -- Storage metadata (from migration 104)
+    archive_bucket text,
+    archive_key text,
+    file_size_bytes bigint,
+
+    FOREIGN KEY (client_id) REFERENCES clients(clientId),
+
+    -- Constraint: storage metadata consistency
+    CONSTRAINT batch_storage_consistent CHECK (
+        (archive_bucket IS NULL AND archive_key IS NULL AND file_size_bytes IS NULL) OR
+        (archive_bucket IS NOT NULL AND archive_key IS NOT NULL)
+    )
+);
+
+-- Create indexes for efficient queries
+CREATE INDEX idx_batch_uploads_client_id ON batch_uploads(client_id);
+CREATE INDEX idx_batch_uploads_status ON batch_uploads(status);
@@ -0,0 +1,6 @@
+-- Migration 005: Folders (DOWN)
+
+DROP INDEX IF EXISTS idx_folders_path;
+DROP INDEX IF EXISTS idx_folders_clientid;
+DROP INDEX IF EXISTS idx_folders_parentid;
+DROP TABLE IF EXISTS folders;
@@ -0,0 +1,26 @@
+-- Migration 005: Folders
+-- Virtual folder hierarchy for document organization
+
+CREATE TABLE folders (
+    id uuid PRIMARY KEY DEFAULT uuid_generate_v7(),
+    path text NOT NULL,
+    parentId uuid,
+    clientId varchar(255) NOT NULL,
+    createdAt timestamp NOT NULL DEFAULT NOW(),
+    createdBy varchar(255) NOT NULL,
+    FOREIGN KEY (parentId) REFERENCES folders(id),
+    FOREIGN KEY (clientId) REFERENCES clients(clientId),
+    UNIQUE(clientId, path),
+    CHECK (id != parentId)
+);
+
+-- Indexes for efficient folder queries
+CREATE INDEX idx_folders_parentid ON folders(parentId);
+CREATE INDEX idx_folders_clientid ON folders(clientId);
+CREATE INDEX idx_folders_path ON folders(path);
+
+-- Add comment to clarify that folders are virtual (database-only, no S3 relationship)
+COMMENT ON TABLE folders IS
+    'Virtual folder hierarchy for document organization. Database-only - has no direct relationship to S3 storage locations.';
+COMMENT ON COLUMN folders.path IS
+    'Virtual folder path (e.g., "/contracts/2025"). Can be renamed without affecting S3 storage.';
@@ -0,0 +1,31 @@
+-- Migration 006: Documents (DOWN)
+
+-- Drop document clean entries first (depends on documentCleans)
+DROP TABLE IF EXISTS documentCleanEntries;
+
+-- Drop document cleans
+DROP TABLE IF EXISTS documentCleans;
+
+-- Drop document entries
+DROP TABLE IF EXISTS documentEntries;
+
+-- Drop documentUploads indexes
+DROP INDEX IF EXISTS idx_documentuploads_folder_id;
+DROP INDEX IF EXISTS idx_documentuploads_clientid;
+DROP INDEX IF EXISTS idx_documentuploads_batch_id;
+DROP INDEX IF EXISTS idx_documentuploads_filename;
+
+-- Drop documentUploads table
+DROP TABLE IF EXISTS documentUploads;
+
+-- Drop documents indexes
+DROP INDEX IF EXISTS idx_documents_folderid;
+DROP INDEX IF EXISTS idx_documents_filename;
+DROP INDEX IF EXISTS idx_documents_batch_id;
+
+-- Drop documents table
+DROP TABLE IF EXISTS documents;
+
+-- Drop enums
+DROP TYPE IF EXISTS cleanMimeType;
+DROP TYPE IF EXISTS cleanFailType;
@@ -0,0 +1,125 @@
+-- Migration 006: Documents (Consolidated)
+-- Document storage and processing tables
+-- Merges: 005_documents + 103 + 105 + 106 + 107 + 108 + 111 + 116 + 118
+-- NOTE: documentTextExtractions and documentTextExtractionEntries removed (no longer used)
+
+-- Enum for document clean failure types
+CREATE TYPE cleanFailType AS ENUM (
+    'invalid_mimetype',
+    'invalid_read',
+    'invalid_read_pages',
+    'zero_page_count',
+    'large_file',
+    'small_dimensions',
+    'large_dimensions',
+    'small_dpi',
+    'large_dpi'
+);
+
+-- Enum for clean document mime types
+CREATE TYPE cleanMimeType AS ENUM ('application/pdf');
+
+-- Main documents table with all columns
+CREATE TABLE documents (
+    id uuid PRIMARY KEY DEFAULT uuid_generate_v7(),
+    clientId varchar(255) NOT NULL,
+    hash text NOT NULL,
+
+    -- Batch reference (from migration 103)
+    batch_id uuid,
+
+    -- Filename (from migration 105)
+    filename TEXT,
+
+    -- Folder organization (from migration 111)
+    folderId uuid,
+    originalPath text,
+
+    -- File size (from migration 118)
+    file_size_bytes BIGINT,
+
+    FOREIGN KEY (clientId) REFERENCES clients(clientId),
+    FOREIGN KEY (batch_id) REFERENCES batch_uploads(id),
+    FOREIGN KEY (folderId) REFERENCES folders(id),
+    UNIQUE(clientId, hash)
+);
+
+-- Indexes for documents table
+CREATE INDEX idx_documents_batch_id ON documents(batch_id);
+CREATE INDEX idx_documents_filename ON documents(filename);
+CREATE INDEX idx_documents_folderid ON documents(folderId);
+
+-- Comments for immutable fields (from migration 111)
+COMMENT ON COLUMN documents.originalPath IS
+    'Original path provided during upload. IMMUTABLE after creation - never modify this value.';
+
+-- Document uploads table (staging area for incoming documents)
+CREATE TABLE documentUploads (
+    id uuid PRIMARY KEY DEFAULT uuid_generate_v7(),
+    bucket text NOT NULL,
+    key text NOT NULL,
+    clientId varchar(255) NOT NULL,
+    part unsignedsmallint NOT NULL,
+    createdAt timestamp NOT NULL,
+
+    -- Filename (from migration 106)
+    filename TEXT,
+
+    -- Batch reference (from migration 107)
+    batch_id uuid,
+
+    -- Folder reference (from migration 116)
+    folder_id uuid,
+
+    FOREIGN KEY (clientId) REFERENCES clients(clientId),
+    FOREIGN KEY (folder_id) REFERENCES folders(id)
+);
+
+-- Indexes for documentUploads
+CREATE INDEX idx_documentuploads_filename ON documentUploads(filename);
+CREATE INDEX idx_documentuploads_batch_id ON documentUploads(batch_id);
+CREATE INDEX idx_documentuploads_clientid ON documentUploads(clientId);
+CREATE INDEX idx_documentuploads_folder_id ON documentUploads(folder_id);
+
+-- Document entries (S3 storage records)
+CREATE TABLE documentEntries (
+    id uuid PRIMARY KEY DEFAULT uuid_generate_v7(),
+    documentId uuid NOT NULL,
+    bucket text NOT NULL,
+    key text NOT NULL,
+    FOREIGN KEY (documentId) REFERENCES documents(id)
+);
+
+-- Comments for immutable fields (from migration 111)
+COMMENT ON COLUMN documentEntries.key IS
+    'S3 object key. IMMUTABLE after creation - never modify this value.';
+COMMENT ON COLUMN documentEntries.bucket IS
+    'S3 bucket name. IMMUTABLE after creation - never modify this value.';
+
+-- Document cleans (processed/cleaned documents)
+CREATE TABLE documentCleans (
+    id uuid PRIMARY KEY DEFAULT uuid_generate_v7(),
+    documentId uuid NOT NULL,
+    bucket text,
+    key text,
+    hash text,
+    mimetype cleanMimeType,
+    fail cleanFailType,
+    FOREIGN KEY (documentId) REFERENCES documents(id),
+    CONSTRAINT bucket_and_key_together CHECK (
+        (bucket IS NULL) = (key IS NULL) AND
+        (bucket IS NULL) = (mimetype IS NULL) AND
+        (bucket IS NULL) = (hash IS NULL)
+    ),
+    CONSTRAINT location_xor_fail CHECK (
+        (bucket IS NOT NULL) != (fail IS NOT NULL)
+    )
+);
+
+-- Document clean entries (version tracking for cleans)
+CREATE TABLE documentCleanEntries (
+    id uuid PRIMARY KEY DEFAULT uuid_generate_v7(),
+    cleanId uuid NOT NULL,
+    version bigint NOT NULL,
+    FOREIGN KEY (cleanId) REFERENCES documentCleans(id)
+);
@@ -0,0 +1,8 @@
+-- Migration 007: Labels (DOWN)
+
+DROP INDEX IF EXISTS idx_documentlabels_document_label_time;
+DROP INDEX IF EXISTS idx_documentlabels_appliedat;
+DROP INDEX IF EXISTS idx_documentlabels_label;
+DROP INDEX IF EXISTS idx_documentlabels_documentid;
+DROP TABLE IF EXISTS documentLabels;
+DROP TABLE IF EXISTS labels;
@@ -0,0 +1,34 @@
+-- Migration 007: Labels
+-- Document labeling system for tracking processing status
+
+-- Create labels lookup table
+CREATE TABLE labels (
+    label text PRIMARY KEY,
+    description text NOT NULL
+);
+
+-- Seed initial label values
+INSERT INTO labels (label, description) VALUES
+    ('Ingested', 'Document has been ingested into the system'),
+    ('OCR_Processed', 'OCR text extraction completed'),
+    ('GenAI_Processed', 'GenAI processing completed'),
+    ('Doczy_AI_Completed', 'Doczy.AI processing completed'),
+    ('Dashboard_Ready', 'Document ready for dashboard display');
+
+-- Create documentLabels junction table
+CREATE TABLE documentLabels (
+    id uuid PRIMARY KEY DEFAULT uuid_generate_v7(),
+    documentId uuid NOT NULL,
+    label text NOT NULL,
+    appliedAt timestamp NOT NULL DEFAULT NOW(),
+    appliedBy varchar(255) NOT NULL,
+    FOREIGN KEY (documentId) REFERENCES documents(id) ON DELETE CASCADE,
+    FOREIGN KEY (label) REFERENCES labels(label)
+);
+
+-- Indexes for efficient label queries
+CREATE INDEX idx_documentlabels_documentid ON documentLabels(documentId);
+CREATE INDEX idx_documentlabels_label ON documentLabels(label);
+CREATE INDEX idx_documentlabels_appliedat ON documentLabels(appliedAt);
+-- Composite index for finding most recent label application
+CREATE INDEX idx_documentlabels_document_label_time ON documentLabels(documentId, label, appliedAt DESC);
@@ -0,0 +1,19 @@
+-- Migration 008: Field Extractions (DOWN)
+
+-- Drop array fields table and indexes
+DROP INDEX IF EXISTS idx_documentfieldextractionarrayfields_id_index;
+DROP INDEX IF EXISTS idx_documentfieldextractionarrayfields_fieldextractionid;
+DROP TABLE IF EXISTS documentFieldExtractionArrayFields;
+
+-- Drop versions table and indexes
+DROP INDEX IF EXISTS idx_unique_version_per_document;
+DROP INDEX IF EXISTS idx_documentfieldextractionversions_documentid;
+DROP INDEX IF EXISTS idx_documentfieldextractionversions_version;
+DROP INDEX IF EXISTS idx_documentfieldextractionversions_fieldextractionid;
+DROP TABLE IF EXISTS documentFieldExtractionVersions;
+
+-- Drop main field extractions table and indexes
+DROP INDEX IF EXISTS idx_documentfieldextractions_filename;
+DROP INDEX IF EXISTS idx_documentfieldextractions_createdby;
+DROP INDEX IF EXISTS idx_documentfieldextractions_documentid;
+DROP TABLE IF EXISTS documentFieldExtractions;
--- a/Show More
+++ b/Show More