Merged in feature/textExtractionsPart1 (pull request #192)

all schema and rest apis for text extraction support * in progress * stage 8 complete * phase 9 completed * phase 9 complete * ongoing - s3 path fix * working * optimize ci build * e2e tests * missing test
2025-11-26 19:23:42 +00:00
parent 2b43799f56
commit c45e1dd427
67 changed files with 16120 additions and 817 deletions
@@ -0,0 +1,533 @@
+package queryapi_test
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	queryapi "queryorchestration/api/queryAPI"
+	"queryorchestration/internal/database/repository"
+	"queryorchestration/internal/folder"
+	"queryorchestration/internal/serviceconfig"
+	"queryorchestration/internal/test"
+
+	"github.com/google/uuid"
+	"github.com/labstack/echo/v4"
+	openapi_types "github.com/oapi-codegen/runtime/types"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+type FoldersTestConfig struct {
+	serviceconfig.BaseConfig
+}
+
+func (c *FoldersTestConfig) GetBackgroundRunner() any { return nil }
+
+func setupFoldersTestController(t *testing.T) (*queryapi.Controllers, *FoldersTestConfig, string) {
+	t.Helper()
+	cfg := &FoldersTestConfig{}
+	test.CreateDB(t, cfg)
+
+	// Create services
+	folderSvc := folder.New(cfg)
+
+	services := &queryapi.Services{
+		Folder: folderSvc,
+	}
+
+	// Create test client with unique ID
+	ctx := t.Context()
+	uniqueID := uuid.New().String()[:8]
+	clientID := fmt.Sprintf("test-client-%s", uniqueID)
+	err := cfg.GetDBQueries().CreateClient(ctx, &repository.CreateClientParams{
+		Clientid: clientID,
+		Name:     fmt.Sprintf("Test Client %s", uniqueID),
+	})
+	require.NoError(t, err)
+
+	ctrl := queryapi.NewControllers(services, nil)
+
+	return ctrl, cfg, clientID
+}
+
+func TestCreateFolder_Success(t *testing.T) {
+	ctrl, _, clientID := setupFoldersTestController(t)
+
+	reqBody := queryapi.FolderCreate{
+		Path:      "/documents/contracts",
+		ClientId:  queryapi.ClientID(clientID),
+		CreatedBy: openapi_types.Email("user@example.com"),
+	}
+
+	body, err := json.Marshal(reqBody)
+	require.NoError(t, err)
+
+	e := echo.New()
+	req := httptest.NewRequest(http.MethodPost, "/folders", bytes.NewReader(body))
+	req.Header.Set(echo.HeaderContentType, echo.MIMEApplicationJSON)
+	rec := httptest.NewRecorder()
+	c := e.NewContext(req, rec)
+
+	err = ctrl.CreateFolder(c)
+	require.NoError(t, err)
+	assert.Equal(t, http.StatusCreated, rec.Code)
+
+	// Parse response
+	var resp queryapi.Folder
+	err = json.Unmarshal(rec.Body.Bytes(), &resp)
+	require.NoError(t, err)
+
+	assert.Equal(t, "/documents/contracts", resp.Path)
+	assert.Equal(t, clientID, string(resp.ClientId))
+	assert.Equal(t, "user@example.com", string(resp.CreatedBy))
+	assert.NotEqual(t, uuid.Nil, uuid.UUID(resp.Id))
+}
+
+func TestCreateFolder_WithParent(t *testing.T) {
+	ctrl, cfg, clientID := setupFoldersTestController(t)
+	ctx := t.Context()
+
+	// Create parent folder first
+	parentFolder, err := cfg.GetDBQueries().CreateFolder(ctx, &repository.CreateFolderParams{
+		Path:      "/documents",
+		Clientid:  clientID,
+		Createdby: "admin@example.com",
+	})
+	require.NoError(t, err)
+
+	reqBody := queryapi.FolderCreate{
+		Path:      "/documents/contracts",
+		ClientId:  queryapi.ClientID(clientID),
+		CreatedBy: openapi_types.Email("user@example.com"),
+		ParentId:  (*openapi_types.UUID)(&parentFolder.ID),
+	}
+
+	body, err := json.Marshal(reqBody)
+	require.NoError(t, err)
+
+	e := echo.New()
+	req := httptest.NewRequest(http.MethodPost, "/folders", bytes.NewReader(body))
+	req.Header.Set(echo.HeaderContentType, echo.MIMEApplicationJSON)
+	rec := httptest.NewRecorder()
+	c := e.NewContext(req, rec)
+
+	err = ctrl.CreateFolder(c)
+	require.NoError(t, err)
+	assert.Equal(t, http.StatusCreated, rec.Code)
+
+	var resp queryapi.Folder
+	err = json.Unmarshal(rec.Body.Bytes(), &resp)
+	require.NoError(t, err)
+
+	// Verify parent ID is set
+	assert.True(t, resp.ParentId.IsSpecified())
+	parentID, err := resp.ParentId.Get()
+	require.NoError(t, err)
+	assert.Equal(t, parentFolder.ID, uuid.UUID(parentID))
+}
+
+func TestCreateFolder_MissingPath(t *testing.T) {
+	ctrl, _, clientID := setupFoldersTestController(t)
+
+	reqBody := queryapi.FolderCreate{
+		Path:      "",
+		ClientId:  queryapi.ClientID(clientID),
+		CreatedBy: openapi_types.Email("user@example.com"),
+	}
+
+	body, err := json.Marshal(reqBody)
+	require.NoError(t, err)
+
+	e := echo.New()
+	req := httptest.NewRequest(http.MethodPost, "/folders", bytes.NewReader(body))
+	req.Header.Set(echo.HeaderContentType, echo.MIMEApplicationJSON)
+	rec := httptest.NewRecorder()
+	c := e.NewContext(req, rec)
+
+	err = ctrl.CreateFolder(c)
+	require.Error(t, err)
+	httpErr, ok := err.(*echo.HTTPError)
+	require.True(t, ok)
+	assert.Equal(t, http.StatusBadRequest, httpErr.Code)
+}
+
+func TestCreateFolder_MissingClientId(t *testing.T) {
+	ctrl, _, _ := setupFoldersTestController(t)
+
+	reqBody := queryapi.FolderCreate{
+		Path:      "/documents",
+		ClientId:  "",
+		CreatedBy: openapi_types.Email("user@example.com"),
+	}
+
+	body, err := json.Marshal(reqBody)
+	require.NoError(t, err)
+
+	e := echo.New()
+	req := httptest.NewRequest(http.MethodPost, "/folders", bytes.NewReader(body))
+	req.Header.Set(echo.HeaderContentType, echo.MIMEApplicationJSON)
+	rec := httptest.NewRecorder()
+	c := e.NewContext(req, rec)
+
+	err = ctrl.CreateFolder(c)
+	require.Error(t, err)
+	httpErr, ok := err.(*echo.HTTPError)
+	require.True(t, ok)
+	assert.Equal(t, http.StatusBadRequest, httpErr.Code)
+}
+
+func TestCreateFolder_MissingCreatedBy(t *testing.T) {
+	ctrl, _, clientID := setupFoldersTestController(t)
+
+	// Send raw JSON with empty createdBy field
+	body := []byte(fmt.Sprintf(`{"path":"/documents","clientId":"%s","createdBy":""}`, clientID))
+
+	e := echo.New()
+	req := httptest.NewRequest(http.MethodPost, "/folders", bytes.NewReader(body))
+	req.Header.Set(echo.HeaderContentType, echo.MIMEApplicationJSON)
+	rec := httptest.NewRecorder()
+	c := e.NewContext(req, rec)
+
+	err := ctrl.CreateFolder(c)
+	require.Error(t, err)
+	httpErr, ok := err.(*echo.HTTPError)
+	require.True(t, ok)
+	assert.Equal(t, http.StatusBadRequest, httpErr.Code)
+}
+
+func TestRenameFolder_Success(t *testing.T) {
+	ctrl, cfg, clientID := setupFoldersTestController(t)
+	ctx := t.Context()
+
+	// Create folder first
+	folderRec, err := cfg.GetDBQueries().CreateFolder(ctx, &repository.CreateFolderParams{
+		Path:      "/old-path",
+		Clientid:  clientID,
+		Createdby: "admin@example.com",
+	})
+	require.NoError(t, err)
+
+	reqBody := queryapi.FolderRename{
+		Path: "/new-path",
+	}
+
+	body, err := json.Marshal(reqBody)
+	require.NoError(t, err)
+
+	e := echo.New()
+	req := httptest.NewRequest(http.MethodPatch, fmt.Sprintf("/folders/%s", folderRec.ID), bytes.NewReader(body))
+	req.Header.Set(echo.HeaderContentType, echo.MIMEApplicationJSON)
+	rec := httptest.NewRecorder()
+	c := e.NewContext(req, rec)
+
+	err = ctrl.RenameFolder(c, openapi_types.UUID(folderRec.ID))
+	require.NoError(t, err)
+	assert.Equal(t, http.StatusNoContent, rec.Code)
+
+	// Verify the folder was renamed
+	updatedFolder, err := cfg.GetDBQueries().GetFolderByID(ctx, folderRec.ID)
+	require.NoError(t, err)
+	assert.Equal(t, "/new-path", updatedFolder.Path)
+}
+
+func TestRenameFolder_MissingPath(t *testing.T) {
+	ctrl, cfg, clientID := setupFoldersTestController(t)
+	ctx := t.Context()
+
+	// Create folder first
+	folderRec, err := cfg.GetDBQueries().CreateFolder(ctx, &repository.CreateFolderParams{
+		Path:      "/old-path",
+		Clientid:  clientID,
+		Createdby: "admin@example.com",
+	})
+	require.NoError(t, err)
+
+	reqBody := queryapi.FolderRename{
+		Path: "",
+	}
+
+	body, err := json.Marshal(reqBody)
+	require.NoError(t, err)
+
+	e := echo.New()
+	req := httptest.NewRequest(http.MethodPatch, fmt.Sprintf("/folders/%s", folderRec.ID), bytes.NewReader(body))
+	req.Header.Set(echo.HeaderContentType, echo.MIMEApplicationJSON)
+	rec := httptest.NewRecorder()
+	c := e.NewContext(req, rec)
+
+	err = ctrl.RenameFolder(c, openapi_types.UUID(folderRec.ID))
+	require.Error(t, err)
+	httpErr, ok := err.(*echo.HTTPError)
+	require.True(t, ok)
+	assert.Equal(t, http.StatusBadRequest, httpErr.Code)
+}
+
+func TestGetFolderDocuments_Success(t *testing.T) {
+	ctrl, cfg, clientID := setupFoldersTestController(t)
+	ctx := t.Context()
+
+	// Create folder
+	folderRec, err := cfg.GetDBQueries().CreateFolder(ctx, &repository.CreateFolderParams{
+		Path:      "/documents",
+		Clientid:  clientID,
+		Createdby: "admin@example.com",
+	})
+	require.NoError(t, err)
+
+	// Create documents in the folder
+	filename1 := "doc1.pdf"
+	docID1, err := cfg.GetDBQueries().CreateDocument(ctx, &repository.CreateDocumentParams{
+		Clientid: clientID,
+		Hash:     fmt.Sprintf("hash1-%s", uuid.New().String()[:8]),
+		Filename: &filename1,
+		Folderid: &folderRec.ID,
+	})
+	require.NoError(t, err)
+
+	filename2 := "doc2.pdf"
+	docID2, err := cfg.GetDBQueries().CreateDocument(ctx, &repository.CreateDocumentParams{
+		Clientid: clientID,
+		Hash:     fmt.Sprintf("hash2-%s", uuid.New().String()[:8]),
+		Filename: &filename2,
+		Folderid: &folderRec.ID,
+	})
+	require.NoError(t, err)
+
+	e := echo.New()
+	req := httptest.NewRequest(http.MethodGet, fmt.Sprintf("/folders/%s/documents", folderRec.ID), nil)
+	rec := httptest.NewRecorder()
+	c := e.NewContext(req, rec)
+
+	err = ctrl.GetFolderDocuments(c, openapi_types.UUID(folderRec.ID))
+	require.NoError(t, err)
+	assert.Equal(t, http.StatusOK, rec.Code)
+
+	var resp struct {
+		Documents []queryapi.DocumentSummary `json:"documents"`
+	}
+	err = json.Unmarshal(rec.Body.Bytes(), &resp)
+	require.NoError(t, err)
+
+	assert.Len(t, resp.Documents, 2)
+
+	// Verify both documents are returned
+	docIDs := make(map[uuid.UUID]bool)
+	for _, doc := range resp.Documents {
+		docIDs[uuid.UUID(doc.Id)] = true
+	}
+	assert.True(t, docIDs[docID1])
+	assert.True(t, docIDs[docID2])
+}
+
+func TestGetFolderDocuments_EmptyFolder(t *testing.T) {
+	ctrl, cfg, clientID := setupFoldersTestController(t)
+	ctx := t.Context()
+
+	// Create folder with no documents
+	folderRec, err := cfg.GetDBQueries().CreateFolder(ctx, &repository.CreateFolderParams{
+		Path:      "/empty-folder",
+		Clientid:  clientID,
+		Createdby: "admin@example.com",
+	})
+	require.NoError(t, err)
+
+	e := echo.New()
+	req := httptest.NewRequest(http.MethodGet, fmt.Sprintf("/folders/%s/documents", folderRec.ID), nil)
+	rec := httptest.NewRecorder()
+	c := e.NewContext(req, rec)
+
+	err = ctrl.GetFolderDocuments(c, openapi_types.UUID(folderRec.ID))
+	require.NoError(t, err)
+	assert.Equal(t, http.StatusOK, rec.Code)
+
+	var resp struct {
+		Documents []queryapi.DocumentSummary `json:"documents"`
+	}
+	err = json.Unmarshal(rec.Body.Bytes(), &resp)
+	require.NoError(t, err)
+
+	assert.Len(t, resp.Documents, 0)
+}
+
+func TestGetFolderMetrics_Success(t *testing.T) {
+	ctrl, cfg, clientID := setupFoldersTestController(t)
+	ctx := t.Context()
+
+	// Create folder
+	folderRec, err := cfg.GetDBQueries().CreateFolder(ctx, &repository.CreateFolderParams{
+		Path:      "/documents",
+		Clientid:  clientID,
+		Createdby: "admin@example.com",
+	})
+	require.NoError(t, err)
+
+	// Create documents in the folder
+	filename1 := "doc1.pdf"
+	docID1, err := cfg.GetDBQueries().CreateDocument(ctx, &repository.CreateDocumentParams{
+		Clientid: clientID,
+		Hash:     fmt.Sprintf("metricshash1-%s", uuid.New().String()[:8]),
+		Filename: &filename1,
+		Folderid: &folderRec.ID,
+	})
+	require.NoError(t, err)
+
+	filename2 := "doc2.pdf"
+	docID2, err := cfg.GetDBQueries().CreateDocument(ctx, &repository.CreateDocumentParams{
+		Clientid: clientID,
+		Hash:     fmt.Sprintf("metricshash2-%s", uuid.New().String()[:8]),
+		Filename: &filename2,
+		Folderid: &folderRec.ID,
+	})
+	require.NoError(t, err)
+
+	filename3 := "doc3.pdf"
+	_, err = cfg.GetDBQueries().CreateDocument(ctx, &repository.CreateDocumentParams{
+		Clientid: clientID,
+		Hash:     fmt.Sprintf("metricshash3-%s", uuid.New().String()[:8]),
+		Filename: &filename3,
+		Folderid: &folderRec.ID,
+	})
+	require.NoError(t, err)
+
+	// Apply labels to documents (using labels from the lookup table)
+	_, err = cfg.GetDBQueries().ApplyLabel(ctx, &repository.ApplyLabelParams{
+		Documentid: docID1,
+		Label:      "Ingested",
+		Appliedby:  "user@example.com",
+	})
+	require.NoError(t, err)
+
+	_, err = cfg.GetDBQueries().ApplyLabel(ctx, &repository.ApplyLabelParams{
+		Documentid: docID2,
+		Label:      "Ingested",
+		Appliedby:  "user@example.com",
+	})
+	require.NoError(t, err)
+
+	_, err = cfg.GetDBQueries().ApplyLabel(ctx, &repository.ApplyLabelParams{
+		Documentid: docID2,
+		Label:      "OCR_Processed",
+		Appliedby:  "user@example.com",
+	})
+	require.NoError(t, err)
+
+	e := echo.New()
+	req := httptest.NewRequest(http.MethodGet, fmt.Sprintf("/folders/%s/metrics", folderRec.ID), nil)
+	rec := httptest.NewRecorder()
+	c := e.NewContext(req, rec)
+
+	err = ctrl.GetFolderMetrics(c, openapi_types.UUID(folderRec.ID))
+	require.NoError(t, err)
+	assert.Equal(t, http.StatusOK, rec.Code)
+
+	var resp queryapi.FolderMetrics
+	err = json.Unmarshal(rec.Body.Bytes(), &resp)
+	require.NoError(t, err)
+
+	assert.Equal(t, folderRec.ID, uuid.UUID(resp.FolderId))
+	assert.Equal(t, int32(3), resp.TotalDocuments)
+	assert.Equal(t, int32(2), resp.ByLabel["Ingested"])
+	assert.Equal(t, int32(1), resp.ByLabel["OCR_Processed"])
+}
+
+func TestGetFolderMetrics_EmptyFolder(t *testing.T) {
+	ctrl, cfg, clientID := setupFoldersTestController(t)
+	ctx := t.Context()
+
+	// Create folder with no documents
+	folderRec, err := cfg.GetDBQueries().CreateFolder(ctx, &repository.CreateFolderParams{
+		Path:      "/empty-metrics",
+		Clientid:  clientID,
+		Createdby: "admin@example.com",
+	})
+	require.NoError(t, err)
+
+	e := echo.New()
+	req := httptest.NewRequest(http.MethodGet, fmt.Sprintf("/folders/%s/metrics", folderRec.ID), nil)
+	rec := httptest.NewRecorder()
+	c := e.NewContext(req, rec)
+
+	err = ctrl.GetFolderMetrics(c, openapi_types.UUID(folderRec.ID))
+	require.NoError(t, err)
+	assert.Equal(t, http.StatusOK, rec.Code)
+
+	var resp queryapi.FolderMetrics
+	err = json.Unmarshal(rec.Body.Bytes(), &resp)
+	require.NoError(t, err)
+
+	assert.Equal(t, int32(0), resp.TotalDocuments)
+	assert.Len(t, resp.ByLabel, 0)
+}
+
+func TestListClientFolders_Success(t *testing.T) {
+	ctrl, cfg, clientID := setupFoldersTestController(t)
+	ctx := t.Context()
+
+	// Create some folders for the client
+	folder1, err := cfg.GetDBQueries().CreateFolder(ctx, &repository.CreateFolderParams{
+		Path:      "/folder1",
+		Clientid:  clientID,
+		Createdby: "admin@example.com",
+	})
+	require.NoError(t, err)
+
+	_, err = cfg.GetDBQueries().CreateFolder(ctx, &repository.CreateFolderParams{
+		Path:      "/folder1/subfolder",
+		Clientid:  clientID,
+		Parentid:  &folder1.ID,
+		Createdby: "admin@example.com",
+	})
+	require.NoError(t, err)
+
+	_, err = cfg.GetDBQueries().CreateFolder(ctx, &repository.CreateFolderParams{
+		Path:      "/folder2",
+		Clientid:  clientID,
+		Createdby: "admin@example.com",
+	})
+	require.NoError(t, err)
+
+	e := echo.New()
+	req := httptest.NewRequest(http.MethodGet, fmt.Sprintf("/client/%s/folders", clientID), nil)
+	rec := httptest.NewRecorder()
+	c := e.NewContext(req, rec)
+
+	err = ctrl.ListClientFolders(c, queryapi.ClientID(clientID))
+	require.NoError(t, err)
+	assert.Equal(t, http.StatusOK, rec.Code)
+
+	var resp queryapi.FolderList
+	err = json.Unmarshal(rec.Body.Bytes(), &resp)
+	require.NoError(t, err)
+
+	// Should have 3 folders
+	assert.Len(t, resp.Folders, 3)
+
+	// Folders should be ordered by path
+	assert.Equal(t, "/folder1", resp.Folders[0].Path)
+	assert.Equal(t, "/folder1/subfolder", resp.Folders[1].Path)
+	assert.Equal(t, "/folder2", resp.Folders[2].Path)
+}
+
+func TestListClientFolders_EmptyResult(t *testing.T) {
+	ctrl, _, clientID := setupFoldersTestController(t)
+
+	e := echo.New()
+	req := httptest.NewRequest(http.MethodGet, fmt.Sprintf("/client/%s/folders", clientID), nil)
+	rec := httptest.NewRecorder()
+	c := e.NewContext(req, rec)
+
+	err := ctrl.ListClientFolders(c, queryapi.ClientID(clientID))
+	require.NoError(t, err)
+	assert.Equal(t, http.StatusOK, rec.Code)
+
+	var resp queryapi.FolderList
+	err = json.Unmarshal(rec.Body.Bytes(), &resp)
+	require.NoError(t, err)
+
+	// Should have empty folders array
+	assert.Len(t, resp.Folders, 0)
+}