Merged in feature/clean (pull request #76)

Clean Set Up

* fail

* starteddb

* constraint

* cleantest

* fixqueries

* fixtests

* storemimetype

* buffer

* tests

* setup

* passingtests

* pdfHElloWorld

* rm

* test

* notodos

* tests

* clean

* testpdf

api/docCleanRunner/runner_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"io"
 	doccleanrunner "queryorchestration/api/docCleanRunner"
 	"queryorchestration/internal/database"
 	"queryorchestration/internal/database/repository"
@@ -15,8 +16,10 @@ import (
 	"queryorchestration/internal/serviceconfig/queue/documenttext"
 	objectstoremock "queryorchestration/mocks/objectstore"
 	queuemock "queryorchestration/mocks/queue"
+	"strings"
 	"testing"
 
+	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/aws/aws-sdk-go-v2/service/sqs"
 	"github.com/aws/aws-sdk-go-v2/service/sqs/types"
 	"github.com/go-playground/validator/v10"
@@ -56,30 +59,46 @@ func TestDocCleanRunner(t *testing.T) {
 	})
 	assert.NotNil(t, runner)
 
-	doc := doccleanrunner.Body{
+	bod := doccleanrunner.Body{
 		ID: uuid.New(),
 	}
-	bodyBytes, err := json.Marshal(doc)
+	bodyBytes, err := json.Marshal(bod)
 	assert.NoError(t, err)
 	body := string(bodyBytes)
 	msg := &types.Message{
 		Body: &body,
 	}
 
+	doc := document.Document{
+		ID:    bod.ID,
+		JobID: uuid.New(),
+		Hash:  "example_hash",
+	}
 	inloc := document.Location{
 		Bucket: "bucket_name",
 		Key:    "/i/am/here",
 	}
+	dbid := database.MustToDBUUID(doc.ID)
 
-	pool.ExpectQuery("name: IsDocumentClean :one").WithArgs(database.MustToDBUUID(doc.ID)).WillReturnRows(
+	pool.ExpectQuery("name: HasDocumentCleanEntry :one").WithArgs(dbid).WillReturnRows(
 		pgxmock.NewRows([]string{"isclean"}).
 			AddRow(false),
 	)
-	pool.ExpectQuery("name: GetDocumentEntry :one").WithArgs(database.MustToDBUUID(doc.ID)).WillReturnRows(
+	pool.ExpectQuery("name: GetDocument :one").WithArgs(dbid).
+		WillReturnRows(
+			pgxmock.NewRows([]string{"id", "jobId", "hash"}).
+				AddRow(dbid, database.MustToDBUUID(doc.JobID), doc.Hash),
+		)
+	pool.ExpectQuery("name: GetDocumentEntry :one").WithArgs(dbid).WillReturnRows(
 		pgxmock.NewRows([]string{"documentId", "bucket", "key"}).
-			AddRow(database.MustToDBUUID(doc.ID), inloc.Bucket, inloc.Key),
+			AddRow(dbid, inloc.Bucket, inloc.Key),
 	)
-	pool.ExpectExec("name: AddDocumentCleanEntry :exec").WithArgs(database.MustToDBUUID(doc.ID), int32(1), inloc.Bucket, inloc.Key).
+	mimeType := "application/pdf"
+	dbmimetype := repository.NullCleanmimetypes{
+		Valid:          true,
+		Cleanmimetypes: repository.Cleanmimetypes(mimeType),
+	}
+	pool.ExpectExec("name: AddDocumentCleanEntry :exec").WithArgs(dbid, int32(1), &inloc.Bucket, &inloc.Key, dbmimetype, repository.NullCleanfailtype{}).
 		WillReturnResult(pgxmock.NewResult("", 1))
 
 	mockSQS.EXPECT().
@@ -92,6 +111,93 @@ func TestDocCleanRunner(t *testing.T) {
 		).
 		Return(&sqs.SendMessageOutput{}, nil)
 
+	mockStore.EXPECT().
+		HeadObject(
+			mock.Anything,
+			mock.MatchedBy(func(in *s3.HeadObjectInput) bool {
+				return *in.Bucket == inloc.Bucket && *in.Key == inloc.Key
+			}),
+			mock.Anything,
+		).
+		Return(&s3.HeadObjectOutput{
+			ContentType: &mimeType,
+		}, nil)
+
+	bodyMsg := io.NopCloser(strings.NewReader(pdfHelloWorld))
+	mockStore.EXPECT().
+		GetObject(
+			mock.Anything,
+			mock.MatchedBy(func(in *s3.GetObjectInput) bool {
+				return *in.Bucket == inloc.Bucket && *in.Key == inloc.Key
+			}),
+			mock.Anything,
+		).
+		Return(&s3.GetObjectOutput{
+			Body: bodyMsg,
+		}, nil)
+
 	err = runner.Process(ctx, msg)
 	assert.NoError(t, err)
 }
+
+const pdfHelloWorld = `%PDF-1.4
+%����
+1 0 obj
+<<
+  /Type /Catalog
+  /Pages 2 0 R
+  /Version /1.4
+>>
+endobj
+2 0 obj
+<<
+  /Type /Pages
+  /Kids [3 0 R]
+  /Count 1
+>>
+endobj
+3 0 obj
+<<
+  /Type /Page
+  /Parent 2 0 R
+  /MediaBox [0 0 612 792]
+  /Resources <<
+    /Font <<
+      /F1 <<
+        /Type /Font
+        /Subtype /Type1
+        /BaseFont /Helvetica
+      >>
+    >>
+  >>
+  /Contents 4 0 R
+>>
+endobj
+4 0 obj
+<<
+  /Length
+  44
+>>
+stream
+BT
+/F1 24 Tf
+100 700 Td
+(Hello World!) Tj
+ET
+endstream
+endobj
+xref
+0 5
+0000000000 65535 f
+0000000015 00000 n
+0000000086 00000 n
+0000000151 00000 n
+0000000376 00000 n
+trailer
+<<
+  /Size 5
+  /Root 1 0 R
+>>
+startxref
+472
+%%EOF`