From 90119a101ce7841ca08362ec45d52418e895d98e Mon Sep 17 00:00:00 2001 From: Michael McGuinness Date: Tue, 4 Mar 2025 11:38:02 +0000 Subject: [PATCH] Merged in feature/parsetypes (pull request #82) Add Parsing Functions * addfuncs --- internal/document/clean/clean.go | 24 +-- internal/document/clean/parsetype.go | 129 ++++++++++++++++ internal/document/clean/parsetypes_test.go | 166 +++++++++++++++++++++ 3 files changed, 300 insertions(+), 19 deletions(-) create mode 100644 internal/document/clean/parsetype.go create mode 100644 internal/document/clean/parsetypes_test.go diff --git a/internal/document/clean/clean.go b/internal/document/clean/clean.go index c936e4f4..17af9289 100644 --- a/internal/document/clean/clean.go +++ b/internal/document/clean/clean.go @@ -127,17 +127,11 @@ func (s *Service) storeClean(ctx context.Context, id uuid.UUID, out *ExecuteClea } if out.failReason != nil { slog.Info("Failed document", "id", id, "reason", *out.failReason) - params.Fail = repository.NullCleanfailtype{ - Cleanfailtype: repository.Cleanfailtype(*out.failReason), - Valid: true, - } + params.Fail = ToDBNullFailType(*out.failReason) } else { params.Bucket = &out.location.Bucket params.Key = &out.location.Key - params.Mimetype = repository.NullCleanmimetype{ - Cleanmimetype: repository.Cleanmimetype(*out.mimetype), - Valid: true, - } + params.Mimetype = ToDBNullMimeType(*out.mimetype) } err := s.cfg.ExecuteDBTransaction(ctx, func(ctx context.Context, q *repository.Queries) error { @@ -183,22 +177,14 @@ func (s *Service) isNewClean(lastEntry *repository.GetMostRecentDocumentCleanEnt } if out.failReason != nil { - if lastEntry.Fail.Valid && *out.failReason == InvalidDocumentReason(lastEntry.Fail.Cleanfailtype) { - return false - } - - return true + return *out.failReason != ParseDBNullFailType(lastEntry.Fail) } if lastEntry.Fail.Valid { return true } - if out.location.Bucket == *lastEntry.Bucket && + return !(out.location.Bucket == *lastEntry.Bucket && out.location.Key == *lastEntry.Key && - *out.mimetype == MimeType(lastEntry.Mimetype.Cleanmimetype) { - return false - } - - return true + *out.mimetype == ParseDBNullMimeType(lastEntry.Mimetype)) } diff --git a/internal/document/clean/parsetype.go b/internal/document/clean/parsetype.go new file mode 100644 index 00000000..3d31bbb7 --- /dev/null +++ b/internal/document/clean/parsetype.go @@ -0,0 +1,129 @@ +package documentclean + +import "queryorchestration/internal/database/repository" + +func ToDBMimeType(t MimeType) repository.Cleanmimetype { + switch t { + case MimeTypePDF: + return repository.CleanmimetypeApplicationPdf + } + + return repository.Cleanmimetype("") +} + +func ParseDBMimeType(t repository.Cleanmimetype) MimeType { + switch t { + case repository.CleanmimetypeApplicationPdf: + return MimeTypePDF + } + + return MimeTypeInvalid +} + +func ToDBFailType(t InvalidDocumentReason) repository.Cleanfailtype { + switch t { + case InvalidDocumentMimeType: + return repository.CleanfailtypeInvalidMimetype + case InvalidDocumentRead: + return repository.CleanfailtypeInvalidRead + case InvalidDocumentReadPages: + return repository.CleanfailtypeInvalidReadPages + case InvalidDocumentZeroPageCount: + return repository.CleanfailtypeZeroPageCount + case InvalidDocumentLargePageCount: + return repository.CleanfailtypeLargePageCount + case InvalidDocumentLargeFile: + return repository.CleanfailtypeLargeFile + case InvalidDocumentSmallDimensions: + return repository.CleanfailtypeSmallDimensions + case InvalidDocumentLargeDimensions: + return repository.CleanfailtypeLargeDimensions + case InvalidDocumentSmallDPI: + return repository.CleanfailtypeSmallDpi + case InvalidDocumentLargeDPI: + return repository.CleanfailtypeLargeDpi + } + + return repository.Cleanfailtype("") +} + +func ParseDBFailType(t repository.Cleanfailtype) InvalidDocumentReason { + switch t { + case repository.CleanfailtypeInvalidMimetype: + return InvalidDocumentMimeType + case repository.CleanfailtypeInvalidRead: + return InvalidDocumentRead + case repository.CleanfailtypeInvalidReadPages: + return InvalidDocumentReadPages + case repository.CleanfailtypeZeroPageCount: + return InvalidDocumentZeroPageCount + case repository.CleanfailtypeLargePageCount: + return InvalidDocumentLargePageCount + case repository.CleanfailtypeLargeFile: + return InvalidDocumentLargeFile + case repository.CleanfailtypeSmallDimensions: + return InvalidDocumentSmallDimensions + case repository.CleanfailtypeLargeDimensions: + return InvalidDocumentLargeDimensions + case repository.CleanfailtypeSmallDpi: + return InvalidDocumentSmallDPI + case repository.CleanfailtypeLargeDpi: + return InvalidDocumentLargeDPI + } + + return InvalidDocumentReason("") +} + +func ToDBNullMimeType(t MimeType) repository.NullCleanmimetype { + mimeType := ToDBMimeType(t) + if mimeType == repository.Cleanmimetype("") { + return repository.NullCleanmimetype{} + } + + return repository.NullCleanmimetype{ + Cleanmimetype: mimeType, + Valid: true, + } +} + +func ParseDBNullMimeType(t repository.NullCleanmimetype) MimeType { + if !t.Valid { + return MimeTypeInvalid + } + + return ParseDBMimeType(t.Cleanmimetype) +} + +func ToDBNullFailType(t InvalidDocumentReason) repository.NullCleanfailtype { + failType := ToDBFailType(t) + if failType == repository.Cleanfailtype("") { + return repository.NullCleanfailtype{} + } + + return repository.NullCleanfailtype{ + Cleanfailtype: failType, + Valid: true, + } +} + +func ParseDBNullFailType(t repository.NullCleanfailtype) InvalidDocumentReason { + if !t.Valid { + return InvalidDocumentReason("") + } + + return ParseDBFailType(t.Cleanfailtype) +} + +// params.Mimetype = repository.NullCleanmimetype{ +// Cleanmimetype: repository.Cleanmimetype(*out.mimetype), +// Valid: true, +// } + +// params.Fail = repository.NullCleanfailtype{ +// Cleanfailtype: repository.Cleanfailtype(*out.failReason), +// Valid: true, +// } + +// if lastEntry.Fail.Valid && *out.failReason == InvalidDocumentReason(lastEntry.Fail.Cleanfailtype) { + +// *out.mimetype == MimeType(lastEntry.Mimetype.Cleanmimetype) { diff --git a/internal/document/clean/parsetypes_test.go b/internal/document/clean/parsetypes_test.go new file mode 100644 index 00000000..2b388bdb --- /dev/null +++ b/internal/document/clean/parsetypes_test.go @@ -0,0 +1,166 @@ +package documentclean_test + +import ( + "queryorchestration/internal/database/repository" + documentclean "queryorchestration/internal/document/clean" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestToDBMimeType(t *testing.T) { + t.Run("application/pdf", func(t *testing.T) { + assert.Equal(t, repository.CleanmimetypeApplicationPdf, documentclean.ToDBMimeType(documentclean.MimeTypePDF)) + }) + t.Run("invalid", func(t *testing.T) { + assert.Equal(t, repository.Cleanmimetype(""), documentclean.ToDBMimeType(documentclean.MimeTypeInvalid)) + }) + t.Run("binary octet", func(t *testing.T) { + assert.Equal(t, repository.Cleanmimetype(""), documentclean.ToDBMimeType(documentclean.MimeTypeBinaryOctetStream)) + }) +} + +func TestToDBNullMimeType(t *testing.T) { + t.Run("application/pdf", func(t *testing.T) { + mimetype := documentclean.ToDBNullMimeType(documentclean.MimeTypePDF) + assert.Equal(t, mimetype.Cleanmimetype, repository.CleanmimetypeApplicationPdf) + assert.True(t, mimetype.Valid) + }) + t.Run("invalid", func(t *testing.T) { + mimetype := documentclean.ToDBNullMimeType(documentclean.MimeTypeInvalid) + assert.False(t, mimetype.Valid) + }) +} + +func TestParseDBMimeType(t *testing.T) { + t.Run("application/pdf", func(t *testing.T) { + assert.Equal(t, documentclean.MimeTypePDF, documentclean.ParseDBMimeType(repository.CleanmimetypeApplicationPdf)) + }) + t.Run("unknown", func(t *testing.T) { + assert.Equal(t, documentclean.MimeTypeInvalid, documentclean.ParseDBMimeType(repository.Cleanmimetype(""))) + }) +} + +func TestParseDBNullMimeType(t *testing.T) { + t.Run("application/pdf", func(t *testing.T) { + assert.Equal(t, documentclean.MimeTypePDF, documentclean.ParseDBNullMimeType(repository.NullCleanmimetype{ + Valid: true, + Cleanmimetype: repository.CleanmimetypeApplicationPdf, + })) + }) + t.Run("not valid", func(t *testing.T) { + assert.Equal(t, documentclean.MimeTypeInvalid, documentclean.ParseDBNullMimeType(repository.NullCleanmimetype{ + Valid: false, + Cleanmimetype: repository.CleanmimetypeApplicationPdf, + })) + }) + t.Run("empty", func(t *testing.T) { + assert.Equal(t, documentclean.MimeTypeInvalid, documentclean.ParseDBNullMimeType(repository.NullCleanmimetype{ + Valid: true, + })) + }) +} + +func TestParseDBFailType(t *testing.T) { + t.Run("invalid mimetype", func(t *testing.T) { + assert.Equal(t, repository.CleanfailtypeInvalidMimetype, documentclean.ToDBFailType(documentclean.InvalidDocumentMimeType)) + }) + t.Run("invalid read", func(t *testing.T) { + assert.Equal(t, repository.CleanfailtypeInvalidRead, documentclean.ToDBFailType(documentclean.InvalidDocumentRead)) + }) + t.Run("invalid read pages", func(t *testing.T) { + assert.Equal(t, repository.CleanfailtypeInvalidReadPages, documentclean.ToDBFailType(documentclean.InvalidDocumentReadPages)) + }) + t.Run("zero page count", func(t *testing.T) { + assert.Equal(t, repository.CleanfailtypeZeroPageCount, documentclean.ToDBFailType(documentclean.InvalidDocumentZeroPageCount)) + }) + t.Run("large page count", func(t *testing.T) { + assert.Equal(t, repository.CleanfailtypeLargePageCount, documentclean.ToDBFailType(documentclean.InvalidDocumentLargePageCount)) + }) + t.Run("large file", func(t *testing.T) { + assert.Equal(t, repository.CleanfailtypeLargeFile, documentclean.ToDBFailType(documentclean.InvalidDocumentLargeFile)) + }) + t.Run("small dimensions", func(t *testing.T) { + assert.Equal(t, repository.CleanfailtypeSmallDimensions, documentclean.ToDBFailType(documentclean.InvalidDocumentSmallDimensions)) + }) + t.Run("large dimensions", func(t *testing.T) { + assert.Equal(t, repository.CleanfailtypeLargeDimensions, documentclean.ToDBFailType(documentclean.InvalidDocumentLargeDimensions)) + }) + t.Run("small dpi", func(t *testing.T) { + assert.Equal(t, repository.CleanfailtypeSmallDpi, documentclean.ToDBFailType(documentclean.InvalidDocumentSmallDPI)) + }) + t.Run("large dpi", func(t *testing.T) { + assert.Equal(t, repository.CleanfailtypeLargeDpi, documentclean.ToDBFailType(documentclean.InvalidDocumentLargeDPI)) + }) + t.Run("unknown", func(t *testing.T) { + assert.Equal(t, repository.Cleanfailtype(""), documentclean.ToDBFailType(documentclean.InvalidDocumentReason(""))) + }) +} + +func TestParseDBNullFailType(t *testing.T) { + t.Run("invalid mimetype", func(t *testing.T) { + assert.Equal(t, documentclean.InvalidDocumentMimeType, documentclean.ParseDBNullFailType(repository.NullCleanfailtype{ + Valid: true, + Cleanfailtype: repository.CleanfailtypeInvalidMimetype, + })) + }) + t.Run("not valid", func(t *testing.T) { + assert.Equal(t, documentclean.InvalidDocumentReason(""), documentclean.ParseDBNullFailType(repository.NullCleanfailtype{ + Valid: false, + Cleanfailtype: repository.CleanfailtypeInvalidMimetype, + })) + }) + t.Run("empty", func(t *testing.T) { + assert.Equal(t, documentclean.InvalidDocumentReason(""), documentclean.ParseDBNullFailType(repository.NullCleanfailtype{ + Valid: true, + })) + }) +} + +func TestToDBFailType(t *testing.T) { + t.Run("invalid mimetype", func(t *testing.T) { + assert.Equal(t, documentclean.InvalidDocumentMimeType, documentclean.ParseDBFailType(repository.CleanfailtypeInvalidMimetype)) + }) + t.Run("invalid read", func(t *testing.T) { + assert.Equal(t, documentclean.InvalidDocumentRead, documentclean.ParseDBFailType(repository.CleanfailtypeInvalidRead)) + }) + t.Run("invalid read pages", func(t *testing.T) { + assert.Equal(t, documentclean.InvalidDocumentReadPages, documentclean.ParseDBFailType(repository.CleanfailtypeInvalidReadPages)) + }) + t.Run("zero page count", func(t *testing.T) { + assert.Equal(t, documentclean.InvalidDocumentZeroPageCount, documentclean.ParseDBFailType(repository.CleanfailtypeZeroPageCount)) + }) + t.Run("large page count", func(t *testing.T) { + assert.Equal(t, documentclean.InvalidDocumentLargePageCount, documentclean.ParseDBFailType(repository.CleanfailtypeLargePageCount)) + }) + t.Run("large file", func(t *testing.T) { + assert.Equal(t, documentclean.InvalidDocumentLargeFile, documentclean.ParseDBFailType(repository.CleanfailtypeLargeFile)) + }) + t.Run("small dimensions", func(t *testing.T) { + assert.Equal(t, documentclean.InvalidDocumentSmallDimensions, documentclean.ParseDBFailType(repository.CleanfailtypeSmallDimensions)) + }) + t.Run("large dimensions", func(t *testing.T) { + assert.Equal(t, documentclean.InvalidDocumentLargeDimensions, documentclean.ParseDBFailType(repository.CleanfailtypeLargeDimensions)) + }) + t.Run("small dpi", func(t *testing.T) { + assert.Equal(t, documentclean.InvalidDocumentSmallDPI, documentclean.ParseDBFailType(repository.CleanfailtypeSmallDpi)) + }) + t.Run("large dpi", func(t *testing.T) { + assert.Equal(t, documentclean.InvalidDocumentLargeDPI, documentclean.ParseDBFailType(repository.CleanfailtypeLargeDpi)) + }) + t.Run("unknown", func(t *testing.T) { + assert.Equal(t, documentclean.InvalidDocumentReason(""), documentclean.ParseDBFailType(repository.Cleanfailtype(""))) + }) +} + +func TestToDBNullFailType(t *testing.T) { + t.Run("invalid mimetype", func(t *testing.T) { + failtype := documentclean.ToDBNullFailType(documentclean.InvalidDocumentMimeType) + assert.Equal(t, repository.CleanfailtypeInvalidMimetype, failtype.Cleanfailtype) + assert.True(t, failtype.Valid) + }) + t.Run("unknown", func(t *testing.T) { + failtype := documentclean.ToDBNullFailType(documentclean.InvalidDocumentReason("")) + assert.False(t, failtype.Valid) + }) +}