jacob-mathison/query-orchestration
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Merged in feature/dimensions (pull request #79)

Browse Source 
Dimension Limits

* dims

* clean
This commit is contained in:
Michael McGuinness
2025-02-28 15:56:47 +00:00
parent 01ec8fd40e
commit ae1abdb31f
4 changed files with 178 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/document/clean/clean.go
+9 -8
View File
@@ -21,13 +21,14 @@ type CleanParams struct {
type InvalidDocumentReason string
const (
InvalidDocumentMimeType InvalidDocumentReason = "invalid_mimetype"
InvalidDocumentRead InvalidDocumentReason = "invalid_read"
InvalidDocumentReadPages InvalidDocumentReason = "invalid_read_pages"
InvalidDocumentZeroPageCount InvalidDocumentReason = "zero_page_count"
InvalidDocumentLargePageCount InvalidDocumentReason = "large_page_count"
InvalidDocumentLargeFile InvalidDocumentReason = "large_file"
InvalidDocumentQuality InvalidDocumentReason = "invalid_quality"
InvalidDocumentMimeType InvalidDocumentReason = "invalid_mimetype"
InvalidDocumentRead InvalidDocumentReason = "invalid_read"
InvalidDocumentReadPages InvalidDocumentReason = "invalid_read_pages"
InvalidDocumentZeroPageCount InvalidDocumentReason = "zero_page_count"
InvalidDocumentLargePageCount InvalidDocumentReason = "large_page_count"
InvalidDocumentLargeFile InvalidDocumentReason = "large_file"
InvalidDocumentSmallDimensions InvalidDocumentReason = "small_dimensions"
InvalidDocumentLargeDimensions InvalidDocumentReason = "large_dimensions"
)
type ExecuteCleanResponse struct {
@@ -66,7 +67,7 @@ func (s *Service) executeCleanTasks(ctx context.Context, params *CleanParams) (*
return nil, err
}
corruptReason := content.isCorrupted(ctx)
corruptReason := content.isCorrupt(ctx)
if corruptReason != nil {
return &ExecuteCleanResponse{
failReason: corruptReason,
internal/document/clean/content.go
+1 -1
View File
@@ -10,7 +10,7 @@ import (
)
type Content interface {
isCorrupted(context.Context) *InvalidDocumentReason
isCorrupt(context.Context) *InvalidDocumentReason
}
func (s *Service) getContent(ctx context.Context, params *CleanParams, mimeType MimeType) (Content, error) {
internal/document/clean/pdf.go
+84 -19
View File
@@ -6,19 +6,24 @@ import (
"github.com/pdfcpu/pdfcpu/pkg/pdfcpu"
"github.com/pdfcpu/pdfcpu/pkg/pdfcpu/model"
"github.com/pdfcpu/pdfcpu/pkg/pdfcpu/types"
)
type PDF struct {
pdf io.ReadSeeker
config *model.Configuration
maxPages int
maxBytes int64
pdf io.ReadSeeker
config *model.Configuration
maxPages int
maxBytes int64
maxDimension int
minDimension int
}
const (
TEXTRACT_SYNC_LIMIT int64 = 10 * 1024 * 1024
TEXTRACT_ASYNC_LIMIT int64 = 500 * 1024 * 1024
TEXTRACT_PAGE_LIMIT = 3000
TEXTRACT_SYNC_LIMIT int64 = 10 * 1024 * 1024
TEXTRACT_ASYNC_LIMIT int64 = 500 * 1024 * 1024
TEXTRACT_PAGE_LIMIT = 3000
TEXTRACT_MAX_DIMENSION = 10000
TEXTRACT_MIN_DIMENSION = 50
)
func NewPDF(buf io.ReadSeeker) *PDF {
@@ -29,37 +34,97 @@ func NewPDF(buf io.ReadSeeker) *PDF {
}
return &PDF{
pdf: buf,
config: config,
maxPages: TEXTRACT_PAGE_LIMIT,
maxBytes: TEXTRACT_SYNC_LIMIT,
pdf: buf,
config: config,
maxPages: TEXTRACT_PAGE_LIMIT,
maxBytes: TEXTRACT_SYNC_LIMIT,
maxDimension: TEXTRACT_MAX_DIMENSION,
minDimension: TEXTRACT_MIN_DIMENSION,
}
}
func (s *PDF) isCorrupted(ctx context.Context) *InvalidDocumentReason {
var reason InvalidDocumentReason
func (s *PDF) isCorrupt(ctx context.Context) *InvalidDocumentReason {
pdfCtx, err := pdfcpu.ReadWithContext(ctx, s.pdf, s.config)
if err != nil {
reason = InvalidDocumentRead
reason := InvalidDocumentRead
return &reason
}
if int64(pdfCtx.Read.ReadFileSize()) > s.maxBytes {
reason = InvalidDocumentLargeFile
reason := InvalidDocumentLargeFile
return &reason
}
err = pdfCtx.EnsurePageCount()
docErr := s.isValidPageCount(pdfCtx)
if docErr != nil {
return docErr
}
docErr = s.isValidDimensions(pdfCtx)
if docErr != nil {
return docErr
}
return nil
}
func (s *PDF) isValidPageCount(pdfCtx *model.Context) *InvalidDocumentReason {
err := pdfCtx.EnsurePageCount()
if err != nil {
reason = InvalidDocumentRead
reason := InvalidDocumentRead
return &reason
}
if pdfCtx.PageCount > s.maxPages {
reason = InvalidDocumentLargePageCount
reason := InvalidDocumentLargePageCount
return &reason
}
return nil
}
func (s *PDF) isValidDimensions(pdfCtx *model.Context) *InvalidDocumentReason {
err := pdfCtx.EnsurePageCount()
if err != nil {
reason := InvalidDocumentRead
return &reason
}
_, err = s.pdf.Seek(0, io.SeekStart)
if err != nil {
reason := InvalidDocumentRead
return &reason
}
pageSet := types.IntSet{}
for pageNum := 1; pageNum <= pdfCtx.PageCount; pageNum++ {
pageSet[pageNum] = true
}
boxes, err := pdfCtx.PageBoundaries(pageSet)
if err != nil {
reason := InvalidDocumentRead
return &reason
}
for _, box := range boxes {
mediaBox := box.MediaBox()
if mediaBox == nil {
continue
}
dims := mediaBox.Dimensions()
if dims.Height > float64(s.maxDimension) ||
dims.Width > float64(s.maxDimension) {
reason := InvalidDocumentLargeDimensions
return &reason
} else if dims.Height < float64(s.minDimension) ||
dims.Width < float64(s.minDimension) {
reason := InvalidDocumentSmallDimensions
return &reason
}
}
return nil
}
internal/document/clean/pdf_test.go
+84 -13
View File
@@ -5,22 +5,85 @@ import (
"strings"
"testing"
"github.com/pdfcpu/pdfcpu/pkg/pdfcpu"
"github.com/stretchr/testify/assert"
)
func TestIsValidDimensions(t *testing.T) {
t.Run("valid", func(t *testing.T) {
ctx := context.Background()
pdf := NewPDF(strings.NewReader(pdfHelloWorld))
pdfCtx, err := pdfcpu.ReadWithContext(ctx, pdf.pdf, pdf.config)
assert.NoError(t, err)
reason := pdf.isValidDimensions(pdfCtx)
assert.Nil(t, reason)
})
t.Run("large dimensions", func(t *testing.T) {
ctx := context.Background()
pdf := NewPDF(strings.NewReader(pdfHelloWorld))
pdfCtx, err := pdfcpu.ReadWithContext(ctx, pdf.pdf, pdf.config)
assert.NoError(t, err)
pdf.maxDimension = 1
reason := pdf.isValidDimensions(pdfCtx)
assert.Equal(t, InvalidDocumentLargeDimensions, *reason)
})
t.Run("small dimensions", func(t *testing.T) {
ctx := context.Background()
pdf := NewPDF(strings.NewReader(pdfHelloWorld))
pdfCtx, err := pdfcpu.ReadWithContext(ctx, pdf.pdf, pdf.config)
assert.NoError(t, err)
pdf.minDimension = TEXTRACT_MAX_DIMENSION
reason := pdf.isValidDimensions(pdfCtx)
assert.Equal(t, InvalidDocumentSmallDimensions, *reason)
})
}
func TestIsValidPageCount(t *testing.T) {
t.Run("valid", func(t *testing.T) {
ctx := context.Background()
pdf := NewPDF(strings.NewReader(pdfHelloWorld))
pdfCtx, err := pdfcpu.ReadWithContext(ctx, pdf.pdf, pdf.config)
assert.NoError(t, err)
reason := pdf.isValidPageCount(pdfCtx)
assert.Nil(t, reason)
})
t.Run("large page count", func(t *testing.T) {
ctx := context.Background()
pdf := NewPDF(strings.NewReader(pdfTwoPages))
pdfCtx, err := pdfcpu.ReadWithContext(ctx, pdf.pdf, pdf.config)
assert.NoError(t, err)
pdf.maxPages = 1
reason := pdf.isValidPageCount(pdfCtx)
assert.Equal(t, InvalidDocumentLargePageCount, *reason)
})
}
func TestIsCorrupt(t *testing.T) {
t.Run("valid", func(t *testing.T) {
ctx := context.Background()
pdf := NewPDF(strings.NewReader(pdfHelloWorld))
reason := pdf.isCorrupted(ctx)
reason := pdf.isCorrupt(ctx)
assert.Nil(t, reason)
})
t.Run("no content", func(t *testing.T) {
ctx := context.Background()
pdf := NewPDF(strings.NewReader("%PDF-1.4 %%EOF"))
reason := pdf.isCorrupted(ctx)
reason := pdf.isCorrupt(ctx)
assert.Equal(t, InvalidDocumentRead, *reason)
})
t.Run("large file size", func(t *testing.T) {
@@ -28,78 +91,86 @@ func TestIsCorrupt(t *testing.T) {
pdf := NewPDF(strings.NewReader(pdfHelloWorld))
pdf.maxBytes = 1
reason := pdf.isCorrupted(ctx)
reason := pdf.isCorrupt(ctx)
assert.Equal(t, InvalidDocumentLargeFile, *reason)
})
t.Run("large dimensions", func(t *testing.T) {
ctx := context.Background()
pdf := NewPDF(strings.NewReader(pdfHelloWorld))
pdf.maxDimension = 1
reason := pdf.isCorrupt(ctx)
assert.Equal(t, InvalidDocumentLargeDimensions, *reason)
})
t.Run("large page count", func(t *testing.T) {
ctx := context.Background()
pdf := NewPDF(strings.NewReader(pdfTwoPages))
pdf.maxPages = 1
reason := pdf.isCorrupted(ctx)
reason := pdf.isCorrupt(ctx)
assert.Equal(t, InvalidDocumentLargePageCount, *reason)
})
t.Run("version 1.0", func(t *testing.T) {
ctx := context.Background()
pdf := NewPDF(strings.NewReader(PDF1_0))
reason := pdf.isCorrupted(ctx)
reason := pdf.isCorrupt(ctx)
assert.Nil(t, reason)
})
t.Run("version 1.1", func(t *testing.T) {
ctx := context.Background()
pdf := NewPDF(strings.NewReader(PDF1_1))
reason := pdf.isCorrupted(ctx)
reason := pdf.isCorrupt(ctx)
assert.Nil(t, reason)
})
t.Run("version 1.2", func(t *testing.T) {
ctx := context.Background()
pdf := NewPDF(strings.NewReader(PDF1_2))
reason := pdf.isCorrupted(ctx)
reason := pdf.isCorrupt(ctx)
assert.Nil(t, reason)
})
t.Run("version 1.3", func(t *testing.T) {
ctx := context.Background()
pdf := NewPDF(strings.NewReader(PDF1_3))
reason := pdf.isCorrupted(ctx)
reason := pdf.isCorrupt(ctx)
assert.Nil(t, reason)
})
t.Run("version 1.4", func(t *testing.T) {
ctx := context.Background()
pdf := NewPDF(strings.NewReader(PDF1_4))
reason := pdf.isCorrupted(ctx)
reason := pdf.isCorrupt(ctx)
assert.Nil(t, reason)
})
t.Run("version 1.5", func(t *testing.T) {
ctx := context.Background()
pdf := NewPDF(strings.NewReader(PDF1_5))
reason := pdf.isCorrupted(ctx)
reason := pdf.isCorrupt(ctx)
assert.Nil(t, reason)
})
t.Run("version 1.6", func(t *testing.T) {
ctx := context.Background()
pdf := NewPDF(strings.NewReader(PDF1_6))
reason := pdf.isCorrupted(ctx)
reason := pdf.isCorrupt(ctx)
assert.Nil(t, reason)
})
t.Run("version 1.7", func(t *testing.T) {
ctx := context.Background()
pdf := NewPDF(strings.NewReader(PDF1_7))
reason := pdf.isCorrupted(ctx)
reason := pdf.isCorrupt(ctx)
assert.Nil(t, reason)
})
t.Run("version 2.0", func(t *testing.T) {
ctx := context.Background()
pdf := NewPDF(strings.NewReader(PDF2_0))
reason := pdf.isCorrupted(ctx)
reason := pdf.isCorrupt(ctx)
assert.Nil(t, reason)
})
}