Jay Brown
ebf47c6013
track file sizes for all documents in system * feature complete needs dev testing
536 lines
14 KiB
Go
536 lines
14 KiB
Go
package documentinit
|
|
|
|
import (
|
|
"bytes"
|
|
"errors"
|
|
"fmt"
|
|
"testing"
|
|
"time"
|
|
|
|
"queryorchestration/internal/database/repository"
|
|
"queryorchestration/internal/document"
|
|
"queryorchestration/internal/serviceconfig/objectstore"
|
|
"queryorchestration/internal/test"
|
|
objectstoremock "queryorchestration/mocks/objectstore"
|
|
queuemock "queryorchestration/mocks/queue"
|
|
|
|
"github.com/stretchr/testify/require"
|
|
|
|
"github.com/aws/aws-sdk-go-v2/aws"
|
|
"github.com/aws/aws-sdk-go-v2/service/s3"
|
|
"github.com/aws/aws-sdk-go-v2/service/sqs"
|
|
"github.com/google/uuid"
|
|
"github.com/pashagolub/pgxmock/v3"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/mock"
|
|
)
|
|
|
|
func TestCreate(t *testing.T) {
|
|
ctx := t.Context()
|
|
|
|
pool, err := pgxmock.NewPool()
|
|
require.NoError(t, err)
|
|
cfg := &DocInitConfig{}
|
|
cfg.DBPool = pool
|
|
cfg.DBQueries = repository.New(pool)
|
|
mockSQS := queuemock.NewMockSQSClient(t)
|
|
cfg.QueueClient = mockSQS
|
|
|
|
// Set up mock S3 client for file size measurement
|
|
mockS3 := objectstoremock.NewMockS3Client(t)
|
|
cfg.StoreClient = mockS3
|
|
|
|
cfg.DocumentSyncURL = "/i/am/here"
|
|
|
|
svc := New(cfg)
|
|
|
|
clientId := "hi"
|
|
doc := document.DocumentSummary{
|
|
ID: uuid.New(),
|
|
ClientID: clientId,
|
|
Hash: "example_hash",
|
|
}
|
|
bucket := "example_bucket"
|
|
key := objectstore.BucketKey{
|
|
ClientID: doc.ClientID,
|
|
CreatedAt: time.Now().UTC(),
|
|
EntityID: uuid.New(),
|
|
Location: objectstore.Import,
|
|
}
|
|
|
|
// Mock HeadObject to return file size
|
|
fileSize := int64(1024)
|
|
mockS3.EXPECT().
|
|
HeadObject(mock.Anything, mock.MatchedBy(func(in *s3.HeadObjectInput) bool {
|
|
return *in.Bucket == bucket && *in.Key == key.String()
|
|
})).
|
|
Return(&s3.HeadObjectOutput{ContentLength: &fileSize}, nil)
|
|
|
|
pool.ExpectQuery("name: GetDocumentIDByHash :one").WithArgs(doc.Hash, doc.ClientID).WillReturnRows(
|
|
pgxmock.NewRows([]string{"id"}),
|
|
)
|
|
pool.ExpectBegin()
|
|
pool.ExpectQuery("name: CreateDocument :one").WithArgs(doc.ClientID, doc.Hash, (*uuid.UUID)(nil), (*string)(nil), (*uuid.UUID)(nil), (*string)(nil), &fileSize).
|
|
WillReturnRows(
|
|
pgxmock.NewRows([]string{"id"}).
|
|
AddRow(doc.ID),
|
|
)
|
|
pool.ExpectExec("name: AddDocumentEntry :exec").WithArgs(doc.ID, bucket, key.String()).
|
|
WillReturnResult(pgxmock.NewResult("", 1))
|
|
pool.ExpectCommit()
|
|
pool.ExpectQuery("name: GetDocumentSummary :one").WithArgs(doc.ID).
|
|
WillReturnRows(
|
|
pgxmock.NewRows([]string{"id", "clientId", "hash"}).
|
|
AddRow(doc.ID, doc.ClientID, doc.Hash),
|
|
)
|
|
pool.ExpectQuery("name: GetClient :one").WithArgs(clientId).WillReturnRows(
|
|
pgxmock.NewRows([]string{"id", "clientId", "canSync"}).
|
|
AddRow(clientId, clientId, true),
|
|
)
|
|
pool.ExpectQuery("name: GetClient :one").WithArgs(clientId).WillReturnRows(
|
|
pgxmock.NewRows([]string{"id", "name", "canSync"}).
|
|
AddRow(clientId, "client_name", true),
|
|
)
|
|
|
|
mockSQS.EXPECT().
|
|
SendMessage(
|
|
mock.Anything,
|
|
mock.MatchedBy(func(in *sqs.SendMessageInput) bool {
|
|
return *in.QueueUrl == cfg.DocumentSyncURL && *in.MessageBody == fmt.Sprintf("{\"id\":\"%s\"}", doc.ID.String())
|
|
}),
|
|
mock.Anything,
|
|
).
|
|
Return(&sqs.SendMessageOutput{}, nil)
|
|
|
|
id, err := svc.Create(ctx, &Create{
|
|
Hash: doc.Hash,
|
|
Key: key,
|
|
Bucket: bucket,
|
|
Filename: nil, // No filename for this test
|
|
})
|
|
require.NoError(t, err)
|
|
assert.Equal(t, doc.ID, id)
|
|
}
|
|
|
|
func TestGetCreateParams(t *testing.T) {
|
|
ctx := t.Context()
|
|
|
|
pool, err := pgxmock.NewPool()
|
|
require.NoError(t, err)
|
|
cfg := &DocInitConfig{}
|
|
cfg.DBPool = pool
|
|
cfg.DBQueries = repository.New(pool)
|
|
|
|
svc := New(cfg)
|
|
|
|
doc := document.DocumentSummary{
|
|
ClientID: "hello",
|
|
Hash: "example_hash",
|
|
}
|
|
bucket := "example_bucket"
|
|
key := objectstore.BucketKey{
|
|
ClientID: doc.ClientID,
|
|
CreatedAt: time.Now().UTC(),
|
|
EntityID: uuid.New(),
|
|
Location: objectstore.Import,
|
|
}
|
|
|
|
pool.ExpectQuery("name: GetDocumentIDByHash :one").WithArgs(doc.Hash, doc.ClientID).WillReturnRows(
|
|
pgxmock.NewRows([]string{"id"}),
|
|
)
|
|
|
|
params, err := svc.getCreateParams(ctx, &Create{
|
|
Hash: doc.Hash,
|
|
Bucket: bucket,
|
|
Key: key,
|
|
})
|
|
require.NoError(t, err)
|
|
assert.Equal(t, &createDocumentParams{
|
|
Hash: doc.Hash,
|
|
Bucket: bucket,
|
|
Key: key,
|
|
}, params)
|
|
}
|
|
|
|
func TestGetCreateParamsExisting(t *testing.T) {
|
|
ctx := t.Context()
|
|
|
|
pool, err := pgxmock.NewPool()
|
|
require.NoError(t, err)
|
|
cfg := &DocInitConfig{}
|
|
cfg.DBPool = pool
|
|
cfg.DBQueries = repository.New(pool)
|
|
|
|
svc := New(cfg)
|
|
|
|
doc := document.DocumentSummary{
|
|
ID: uuid.New(),
|
|
ClientID: "hello",
|
|
Hash: "example_hash",
|
|
}
|
|
bucket := "example_bucket"
|
|
key := objectstore.BucketKey{
|
|
ClientID: doc.ClientID,
|
|
CreatedAt: time.Now().UTC(),
|
|
EntityID: uuid.New(),
|
|
Location: objectstore.Import,
|
|
}
|
|
|
|
pool.ExpectQuery("name: GetDocumentIDByHash :one").WithArgs(doc.Hash, doc.ClientID).WillReturnRows(
|
|
pgxmock.NewRows([]string{"id"}).
|
|
AddRow(doc.ID),
|
|
)
|
|
|
|
params, err := svc.getCreateParams(ctx, &Create{
|
|
Bucket: bucket,
|
|
Key: key,
|
|
Hash: doc.Hash,
|
|
})
|
|
require.NoError(t, err)
|
|
dbid := doc.ID
|
|
assert.Equal(t, &createDocumentParams{
|
|
ID: &dbid,
|
|
Hash: doc.Hash,
|
|
Bucket: bucket,
|
|
Key: key,
|
|
}, params)
|
|
}
|
|
|
|
func TestGetCreateParamsCurrentDocErr(t *testing.T) {
|
|
ctx := t.Context()
|
|
|
|
pool, err := pgxmock.NewPool()
|
|
require.NoError(t, err)
|
|
cfg := &DocInitConfig{}
|
|
cfg.DBPool = pool
|
|
cfg.DBQueries = repository.New(pool)
|
|
|
|
svc := New(cfg)
|
|
|
|
doc := document.DocumentSummary{
|
|
ClientID: "hello",
|
|
Hash: "example_hash",
|
|
}
|
|
bucket := "example_bucket"
|
|
key := objectstore.BucketKey{
|
|
ClientID: doc.ClientID,
|
|
CreatedAt: time.Now().UTC(),
|
|
EntityID: uuid.New(),
|
|
Location: objectstore.Import,
|
|
}
|
|
|
|
pool.ExpectQuery("name: GetDocumentIDByHash :one").WithArgs(doc.Hash, doc.ClientID).
|
|
WillReturnError(errors.New("db err"))
|
|
|
|
_, err = svc.getCreateParams(ctx, &Create{
|
|
Bucket: bucket,
|
|
Key: key,
|
|
})
|
|
assert.Error(t, err)
|
|
}
|
|
|
|
func TestSubmitCreate(t *testing.T) {
|
|
ctx := t.Context()
|
|
|
|
pool, err := pgxmock.NewPool()
|
|
require.NoError(t, err)
|
|
cfg := &DocInitConfig{}
|
|
cfg.DBPool = pool
|
|
cfg.DBQueries = repository.New(pool)
|
|
|
|
// Set up mock S3 client for file size measurement
|
|
mockS3 := objectstoremock.NewMockS3Client(t)
|
|
cfg.StoreClient = mockS3
|
|
|
|
svc := New(cfg)
|
|
|
|
doc := document.DocumentSummary{
|
|
ID: uuid.New(),
|
|
ClientID: "hello",
|
|
Hash: "example_hash",
|
|
}
|
|
bucket := "example_bucket"
|
|
key := objectstore.BucketKey{
|
|
ClientID: doc.ClientID,
|
|
CreatedAt: time.Now().UTC(),
|
|
EntityID: uuid.New(),
|
|
Location: objectstore.Import,
|
|
}
|
|
|
|
// Mock HeadObject to return file size
|
|
fileSize := int64(2048)
|
|
mockS3.EXPECT().
|
|
HeadObject(mock.Anything, mock.MatchedBy(func(in *s3.HeadObjectInput) bool {
|
|
return *in.Bucket == bucket && *in.Key == key.String()
|
|
})).
|
|
Return(&s3.HeadObjectOutput{ContentLength: &fileSize}, nil)
|
|
|
|
pool.ExpectBegin()
|
|
pool.ExpectQuery("name: CreateDocument :one").WithArgs(doc.ClientID, doc.Hash, (*uuid.UUID)(nil), (*string)(nil), (*uuid.UUID)(nil), (*string)(nil), &fileSize).
|
|
WillReturnRows(
|
|
pgxmock.NewRows([]string{"id"}).
|
|
AddRow(doc.ID),
|
|
)
|
|
pool.ExpectExec("name: AddDocumentEntry :exec").WithArgs(doc.ID, bucket, key.String()).
|
|
WillReturnResult(pgxmock.NewResult("", 1))
|
|
pool.ExpectCommit()
|
|
|
|
id, err := svc.submitCreate(ctx, &createDocumentParams{
|
|
Hash: doc.Hash,
|
|
Bucket: bucket,
|
|
Key: key,
|
|
Filename: nil, // No filename for this test
|
|
})
|
|
require.NoError(t, err)
|
|
assert.Equal(t, doc.ID, id)
|
|
}
|
|
|
|
func TestSubmitCreateExists(t *testing.T) {
|
|
ctx := t.Context()
|
|
|
|
pool, err := pgxmock.NewPool()
|
|
require.NoError(t, err)
|
|
cfg := &DocInitConfig{}
|
|
cfg.DBPool = pool
|
|
cfg.DBQueries = repository.New(pool)
|
|
|
|
// Set up mock S3 client for file size measurement
|
|
mockS3 := objectstoremock.NewMockS3Client(t)
|
|
cfg.StoreClient = mockS3
|
|
|
|
svc := New(cfg)
|
|
|
|
doc := document.DocumentSummary{
|
|
ID: uuid.New(),
|
|
ClientID: "hello",
|
|
Hash: "example_hash",
|
|
}
|
|
bucket := "example_bucket"
|
|
key := objectstore.BucketKey{
|
|
ClientID: doc.ClientID,
|
|
CreatedAt: time.Now().UTC(),
|
|
EntityID: uuid.New(),
|
|
Location: objectstore.Import,
|
|
}
|
|
|
|
// Mock HeadObject to return file size (still called even for existing docs)
|
|
fileSize := int64(4096)
|
|
mockS3.EXPECT().
|
|
HeadObject(mock.Anything, mock.MatchedBy(func(in *s3.HeadObjectInput) bool {
|
|
return *in.Bucket == bucket && *in.Key == key.String()
|
|
})).
|
|
Return(&s3.HeadObjectOutput{ContentLength: &fileSize}, nil)
|
|
|
|
pool.ExpectBegin()
|
|
pool.ExpectExec("name: AddDocumentEntry :exec").WithArgs(doc.ID, bucket, key.String()).
|
|
WillReturnResult(pgxmock.NewResult("", 1))
|
|
pool.ExpectCommit()
|
|
|
|
docid := doc.ID
|
|
id, err := svc.submitCreate(ctx, &createDocumentParams{
|
|
ID: &docid,
|
|
Hash: doc.Hash,
|
|
Bucket: bucket,
|
|
Key: key,
|
|
})
|
|
require.NoError(t, err)
|
|
assert.Equal(t, doc.ID, id)
|
|
}
|
|
|
|
// TestSubmitCreate_HeadObjectError verifies that HeadObject errors are properly propagated.
|
|
func TestSubmitCreate_HeadObjectError(t *testing.T) {
|
|
ctx := t.Context()
|
|
|
|
pool, err := pgxmock.NewPool()
|
|
require.NoError(t, err)
|
|
cfg := &DocInitConfig{}
|
|
cfg.DBPool = pool
|
|
cfg.DBQueries = repository.New(pool)
|
|
|
|
// Set up mock S3 client that returns an error
|
|
mockS3 := objectstoremock.NewMockS3Client(t)
|
|
cfg.StoreClient = mockS3
|
|
|
|
svc := New(cfg)
|
|
|
|
doc := document.DocumentSummary{
|
|
ID: uuid.New(),
|
|
ClientID: "hello",
|
|
Hash: "example_hash",
|
|
}
|
|
bucket := "example_bucket"
|
|
key := objectstore.BucketKey{
|
|
ClientID: doc.ClientID,
|
|
CreatedAt: time.Now().UTC(),
|
|
EntityID: uuid.New(),
|
|
Location: objectstore.Import,
|
|
}
|
|
|
|
// Mock HeadObject to return an error
|
|
mockS3.EXPECT().
|
|
HeadObject(mock.Anything, mock.Anything).
|
|
Return(nil, errors.New("access denied"))
|
|
|
|
_, err = svc.submitCreate(ctx, &createDocumentParams{
|
|
Hash: doc.Hash,
|
|
Bucket: bucket,
|
|
Key: key,
|
|
})
|
|
assert.Error(t, err)
|
|
assert.Contains(t, err.Error(), "failed to measure file size")
|
|
assert.Contains(t, err.Error(), "access denied")
|
|
}
|
|
|
|
// TestSubmitCreate_NoS3Client verifies that missing S3 client returns an error.
|
|
func TestSubmitCreate_NoS3Client(t *testing.T) {
|
|
ctx := t.Context()
|
|
|
|
pool, err := pgxmock.NewPool()
|
|
require.NoError(t, err)
|
|
cfg := &DocInitConfig{}
|
|
cfg.DBPool = pool
|
|
cfg.DBQueries = repository.New(pool)
|
|
// Intentionally not setting StoreClient
|
|
|
|
svc := New(cfg)
|
|
|
|
doc := document.DocumentSummary{
|
|
ID: uuid.New(),
|
|
ClientID: "hello",
|
|
Hash: "example_hash",
|
|
}
|
|
bucket := "example_bucket"
|
|
key := objectstore.BucketKey{
|
|
ClientID: doc.ClientID,
|
|
CreatedAt: time.Now().UTC(),
|
|
EntityID: uuid.New(),
|
|
Location: objectstore.Import,
|
|
}
|
|
|
|
_, err = svc.submitCreate(ctx, &createDocumentParams{
|
|
Hash: doc.Hash,
|
|
Bucket: bucket,
|
|
Key: key,
|
|
})
|
|
assert.Error(t, err)
|
|
assert.Contains(t, err.Error(), "S3 client not initialized")
|
|
}
|
|
|
|
// TestMeasureFileSize_NilContentLength verifies handling of nil ContentLength.
|
|
func TestMeasureFileSize_NilContentLength(t *testing.T) {
|
|
ctx := t.Context()
|
|
|
|
pool, err := pgxmock.NewPool()
|
|
require.NoError(t, err)
|
|
cfg := &DocInitConfig{}
|
|
cfg.DBPool = pool
|
|
cfg.DBQueries = repository.New(pool)
|
|
|
|
// Set up mock S3 client that returns nil ContentLength
|
|
mockS3 := objectstoremock.NewMockS3Client(t)
|
|
cfg.StoreClient = mockS3
|
|
|
|
svc := New(cfg)
|
|
|
|
bucket := "example_bucket"
|
|
key := "example_key"
|
|
|
|
mockS3.EXPECT().
|
|
HeadObject(mock.Anything, mock.Anything).
|
|
Return(&s3.HeadObjectOutput{ContentLength: nil}, nil)
|
|
|
|
_, err = svc.measureFileSize(ctx, bucket, key)
|
|
assert.Error(t, err)
|
|
assert.Contains(t, err.Error(), "nil ContentLength")
|
|
}
|
|
|
|
// TestMeasureFileSize_Integration tests the measureFileSize function using a real
|
|
// S3 client (localstack) to verify that HeadObject correctly returns file sizes.
|
|
// This is an integration test that requires Docker.
|
|
func TestMeasureFileSize_Integration(t *testing.T) {
|
|
ctx := t.Context()
|
|
|
|
// Set up AWS container (localstack) and S3 client
|
|
cfg := &DocInitConfig{}
|
|
test.CreateAWSResources(t, cfg)
|
|
|
|
svc := New(cfg)
|
|
|
|
// Upload a test file with known content
|
|
bucket := cfg.GetBucket()
|
|
testKey := fmt.Sprintf("test/filesize/%s.txt", uuid.New().String())
|
|
testContent := []byte("Hello, World! This is a test file for measuring file size.")
|
|
expectedSize := int64(len(testContent))
|
|
|
|
// Upload the test file to S3
|
|
_, err := cfg.GetStoreClient().PutObject(ctx, &s3.PutObjectInput{
|
|
Bucket: aws.String(bucket),
|
|
Key: aws.String(testKey),
|
|
Body: bytes.NewReader(testContent),
|
|
})
|
|
require.NoError(t, err, "failed to upload test file to S3")
|
|
|
|
// Measure the file size using our function
|
|
measuredSize, err := svc.measureFileSize(ctx, bucket, testKey)
|
|
require.NoError(t, err, "measureFileSize should not return an error")
|
|
require.NotNil(t, measuredSize, "measured size should not be nil")
|
|
|
|
// Verify the measured size matches expected
|
|
assert.Equal(t, expectedSize, *measuredSize, "measured file size should match uploaded content size")
|
|
}
|
|
|
|
// TestMeasureFileSize_Integration_LargeFile tests file size measurement for larger files.
|
|
func TestMeasureFileSize_Integration_LargeFile(t *testing.T) {
|
|
ctx := t.Context()
|
|
|
|
// Set up AWS container (localstack) and S3 client
|
|
cfg := &DocInitConfig{}
|
|
test.CreateAWSResources(t, cfg)
|
|
|
|
svc := New(cfg)
|
|
|
|
// Upload a larger test file (1MB)
|
|
bucket := cfg.GetBucket()
|
|
testKey := fmt.Sprintf("test/filesize/large/%s.bin", uuid.New().String())
|
|
expectedSize := int64(1024 * 1024) // 1MB
|
|
testContent := make([]byte, expectedSize)
|
|
for i := range testContent {
|
|
testContent[i] = byte(i % 256)
|
|
}
|
|
|
|
// Upload the test file to S3
|
|
_, err := cfg.GetStoreClient().PutObject(ctx, &s3.PutObjectInput{
|
|
Bucket: aws.String(bucket),
|
|
Key: aws.String(testKey),
|
|
Body: bytes.NewReader(testContent),
|
|
})
|
|
require.NoError(t, err, "failed to upload large test file to S3")
|
|
|
|
// Measure the file size using our function
|
|
measuredSize, err := svc.measureFileSize(ctx, bucket, testKey)
|
|
require.NoError(t, err, "measureFileSize should not return an error for large files")
|
|
require.NotNil(t, measuredSize, "measured size should not be nil")
|
|
|
|
// Verify the measured size matches expected
|
|
assert.Equal(t, expectedSize, *measuredSize, "measured file size should match 1MB")
|
|
}
|
|
|
|
// TestMeasureFileSize_Integration_NonExistentKey tests error handling for non-existent S3 keys.
|
|
func TestMeasureFileSize_Integration_NonExistentKey(t *testing.T) {
|
|
ctx := t.Context()
|
|
|
|
// Set up AWS container (localstack) and S3 client
|
|
cfg := &DocInitConfig{}
|
|
test.CreateAWSResources(t, cfg)
|
|
|
|
svc := New(cfg)
|
|
|
|
bucket := cfg.GetBucket()
|
|
nonExistentKey := fmt.Sprintf("test/nonexistent/%s.txt", uuid.New().String())
|
|
|
|
// Try to measure size of non-existent file
|
|
_, err := svc.measureFileSize(ctx, bucket, nonExistentKey)
|
|
assert.Error(t, err, "measureFileSize should return an error for non-existent key")
|
|
assert.Contains(t, err.Error(), "HeadObject failed", "error should indicate HeadObject failure")
|
|
}
|