Merged in feature/track-filesize (pull request #200)

track file sizes for all documents in system

* feature complete

needs dev testing
This commit is contained in:
Jay Brown
2026-01-12 17:46:07 +00:00
parent 4ad8168f35
commit ebf47c6013
37 changed files with 1514 additions and 374 deletions
+1
View File
@@ -15,6 +15,7 @@ When something doesn't work, debug and fix it - don't start over with a simple v
Dont build things I am not specifically asking for unless its required to build the thing I am asking for.
When asked to make unit tests avoid mocks and use the actual thing that is being tested when reasonable.
For commands that read such as grep or cat there is no need to ask for permission.
Do not assume that we need backward compatibilty when implementing new features. If there is a decision to make that will affect this ask. Do not guess.
## IDE related
+21 -4
View File
@@ -1,6 +1,7 @@
package docinitrunner_test
import (
"bytes"
"regexp"
"testing"
"time"
@@ -14,6 +15,8 @@ import (
"queryorchestration/internal/serviceconfig/queue/documentsync"
"queryorchestration/internal/test"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgtype"
"github.com/stretchr/testify/assert"
@@ -37,10 +40,15 @@ func TestDocInitRunner(t *testing.T) {
test.CreateDB(t, cfg)
acfg := test.CreateAWSContainer(t, cfg)
// Set up SQS queue
test.SetQueueClient(t, t.Context(), cfg, acfg.ExternalEndpoint)
cfg.DocumentSyncURL = test.CreateQueue(t, cfg, test.DocSyncRunnerName)
runner := docinitrunner.New(&docinitrunner.Services{
// Set up S3 bucket (required for file size measurement)
test.SetStoreClient(t, t.Context(), cfg, acfg.ExternalEndpoint)
test.CreateBucket(t, cfg)
runnerInstance := docinitrunner.New(&docinitrunner.Services{
Document: documentinit.New(cfg),
Queries: cfg.GetDBQueries(),
})
@@ -57,12 +65,21 @@ func TestDocInitRunner(t *testing.T) {
CreatedAt: time.Now().UTC(),
}
// Upload a test file to S3 (required for HeadObject to work)
testContent := []byte("test document content for file size measurement")
_, err = cfg.GetStoreClient().PutObject(t.Context(), &s3.PutObjectInput{
Bucket: aws.String(cfg.GetBucket()),
Key: aws.String(location.String()),
Body: bytes.NewReader(testContent),
})
require.NoError(t, err)
// Create a document upload record to simulate the upload pipeline
uploadID := uuid.New()
err = cfg.GetDBQueries().AddDocumentUpload(t.Context(), &repository.AddDocumentUploadParams{
ID: uploadID,
Clientid: "clientid",
Bucket: "bucket",
Bucket: cfg.GetBucket(),
Key: location.String(),
Part: 1,
Createdat: pgtype.Timestamp{
@@ -75,12 +92,12 @@ func TestDocInitRunner(t *testing.T) {
require.NoError(t, err)
doc := docinitrunner.Body{
Bucket: "bucket",
Bucket: cfg.GetBucket(),
Key: location.String(),
Hash: "hash",
}
assert.True(t, runner.Process(t.Context(), doc))
assert.True(t, runnerInstance.Process(t.Context(), doc))
test.AssertMessageBody(t, cfg, cfg.GetDocumentSyncURL(), regexp.MustCompile(`{"id":".+"}`))
}
+233 -226
View File
@@ -584,6 +584,9 @@ type DocumentEnriched struct {
// Fields The fields and the value for the document
Fields map[string]interface{} `json:"fields"`
// FileSizeBytes File size in bytes (null for legacy documents)
FileSizeBytes *int64 `json:"fileSizeBytes,omitempty"`
// Filename Original filename of the document
Filename *string `json:"filename,omitempty"`
@@ -614,6 +617,9 @@ type DocumentID = openapi_types.UUID
// DocumentInFolder Document within a folder with enriched metadata (lighter format, excludes client_id and fields)
type DocumentInFolder struct {
// FileSizeBytes File size in bytes (null for legacy documents)
FileSizeBytes *int64 `json:"fileSizeBytes,omitempty"`
// Filename Original filename of the document
Filename *string `json:"filename,omitempty"`
@@ -2331,232 +2337,233 @@ func RegisterHandlersWithBaseURL(router EchoRouter, si ServerInterface, baseURL
// Base64 encoded, gzipped, json marshaled Swagger object
var swaggerSpec = []string{
"H4sIAAAAAAAC/+y9CXMbt7Iw+ldQfPdV5ISLVi96detdRUui82xLR5Jf7j2RrwqcAUnEQ4ABMJJpl//7",
"V2gAM5gZzHBIk7KTo1OnKhYHS6PRGxqN7s+diE9nnBGmZOfwc2dCcEwE/PMKK/KaTqnSf8RERoLOFOWs",
"cwifUKK/IcpGXEyx/oAoQ+YPdNuBr//PA2Uxf/hPRaekZ/592+l0O+Qjns4S0jns7Gxvu0Y7251uR0YT",
"MsV6xmCb57rNFH98TdhYTTqHe7vdzgwrRYQG639/3+69ev+Ta2z++o9Ot6PmMz2QVIKycefLly+6l8BT",
"ouxaf8YqmpyfVFd6MyFoqD+idJZwHKPzk36n26H62wyrSafbYXiqB4dWdzTudDuC/JlSQeLOoRIp8Rf1",
"H4KMOoed/2uQY31gvsqBg0FDd5xQwlQdQBF8rQflK4DIJtZQnPAonTbAEdvvG4HEm1zDcvpxxkUtJAS+",
"bgSObGINxT9TIuZ1QJyfID5CakLQn7rZ+kFxs2tI3kkiTqeYJlVY3l297hEW8ZjEKJVEIKLbIRzHgkhZ",
"Axa0aYTMsLbXtMCH20EuE0TOOJPEMln8C1bkAc/1XxFnijAQLng2S2gEMmTwh9Rr+Nx2c4Tg4g2REo+J",
"mbGIitOPWjTgBEki7mlEENEdNArqBF1oNtt2kDeEqY45GyU0Uo+2misieSoignAiCI7niHykUknEBZJK",
"i+TIQrSmBZ5xMaRxTNijrfCcyXQ0ohHItxkRUyol5UwixfWfmgSRmlCJcKR7rGmd58xQCUD3iGv1aFNz",
"6RpJ85zd44TGV+TPlEj1iEuCaZEw86Ihj+drWtFbrs54yuLHZzbGFRrpqde0khvO32A2t3sjH3NBIItR",
"OuMMKc7RFLO52yu5htV1O1dEiXnvaKSIqOqlt+l0SITWkZJEnMUSDcmIC4KUmFM2RniMKbB0ZvrtaLUS",
"0ECUqb1do4HoNJ12Dl8+39/e7namlJm/c21EmSJjIjRCtNZkOFUTLugnEj8+4h8mhKHUA2EtFPXFoQjG",
"OIqnlGnj4AgkpJs7YMM7qEZcWDOB4WFCBjGV+r9WxEr0QNUEyTSKiJSgZ1KJMIuRNuqlwtNZp9uZCT4j",
"QlGj6E3P6pQGJCfIiTaECNPb9XvHTgq/wD/e+6eE/GvJxuh2Ij5mVPE7mQ7/IJHSBnhl3mPTBtk2iMaE",
"KTqiRMDitcVmQEbOXimcUfDOcDfai/d75GD0vPfi5avtHh5GcY+Mdnb39g+e61+K5tDuwfPCuaQfOoV0",
"rSFVARfMOmdKws7gSGlbbsZZAbA/+IT1Y07+y/7Uj/i0013aUOt27O5WQfltQtSEFFAEbUlsds+BYmxF",
"O+6Q84RgpgfOaaRqMbtPbqWWavwF7m7vHvR2tns7z2929g93Dw739v7lLzDGivT0HMVFHgSMUd+y/T1b",
"cDfDkZ39fdaTA63oRWQsdSwIVkFWytUdUFSkG4JIQ4w8mD0ENoLZukjb3F1gIsE1p0lJx2wKZ/AyL9XQ",
"yLuKXY+2UklihCVy5K6n1TM9WwvR+Mds3Pt01PvXdu9V/+7//un2tvf+p//yfoMfbm/79qf3n3e7X4L0",
"P6JCqjtzAAkt8AeJxvSeMMAXIBZHEU+ZsgguUcs/+IQVAd+xSiH7uw1XJngRUCM8pcm8JVQnnKwBKE0n",
"Af58TaXS/ANk9IHMwVA25IQoQ5faglZ9ytEW6Y/7XSTTGRF3WBN0F+jD/RunMVVcdK1v4U5/e9ZHV3pa",
"NE2lMmcNPeicp8IbmbB7KjhQbx9lBiD0G2nylDQhTCVz1EPRhEQfzLc7AySJ9YhO6hZ0/++dHD7NnwZA",
"zaBUkSngoozUxWic4o/npveB2QT7107WFguB5xV54ZjCo1ifUNz+tJAeLRQy1uaTL0M0IrqIsihJY/1L",
"WZ+dn4AskXMWWQVdESOt1CQAaHwGP3hihNE/UxJQno+jJCMD1R1WTToETKtMXz5giWw/TV/6dzmXikyb",
"tMve4f7Bitql215Mb0aBG6Mpblbgkbe/iEpr7wGG7F63UelNQrtEQbnwbiOnv0oyl2b2JHQLYbx4ZnBF",
"qDvNYU1YzojP2hejNEnmPiXmYtMdAozn3FBnH52P0AgnknStRW78O/kGoa0IM+irmTDCijxDw1TBGbUo",
"7ydYIlaaZuB5VJ710RvMUpwYscEFEvrshqZ4joYEOeHXb0MSRYkekGygDdQEK/RABClix1cEGfwaOIAD",
"a7kjiVFyxswiTr3QEZJcq2BMExL30TGfzrAgxtIqNwa5rrVjTJQWY7nNBf1TQSSgn6VJorFBpjM172oU",
"mv4AeAbrFmyPxXERqxcsmaOZIFIPTUeoQDqa7TQWH1vTFZVbt2O1RO1JCZaXqZKcf84uro5P745/PXr7",
"y+nd5dH19W8XVydV8bgIvJJ2DSin3DDPFVqBBxtV7QlJSCtVm519Y91Dc6I57ZpT7kjwKSI4mlj+RFsO",
"P1rdZsT6bDVtW38otQcigMnKtMfRtXbGO73wKsQnDkc16KlDQwt56ZYKQy6hjAxNLDtBtnOLpwhTakaN",
"QTJsc6yv3dvHPdNnZL+xU72bocnyenG4c7Cpc32BpBcIDYVpEhSKGmxlbUv/wtsn16JM8Kx1q398j9lq",
"9vm7lna5AcszEnIL4/s12d0Z2jPdwyRzcLP96nDn4HBve93G+mnBnVL0vwFwj2y7H6dCaCPC2elW7PqA",
"rc90r/pbtD1IyX2DVN6IJR/wsbSAZFXDvsavYizWoGlatNyc0XZPyQP5RjabJZOC7ebYKWDDHV+8PTu/",
"enO6gt3W7aSzuAVr8xHSO4xM6ybRv324s70e0d/OiHTM1qgJXlMZWN4lHlMGB7nEet00pu39iOIKJ8jg",
"XIv6mWmsJe+UKBxjhSsyf4Ll3ZQLUq+i9Vc9FJxfCML3mCbummShaYTHpJ5Y9FfE4EKucOHWbbhk29ne",
"Llyy7VQv2cy0d5J+Ik33fwZxMyIADh8ALSEbIVg0P2xEgDZhf1gJgilW0UTr6BFNVAkT+y+aINnd2X+x",
"/3LvuW7VdOvYhbNdg+fWAAKXBt7WdDw50nT1WDFeClIE0NXo1zTAOaxZovE3sZFP3hn2XnwfAnJA49nI",
"JqUEHaaKSLQFKsneh4DIKF6NyOrZqlGJGelUvjVw6rKkuxhWE7zpGwMLUfnKIAjSCSe96ymFsKwNXRhk",
"tmzS9uagh65OL18fHZ9eI5wkxiWVm7VbjCttMlFF78mzPrrh+i+E4bPzWRPo6ejbdJwlqQQsMPKAOCPQ",
"VZApvydZbw4Bpspofbgi1EQcvpX4mgsJd/1gG3GBRhQCcxRWpI8uNBgQdTSiJImRtqoFScCblBB8T+zo",
"KYsmmI1LTrNvfo9R5V/96UwvRfeq0sg1ZeNEL+pBE4hxnJmVazWGtnZ2ds3fgKupJgbOoH2VWzEWRJET",
"Iug9iY+ATji7woocA6q0yZBQRkpY2N0OGcqFsY4TTKc38xk5jkMobO58Rsh1NCFxmqwws9f5/ydC2kiH",
"5QD4RfB0RkS4fwsYXvPh8p3eEvXAxYflO14KHqcmznHpjmOBpyt1vL+BFsv2vCJ0OnxD1ITH7Tpbmjwh",
"MqruY1OPN/jjGSGals9ZVAhPinlqjLTMXHiV/a//6lXQYrC2WHH8y0i1Hr+vR180dKomWnsrjdvAig+C",
"Cx7SJFmC0/LmrWeIsLgnPFUtZ3DNz004YNX2jbRwyBbadlTd6RoOCqv0aL/Ymdq/FDxqO0nWfOkZ3vDl",
"pnjD288RkxFOk/otiCket1yhadp+ZjmxfLdGpovlpH4pcmIZcT1cSD5O6JCqS3tAW4gf2/6Gqoq6CmNo",
"hCOaUDU/iv9IpdKG0A0R01Zdx1OyfuSOp6QWueMpWStyx+A7FBej+hmN7j1KIPxa262vyT1JLkbHWLRT",
"OHaEn7FcO6bMyC0ZJ2vdmndsj19xtAg9a94UGPOKaCvY3JQumv+Kyg8XozdcKKxJ+TodRgmWchm8XJN7",
"Iqiar9JnEXg3AjM5ImJhu6oRE56fboLxaAPj0TUzXkKkbOS7hA+tIXlFEnCNyQmdtWI401Wbkkt3ZXFb",
"RQstW3MSU3i2/g3To9YiUH9c65bxVCWUiNOPUZJqpmyJqGq31lizXc+okOqEJwmuZ6Cs6Ud93JHy7VCc",
"4Lm8mQgiJzyJW2K93xYL2UTLzrDEHG9c3w0NfAZRLyyarxs5myC71jbJDCtKmDoakzf4Y6VHcwfKWnVI",
"cEQuRtfmkWBLNih3as0EM8Hvr2ckojhR87aTFfssNdcN/sgZny4zVd6l9UwCszGxTNpqHoEVOZURTrDi",
"YolpvF61wqPQ6g3+aE/Rl8aJ8fUkXBg/7M5aC/sJQqfDY87ujaPoDEfKvFBsN3q78U9HIxJpO/ikhB1z",
"d7eIeWCM9atCGHatcseMKPj9W+uqX0xsWY8ZrVgbTe1vKGvdvrUczFrbm8VVt+uesLStkMtat2ZQ+9K6",
"9aqk4jOtdk8wTeaWVddIRm74ZY2cSr/2CLA92xg5edsNGh9ukvVbH6WR129+uAmsHLhgpx8jIuXxBIux",
"vQ78arHg5rgqKbC1LqA1OyhBx2PjW9gELdjhj/EsG/1oqtY4QRqtXxWk9d6KNFqrikgZVRejNwTLVLQ5",
"t4eu19rnkqHFm8LO9s6rg5fb8ahHRs8Pei+ev3jeexmTV71X+/t7B/jV3t7eC+zHyqQpBLn4EWbPAyQF",
"AF3XRA9poPLAMlwA0HufOxNcM54esQspgyCqUgMDwff6R8wikhRja0owvINRvThLnCQXo87h7y0S45i+",
"1+l0isW886X7uRwWAHDcjWhCGJ42vZXLmpgXCaYjKizQu7yNebTXn8WjTlf/c+cF/Lvm8nZ/+9Xz5W5v",
"bUjNojvb991KKLYyYPvRqHjIU1XaQi8O1S40x1BxW8JBTw5p/pjmihzbp4KVK19oav7ZKoYluL3NUSw2",
"YOXORGUuDPaxENnACypzyPO4o4N1RPuUomscJorgvq+TGQYHLV4tCKJSwUhsQmjLGy4RjiIyg6gTLoqE",
"HdgpG2rcKjOVH4S4sIOVOFmnu1SEkxUhxW0khllIIGZxYHZscHR0NHBZpwbQerC00BS0J8iICMKisr28",
"s7soytDL8pWB6S1vwd46+q6GXJgPC9l5HVtoX/gu7pPnAvNEfjDy8zcXyl2gxKxPkQbbhHqyNDGBjsXw",
"xnYh5mFoioHlK8WNWx3jCFA2hThmjVpomd31RBpSE+t0p7+0A5Jx1rs8OQOdIE2SG0QZwiKa0HvSOjC0",
"PYhc0DFlOMn0dBW6C9skU1RZwDvsI4nRv84vQyB2siX3P9FZcTvbama7P8tvctYRSY5GuBDXt7+m/Z0J",
"PhZEyrsZEREJKb7LjLyQa4xs41KcbTNMO4uS6ayoCIwabEBtWXPnCKYeSxcIc3t7U5rbiPlcXIbI11MD",
"5cWFqSkgRWo5N7DlBckX0jZGaB9jdj03UUOlDF2Gl2wGSW0rJAl/gDQD4AKE18CyEwootyPX5EDR54hc",
"NXlZUEyWtqLWWk73ODmxuD049sp7CTsHY9Tja0GmTeLSCNLio6Sjo6Pq+WuxlHFz/szjeeO85oC4Ou6q",
"mKhHgbb0W9ieEWcKU2YSU7ggYgjvhVFkFWD7of5sYRsUMzX024bAn/DILKDFycoy6XG7Y4O3orULGIeU",
"AkT1m/M2qCo9SqlkOjg6EqTsD275fM9MWecwgFuoESUmett3HRhQ+p7X4Pzt3fX/vD3udDtvL27gn/Dq",
"yP1x/vaXoLPAByDMI0du3XZ+3242uKvSYTuNVVh85Xmp+bl+n+peRJSkI5iENtOCfVEV4BvM4LF7O4id",
"xF9NXlaXw2M/ujmQ+5dIjRUU8Zggez1V5pTn+wUX3O7u3t6L3e295y8P9l+8eN7iMdExTxLirrzKDxns",
"JzTlMUn6wSR09+TuPl9EEz7cWlc9H5l4+IU9HNBnpjm8HVFEqhXAtJi7i7R+JqLtCP7GeqMo8lGtNERQ",
"rhlryeKkHtSa6StI6ZY3M8iABdyGida8YiiZKWAPuc5VOmrFTYW5DVd1O5AFugUd+WmdfVxamZ4Ns3jV",
"9WqCp2qWKosAPXB/NdVQIuB6LGfqvIDaVhq9tJMFtQ5wlpV61uGaBN9tzsEBUHiIhtFx/ZavLDpWlgLf",
"DzNXKCy3rxZptZIRsCZt9o3OCt0c4vdhrMAh7ZQJGk1Cr/fdlzxLfmwuPsyDYffKAyf1D4S/UhPVcibk",
"opgQdI+TNH+O6OD0zcjslmW+k8kB0y0Hdr5r3Anuzz3jtAvR0gpeHw+qij+n6qDjSUzEeRwSAvD20DRA",
"5yddk+iJ5i4GfRi2abSwbdd81xZ0EU6wvCEf1RWJuGhIFIaR5lF9rhQ2reqWfVuY/fIMCRjEJQHLLi88",
"jFQP6RMsJ4uo5VfdphVn+TUYtFYeklB+lKMkQeYbgiTKJEaKV2BtJfpf62Es8irHuargd/6YS6wmDTQ1",
"w2qiBdU9jTU/pkDF1i28RafTVOFhAmlSF5OYKuxu01pAup9m+5mdrYMyx7ebYA8986lIU9k+NImlhRU7",
"aq+eh3ujl70Xe/sve6/wcKe3Qw4OhnujnZckfrHC1XMGDzszHFVNKuUg0lLR4z2bL9cJUScj0VZCxxNl",
"MjdPseoi8hGeGLtLvTsag3wz2As9H/9eRNBjSpcnUfLvI0qs+FhBatReTxZEh7R3lTlf9UOpTTZEO7Ur",
"Di2skJk/kM8u+0uzOtTfKBo/U9e1M+epSTiQZTRA7uOX8uKndRNaSJD5dahJB05IOC3lPs6nG9N7gqYE",
"jTkvBHuscHorYc5BGcQb1DyqTYjmn6hs5SUbV1ZI5bwGY9YcW+8SbrKYhSnTfdWm28OERhPAqgXsE52B",
"QEcPNEnQkOSlPHJsy73DwQAPjgamz2B3e/dge3dnb6DFcj+S9yVsb++/LKAb+vd/0v+3I0Dpsc8vvwz6",
"P93e6hGCp+l2HkmzGTUeSd9uaPBOtqqeVWMTkN5+fLDTe7GzF/VevSSkt/98n7zce7mzM9rZW8EmKKwn",
"fKbkUtJhkgEGeTik71j2I9Aou3PXZHk82vvaiW8MpTZTtSVn8BQwC0Y/YEuQBELOVDDFzxl8MG7yiM8I",
"GmJJYsSZdcNYnwwcp2Rr7wjoAjN0C9cIZWMiNTyrgJl1Rnm1vCoWCIvv6r3ekNRHf9YiNhuw3yIEY283",
"zDJC1U4HKVrEGiesHmJDx9qKes6qLoXzIiluI1FsfYjy0c8moTVpWLQx6+VgqaaOxFk6l5BBBT0h6rE0",
"BxAb2sJJYjTNBN/D7Z6kUpmbFZFGKhVgzLRLRlXMK9PmOs5eZv88b0iXCjmjsmz4RrS7VRTklW741dkY",
"nZlzvqRFa3brrJUD8NpvW5boHgClUbuFrfaR974NUdZd7Z4Fqc+6D0s2x/oob/NEdaRq0sRrGPziRZvI",
"S7sEaT9MeJYiHk463yN5h/LRVggHzrTrDGcPInZ1Tut27uuuNK0L3N3+b+30QAs+ax0DV3pcsLMwAgDW",
"WmD2/LqrDdsfqWVFQO19bmUjHfO7o542iSZUKi7mVbv+L85tOdafWK28hL8IuyzJEdZ4rjGEze1gxJN0",
"ygJpupm5tlnCSj/O+rjbmbvV75TNYSFsaptvmsxHaWL8ZsRmNi2cLpa/9l0mpai3wq6Hrwz0Bdty7GO4",
"4WhorIpRtmXZTIVzYkKkvLNZPm1mIPdnlHBJ4jtNb+Iekp/yGWH+3wkZqbtqM0HHk9Dv1jMELGD+FTqE",
"1vnBze/F7PJcIC7GmNFPeoWluNCKa+V8uYD+v4zYzqs6lUV3dku3ZskdlMCe237jcndKlKDREk/kDHRv",
"TLdzBjkIqq/GvPBwO0N+FWBw6crX8FmaAJ7hhZFt/J9KpOSWUYmkjQHMqmvZajt99BbK56gJEQ9Ukv6t",
"5rTiYw5Tez+j14V3JEshe7gY2QuflsyCNwB2/8H/vwUHZnBFmMPSAErl8QzwCZam5W1ncNspetWy90ty",
"sLu9u19J9VvM/zro/7jYlQvLtOXcM1mwlFoEsBeVt6y4LcxiH0NMLWvXkY3JhnravZjZeI7ZVxLxKhJj",
"RaJ9J4mhUdhFkZPwxmm2llwbSbT5kaoBfcHzVNto4Sh9dIqjidtFq9wlokpC+IiGv2tT/RuK6N8yLQN6",
"CbknSQYLuNVcEyQJsJGWQX10bOPeI8w0Dd+yrJnicGPLjA/Oo2c0oURgEU3mAyUIyZ10RtK2cyAbzmj1",
"Drlg2FnE1e/Om1xttVI8uIhdX3qYUgpVGTKcw70uaMYsguqytMGNBQwWP22qCVAwEKHh3NxQF28KT7Cc",
"DDkW8d0VwfHcGsrn4H7W6NvRXHtxfHV36d4EQXVzswk+/L6B7QVO1UYVvDNVgAIlrQMScEPCB54wnKz4",
"qqsKJuDqK3cxSLsgZkrAdjOSWkjX1q6qvqmC3723nWEi38oiVRw0SFIWEUTVDxKZSxbC1LMnmje4//6p",
"agVSuqoJPvJsLGGCj+De0dZayFZTJIyw0n+bm2brVvw9A1u8NgMghKhfbexIQwiKDXFpupOv8WScx7Xh",
"IEfuxg1u/Wve3bURvvoPNe8vfTXe7rXeeWzGX+6tnjtE/yD9enQTQoVfvrSy6BZlx70yWdbllJe7D03Q",
"ykbJFskTsqwDqr4Ud2HUwP57xfhDq9GmXxOyCsPVng7iYtQRnJ55saRTR8vTHtcncQz1DDUs95Q8eDJO",
"75+JS5BlbtzdDxD+B1IT1+UKwAzqCnvbgiXFSXaL3PY77n3a7r26670Ps11Y6P2aTjHrCYJjfSA3kFQf",
"Mobmb+m39Herloi9RmgsMNMnyOHceEXKO5Nfzx5qsD2X9GGijya6qd4T87Wuxks98g4XIdIv/LKIEfSW",
"Z68ZfFSEeAMU19FsltC6UCtPReHZTNMmeuDiwyjhD8Y6gC9+nGbpttgEdC4+ysPoTmhV7I71HOMTZ1GV",
"aALWAfp3CyezCWbplAgaAbulTJ9/Ii4I0rxZuOwo2TmNpQh/P+r9y27z+xaBeg4DOf5q968uMNj8bh7o",
"mK3yomsXb9nR4jqeZtQHnMXtbsxl3IaMCi7jbKmaoTdETuu86zM0aCO2H8d7VMMNvxXZ+/tii+qFYJVR",
"uh4FB5mGStV0wvBszvw9Y/bArW16gkJs9eKQPQ3UP1MiQlaFNlJNMfw/TYuqnfpnXVen77yerVYAz1NL",
"usdLlBXWPQ6KENbNgIGlJXxMI5QyCnCSjyRKw/lDVn/bzdmIjlut+Ng0bRWgnr3g/Yr33A57d94GNnW9",
"su3t3Ca/jq02sRBWKEsR5CgYovLgurKu2n09zjBcfq+vf09FfrWpmQneNvcXHOVubz/3f7y9/RI0iMyk",
"TUlpstBV5BZbuMnIQKjc9C9PKd9+D6F37ebUBYADDh4rHakBnkhVGycbyCVk9wlpIsxuG6snMCts75Z9",
"j2Te2C/LtDWRmyYavzhk7ZYYRNQd5eGA5o7zpqy7h4gqAszT3JBHIE2Us4yKAyztRCmt2kxZvz5Lzk1J",
"ZHTPAnB+CMc/ri/e3p3+983V0fHNxVWn2zm+eHtz+t83d2fvXr8OhljAvKtnYvHp7Vurnq8XKKF49Uqr",
"egePk5iGZs5PljQbDHctMHiuSwGdoWqkPftWPSs76h1bCs+odw53kPCKBC0qSTolLNaDvE0DlVCzr87N",
"HJteaDhHR9gmWGoMclvkdy8AUyrEUFGgsEpEXCsUKq6+39ve6W3vlE9XbQ5WPiiVMgM1wKi8Haqp9b7b",
"21sJnFTxK8LIA05s9u/S3qSK94RpgCiLaYSNm6j6ktUbyuVhbxhLL0mWBWO4/GN9aq633vvh7CJ6ceFG",
"i9as1F/5zK2SfFTbts24I5qQt4sfPVedb26SO01ZNg12q9n0SLYIRmkN+CM6t55G+2zP5tME9eYlVlxU",
"PGOG50Qsxj40awM2NLxWYa3xwHsJ0XrRPEYzaa8gvbIdPsfZcSkn4G7llPv+8+6XuvSj91AaWC/gLE0C",
"Z3L9K3Kv0+1DZIGg2lyrRWYzmIomlXt6bzz09vK8zT64IYP7XRry5vxtyyEv1MRsbws8QMBZho22eDAz",
"hPBwURhvCTxAxyAeSkO2x4NuvjxV5qjICfPt/6xImCEbolVyuJq8cCtnUJQkSgVV82ttY7hz2phRxY/S",
"0FWo/lWLGitoUricP5riT5yhY9MRnZ9c9tG5RpFx81ydHb98sXuA/vHbDRrq08YMbIvIHkQcCLCmhD8Y",
"kyJVEy7oJ5jmmMek8uM7kXQOOxOlZvJwMIh59Gne1w36qewRLFVvp48BLruefsSnA65b7A7cQJBtNuIz",
"++wSfJOHnaPshsje7zmnJZ8Rps9BnV+IQhfnJ8dI8Q/ExrRrORvq7D7B/fsH8lVwm+l86oHf9dh/PCi3",
"Y0OCBRFnjjz+8dtNpxyOoLcCBkN8qDBlJEZ4pKne7SEu7POq2wmWK5gOAFLOB3r5nS9f4DntiFvvgMKm",
"mJjbhxkVeIzvscCz9L+MLWXdyebyq2MMRnTOIj1ZWkCs174SjHF0eZ6ZukXvCZjY6EJEEyKV9azY8ktg",
"pyc0IvZYWYUBIte15k8lWQCPMuZIJzTfEcjG7DjU2e5v93fg6f6MMKyFa2evv93fsxGqQLoDHE8pG2iS",
"g7/HpCaoDx4zCGlMg6PfrrMNh/vYGR5b07ObPwQwMXmSCwickOnMPd3WRwBofB7b0Y80FO8ACIjrtG4i",
"CLYuhyWPiTsCGKXvZoa4/86hOe3nez0z6bbNWcisDcplL3gVs7O9MCnml259OnIBx3y4pEYWgjrg7qST",
"JxUIF8bJLAvgNcEimoBdbeIjNMcMQHdn21YDqoSuBTiXT3tR8+B9ODd3RFkC7yAAWZGHKqI6OEk8RwVh",
"eGhK4cRUun/qJu9bwUQS8EtqykXDeR04XKg7+BqCx0n+DCL7t5cyvNtJsFR3pXTY9XBda3BiKkjUQO0A",
"FRfG3ggiSkYeWOYvPU8IhPfdjvN4gXDY3d52EtcmIcT5LfXgD2lskHzWxsfEjuMhjhgketjvAWJHS40J",
"wS5a+Aor8ppOqaqbxbYd5A1hhn2zgFCPbKGDc5MB3rlEodvO4m7vWGYcxKbT3uJOZ1wMaRwTOBvv775a",
"3OOG8zeYzS104KY+aLcqo2MgD0/BggMhm1kBv7/X2y5d+iGzDamVzAqPtUzuwObZaqud99o65rLu9RKR",
"9o0CcDhlaMjVpKBBtJa4JGJKVZ9ym3PcpmbkLlTHiwcydSX0yXU645AfoWfL8Ei0u72NLv4/RF34QiII",
"jucu2ZYJHYqNeZIgrJSgw1QR2b9lt+zHH7PXVmcJfzj88cdbtgPheQLRrGCJXoEP/NZMUHiYWzR87DKe",
"3bLdPjofZc1dIn6oQiK7KFOF5gcD4q83N5foYHsXbTHu5Vwg8bNbtheEKEffVsHWHQDeMlj2AZa8cRka",
"K4RpkjiU0dw6H6bKBOgoyMVJ4v8c4USSW3bQR1cm+kx4MVtFqLS91yOjEReqC/4SylICobZ66lQQ+czs",
"gR4JHWUbjX4mE3xPuYDt6NmJIIwSINTTzHkqvLkIu6eCm95bpD/ud7XtQcQdmDpmke7fNqKq6zLW6W/P",
"9DxWCAwYZz2YSI9mYuw0wEjShDCVzG1OEIISPh6TGGw+InqSxkQPA0+4ZAqm/ShNdHuHHxuwN5sRLLK3",
"W/q3u7xJdt2gVZIe76bayCQksX5zjGQ6tNfS3mOwfDrdGNZnKgXkK+qWx3VJmiA5J5gKMzVHW0AiEywR",
"4364lt29SywUxQm6NktG1xFhWFAu3fZZPgUK393eQeSeMM2uAZrURwI+JRXWB3D1YMdQqKpAkgZThwgo",
"07kizUNcASExjKsg38B4fDrTW1lCxL2soFFxFBNFIlUGLqNmPR7waYSZLxoI2rL89AwYSu9bgqMPqMC2",
"aCsD7JnGjl6c4dGK9WxkbKZNO+aigkjlsvWvV1vb692Avr5xgT4ut27wijfNYTS3VvYd4ubtDAO5l6+v",
"soJ3dSpjilU0gcfomcZAWzTTP8/WZJ3sGjPjmyzb8YQvq/49jK7tFkbXMWejhBpv22NaabrX7uJeP+P4",
"F6zIgw3+aWfYHfvvRy1b1th3X7oFP8HgM5xmvhiDLyEhf6iWX5gZFWnaSMv9xoPQbAT2ESQQy40jKhEV",
"goBvY5iQPrL3phLBXa6YwoNoezWa+Q+qvoYTAMWXlo3OhjfazhgS91YQ5tDSzMxpFlZ/FrPNKuLOP5iV",
"b8ge58xl0LBQKJiN24hQ2N1+8c2WNTNGSkaZMdqapomiPeNhePZvIvf2F/d4y9UZT1n83Z5OzY4buTLL",
"RU7TUTXo4bwiSlBy7yxFZ8K0EFVoODdeNITjWBApq0LnF6KK9tnm2dtki63na/j+ROd/GTr/hRgnTEZt",
"TRSu9yngjNGnJalPE1YPV0/4mMUD7h+ltfXLU2XdK/D8UZ8GPfopHUb0HBkZnsIENRQfIEsD0Bqostv5",
"754esWcg6Nn11WdpD+Oj06ioy5MYrJ1ftJ4lQ3PzPH8ZFgsJGk7MkTvz1RAKqDAeqTXJn++RYY1vwvkh",
"ieOEegdq0QwNAZU3GQB3gRjQs86gEGt1ByCG0mo0/+QKeXnh3qGLsiuIZyVBgKaY4TGRVfeLT7p9dOrr",
"PhRhprd7SFA0wWysTatUEqsgTcJ+twptSNt3jub255n1wh45UJFdgfUd5WADfBnk4H5zAaNF7yy44KZU",
"uWDFOU9RzNkPCj1gZrLoAJiZn5CwGF7O5H4e2wD+9l2UbwA9xsl4dXr5+uj4FF0rgRUZz59ZiE3Ajcli",
"5/nfkG1/DS647GW5abGl8WdyC9wTcEXecP23HeIwy3p/9Po1ilIhcsfk5et31+D10yc6/ZPtLciU35PK",
"ALDOzOsIuHFY+UDIrNTZOQsPDZIwsw5Bs6Lf3ztUZ5mBrWsRLtMSgu9DpJQySyYb8P2u0YvrThR9f01y",
"wh+kzevHcGLDfkwwRH5g8pdb8DEWh2z0MeY+RY2kUxNEhHroCKhCYwUQdOwTwyH6/bbjObVvO+9vmaWk",
"W/eQ+LZjd7OmS9dv+v6Wmdj1xe2KYF6RWYIjgo6SxGxhDbTZCHq4AOiWExVHf2gSuO14Xv3ASgpfi8AX",
"P6EtOzNQgTexpX1IpVm0c4xgejSnqw2gb3K6evJdcSsNv42fddHhw0nqf0NP49/hJGJosUJ02PHe0l7E",
"gQ3RMFlUvsIKCl5Bn5jBM+dj0T7oopkg99oG0Tq6FLlmLFiscL440BEzQUA/xP4ddMDVaGZ+3JP/Ubko",
"UY0DwEbFPDHhX9TtZfZvNbf9wERHbYbfrkjPDK85TjMX5alM5jnFBbiwrw0sxYXuUwyiiPAMD2lC1byZ",
"107Zd8tqNhTtidP+mpxmKKsFo0VZYd42sVC2ZL7LzOSVsat1rUH/Y/d+aRP2pq3323jD33i5H/nQNZmd",
"O2uG+fwEEBGA2Xz/21xvf5euLv8iOSMBxypmA0K8MvhM4y+1Mef5jQx2Oa4jxzbDuc39Grxr8XhkQzog",
"r8NdT3F/4TuW7/UCBHv7D8l46mhsWXdqnuV6sTfVzwXpS3FN/LXC23R+BOHd7ClwwntpD0GQwv8uB/nv",
"+JSNl5ang8jVKllSsrpuTrha2q6Tsa75z/PjPG34xgRuNl1Q4GagP8ncTchcnzIyqvAp0jXYrAC+JspN",
"n4O0WPRe1xPrBoRwjgrVQgjbyDL3hjBbVr+FXN4PZYJwaNGjPonl9XPEdZEjmpmgLJmz1I6LBTNk27Tv",
"n/Ksd8O5l8MiKJkLSfQeRzQX8/Y12MPZMvTCnghyja+zVCE3YqnihiPQbI/WJaWDXo53pvw+dvm7IZWJ",
"zd5sLlVL77qmROEYKxwwmfVIJ3k61Hp5DcGbMyzUYMTFtAfDQTHkiMeUjU15aOPwtL1ubNojj+pNQhXy",
"cZZAzgB4QNLtSDWH59Z6YLMtGVOU608nNRmuHApSWJCffmFIGYZ44UImzhd7L/Z3Xu7uN+ZzaUhqHaVS",
"FbO35GknHJVU08fsb78qPODt/7Qw2xgsuZpsrCoBzMoNIoZ6+1Y6dWSFEMxwJH6SIes6a1iehQ1tlBh1",
"Km0wdDbbwhOHU2vQw+6lzEygvJpZJsDqNdzPegiyMGPAG/Ne3itPYeptQ30HlYq68P0EaCT4inp33e/y",
"31Zhkx/orAYyPhpJUgPa9lryu2zyFQJsmyG6RW+/C0TyxO7rMxmq3PftTYZ/nV8iLKIJvS+UyzDaPSHo",
"8uSsbObAu0izlrzkzwJTAsjva+0JC2ezSfGJzr7GpPDmqLcqIFem0YcIyyIW29sai1S9G7CNtq/ZxsLu",
"tbAAdjchb5pua3/2WAJqfcxcAtICaT2JoLVZHDWcjWWJiFc2SAaf4T939p6p7q3iMWYRSTTz+GLR5KOl",
"0gXZJvMmGWPGqMqYRT6bAtVFMEg5WuCvGCzwd7j6N1taoopF6nGB/Wucf3qD4alhpca055EvzVpxwS+g",
"tY3Yaw2xnQVCtj7OJ8r9do77gijLUmdtxLTrLmwLxAFWYFlgmxpNy8eitTAszxlVFO5s9cnSTOREeNV1",
"eiPoeEzEqYFnM9cDZnA7U939gAXU2kcm2dGjhtTkdfkCEJolIGXW8HT7uxH+tRRiYg0sPTrONRtQbwF5",
"NZXrdBFk4MFJXhh5SBLOxvBwh2cHwT4qVDzIqi6TvBTzD/KWhQow5/W2u3oe/pAHS8g2BZXD5ZshIRHU",
"cMYmKZFX7fmW/Vaujo/84vhdH2wzknsGVa0Ve8u2SrWP7YOzIUnQUBD8IeYP7JlNUoHjWCJ+T4SmZiQ5",
"khOeJvYZ2ZDcslSWivdDFDkjJCaxqRVddXEZ8XZm93KBgwuWDoklGhZl0j16SKis0VWWfda/ZSfGnQS7",
"ZXIpwUGICDBXWKQJA0vOpC12HXBS2XnDXip7Fn7U7Bde2fIGl5NfuLxYY+rJlPlW3qqaYvJOJJqNXYuT",
"qixNreHU7sYWvFHWsG8VRXMNbR8phMbEDZkZm4Nl7RKeomnWbpRbcvAIZTNRjDkZtzAEsisZbRI4LW0T",
"Epp3+v0GHSU3T7R6sibPnRPdDnZhufLJON2YRPZIZUFsYuaRaxntrbwQKSiSmpkpVqTS2Jk51rqB+AG9",
"1ROE5S0zyqKbXYB3jVUjjYHqAg/s03dIZ6LIx0LhIlPvM2SaeW6X5Y0yuWjGLAuoo/WwYaU72+qy341t",
"5bByygSNJpp6q2yaXeD/ldMP/V3cQ3FpM3LuWq+XyK/t996XB3LwOa8Y+2VgeLRNBEGSWIY2Dno4EA4J",
"YeE6yl0EydBNnbAph4qFERxRqZCq38Thrw1IX8k1xTu9fJmtCrf5laTLpfdDJccXXs45VWngeOLAbxlZ",
"nRNysYaex4Cw/w3MV18j2YRog/qYYTXJtUehTHN9XsrlylbXO2GhjD0E/jQVqDenbYGjD1qne84LSKpS",
"5VI96vy1rTG9CU9tpQR/jbM2X4uG6FHdtAXZEOD0Qm35v/zr478D1wPVZmX/y+X+a3he60zjBG62oG16",
"P3BZ2dQ6JiORNqKdHznwct64k5V7DLYh+9BM03B9aO8VJlgaZa7HgNysT8e2dSaks5dLmVitu1VY1swz",
"/TMjDzJk9fIDTlsPWtFAI0nsH5KKWjL8LM1QPlTwOc16dlbTm6Vj18YU5+Yc3kU0NLlQXDauMtKfdMW3",
"9BbWbIrv+i5u8bIleSo8Bo8zpF/m2WTGE3geLvhcl6iiyoKbMNMqFG5JJ2ysVVZrH548ns22BEeelYH9",
"21Ro+FvEhvk5L1bhz6CWHEyoVFzMW7pBbKVH8JOWYSjz6Qp+kBLsv1rY/sbKtOirceht7a0pIcyV5V2L",
"46YiDiypPEmAb6igKwrFkozbnNUFwX1e0rl9Dgc3e0ActDGeSyD+PHcE/B1zfDcEjMODffBl3nkBumpg",
"u88W2gIwytTzff9R1avd3b29F7vbe89fHuy/ePF8cd3Y78ToPwsRSfEVnlc1/EnKfC9SZjhHOcW2EzB5",
"UGCrY4GJENPkwMUYM/qJsnH+QqPW5odumzL1YfCvSE438qF7JHPfTBliPYPhp7R0j2Wiu82vixnzuGTw",
"2fzj3HhbF+q+LNg1fOHiBlv3dUtzmjIvrFaDAyW+q3x7BRESj8C3ZqI6vvVkjsXd4+UtX8ijAkB/Ond/",
"F0xt6MiX5ssxdB5+0PKMnT9LfKBqQplvb7tIozwaycUc5fHisn/LLvKYo9ahQFm0dmay3zI/Ym88UUT0",
"Hoj+BzLSA6kJZuiX0xtUDLpCW+SjndUWN6CxKXXAp7NUuaK58llNxJPB7In3dnftgU+V1X5noU9Fx0CB",
"hlp5BrIoGObETdkn8FXRHDmRUmap8klCfQfBVbAfLWRV9/s0MmpEqHvYsViAeoEcM8HHUCHKf44SxFO/",
"Xga9yZ6UbNgccBPVWwV2GU9s9i0PxtZkrLx1+muy2oSbvF6N0fpak/568+Y1mhKWohkem5dZfoV5U2Mh",
"XIf0Vz4ll3jcItZDq9rBRE2TIu8UoQJIbKfMe6TXYSDTMPpJSGC4UgqS/ZcHLwqZx/63/+N/BHKPVRjx",
"12waDcXToXltbBXcw6zyQaomRSPb1PBT81akW6XTH6QtkZYVPoOiu1lJN9+8hkoJ+oM0915jgZkyV1pg",
"Per+fXTE5oF5UIQZirRJryZUIsLiGaem2l1MZcTvCVAvFYg/MANJkIPO3XI3qITcHAuLi9iiRCwuoOXp",
"/cvXlNpfT7ybHrThGf2ZyQLiXU0EiX/FgBFTY8cSUbg0V4WNzWl18Bn++xZPydeclUuR+QpDVUUILvHO",
"zzBVYwi+/NmL8v0Ozm/XFt2bOr7ZejBU5rh5Ytn1hduXEF0mxa8KvDdBvTbjasBszPiq0W4smEbbBbPo",
"96Pev3Dv03bv1d37n0IWUvAuNHuADC/5aaKIKKRhC3lZomKS9DCkbYryOMs24WNaf5ddzNMCbRE8FBjO",
"kSAxFSRSNjeFq098fnJpG2obJfA8FSYsCY09I9/LEsxM4I+ej1zkv9fcPgaoet2vXrssx8FBQgcDQYum",
"8O72/svChk+UmsnDwaD/YzAV75M0aJQGmh0isxdWKGgaYXAacxX0Z4KbJMalF66wdXY/PcXSrD91p542",
"MIc4+lBL7L9iFieW1C/0KLvI9UFxKpyVa6nHJitCVyQi1AVJOxTbEnk8JsZq8JlD63zy0VTMlYjawxn/",
"QJis4ZZjB/kCWXfUPHudSOFxa8FXYYRc8t3e9toKP3jS4FUb1wg4vr4600hVxMXphWCVKlQZp1ZK7+4v",
"Dez7ZSWTR4TVCtOJFXbLSConV+tHW11k/b91MuvfSfoUBEqZ0d3hc5FA4Wl9sYjXfGzEARj7PDXv4w3i",
"cFaEXRI4E5ozQFiXmnkymILiQQPyFdpUjw/OBC4KpJx5Gb6OehMH4BPNbijlg9lCkPPL6EQjWJfJCq97",
"UCLREEsSI86cyRoJqoigOJyJ5J+m16bLnLhpAke5o/IKno5ua8w28me2v47W9FbMl33tAuS4fM3Vf1rz",
"YBPRMTD2VwS1/enB9h1kh4TlPEW1PVZUm9v9Gq7IRPAKlVYNrzQXWs0ZY0NC10xQS2dPCco2UO4v2/hC",
"jb+qvF3qeTB0X76+qiewW5RX3bicbi6uasqhIng1maePjIkEoQ1L6a9W+chQ+1O51Ucot7qURB0oIldI",
"XV1ghqDZcvqRRKmxW/QUSKTM5EKrs2Dy0QOJrYlUG2cNPcmC17cGdrOeJXNbb28C2Pqr1RsDo0yTpzqF",
"68porVG6mL1gPHEf9jxeCh6nJmzUNOp0O6lIIK7EeMpjHn2a9yM+HdzvAHvZeaq18iyDaKs6Aamqjeos",
"n6F1BBbTGVZdjDXDuEqg/kjl6qBtB/OvauxYlaRsy46VpdOFfIjmfY87y9s5ilFTS09g01rp8bOsTy67",
"Uz5J4Y5t+UWUHzx70IefQrWdIj9t2vEKVNp2FJNpxRulmGKl7TBeIEthh3xnS9uxwFM4xQyPQfSa1A7x",
"lDIqlSiPr3/PJnj/5f8EAAD//+2UaeGdXAEA",
"H4sIAAAAAAAC/+y9C3Mbt7Iw+FdQ3LsVOeFDoiQ/tHVrr6JHorO2pSPJm3tP5KsCZ0AS8RBgAIxk2uX/",
"/hUawAxmBjMcyqTs5OjUqYrFwaPR6Bcaje7PnYjP5pwRpmTn4HNnSnBMBPzzEivyms6o0n/EREaCzhXl",
"rHMAn1CivyHKxlzMsP6AKEPmD3TTga//zz1lMb//T0VnpGf+fdPpdDvkI57NE9I56Oxsb7tGO9udbkdG",
"UzLDesZgm+e6zQx/fE3YRE07B7vDbmeOlSJCg/W/v2/3Xr3/yTU2f/1Hp9tRi7keSCpB2aTz5csX3Uvg",
"GVF2rT9jFU3PjqsrvZ4SNNIfUTpPOI7R2XG/0+1Q/W2O1bTT7TA804NDq1sad7odQf5MqSBx50CJlPiL",
"+g9Bxp2Dzv81yLE+MF/lwMGgoTtKKGGqDqAIvtaD8hVAZBNrKI55lM4a4Ijt941A4k2uYTn5OOeiFhIC",
"XzcCRzaxhuKfKRGLOiDOjhEfIzUl6E/dbP2guNk1JO8kESczTJMqLO8uX/cIi3hMYpRKIhDR7RCOY0Gk",
"rAEL2jRCZljba1rgw+0glwki55xJYpks/gUrco8X+q+IM0UYCBc8nyc0Ahky+EPqNXxuuzlCcPGGSIkn",
"xMxYRMXJRy0acIIkEXc0IojoDhoFdYIuNJttO8gbwlRHnI0TGqlHW80lkTwVEUE4EQTHC0Q+Uqkk4gJJ",
"pUVyZCFa0wJPuRjROCbs0VZ4xmQ6HtMI5NuciBmVknImkeL6T02CSE2pRDjSPda0zjNmqASge8S1erSp",
"uXSNpHnG7nBC40vyZ0qkesQlwbRImHnRiMeLNa3oLVenPGXx4zMb4wqN9dRrWsk1528wW9i9kY+5IJDF",
"KJ1zhhTnaIbZwu2VXMPqup1LosSidzhWRFT10tt0NiJC60hJIs5iiUZkzAVBSiwomyA8wRRYOjP9drRa",
"CWggytTu0GggOktnnYOXz/e2t7udGWXm71wbUabIhAiNEK01GU7VlAv6icSPj/j7KWEo9UBYC0V9cSiC",
"MQ7jGWXaODgECenmDtjwDqoxF9ZMYHiUkEFMpf6vFbES3VM1RTKNIiIl6JlUIsxipI16qfBs3ul25oLP",
"iVDUKHrTszqlAckJcqINIcL0dv3esZPCL/CP9/4pIf9asjG6nYhPGFX8VqajP0iktAFemffItEG2DaIx",
"YYqOKRGweG2xGZCRs1cKZxS8MxpGu/Fej+yPn/devHy13cOjKO6R8c5wd2//uf6laA4N958XziX90Cmk",
"aw2pCrhg1jlTEnYGR0rbcnPOCoD9waesH3PyX/anfsRnne7Khlq3Y3e3CspvU6KmpIAiaEtis3sOFGMr",
"2nFHnCcEMz1wTiNVi9l9ciu1VOMvcLg93O/tbPd2nl/v7B0M9w92d//lLzDGivT0HMVF7geMUd+y/T1b",
"cDfDkZ39fdaTA63oRWQsdSQIVkFWytUdUFSkG4JIQ4zcmz0ENoLZukjb3F1gIsE1p0lJJ2wGZ/AyL9XQ",
"yLuKXY+2UklihCVy5K6n1TM9WwvR+Mds3Pt02PvXdu9V//b//unmpvf+p//yfoMfbm769qf3n4fdL0H6",
"H1Mh1a05gIQW+INEE3pHGOALEIujiKdMWQSXqOUffMqKgO9YpZD93YYrE7wMqDGe0WTREqpjTtYAlKaT",
"AH++plJp/gEy+kAWYCgbckKUoQttQas+5WiL9Cf9LpLpnIhbrAm6C/Th/o3TmCouuta3cKu/PeujSz0t",
"mqVSmbOGHnTBU+GNTNgdFRyot48yAxD6jTV5SpoQppIF6qFoSqIP5tutAZLEekQndQu6//dODp/mTwOg",
"ZlCqyAxwUUbqcjTO8Mcz03vfbIL9aydri4XAi4q8cEzhUaxPKG5/WkiPFgoZa/PJlyEaEV1EWZSksf6l",
"rM/OjkGWyAWLrIKuiJFWahIAND6DHzwxwuifKQkoz8dRkpGB6harJh0CplWmL++xRLafpi/9u1xIRWZN",
"2mX3YG//gdql215Mb0aBG6Mpblbgkbe/iEpr7wGG7F63UelNQrtEQbnwbiOnv0oyl2b2JHQLYbx8ZnBF",
"qFvNYU1YzojP2hfjNEkWPiXmYtMdAozn3FBnH52N0RgnknStRW78O/kGoa0IM+irmTDCijxDo1TBGbUo",
"76dYIlaaZuB5VJ710RvMUpwYscEFEvrshmZ4gUYEOeHXb0MSRYkekGygDdQUK3RPBClix1cEGfwaOIAD",
"a7kjiVFyxswiTr3QMZJcq2BMExL30RGfzbEgxtIqNwa5rrVjTJQWY7nNBf1TQSSgn6VJorFBZnO16GoU",
"mv4AeAbrFmyPxXERq+csWaC5IFIPTceoQDqa7TQWH1vTFZVbt2O1RO1JCZaXqZKcf07PL49Obo9+PXz7",
"y8ntxeHV1W/nl8dV8bgMvJJ2DSin3DDPFVqBBxtV7TFJSCtVm519Y91Dc6I57ZpT7ljwGSI4mlr+RFsO",
"P1rdZsT67GHatv5Qag9EAJOVaY+ja+2Mt3rhVYiPHY5q0FOHhhby0i0VhlxBGRmaWHWCbOeWTxGm1Iwa",
"g2TY5lhfu7ePe6bPyH5jp3o3Q5Pl9eJgZ39T5/oCSS8RGgrTJCgUNdjK2pb+hbdPrkWZ4FnrVv/4HrOH",
"2efvWtrlBizPSMgtjO/XZHdnaM90D5PM/vX2q4Od/YPd7XUb6ycFd0rR/wbAPbLtfpQKoY0IZ6dbsesD",
"tj7Tvepv0fYgJXcNUnkjlnzAx9ICkoca9jV+FWOxBk3TouXmjLY7Su7JN7LZLJkUbDfHTgEb7uj87enZ",
"5ZuTB9ht3U46j1uwNh8jvcPItG4S/dsHO9vrEf3tjEjHbI2a4DWVgeVd4AllcJBLrNdNY9rejyiucIIM",
"zrWon5vGWvLOiMIxVrgi86dY3s64IPUqWn/VQ8H5hSB8h2nirkmWmkZ4QuqJRX9FDC7kChdu3YZLtp3t",
"7cIl2071ks1MeyvpJ9J0/2cQNycC4PAB0BKyEYJl88NGBGgT9oeVIJhhFU21jh7TRJUwsfeiCZLhzt6L",
"vZe7z3WrplvHLpztGjy3BhC4NPC2puPJkaarx4rxUpAigK5Gv6YBzmHNEo2/iY188s6w9/L7EJADGs9G",
"Nikl6ChVRKItUEn2PgRERvFqRFbPVo1KzEin8q2BU5cl3cWwmuJN3xhYiMpXBkGQjjnpXc0ohGVt6MIg",
"s2WTtjcHPXR5cvH68OjkCuEkMS6p3KzdYlxpk4kqekee9dE1138hDJ+dz5pAT0ffpuM8SSVggZF7xBmB",
"roLM+B3JenMIMFVG68MVoSbi8K3E11xIuOsH24gLNKYQmKOwIn10rsGAqKMxJUmMtFUtSALepITgO2JH",
"T1k0xWxScpp983uMKv/qT6d6KbpXlUauKJskelH3mkCM48ysXKsxtLWzMzR/A65mmhg4g/ZVbsVYEEWO",
"iaB3JD4EOuHsEityBKjSJkNCGSlhYbgdMpQLYx0lmM6uF3NyFIdQ2Nz5lJCraEriNHnAzF7n/58IaSMd",
"VgPgF8HTORHh/i1geM1Hq3d6S9Q9Fx9W73gheJyaOMeVO04Enj2o4901tFi15yWhs9EboqY8btfZ0uQx",
"kVF1H5t6vMEfTwnRtHzGokJ4UsxTY6Rl5sKr7H/9V6+CFoO1xYrjX0Sq9fh9PfqyoVM11dpbadwGVrwf",
"XPCIJskKnJY3bz1DhMUd4alqOYNrfmbCAau2b6SFQ7bQtqPqTldwUHhIj/aLnau9C8GjtpNkzVee4Q1f",
"bYo3vP0cMRnjNKnfgpjiScsVmqbtZ5ZTy3drZLpYTuuXIqeWEdfDheTjlI6ourAHtKX4se2vqaqoqzCG",
"xjiiCVWLw/iPVCptCF0TMWvVdTIj60fuZEZqkTuZkbUidwK+Q3E+rp/R6N7DBMKvtd36mtyR5Hx8hEU7",
"hWNH+BnLtWPKjNyScbLWrXnH9vgVR8vQs+ZNgTEvibaCzU3psvkvqfxwPn7DhcKalK/SUZRgKVfByxW5",
"I4KqxUP6LAPvWmAmx0QsbVc1YsLz000wHm1gPLpmxkuIlI18l/CRNSQvSQKuMTml81YMZ7pqU3Llrixu",
"q2ihZWtOYgrP179hetRaBOqPa90ynqqEEnHyMUpSzZQtEVXt1hprtuspFVId8yTB9QyUNf2ojztSvh2J",
"Y7yQ11NB5JQncUus99tiIZto1RlWmOON67uhgU8h6oVFi3UjZxNk19ommWNFCVOHE/IGf6z0aO5AWasO",
"CY7I+fjKPBJsyQblTq2ZYC743dWcRBQnatF2smKflea6xh8547NVpsq7tJ5JYDYhlklbzSOwIicywglW",
"XKwwjderVngUWr3BH+0p+sI4Mb6ehAvjh91Za2E/QehsdMTZnXEUneJImReK7UZvN/7JeEwibQcfl7Bj",
"7u6WMQ+MsX5VCMOuVe6YEQW/e2td9cuJLesxpxVro6n9NWWt27eWg1lre7P40O26IyxtK+Sy1q0Z1L60",
"br0qqfhcq91jTJOFZdU1kpEbflUjp9KvPQJszzZGTt52g8aHm2T91kdp5PWbH24CKwfO2cnHiEh5NMVi",
"Yq8Dv1osuDkuSwpsrQtozQ5K0MnE+BY2QQt2+CM8z0Y/nKk1TpBG61cFab23Io3WqiJSRtX5+A3BMhVt",
"zu2h67X2uWRo8aaws73zav/ldjzukfHz/d6L5y+e917G5FXv1d7e7j5+tbu7+wL7sTJpCkEufoTZ8wBJ",
"AUBXNdFDGqg8sAwXAPTe584F14ynR+xCyiCIqtTAQPC9/hGziCTF2JoSDO9gVC/OEifJ+bhz8HuLxDim",
"71U6m2Gx6Hzpfi6HBQAct2OaEIZnTW/lsibmRYLpiAoL9C5vYx7t9ufxuNPV/9x5Af+uubzd2371fLXb",
"WxtSs+zO9n23EoqtDNh+NCoe8VSVttCLQ7ULzTFU3JZw0JNDmj+muSLH9qlg5coXmpp/tophCW5vcxSL",
"DVi5NVGZS4N9LEQ28ILKHPI87mh/HdE+pegah4kiuO/rZIbBQYtXC4KoVDASmxDa8oZLhKOIzCHqhIsi",
"YQd2yoYat8pM5QchLu1gJU7W6TYV4WRFSHEbiWEWEohZHJgdGxweHg5c1qkBtB6sLDQF7QkyJoKwqGwv",
"7wyXRRl6Wb4yML3lLdlbR9/VkAvzYSk7r2ML7Qvf5X3yXGCeyA9Gfv7mQrkLlJj1KdJgm1BPliYm0LEY",
"3tguxDwMTTGw/EFx41bHOAKUTSGOWaMWWma4nkhDamKdbvWXdkAyznoXx6egE6RJcoMoQ1hEU3pHWgeG",
"tgeRCzqhDCeZnq5Cd26bZIoqC3iHfSQx+tfZRQjETrbk/ic6L25nW81s92f1Tc46IsnRGBfi+vbWtL9z",
"wSeCSHk7JyIiIcV3kZEXco2RbVyKs22GaWdZMp0HKgKjBhtQW9bcOYKpx9IFwtze3pTmNmI+F5ch8vXU",
"QHlxYWoKSJFazg1seUHyhbSNEdpHmF0tTNRQKUOX4SWbQVLbCknC7yHNALgA4TWw7IQCyu3INTlQ9Dki",
"V01eFhSTpa2otVbTPU5OLG8Pjr3yXsLOwRj1+FqSaZO4NIK0+Cjp8PCwev5aLmXcnD/zeNE4rzkgPhx3",
"VUzUo0Bb+i1sz4gzhSkziSlcEDGE98Iosgqw/VB/trANipka+m1D4I95ZBbQ4mRlmfSo3bHBW9HaBYxD",
"SgGi+s15G1SVHqVUMh0cHgpS9ge3fL5npqxzGMAt1JgSE73tuw4MKH3Pa3D29vbqf94edbqdt+fX8E94",
"deT+OHv7S9BZ4AMQ5pFDt247v283G9xV6bCdxiosvvK81Pxcv091LyJK0hFMQptpwb6oCvANZvDYvR3E",
"TuI/TF5Wl8NjP7o5kPuXSI0VFPGYIHs9VeaU53sFF9xwuLv7Yri9+/zl/t6LF89bPCY64klC3JVX+SGD",
"/YRmPCZJP5iE7o7c3uWLaMKHW+tDz0cmHn5pDwf0qWkOb0cUkeoBYFrM3UZaPxPRdgR/Y71RFPmoHjRE",
"UK4Za8nipB7UmukrSOmWNzPIgAXchonWvGIomSlgD7nOVTpqxU2FuQ1XdTuQBboFHflpnX1cWpmeDbN8",
"1fVqgqdqniqLAD1w/2GqoUTA9VjO1HkBta00emknC2od4Cwr9azDFQm+21yAA6DwEA2jo/otf7DoeLAU",
"+H6YuUJhuX21TKuVjIA1abNvdFbo5hC/D2MFDmknTNBoGnq9777kWfJjc/FhHgy7Vx44qX8g/JWaqJYz",
"IRfFlKA7nKT5c0QHp29GZrcsi51MDphuObCLoXEnuD93jdMuREv62HxFP5GfFyp0Q3NKE4Ik/US07Bgt",
"4IEo5GPSICZkgqNF7hF41mhs7A1f7b16/mL4an+ZI+MBnigPUxUfU9VpyJOYiLM4JJjgPaRpgM6Ouyb5",
"FM3dHvqAblN7Yduu+f4v6LacYnlNPqpLEnHRkLwMIy039FlX2FSvW/a9Y/bLMyRgEJeYLLtQ8TBSdRxM",
"sZwuo+BfdZtW3O7XhdCWwoiEcrYcJgky3xAkdiYxUrwCayt19FoPY5FXOWJWlZHzEV1gNW2gqTlWUy08",
"72isZUQKnGVd1Vt0NksVHiWQunU5ianC7jatBTTOSbaf2Xk/KAd9Ww720DPpijSV7UOTqFxaRaT2Ony0",
"O37Ze7G797L3Co92ejtkf3+0O955SeIXD7gOz+Bhp4ajqomuHERaUnu8Z3P4OsHu5DbaSuhkqkw26RlW",
"XUQ+wrNnd9F4S2OQuQZ7oSftT2KxXiw+psR7Em//PuLNirQHSLLaa9yCOJP2Tjfn9X4oBcyGaKd2xaGF",
"FSoYBPL+ZX9pVoc6JUUjcea6dhY8NYkZsswPyH38Ul78rG5CCwkyv4406cBJEqelHNH5dBN6R9CMoAnn",
"haCYB5xyS5hzUAbxBrWhahPH+SdPW6HKxt8VUl6vweg3x/vbhJtsb2HKdF+1Grmf0mgKWLWAfaJzEOjo",
"niYJGpG85EmObbl7MBjgweHA9BkMt4f728Od3YEWy/1I3pWwvb33soBu6N//Sf/fjgAl2j6//DLo/3Rz",
"o0cIeh3aeW7NZtR4bn1bpsGL26rKWI2dQnp78f5O78XObtR79ZKQ3t7zPfJy9+XOznhn9wF2SmE94bM3",
"l5KOkgwwyFcifQe8H6lH2a27Tszj9t7XTnxtKLWZqi05g0eFWTD6AfuGJBCap4KpkE7hg7lOiPicoBGW",
"JEacWXeV9V3BsVO29iKBLjBDt3AhUTYhUsPzEDCzziivKljFAmHxbf3tACQ/0p+1iM0G7LcIVdkdhllG",
"qNrpIJWNWOOE1cN+6PhfUc9Zdapw/ijFbcSOraNRPo7aZL0mXY02sL1cNdUUmzhLexMyqKAnRIeW5gBi",
"Q1s4SYymmeI7uAWVVCpzAyXSSKUCjJl2SbuK+XfaXFvaS/+fFw1pZSG3VlY1wIh2t4qCvNINvzprpTNz",
"zla0aM1unbZylF75bcsS3QOgNGq3sNU+8t63Icq6K/DTIPVZN2vJ5lgf5W2eqA5VTTp9DYNf5GkT+XtX",
"IO37Kc9S6cNJ53sk71De3grhwJl2nWH/QcQ+nNO6nbu6q197VeCiJLZ2eqAFn7WOFSw9wthZGikBay0w",
"e34t2IbtD9WqIqD23ruykY753VFPm0RTKhUXi6pd/xfnthzrT6xWXsJfhF1W5AhrPNcYwuYWNeJJOmOB",
"dObMXG+tYKUfZX3cLdbtw+/ezWEhbGqbb5rMx2li/GbEZoAtnC5Wvx5fJfWqt8Kuh68M9CXbcuRjuOFo",
"aKyKcbZl2UyFc2JCpLy12VBtBiX3Z5RwSeJbTW/iDpLE8jlh/t8JGavbajNBJ9PQ79YzBCxg/hU6hNb5",
"5s3vxSz8XCAuJpjRT3qFpfjZimvlbLWHD38ZsZ1XvyqL7uzmcM2SOyiBPbf9xuXujChBoxWeEhro3phu",
"ZwxyNVRf13lh9HaG/CrA4NKV+eHzNAE8w0ss2/g/lUjJDaMSSRsrmVUhs1WJ+ugtlBlSUyLuqST9G81p",
"xUcvkMdEZPS69I5kJWSPliN76ROcefAGwO4/+P+34MAMrghzWBpASUGeAT7F0rS86QxuOkWvWvbOSw6G",
"28O9SkrkYp7cQf/H5a5cWKYte5/JgpXUIoC9rAxoxW1hFvsYYmpVu45sTDbU0+753Ma9zL+SiB8iMR5I",
"tO8kMTQKuyhyEt44zdaSayOJNj/mNaAvecZrGy0dpY9OcDR1u2iVu0RUSQhp0fB3bUkEQxH9G6ZlQC8h",
"dyTJYAG3mmuCJAE20jKoj47s+4AIM03DNyxrpjjc2DLjg/PoGU0pEVhE08VACUJyJ52RtO0cyIYzWr3X",
"Lhh2FnH1u/MmV1utFA8uYteXHqbkRFWGjBZwrwuaMYs0uyhtcGOhh+VPwGqCJgxEaLQwN9TFm8JjLKcj",
"jkV8e0lwvLCG8hm4nzX6djTXnh9d3l64t1NQBd5sgg+/b2B7AWa1UQXvTLWkQOnvgATckPCBpx7HD3z9",
"VgUTcPWVuxikXRAzJWC7GUktpWtrV1XfnsHv3hvYMJFvZdEzDhokKYsIouoHicwlC2Hq2RPNG9x//1T1",
"AFK6rAk+8mwsYYKP4N7R1qTIVlMkjLDSf5ubZutW/D0DW7w2AyCEqF9t7EhDCIoNcWm6k6/xZJzFteEg",
"h+7GDW79a94nthG++g+16K98Nd7uVeNZbMZf7U2jO0T/IP26fVNChV/mtbLoFuXZvXJi1uVk7HGoHhWY",
"oJWNki2SJ2RVB1R9yfLCqIH9h5ox9avRpl8TsgrD1Z4O4mLUEZyeebH0VUfL0x7XJ3EMdR81LHeU3Hsy",
"Tu+fiUuQZW4c7gUI/wOpietyhXIGdQXQbWGX4iTDIrf9jnuftnuvbnvvw2wXFnq/pjPMeoLgWB/IDSTV",
"B5+h+Vv6Lf3dqiVirxGaCMz0CXK0MF6R8s7k17MHGmzPJX2Q6KOJbqr3xHytq4VTj7yDZYj0C+QsYwS9",
"5dmrDx8VId4AxXU4nye0LtTKU1F4Pte0ie65+DBO+L2xDuCLH6dZui02AZ3Lj/IwuhNaFbtjPcf4xFlU",
"JZqAdYD+3cLJfIpZOiOCRsBuKdPnn4gLgjRvFi47SnZOY8nG3w97/7Lb/L5FoJ7DQI6/2v2rCww2v5uH",
"TGarvOja5Vt2uLzeqRn1HmdxuxtzGbcho4LLOFuqZugNkdM67/oMDdqI7cfxHtVww29F9v6+2KJ6IVhl",
"lK5HwUGmoVI1nTA8mzN/95k9BGybxqEQW708ZE8D9c+UiJBVoY1UAlrrT9Oiaqf+WdfV6TuvZ6sVwDPe",
"ku7xEoqFdY+DIoR1M2BgaQmf0AiljAKc5COJ0nCelYe/gedsTCetVnxkmrYKUM9eOn/Fu3eHvVtvA5u6",
"Xtr2dm6Th8hW5VgKK5TvCHIUDFF5mF5ZV+2+HmUYLuc10L+nIr/a1MwEb8D7S45yNzef+z/e3HwJGkRm",
"0qbkPVnoKnKLLdxkZCBUbvpXp5Rvv4fQu3Zz6gLAAQePlbbVAE+kqo2TDeRcsvuENBFmt43VE5gVtrer",
"vkcyuQhWZdqayE0TjV8csnZLDCLqjvJwQHPHeVP+3kNEFQHmCXPII5AmyllGxQFWdqKUVm2mrF+fJeem",
"ZDu6ZwE4P4TjH1fnb29P/vv68vDo+vyy0+0cnb+9Pvnv69vTd69fB0MsYN6HZ6zx6e1bq56vFyihePVK",
"q3oHj5OYhmbOjlc0Gwx3LTF4rkoBnaGqrT37pj8rz+odWwpPu3cOdpDwiiktK906IyzWg7xNAxVjs6/O",
"zRybXmi0QIfYJqJqDHJb5ncvAFMqWFFRoLBKRFwrFCpCv9fb3ult75RPV20OVj4olXIMNcCovB2qqYk/",
"7O0+CJxU8UvCyD1ObJb00t6kiveEaYAoi2mEjZuo+pLVG8rlq28YSy9JlgVjuExmfQqzt9774ewienmB",
"S4vWrCRi+cytknxU27bNuGOakLfLHz1XnW9ukltNWTZdeKvZ9Ei2WEhpDfgjOrOeRvtsz+YdBfXmJaBc",
"VmRkjhdELMc+NGsDNjS8UmGtcc97CdF60TxGM+nBIA21HT7H2VEpd+Kwcsp9/3n4pS5N6x2UUNYLOE2T",
"wJlc/4rc63T7EFkgqMrXapHZDKbyS+We3hsPvb04a7MPbsjgfpeGvD5723LIczU129sCDxBwlmGjLR7M",
"DCE8nBfGWwEP0DGIh9KQ7fGgm69OlTkqcsJ8+z8PJMyQDdEqiV5N/rwHZ5qUJEoFVYsrbWO4c9qEUcUP",
"09BVqP5VixoraFK4nD+c4U+coSPTEZ0dX/TRmUaRcfNcnh69fDHcR//47RqN9GljDrZFZA8iDgRYU8Lv",
"jUmRqikX9BNMc8RjUvnxnUg6B52pUnN5MBjEPPq06OsG/VT2CJaqt9PHAJddTz/iswHXLYYDNxBk5Y34",
"3D67BN/kQecwuyGy93vOacnnhOlzUOcXotD52fERUvwDsTHtWs6GOrtPcP/+gXwV3GY6n3rgdz32H/fK",
"7diIYEHEqSOPf/x23SmHI+itgMEQHylMGYkRHmuqd3uIC/v80O0EyxVMBwAp5wO9/M6XL/Ccdsytd0Bh",
"U3TN7cOcCjzBd1jgefpfxpay7mRz+dUxBiM6Y5GeLC0g1mtfCcY4vDjLTN2i9wRMbHQuoimRynpWbJkq",
"sNMTGhF7rKzCAJHrWvOnkiyBRxlzpBOa7xBkY3Yc6mz3t/s78HR/ThjWwrWz29/u79oIVSDdAY5nlA00",
"ycHfE1IT1AePGYQ0psHhb1fZhsN97BxPrOnZzR8CmJg8yQUETsh07p5u6yMAND6L7eiHGop3AATEdVo3",
"EQRbl8OSJ8QdAYzSdzND3H/nwJz2872em7Tk5ixk1gZlxZe8itnZXpo89Eu3Pm27gGM+XFIjC0EdcLfS",
"yZMKhEvjZFYF8IpgEU3BrjbxEZpjBqC7s22rAVVC1wKcq6e9qHnwPlqYO6Is0XkQgKwYRhVRHZwknqOC",
"MDwyJYNiKt0/dZP3rWAiCfglNeWi0aIOHC7ULXwNweMkfwaR/dtLrd7tJFiq21La8Hq4rjQ4MRUkaqB2",
"gIoLY28EESUjDyzzl54nBML7bsd5vEA4DLe3ncS1yRpxfks9+EMaGySftfExseN4iCMGiR72e4DY0VJj",
"SrCLFr7EirymM6rqZrFtB3lDmGHPLCDUI1vo4MxkyncuUei2s7zbO5YZB7HptLu80ykXIxrHBM7Ge8NX",
"y3tcc/4Gs4WFDtzU++1WZXQM5OEpWHAgZDMr4Pf3etulSz9ktiG1klnhiZbJHdg8W5W2815bx1zWvV4i",
"0r5RAA6nDI24mhY0iNYSF0TMqOpTbnOz2xSW3IXqePFApv6GPrnO5hzyI/RsuSKJhtvb6Pz/Q9SFLySC",
"4Hjhkm2Z0KHYmCcJwkoJOkoVkf0bdsN+/DF7bXWa8PuDH3+8YTsQnicQzQq76BX4wG/NBYWHuUXDxy7j",
"2Q0b9tHZOGvuChZAtRbZRZkqND8YEH+9vr5A+9tDtMW4l3OBxM9u2G4Qohx9WwVbdwB4y2DZA1jyxmVo",
"rBCmSeJQRnPrfJQqE6CjIGcpif9zjBNJbth+H12a6DPhxWwVodL2Xo+Mx1yoLvhLKEsJhNrqqVNB5DOz",
"B3okdJhtNPqZTPEd5QK2o2cngjBKgFBPs+Cp8OYi7I4Kbnpvkf6k39W2BxG3YOqYRbp/24iqrsuip789",
"0/NYITBgnPVgIj2aibHTACNJE8JUsrA5QQhK+GRCYrD5iOhJGhM9DDzhkimY9uM00e0dfmzA3nxOsMje",
"bunfbvMm2XWDVkl6vOtqI5OQxPrNMZLpyF5Le4/B8ul0Y1ifqaiQr6hbHtclaYI0gGAqzNUCbQGJTLFE",
"jPvhWnb3LrBQFCfoyiwZXUWEYUG5dNtn+RQofLi9g8gdYZpdAzSpjwR8RiqsD+DqwY6goFeBJA2mDhBQ",
"pnNFmoe4AkJiGFdBvoHx+Gyut7KEiDtZQaPiKCaKRKoMXEbNejzg0wgzXzQQtGX56RkwlN63BEcfUIFt",
"0VYG2DONHb04w6MV69nI2EybdsxFBZHKVTVYr7a217sBfX3tAn1cDuLgFW+aw2hurew7xM3bGQZyL19f",
"ZQXv6lTGDKtoCo/RM42Btmimf56tyToZGjPjmyzb8YQvq/49jK7tFkbXEWfjhBpv22NaabrXcHmvn3H8",
"C1bk3gb/tDPsjvz3o5Yta+y7L92Cn2DwGU4zX4zBl5CQP1TLL8yMijRtpOV+40FoNgL7CBKI5cYRlYgK",
"QcC3MUpIH9l7U4ngLlfM4EG0vRrN/AdVX8MxgOJLy0ZnwxttZ4yIeysIc2hpZuY0C6s/i9lmFXHnH8zK",
"N2SPc+YyaFgqFMzGbUQoDLdffLNlzY2RklFmjLZmaaJoz3gYnv2byL295T3ecnXKUxZ/t6dTs+NGrsxz",
"kdN0VA16OC+JEpTcOUvRmTAtRBUaLYwXDeE4FkTKqtD5haiifbZ59jbZYuv5Gr4/0flfhs5/IcYJk1Fb",
"E4XrfQo4Y/RpSerThNXD1RM+ZvGA+0dpbf3yVFn3Cjx/1KdBj35KhxE9R0aGJzBBDcUHyNIAtAaq7Hb+",
"u6dH7BkIenZ99Vnaw/joNCrq8iQGa2fnrWfJ0Nw8z1+GxUKChhNz5M58NYQCKoxHak3y53tkWOObcH5I",
"4jih3oFaNENDQOVNBsBdIAb0rHMoWFvdAYihtBrNP7lCXl64d+ii7AriWUkQoBlmeEJk1f3ik24fnfi6",
"D0WY6e0eERRNMZto0yqVxCpIk7DfrUIb0vado7n9eWa9sIcOVGRXYH1HOdgAXwY5uN9cwGjROwsuuBlV",
"LlhxwVMUc/aDQveYmSw6AGbmJyQshpczuZ/HNoC/fRflG0CPcTJenly8Pjw6QVdKYEUmi2cWYhNwY7LY",
"ef43ZNtfgQsue1luWmxp/JncAncEXJHXXP9thzjIst4fvn6NolSI3DF58frdFXj99IlO/2R7CzLjd6Qy",
"AKwz8zoCbhxWPhAyL3V2zsIDgyTMrEPQrOj39w7VWWZg61qEy7SE4LsQKaXMkskGfL9r9OK6E0XfX5Oc",
"8ntp8/oxnNiwHxMMkR+Y/OUWfIzFIRt9jLlPUSPpxAQRoR46BKrQWAEEHfnEcIB+v+l4Tu2bzvsbZinp",
"xj0kvunY3azp0vWbvr9hJnZ9ebsimJdknuCIoMMkMVtYA202gh4uALrlRMXRH5oEbjqeVz+wksLXIvDF",
"T2jLzgxU4E1saR9SaRbtHCOYHs3pagPom5yunnxX3ErDb+NnXXb4cJL639DT+Hc4iRharBAddry3shdx",
"YEM0TBaVr7CCglfQx2bwzPlYtA+6aC7InbZBtI4uRa4ZCxYrnC8OdMRcENAPsX8HHXA1mpkf9+R/WC5K",
"VOMAsFExT0z4F3V7mf17mNt+YKKjNsNvl6Rnhtccp5mL8lQmi5ziAlzY1waW4kL3KQZRRHiORzShatHM",
"ayfsu2U1G4r2xGl/TU4zlNWC0aKsgHGbWCjTOsvM5JWxq3WtQf8j935pE/amrYvceMPfeLkf+dA1mZ07",
"a4b57BgQEYDZfP/bXG9/l64u/yI5IwHHKmYDQrwy+EzjL7Ux5/mNDHY5riPHNqOFzf0avGvxeGRDOiCv",
"V15PcX/hO5bv9QIEe/sPyXjqaGxVd2qe5Xq5N9XPBelLcU38tcLbdH4E4d3sKXDCe2UPQZDC/y4H+e/4",
"lI1XlqeDyNUqWVGyum5OuFrarpOxrvnPi6M8bfjGBG42XVDgZqA/ydxNyFyfMjKq8CnSNdisAL4iyk2f",
"g7Rc9F7VE+sGhHCOCtVCCNvIMveGMFtWv4Vc3gtlgnBo0aM+ieX1c8RVkSOamaAsmbPUjssFM2TbtO+f",
"8qx3o4WXwyIomQtJ9B5HNBfz9jXYw9ky9MKeCHKNr7NUITdiqeKGI9Bsj9YlpYNejnem/D52+bshlYnN",
"3mwuVUvvumZE4RgrHDCZ9UjHeTrUenkNwZtzLNRgzMWsB8NBMeSIx5RNTHlo4/C0va5t2iOP6k1CFfJx",
"nkDOAHhA0u1ItYDn1npgsy0ZU5TrTyc1Ga4cClJYkJ9+YUQZhnjhQibOF7sv9nZeDvca87k0JLWOUqmK",
"2VvytBOOSqrpY/a2XxUe8PZ/WpptDJZcTTZWlQBm5QYRI719Dzp1ZIUQzHAkfpIh6zprWJ6FDW2UGHUq",
"bTByNtvSE4dTa9DD7qXMTKC8mlkmwOo13M96CLI0Y8Ab817eK09h6m1DfQeVirrw/QRoJPiKerjud/lv",
"q7DJD3ReAxkfjyWpAW17LfldNvkKAbbNEN2yt98FInli9/WZDFXu+/Ymw7/OLhAW0ZTeFcplGO2eEHRx",
"fFo2c+BdpFlLXvJniSkB5Pe19oSFs9mk+ETnX2NSeHPUWxWQK9PoQ4RlEYvtbY1lqt4N2Ebb12xjYfda",
"WADDTcibptvanz2WgFofc5eAtEBaTyJobRZHDWdjWSLiBxskg8/wn1t7z1T3VvEIs4gkmnl8sWjy0VLp",
"gmyTRZOMMWNUZcwyn02B6iIYpBwt8FcMFvg7XP2bLS1RxTL1uMT+Nc4/vcHw1LBSY9rzyJdmrbjgl9Da",
"Ruy1htjOAiFbH+cT5X47x31BlGWpszZi2nWXtgXiACuwLLBNjabVY9FaGJZnjCoKd7b6ZGkmciK86jq9",
"FnQyIeLEwLOZ6wEzuJ2p7n7AAmrtI5Ps6FFDavK6fAEIzRKQMmt4uv3dCP9aCjGxBpYeHeeaDai3gLya",
"ynW6CDLw4CQvjDwiCWcTeLjDs4NgHxUqHmRVl0leivkHecNCBZjzettdPQ+/z4MlZJuCyuHyzZCQCGo4",
"Y5OUyKv2fMN+K1fHR35x/K4PthnJPYOq1oq9YVul2sf2wdmIJGgkCP4Q83v2zCapwHEsEb8jQlMzkhzJ",
"KU8T+4xsRG5YKkvF+yGKnBESk9jUiq66uIx4O7V7ucTBBUuHxBINizLpHj0kVNboKss+69+wY+NOgt0y",
"uZTgIEQEmCss0oSBJWfSFrsOOKnsvGEvlT0LP2r2C69seYPLyS9cXqwx9WTKfCtvVU0xeScSzcauxUlV",
"lqbWcGp3YwveKGvYt4qiuYK2jxRCY+KGzIzNwbJ2CU/RNGs3yi05eISymSjGnIxbGALZlYw2CZyWtgkJ",
"zTv9foOOkpsnWj1Zk+fOiW4Hu7Bc+WScbkwie6SyJDYx88i1jPZWXogUFEnNzBQrUmnszBxr3UD8gN7q",
"KcLyhhll0c0uwLvGqpHGQHWBB/bpO6QzUeRjoXCRqfcZMs08t8vqRplcNmOWBdTRetiw0p1tddnvxrZy",
"WDlhgkZTTb1VNs0u8P/K6Yf+Lu6huLQZOXet10vk1/Z778sDOficV4z9MjA82iaCIEksQxsHPRwIR4Sw",
"cB3lLoJk6KZO2IxDxcIIjqhUSNVv4vDXBqSv5JrinV6+zFaF2/xK0uXS+6GS40sv55yqNHA8ceC3jKzO",
"CblYQ89jQNj/Buarr5FsQrRBfcyxmubao1CmuT4v5Wplq+udsFDGHgJ/mgrUm9O2wNEHrdM95wUkValy",
"qR518drWmN6Ep7ZSgr/GWZuvRUP0qG7agmwIcHqhtvxf/vXx34HrgWqzsv/lcv81PK91pnECN1vQNr0f",
"uKxsah2TkUgb0c6PHHg5b9zJyj0G25B9aKZpuD609wpTLI0y12NAbtanY9s6E9LZy6VMrNbdKqxq5pn+",
"mZEHGbJ6+QGnrQetaKCRJPYPSUUtGX6WZigfKvicZD07D9ObpWPXxhTn5hzeRTQ0uVBcNq4y0p90xbf0",
"FtZsiu/6Lm7xqiV5KjwGjzOkX+bZZMYTeBEu+FyXqKLKgpsw0yoUbkknbKxVVmsfnjyezbYCR56Wgf3b",
"VGj4W8SG+TkvHsKfQS05mFKpuFi0dIPYSo/gJy3DUObTB/hBSrD/amH7GyvToq/Gobe1t6aEMFeWdy2O",
"m4o4sKTyJAG+oYKuKBRLMm5zHi4I7vKSzu1zOLjZA+KgjfFcAvHnhSPg75jjuyFgHB7sgy/zzgvQVQPb",
"XbbQFoBRpp7v+Y+qXg2Hu7svhtu7z1/u77148Xx53djvxOg/DRFJ8RWeVzX8Scp8L1JmtEA5xbYTMHlQ",
"YKtjgYkQ0+TAxQQz+omySf5Co9bmh26bMvVh8K9ITjf2oXskc99MGWI9g+GntHSPZaK7za+LGfO4ZPDZ",
"/OPMeFuX6r4s2DV84eIGW/d1S3OaMi+sVoMDJb6rfHsJERKPwLdmojq+9WSOxd3j5S1fyqMCQH86d38X",
"TG3oyJfmqzF0Hn7Q8oydP0u8p2pKmW9vu0ijPBrJxRzl8eKyf8PO85ij1qFAWbR2ZrLfMD9ibzJVRPTu",
"if4HMtIDqSlm6JeTa1QMukJb5KOd1RY3oLEpdcBn81S5ornyWU3Ek8Hssfd2d+2BT5XVfmehT0XHQIGG",
"WnkGsigY5sRN2SfwVdEcOZFSZqnySUJ9B8FVsB8tZFX3+zQyakSoe9ixXIB6gRxzwSdQIcp/jhLEU79e",
"Br3JnpRs2BxwE9VbBXYZT2z2LQ/G1mSsvHX6a7LalJu8Xo3R+lqT/nr95jWaEZaiOZ6Yl1l+hXlTYyFc",
"h/RXPiMXeNIi1kOr2sFUzZIi7xShAkhsp8x7pNdhINMw+klIYLhSCpK9l/svCpnH/rf/438Eco9VGPHX",
"bBoNxdOheW1sFdzDrPJBqqZFI9vU8FOLVqRbpdMfpC2RlhU+g6K7WUk337yGSgn6gzT3XhOBmTJXWmA9",
"6v59dMgWgXlQhBmKtEmvplQiwuI5p6baXUxlxO8IUC8ViN8zA0mQg87ccjeohNwcS4uL2KJELC6g5en9",
"y9eU2l9PvJsetOEZ/anJAuJdTQSJ/4EBI6bGjiWicGmuChub0+rgM/z3LZ6RrzkrlyLzFYaqihBc4p2f",
"YarGEHz5sxfl+x2c364sujd1fLP1YKjMcfPEsusLty8hukyKXxV4b4J6bcbVgNmY8VWj3VgwjbYLZtHv",
"h71/4d6n7d6r2/c/hSyk4F1o9gAZXvLTRBFRSMMW8rJExSTpYUjbFOVxlm3CJ7T+LruYpwXaIngoMFog",
"QWIqSKRsbgpXn/js+MI21DZK4HkqTFgSGrtGvpclmJnAHz0fuch/r7l9DFD1ul++dlmOg4OEDgaCFk3h",
"4fbey8KGT5Way4PBoP9jMBXvkzRolAaaHSKzF1YoaBphcBpzFfTngpskxqUXrrB1dj89xdKsP3WnnjYw",
"Rzj6UEvsv2IWJ5bUz/UoQ+T6oDgVzsq11GOTFaFLEhHqgqQdim2JPB4TYzX4zKF1PvloKuZKRO3hjH8g",
"TNZwy5GDfImsO2yevU6k8Li14KswQi75bm56bYUfPGnwqo1rBBxdXZ5qpCri4vRCsEoVqoxTK6WHeysD",
"+35VyeQRYbXCdGKF3SqSysnV+tEeLrL+3zqZ9e8kfQoCpczo7vC5TKDwtL5YxGs+MeIAjH2emvfxBnE4",
"K8IuCZwJzRkgrEvNPBlMQfGgAfkKbarHB2cCFwVSzrwMX0e9iQPwiWY3lPLBbCHI+VV0ohGsq2SF1z0o",
"kWiEJYkRZ85kjQRVRFAczkTyT9Nr02VO3DSBo9xheQVPR7c1Zhv5M9tfR2t6KxarvnYBcly95uo/rXmw",
"iegYGPsrgtr+9GD7DrJDwnKeotoeK6rN7X4NV2Qi+AGVVg2vNBdazRljQ0LXTFBLZ08JyjZQ7i/b+EKN",
"v6q8Xel5MHRfvb6qJ7BblFfduJxuLq5qyqEieDWZp4+MiQShDUvpP6zykaH2p3Krj1BudSWJOlBEPiB1",
"dYEZgmbLyUcSpcZu0VMgkTKTC63OgslHDyS2JlJtnDX0JEte3xrYzXpWzG29vQlg669Wrw2MMk2e6hSu",
"K6O1Ruly9oLxxF3Y83gheJyasFHTqNPtpCKBuBLjKY959GnRj/hscLcD7GXnqdbKswyireoEpKo2qrN8",
"htYRWExnWHUx1gzjKoH6I5Wrg7YdzL+qsWNVkrKtOlaWThfyIZr3Pe4sb+coRk2tPIFNa6XHz7I+uexO",
"+SSFO7bVF1F+8OxBH34K1XaK/LRpxytQadtRTKYVb5RiipW2w3iBLIUd8p0tbccCT+EMMzwB0WtSO8Qz",
"yqhUojy+/j2b4P2X/xMAAP//akDiTcVdAQA=",
}
// GetSwagger returns the content of the embedded swagger specification file
+3
View File
@@ -107,6 +107,9 @@ func (s *Controllers) GetDocument(ctx echo.Context, id DocumentID, params GetDoc
if document.OriginalPath != nil {
response.OriginalPath = document.OriginalPath
}
if document.FileSizeBytes != nil {
response.FileSizeBytes = document.FileSizeBytes
}
// Include text record if requested and available
if document.TextRecord != nil {
+80
View File
@@ -627,6 +627,86 @@ func TestUploadDocumentBatch_Part1_TooLarge(t *testing.T) {
assert.Contains(t, httpErr.Message, "file too large")
}
// TestGetDocument_IncludesFileSize verifies that file sizes are returned
// in the document response when documents have file_size_bytes set.
func TestGetDocument_IncludesFileSize(t *testing.T) {
cfg := &ControllerConfig{}
test.CreateDB(t, cfg)
initializeTestConfig(t, cfg)
svc := createControllerServices(cfg)
cons := queryapi.NewControllers(svc, cfg)
// Create test client with root folder (unique ID to avoid conflicts)
clientID := fmt.Sprintf("filesize_client_%s", uuid.New().String()[:8])
test.CreateTestClient(t, cfg, clientID, "FileSize Client")
// Create document with file size
filename := "measured-doc.pdf"
fileSize := int64(1048576) // 1MB
docId, err := cfg.GetDBQueries().CreateDocument(t.Context(), &repository.CreateDocumentParams{
Clientid: clientID,
Hash: fmt.Sprintf("filesizehash-%s", uuid.New().String()[:8]),
Filename: &filename,
FileSizeBytes: &fileSize,
})
require.NoError(t, err)
ctx, rec := createContext(t)
err = cons.GetDocument(ctx, docId, queryapi.GetDocumentParams{})
require.NoError(t, err)
assert.Equal(t, http.StatusOK, rec.Code)
var result queryapi.DocumentEnriched
err = json.Unmarshal(rec.Body.Bytes(), &result)
require.NoError(t, err)
// Verify file size is present and correct
assert.Equal(t, docId, result.Id)
require.NotNil(t, result.FileSizeBytes, "document should have file size")
assert.Equal(t, fileSize, *result.FileSizeBytes, "file size should be 1048576 (1MB)")
}
// TestGetDocument_NilFileSize verifies that documents without file sizes
// (legacy documents) return nil for fileSizeBytes.
func TestGetDocument_NilFileSize(t *testing.T) {
cfg := &ControllerConfig{}
test.CreateDB(t, cfg)
initializeTestConfig(t, cfg)
svc := createControllerServices(cfg)
cons := queryapi.NewControllers(svc, cfg)
// Create test client with root folder (unique ID to avoid conflicts)
clientID := fmt.Sprintf("legacy_client_%s", uuid.New().String()[:8])
test.CreateTestClient(t, cfg, clientID, "Legacy Client")
// Create document WITHOUT file size (simulating legacy document)
filename := "legacy-document.pdf"
docId, err := cfg.GetDBQueries().CreateDocument(t.Context(), &repository.CreateDocumentParams{
Clientid: clientID,
Hash: fmt.Sprintf("legacyhash-%s", uuid.New().String()[:8]),
Filename: &filename,
FileSizeBytes: nil, // No file size
})
require.NoError(t, err)
ctx, rec := createContext(t)
err = cons.GetDocument(ctx, docId, queryapi.GetDocumentParams{})
require.NoError(t, err)
assert.Equal(t, http.StatusOK, rec.Code)
var result queryapi.DocumentEnriched
err = json.Unmarshal(rec.Body.Bytes(), &result)
require.NoError(t, err)
// Verify file size is nil for the legacy document
assert.Equal(t, docId, result.Id)
assert.Nil(t, result.FileSizeBytes, "legacy document should have nil file size")
}
// Helper function to create test ZIP with multiple PDFs
func createTestZIPFile(t *testing.T, pdfCount int) *bytes.Reader {
var buf bytes.Buffer
+3
View File
@@ -117,6 +117,9 @@ func (s *Controllers) GetFolderDocuments(ctx echo.Context, folderId openapi_type
if doc.OriginalPath != nil {
apiDocs[i].OriginalPath = doc.OriginalPath
}
if doc.FileSizeBytes != nil {
apiDocs[i].FileSizeBytes = doc.FileSizeBytes
}
// Include text record if available
if doc.TextRecord != nil {
+115
View File
@@ -353,6 +353,121 @@ func TestGetFolderDocuments_EmptyFolder(t *testing.T) {
assert.Len(t, resp.Documents, 0)
}
// TestGetFolderDocuments_IncludesFileSize verifies that file sizes are returned
// in the folder documents response when documents have file_size_bytes set.
func TestGetFolderDocuments_IncludesFileSize(t *testing.T) {
ctrl, cfg, clientID := setupFoldersTestController(t)
ctx := t.Context()
// Create folder
folderRec, err := cfg.GetDBQueries().CreateFolder(ctx, &repository.CreateFolderParams{
Path: "/documents-with-sizes",
Clientid: clientID,
Createdby: "admin@example.com",
})
require.NoError(t, err)
// Create documents with file sizes
filename1 := "small-doc.pdf"
fileSize1 := int64(1024) // 1KB
docID1, err := cfg.GetDBQueries().CreateDocument(ctx, &repository.CreateDocumentParams{
Clientid: clientID,
Hash: fmt.Sprintf("sizehash1-%s", uuid.New().String()[:8]),
Filename: &filename1,
Folderid: &folderRec.ID,
FileSizeBytes: &fileSize1,
})
require.NoError(t, err)
filename2 := "large-doc.pdf"
fileSize2 := int64(5242880) // 5MB
docID2, err := cfg.GetDBQueries().CreateDocument(ctx, &repository.CreateDocumentParams{
Clientid: clientID,
Hash: fmt.Sprintf("sizehash2-%s", uuid.New().String()[:8]),
Filename: &filename2,
Folderid: &folderRec.ID,
FileSizeBytes: &fileSize2,
})
require.NoError(t, err)
e := echo.New()
req := httptest.NewRequest(http.MethodGet, fmt.Sprintf("/folders/%s/documents", folderRec.ID), nil)
rec := httptest.NewRecorder()
c := e.NewContext(req, rec)
err = ctrl.GetFolderDocuments(c, openapi_types.UUID(folderRec.ID), queryapi.GetFolderDocumentsParams{})
require.NoError(t, err)
assert.Equal(t, http.StatusOK, rec.Code)
var resp struct {
Documents []queryapi.DocumentInFolder `json:"documents"`
}
err = json.Unmarshal(rec.Body.Bytes(), &resp)
require.NoError(t, err)
assert.Len(t, resp.Documents, 2)
// Build a map of document ID to file size for verification
docSizes := make(map[uuid.UUID]*int64)
for _, doc := range resp.Documents {
docSizes[uuid.UUID(doc.Id)] = doc.FileSizeBytes
}
// Verify file sizes are present and correct
require.NotNil(t, docSizes[docID1], "document 1 should have file size")
assert.Equal(t, fileSize1, *docSizes[docID1], "document 1 file size should be 1024")
require.NotNil(t, docSizes[docID2], "document 2 should have file size")
assert.Equal(t, fileSize2, *docSizes[docID2], "document 2 file size should be 5242880")
}
// TestGetFolderDocuments_NilFileSize verifies that documents without file sizes
// (legacy documents) return nil for fileSizeBytes.
func TestGetFolderDocuments_NilFileSize(t *testing.T) {
ctrl, cfg, clientID := setupFoldersTestController(t)
ctx := t.Context()
// Create folder
folderRec, err := cfg.GetDBQueries().CreateFolder(ctx, &repository.CreateFolderParams{
Path: "/documents-no-sizes",
Clientid: clientID,
Createdby: "admin@example.com",
})
require.NoError(t, err)
// Create document WITHOUT file size (simulating legacy document)
filename := "legacy-doc.pdf"
docID, err := cfg.GetDBQueries().CreateDocument(ctx, &repository.CreateDocumentParams{
Clientid: clientID,
Hash: fmt.Sprintf("legacyhash-%s", uuid.New().String()[:8]),
Filename: &filename,
Folderid: &folderRec.ID,
FileSizeBytes: nil, // No file size
})
require.NoError(t, err)
e := echo.New()
req := httptest.NewRequest(http.MethodGet, fmt.Sprintf("/folders/%s/documents", folderRec.ID), nil)
rec := httptest.NewRecorder()
c := e.NewContext(req, rec)
err = ctrl.GetFolderDocuments(c, openapi_types.UUID(folderRec.ID), queryapi.GetFolderDocumentsParams{})
require.NoError(t, err)
assert.Equal(t, http.StatusOK, rec.Code)
var resp struct {
Documents []queryapi.DocumentInFolder `json:"documents"`
}
err = json.Unmarshal(rec.Body.Bytes(), &resp)
require.NoError(t, err)
require.Len(t, resp.Documents, 1)
// Verify file size is nil for the legacy document
assert.Equal(t, docID, uuid.UUID(resp.Documents[0].Id))
assert.Nil(t, resp.Documents[0].FileSizeBytes, "legacy document should have nil file size")
}
func TestGetFolderMetrics_Success(t *testing.T) {
ctrl, cfg, clientID := setupFoldersTestController(t)
ctx := t.Context()
+8
View File
@@ -24,6 +24,7 @@ import (
documentinit "queryorchestration/internal/document/init"
"queryorchestration/internal/server/runner"
"queryorchestration/internal/serviceconfig/build"
"queryorchestration/internal/serviceconfig/objectstore"
documentsyncc "queryorchestration/internal/serviceconfig/queue/documentsync"
_ "github.com/lib/pq"
@@ -32,6 +33,7 @@ import (
type DocInitConfig struct {
runner.BaseConfig[docinitrunner.Body]
documentsyncc.DocSyncConfig
objectstore.ObjectStoreConfig // Provides GetStoreClient() for S3 access to measure file size
}
func main() {
@@ -57,6 +59,12 @@ func main() {
os.Exit(1)
}
// Initialize S3 client for file size measurement AFTER runner.New() parses env vars
if err := cfg.SetStoreClient(ctx); err != nil {
slog.Error("failed to initialize S3 client", "error", err)
os.Exit(1)
}
// optional add shutdown metrics
defer func() {
err = cfg.GetMetrics().RecordLifecycleEvent(1.0, "shutdown", "main")
Executable
+25
View File
@@ -0,0 +1,25 @@
#!/bin/bash
# Database connection parameters for test database
DB_HOST="localhost"
DB_PORT="5430"
DB_USER="postgres"
DB_PASSWORD="pass"
DB_NAME="query_orchestration"
# Export password to avoid prompt
export PGPASSWORD="${DB_PASSWORD}"
# Connect to the database
echo "Connecting to test database at ${DB_HOST}:${DB_PORT}..."
echo "Database: ${DB_NAME}"
echo "User: ${DB_USER}"
echo ""
echo "Useful commands:"
echo " \\dt - List all tables"
echo " \\d tablename - Describe table structure"
echo " \\l - List all databases"
echo " \\q - Quit"
echo ""
psql -h "${DB_HOST}" -p "${DB_PORT}" -U "${DB_USER}" -d "${DB_NAME}"
+11 -10
View File
@@ -28,7 +28,7 @@ services:
DB_NOSSL: ${DB_NOSSL}
AWS_ENDPOINT_URL: "http://localstack:4566"
AWS_S3_USE_PATH_STYLE: true
DISABLE_AUTH: true
DISABLE_AUTH: ${DISABLE_AUTH:-true}
COGNITO_USER_POOL_ID: "fillin"
COGNITO_CLIENT_SECRET: "fillin"
COGNITO_DOMAIN: "fillin"
@@ -56,6 +56,7 @@ services:
AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
AWS_SESSION_TOKEN: ${AWS_SESSION_TOKEN}
AWS_REGION: ${AWS_REGION}
BUCKET: ${BUCKET_IN}
PGUSER: ${PGUSER}
PGPASSWORD: ${PGPASSWORD}
PGHOST: aws-db
@@ -64,7 +65,7 @@ services:
DB_NOSSL: ${DB_NOSSL}
AWS_ENDPOINT_URL: "http://localstack:4566"
AWS_S3_USE_PATH_STYLE: true
DISABLE_AUTH: true
DISABLE_AUTH: ${DISABLE_AUTH:-true}
COGNITO_USER_POOL_ID: "fillin"
COGNITO_CLIENT_SECRET: "fillin"
COGNITO_DOMAIN: "fillin"
@@ -100,7 +101,7 @@ services:
DB_NOSSL: ${DB_NOSSL}
AWS_ENDPOINT_URL: "http://localstack:4566"
AWS_S3_USE_PATH_STYLE: true
DISABLE_AUTH: true
DISABLE_AUTH: ${DISABLE_AUTH:-true}
COGNITO_USER_POOL_ID: "fillin"
COGNITO_CLIENT_SECRET: "fillin"
COGNITO_DOMAIN: "fillin"
@@ -136,7 +137,7 @@ services:
DB_NOSSL: ${DB_NOSSL}
AWS_ENDPOINT_URL: "http://localstack:4566"
AWS_S3_USE_PATH_STYLE: true
DISABLE_AUTH: true
DISABLE_AUTH: ${DISABLE_AUTH:-true}
COGNITO_USER_POOL_ID: "fillin"
COGNITO_CLIENT_SECRET: "fillin"
COGNITO_DOMAIN: "fillin"
@@ -178,7 +179,7 @@ services:
AWS_ENDPOINT_URL_S3: "http://localstack:4566"
AWS_ENDPOINT_URL_SQS: "http://localstack:4566"
AWS_S3_USE_PATH_STYLE: true
DISABLE_AUTH: true
DISABLE_AUTH: ${DISABLE_AUTH:-true}
COGNITO_USER_POOL_ID: "fillin"
COGNITO_CLIENT_SECRET: "fillin"
COGNITO_DOMAIN: "fillin"
@@ -215,7 +216,7 @@ services:
DB_NOSSL: ${DB_NOSSL}
AWS_ENDPOINT_URL: "http://localstack:4566"
AWS_S3_USE_PATH_STYLE: true
DISABLE_AUTH: true
DISABLE_AUTH: ${DISABLE_AUTH:-true}
COGNITO_USER_POOL_ID: "fillin"
COGNITO_CLIENT_SECRET: "fillin"
COGNITO_DOMAIN: "fillin"
@@ -251,7 +252,7 @@ services:
DB_NOSSL: ${DB_NOSSL}
AWS_ENDPOINT_URL: "http://localstack:4566"
AWS_S3_USE_PATH_STYLE: true
DISABLE_AUTH: true
DISABLE_AUTH: ${DISABLE_AUTH:-true}
COGNITO_USER_POOL_ID: "fillin"
COGNITO_CLIENT_SECRET: "fillin"
COGNITO_DOMAIN: "fillin"
@@ -287,7 +288,7 @@ services:
DB_NOSSL: ${DB_NOSSL}
AWS_ENDPOINT_URL: "http://localstack:4566"
AWS_S3_USE_PATH_STYLE: true
DISABLE_AUTH: true
DISABLE_AUTH: ${DISABLE_AUTH:-true}
COGNITO_USER_POOL_ID: "fillin"
COGNITO_CLIENT_SECRET: "fillin"
COGNITO_DOMAIN: "fillin"
@@ -324,7 +325,7 @@ services:
DB_NOSSL: ${DB_NOSSL}
AWS_ENDPOINT_URL: "http://localstack:4566"
AWS_S3_USE_PATH_STYLE: true
DISABLE_AUTH: true
DISABLE_AUTH: ${DISABLE_AUTH:-true}
COGNITO_USER_POOL_ID: "fillin"
COGNITO_CLIENT_SECRET: "fillin"
COGNITO_DOMAIN: "fillin"
@@ -360,7 +361,7 @@ services:
DB_NOSSL: ${DB_NOSSL}
AWS_ENDPOINT_URL: "http://localstack:4566"
AWS_S3_USE_PATH_STYLE: true
DISABLE_AUTH: true
DISABLE_AUTH: ${DISABLE_AUTH:-true}
COGNITO_USER_POOL_ID: "fillin"
COGNITO_CLIENT_SECRET: "fillin"
COGNITO_DOMAIN: "fillin"
+11 -10
View File
@@ -28,7 +28,7 @@ services:
DB_NOSSL: ${DB_NOSSL}
AWS_ENDPOINT_URL: "http://localstack:4566"
AWS_S3_USE_PATH_STYLE: true
DISABLE_AUTH: true
DISABLE_AUTH: ${DISABLE_AUTH:-true}
COGNITO_USER_POOL_ID: "fillin"
COGNITO_CLIENT_SECRET: "fillin"
COGNITO_DOMAIN: "fillin"
@@ -56,6 +56,7 @@ services:
AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
AWS_SESSION_TOKEN: ${AWS_SESSION_TOKEN}
AWS_REGION: ${AWS_REGION}
BUCKET: ${BUCKET_IN}
PGUSER: ${PGUSER}
PGPASSWORD: ${PGPASSWORD}
PGHOST: db
@@ -64,7 +65,7 @@ services:
DB_NOSSL: ${DB_NOSSL}
AWS_ENDPOINT_URL: "http://localstack:4566"
AWS_S3_USE_PATH_STYLE: true
DISABLE_AUTH: true
DISABLE_AUTH: ${DISABLE_AUTH:-true}
COGNITO_USER_POOL_ID: "fillin"
COGNITO_CLIENT_SECRET: "fillin"
COGNITO_DOMAIN: "fillin"
@@ -100,7 +101,7 @@ services:
DB_NOSSL: ${DB_NOSSL}
AWS_ENDPOINT_URL: "http://localstack:4566"
AWS_S3_USE_PATH_STYLE: true
DISABLE_AUTH: true
DISABLE_AUTH: ${DISABLE_AUTH:-true}
COGNITO_USER_POOL_ID: "fillin"
COGNITO_CLIENT_SECRET: "fillin"
COGNITO_DOMAIN: "fillin"
@@ -136,7 +137,7 @@ services:
DB_NOSSL: ${DB_NOSSL}
AWS_ENDPOINT_URL: "http://localstack:4566"
AWS_S3_USE_PATH_STYLE: true
DISABLE_AUTH: true
DISABLE_AUTH: ${DISABLE_AUTH:-true}
COGNITO_USER_POOL_ID: "fillin"
COGNITO_CLIENT_SECRET: "fillin"
COGNITO_DOMAIN: "fillin"
@@ -173,7 +174,7 @@ services:
DB_NOSSL: ${DB_NOSSL}
AWS_ENDPOINT_URL: "http://localstack:4566"
AWS_S3_USE_PATH_STYLE: true
DISABLE_AUTH: true
DISABLE_AUTH: ${DISABLE_AUTH:-true}
COGNITO_USER_POOL_ID: "fillin"
COGNITO_CLIENT_SECRET: "fillin"
COGNITO_DOMAIN: "fillin"
@@ -210,7 +211,7 @@ services:
DB_NOSSL: ${DB_NOSSL}
AWS_ENDPOINT_URL: "http://localstack:4566"
AWS_S3_USE_PATH_STYLE: true
DISABLE_AUTH: true
DISABLE_AUTH: ${DISABLE_AUTH:-true}
COGNITO_USER_POOL_ID: "fillin"
COGNITO_CLIENT_SECRET: "fillin"
COGNITO_DOMAIN: "fillin"
@@ -246,7 +247,7 @@ services:
DB_NOSSL: ${DB_NOSSL}
AWS_ENDPOINT_URL: "http://localstack:4566"
AWS_S3_USE_PATH_STYLE: true
DISABLE_AUTH: true
DISABLE_AUTH: ${DISABLE_AUTH:-true}
COGNITO_USER_POOL_ID: "fillin"
COGNITO_CLIENT_SECRET: "fillin"
COGNITO_DOMAIN: "fillin"
@@ -285,7 +286,7 @@ services:
DB_NOSSL: ${DB_NOSSL}
AWS_ENDPOINT_URL: "http://localstack:4566"
AWS_S3_USE_PATH_STYLE: true
DISABLE_AUTH: false
DISABLE_AUTH: ${DISABLE_AUTH:-true}
COGNITO_USER_POOL_ID: ${COGNITO_USER_POOL_ID}
COGNITO_CLIENT_SECRET: ${COGNITO_CLIENT_SECRET}
COGNITO_DOMAIN: ${COGNITO_DOMAIN}
@@ -326,7 +327,7 @@ services:
DB_NOSSL: ${DB_NOSSL}
AWS_ENDPOINT_URL: "http://localstack:4566"
AWS_S3_USE_PATH_STYLE: true
DISABLE_AUTH: true
DISABLE_AUTH: ${DISABLE_AUTH:-true}
COGNITO_USER_POOL_ID: "fillin"
COGNITO_CLIENT_SECRET: "fillin"
COGNITO_DOMAIN: "fillin"
@@ -362,7 +363,7 @@ services:
DB_NOSSL: ${DB_NOSSL}
AWS_ENDPOINT_URL: "http://localstack:4566"
AWS_S3_USE_PATH_STYLE: true
DISABLE_AUTH: true
DISABLE_AUTH: ${DISABLE_AUTH:-true}
COGNITO_USER_POOL_ID: "fillin"
COGNITO_CLIENT_SECRET: "fillin"
COGNITO_DOMAIN: "fillin"
+114
View File
@@ -472,6 +472,120 @@ Queue Processing: < 1 second per message
- Resource utilization trends
```
## AWS IAM Requirements
### Required S3 Permissions
The document processing pipeline requires specific S3 permissions for file operations. Below are the minimum IAM permissions required for each service.
#### docInitRunner Service
The `docInitRunner` service requires `s3:GetObject` permission to measure file sizes during document initialization. This is used by the `HeadObject` operation to retrieve file metadata.
```json
{
"Version": "2012-10-17",
"Statement": [
{
"Sid": "DocumentInitS3Permissions",
"Effect": "Allow",
"Action": [
"s3:GetObject",
"s3:PutObject",
"s3:HeadObject"
],
"Resource": [
"arn:aws:s3:::${BUCKET_NAME}/*"
]
},
{
"Sid": "DocumentInitBucketAccess",
"Effect": "Allow",
"Action": [
"s3:ListBucket"
],
"Resource": [
"arn:aws:s3:::${BUCKET_NAME}"
]
}
]
}
```
**Note**: The `s3:GetObject` permission is required for the `HeadObject` API call that measures file sizes. Without this permission, document initialization will fail with an access denied error.
#### File Size Tracking (January 2026)
Documents now track file size at initialization time. The `docInitRunner` service measures file size using S3 `HeadObject` before storing documents. This requires:
- **Permission**: `s3:GetObject` on the document bucket
- **Environment**: `BUCKET` environment variable must be set
- **Error Handling**: If HeadObject fails, document creation will fail (not silently ignored)
#### Compose File Configuration
For local development with localstack, the bucket is configured via environment variables:
```yaml
# deployments/compose.local.yaml
docInitRunner:
environment:
BUCKET: ${BUCKET_IN} # Maps to the "documentin" bucket
```
For production deployments, ensure the IAM role attached to the service has the required S3 permissions above.
### SQS Permissions
Queue consumer services require SQS permissions:
```json
{
"Version": "2012-10-17",
"Statement": [
{
"Sid": "SQSConsumerPermissions",
"Effect": "Allow",
"Action": [
"sqs:ReceiveMessage",
"sqs:DeleteMessage",
"sqs:GetQueueAttributes",
"sqs:ChangeMessageVisibility"
],
"Resource": [
"arn:aws:sqs:${REGION}:${ACCOUNT_ID}:document_init",
"arn:aws:sqs:${REGION}:${ACCOUNT_ID}:document_sync",
"arn:aws:sqs:${REGION}:${ACCOUNT_ID}:document_clean",
"arn:aws:sqs:${REGION}:${ACCOUNT_ID}:document_text"
]
}
]
}
```
### Textract Permissions
For OCR text extraction, additional Textract permissions are required:
```json
{
"Version": "2012-10-17",
"Statement": [
{
"Sid": "TextractPermissions",
"Effect": "Allow",
"Action": [
"textract:StartDocumentTextDetection",
"textract:GetDocumentTextDetection",
"textract:StartDocumentAnalysis",
"textract:GetDocumentAnalysis"
],
"Resource": "*"
}
]
}
```
## Security Operations
### Security Monitoring
+1 -1
View File
@@ -33,6 +33,7 @@ require (
github.com/tidwall/gjson v1.18.0
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.34.0
go.opentelemetry.io/otel/sdk v1.34.0
golang.org/x/sync v0.12.0
golang.org/x/time v0.11.0
)
@@ -127,7 +128,6 @@ require (
golang.org/x/image v0.25.0 // indirect
golang.org/x/mod v0.23.0 // indirect
golang.org/x/oauth2 v0.27.0 // indirect
golang.org/x/sync v0.12.0 // indirect
golang.org/x/term v0.30.0 // indirect
golang.org/x/tools v0.30.0 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20250303144028-a0af3efb3deb // indirect
@@ -0,0 +1,2 @@
-- Remove file_size_bytes column from documents table
ALTER TABLE documents DROP COLUMN file_size_bytes;
@@ -0,0 +1,4 @@
-- Add file_size_bytes column to documents table
-- Stores the size of the document in bytes as measured from S3 HeadObject
-- NULL for legacy documents (before this feature was added)
ALTER TABLE documents ADD COLUMN file_size_bytes BIGINT;
+3 -2
View File
@@ -34,7 +34,7 @@ SELECT id, hash from documents where clientId = @clientId;
SELECT id, hash, filename from documents where batch_id = @batch_id;
-- name: CreateDocument :one
INSERT INTO documents (clientId, hash, batch_id, filename, folderId, originalPath) VALUES ($1, $2, $3, $4, $5, $6) RETURNING id;
INSERT INTO documents (clientId, hash, batch_id, filename, folderId, originalPath, file_size_bytes) VALUES ($1, $2, $3, $4, $5, $6, $7) RETURNING id;
-- name: AddDocumentEntry :exec
INSERT INTO documentEntries (documentId, bucket, key) VALUES ($1, $2, $3);
@@ -85,6 +85,7 @@ SELECT
d.hash,
d.folderId,
d.filename,
d.originalPath
d.originalPath,
d.file_size_bytes
FROM documents d
WHERE d.id = $1;
+2 -1
View File
@@ -95,7 +95,8 @@ SELECT
d.hash,
d.folderId,
d.filename,
d.originalPath
d.originalPath,
d.file_size_bytes
FROM documents d
WHERE d.folderId = $1
ORDER BY d.id;
+22 -16
View File
@@ -65,21 +65,22 @@ func (q *Queries) AddDocumentUpload(ctx context.Context, arg *AddDocumentUploadP
}
const createDocument = `-- name: CreateDocument :one
INSERT INTO documents (clientId, hash, batch_id, filename, folderId, originalPath) VALUES ($1, $2, $3, $4, $5, $6) RETURNING id
INSERT INTO documents (clientId, hash, batch_id, filename, folderId, originalPath, file_size_bytes) VALUES ($1, $2, $3, $4, $5, $6, $7) RETURNING id
`
type CreateDocumentParams struct {
Clientid string `db:"clientid"`
Hash string `db:"hash"`
BatchID *uuid.UUID `db:"batch_id"`
Filename *string `db:"filename"`
Folderid *uuid.UUID `db:"folderid"`
Originalpath *string `db:"originalpath"`
Clientid string `db:"clientid"`
Hash string `db:"hash"`
BatchID *uuid.UUID `db:"batch_id"`
Filename *string `db:"filename"`
Folderid *uuid.UUID `db:"folderid"`
Originalpath *string `db:"originalpath"`
FileSizeBytes *int64 `db:"file_size_bytes"`
}
// CreateDocument
//
// INSERT INTO documents (clientId, hash, batch_id, filename, folderId, originalPath) VALUES ($1, $2, $3, $4, $5, $6) RETURNING id
// INSERT INTO documents (clientId, hash, batch_id, filename, folderId, originalPath, file_size_bytes) VALUES ($1, $2, $3, $4, $5, $6, $7) RETURNING id
func (q *Queries) CreateDocument(ctx context.Context, arg *CreateDocumentParams) (uuid.UUID, error) {
row := q.db.QueryRow(ctx, createDocument,
arg.Clientid,
@@ -88,6 +89,7 @@ func (q *Queries) CreateDocument(ctx context.Context, arg *CreateDocumentParams)
arg.Filename,
arg.Folderid,
arg.Originalpath,
arg.FileSizeBytes,
)
var id uuid.UUID
err := row.Scan(&id)
@@ -101,18 +103,20 @@ SELECT
d.hash,
d.folderId,
d.filename,
d.originalPath
d.originalPath,
d.file_size_bytes
FROM documents d
WHERE d.id = $1
`
type GetDocumentEnrichedRow struct {
ID uuid.UUID `db:"id"`
Clientid string `db:"clientid"`
Hash string `db:"hash"`
Folderid *uuid.UUID `db:"folderid"`
Filename *string `db:"filename"`
Originalpath *string `db:"originalpath"`
ID uuid.UUID `db:"id"`
Clientid string `db:"clientid"`
Hash string `db:"hash"`
Folderid *uuid.UUID `db:"folderid"`
Filename *string `db:"filename"`
Originalpath *string `db:"originalpath"`
FileSizeBytes *int64 `db:"file_size_bytes"`
}
// Retrieves a document with extended metadata including folder and filename
@@ -123,7 +127,8 @@ type GetDocumentEnrichedRow struct {
// d.hash,
// d.folderId,
// d.filename,
// d.originalPath
// d.originalPath,
// d.file_size_bytes
// FROM documents d
// WHERE d.id = $1
func (q *Queries) GetDocumentEnriched(ctx context.Context, id uuid.UUID) (*GetDocumentEnrichedRow, error) {
@@ -136,6 +141,7 @@ func (q *Queries) GetDocumentEnriched(ctx context.Context, id uuid.UUID) (*GetDo
&i.Folderid,
&i.Filename,
&i.Originalpath,
&i.FileSizeBytes,
)
return &i, err
}
+14 -9
View File
@@ -89,14 +89,14 @@ func (q *Queries) GetClientRootFolder(ctx context.Context, clientid string) (*Fo
}
const getDocumentsByFolder = `-- name: GetDocumentsByFolder :many
SELECT id, clientid, hash, batch_id, filename, folderid, originalpath FROM documents
SELECT id, clientid, hash, batch_id, filename, folderid, originalpath, file_size_bytes FROM documents
WHERE folderId = $1
ORDER BY id
`
// GetDocumentsByFolder
//
// SELECT id, clientid, hash, batch_id, filename, folderid, originalpath FROM documents
// SELECT id, clientid, hash, batch_id, filename, folderid, originalpath, file_size_bytes FROM documents
// WHERE folderId = $1
// ORDER BY id
func (q *Queries) GetDocumentsByFolder(ctx context.Context, folderid *uuid.UUID) ([]*Document, error) {
@@ -116,6 +116,7 @@ func (q *Queries) GetDocumentsByFolder(ctx context.Context, folderid *uuid.UUID)
&i.Filename,
&i.Folderid,
&i.Originalpath,
&i.FileSizeBytes,
); err != nil {
return nil, err
}
@@ -133,18 +134,20 @@ SELECT
d.hash,
d.folderId,
d.filename,
d.originalPath
d.originalPath,
d.file_size_bytes
FROM documents d
WHERE d.folderId = $1
ORDER BY d.id
`
type GetDocumentsByFolderEnrichedRow struct {
ID uuid.UUID `db:"id"`
Hash string `db:"hash"`
Folderid *uuid.UUID `db:"folderid"`
Filename *string `db:"filename"`
Originalpath *string `db:"originalpath"`
ID uuid.UUID `db:"id"`
Hash string `db:"hash"`
Folderid *uuid.UUID `db:"folderid"`
Filename *string `db:"filename"`
Originalpath *string `db:"originalpath"`
FileSizeBytes *int64 `db:"file_size_bytes"`
}
// Gets documents in a folder with all enriched metadata fields
@@ -154,7 +157,8 @@ type GetDocumentsByFolderEnrichedRow struct {
// d.hash,
// d.folderId,
// d.filename,
// d.originalPath
// d.originalPath,
// d.file_size_bytes
// FROM documents d
// WHERE d.folderId = $1
// ORDER BY d.id
@@ -173,6 +177,7 @@ func (q *Queries) GetDocumentsByFolderEnriched(ctx context.Context, folderid *uu
&i.Folderid,
&i.Filename,
&i.Originalpath,
&i.FileSizeBytes,
); err != nil {
return nil, err
}
+6 -4
View File
@@ -191,7 +191,7 @@ func (q *Queries) GetDocumentLabels(ctx context.Context, documentid uuid.UUID) (
}
const getDocumentsByLabel = `-- name: GetDocumentsByLabel :many
SELECT DISTINCT d.id, d.clientid, d.hash, d.batch_id, d.filename, d.folderid, d.originalpath
SELECT DISTINCT d.id, d.clientid, d.hash, d.batch_id, d.filename, d.folderid, d.originalpath, d.file_size_bytes
FROM documents d
INNER JOIN documentLabels dl ON d.id = dl.documentId
WHERE d.clientId = $1 AND dl.label = $2
@@ -205,7 +205,7 @@ type GetDocumentsByLabelParams struct {
// GetDocumentsByLabel
//
// SELECT DISTINCT d.id, d.clientid, d.hash, d.batch_id, d.filename, d.folderid, d.originalpath
// SELECT DISTINCT d.id, d.clientid, d.hash, d.batch_id, d.filename, d.folderid, d.originalpath, d.file_size_bytes
// FROM documents d
// INNER JOIN documentLabels dl ON d.id = dl.documentId
// WHERE d.clientId = $1 AND dl.label = $2
@@ -227,6 +227,7 @@ func (q *Queries) GetDocumentsByLabel(ctx context.Context, arg *GetDocumentsByLa
&i.Filename,
&i.Folderid,
&i.Originalpath,
&i.FileSizeBytes,
); err != nil {
return nil, err
}
@@ -239,7 +240,7 @@ func (q *Queries) GetDocumentsByLabel(ctx context.Context, arg *GetDocumentsByLa
}
const getDocumentsByLabelAndFolder = `-- name: GetDocumentsByLabelAndFolder :many
SELECT DISTINCT d.id, d.clientid, d.hash, d.batch_id, d.filename, d.folderid, d.originalpath
SELECT DISTINCT d.id, d.clientid, d.hash, d.batch_id, d.filename, d.folderid, d.originalpath, d.file_size_bytes
FROM documents d
INNER JOIN documentLabels dl ON d.id = dl.documentId
WHERE d.folderId = $1 AND dl.label = $2
@@ -253,7 +254,7 @@ type GetDocumentsByLabelAndFolderParams struct {
// GetDocumentsByLabelAndFolder
//
// SELECT DISTINCT d.id, d.clientid, d.hash, d.batch_id, d.filename, d.folderid, d.originalpath
// SELECT DISTINCT d.id, d.clientid, d.hash, d.batch_id, d.filename, d.folderid, d.originalpath, d.file_size_bytes
// FROM documents d
// INNER JOIN documentLabels dl ON d.id = dl.documentId
// WHERE d.folderId = $1 AND dl.label = $2
@@ -275,6 +276,7 @@ func (q *Queries) GetDocumentsByLabelAndFolder(ctx context.Context, arg *GetDocu
&i.Filename,
&i.Folderid,
&i.Originalpath,
&i.FileSizeBytes,
); err != nil {
return nil, err
}
+2 -1
View File
@@ -425,7 +425,8 @@ type Document struct {
Filename *string `db:"filename"`
Folderid *uuid.UUID `db:"folderid"`
// Original path provided during upload. IMMUTABLE after creation - never modify this value.
Originalpath *string `db:"originalpath"`
Originalpath *string `db:"originalpath"`
FileSizeBytes *int64 `db:"file_size_bytes"`
}
type Documentclean struct {
+1
View File
@@ -103,6 +103,7 @@ func (s *Service) GetEnriched(ctx context.Context, id uuid.UUID, includeTextReco
Filename: docEnriched.Filename,
OriginalPath: docEnriched.Originalpath,
Labels: labels,
FileSizeBytes: docEnriched.FileSizeBytes,
}
// 5. Optionally fetch full text record if requested and exists
+53 -7
View File
@@ -4,6 +4,7 @@ import (
"context"
"database/sql"
"errors"
"fmt"
"log/slog"
docsyncrunner "queryorchestration/api/docSyncRunner"
@@ -11,6 +12,8 @@ import (
"queryorchestration/internal/serviceconfig/objectstore"
"queryorchestration/internal/serviceconfig/queue"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/google/uuid"
)
@@ -85,16 +88,24 @@ func (s *Service) getCreateParams(ctx context.Context, doc *Create) (*createDocu
func (s *Service) submitCreate(ctx context.Context, params *createDocumentParams) (uuid.UUID, error) {
var id uuid.UUID
err := s.cfg.ExecuteDBTransaction(ctx, func(ctx context.Context, q *repository.Queries) error {
// Measure file size from S3 (trusted source - we measure it ourselves)
// The S3 client must be initialized - file size tracking is required
fileSizeBytes, err := s.measureFileSize(ctx, params.Bucket, params.Key.String())
if err != nil {
return uuid.Nil, fmt.Errorf("failed to measure file size: %w", err)
}
err = s.cfg.ExecuteDBTransaction(ctx, func(ctx context.Context, q *repository.Queries) error {
var dbid uuid.UUID
if params.ID == nil {
createid, err := s.cfg.GetDBQueries().CreateDocument(ctx, &repository.CreateDocumentParams{
Clientid: params.Key.ClientID,
Hash: params.Hash,
BatchID: params.Key.BatchID,
Filename: params.Filename,
Folderid: params.FolderID,
Originalpath: params.OriginalPath,
Clientid: params.Key.ClientID,
Hash: params.Hash,
BatchID: params.Key.BatchID,
Filename: params.Filename,
Folderid: params.FolderID,
Originalpath: params.OriginalPath,
FileSizeBytes: fileSizeBytes,
})
if err != nil {
return err
@@ -127,3 +138,38 @@ func (s *Service) submitCreate(ctx context.Context, params *createDocumentParams
return id, nil
}
// measureFileSize retrieves the file size from S3 using HeadObject.
// Returns the file size in bytes, or an error if the S3 client is not configured
// or if the HeadObject call fails.
//
// Parameters:
// - ctx: Context for the S3 operation
// - bucket: S3 bucket name
// - key: S3 object key
//
// Returns:
// - *int64: Pointer to file size in bytes
// - error: Error if S3 client is nil or HeadObject fails
//
// Note: This operation requires s3:GetObject permission on the bucket/key.
func (s *Service) measureFileSize(ctx context.Context, bucket, key string) (*int64, error) {
storeClient := s.cfg.GetStoreClient()
if storeClient == nil {
return nil, errors.New("S3 client not initialized - required for file size measurement")
}
headOutput, err := storeClient.HeadObject(ctx, &s3.HeadObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(key),
})
if err != nil {
return nil, fmt.Errorf("HeadObject failed for s3://%s/%s: %w", bucket, key, err)
}
if headOutput.ContentLength == nil {
return nil, fmt.Errorf("HeadObject returned nil ContentLength for s3://%s/%s", bucket, key)
}
return headOutput.ContentLength, nil
}
+241 -2
View File
@@ -1,6 +1,7 @@
package documentinit
import (
"bytes"
"errors"
"fmt"
"testing"
@@ -9,10 +10,14 @@ import (
"queryorchestration/internal/database/repository"
"queryorchestration/internal/document"
"queryorchestration/internal/serviceconfig/objectstore"
"queryorchestration/internal/test"
objectstoremock "queryorchestration/mocks/objectstore"
queuemock "queryorchestration/mocks/queue"
"github.com/stretchr/testify/require"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/sqs"
"github.com/google/uuid"
"github.com/pashagolub/pgxmock/v3"
@@ -30,6 +35,11 @@ func TestCreate(t *testing.T) {
cfg.DBQueries = repository.New(pool)
mockSQS := queuemock.NewMockSQSClient(t)
cfg.QueueClient = mockSQS
// Set up mock S3 client for file size measurement
mockS3 := objectstoremock.NewMockS3Client(t)
cfg.StoreClient = mockS3
cfg.DocumentSyncURL = "/i/am/here"
svc := New(cfg)
@@ -48,11 +58,19 @@ func TestCreate(t *testing.T) {
Location: objectstore.Import,
}
// Mock HeadObject to return file size
fileSize := int64(1024)
mockS3.EXPECT().
HeadObject(mock.Anything, mock.MatchedBy(func(in *s3.HeadObjectInput) bool {
return *in.Bucket == bucket && *in.Key == key.String()
})).
Return(&s3.HeadObjectOutput{ContentLength: &fileSize}, nil)
pool.ExpectQuery("name: GetDocumentIDByHash :one").WithArgs(doc.Hash, doc.ClientID).WillReturnRows(
pgxmock.NewRows([]string{"id"}),
)
pool.ExpectBegin()
pool.ExpectQuery("name: CreateDocument :one").WithArgs(doc.ClientID, doc.Hash, (*uuid.UUID)(nil), (*string)(nil), (*uuid.UUID)(nil), (*string)(nil)).
pool.ExpectQuery("name: CreateDocument :one").WithArgs(doc.ClientID, doc.Hash, (*uuid.UUID)(nil), (*string)(nil), (*uuid.UUID)(nil), (*string)(nil), &fileSize).
WillReturnRows(
pgxmock.NewRows([]string{"id"}).
AddRow(doc.ID),
@@ -220,6 +238,10 @@ func TestSubmitCreate(t *testing.T) {
cfg.DBPool = pool
cfg.DBQueries = repository.New(pool)
// Set up mock S3 client for file size measurement
mockS3 := objectstoremock.NewMockS3Client(t)
cfg.StoreClient = mockS3
svc := New(cfg)
doc := document.DocumentSummary{
@@ -235,8 +257,16 @@ func TestSubmitCreate(t *testing.T) {
Location: objectstore.Import,
}
// Mock HeadObject to return file size
fileSize := int64(2048)
mockS3.EXPECT().
HeadObject(mock.Anything, mock.MatchedBy(func(in *s3.HeadObjectInput) bool {
return *in.Bucket == bucket && *in.Key == key.String()
})).
Return(&s3.HeadObjectOutput{ContentLength: &fileSize}, nil)
pool.ExpectBegin()
pool.ExpectQuery("name: CreateDocument :one").WithArgs(doc.ClientID, doc.Hash, (*uuid.UUID)(nil), (*string)(nil), (*uuid.UUID)(nil), (*string)(nil)).
pool.ExpectQuery("name: CreateDocument :one").WithArgs(doc.ClientID, doc.Hash, (*uuid.UUID)(nil), (*string)(nil), (*uuid.UUID)(nil), (*string)(nil), &fileSize).
WillReturnRows(
pgxmock.NewRows([]string{"id"}).
AddRow(doc.ID),
@@ -264,6 +294,10 @@ func TestSubmitCreateExists(t *testing.T) {
cfg.DBPool = pool
cfg.DBQueries = repository.New(pool)
// Set up mock S3 client for file size measurement
mockS3 := objectstoremock.NewMockS3Client(t)
cfg.StoreClient = mockS3
svc := New(cfg)
doc := document.DocumentSummary{
@@ -279,6 +313,14 @@ func TestSubmitCreateExists(t *testing.T) {
Location: objectstore.Import,
}
// Mock HeadObject to return file size (still called even for existing docs)
fileSize := int64(4096)
mockS3.EXPECT().
HeadObject(mock.Anything, mock.MatchedBy(func(in *s3.HeadObjectInput) bool {
return *in.Bucket == bucket && *in.Key == key.String()
})).
Return(&s3.HeadObjectOutput{ContentLength: &fileSize}, nil)
pool.ExpectBegin()
pool.ExpectExec("name: AddDocumentEntry :exec").WithArgs(doc.ID, bucket, key.String()).
WillReturnResult(pgxmock.NewResult("", 1))
@@ -294,3 +336,200 @@ func TestSubmitCreateExists(t *testing.T) {
require.NoError(t, err)
assert.Equal(t, doc.ID, id)
}
// TestSubmitCreate_HeadObjectError verifies that HeadObject errors are properly propagated.
func TestSubmitCreate_HeadObjectError(t *testing.T) {
ctx := t.Context()
pool, err := pgxmock.NewPool()
require.NoError(t, err)
cfg := &DocInitConfig{}
cfg.DBPool = pool
cfg.DBQueries = repository.New(pool)
// Set up mock S3 client that returns an error
mockS3 := objectstoremock.NewMockS3Client(t)
cfg.StoreClient = mockS3
svc := New(cfg)
doc := document.DocumentSummary{
ID: uuid.New(),
ClientID: "hello",
Hash: "example_hash",
}
bucket := "example_bucket"
key := objectstore.BucketKey{
ClientID: doc.ClientID,
CreatedAt: time.Now().UTC(),
EntityID: uuid.New(),
Location: objectstore.Import,
}
// Mock HeadObject to return an error
mockS3.EXPECT().
HeadObject(mock.Anything, mock.Anything).
Return(nil, errors.New("access denied"))
_, err = svc.submitCreate(ctx, &createDocumentParams{
Hash: doc.Hash,
Bucket: bucket,
Key: key,
})
assert.Error(t, err)
assert.Contains(t, err.Error(), "failed to measure file size")
assert.Contains(t, err.Error(), "access denied")
}
// TestSubmitCreate_NoS3Client verifies that missing S3 client returns an error.
func TestSubmitCreate_NoS3Client(t *testing.T) {
ctx := t.Context()
pool, err := pgxmock.NewPool()
require.NoError(t, err)
cfg := &DocInitConfig{}
cfg.DBPool = pool
cfg.DBQueries = repository.New(pool)
// Intentionally not setting StoreClient
svc := New(cfg)
doc := document.DocumentSummary{
ID: uuid.New(),
ClientID: "hello",
Hash: "example_hash",
}
bucket := "example_bucket"
key := objectstore.BucketKey{
ClientID: doc.ClientID,
CreatedAt: time.Now().UTC(),
EntityID: uuid.New(),
Location: objectstore.Import,
}
_, err = svc.submitCreate(ctx, &createDocumentParams{
Hash: doc.Hash,
Bucket: bucket,
Key: key,
})
assert.Error(t, err)
assert.Contains(t, err.Error(), "S3 client not initialized")
}
// TestMeasureFileSize_NilContentLength verifies handling of nil ContentLength.
func TestMeasureFileSize_NilContentLength(t *testing.T) {
ctx := t.Context()
pool, err := pgxmock.NewPool()
require.NoError(t, err)
cfg := &DocInitConfig{}
cfg.DBPool = pool
cfg.DBQueries = repository.New(pool)
// Set up mock S3 client that returns nil ContentLength
mockS3 := objectstoremock.NewMockS3Client(t)
cfg.StoreClient = mockS3
svc := New(cfg)
bucket := "example_bucket"
key := "example_key"
mockS3.EXPECT().
HeadObject(mock.Anything, mock.Anything).
Return(&s3.HeadObjectOutput{ContentLength: nil}, nil)
_, err = svc.measureFileSize(ctx, bucket, key)
assert.Error(t, err)
assert.Contains(t, err.Error(), "nil ContentLength")
}
// TestMeasureFileSize_Integration tests the measureFileSize function using a real
// S3 client (localstack) to verify that HeadObject correctly returns file sizes.
// This is an integration test that requires Docker.
func TestMeasureFileSize_Integration(t *testing.T) {
ctx := t.Context()
// Set up AWS container (localstack) and S3 client
cfg := &DocInitConfig{}
test.CreateAWSResources(t, cfg)
svc := New(cfg)
// Upload a test file with known content
bucket := cfg.GetBucket()
testKey := fmt.Sprintf("test/filesize/%s.txt", uuid.New().String())
testContent := []byte("Hello, World! This is a test file for measuring file size.")
expectedSize := int64(len(testContent))
// Upload the test file to S3
_, err := cfg.GetStoreClient().PutObject(ctx, &s3.PutObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(testKey),
Body: bytes.NewReader(testContent),
})
require.NoError(t, err, "failed to upload test file to S3")
// Measure the file size using our function
measuredSize, err := svc.measureFileSize(ctx, bucket, testKey)
require.NoError(t, err, "measureFileSize should not return an error")
require.NotNil(t, measuredSize, "measured size should not be nil")
// Verify the measured size matches expected
assert.Equal(t, expectedSize, *measuredSize, "measured file size should match uploaded content size")
}
// TestMeasureFileSize_Integration_LargeFile tests file size measurement for larger files.
func TestMeasureFileSize_Integration_LargeFile(t *testing.T) {
ctx := t.Context()
// Set up AWS container (localstack) and S3 client
cfg := &DocInitConfig{}
test.CreateAWSResources(t, cfg)
svc := New(cfg)
// Upload a larger test file (1MB)
bucket := cfg.GetBucket()
testKey := fmt.Sprintf("test/filesize/large/%s.bin", uuid.New().String())
expectedSize := int64(1024 * 1024) // 1MB
testContent := make([]byte, expectedSize)
for i := range testContent {
testContent[i] = byte(i % 256)
}
// Upload the test file to S3
_, err := cfg.GetStoreClient().PutObject(ctx, &s3.PutObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(testKey),
Body: bytes.NewReader(testContent),
})
require.NoError(t, err, "failed to upload large test file to S3")
// Measure the file size using our function
measuredSize, err := svc.measureFileSize(ctx, bucket, testKey)
require.NoError(t, err, "measureFileSize should not return an error for large files")
require.NotNil(t, measuredSize, "measured size should not be nil")
// Verify the measured size matches expected
assert.Equal(t, expectedSize, *measuredSize, "measured file size should match 1MB")
}
// TestMeasureFileSize_Integration_NonExistentKey tests error handling for non-existent S3 keys.
func TestMeasureFileSize_Integration_NonExistentKey(t *testing.T) {
ctx := t.Context()
// Set up AWS container (localstack) and S3 client
cfg := &DocInitConfig{}
test.CreateAWSResources(t, cfg)
svc := New(cfg)
bucket := cfg.GetBucket()
nonExistentKey := fmt.Sprintf("test/nonexistent/%s.txt", uuid.New().String())
// Try to measure size of non-existent file
_, err := svc.measureFileSize(ctx, bucket, nonExistentKey)
assert.Error(t, err, "measureFileSize should return an error for non-existent key")
assert.Contains(t, err.Error(), "HeadObject failed", "error should indicate HeadObject failure")
}
+2
View File
@@ -2,12 +2,14 @@ package documentinit
import (
"queryorchestration/internal/serviceconfig"
"queryorchestration/internal/serviceconfig/objectstore"
documentsync "queryorchestration/internal/serviceconfig/queue/documentsync"
)
type ConfigProvider interface {
serviceconfig.ConfigProvider
documentsync.ConfigProvider
objectstore.ConfigProvider // Provides GetStoreClient() for S3 access to measure file size
}
type Service struct {
+2
View File
@@ -4,6 +4,7 @@ import (
"testing"
"queryorchestration/internal/serviceconfig"
"queryorchestration/internal/serviceconfig/objectstore"
"queryorchestration/internal/serviceconfig/queue/documentsync"
"github.com/stretchr/testify/assert"
@@ -12,6 +13,7 @@ import (
type DocInitConfig struct {
serviceconfig.BaseConfig
documentsync.DocSyncConfig
objectstore.ObjectStoreConfig
}
func TestNewDocumentInitService(t *testing.T) {
+1
View File
@@ -36,6 +36,7 @@ type DocumentEnriched struct {
OriginalPath *string
Labels []LabelRecord
TextRecord *TextRecord // Only populated when includeTextRecord=true
FileSizeBytes *int64 // File size in bytes (nil for legacy documents)
}
// LabelRecord represents a label applied to a document.
+2
View File
@@ -248,6 +248,7 @@ type DocumentInFolder struct {
OriginalPath *string
Labels []LabelRecord
TextRecord *TextRecord // Only populated when includeTextRecord=true
FileSizeBytes *int64 // File size in bytes (nil for legacy documents)
}
// LabelRecord represents a label applied to a document.
@@ -368,6 +369,7 @@ func (s *Service) GetDocumentsEnriched(ctx context.Context, folderID uuid.UUID,
OriginalPath: doc.Originalpath,
Labels: labels,
TextRecord: textRecordsMap[doc.ID],
FileSizeBytes: doc.FileSizeBytes,
}
}
+152
View File
@@ -0,0 +1,152 @@
// Package test provides configuration utilities for test infrastructure.
// This file centralizes timeout configuration for CI/CD stability.
package test
import (
"fmt"
"log/slog"
"net/http"
"os"
"time"
)
// GetHTTPTimeout returns the HTTP client timeout for API calls.
// Default is 30 seconds, configurable via E2E_HTTP_TIMEOUT environment variable.
// In CI environments, this timeout ensures that stalled HTTP calls don't block
// indefinitely, which was a root cause of test hangs.
func GetHTTPTimeout() time.Duration {
if t := os.Getenv("E2E_HTTP_TIMEOUT"); t != "" {
if d, err := time.ParseDuration(t); err == nil {
return d
}
}
return 30 * time.Second
}
// GetPollingTimeout returns the timeout for polling operations like waiting
// for client status or mock endpoint verification.
// Default is 60 seconds locally, 180 seconds in CI (detected via CI or
// BITBUCKET_PIPELINE_UUID environment variables).
// Configurable via E2E_WAIT_TIMEOUT environment variable.
func GetPollingTimeout() time.Duration {
if t := os.Getenv("E2E_WAIT_TIMEOUT"); t != "" {
if d, err := time.ParseDuration(t); err == nil {
return d
}
}
if isCI() {
return 180 * time.Second // 3 minutes in CI
}
return 60 * time.Second // 1 minute locally
}
// GetPollInterval returns the interval between polling attempts.
// Default is 500ms, configurable via E2E_POLL_INTERVAL environment variable.
func GetPollInterval() time.Duration {
if t := os.Getenv("E2E_POLL_INTERVAL"); t != "" {
if d, err := time.ParseDuration(t); err == nil {
return d
}
}
return 500 * time.Millisecond
}
// GetHTTPClient returns an HTTP client with the configured timeout.
// This should be used instead of &http.Client{} to ensure all HTTP operations
// have bounded timeouts and won't hang indefinitely.
func GetHTTPClient() *http.Client {
return &http.Client{
Timeout: GetHTTPTimeout(),
}
}
// isCI returns true if running in a CI environment.
// Detects CI via the CI or BITBUCKET_PIPELINE_UUID environment variables.
func isCI() bool {
return os.Getenv("CI") != "" || os.Getenv("BITBUCKET_PIPELINE_UUID") != ""
}
// WaitForExternalReadiness waits for a service to be reachable from outside Docker.
// This addresses the race condition where a container's internal health check passes
// but the external port mapping isn't yet ready. This is particularly important in
// Docker-in-Docker (DinD) environments like Bitbucket Pipelines.
//
// Parameters:
// - uri: The base URI of the service (e.g., "http://localhost:32768")
// - healthPath: The health check endpoint path (e.g., "/health" or "/liveness/probe")
// - timeout: Maximum time to wait for the service to become reachable
//
// Returns an error if the service is not reachable within the timeout period.
func WaitForExternalReadiness(uri string, healthPath string, timeout time.Duration) error {
client := &http.Client{Timeout: 2 * time.Second}
deadline := time.Now().Add(timeout)
fullURL := fmt.Sprintf("%s%s", uri, healthPath)
slog.Info("Waiting for external readiness", "url", fullURL, "timeout", timeout)
var lastErr error
attempts := 0
for time.Now().Before(deadline) {
attempts++
resp, err := client.Get(fullURL)
if err == nil && resp.StatusCode == http.StatusOK {
resp.Body.Close()
slog.Info("Service is externally reachable", "url", fullURL, "attempts", attempts)
return nil
}
if err != nil {
lastErr = err
} else {
lastErr = fmt.Errorf("unexpected status code: %d", resp.StatusCode)
resp.Body.Close()
}
time.Sleep(500 * time.Millisecond)
}
return fmt.Errorf("service not reachable from test client after %v (%d attempts): %w", timeout, attempts, lastErr)
}
// WaitForAPIRoutes waits for API routes to be fully registered.
// This addresses a race condition where the health endpoint returns 200 but other
// API routes haven't been registered yet, causing "404 page not found" errors.
// This is particularly important in slow CI environments where route registration
// may take longer after the health endpoint becomes available.
//
// Parameters:
// - uri: The base URI of the service (e.g., "http://localhost:32768")
// - timeout: Maximum time to wait for routes to be registered
//
// Returns an error if routes are not available within the timeout period.
func WaitForAPIRoutes(uri string, timeout time.Duration) error {
client := &http.Client{Timeout: 2 * time.Second}
deadline := time.Now().Add(timeout)
// Check /query endpoint - should return 200 with empty array when routes are ready
// A "404 page not found" means routes aren't registered yet
routeURL := fmt.Sprintf("%s/query", uri)
slog.Info("Waiting for API routes to be registered", "url", routeURL, "timeout", timeout)
var lastErr error
attempts := 0
for time.Now().Before(deadline) {
attempts++
resp, err := client.Get(routeURL)
if err == nil {
// Any response other than connection error means routes are registered
// 200 = success, 401 = auth required (but route exists), etc.
// Only "404 page not found" indicates routes not yet registered
if resp.StatusCode != http.StatusNotFound {
resp.Body.Close()
slog.Info("API routes are registered", "url", routeURL, "attempts", attempts, "status", resp.StatusCode)
return nil
}
lastErr = fmt.Errorf("routes not registered yet: %d", resp.StatusCode)
resp.Body.Close()
} else {
lastErr = err
}
time.Sleep(500 * time.Millisecond)
}
return fmt.Errorf("API routes not registered after %v (%d attempts): %w", timeout, attempts, lastErr)
}
+219
View File
@@ -0,0 +1,219 @@
package test
import (
"net/http"
"net/http/httptest"
"testing"
"time"
"github.com/stretchr/testify/assert"
)
func TestGetHTTPTimeout(t *testing.T) {
// Test default value
timeout := GetHTTPTimeout()
assert.Equal(t, 30*time.Second, timeout, "default HTTP timeout should be 30 seconds")
// Test with environment variable
t.Setenv("E2E_HTTP_TIMEOUT", "45s")
timeout = GetHTTPTimeout()
assert.Equal(t, 45*time.Second, timeout, "HTTP timeout should respect E2E_HTTP_TIMEOUT env var")
// Test with invalid environment variable (should fall back to default)
t.Setenv("E2E_HTTP_TIMEOUT", "invalid")
timeout = GetHTTPTimeout()
assert.Equal(t, 30*time.Second, timeout, "invalid E2E_HTTP_TIMEOUT should fall back to default")
}
func TestGetPollingTimeout(t *testing.T) {
// Clear CI env vars to ensure we test non-CI default first
t.Setenv("CI", "")
t.Setenv("BITBUCKET_PIPELINE_UUID", "")
// Test default value (non-CI)
timeout := GetPollingTimeout()
assert.Equal(t, 60*time.Second, timeout, "default polling timeout should be 60 seconds")
// Test with environment variable override
t.Setenv("E2E_WAIT_TIMEOUT", "120s")
timeout = GetPollingTimeout()
assert.Equal(t, 120*time.Second, timeout, "polling timeout should respect E2E_WAIT_TIMEOUT env var")
// Test CI detection
t.Setenv("E2E_WAIT_TIMEOUT", "")
t.Setenv("CI", "true")
timeout = GetPollingTimeout()
assert.Equal(t, 180*time.Second, timeout, "polling timeout in CI should be 180 seconds")
// Test BITBUCKET_PIPELINE_UUID detection
t.Setenv("CI", "")
t.Setenv("BITBUCKET_PIPELINE_UUID", "some-uuid")
timeout = GetPollingTimeout()
assert.Equal(t, 180*time.Second, timeout, "polling timeout with BITBUCKET_PIPELINE_UUID should be 180 seconds")
}
func TestGetPollInterval(t *testing.T) {
// Test default value
interval := GetPollInterval()
assert.Equal(t, 500*time.Millisecond, interval, "default poll interval should be 500ms")
// Test with environment variable
t.Setenv("E2E_POLL_INTERVAL", "1s")
interval = GetPollInterval()
assert.Equal(t, 1*time.Second, interval, "poll interval should respect E2E_POLL_INTERVAL env var")
// Test with invalid environment variable (should fall back to default)
t.Setenv("E2E_POLL_INTERVAL", "invalid")
interval = GetPollInterval()
assert.Equal(t, 500*time.Millisecond, interval, "invalid E2E_POLL_INTERVAL should fall back to default")
}
func TestGetHTTPClient(t *testing.T) {
client := GetHTTPClient()
assert.NotNil(t, client, "GetHTTPClient should return a non-nil client")
assert.Equal(t, GetHTTPTimeout(), client.Timeout, "HTTP client timeout should match GetHTTPTimeout()")
}
func TestIsCI(t *testing.T) {
// Clear CI env vars to ensure consistent test behavior
t.Setenv("CI", "")
t.Setenv("BITBUCKET_PIPELINE_UUID", "")
// Test non-CI environment
assert.False(t, isCI(), "isCI should return false when CI env vars are not set")
// Test with CI env var
t.Setenv("CI", "true")
assert.True(t, isCI(), "isCI should return true when CI is set")
// Test with BITBUCKET_PIPELINE_UUID
t.Setenv("CI", "")
t.Setenv("BITBUCKET_PIPELINE_UUID", "some-uuid")
assert.True(t, isCI(), "isCI should return true when BITBUCKET_PIPELINE_UUID is set")
}
func TestWaitForExternalReadiness(t *testing.T) {
t.Run("success on first attempt", func(t *testing.T) {
// Create a test server that returns 200 OK
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/health" {
w.WriteHeader(http.StatusOK)
return
}
w.WriteHeader(http.StatusNotFound)
}))
defer server.Close()
err := WaitForExternalReadiness(server.URL, "/health", 5*time.Second)
assert.NoError(t, err, "should succeed when server is immediately ready")
})
t.Run("success after delay", func(t *testing.T) {
attempts := 0
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/health" {
attempts++
if attempts < 3 {
w.WriteHeader(http.StatusServiceUnavailable)
return
}
w.WriteHeader(http.StatusOK)
return
}
w.WriteHeader(http.StatusNotFound)
}))
defer server.Close()
err := WaitForExternalReadiness(server.URL, "/health", 5*time.Second)
assert.NoError(t, err, "should succeed after retries")
assert.GreaterOrEqual(t, attempts, 3, "should have made multiple attempts")
})
t.Run("timeout on non-responsive server", func(t *testing.T) {
// Create a server that always returns 503
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusServiceUnavailable)
}))
defer server.Close()
err := WaitForExternalReadiness(server.URL, "/health", 2*time.Second)
assert.Error(t, err, "should fail when server never becomes ready")
assert.Contains(t, err.Error(), "not reachable", "error should indicate service not reachable")
})
t.Run("timeout on unreachable server", func(t *testing.T) {
// Use an invalid URL that will fail to connect
err := WaitForExternalReadiness("http://127.0.0.1:1", "/health", 2*time.Second)
assert.Error(t, err, "should fail when server is unreachable")
assert.Contains(t, err.Error(), "not reachable", "error should indicate service not reachable")
})
}
func TestWaitForAPIRoutes(t *testing.T) {
t.Run("success when routes return 200", func(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/query" {
w.WriteHeader(http.StatusOK)
_, _ = w.Write([]byte("[]"))
return
}
w.WriteHeader(http.StatusNotFound)
}))
defer server.Close()
err := WaitForAPIRoutes(server.URL, 5*time.Second)
assert.NoError(t, err, "should succeed when routes return 200")
})
t.Run("success after routes become available", func(t *testing.T) {
attempts := 0
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/query" {
attempts++
if attempts < 3 {
// Simulate "404 page not found" before routes are registered
w.WriteHeader(http.StatusNotFound)
_, _ = w.Write([]byte("404 page not found"))
return
}
w.WriteHeader(http.StatusOK)
_, _ = w.Write([]byte("[]"))
return
}
w.WriteHeader(http.StatusNotFound)
}))
defer server.Close()
err := WaitForAPIRoutes(server.URL, 5*time.Second)
assert.NoError(t, err, "should succeed after routes become available")
assert.GreaterOrEqual(t, attempts, 3, "should have made multiple attempts")
})
t.Run("success when routes return 401", func(t *testing.T) {
// 401 means the route exists but requires auth - route is registered
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/query" {
w.WriteHeader(http.StatusUnauthorized)
return
}
w.WriteHeader(http.StatusNotFound)
}))
defer server.Close()
err := WaitForAPIRoutes(server.URL, 5*time.Second)
assert.NoError(t, err, "should succeed when routes return 401 (route exists)")
})
t.Run("timeout when routes never register", func(t *testing.T) {
// Server always returns 404 - routes never registered
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusNotFound)
_, _ = w.Write([]byte("404 page not found"))
}))
defer server.Close()
err := WaitForAPIRoutes(server.URL, 2*time.Second)
assert.Error(t, err, "should fail when routes never register")
assert.Contains(t, err.Error(), "not registered", "error should indicate routes not registered")
})
}
+18 -1
View File
@@ -70,7 +70,24 @@ func CreateFullNetwork(t testing.TB, ctx context.Context, cfg FullDependenciesCo
wg.Wait()
qService, err := queryapi.NewClientWithResponses(apiContainers[QueryAPIName].URI)
// Wait for external readiness before creating the client.
// This addresses the race condition where containers report healthy internally
// but the external port mapping isn't fully ready, causing 404 errors in CI.
err := WaitForExternalReadiness(apiContainers[QueryAPIName].URI, "/health", GetPollingTimeout())
require.NoError(t, err, "QueryAPI not externally reachable")
// Wait for API routes to be registered.
// The health endpoint may return 200 before all routes are registered,
// causing "404 page not found" errors on actual API calls.
err = WaitForAPIRoutes(apiContainers[QueryAPIName].URI, GetPollingTimeout())
require.NoError(t, err, "QueryAPI routes not registered")
// Create API client with bounded HTTP timeout to prevent indefinite hangs
// that were causing CI failures (goroutines blocking until 15-minute global timeout)
qService, err := queryapi.NewClientWithResponses(
apiContainers[QueryAPIName].URI,
queryapi.WithHTTPClient(GetHTTPClient()),
)
require.NoError(t, err)
return Network{
+8 -6
View File
@@ -103,7 +103,7 @@ func CreateMockServer(t testing.TB) *MockServer {
require.NoError(t, err)
return &MockServer{
Client: &http.Client{},
Client: GetHTTPClient(),
Internal: Address(fmt.Sprintf("http://%s:%d", mockServerAlias, port.Int())),
External: Address(fmt.Sprintf("http://%s:%d", host, externalPort.Int())),
Container: container,
@@ -193,11 +193,12 @@ func WaitForMockEndpoint(t testing.TB, server *MockServer, request MockRequest)
verifyURL := fmt.Sprintf("%s/mockserver/verify", server.External)
clientTimeout := 500 * time.Millisecond
client := &http.Client{}
// Use configured HTTP client with timeout to prevent indefinite hangs
client := GetHTTPClient()
timeout := time.After(60 * time.Second)
ticker := time.NewTicker(clientTimeout)
// Use configurable polling timeout (longer in CI environments)
timeout := time.After(GetPollingTimeout())
ticker := time.NewTicker(GetPollInterval())
defer ticker.Stop()
slog.Info("Attempting to process request", "body", jsonData)
@@ -238,7 +239,8 @@ func retrieveMatchingRequest(t testing.TB, server *MockServer, request MockReque
req, err := http.NewRequest("PUT", retrieveURL, nil)
require.NoError(t, err)
client := &http.Client{}
// Use configured HTTP client with timeout to prevent indefinite hangs
client := GetHTTPClient()
resp, err := client.Do(req)
require.NoError(t, err)
defer resp.Body.Close()
+4 -2
View File
@@ -88,8 +88,10 @@ func SetQueryForClient(t testing.TB, client queryapi.ClientWithResponsesInterfac
func WaitForClientStatus(t testing.TB, ctx context.Context, service queryapi.ClientWithResponsesInterface, id string, status queryapi.ClientStatus) {
t.Helper()
timeout := time.After(60 * time.Second)
ticker := time.NewTicker(500 * time.Millisecond) // Increased from 100ms to avoid rate limiting (2 req/s < 5 req/s limit)
// Use configurable polling timeout (longer in CI environments to handle slower infrastructure)
// and configurable poll interval to balance responsiveness with rate limiting concerns
timeout := time.After(test.GetPollingTimeout())
ticker := time.NewTicker(test.GetPollInterval())
defer ticker.Stop()
for {
+6
View File
@@ -582,6 +582,9 @@ type DocumentEnriched struct {
// Fields The fields and the value for the document
Fields map[string]interface{} `json:"fields"`
// FileSizeBytes File size in bytes (null for legacy documents)
FileSizeBytes *int64 `json:"fileSizeBytes,omitempty"`
// Filename Original filename of the document
Filename *string `json:"filename,omitempty"`
@@ -612,6 +615,9 @@ type DocumentID = openapi_types.UUID
// DocumentInFolder Document within a folder with enriched metadata (lighter format, excludes client_id and fields)
type DocumentInFolder struct {
// FileSizeBytes File size in bytes (null for legacy documents)
FileSizeBytes *int64 `json:"fileSizeBytes,omitempty"`
// Filename Original filename of the document
Filename *string `json:"filename,omitempty"`
+16
View File
@@ -43,11 +43,27 @@ tasks:
cmds:
- rm -rf mocks/*
- go tool mockery --log-level=""
cleanup:
desc: "Remove stale test containers from previous runs"
cmds:
- |
echo "Cleaning up stale test containers..."
# Remove specific test containers if they exist (running or stopped)
for container in postgres_test_queryorchestration localstack_test_queryorchestration mockserver_test_queryorchestration; do
if docker ps -a --format '{{.Names}}' | grep -q "^${container}$"; then
echo "Removing container: $container"
docker rm -f "$container" 2>/dev/null || true
fi
done
# Also remove any queryorchestration API/runner containers
docker ps -a --format '{{.Names}}' | grep -E '_test_queryorchestration' | xargs -r docker rm -f 2>/dev/null || true
echo "Cleanup complete"
functional:
vars:
INCREMENTAL: "{{.INCREMENTAL | default \"true\"}}"
cmds:
- mkdir -p {{.OUT_DIR}}
- task: cleanup
- task: wait
- |
# Determine test mode first
+12
View File
@@ -2492,6 +2492,12 @@ components:
description: "All labels applied to this document"
textRecord:
$ref: "#/components/schemas/FieldExtractionResponse"
fileSizeBytes:
type: integer
format: int64
minimum: 0
maximum: 4294967295
description: "File size in bytes (null for legacy documents)"
DocumentInFolder:
description: Document within a folder with enriched metadata (lighter format, excludes client_id and fields)
@@ -2530,6 +2536,12 @@ components:
description: "All labels applied to this document"
textRecord:
$ref: "#/components/schemas/FieldExtractionResponse"
fileSizeBytes:
type: integer
format: int64
minimum: 0
maximum: 4294967295
description: "File size in bytes (null for legacy documents)"
ClientCreate:
description: The properties for creation.
+96 -72
View File
@@ -1,7 +1,6 @@
package endtoend_test
import (
"sync"
"testing"
"queryorchestration/internal/serviceconfig"
@@ -14,6 +13,7 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"golang.org/x/sync/errgroup"
)
type Config struct {
@@ -29,46 +29,54 @@ func TestProcess(t *testing.T) {
net, clean := test.CreateFullNetwork(t, t.Context(), cfg)
defer clean()
var wg sync.WaitGroup
var clientId queryapi.ClientID
var jsonId queryapi.QueryID
var textractExpectation test.MockExpectation
textractBody := "Hello World"
wg.Add(2)
go func(t testing.TB) {
// Use errgroup to properly propagate errors and avoid indefinite hangs
// If any goroutine fails, g.Wait() returns the error immediately
g := new(errgroup.Group)
g.Go(func() error {
_, jsonId = queryapitest.CreateDependentQueries(t, net.Client)
wg.Done()
}(t)
go func(t testing.TB) {
return nil
})
g.Go(func() error {
clientId = queryapitest.CreateClientWithSync(t, net.Client)
wg.Done()
}(t)
return nil
})
wg.Wait()
if err := g.Wait(); err != nil {
t.Fatalf("setup phase 1 failed: %v", err)
}
wg.Add(3)
go func(t testing.TB) {
g = new(errgroup.Group)
g.Go(func() error {
textractExpectation = test.CreateDetectDocumentTextExpectation(
t,
net.Dependencies.MockServer,
textractBody,
)
wg.Done()
}(t)
go func(t testing.TB) {
return nil
})
g.Go(func() error {
queryapitest.SetQueryForClient(t, net.Client, clientId, jsonId)
wg.Done()
}(t)
go func(t testing.TB) {
return nil
})
g.Go(func() error {
queryapitest.CreateFile(t, net.Client, queryapitest.File{
Filename: "helloworld.pdf",
Content: []byte(pdfHelloWorld),
ClientID: clientId,
})
wg.Done()
}(t)
return nil
})
wg.Wait()
if err := g.Wait(); err != nil {
t.Fatalf("setup phase 2 failed: %v", err)
}
test.WaitForMockEndpoint(t, net.Dependencies.MockServer, textractExpectation.Request)
queryapitest.WaitForClientStatus(t, t.Context(), net.Client, clientId, queryapi.INSYNC)
@@ -112,46 +120,52 @@ func TestProcess(t *testing.T) {
net, clean := test.CreateFullNetwork(t, t.Context(), cfg)
defer clean()
var wg sync.WaitGroup
var clientId queryapi.ClientID
var jsonId queryapi.QueryID
var textractExpectation test.MockExpectation
textractBody := "Hello World"
wg.Add(2)
go func(t testing.TB) {
g := new(errgroup.Group)
g.Go(func() error {
_, jsonId = queryapitest.CreateDependentQueries(t, net.Client)
wg.Done()
}(t)
go func(t testing.TB) {
return nil
})
g.Go(func() error {
clientId = queryapitest.CreateClientWithSync(t, net.Client)
wg.Done()
}(t)
return nil
})
wg.Wait()
if err := g.Wait(); err != nil {
t.Fatalf("setup phase 1 failed: %v", err)
}
wg.Add(3)
go func(t testing.TB) {
g = new(errgroup.Group)
g.Go(func() error {
queryapitest.SetQueryForClient(t, net.Client, clientId, jsonId)
wg.Done()
}(t)
go func(t testing.TB) {
return nil
})
g.Go(func() error {
queryapitest.CreateFile(t, net.Client, queryapitest.File{
Filename: "helloworld.pdf",
Content: []byte(pdfHelloWorld),
ClientID: clientId,
})
wg.Done()
}(t)
go func(t testing.TB) {
return nil
})
g.Go(func() error {
textractExpectation = test.CreateDetectDocumentTextExpectation(
t,
net.Dependencies.MockServer,
textractBody,
)
wg.Done()
}(t)
return nil
})
wg.Wait()
if err := g.Wait(); err != nil {
t.Fatalf("setup phase 2 failed: %v", err)
}
test.WaitForMockEndpoint(t, net.Dependencies.MockServer, textractExpectation.Request)
queryapitest.WaitForClientStatus(t, t.Context(), net.Client, clientId, queryapi.INSYNC)
@@ -194,13 +208,14 @@ func TestProcess(t *testing.T) {
net, clean := test.CreateFullNetwork(t, t.Context(), cfg)
defer clean()
var wg sync.WaitGroup
var clientId queryapi.ClientID
var jsonId queryapi.QueryID
var textractExpectation test.MockExpectation
textractBody := "Hello World"
wg.Add(2)
go func(t testing.TB) {
g := new(errgroup.Group)
g.Go(func() error {
_, jsonId = queryapitest.CreateDependentQueries(t, net.Client)
jcfg := `{"path":"keytwo"}`
@@ -208,39 +223,43 @@ func TestProcess(t *testing.T) {
Config: &jcfg,
})
require.NoError(t, err)
wg.Done()
}(t)
go func(t testing.TB) {
return nil
})
g.Go(func() error {
clientId = queryapitest.CreateClientWithSync(t, net.Client)
wg.Done()
}(t)
return nil
})
wg.Wait()
if err := g.Wait(); err != nil {
t.Fatalf("setup phase 1 failed: %v", err)
}
wg.Add(3)
go func(t testing.TB) {
g = new(errgroup.Group)
g.Go(func() error {
textractExpectation = test.CreateDetectDocumentTextExpectation(
t,
net.Dependencies.MockServer,
textractBody,
)
wg.Done()
}(t)
go func(t testing.TB) {
return nil
})
g.Go(func() error {
queryapitest.SetQueryForClient(t, net.Client, clientId, jsonId)
wg.Done()
}(t)
go func(t testing.TB) {
return nil
})
g.Go(func() error {
queryapitest.CreateFile(t, net.Client, queryapitest.File{
Filename: "helloworld.pdf",
Content: []byte(pdfHelloWorld),
ClientID: clientId,
})
wg.Done()
}(t)
return nil
})
wg.Wait()
if err := g.Wait(); err != nil {
t.Fatalf("setup phase 2 failed: %v", err)
}
test.WaitForMockEndpoint(t, net.Dependencies.MockServer, textractExpectation.Request)
queryapitest.WaitForClientStatus(t, t.Context(), net.Client, clientId, queryapi.INSYNC)
@@ -250,28 +269,33 @@ func TestProcess(t *testing.T) {
assert.Len(t, *docs.JSON200, 1)
doc := (*docs.JSON200)[0]
wg.Add(2)
go func(t testing.TB) {
g = new(errgroup.Group)
// Capture doc.Id to avoid closure issues
docId := doc.Id
g.Go(func() error {
testRes, err := net.Client.TestQueryWithResponse(t.Context(), jsonId, queryapi.QueryTestRequest{
QueryVersion: 1,
DocumentId: doc.Id,
DocumentId: docId,
})
require.NoError(t, err)
assert.Equal(t, "valueone", testRes.JSON200.Value)
wg.Done()
}(t)
return nil
})
go func(t testing.TB) {
g.Go(func() error {
testRes, err := net.Client.TestQueryWithResponse(t.Context(), jsonId, queryapi.QueryTestRequest{
QueryVersion: 2,
DocumentId: doc.Id,
DocumentId: docId,
})
require.NoError(t, err)
assert.Equal(t, "valuetwo", testRes.JSON200.Value)
wg.Done()
}(t)
return nil
})
wg.Wait()
if err := g.Wait(); err != nil {
t.Fatalf("test phase failed: %v", err)
}
})
}