diff --git a/CLAUDE.MD b/CLAUDE.MD index 283813e9..cb0a1ef5 100644 --- a/CLAUDE.MD +++ b/CLAUDE.MD @@ -15,6 +15,7 @@ When something doesn't work, debug and fix it - don't start over with a simple v Don’t build things I am not specifically asking for unless its required to build the thing I am asking for. When asked to make unit tests avoid mocks and use the actual thing that is being tested when reasonable. For commands that read such as grep or cat there is no need to ask for permission. +Do not assume that we need backward compatibilty when implementing new features. If there is a decision to make that will affect this ask. Do not guess. ## IDE related diff --git a/api/docInitRunner/runner_test.go b/api/docInitRunner/runner_test.go index 2a1f2b80..2613e154 100644 --- a/api/docInitRunner/runner_test.go +++ b/api/docInitRunner/runner_test.go @@ -1,6 +1,7 @@ package docinitrunner_test import ( + "bytes" "regexp" "testing" "time" @@ -14,6 +15,8 @@ import ( "queryorchestration/internal/serviceconfig/queue/documentsync" "queryorchestration/internal/test" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/google/uuid" "github.com/jackc/pgx/v5/pgtype" "github.com/stretchr/testify/assert" @@ -37,10 +40,15 @@ func TestDocInitRunner(t *testing.T) { test.CreateDB(t, cfg) acfg := test.CreateAWSContainer(t, cfg) + // Set up SQS queue test.SetQueueClient(t, t.Context(), cfg, acfg.ExternalEndpoint) cfg.DocumentSyncURL = test.CreateQueue(t, cfg, test.DocSyncRunnerName) - runner := docinitrunner.New(&docinitrunner.Services{ + // Set up S3 bucket (required for file size measurement) + test.SetStoreClient(t, t.Context(), cfg, acfg.ExternalEndpoint) + test.CreateBucket(t, cfg) + + runnerInstance := docinitrunner.New(&docinitrunner.Services{ Document: documentinit.New(cfg), Queries: cfg.GetDBQueries(), }) @@ -57,12 +65,21 @@ func TestDocInitRunner(t *testing.T) { CreatedAt: time.Now().UTC(), } + // Upload a test file to S3 (required for HeadObject to work) + testContent := []byte("test document content for file size measurement") + _, err = cfg.GetStoreClient().PutObject(t.Context(), &s3.PutObjectInput{ + Bucket: aws.String(cfg.GetBucket()), + Key: aws.String(location.String()), + Body: bytes.NewReader(testContent), + }) + require.NoError(t, err) + // Create a document upload record to simulate the upload pipeline uploadID := uuid.New() err = cfg.GetDBQueries().AddDocumentUpload(t.Context(), &repository.AddDocumentUploadParams{ ID: uploadID, Clientid: "clientid", - Bucket: "bucket", + Bucket: cfg.GetBucket(), Key: location.String(), Part: 1, Createdat: pgtype.Timestamp{ @@ -75,12 +92,12 @@ func TestDocInitRunner(t *testing.T) { require.NoError(t, err) doc := docinitrunner.Body{ - Bucket: "bucket", + Bucket: cfg.GetBucket(), Key: location.String(), Hash: "hash", } - assert.True(t, runner.Process(t.Context(), doc)) + assert.True(t, runnerInstance.Process(t.Context(), doc)) test.AssertMessageBody(t, cfg, cfg.GetDocumentSyncURL(), regexp.MustCompile(`{"id":".+"}`)) } diff --git a/api/queryAPI/api.gen.go b/api/queryAPI/api.gen.go index a5c7fd17..67604607 100644 --- a/api/queryAPI/api.gen.go +++ b/api/queryAPI/api.gen.go @@ -584,6 +584,9 @@ type DocumentEnriched struct { // Fields The fields and the value for the document Fields map[string]interface{} `json:"fields"` + // FileSizeBytes File size in bytes (null for legacy documents) + FileSizeBytes *int64 `json:"fileSizeBytes,omitempty"` + // Filename Original filename of the document Filename *string `json:"filename,omitempty"` @@ -614,6 +617,9 @@ type DocumentID = openapi_types.UUID // DocumentInFolder Document within a folder with enriched metadata (lighter format, excludes client_id and fields) type DocumentInFolder struct { + // FileSizeBytes File size in bytes (null for legacy documents) + FileSizeBytes *int64 `json:"fileSizeBytes,omitempty"` + // Filename Original filename of the document Filename *string `json:"filename,omitempty"` @@ -2331,232 +2337,233 @@ func RegisterHandlersWithBaseURL(router EchoRouter, si ServerInterface, baseURL // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/+y9CXMbt7Iw+ldQfPdV5ISLVi96detdRUui82xLR5Jf7j2RrwqcAUnEQ4ABMJJpl//7", - "V2gAM5gZzHBIk7KTo1OnKhYHS6PRGxqN7s+diE9nnBGmZOfwc2dCcEwE/PMKK/KaTqnSf8RERoLOFOWs", - "cwifUKK/IcpGXEyx/oAoQ+YPdNuBr//PA2Uxf/hPRaekZ/592+l0O+Qjns4S0jns7Gxvu0Y7251uR0YT", - "MsV6xmCb57rNFH98TdhYTTqHe7vdzgwrRYQG639/3+69ev+Ta2z++o9Ot6PmMz2QVIKycefLly+6l8BT", - "ouxaf8YqmpyfVFd6MyFoqD+idJZwHKPzk36n26H62wyrSafbYXiqB4dWdzTudDuC/JlSQeLOoRIp8Rf1", - "H4KMOoed/2uQY31gvsqBg0FDd5xQwlQdQBF8rQflK4DIJtZQnPAonTbAEdvvG4HEm1zDcvpxxkUtJAS+", - "bgSObGINxT9TIuZ1QJyfID5CakLQn7rZ+kFxs2tI3kkiTqeYJlVY3l297hEW8ZjEKJVEIKLbIRzHgkhZ", - "Axa0aYTMsLbXtMCH20EuE0TOOJPEMln8C1bkAc/1XxFnijAQLng2S2gEMmTwh9Rr+Nx2c4Tg4g2REo+J", - "mbGIitOPWjTgBEki7mlEENEdNArqBF1oNtt2kDeEqY45GyU0Uo+2misieSoignAiCI7niHykUknEBZJK", - "i+TIQrSmBZ5xMaRxTNijrfCcyXQ0ohHItxkRUyol5UwixfWfmgSRmlCJcKR7rGmd58xQCUD3iGv1aFNz", - "6RpJ85zd44TGV+TPlEj1iEuCaZEw86Ihj+drWtFbrs54yuLHZzbGFRrpqde0khvO32A2t3sjH3NBIItR", - "OuMMKc7RFLO52yu5htV1O1dEiXnvaKSIqOqlt+l0SITWkZJEnMUSDcmIC4KUmFM2RniMKbB0ZvrtaLUS", - "0ECUqb1do4HoNJ12Dl8+39/e7namlJm/c21EmSJjIjRCtNZkOFUTLugnEj8+4h8mhKHUA2EtFPXFoQjG", - "OIqnlGnj4AgkpJs7YMM7qEZcWDOB4WFCBjGV+r9WxEr0QNUEyTSKiJSgZ1KJMIuRNuqlwtNZp9uZCT4j", - "QlGj6E3P6pQGJCfIiTaECNPb9XvHTgq/wD/e+6eE/GvJxuh2Ij5mVPE7mQ7/IJHSBnhl3mPTBtk2iMaE", - "KTqiRMDitcVmQEbOXimcUfDOcDfai/d75GD0vPfi5avtHh5GcY+Mdnb39g+e61+K5tDuwfPCuaQfOoV0", - "rSFVARfMOmdKws7gSGlbbsZZAbA/+IT1Y07+y/7Uj/i0013aUOt27O5WQfltQtSEFFAEbUlsds+BYmxF", - "O+6Q84RgpgfOaaRqMbtPbqWWavwF7m7vHvR2tns7z2929g93Dw739v7lLzDGivT0HMVFHgSMUd+y/T1b", - "cDfDkZ39fdaTA63oRWQsdSwIVkFWytUdUFSkG4JIQ4w8mD0ENoLZukjb3F1gIsE1p0lJx2wKZ/AyL9XQ", - "yLuKXY+2UklihCVy5K6n1TM9WwvR+Mds3Pt01PvXdu9V/+7//un2tvf+p//yfoMfbm/79qf3n3e7X4L0", - "P6JCqjtzAAkt8AeJxvSeMMAXIBZHEU+ZsgguUcs/+IQVAd+xSiH7uw1XJngRUCM8pcm8JVQnnKwBKE0n", - "Af58TaXS/ANk9IHMwVA25IQoQ5faglZ9ytEW6Y/7XSTTGRF3WBN0F+jD/RunMVVcdK1v4U5/e9ZHV3pa", - "NE2lMmcNPeicp8IbmbB7KjhQbx9lBiD0G2nylDQhTCVz1EPRhEQfzLc7AySJ9YhO6hZ0/++dHD7NnwZA", - "zaBUkSngoozUxWic4o/npveB2QT7107WFguB5xV54ZjCo1ifUNz+tJAeLRQy1uaTL0M0IrqIsihJY/1L", - "WZ+dn4AskXMWWQVdESOt1CQAaHwGP3hihNE/UxJQno+jJCMD1R1WTToETKtMXz5giWw/TV/6dzmXikyb", - "tMve4f7Bitql215Mb0aBG6Mpblbgkbe/iEpr7wGG7F63UelNQrtEQbnwbiOnv0oyl2b2JHQLYbx4ZnBF", - "qDvNYU1YzojP2hejNEnmPiXmYtMdAozn3FBnH52P0AgnknStRW78O/kGoa0IM+irmTDCijxDw1TBGbUo", - "7ydYIlaaZuB5VJ710RvMUpwYscEFEvrshqZ4joYEOeHXb0MSRYkekGygDdQEK/RABClix1cEGfwaOIAD", - "a7kjiVFyxswiTr3QEZJcq2BMExL30TGfzrAgxtIqNwa5rrVjTJQWY7nNBf1TQSSgn6VJorFBpjM172oU", - "mv4AeAbrFmyPxXERqxcsmaOZIFIPTUeoQDqa7TQWH1vTFZVbt2O1RO1JCZaXqZKcf84uro5P745/PXr7", - "y+nd5dH19W8XVydV8bgIvJJ2DSin3DDPFVqBBxtV7QlJSCtVm519Y91Dc6I57ZpT7kjwKSI4mlj+RFsO", - "P1rdZsT6bDVtW38otQcigMnKtMfRtXbGO73wKsQnDkc16KlDQwt56ZYKQy6hjAxNLDtBtnOLpwhTakaN", - "QTJsc6yv3dvHPdNnZL+xU72bocnyenG4c7Cpc32BpBcIDYVpEhSKGmxlbUv/wtsn16JM8Kx1q398j9lq", - "9vm7lna5AcszEnIL4/s12d0Z2jPdwyRzcLP96nDn4HBve93G+mnBnVL0vwFwj2y7H6dCaCPC2elW7PqA", - "rc90r/pbtD1IyX2DVN6IJR/wsbSAZFXDvsavYizWoGlatNyc0XZPyQP5RjabJZOC7ebYKWDDHV+8PTu/", - "enO6gt3W7aSzuAVr8xHSO4xM6ybRv324s70e0d/OiHTM1qgJXlMZWN4lHlMGB7nEet00pu39iOIKJ8jg", - "XIv6mWmsJe+UKBxjhSsyf4Ll3ZQLUq+i9Vc9FJxfCML3mCbummShaYTHpJ5Y9FfE4EKucOHWbbhk29ne", - "Llyy7VQv2cy0d5J+Ik33fwZxMyIADh8ALSEbIVg0P2xEgDZhf1gJgilW0UTr6BFNVAkT+y+aINnd2X+x", - "/3LvuW7VdOvYhbNdg+fWAAKXBt7WdDw50nT1WDFeClIE0NXo1zTAOaxZovE3sZFP3hn2XnwfAnJA49nI", - "JqUEHaaKSLQFKsneh4DIKF6NyOrZqlGJGelUvjVw6rKkuxhWE7zpGwMLUfnKIAjSCSe96ymFsKwNXRhk", - "tmzS9uagh65OL18fHZ9eI5wkxiWVm7VbjCttMlFF78mzPrrh+i+E4bPzWRPo6ejbdJwlqQQsMPKAOCPQ", - "VZApvydZbw4Bpspofbgi1EQcvpX4mgsJd/1gG3GBRhQCcxRWpI8uNBgQdTSiJImRtqoFScCblBB8T+zo", - "KYsmmI1LTrNvfo9R5V/96UwvRfeq0sg1ZeNEL+pBE4hxnJmVazWGtnZ2ds3fgKupJgbOoH2VWzEWRJET", - "Iug9iY+ATji7woocA6q0yZBQRkpY2N0OGcqFsY4TTKc38xk5jkMobO58Rsh1NCFxmqwws9f5/ydC2kiH", - "5QD4RfB0RkS4fwsYXvPh8p3eEvXAxYflO14KHqcmznHpjmOBpyt1vL+BFsv2vCJ0OnxD1ITH7Tpbmjwh", - "MqruY1OPN/jjGSGals9ZVAhPinlqjLTMXHiV/a//6lXQYrC2WHH8y0i1Hr+vR180dKomWnsrjdvAig+C", - "Cx7SJFmC0/LmrWeIsLgnPFUtZ3DNz004YNX2jbRwyBbadlTd6RoOCqv0aL/Ymdq/FDxqO0nWfOkZ3vDl", - "pnjD288RkxFOk/otiCket1yhadp+ZjmxfLdGpovlpH4pcmIZcT1cSD5O6JCqS3tAW4gf2/6Gqoq6CmNo", - "hCOaUDU/iv9IpdKG0A0R01Zdx1OyfuSOp6QWueMpWStyx+A7FBej+hmN7j1KIPxa262vyT1JLkbHWLRT", - "OHaEn7FcO6bMyC0ZJ2vdmndsj19xtAg9a94UGPOKaCvY3JQumv+Kyg8XozdcKKxJ+TodRgmWchm8XJN7", - "Iqiar9JnEXg3AjM5ImJhu6oRE56fboLxaAPj0TUzXkKkbOS7hA+tIXlFEnCNyQmdtWI401Wbkkt3ZXFb", - "RQstW3MSU3i2/g3To9YiUH9c65bxVCWUiNOPUZJqpmyJqGq31lizXc+okOqEJwmuZ6Cs6Ud93JHy7VCc", - "4Lm8mQgiJzyJW2K93xYL2UTLzrDEHG9c3w0NfAZRLyyarxs5myC71jbJDCtKmDoakzf4Y6VHcwfKWnVI", - "cEQuRtfmkWBLNih3as0EM8Hvr2ckojhR87aTFfssNdcN/sgZny4zVd6l9UwCszGxTNpqHoEVOZURTrDi", - "YolpvF61wqPQ6g3+aE/Rl8aJ8fUkXBg/7M5aC/sJQqfDY87ujaPoDEfKvFBsN3q78U9HIxJpO/ikhB1z", - "d7eIeWCM9atCGHatcseMKPj9W+uqX0xsWY8ZrVgbTe1vKGvdvrUczFrbm8VVt+uesLStkMtat2ZQ+9K6", - "9aqk4jOtdk8wTeaWVddIRm74ZY2cSr/2CLA92xg5edsNGh9ukvVbH6WR129+uAmsHLhgpx8jIuXxBIux", - "vQ78arHg5rgqKbC1LqA1OyhBx2PjW9gELdjhj/EsG/1oqtY4QRqtXxWk9d6KNFqrikgZVRejNwTLVLQ5", - "t4eu19rnkqHFm8LO9s6rg5fb8ahHRs8Pei+ev3jeexmTV71X+/t7B/jV3t7eC+zHyqQpBLn4EWbPAyQF", - "AF3XRA9poPLAMlwA0HufOxNcM54esQspgyCqUgMDwff6R8wikhRja0owvINRvThLnCQXo87h7y0S45i+", - "1+l0isW886X7uRwWAHDcjWhCGJ42vZXLmpgXCaYjKizQu7yNebTXn8WjTlf/c+cF/Lvm8nZ/+9Xz5W5v", - "bUjNojvb991KKLYyYPvRqHjIU1XaQi8O1S40x1BxW8JBTw5p/pjmihzbp4KVK19oav7ZKoYluL3NUSw2", - "YOXORGUuDPaxENnACypzyPO4o4N1RPuUomscJorgvq+TGQYHLV4tCKJSwUhsQmjLGy4RjiIyg6gTLoqE", - "HdgpG2rcKjOVH4S4sIOVOFmnu1SEkxUhxW0khllIIGZxYHZscHR0NHBZpwbQerC00BS0J8iICMKisr28", - "s7soytDL8pWB6S1vwd46+q6GXJgPC9l5HVtoX/gu7pPnAvNEfjDy8zcXyl2gxKxPkQbbhHqyNDGBjsXw", - "xnYh5mFoioHlK8WNWx3jCFA2hThmjVpomd31RBpSE+t0p7+0A5Jx1rs8OQOdIE2SG0QZwiKa0HvSOjC0", - "PYhc0DFlOMn0dBW6C9skU1RZwDvsI4nRv84vQyB2siX3P9FZcTvbama7P8tvctYRSY5GuBDXt7+m/Z0J", - "PhZEyrsZEREJKb7LjLyQa4xs41KcbTNMO4uS6ayoCIwabEBtWXPnCKYeSxcIc3t7U5rbiPlcXIbI11MD", - "5cWFqSkgRWo5N7DlBckX0jZGaB9jdj03UUOlDF2Gl2wGSW0rJAl/gDQD4AKE18CyEwootyPX5EDR54hc", - "NXlZUEyWtqLWWk73ODmxuD049sp7CTsHY9Tja0GmTeLSCNLio6Sjo6Pq+WuxlHFz/szjeeO85oC4Ou6q", - "mKhHgbb0W9ieEWcKU2YSU7ggYgjvhVFkFWD7of5sYRsUMzX024bAn/DILKDFycoy6XG7Y4O3orULGIeU", - "AkT1m/M2qCo9SqlkOjg6EqTsD275fM9MWecwgFuoESUmett3HRhQ+p7X4Pzt3fX/vD3udDtvL27gn/Dq", - "yP1x/vaXoLPAByDMI0du3XZ+3242uKvSYTuNVVh85Xmp+bl+n+peRJSkI5iENtOCfVEV4BvM4LF7O4id", - "xF9NXlaXw2M/ujmQ+5dIjRUU8Zggez1V5pTn+wUX3O7u3t6L3e295y8P9l+8eN7iMdExTxLirrzKDxns", - "JzTlMUn6wSR09+TuPl9EEz7cWlc9H5l4+IU9HNBnpjm8HVFEqhXAtJi7i7R+JqLtCP7GeqMo8lGtNERQ", - "rhlryeKkHtSa6StI6ZY3M8iABdyGida8YiiZKWAPuc5VOmrFTYW5DVd1O5AFugUd+WmdfVxamZ4Ns3jV", - "9WqCp2qWKosAPXB/NdVQIuB6LGfqvIDaVhq9tJMFtQ5wlpV61uGaBN9tzsEBUHiIhtFx/ZavLDpWlgLf", - "DzNXKCy3rxZptZIRsCZt9o3OCt0c4vdhrMAh7ZQJGk1Cr/fdlzxLfmwuPsyDYffKAyf1D4S/UhPVcibk", - "opgQdI+TNH+O6OD0zcjslmW+k8kB0y0Hdr5r3Anuzz3jtAvR0gpeHw+qij+n6qDjSUzEeRwSAvD20DRA", - "5yddk+iJ5i4GfRi2abSwbdd81xZ0EU6wvCEf1RWJuGhIFIaR5lF9rhQ2reqWfVuY/fIMCRjEJQHLLi88", - "jFQP6RMsJ4uo5VfdphVn+TUYtFYeklB+lKMkQeYbgiTKJEaKV2BtJfpf62Es8irHuargd/6YS6wmDTQ1", - "w2qiBdU9jTU/pkDF1i28RafTVOFhAmlSF5OYKuxu01pAup9m+5mdrYMyx7ebYA8986lIU9k+NImlhRU7", - "aq+eh3ujl70Xe/sve6/wcKe3Qw4OhnujnZckfrHC1XMGDzszHFVNKuUg0lLR4z2bL9cJUScj0VZCxxNl", - "MjdPseoi8hGeGLtLvTsag3wz2As9H/9eRNBjSpcnUfLvI0qs+FhBatReTxZEh7R3lTlf9UOpTTZEO7Ur", - "Di2skJk/kM8u+0uzOtTfKBo/U9e1M+epSTiQZTRA7uOX8uKndRNaSJD5dahJB05IOC3lPs6nG9N7gqYE", - "jTkvBHuscHorYc5BGcQb1DyqTYjmn6hs5SUbV1ZI5bwGY9YcW+8SbrKYhSnTfdWm28OERhPAqgXsE52B", - "QEcPNEnQkOSlPHJsy73DwQAPjgamz2B3e/dge3dnb6DFcj+S9yVsb++/LKAb+vd/0v+3I0Dpsc8vvwz6", - "P93e6hGCp+l2HkmzGTUeSd9uaPBOtqqeVWMTkN5+fLDTe7GzF/VevSSkt/98n7zce7mzM9rZW8EmKKwn", - "fKbkUtJhkgEGeTik71j2I9Aou3PXZHk82vvaiW8MpTZTtSVn8BQwC0Y/YEuQBELOVDDFzxl8MG7yiM8I", - "GmJJYsSZdcNYnwwcp2Rr7wjoAjN0C9cIZWMiNTyrgJl1Rnm1vCoWCIvv6r3ekNRHf9YiNhuw3yIEY283", - "zDJC1U4HKVrEGiesHmJDx9qKes6qLoXzIiluI1FsfYjy0c8moTVpWLQx6+VgqaaOxFk6l5BBBT0h6rE0", - "BxAb2sJJYjTNBN/D7Z6kUpmbFZFGKhVgzLRLRlXMK9PmOs5eZv88b0iXCjmjsmz4RrS7VRTklW741dkY", - "nZlzvqRFa3brrJUD8NpvW5boHgClUbuFrfaR974NUdZd7Z4Fqc+6D0s2x/oob/NEdaRq0sRrGPziRZvI", - "S7sEaT9MeJYiHk463yN5h/LRVggHzrTrDGcPInZ1Tut27uuuNK0L3N3+b+30QAs+ax0DV3pcsLMwAgDW", - "WmD2/LqrDdsfqWVFQO19bmUjHfO7o542iSZUKi7mVbv+L85tOdafWK28hL8IuyzJEdZ4rjGEze1gxJN0", - "ygJpupm5tlnCSj/O+rjbmbvV75TNYSFsaptvmsxHaWL8ZsRmNi2cLpa/9l0mpai3wq6Hrwz0Bdty7GO4", - "4WhorIpRtmXZTIVzYkKkvLNZPm1mIPdnlHBJ4jtNb+Iekp/yGWH+3wkZqbtqM0HHk9Dv1jMELGD+FTqE", - "1vnBze/F7PJcIC7GmNFPeoWluNCKa+V8uYD+v4zYzqs6lUV3dku3ZskdlMCe237jcndKlKDREk/kDHRv", - "TLdzBjkIqq/GvPBwO0N+FWBw6crX8FmaAJ7hhZFt/J9KpOSWUYmkjQHMqmvZajt99BbK56gJEQ9Ukv6t", - "5rTiYw5Tez+j14V3JEshe7gY2QuflsyCNwB2/8H/vwUHZnBFmMPSAErl8QzwCZam5W1ncNspetWy90ty", - "sLu9u19J9VvM/zro/7jYlQvLtOXcM1mwlFoEsBeVt6y4LcxiH0NMLWvXkY3JhnravZjZeI7ZVxLxKhJj", - "RaJ9J4mhUdhFkZPwxmm2llwbSbT5kaoBfcHzVNto4Sh9dIqjidtFq9wlokpC+IiGv2tT/RuK6N8yLQN6", - "CbknSQYLuNVcEyQJsJGWQX10bOPeI8w0Dd+yrJnicGPLjA/Oo2c0oURgEU3mAyUIyZ10RtK2cyAbzmj1", - "Drlg2FnE1e/Om1xttVI8uIhdX3qYUgpVGTKcw70uaMYsguqytMGNBQwWP22qCVAwEKHh3NxQF28KT7Cc", - "DDkW8d0VwfHcGsrn4H7W6NvRXHtxfHV36d4EQXVzswk+/L6B7QVO1UYVvDNVgAIlrQMScEPCB54wnKz4", - "qqsKJuDqK3cxSLsgZkrAdjOSWkjX1q6qvqmC3723nWEi38oiVRw0SFIWEUTVDxKZSxbC1LMnmje4//6p", - "agVSuqoJPvJsLGGCj+De0dZayFZTJIyw0n+bm2brVvw9A1u8NgMghKhfbexIQwiKDXFpupOv8WScx7Xh", - "IEfuxg1u/Wve3bURvvoPNe8vfTXe7rXeeWzGX+6tnjtE/yD9enQTQoVfvrSy6BZlx70yWdbllJe7D03Q", - "ykbJFskTsqwDqr4Ud2HUwP57xfhDq9GmXxOyCsPVng7iYtQRnJ55saRTR8vTHtcncQz1DDUs95Q8eDJO", - "75+JS5BlbtzdDxD+B1IT1+UKwAzqCnvbgiXFSXaL3PY77n3a7r26670Ps11Y6P2aTjHrCYJjfSA3kFQf", - "Mobmb+m39Herloi9RmgsMNMnyOHceEXKO5Nfzx5qsD2X9GGijya6qd4T87Wuxks98g4XIdIv/LKIEfSW", - "Z68ZfFSEeAMU19FsltC6UCtPReHZTNMmeuDiwyjhD8Y6gC9+nGbpttgEdC4+ysPoTmhV7I71HOMTZ1GV", - "aALWAfp3CyezCWbplAgaAbulTJ9/Ii4I0rxZuOwo2TmNpQh/P+r9y27z+xaBeg4DOf5q968uMNj8bh7o", - "mK3yomsXb9nR4jqeZtQHnMXtbsxl3IaMCi7jbKmaoTdETuu86zM0aCO2H8d7VMMNvxXZ+/tii+qFYJVR", - "uh4FB5mGStV0wvBszvw9Y/bArW16gkJs9eKQPQ3UP1MiQlaFNlJNMfw/TYuqnfpnXVen77yerVYAz1NL", - "usdLlBXWPQ6KENbNgIGlJXxMI5QyCnCSjyRKw/lDVn/bzdmIjlut+Ng0bRWgnr3g/Yr33A57d94GNnW9", - "su3t3Ca/jq02sRBWKEsR5CgYovLgurKu2n09zjBcfq+vf09FfrWpmQneNvcXHOVubz/3f7y9/RI0iMyk", - "TUlpstBV5BZbuMnIQKjc9C9PKd9+D6F37ebUBYADDh4rHakBnkhVGycbyCVk9wlpIsxuG6snMCts75Z9", - "j2Te2C/LtDWRmyYavzhk7ZYYRNQd5eGA5o7zpqy7h4gqAszT3JBHIE2Us4yKAyztRCmt2kxZvz5Lzk1J", - "ZHTPAnB+CMc/ri/e3p3+983V0fHNxVWn2zm+eHtz+t83d2fvXr8OhljAvKtnYvHp7Vurnq8XKKF49Uqr", - "egePk5iGZs5PljQbDHctMHiuSwGdoWqkPftWPSs76h1bCs+odw53kPCKBC0qSTolLNaDvE0DlVCzr87N", - "HJteaDhHR9gmWGoMclvkdy8AUyrEUFGgsEpEXCsUKq6+39ve6W3vlE9XbQ5WPiiVMgM1wKi8Haqp9b7b", - "21sJnFTxK8LIA05s9u/S3qSK94RpgCiLaYSNm6j6ktUbyuVhbxhLL0mWBWO4/GN9aq633vvh7CJ6ceFG", - "i9as1F/5zK2SfFTbts24I5qQt4sfPVedb26SO01ZNg12q9n0SLYIRmkN+CM6t55G+2zP5tME9eYlVlxU", - "PGOG50Qsxj40awM2NLxWYa3xwHsJ0XrRPEYzaa8gvbIdPsfZcSkn4G7llPv+8+6XuvSj91AaWC/gLE0C", - "Z3L9K3Kv0+1DZIGg2lyrRWYzmIomlXt6bzz09vK8zT64IYP7XRry5vxtyyEv1MRsbws8QMBZho22eDAz", - "hPBwURhvCTxAxyAeSkO2x4NuvjxV5qjICfPt/6xImCEbolVyuJq8cCtnUJQkSgVV82ttY7hz2phRxY/S", - "0FWo/lWLGitoUricP5riT5yhY9MRnZ9c9tG5RpFx81ydHb98sXuA/vHbDRrq08YMbIvIHkQcCLCmhD8Y", - "kyJVEy7oJ5jmmMek8uM7kXQOOxOlZvJwMIh59Gne1w36qewRLFVvp48BLruefsSnA65b7A7cQJBtNuIz", - "++wSfJOHnaPshsje7zmnJZ8Rps9BnV+IQhfnJ8dI8Q/ExrRrORvq7D7B/fsH8lVwm+l86oHf9dh/PCi3", - "Y0OCBRFnjjz+8dtNpxyOoLcCBkN8qDBlJEZ4pKne7SEu7POq2wmWK5gOAFLOB3r5nS9f4DntiFvvgMKm", - "mJjbhxkVeIzvscCz9L+MLWXdyebyq2MMRnTOIj1ZWkCs174SjHF0eZ6ZukXvCZjY6EJEEyKV9azY8ktg", - "pyc0IvZYWYUBIte15k8lWQCPMuZIJzTfEcjG7DjU2e5v93fg6f6MMKyFa2evv93fsxGqQLoDHE8pG2iS", - "g7/HpCaoDx4zCGlMg6PfrrMNh/vYGR5b07ObPwQwMXmSCwickOnMPd3WRwBofB7b0Y80FO8ACIjrtG4i", - "CLYuhyWPiTsCGKXvZoa4/86hOe3nez0z6bbNWcisDcplL3gVs7O9MCnml259OnIBx3y4pEYWgjrg7qST", - "JxUIF8bJLAvgNcEimoBdbeIjNMcMQHdn21YDqoSuBTiXT3tR8+B9ODd3RFkC7yAAWZGHKqI6OEk8RwVh", - "eGhK4cRUun/qJu9bwUQS8EtqykXDeR04XKg7+BqCx0n+DCL7t5cyvNtJsFR3pXTY9XBda3BiKkjUQO0A", - "FRfG3ggiSkYeWOYvPU8IhPfdjvN4gXDY3d52EtcmIcT5LfXgD2lskHzWxsfEjuMhjhgketjvAWJHS40J", - "wS5a+Aor8ppOqaqbxbYd5A1hhn2zgFCPbKGDc5MB3rlEodvO4m7vWGYcxKbT3uJOZ1wMaRwTOBvv775a", - "3OOG8zeYzS104KY+aLcqo2MgD0/BggMhm1kBv7/X2y5d+iGzDamVzAqPtUzuwObZaqud99o65rLu9RKR", - "9o0CcDhlaMjVpKBBtJa4JGJKVZ9ym3PcpmbkLlTHiwcydSX0yXU645AfoWfL8Ei0u72NLv4/RF34QiII", - "jucu2ZYJHYqNeZIgrJSgw1QR2b9lt+zHH7PXVmcJfzj88cdbtgPheQLRrGCJXoEP/NZMUHiYWzR87DKe", - "3bLdPjofZc1dIn6oQiK7KFOF5gcD4q83N5foYHsXbTHu5Vwg8bNbtheEKEffVsHWHQDeMlj2AZa8cRka", - "K4RpkjiU0dw6H6bKBOgoyMVJ4v8c4USSW3bQR1cm+kx4MVtFqLS91yOjEReqC/4SylICobZ66lQQ+czs", - "gR4JHWUbjX4mE3xPuYDt6NmJIIwSINTTzHkqvLkIu6eCm95bpD/ud7XtQcQdmDpmke7fNqKq6zLW6W/P", - "9DxWCAwYZz2YSI9mYuw0wEjShDCVzG1OEIISPh6TGGw+InqSxkQPA0+4ZAqm/ShNdHuHHxuwN5sRLLK3", - "W/q3u7xJdt2gVZIe76bayCQksX5zjGQ6tNfS3mOwfDrdGNZnKgXkK+qWx3VJmiA5J5gKMzVHW0AiEywR", - "4364lt29SywUxQm6NktG1xFhWFAu3fZZPgUK393eQeSeMM2uAZrURwI+JRXWB3D1YMdQqKpAkgZThwgo", - "07kizUNcASExjKsg38B4fDrTW1lCxL2soFFxFBNFIlUGLqNmPR7waYSZLxoI2rL89AwYSu9bgqMPqMC2", - "aCsD7JnGjl6c4dGK9WxkbKZNO+aigkjlsvWvV1vb692Avr5xgT4ut27wijfNYTS3VvYd4ubtDAO5l6+v", - "soJ3dSpjilU0gcfomcZAWzTTP8/WZJ3sGjPjmyzb8YQvq/49jK7tFkbXMWejhBpv22NaabrX7uJeP+P4", - "F6zIgw3+aWfYHfvvRy1b1th3X7oFP8HgM5xmvhiDLyEhf6iWX5gZFWnaSMv9xoPQbAT2ESQQy40jKhEV", - "goBvY5iQPrL3phLBXa6YwoNoezWa+Q+qvoYTAMWXlo3OhjfazhgS91YQ5tDSzMxpFlZ/FrPNKuLOP5iV", - "b8ge58xl0LBQKJiN24hQ2N1+8c2WNTNGSkaZMdqapomiPeNhePZvIvf2F/d4y9UZT1n83Z5OzY4buTLL", - "RU7TUTXo4bwiSlBy7yxFZ8K0EFVoODdeNITjWBApq0LnF6KK9tnm2dtki63na/j+ROd/GTr/hRgnTEZt", - "TRSu9yngjNGnJalPE1YPV0/4mMUD7h+ltfXLU2XdK/D8UZ8GPfopHUb0HBkZnsIENRQfIEsD0Bqostv5", - "754esWcg6Nn11WdpD+Oj06ioy5MYrJ1ftJ4lQ3PzPH8ZFgsJGk7MkTvz1RAKqDAeqTXJn++RYY1vwvkh", - "ieOEegdq0QwNAZU3GQB3gRjQs86gEGt1ByCG0mo0/+QKeXnh3qGLsiuIZyVBgKaY4TGRVfeLT7p9dOrr", - "PhRhprd7SFA0wWysTatUEqsgTcJ+twptSNt3jub255n1wh45UJFdgfUd5WADfBnk4H5zAaNF7yy44KZU", - "uWDFOU9RzNkPCj1gZrLoAJiZn5CwGF7O5H4e2wD+9l2UbwA9xsl4dXr5+uj4FF0rgRUZz59ZiE3Ajcli", - "5/nfkG1/DS647GW5abGl8WdyC9wTcEXecP23HeIwy3p/9Po1ilIhcsfk5et31+D10yc6/ZPtLciU35PK", - "ALDOzOsIuHFY+UDIrNTZOQsPDZIwsw5Bs6Lf3ztUZ5mBrWsRLtMSgu9DpJQySyYb8P2u0YvrThR9f01y", - "wh+kzevHcGLDfkwwRH5g8pdb8DEWh2z0MeY+RY2kUxNEhHroCKhCYwUQdOwTwyH6/bbjObVvO+9vmaWk", - "W/eQ+LZjd7OmS9dv+v6Wmdj1xe2KYF6RWYIjgo6SxGxhDbTZCHq4AOiWExVHf2gSuO14Xv3ASgpfi8AX", - "P6EtOzNQgTexpX1IpVm0c4xgejSnqw2gb3K6evJdcSsNv42fddHhw0nqf0NP49/hJGJosUJ02PHe0l7E", - "gQ3RMFlUvsIKCl5Bn5jBM+dj0T7oopkg99oG0Tq6FLlmLFiscL440BEzQUA/xP4ddMDVaGZ+3JP/Ubko", - "UY0DwEbFPDHhX9TtZfZvNbf9wERHbYbfrkjPDK85TjMX5alM5jnFBbiwrw0sxYXuUwyiiPAMD2lC1byZ", - "107Zd8tqNhTtidP+mpxmKKsFo0VZYd42sVC2ZL7LzOSVsat1rUH/Y/d+aRP2pq3323jD33i5H/nQNZmd", - "O2uG+fwEEBGA2Xz/21xvf5euLv8iOSMBxypmA0K8MvhM4y+1Mef5jQx2Oa4jxzbDuc39Grxr8XhkQzog", - "r8NdT3F/4TuW7/UCBHv7D8l46mhsWXdqnuV6sTfVzwXpS3FN/LXC23R+BOHd7ClwwntpD0GQwv8uB/nv", - "+JSNl5ang8jVKllSsrpuTrha2q6Tsa75z/PjPG34xgRuNl1Q4GagP8ncTchcnzIyqvAp0jXYrAC+JspN", - "n4O0WPRe1xPrBoRwjgrVQgjbyDL3hjBbVr+FXN4PZYJwaNGjPonl9XPEdZEjmpmgLJmz1I6LBTNk27Tv", - "n/Ksd8O5l8MiKJkLSfQeRzQX8/Y12MPZMvTCnghyja+zVCE3YqnihiPQbI/WJaWDXo53pvw+dvm7IZWJ", - "zd5sLlVL77qmROEYKxwwmfVIJ3k61Hp5DcGbMyzUYMTFtAfDQTHkiMeUjU15aOPwtL1ubNojj+pNQhXy", - "cZZAzgB4QNLtSDWH59Z6YLMtGVOU608nNRmuHApSWJCffmFIGYZ44UImzhd7L/Z3Xu7uN+ZzaUhqHaVS", - "FbO35GknHJVU08fsb78qPODt/7Qw2xgsuZpsrCoBzMoNIoZ6+1Y6dWSFEMxwJH6SIes6a1iehQ1tlBh1", - "Km0wdDbbwhOHU2vQw+6lzEygvJpZJsDqNdzPegiyMGPAG/Ne3itPYeptQ30HlYq68P0EaCT4inp33e/y", - "31Zhkx/orAYyPhpJUgPa9lryu2zyFQJsmyG6RW+/C0TyxO7rMxmq3PftTYZ/nV8iLKIJvS+UyzDaPSHo", - "8uSsbObAu0izlrzkzwJTAsjva+0JC2ezSfGJzr7GpPDmqLcqIFem0YcIyyIW29sai1S9G7CNtq/ZxsLu", - "tbAAdjchb5pua3/2WAJqfcxcAtICaT2JoLVZHDWcjWWJiFc2SAaf4T939p6p7q3iMWYRSTTz+GLR5KOl", - "0gXZJvMmGWPGqMqYRT6bAtVFMEg5WuCvGCzwd7j6N1taoopF6nGB/Wucf3qD4alhpca055EvzVpxwS+g", - "tY3Yaw2xnQVCtj7OJ8r9do77gijLUmdtxLTrLmwLxAFWYFlgmxpNy8eitTAszxlVFO5s9cnSTOREeNV1", - "eiPoeEzEqYFnM9cDZnA7U939gAXU2kcm2dGjhtTkdfkCEJolIGXW8HT7uxH+tRRiYg0sPTrONRtQbwF5", - "NZXrdBFk4MFJXhh5SBLOxvBwh2cHwT4qVDzIqi6TvBTzD/KWhQow5/W2u3oe/pAHS8g2BZXD5ZshIRHU", - "cMYmKZFX7fmW/Vaujo/84vhdH2wzknsGVa0Ve8u2SrWP7YOzIUnQUBD8IeYP7JlNUoHjWCJ+T4SmZiQ5", - "khOeJvYZ2ZDcslSWivdDFDkjJCaxqRVddXEZ8XZm93KBgwuWDoklGhZl0j16SKis0VWWfda/ZSfGnQS7", - "ZXIpwUGICDBXWKQJA0vOpC12HXBS2XnDXip7Fn7U7Bde2fIGl5NfuLxYY+rJlPlW3qqaYvJOJJqNXYuT", - "qixNreHU7sYWvFHWsG8VRXMNbR8phMbEDZkZm4Nl7RKeomnWbpRbcvAIZTNRjDkZtzAEsisZbRI4LW0T", - "Epp3+v0GHSU3T7R6sibPnRPdDnZhufLJON2YRPZIZUFsYuaRaxntrbwQKSiSmpkpVqTS2Jk51rqB+AG9", - "1ROE5S0zyqKbXYB3jVUjjYHqAg/s03dIZ6LIx0LhIlPvM2SaeW6X5Y0yuWjGLAuoo/WwYaU72+qy341t", - "5bByygSNJpp6q2yaXeD/ldMP/V3cQ3FpM3LuWq+XyK/t996XB3LwOa8Y+2VgeLRNBEGSWIY2Dno4EA4J", - "YeE6yl0EydBNnbAph4qFERxRqZCq38Thrw1IX8k1xTu9fJmtCrf5laTLpfdDJccXXs45VWngeOLAbxlZ", - "nRNysYaex4Cw/w3MV18j2YRog/qYYTXJtUehTHN9XsrlylbXO2GhjD0E/jQVqDenbYGjD1qne84LSKpS", - "5VI96vy1rTG9CU9tpQR/jbM2X4uG6FHdtAXZEOD0Qm35v/zr478D1wPVZmX/y+X+a3he60zjBG62oG16", - "P3BZ2dQ6JiORNqKdHznwct64k5V7DLYh+9BM03B9aO8VJlgaZa7HgNysT8e2dSaks5dLmVitu1VY1swz", - "/TMjDzJk9fIDTlsPWtFAI0nsH5KKWjL8LM1QPlTwOc16dlbTm6Vj18YU5+Yc3kU0NLlQXDauMtKfdMW3", - "9BbWbIrv+i5u8bIleSo8Bo8zpF/m2WTGE3geLvhcl6iiyoKbMNMqFG5JJ2ysVVZrH548ns22BEeelYH9", - "21Ro+FvEhvk5L1bhz6CWHEyoVFzMW7pBbKVH8JOWYSjz6Qp+kBLsv1rY/sbKtOirceht7a0pIcyV5V2L", - "46YiDiypPEmAb6igKwrFkozbnNUFwX1e0rl9Dgc3e0ActDGeSyD+PHcE/B1zfDcEjMODffBl3nkBumpg", - "u88W2gIwytTzff9R1avd3b29F7vbe89fHuy/ePF8cd3Y78ToPwsRSfEVnlc1/EnKfC9SZjhHOcW2EzB5", - "UGCrY4GJENPkwMUYM/qJsnH+QqPW5odumzL1YfCvSE438qF7JHPfTBliPYPhp7R0j2Wiu82vixnzuGTw", - "2fzj3HhbF+q+LNg1fOHiBlv3dUtzmjIvrFaDAyW+q3x7BRESj8C3ZqI6vvVkjsXd4+UtX8ijAkB/Ond/", - "F0xt6MiX5ssxdB5+0PKMnT9LfKBqQplvb7tIozwaycUc5fHisn/LLvKYo9ahQFm0dmay3zI/Ym88UUT0", - "Hoj+BzLSA6kJZuiX0xtUDLpCW+SjndUWN6CxKXXAp7NUuaK58llNxJPB7In3dnftgU+V1X5noU9Fx0CB", - "hlp5BrIoGObETdkn8FXRHDmRUmap8klCfQfBVbAfLWRV9/s0MmpEqHvYsViAeoEcM8HHUCHKf44SxFO/", - "Xga9yZ6UbNgccBPVWwV2GU9s9i0PxtZkrLx1+muy2oSbvF6N0fpak/568+Y1mhKWohkem5dZfoV5U2Mh", - "XIf0Vz4ll3jcItZDq9rBRE2TIu8UoQJIbKfMe6TXYSDTMPpJSGC4UgqS/ZcHLwqZx/63/+N/BHKPVRjx", - "12waDcXToXltbBXcw6zyQaomRSPb1PBT81akW6XTH6QtkZYVPoOiu1lJN9+8hkoJ+oM0915jgZkyV1pg", - "Per+fXTE5oF5UIQZirRJryZUIsLiGaem2l1MZcTvCVAvFYg/MANJkIPO3XI3qITcHAuLi9iiRCwuoOXp", - "/cvXlNpfT7ybHrThGf2ZyQLiXU0EiX/FgBFTY8cSUbg0V4WNzWl18Bn++xZPydeclUuR+QpDVUUILvHO", - "zzBVYwi+/NmL8v0Ozm/XFt2bOr7ZejBU5rh5Ytn1hduXEF0mxa8KvDdBvTbjasBszPiq0W4smEbbBbPo", - "96Pev3Dv03bv1d37n0IWUvAuNHuADC/5aaKIKKRhC3lZomKS9DCkbYryOMs24WNaf5ddzNMCbRE8FBjO", - "kSAxFSRSNjeFq098fnJpG2obJfA8FSYsCY09I9/LEsxM4I+ej1zkv9fcPgaoet2vXrssx8FBQgcDQYum", - "8O72/svChk+UmsnDwaD/YzAV75M0aJQGmh0isxdWKGgaYXAacxX0Z4KbJMalF66wdXY/PcXSrD91p542", - "MIc4+lBL7L9iFieW1C/0KLvI9UFxKpyVa6nHJitCVyQi1AVJOxTbEnk8JsZq8JlD63zy0VTMlYjawxn/", - "QJis4ZZjB/kCWXfUPHudSOFxa8FXYYRc8t3e9toKP3jS4FUb1wg4vr4600hVxMXphWCVKlQZp1ZK7+4v", - "Dez7ZSWTR4TVCtOJFXbLSConV+tHW11k/b91MuvfSfoUBEqZ0d3hc5FA4Wl9sYjXfGzEARj7PDXv4w3i", - "cFaEXRI4E5ozQFiXmnkymILiQQPyFdpUjw/OBC4KpJx5Gb6OehMH4BPNbijlg9lCkPPL6EQjWJfJCq97", - "UCLREEsSI86cyRoJqoigOJyJ5J+m16bLnLhpAke5o/IKno5ua8w28me2v47W9FbMl33tAuS4fM3Vf1rz", - "YBPRMTD2VwS1/enB9h1kh4TlPEW1PVZUm9v9Gq7IRPAKlVYNrzQXWs0ZY0NC10xQS2dPCco2UO4v2/hC", - "jb+qvF3qeTB0X76+qiewW5RX3bicbi6uasqhIng1maePjIkEoQ1L6a9W+chQ+1O51Ucot7qURB0oIldI", - "XV1ghqDZcvqRRKmxW/QUSKTM5EKrs2Dy0QOJrYlUG2cNPcmC17cGdrOeJXNbb28C2Pqr1RsDo0yTpzqF", - "68porVG6mL1gPHEf9jxeCh6nJmzUNOp0O6lIIK7EeMpjHn2a9yM+HdzvAHvZeaq18iyDaKs6Aamqjeos", - "n6F1BBbTGVZdjDXDuEqg/kjl6qBtB/OvauxYlaRsy46VpdOFfIjmfY87y9s5ilFTS09g01rp8bOsTy67", - "Uz5J4Y5t+UWUHzx70IefQrWdIj9t2vEKVNp2FJNpxRulmGKl7TBeIEthh3xnS9uxwFM4xQyPQfSa1A7x", - "lDIqlSiPr3/PJnj/5f8EAAD//+2UaeGdXAEA", + "H4sIAAAAAAAC/+y9C3Mbt7Iw+FdQ3LsVOeFDoiQ/tHVrr6JHorO2pSPJm3tP5KsCZ0AS8RBgAIxk2uX/", + "/hUawAxmBjMcyqTs5OjUqYrFwaPR6Bcaje7PnYjP5pwRpmTn4HNnSnBMBPzzEivyms6o0n/EREaCzhXl", + "rHMAn1CivyHKxlzMsP6AKEPmD3TTga//zz1lMb//T0VnpGf+fdPpdDvkI57NE9I56Oxsb7tGO9udbkdG", + "UzLDesZgm+e6zQx/fE3YRE07B7vDbmeOlSJCg/W/v2/3Xr3/yTU2f/1Hp9tRi7keSCpB2aTz5csX3Uvg", + "GVF2rT9jFU3PjqsrvZ4SNNIfUTpPOI7R2XG/0+1Q/W2O1bTT7TA804NDq1sad7odQf5MqSBx50CJlPiL", + "+g9Bxp2Dzv81yLE+MF/lwMGgoTtKKGGqDqAIvtaD8hVAZBNrKI55lM4a4Ijt941A4k2uYTn5OOeiFhIC", + "XzcCRzaxhuKfKRGLOiDOjhEfIzUl6E/dbP2guNk1JO8kESczTJMqLO8uX/cIi3hMYpRKIhDR7RCOY0Gk", + "rAEL2jRCZljba1rgw+0glwki55xJYpks/gUrco8X+q+IM0UYCBc8nyc0Ahky+EPqNXxuuzlCcPGGSIkn", + "xMxYRMXJRy0acIIkEXc0IojoDhoFdYIuNJttO8gbwlRHnI0TGqlHW80lkTwVEUE4EQTHC0Q+Uqkk4gJJ", + "pUVyZCFa0wJPuRjROCbs0VZ4xmQ6HtMI5NuciBmVknImkeL6T02CSE2pRDjSPda0zjNmqASge8S1erSp", + "uXSNpHnG7nBC40vyZ0qkesQlwbRImHnRiMeLNa3oLVenPGXx4zMb4wqN9dRrWsk1528wW9i9kY+5IJDF", + "KJ1zhhTnaIbZwu2VXMPqup1LosSidzhWRFT10tt0NiJC60hJIs5iiUZkzAVBSiwomyA8wRRYOjP9drRa", + "CWggytTu0GggOktnnYOXz/e2t7udGWXm71wbUabIhAiNEK01GU7VlAv6icSPj/j7KWEo9UBYC0V9cSiC", + "MQ7jGWXaODgECenmDtjwDqoxF9ZMYHiUkEFMpf6vFbES3VM1RTKNIiIl6JlUIsxipI16qfBs3ul25oLP", + "iVDUKHrTszqlAckJcqINIcL0dv3esZPCL/CP9/4pIf9asjG6nYhPGFX8VqajP0iktAFemffItEG2DaIx", + "YYqOKRGweG2xGZCRs1cKZxS8MxpGu/Fej+yPn/devHy13cOjKO6R8c5wd2//uf6laA4N958XziX90Cmk", + "aw2pCrhg1jlTEnYGR0rbcnPOCoD9waesH3PyX/anfsRnne7Khlq3Y3e3CspvU6KmpIAiaEtis3sOFGMr", + "2nFHnCcEMz1wTiNVi9l9ciu1VOMvcLg93O/tbPd2nl/v7B0M9w92d//lLzDGivT0HMVF7geMUd+y/T1b", + "cDfDkZ39fdaTA63oRWQsdSQIVkFWytUdUFSkG4JIQ4zcmz0ENoLZukjb3F1gIsE1p0lJJ2wGZ/AyL9XQ", + "yLuKXY+2UklihCVy5K6n1TM9WwvR+Mds3Pt02PvXdu9V//b//unmpvf+p//yfoMfbm769qf3n4fdL0H6", + "H1Mh1a05gIQW+INEE3pHGOALEIujiKdMWQSXqOUffMqKgO9YpZD93YYrE7wMqDGe0WTREqpjTtYAlKaT", + "AH++plJp/gEy+kAWYCgbckKUoQttQas+5WiL9Cf9LpLpnIhbrAm6C/Th/o3TmCouuta3cKu/PeujSz0t", + "mqVSmbOGHnTBU+GNTNgdFRyot48yAxD6jTV5SpoQppIF6qFoSqIP5tutAZLEekQndQu6//dODp/mTwOg", + "ZlCqyAxwUUbqcjTO8Mcz03vfbIL9aydri4XAi4q8cEzhUaxPKG5/WkiPFgoZa/PJlyEaEV1EWZSksf6l", + "rM/OjkGWyAWLrIKuiJFWahIAND6DHzwxwuifKQkoz8dRkpGB6harJh0CplWmL++xRLafpi/9u1xIRWZN", + "2mX3YG//gdql215Mb0aBG6Mpblbgkbe/iEpr7wGG7F63UelNQrtEQbnwbiOnv0oyl2b2JHQLYbx8ZnBF", + "qFvNYU1YzojP2hfjNEkWPiXmYtMdAozn3FBnH52N0RgnknStRW78O/kGoa0IM+irmTDCijxDo1TBGbUo", + "76dYIlaaZuB5VJ710RvMUpwYscEFEvrshmZ4gUYEOeHXb0MSRYkekGygDdQUK3RPBClix1cEGfwaOIAD", + "a7kjiVFyxswiTr3QMZJcq2BMExL30RGfzbEgxtIqNwa5rrVjTJQWY7nNBf1TQSSgn6VJorFBZnO16GoU", + "mv4AeAbrFmyPxXERq+csWaC5IFIPTceoQDqa7TQWH1vTFZVbt2O1RO1JCZaXqZKcf07PL49Obo9+PXz7", + "y8ntxeHV1W/nl8dV8bgMvJJ2DSin3DDPFVqBBxtV7TFJSCtVm519Y91Dc6I57ZpT7ljwGSI4mlr+RFsO", + "P1rdZsT67GHatv5Qag9EAJOVaY+ja+2Mt3rhVYiPHY5q0FOHhhby0i0VhlxBGRmaWHWCbOeWTxGm1Iwa", + "g2TY5lhfu7ePe6bPyH5jp3o3Q5Pl9eJgZ39T5/oCSS8RGgrTJCgUNdjK2pb+hbdPrkWZ4FnrVv/4HrOH", + "2efvWtrlBizPSMgtjO/XZHdnaM90D5PM/vX2q4Od/YPd7XUb6ycFd0rR/wbAPbLtfpQKoY0IZ6dbsesD", + "tj7Tvepv0fYgJXcNUnkjlnzAx9ICkoca9jV+FWOxBk3TouXmjLY7Su7JN7LZLJkUbDfHTgEb7uj87enZ", + "5ZuTB9ht3U46j1uwNh8jvcPItG4S/dsHO9vrEf3tjEjHbI2a4DWVgeVd4AllcJBLrNdNY9rejyiucIIM", + "zrWon5vGWvLOiMIxVrgi86dY3s64IPUqWn/VQ8H5hSB8h2nirkmWmkZ4QuqJRX9FDC7kChdu3YZLtp3t", + "7cIl2071ks1MeyvpJ9J0/2cQNycC4PAB0BKyEYJl88NGBGgT9oeVIJhhFU21jh7TRJUwsfeiCZLhzt6L", + "vZe7z3WrplvHLpztGjy3BhC4NPC2puPJkaarx4rxUpAigK5Gv6YBzmHNEo2/iY188s6w9/L7EJADGs9G", + "Nikl6ChVRKItUEn2PgRERvFqRFbPVo1KzEin8q2BU5cl3cWwmuJN3xhYiMpXBkGQjjnpXc0ohGVt6MIg", + "s2WTtjcHPXR5cvH68OjkCuEkMS6p3KzdYlxpk4kqekee9dE1138hDJ+dz5pAT0ffpuM8SSVggZF7xBmB", + "roLM+B3JenMIMFVG68MVoSbi8K3E11xIuOsH24gLNKYQmKOwIn10rsGAqKMxJUmMtFUtSALepITgO2JH", + "T1k0xWxScpp983uMKv/qT6d6KbpXlUauKJskelH3mkCM48ysXKsxtLWzMzR/A65mmhg4g/ZVbsVYEEWO", + "iaB3JD4EOuHsEityBKjSJkNCGSlhYbgdMpQLYx0lmM6uF3NyFIdQ2Nz5lJCraEriNHnAzF7n/58IaSMd", + "VgPgF8HTORHh/i1geM1Hq3d6S9Q9Fx9W73gheJyaOMeVO04Enj2o4901tFi15yWhs9EboqY8btfZ0uQx", + "kVF1H5t6vMEfTwnRtHzGokJ4UsxTY6Rl5sKr7H/9V6+CFoO1xYrjX0Sq9fh9PfqyoVM11dpbadwGVrwf", + "XPCIJskKnJY3bz1DhMUd4alqOYNrfmbCAau2b6SFQ7bQtqPqTldwUHhIj/aLnau9C8GjtpNkzVee4Q1f", + "bYo3vP0cMRnjNKnfgpjiScsVmqbtZ5ZTy3drZLpYTuuXIqeWEdfDheTjlI6ourAHtKX4se2vqaqoqzCG", + "xjiiCVWLw/iPVCptCF0TMWvVdTIj60fuZEZqkTuZkbUidwK+Q3E+rp/R6N7DBMKvtd36mtyR5Hx8hEU7", + "hWNH+BnLtWPKjNyScbLWrXnH9vgVR8vQs+ZNgTEvibaCzU3psvkvqfxwPn7DhcKalK/SUZRgKVfByxW5", + "I4KqxUP6LAPvWmAmx0QsbVc1YsLz000wHm1gPLpmxkuIlI18l/CRNSQvSQKuMTml81YMZ7pqU3Llrixu", + "q2ihZWtOYgrP179hetRaBOqPa90ynqqEEnHyMUpSzZQtEVXt1hprtuspFVId8yTB9QyUNf2ojztSvh2J", + "Y7yQ11NB5JQncUus99tiIZto1RlWmOON67uhgU8h6oVFi3UjZxNk19ommWNFCVOHE/IGf6z0aO5AWasO", + "CY7I+fjKPBJsyQblTq2ZYC743dWcRBQnatF2smKflea6xh8547NVpsq7tJ5JYDYhlklbzSOwIicywglW", + "XKwwjderVngUWr3BH+0p+sI4Mb6ehAvjh91Za2E/QehsdMTZnXEUneJImReK7UZvN/7JeEwibQcfl7Bj", + "7u6WMQ+MsX5VCMOuVe6YEQW/e2td9cuJLesxpxVro6n9NWWt27eWg1lre7P40O26IyxtK+Sy1q0Z1L60", + "br0qqfhcq91jTJOFZdU1kpEbflUjp9KvPQJszzZGTt52g8aHm2T91kdp5PWbH24CKwfO2cnHiEh5NMVi", + "Yq8Dv1osuDkuSwpsrQtozQ5K0MnE+BY2QQt2+CM8z0Y/nKk1TpBG61cFab23Io3WqiJSRtX5+A3BMhVt", + "zu2h67X2uWRo8aaws73zav/ldjzukfHz/d6L5y+e917G5FXv1d7e7j5+tbu7+wL7sTJpCkEufoTZ8wBJ", + "AUBXNdFDGqg8sAwXAPTe584F14ynR+xCyiCIqtTAQPC9/hGziCTF2JoSDO9gVC/OEifJ+bhz8HuLxDim", + "71U6m2Gx6Hzpfi6HBQAct2OaEIZnTW/lsibmRYLpiAoL9C5vYx7t9ufxuNPV/9x5Af+uubzd2371fLXb", + "WxtSs+zO9n23EoqtDNh+NCoe8VSVttCLQ7ULzTFU3JZw0JNDmj+muSLH9qlg5coXmpp/tophCW5vcxSL", + "DVi5NVGZS4N9LEQ28ILKHPI87mh/HdE+pegah4kiuO/rZIbBQYtXC4KoVDASmxDa8oZLhKOIzCHqhIsi", + "YQd2yoYat8pM5QchLu1gJU7W6TYV4WRFSHEbiWEWEohZHJgdGxweHg5c1qkBtB6sLDQF7QkyJoKwqGwv", + "7wyXRRl6Wb4yML3lLdlbR9/VkAvzYSk7r2ML7Qvf5X3yXGCeyA9Gfv7mQrkLlJj1KdJgm1BPliYm0LEY", + "3tguxDwMTTGw/EFx41bHOAKUTSGOWaMWWma4nkhDamKdbvWXdkAyznoXx6egE6RJcoMoQ1hEU3pHWgeG", + "tgeRCzqhDCeZnq5Cd26bZIoqC3iHfSQx+tfZRQjETrbk/ic6L25nW81s92f1Tc46IsnRGBfi+vbWtL9z", + "wSeCSHk7JyIiIcV3kZEXco2RbVyKs22GaWdZMp0HKgKjBhtQW9bcOYKpx9IFwtze3pTmNmI+F5ch8vXU", + "QHlxYWoKSJFazg1seUHyhbSNEdpHmF0tTNRQKUOX4SWbQVLbCknC7yHNALgA4TWw7IQCyu3INTlQ9Dki", + "V01eFhSTpa2otVbTPU5OLG8Pjr3yXsLOwRj1+FqSaZO4NIK0+Cjp8PCwev5aLmXcnD/zeNE4rzkgPhx3", + "VUzUo0Bb+i1sz4gzhSkziSlcEDGE98Iosgqw/VB/trANipka+m1D4I95ZBbQ4mRlmfSo3bHBW9HaBYxD", + "SgGi+s15G1SVHqVUMh0cHgpS9ge3fL5npqxzGMAt1JgSE73tuw4MKH3Pa3D29vbqf94edbqdt+fX8E94", + "deT+OHv7S9BZ4AMQ5pFDt247v283G9xV6bCdxiosvvK81Pxcv091LyJK0hFMQptpwb6oCvANZvDYvR3E", + "TuI/TF5Wl8NjP7o5kPuXSI0VFPGYIHs9VeaU53sFF9xwuLv7Yri9+/zl/t6LF89bPCY64klC3JVX+SGD", + "/YRmPCZJP5iE7o7c3uWLaMKHW+tDz0cmHn5pDwf0qWkOb0cUkeoBYFrM3UZaPxPRdgR/Y71RFPmoHjRE", + "UK4Za8nipB7UmukrSOmWNzPIgAXchonWvGIomSlgD7nOVTpqxU2FuQ1XdTuQBboFHflpnX1cWpmeDbN8", + "1fVqgqdqniqLAD1w/2GqoUTA9VjO1HkBta00emknC2od4Cwr9azDFQm+21yAA6DwEA2jo/otf7DoeLAU", + "+H6YuUJhuX21TKuVjIA1abNvdFbo5hC/D2MFDmknTNBoGnq9777kWfJjc/FhHgy7Vx44qX8g/JWaqJYz", + "IRfFlKA7nKT5c0QHp29GZrcsi51MDphuObCLoXEnuD93jdMuREv62HxFP5GfFyp0Q3NKE4Ik/US07Bgt", + "4IEo5GPSICZkgqNF7hF41mhs7A1f7b16/mL4an+ZI+MBnigPUxUfU9VpyJOYiLM4JJjgPaRpgM6Ouyb5", + "FM3dHvqAblN7Yduu+f4v6LacYnlNPqpLEnHRkLwMIy039FlX2FSvW/a9Y/bLMyRgEJeYLLtQ8TBSdRxM", + "sZwuo+BfdZtW3O7XhdCWwoiEcrYcJgky3xAkdiYxUrwCayt19FoPY5FXOWJWlZHzEV1gNW2gqTlWUy08", + "72isZUQKnGVd1Vt0NksVHiWQunU5ianC7jatBTTOSbaf2Xk/KAd9Ww720DPpijSV7UOTqFxaRaT2Ony0", + "O37Ze7G797L3Co92ejtkf3+0O955SeIXD7gOz+Bhp4ajqomuHERaUnu8Z3P4OsHu5DbaSuhkqkw26RlW", + "XUQ+wrNnd9F4S2OQuQZ7oSftT2KxXiw+psR7Em//PuLNirQHSLLaa9yCOJP2Tjfn9X4oBcyGaKd2xaGF", + "FSoYBPL+ZX9pVoc6JUUjcea6dhY8NYkZsswPyH38Ul78rG5CCwkyv4406cBJEqelHNH5dBN6R9CMoAnn", + "haCYB5xyS5hzUAbxBrWhahPH+SdPW6HKxt8VUl6vweg3x/vbhJtsb2HKdF+1Grmf0mgKWLWAfaJzEOjo", + "niYJGpG85EmObbl7MBjgweHA9BkMt4f728Od3YEWy/1I3pWwvb33soBu6N//Sf/fjgAl2j6//DLo/3Rz", + "o0cIeh3aeW7NZtR4bn1bpsGL26rKWI2dQnp78f5O78XObtR79ZKQ3t7zPfJy9+XOznhn9wF2SmE94bM3", + "l5KOkgwwyFcifQe8H6lH2a27Tszj9t7XTnxtKLWZqi05g0eFWTD6AfuGJBCap4KpkE7hg7lOiPicoBGW", + "JEacWXeV9V3BsVO29iKBLjBDt3AhUTYhUsPzEDCzziivKljFAmHxbf3tACQ/0p+1iM0G7LcIVdkdhllG", + "qNrpIJWNWOOE1cN+6PhfUc9Zdapw/ijFbcSOraNRPo7aZL0mXY02sL1cNdUUmzhLexMyqKAnRIeW5gBi", + "Q1s4SYymmeI7uAWVVCpzAyXSSKUCjJl2SbuK+XfaXFvaS/+fFw1pZSG3VlY1wIh2t4qCvNINvzprpTNz", + "zla0aM1unbZylF75bcsS3QOgNGq3sNU+8t63Icq6K/DTIPVZN2vJ5lgf5W2eqA5VTTp9DYNf5GkT+XtX", + "IO37Kc9S6cNJ53sk71De3grhwJl2nWH/QcQ+nNO6nbu6q197VeCiJLZ2eqAFn7WOFSw9wthZGikBay0w", + "e34t2IbtD9WqIqD23ruykY753VFPm0RTKhUXi6pd/xfnthzrT6xWXsJfhF1W5AhrPNcYwuYWNeJJOmOB", + "dObMXG+tYKUfZX3cLdbtw+/ezWEhbGqbb5rMx2li/GbEZoAtnC5Wvx5fJfWqt8Kuh68M9CXbcuRjuOFo", + "aKyKcbZl2UyFc2JCpLy12VBtBiX3Z5RwSeJbTW/iDpLE8jlh/t8JGavbajNBJ9PQ79YzBCxg/hU6hNb5", + "5s3vxSz8XCAuJpjRT3qFpfjZimvlbLWHD38ZsZ1XvyqL7uzmcM2SOyiBPbf9xuXujChBoxWeEhro3phu", + "ZwxyNVRf13lh9HaG/CrA4NKV+eHzNAE8w0ss2/g/lUjJDaMSSRsrmVUhs1WJ+ugtlBlSUyLuqST9G81p", + "xUcvkMdEZPS69I5kJWSPliN76ROcefAGwO4/+P+34MAMrghzWBpASUGeAT7F0rS86QxuOkWvWvbOSw6G", + "28O9SkrkYp7cQf/H5a5cWKYte5/JgpXUIoC9rAxoxW1hFvsYYmpVu45sTDbU0+753Ma9zL+SiB8iMR5I", + "tO8kMTQKuyhyEt44zdaSayOJNj/mNaAvecZrGy0dpY9OcDR1u2iVu0RUSQhp0fB3bUkEQxH9G6ZlQC8h", + "dyTJYAG3mmuCJAE20jKoj47s+4AIM03DNyxrpjjc2DLjg/PoGU0pEVhE08VACUJyJ52RtO0cyIYzWr3X", + "Lhh2FnH1u/MmV1utFA8uYteXHqbkRFWGjBZwrwuaMYs0uyhtcGOhh+VPwGqCJgxEaLQwN9TFm8JjLKcj", + "jkV8e0lwvLCG8hm4nzX6djTXnh9d3l64t1NQBd5sgg+/b2B7AWa1UQXvTLWkQOnvgATckPCBpx7HD3z9", + "VgUTcPWVuxikXRAzJWC7GUktpWtrV1XfnsHv3hvYMJFvZdEzDhokKYsIouoHicwlC2Hq2RPNG9x//1T1", + "AFK6rAk+8mwsYYKP4N7R1qTIVlMkjLDSf5ubZutW/D0DW7w2AyCEqF9t7EhDCIoNcWm6k6/xZJzFteEg", + "h+7GDW79a94nthG++g+16K98Nd7uVeNZbMZf7U2jO0T/IP26fVNChV/mtbLoFuXZvXJi1uVk7HGoHhWY", + "oJWNki2SJ2RVB1R9yfLCqIH9h5ox9avRpl8TsgrD1Z4O4mLUEZyeebH0VUfL0x7XJ3EMdR81LHeU3Hsy", + "Tu+fiUuQZW4c7gUI/wOpietyhXIGdQXQbWGX4iTDIrf9jnuftnuvbnvvw2wXFnq/pjPMeoLgWB/IDSTV", + "B5+h+Vv6Lf3dqiVirxGaCMz0CXK0MF6R8s7k17MHGmzPJX2Q6KOJbqr3xHytq4VTj7yDZYj0C+QsYwS9", + "5dmrDx8VId4AxXU4nye0LtTKU1F4Pte0ie65+DBO+L2xDuCLH6dZui02AZ3Lj/IwuhNaFbtjPcf4xFlU", + "JZqAdYD+3cLJfIpZOiOCRsBuKdPnn4gLgjRvFi47SnZOY8nG3w97/7Lb/L5FoJ7DQI6/2v2rCww2v5uH", + "TGarvOja5Vt2uLzeqRn1HmdxuxtzGbcho4LLOFuqZugNkdM67/oMDdqI7cfxHtVww29F9v6+2KJ6IVhl", + "lK5HwUGmoVI1nTA8mzN/95k9BGybxqEQW708ZE8D9c+UiJBVoY1UAlrrT9Oiaqf+WdfV6TuvZ6sVwDPe", + "ku7xEoqFdY+DIoR1M2BgaQmf0AiljAKc5COJ0nCelYe/gedsTCetVnxkmrYKUM9eOn/Fu3eHvVtvA5u6", + "Xtr2dm6Th8hW5VgKK5TvCHIUDFF5mF5ZV+2+HmUYLuc10L+nIr/a1MwEb8D7S45yNzef+z/e3HwJGkRm", + "0qbkPVnoKnKLLdxkZCBUbvpXp5Rvv4fQu3Zz6gLAAQePlbbVAE+kqo2TDeRcsvuENBFmt43VE5gVtrer", + "vkcyuQhWZdqayE0TjV8csnZLDCLqjvJwQHPHeVP+3kNEFQHmCXPII5AmyllGxQFWdqKUVm2mrF+fJeem", + "ZDu6ZwE4P4TjH1fnb29P/vv68vDo+vyy0+0cnb+9Pvnv69vTd69fB0MsYN6HZ6zx6e1bq56vFyihePVK", + "q3oHj5OYhmbOjlc0Gwx3LTF4rkoBnaGqrT37pj8rz+odWwpPu3cOdpDwiiktK906IyzWg7xNAxVjs6/O", + "zRybXmi0QIfYJqJqDHJb5ncvAFMqWFFRoLBKRFwrFCpCv9fb3ult75RPV20OVj4olXIMNcCovB2qqYk/", + "7O0+CJxU8UvCyD1ObJb00t6kiveEaYAoi2mEjZuo+pLVG8rlq28YSy9JlgVjuExmfQqzt9774ewienmB", + "S4vWrCRi+cytknxU27bNuGOakLfLHz1XnW9ukltNWTZdeKvZ9Ei2WEhpDfgjOrOeRvtsz+YdBfXmJaBc", + "VmRkjhdELMc+NGsDNjS8UmGtcc97CdF60TxGM+nBIA21HT7H2VEpd+Kwcsp9/3n4pS5N6x2UUNYLOE2T", + "wJlc/4rc63T7EFkgqMrXapHZDKbyS+We3hsPvb04a7MPbsjgfpeGvD5723LIczU129sCDxBwlmGjLR7M", + "DCE8nBfGWwEP0DGIh9KQ7fGgm69OlTkqcsJ8+z8PJMyQDdEqiV5N/rwHZ5qUJEoFVYsrbWO4c9qEUcUP", + "09BVqP5VixoraFK4nD+c4U+coSPTEZ0dX/TRmUaRcfNcnh69fDHcR//47RqN9GljDrZFZA8iDgRYU8Lv", + "jUmRqikX9BNMc8RjUvnxnUg6B52pUnN5MBjEPPq06OsG/VT2CJaqt9PHAJddTz/iswHXLYYDNxBk5Y34", + "3D67BN/kQecwuyGy93vOacnnhOlzUOcXotD52fERUvwDsTHtWs6GOrtPcP/+gXwV3GY6n3rgdz32H/fK", + "7diIYEHEqSOPf/x23SmHI+itgMEQHylMGYkRHmuqd3uIC/v80O0EyxVMBwAp5wO9/M6XL/Ccdsytd0Bh", + "U3TN7cOcCjzBd1jgefpfxpay7mRz+dUxBiM6Y5GeLC0g1mtfCcY4vDjLTN2i9wRMbHQuoimRynpWbJkq", + "sNMTGhF7rKzCAJHrWvOnkiyBRxlzpBOa7xBkY3Yc6mz3t/s78HR/ThjWwrWz29/u79oIVSDdAY5nlA00", + "ycHfE1IT1AePGYQ0psHhb1fZhsN97BxPrOnZzR8CmJg8yQUETsh07p5u6yMAND6L7eiHGop3AATEdVo3", + "EQRbl8OSJ8QdAYzSdzND3H/nwJz2872em7Tk5ixk1gZlxZe8itnZXpo89Eu3Pm27gGM+XFIjC0EdcLfS", + "yZMKhEvjZFYF8IpgEU3BrjbxEZpjBqC7s22rAVVC1wKcq6e9qHnwPlqYO6Is0XkQgKwYRhVRHZwknqOC", + "MDwyJYNiKt0/dZP3rWAiCfglNeWi0aIOHC7ULXwNweMkfwaR/dtLrd7tJFiq21La8Hq4rjQ4MRUkaqB2", + "gIoLY28EESUjDyzzl54nBML7bsd5vEA4DLe3ncS1yRpxfks9+EMaGySftfExseN4iCMGiR72e4DY0VJj", + "SrCLFr7EirymM6rqZrFtB3lDmGHPLCDUI1vo4MxkyncuUei2s7zbO5YZB7HptLu80ykXIxrHBM7Ge8NX", + "y3tcc/4Gs4WFDtzU++1WZXQM5OEpWHAgZDMr4Pf3etulSz9ktiG1klnhiZbJHdg8W5W2815bx1zWvV4i", + "0r5RAA6nDI24mhY0iNYSF0TMqOpTbnOz2xSW3IXqePFApv6GPrnO5hzyI/RsuSKJhtvb6Pz/Q9SFLySC", + "4Hjhkm2Z0KHYmCcJwkoJOkoVkf0bdsN+/DF7bXWa8PuDH3+8YTsQnicQzQq76BX4wG/NBYWHuUXDxy7j", + "2Q0b9tHZOGvuChZAtRbZRZkqND8YEH+9vr5A+9tDtMW4l3OBxM9u2G4Qohx9WwVbdwB4y2DZA1jyxmVo", + "rBCmSeJQRnPrfJQqE6CjIGcpif9zjBNJbth+H12a6DPhxWwVodL2Xo+Mx1yoLvhLKEsJhNrqqVNB5DOz", + "B3okdJhtNPqZTPEd5QK2o2cngjBKgFBPs+Cp8OYi7I4Kbnpvkf6k39W2BxG3YOqYRbp/24iqrsuip789", + "0/NYITBgnPVgIj2aibHTACNJE8JUsrA5QQhK+GRCYrD5iOhJGhM9DDzhkimY9uM00e0dfmzA3nxOsMje", + "bunfbvMm2XWDVkl6vOtqI5OQxPrNMZLpyF5Le4/B8ul0Y1ifqaiQr6hbHtclaYI0gGAqzNUCbQGJTLFE", + "jPvhWnb3LrBQFCfoyiwZXUWEYUG5dNtn+RQofLi9g8gdYZpdAzSpjwR8RiqsD+DqwY6goFeBJA2mDhBQ", + "pnNFmoe4AkJiGFdBvoHx+Gyut7KEiDtZQaPiKCaKRKoMXEbNejzg0wgzXzQQtGX56RkwlN63BEcfUIFt", + "0VYG2DONHb04w6MV69nI2EybdsxFBZHKVTVYr7a217sBfX3tAn1cDuLgFW+aw2hurew7xM3bGQZyL19f", + "ZQXv6lTGDKtoCo/RM42Btmimf56tyToZGjPjmyzb8YQvq/49jK7tFkbXEWfjhBpv22NaabrXcHmvn3H8", + "C1bk3gb/tDPsjvz3o5Yta+y7L92Cn2DwGU4zX4zBl5CQP1TLL8yMijRtpOV+40FoNgL7CBKI5cYRlYgK", + "QcC3MUpIH9l7U4ngLlfM4EG0vRrN/AdVX8MxgOJLy0ZnwxttZ4yIeysIc2hpZuY0C6s/i9lmFXHnH8zK", + "N2SPc+YyaFgqFMzGbUQoDLdffLNlzY2RklFmjLZmaaJoz3gYnv2byL295T3ecnXKUxZ/t6dTs+NGrsxz", + "kdN0VA16OC+JEpTcOUvRmTAtRBUaLYwXDeE4FkTKqtD5haiifbZ59jbZYuv5Gr4/0flfhs5/IcYJk1Fb", + "E4XrfQo4Y/RpSerThNXD1RM+ZvGA+0dpbf3yVFn3Cjx/1KdBj35KhxE9R0aGJzBBDcUHyNIAtAaq7Hb+", + "u6dH7BkIenZ99Vnaw/joNCrq8iQGa2fnrWfJ0Nw8z1+GxUKChhNz5M58NYQCKoxHak3y53tkWOObcH5I", + "4jih3oFaNENDQOVNBsBdIAb0rHMoWFvdAYihtBrNP7lCXl64d+ii7AriWUkQoBlmeEJk1f3ik24fnfi6", + "D0WY6e0eERRNMZto0yqVxCpIk7DfrUIb0vado7n9eWa9sIcOVGRXYH1HOdgAXwY5uN9cwGjROwsuuBlV", + "LlhxwVMUc/aDQveYmSw6AGbmJyQshpczuZ/HNoC/fRflG0CPcTJenly8Pjw6QVdKYEUmi2cWYhNwY7LY", + "ef43ZNtfgQsue1luWmxp/JncAncEXJHXXP9thzjIst4fvn6NolSI3DF58frdFXj99IlO/2R7CzLjd6Qy", + "AKwz8zoCbhxWPhAyL3V2zsIDgyTMrEPQrOj39w7VWWZg61qEy7SE4LsQKaXMkskGfL9r9OK6E0XfX5Oc", + "8ntp8/oxnNiwHxMMkR+Y/OUWfIzFIRt9jLlPUSPpxAQRoR46BKrQWAEEHfnEcIB+v+l4Tu2bzvsbZinp", + "xj0kvunY3azp0vWbvr9hJnZ9ebsimJdknuCIoMMkMVtYA202gh4uALrlRMXRH5oEbjqeVz+wksLXIvDF", + "T2jLzgxU4E1saR9SaRbtHCOYHs3pagPom5yunnxX3ErDb+NnXXb4cJL639DT+Hc4iRharBAddry3shdx", + "YEM0TBaVr7CCglfQx2bwzPlYtA+6aC7InbZBtI4uRa4ZCxYrnC8OdMRcENAPsX8HHXA1mpkf9+R/WC5K", + "VOMAsFExT0z4F3V7mf17mNt+YKKjNsNvl6Rnhtccp5mL8lQmi5ziAlzY1waW4kL3KQZRRHiORzShatHM", + "ayfsu2U1G4r2xGl/TU4zlNWC0aKsgHGbWCjTOsvM5JWxq3WtQf8j935pE/amrYvceMPfeLkf+dA1mZ07", + "a4b57BgQEYDZfP/bXG9/l64u/yI5IwHHKmYDQrwy+EzjL7Ux5/mNDHY5riPHNqOFzf0avGvxeGRDOiCv", + "V15PcX/hO5bv9QIEe/sPyXjqaGxVd2qe5Xq5N9XPBelLcU38tcLbdH4E4d3sKXDCe2UPQZDC/y4H+e/4", + "lI1XlqeDyNUqWVGyum5OuFrarpOxrvnPi6M8bfjGBG42XVDgZqA/ydxNyFyfMjKq8CnSNdisAL4iyk2f", + "g7Rc9F7VE+sGhHCOCtVCCNvIMveGMFtWv4Vc3gtlgnBo0aM+ieX1c8RVkSOamaAsmbPUjssFM2TbtO+f", + "8qx3o4WXwyIomQtJ9B5HNBfz9jXYw9ky9MKeCHKNr7NUITdiqeKGI9Bsj9YlpYNejnem/D52+bshlYnN", + "3mwuVUvvumZE4RgrHDCZ9UjHeTrUenkNwZtzLNRgzMWsB8NBMeSIx5RNTHlo4/C0va5t2iOP6k1CFfJx", + "nkDOAHhA0u1ItYDn1npgsy0ZU5TrTyc1Ga4cClJYkJ9+YUQZhnjhQibOF7sv9nZeDvca87k0JLWOUqmK", + "2VvytBOOSqrpY/a2XxUe8PZ/WpptDJZcTTZWlQBm5QYRI719Dzp1ZIUQzHAkfpIh6zprWJ6FDW2UGHUq", + "bTByNtvSE4dTa9DD7qXMTKC8mlkmwOo13M96CLI0Y8Ab817eK09h6m1DfQeVirrw/QRoJPiKerjud/lv", + "q7DJD3ReAxkfjyWpAW17LfldNvkKAbbNEN2yt98FInli9/WZDFXu+/Ymw7/OLhAW0ZTeFcplGO2eEHRx", + "fFo2c+BdpFlLXvJniSkB5Pe19oSFs9mk+ETnX2NSeHPUWxWQK9PoQ4RlEYvtbY1lqt4N2Ebb12xjYfda", + "WADDTcibptvanz2WgFofc5eAtEBaTyJobRZHDWdjWSLiBxskg8/wn1t7z1T3VvEIs4gkmnl8sWjy0VLp", + "gmyTRZOMMWNUZcwyn02B6iIYpBwt8FcMFvg7XP2bLS1RxTL1uMT+Nc4/vcHw1LBSY9rzyJdmrbjgl9Da", + "Ruy1htjOAiFbH+cT5X47x31BlGWpszZi2nWXtgXiACuwLLBNjabVY9FaGJZnjCoKd7b6ZGkmciK86jq9", + "FnQyIeLEwLOZ6wEzuJ2p7n7AAmrtI5Ps6FFDavK6fAEIzRKQMmt4uv3dCP9aCjGxBpYeHeeaDai3gLya", + "ynW6CDLw4CQvjDwiCWcTeLjDs4NgHxUqHmRVl0leivkHecNCBZjzettdPQ+/z4MlZJuCyuHyzZCQCGo4", + "Y5OUyKv2fMN+K1fHR35x/K4PthnJPYOq1oq9YVul2sf2wdmIJGgkCP4Q83v2zCapwHEsEb8jQlMzkhzJ", + "KU8T+4xsRG5YKkvF+yGKnBESk9jUiq66uIx4O7V7ucTBBUuHxBINizLpHj0kVNboKss+69+wY+NOgt0y", + "uZTgIEQEmCss0oSBJWfSFrsOOKnsvGEvlT0LP2r2C69seYPLyS9cXqwx9WTKfCtvVU0xeScSzcauxUlV", + "lqbWcGp3YwveKGvYt4qiuYK2jxRCY+KGzIzNwbJ2CU/RNGs3yi05eISymSjGnIxbGALZlYw2CZyWtgkJ", + "zTv9foOOkpsnWj1Zk+fOiW4Hu7Bc+WScbkwie6SyJDYx88i1jPZWXogUFEnNzBQrUmnszBxr3UD8gN7q", + "KcLyhhll0c0uwLvGqpHGQHWBB/bpO6QzUeRjoXCRqfcZMs08t8vqRplcNmOWBdTRetiw0p1tddnvxrZy", + "WDlhgkZTTb1VNs0u8P/K6Yf+Lu6huLQZOXet10vk1/Z778sDOficV4z9MjA82iaCIEksQxsHPRwIR4Sw", + "cB3lLoJk6KZO2IxDxcIIjqhUSNVv4vDXBqSv5JrinV6+zFaF2/xK0uXS+6GS40sv55yqNHA8ceC3jKzO", + "CblYQ89jQNj/Buarr5FsQrRBfcyxmubao1CmuT4v5Wplq+udsFDGHgJ/mgrUm9O2wNEHrdM95wUkValy", + "qR518drWmN6Ep7ZSgr/GWZuvRUP0qG7agmwIcHqhtvxf/vXx34HrgWqzsv/lcv81PK91pnECN1vQNr0f", + "uKxsah2TkUgb0c6PHHg5b9zJyj0G25B9aKZpuD609wpTLI0y12NAbtanY9s6E9LZy6VMrNbdKqxq5pn+", + "mZEHGbJ6+QGnrQetaKCRJPYPSUUtGX6WZigfKvicZD07D9ObpWPXxhTn5hzeRTQ0uVBcNq4y0p90xbf0", + "FtZsiu/6Lm7xqiV5KjwGjzOkX+bZZMYTeBEu+FyXqKLKgpsw0yoUbkknbKxVVmsfnjyezbYCR56Wgf3b", + "VGj4W8SG+TkvHsKfQS05mFKpuFi0dIPYSo/gJy3DUObTB/hBSrD/amH7GyvToq/Gobe1t6aEMFeWdy2O", + "m4o4sKTyJAG+oYKuKBRLMm5zHi4I7vKSzu1zOLjZA+KgjfFcAvHnhSPg75jjuyFgHB7sgy/zzgvQVQPb", + "XbbQFoBRpp7v+Y+qXg2Hu7svhtu7z1/u77148Xx53djvxOg/DRFJ8RWeVzX8Scp8L1JmtEA5xbYTMHlQ", + "YKtjgYkQ0+TAxQQz+omySf5Co9bmh26bMvVh8K9ITjf2oXskc99MGWI9g+GntHSPZaK7za+LGfO4ZPDZ", + "/OPMeFuX6r4s2DV84eIGW/d1S3OaMi+sVoMDJb6rfHsJERKPwLdmojq+9WSOxd3j5S1fyqMCQH86d38X", + "TG3oyJfmqzF0Hn7Q8oydP0u8p2pKmW9vu0ijPBrJxRzl8eKyf8PO85ij1qFAWbR2ZrLfMD9ibzJVRPTu", + "if4HMtIDqSlm6JeTa1QMukJb5KOd1RY3oLEpdcBn81S5ornyWU3Ek8Hssfd2d+2BT5XVfmehT0XHQIGG", + "WnkGsigY5sRN2SfwVdEcOZFSZqnySUJ9B8FVsB8tZFX3+zQyakSoe9ixXIB6gRxzwSdQIcp/jhLEU79e", + "Br3JnpRs2BxwE9VbBXYZT2z2LQ/G1mSsvHX6a7LalJu8Xo3R+lqT/nr95jWaEZaiOZ6Yl1l+hXlTYyFc", + "h/RXPiMXeNIi1kOr2sFUzZIi7xShAkhsp8x7pNdhINMw+klIYLhSCpK9l/svCpnH/rf/438Eco9VGPHX", + "bBoNxdOheW1sFdzDrPJBqqZFI9vU8FOLVqRbpdMfpC2RlhU+g6K7WUk337yGSgn6gzT3XhOBmTJXWmA9", + "6v59dMgWgXlQhBmKtEmvplQiwuI5p6baXUxlxO8IUC8ViN8zA0mQg87ccjeohNwcS4uL2KJELC6g5en9", + "y9eU2l9PvJsetOEZ/anJAuJdTQSJ/4EBI6bGjiWicGmuChub0+rgM/z3LZ6RrzkrlyLzFYaqihBc4p2f", + "YarGEHz5sxfl+x2c364sujd1fLP1YKjMcfPEsusLty8hukyKXxV4b4J6bcbVgNmY8VWj3VgwjbYLZtHv", + "h71/4d6n7d6r2/c/hSyk4F1o9gAZXvLTRBFRSMMW8rJExSTpYUjbFOVxlm3CJ7T+LruYpwXaIngoMFog", + "QWIqSKRsbgpXn/js+MI21DZK4HkqTFgSGrtGvpclmJnAHz0fuch/r7l9DFD1ul++dlmOg4OEDgaCFk3h", + "4fbey8KGT5Way4PBoP9jMBXvkzRolAaaHSKzF1YoaBphcBpzFfTngpskxqUXrrB1dj89xdKsP3WnnjYw", + "Rzj6UEvsv2IWJ5bUz/UoQ+T6oDgVzsq11GOTFaFLEhHqgqQdim2JPB4TYzX4zKF1PvloKuZKRO3hjH8g", + "TNZwy5GDfImsO2yevU6k8Li14KswQi75bm56bYUfPGnwqo1rBBxdXZ5qpCri4vRCsEoVqoxTK6WHeysD", + "+35VyeQRYbXCdGKF3SqSysnV+tEeLrL+3zqZ9e8kfQoCpczo7vC5TKDwtL5YxGs+MeIAjH2emvfxBnE4", + "K8IuCZwJzRkgrEvNPBlMQfGgAfkKbarHB2cCFwVSzrwMX0e9iQPwiWY3lPLBbCHI+VV0ohGsq2SF1z0o", + "kWiEJYkRZ85kjQRVRFAczkTyT9Nr02VO3DSBo9xheQVPR7c1Zhv5M9tfR2t6KxarvnYBcly95uo/rXmw", + "iegYGPsrgtr+9GD7DrJDwnKeotoeK6rN7X4NV2Qi+AGVVg2vNBdazRljQ0LXTFBLZ08JyjZQ7i/b+EKN", + "v6q8Xel5MHRfvb6qJ7BblFfduJxuLq5qyqEieDWZp4+MiQShDUvpP6zykaH2p3Krj1BudSWJOlBEPiB1", + "dYEZgmbLyUcSpcZu0VMgkTKTC63OgslHDyS2JlJtnDX0JEte3xrYzXpWzG29vQlg669Wrw2MMk2e6hSu", + "K6O1Ruly9oLxxF3Y83gheJyasFHTqNPtpCKBuBLjKY959GnRj/hscLcD7GXnqdbKswyireoEpKo2qrN8", + "htYRWExnWHUx1gzjKoH6I5Wrg7YdzL+qsWNVkrKtOlaWThfyIZr3Pe4sb+coRk2tPIFNa6XHz7I+uexO", + "+SSFO7bVF1F+8OxBH34K1XaK/LRpxytQadtRTKYVb5RiipW2w3iBLIUd8p0tbccCT+EMMzwB0WtSO8Qz", + "yqhUojy+/j2b4P2X/xMAAP//akDiTcVdAQA=", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/api/queryAPI/documents.go b/api/queryAPI/documents.go index a66e5e1a..14d7e66f 100644 --- a/api/queryAPI/documents.go +++ b/api/queryAPI/documents.go @@ -107,6 +107,9 @@ func (s *Controllers) GetDocument(ctx echo.Context, id DocumentID, params GetDoc if document.OriginalPath != nil { response.OriginalPath = document.OriginalPath } + if document.FileSizeBytes != nil { + response.FileSizeBytes = document.FileSizeBytes + } // Include text record if requested and available if document.TextRecord != nil { diff --git a/api/queryAPI/documents_test.go b/api/queryAPI/documents_test.go index f5941a98..a06822ae 100644 --- a/api/queryAPI/documents_test.go +++ b/api/queryAPI/documents_test.go @@ -627,6 +627,86 @@ func TestUploadDocumentBatch_Part1_TooLarge(t *testing.T) { assert.Contains(t, httpErr.Message, "file too large") } +// TestGetDocument_IncludesFileSize verifies that file sizes are returned +// in the document response when documents have file_size_bytes set. +func TestGetDocument_IncludesFileSize(t *testing.T) { + cfg := &ControllerConfig{} + test.CreateDB(t, cfg) + initializeTestConfig(t, cfg) + + svc := createControllerServices(cfg) + cons := queryapi.NewControllers(svc, cfg) + + // Create test client with root folder (unique ID to avoid conflicts) + clientID := fmt.Sprintf("filesize_client_%s", uuid.New().String()[:8]) + test.CreateTestClient(t, cfg, clientID, "FileSize Client") + + // Create document with file size + filename := "measured-doc.pdf" + fileSize := int64(1048576) // 1MB + docId, err := cfg.GetDBQueries().CreateDocument(t.Context(), &repository.CreateDocumentParams{ + Clientid: clientID, + Hash: fmt.Sprintf("filesizehash-%s", uuid.New().String()[:8]), + Filename: &filename, + FileSizeBytes: &fileSize, + }) + require.NoError(t, err) + + ctx, rec := createContext(t) + + err = cons.GetDocument(ctx, docId, queryapi.GetDocumentParams{}) + require.NoError(t, err) + assert.Equal(t, http.StatusOK, rec.Code) + + var result queryapi.DocumentEnriched + err = json.Unmarshal(rec.Body.Bytes(), &result) + require.NoError(t, err) + + // Verify file size is present and correct + assert.Equal(t, docId, result.Id) + require.NotNil(t, result.FileSizeBytes, "document should have file size") + assert.Equal(t, fileSize, *result.FileSizeBytes, "file size should be 1048576 (1MB)") +} + +// TestGetDocument_NilFileSize verifies that documents without file sizes +// (legacy documents) return nil for fileSizeBytes. +func TestGetDocument_NilFileSize(t *testing.T) { + cfg := &ControllerConfig{} + test.CreateDB(t, cfg) + initializeTestConfig(t, cfg) + + svc := createControllerServices(cfg) + cons := queryapi.NewControllers(svc, cfg) + + // Create test client with root folder (unique ID to avoid conflicts) + clientID := fmt.Sprintf("legacy_client_%s", uuid.New().String()[:8]) + test.CreateTestClient(t, cfg, clientID, "Legacy Client") + + // Create document WITHOUT file size (simulating legacy document) + filename := "legacy-document.pdf" + docId, err := cfg.GetDBQueries().CreateDocument(t.Context(), &repository.CreateDocumentParams{ + Clientid: clientID, + Hash: fmt.Sprintf("legacyhash-%s", uuid.New().String()[:8]), + Filename: &filename, + FileSizeBytes: nil, // No file size + }) + require.NoError(t, err) + + ctx, rec := createContext(t) + + err = cons.GetDocument(ctx, docId, queryapi.GetDocumentParams{}) + require.NoError(t, err) + assert.Equal(t, http.StatusOK, rec.Code) + + var result queryapi.DocumentEnriched + err = json.Unmarshal(rec.Body.Bytes(), &result) + require.NoError(t, err) + + // Verify file size is nil for the legacy document + assert.Equal(t, docId, result.Id) + assert.Nil(t, result.FileSizeBytes, "legacy document should have nil file size") +} + // Helper function to create test ZIP with multiple PDFs func createTestZIPFile(t *testing.T, pdfCount int) *bytes.Reader { var buf bytes.Buffer diff --git a/api/queryAPI/folders.go b/api/queryAPI/folders.go index 54d6ff24..39f54c4a 100644 --- a/api/queryAPI/folders.go +++ b/api/queryAPI/folders.go @@ -117,6 +117,9 @@ func (s *Controllers) GetFolderDocuments(ctx echo.Context, folderId openapi_type if doc.OriginalPath != nil { apiDocs[i].OriginalPath = doc.OriginalPath } + if doc.FileSizeBytes != nil { + apiDocs[i].FileSizeBytes = doc.FileSizeBytes + } // Include text record if available if doc.TextRecord != nil { diff --git a/api/queryAPI/folders_test.go b/api/queryAPI/folders_test.go index 536934c5..5cee75cf 100644 --- a/api/queryAPI/folders_test.go +++ b/api/queryAPI/folders_test.go @@ -353,6 +353,121 @@ func TestGetFolderDocuments_EmptyFolder(t *testing.T) { assert.Len(t, resp.Documents, 0) } +// TestGetFolderDocuments_IncludesFileSize verifies that file sizes are returned +// in the folder documents response when documents have file_size_bytes set. +func TestGetFolderDocuments_IncludesFileSize(t *testing.T) { + ctrl, cfg, clientID := setupFoldersTestController(t) + ctx := t.Context() + + // Create folder + folderRec, err := cfg.GetDBQueries().CreateFolder(ctx, &repository.CreateFolderParams{ + Path: "/documents-with-sizes", + Clientid: clientID, + Createdby: "admin@example.com", + }) + require.NoError(t, err) + + // Create documents with file sizes + filename1 := "small-doc.pdf" + fileSize1 := int64(1024) // 1KB + docID1, err := cfg.GetDBQueries().CreateDocument(ctx, &repository.CreateDocumentParams{ + Clientid: clientID, + Hash: fmt.Sprintf("sizehash1-%s", uuid.New().String()[:8]), + Filename: &filename1, + Folderid: &folderRec.ID, + FileSizeBytes: &fileSize1, + }) + require.NoError(t, err) + + filename2 := "large-doc.pdf" + fileSize2 := int64(5242880) // 5MB + docID2, err := cfg.GetDBQueries().CreateDocument(ctx, &repository.CreateDocumentParams{ + Clientid: clientID, + Hash: fmt.Sprintf("sizehash2-%s", uuid.New().String()[:8]), + Filename: &filename2, + Folderid: &folderRec.ID, + FileSizeBytes: &fileSize2, + }) + require.NoError(t, err) + + e := echo.New() + req := httptest.NewRequest(http.MethodGet, fmt.Sprintf("/folders/%s/documents", folderRec.ID), nil) + rec := httptest.NewRecorder() + c := e.NewContext(req, rec) + + err = ctrl.GetFolderDocuments(c, openapi_types.UUID(folderRec.ID), queryapi.GetFolderDocumentsParams{}) + require.NoError(t, err) + assert.Equal(t, http.StatusOK, rec.Code) + + var resp struct { + Documents []queryapi.DocumentInFolder `json:"documents"` + } + err = json.Unmarshal(rec.Body.Bytes(), &resp) + require.NoError(t, err) + + assert.Len(t, resp.Documents, 2) + + // Build a map of document ID to file size for verification + docSizes := make(map[uuid.UUID]*int64) + for _, doc := range resp.Documents { + docSizes[uuid.UUID(doc.Id)] = doc.FileSizeBytes + } + + // Verify file sizes are present and correct + require.NotNil(t, docSizes[docID1], "document 1 should have file size") + assert.Equal(t, fileSize1, *docSizes[docID1], "document 1 file size should be 1024") + + require.NotNil(t, docSizes[docID2], "document 2 should have file size") + assert.Equal(t, fileSize2, *docSizes[docID2], "document 2 file size should be 5242880") +} + +// TestGetFolderDocuments_NilFileSize verifies that documents without file sizes +// (legacy documents) return nil for fileSizeBytes. +func TestGetFolderDocuments_NilFileSize(t *testing.T) { + ctrl, cfg, clientID := setupFoldersTestController(t) + ctx := t.Context() + + // Create folder + folderRec, err := cfg.GetDBQueries().CreateFolder(ctx, &repository.CreateFolderParams{ + Path: "/documents-no-sizes", + Clientid: clientID, + Createdby: "admin@example.com", + }) + require.NoError(t, err) + + // Create document WITHOUT file size (simulating legacy document) + filename := "legacy-doc.pdf" + docID, err := cfg.GetDBQueries().CreateDocument(ctx, &repository.CreateDocumentParams{ + Clientid: clientID, + Hash: fmt.Sprintf("legacyhash-%s", uuid.New().String()[:8]), + Filename: &filename, + Folderid: &folderRec.ID, + FileSizeBytes: nil, // No file size + }) + require.NoError(t, err) + + e := echo.New() + req := httptest.NewRequest(http.MethodGet, fmt.Sprintf("/folders/%s/documents", folderRec.ID), nil) + rec := httptest.NewRecorder() + c := e.NewContext(req, rec) + + err = ctrl.GetFolderDocuments(c, openapi_types.UUID(folderRec.ID), queryapi.GetFolderDocumentsParams{}) + require.NoError(t, err) + assert.Equal(t, http.StatusOK, rec.Code) + + var resp struct { + Documents []queryapi.DocumentInFolder `json:"documents"` + } + err = json.Unmarshal(rec.Body.Bytes(), &resp) + require.NoError(t, err) + + require.Len(t, resp.Documents, 1) + + // Verify file size is nil for the legacy document + assert.Equal(t, docID, uuid.UUID(resp.Documents[0].Id)) + assert.Nil(t, resp.Documents[0].FileSizeBytes, "legacy document should have nil file size") +} + func TestGetFolderMetrics_Success(t *testing.T) { ctrl, cfg, clientID := setupFoldersTestController(t) ctx := t.Context() diff --git a/cmd/docInitRunner/main.go b/cmd/docInitRunner/main.go index 92e64a70..fd893206 100644 --- a/cmd/docInitRunner/main.go +++ b/cmd/docInitRunner/main.go @@ -24,6 +24,7 @@ import ( documentinit "queryorchestration/internal/document/init" "queryorchestration/internal/server/runner" "queryorchestration/internal/serviceconfig/build" + "queryorchestration/internal/serviceconfig/objectstore" documentsyncc "queryorchestration/internal/serviceconfig/queue/documentsync" _ "github.com/lib/pq" @@ -32,6 +33,7 @@ import ( type DocInitConfig struct { runner.BaseConfig[docinitrunner.Body] documentsyncc.DocSyncConfig + objectstore.ObjectStoreConfig // Provides GetStoreClient() for S3 access to measure file size } func main() { @@ -57,6 +59,12 @@ func main() { os.Exit(1) } + // Initialize S3 client for file size measurement AFTER runner.New() parses env vars + if err := cfg.SetStoreClient(ctx); err != nil { + slog.Error("failed to initialize S3 client", "error", err) + os.Exit(1) + } + // optional add shutdown metrics defer func() { err = cfg.GetMetrics().RecordLifecycleEvent(1.0, "shutdown", "main") diff --git a/connect.db.sh b/connect.db.sh new file mode 100755 index 00000000..80bdf4e9 --- /dev/null +++ b/connect.db.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +# Database connection parameters for test database +DB_HOST="localhost" +DB_PORT="5430" +DB_USER="postgres" +DB_PASSWORD="pass" +DB_NAME="query_orchestration" + +# Export password to avoid prompt +export PGPASSWORD="${DB_PASSWORD}" + +# Connect to the database +echo "Connecting to test database at ${DB_HOST}:${DB_PORT}..." +echo "Database: ${DB_NAME}" +echo "User: ${DB_USER}" +echo "" +echo "Useful commands:" +echo " \\dt - List all tables" +echo " \\d tablename - Describe table structure" +echo " \\l - List all databases" +echo " \\q - Quit" +echo "" + +psql -h "${DB_HOST}" -p "${DB_PORT}" -U "${DB_USER}" -d "${DB_NAME}" \ No newline at end of file diff --git a/deployments/compose.aws.yaml b/deployments/compose.aws.yaml index d95f22f8..525408ec 100644 --- a/deployments/compose.aws.yaml +++ b/deployments/compose.aws.yaml @@ -28,7 +28,7 @@ services: DB_NOSSL: ${DB_NOSSL} AWS_ENDPOINT_URL: "http://localstack:4566" AWS_S3_USE_PATH_STYLE: true - DISABLE_AUTH: true + DISABLE_AUTH: ${DISABLE_AUTH:-true} COGNITO_USER_POOL_ID: "fillin" COGNITO_CLIENT_SECRET: "fillin" COGNITO_DOMAIN: "fillin" @@ -56,6 +56,7 @@ services: AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY} AWS_SESSION_TOKEN: ${AWS_SESSION_TOKEN} AWS_REGION: ${AWS_REGION} + BUCKET: ${BUCKET_IN} PGUSER: ${PGUSER} PGPASSWORD: ${PGPASSWORD} PGHOST: aws-db @@ -64,7 +65,7 @@ services: DB_NOSSL: ${DB_NOSSL} AWS_ENDPOINT_URL: "http://localstack:4566" AWS_S3_USE_PATH_STYLE: true - DISABLE_AUTH: true + DISABLE_AUTH: ${DISABLE_AUTH:-true} COGNITO_USER_POOL_ID: "fillin" COGNITO_CLIENT_SECRET: "fillin" COGNITO_DOMAIN: "fillin" @@ -100,7 +101,7 @@ services: DB_NOSSL: ${DB_NOSSL} AWS_ENDPOINT_URL: "http://localstack:4566" AWS_S3_USE_PATH_STYLE: true - DISABLE_AUTH: true + DISABLE_AUTH: ${DISABLE_AUTH:-true} COGNITO_USER_POOL_ID: "fillin" COGNITO_CLIENT_SECRET: "fillin" COGNITO_DOMAIN: "fillin" @@ -136,7 +137,7 @@ services: DB_NOSSL: ${DB_NOSSL} AWS_ENDPOINT_URL: "http://localstack:4566" AWS_S3_USE_PATH_STYLE: true - DISABLE_AUTH: true + DISABLE_AUTH: ${DISABLE_AUTH:-true} COGNITO_USER_POOL_ID: "fillin" COGNITO_CLIENT_SECRET: "fillin" COGNITO_DOMAIN: "fillin" @@ -178,7 +179,7 @@ services: AWS_ENDPOINT_URL_S3: "http://localstack:4566" AWS_ENDPOINT_URL_SQS: "http://localstack:4566" AWS_S3_USE_PATH_STYLE: true - DISABLE_AUTH: true + DISABLE_AUTH: ${DISABLE_AUTH:-true} COGNITO_USER_POOL_ID: "fillin" COGNITO_CLIENT_SECRET: "fillin" COGNITO_DOMAIN: "fillin" @@ -215,7 +216,7 @@ services: DB_NOSSL: ${DB_NOSSL} AWS_ENDPOINT_URL: "http://localstack:4566" AWS_S3_USE_PATH_STYLE: true - DISABLE_AUTH: true + DISABLE_AUTH: ${DISABLE_AUTH:-true} COGNITO_USER_POOL_ID: "fillin" COGNITO_CLIENT_SECRET: "fillin" COGNITO_DOMAIN: "fillin" @@ -251,7 +252,7 @@ services: DB_NOSSL: ${DB_NOSSL} AWS_ENDPOINT_URL: "http://localstack:4566" AWS_S3_USE_PATH_STYLE: true - DISABLE_AUTH: true + DISABLE_AUTH: ${DISABLE_AUTH:-true} COGNITO_USER_POOL_ID: "fillin" COGNITO_CLIENT_SECRET: "fillin" COGNITO_DOMAIN: "fillin" @@ -287,7 +288,7 @@ services: DB_NOSSL: ${DB_NOSSL} AWS_ENDPOINT_URL: "http://localstack:4566" AWS_S3_USE_PATH_STYLE: true - DISABLE_AUTH: true + DISABLE_AUTH: ${DISABLE_AUTH:-true} COGNITO_USER_POOL_ID: "fillin" COGNITO_CLIENT_SECRET: "fillin" COGNITO_DOMAIN: "fillin" @@ -324,7 +325,7 @@ services: DB_NOSSL: ${DB_NOSSL} AWS_ENDPOINT_URL: "http://localstack:4566" AWS_S3_USE_PATH_STYLE: true - DISABLE_AUTH: true + DISABLE_AUTH: ${DISABLE_AUTH:-true} COGNITO_USER_POOL_ID: "fillin" COGNITO_CLIENT_SECRET: "fillin" COGNITO_DOMAIN: "fillin" @@ -360,7 +361,7 @@ services: DB_NOSSL: ${DB_NOSSL} AWS_ENDPOINT_URL: "http://localstack:4566" AWS_S3_USE_PATH_STYLE: true - DISABLE_AUTH: true + DISABLE_AUTH: ${DISABLE_AUTH:-true} COGNITO_USER_POOL_ID: "fillin" COGNITO_CLIENT_SECRET: "fillin" COGNITO_DOMAIN: "fillin" diff --git a/deployments/compose.local.yaml b/deployments/compose.local.yaml index 92636eff..f0a1fe4d 100644 --- a/deployments/compose.local.yaml +++ b/deployments/compose.local.yaml @@ -28,7 +28,7 @@ services: DB_NOSSL: ${DB_NOSSL} AWS_ENDPOINT_URL: "http://localstack:4566" AWS_S3_USE_PATH_STYLE: true - DISABLE_AUTH: true + DISABLE_AUTH: ${DISABLE_AUTH:-true} COGNITO_USER_POOL_ID: "fillin" COGNITO_CLIENT_SECRET: "fillin" COGNITO_DOMAIN: "fillin" @@ -56,6 +56,7 @@ services: AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY} AWS_SESSION_TOKEN: ${AWS_SESSION_TOKEN} AWS_REGION: ${AWS_REGION} + BUCKET: ${BUCKET_IN} PGUSER: ${PGUSER} PGPASSWORD: ${PGPASSWORD} PGHOST: db @@ -64,7 +65,7 @@ services: DB_NOSSL: ${DB_NOSSL} AWS_ENDPOINT_URL: "http://localstack:4566" AWS_S3_USE_PATH_STYLE: true - DISABLE_AUTH: true + DISABLE_AUTH: ${DISABLE_AUTH:-true} COGNITO_USER_POOL_ID: "fillin" COGNITO_CLIENT_SECRET: "fillin" COGNITO_DOMAIN: "fillin" @@ -100,7 +101,7 @@ services: DB_NOSSL: ${DB_NOSSL} AWS_ENDPOINT_URL: "http://localstack:4566" AWS_S3_USE_PATH_STYLE: true - DISABLE_AUTH: true + DISABLE_AUTH: ${DISABLE_AUTH:-true} COGNITO_USER_POOL_ID: "fillin" COGNITO_CLIENT_SECRET: "fillin" COGNITO_DOMAIN: "fillin" @@ -136,7 +137,7 @@ services: DB_NOSSL: ${DB_NOSSL} AWS_ENDPOINT_URL: "http://localstack:4566" AWS_S3_USE_PATH_STYLE: true - DISABLE_AUTH: true + DISABLE_AUTH: ${DISABLE_AUTH:-true} COGNITO_USER_POOL_ID: "fillin" COGNITO_CLIENT_SECRET: "fillin" COGNITO_DOMAIN: "fillin" @@ -173,7 +174,7 @@ services: DB_NOSSL: ${DB_NOSSL} AWS_ENDPOINT_URL: "http://localstack:4566" AWS_S3_USE_PATH_STYLE: true - DISABLE_AUTH: true + DISABLE_AUTH: ${DISABLE_AUTH:-true} COGNITO_USER_POOL_ID: "fillin" COGNITO_CLIENT_SECRET: "fillin" COGNITO_DOMAIN: "fillin" @@ -210,7 +211,7 @@ services: DB_NOSSL: ${DB_NOSSL} AWS_ENDPOINT_URL: "http://localstack:4566" AWS_S3_USE_PATH_STYLE: true - DISABLE_AUTH: true + DISABLE_AUTH: ${DISABLE_AUTH:-true} COGNITO_USER_POOL_ID: "fillin" COGNITO_CLIENT_SECRET: "fillin" COGNITO_DOMAIN: "fillin" @@ -246,7 +247,7 @@ services: DB_NOSSL: ${DB_NOSSL} AWS_ENDPOINT_URL: "http://localstack:4566" AWS_S3_USE_PATH_STYLE: true - DISABLE_AUTH: true + DISABLE_AUTH: ${DISABLE_AUTH:-true} COGNITO_USER_POOL_ID: "fillin" COGNITO_CLIENT_SECRET: "fillin" COGNITO_DOMAIN: "fillin" @@ -285,7 +286,7 @@ services: DB_NOSSL: ${DB_NOSSL} AWS_ENDPOINT_URL: "http://localstack:4566" AWS_S3_USE_PATH_STYLE: true - DISABLE_AUTH: false + DISABLE_AUTH: ${DISABLE_AUTH:-true} COGNITO_USER_POOL_ID: ${COGNITO_USER_POOL_ID} COGNITO_CLIENT_SECRET: ${COGNITO_CLIENT_SECRET} COGNITO_DOMAIN: ${COGNITO_DOMAIN} @@ -326,7 +327,7 @@ services: DB_NOSSL: ${DB_NOSSL} AWS_ENDPOINT_URL: "http://localstack:4566" AWS_S3_USE_PATH_STYLE: true - DISABLE_AUTH: true + DISABLE_AUTH: ${DISABLE_AUTH:-true} COGNITO_USER_POOL_ID: "fillin" COGNITO_CLIENT_SECRET: "fillin" COGNITO_DOMAIN: "fillin" @@ -362,7 +363,7 @@ services: DB_NOSSL: ${DB_NOSSL} AWS_ENDPOINT_URL: "http://localstack:4566" AWS_S3_USE_PATH_STYLE: true - DISABLE_AUTH: true + DISABLE_AUTH: ${DISABLE_AUTH:-true} COGNITO_USER_POOL_ID: "fillin" COGNITO_CLIENT_SECRET: "fillin" COGNITO_DOMAIN: "fillin" diff --git a/docs/ai.generated/08-operations-guide.md b/docs/ai.generated/08-operations-guide.md index c2086141..59e1733c 100644 --- a/docs/ai.generated/08-operations-guide.md +++ b/docs/ai.generated/08-operations-guide.md @@ -472,6 +472,120 @@ Queue Processing: < 1 second per message - Resource utilization trends ``` +## AWS IAM Requirements + +### Required S3 Permissions + +The document processing pipeline requires specific S3 permissions for file operations. Below are the minimum IAM permissions required for each service. + +#### docInitRunner Service + +The `docInitRunner` service requires `s3:GetObject` permission to measure file sizes during document initialization. This is used by the `HeadObject` operation to retrieve file metadata. + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "DocumentInitS3Permissions", + "Effect": "Allow", + "Action": [ + "s3:GetObject", + "s3:PutObject", + "s3:HeadObject" + ], + "Resource": [ + "arn:aws:s3:::${BUCKET_NAME}/*" + ] + }, + { + "Sid": "DocumentInitBucketAccess", + "Effect": "Allow", + "Action": [ + "s3:ListBucket" + ], + "Resource": [ + "arn:aws:s3:::${BUCKET_NAME}" + ] + } + ] +} +``` + +**Note**: The `s3:GetObject` permission is required for the `HeadObject` API call that measures file sizes. Without this permission, document initialization will fail with an access denied error. + +#### File Size Tracking (January 2026) + +Documents now track file size at initialization time. The `docInitRunner` service measures file size using S3 `HeadObject` before storing documents. This requires: + +- **Permission**: `s3:GetObject` on the document bucket +- **Environment**: `BUCKET` environment variable must be set +- **Error Handling**: If HeadObject fails, document creation will fail (not silently ignored) + +#### Compose File Configuration + +For local development with localstack, the bucket is configured via environment variables: + +```yaml +# deployments/compose.local.yaml +docInitRunner: + environment: + BUCKET: ${BUCKET_IN} # Maps to the "documentin" bucket +``` + +For production deployments, ensure the IAM role attached to the service has the required S3 permissions above. + +### SQS Permissions + +Queue consumer services require SQS permissions: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "SQSConsumerPermissions", + "Effect": "Allow", + "Action": [ + "sqs:ReceiveMessage", + "sqs:DeleteMessage", + "sqs:GetQueueAttributes", + "sqs:ChangeMessageVisibility" + ], + "Resource": [ + "arn:aws:sqs:${REGION}:${ACCOUNT_ID}:document_init", + "arn:aws:sqs:${REGION}:${ACCOUNT_ID}:document_sync", + "arn:aws:sqs:${REGION}:${ACCOUNT_ID}:document_clean", + "arn:aws:sqs:${REGION}:${ACCOUNT_ID}:document_text" + ] + } + ] +} +``` + +### Textract Permissions + +For OCR text extraction, additional Textract permissions are required: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "TextractPermissions", + "Effect": "Allow", + "Action": [ + "textract:StartDocumentTextDetection", + "textract:GetDocumentTextDetection", + "textract:StartDocumentAnalysis", + "textract:GetDocumentAnalysis" + ], + "Resource": "*" + } + ] +} +``` + ## Security Operations ### Security Monitoring diff --git a/go.mod b/go.mod index 1d095b46..403fe9e3 100644 --- a/go.mod +++ b/go.mod @@ -33,6 +33,7 @@ require ( github.com/tidwall/gjson v1.18.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.34.0 go.opentelemetry.io/otel/sdk v1.34.0 + golang.org/x/sync v0.12.0 golang.org/x/time v0.11.0 ) @@ -127,7 +128,6 @@ require ( golang.org/x/image v0.25.0 // indirect golang.org/x/mod v0.23.0 // indirect golang.org/x/oauth2 v0.27.0 // indirect - golang.org/x/sync v0.12.0 // indirect golang.org/x/term v0.30.0 // indirect golang.org/x/tools v0.30.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20250303144028-a0af3efb3deb // indirect diff --git a/internal/database/migrations/00000000000118_add_document_file_size.down.sql b/internal/database/migrations/00000000000118_add_document_file_size.down.sql new file mode 100644 index 00000000..4ff05c19 --- /dev/null +++ b/internal/database/migrations/00000000000118_add_document_file_size.down.sql @@ -0,0 +1,2 @@ +-- Remove file_size_bytes column from documents table +ALTER TABLE documents DROP COLUMN file_size_bytes; diff --git a/internal/database/migrations/00000000000118_add_document_file_size.up.sql b/internal/database/migrations/00000000000118_add_document_file_size.up.sql new file mode 100644 index 00000000..80d5ac9d --- /dev/null +++ b/internal/database/migrations/00000000000118_add_document_file_size.up.sql @@ -0,0 +1,4 @@ +-- Add file_size_bytes column to documents table +-- Stores the size of the document in bytes as measured from S3 HeadObject +-- NULL for legacy documents (before this feature was added) +ALTER TABLE documents ADD COLUMN file_size_bytes BIGINT; diff --git a/internal/database/queries/document.sql b/internal/database/queries/document.sql index 9b0311c1..10b8f5f2 100644 --- a/internal/database/queries/document.sql +++ b/internal/database/queries/document.sql @@ -34,7 +34,7 @@ SELECT id, hash from documents where clientId = @clientId; SELECT id, hash, filename from documents where batch_id = @batch_id; -- name: CreateDocument :one -INSERT INTO documents (clientId, hash, batch_id, filename, folderId, originalPath) VALUES ($1, $2, $3, $4, $5, $6) RETURNING id; +INSERT INTO documents (clientId, hash, batch_id, filename, folderId, originalPath, file_size_bytes) VALUES ($1, $2, $3, $4, $5, $6, $7) RETURNING id; -- name: AddDocumentEntry :exec INSERT INTO documentEntries (documentId, bucket, key) VALUES ($1, $2, $3); @@ -85,6 +85,7 @@ SELECT d.hash, d.folderId, d.filename, - d.originalPath + d.originalPath, + d.file_size_bytes FROM documents d WHERE d.id = $1; diff --git a/internal/database/queries/folders.sql b/internal/database/queries/folders.sql index de96b6ac..f43fa713 100644 --- a/internal/database/queries/folders.sql +++ b/internal/database/queries/folders.sql @@ -95,7 +95,8 @@ SELECT d.hash, d.folderId, d.filename, - d.originalPath + d.originalPath, + d.file_size_bytes FROM documents d WHERE d.folderId = $1 ORDER BY d.id; diff --git a/internal/database/repository/document.sql.go b/internal/database/repository/document.sql.go index a9d02d0e..d2207133 100644 --- a/internal/database/repository/document.sql.go +++ b/internal/database/repository/document.sql.go @@ -65,21 +65,22 @@ func (q *Queries) AddDocumentUpload(ctx context.Context, arg *AddDocumentUploadP } const createDocument = `-- name: CreateDocument :one -INSERT INTO documents (clientId, hash, batch_id, filename, folderId, originalPath) VALUES ($1, $2, $3, $4, $5, $6) RETURNING id +INSERT INTO documents (clientId, hash, batch_id, filename, folderId, originalPath, file_size_bytes) VALUES ($1, $2, $3, $4, $5, $6, $7) RETURNING id ` type CreateDocumentParams struct { - Clientid string `db:"clientid"` - Hash string `db:"hash"` - BatchID *uuid.UUID `db:"batch_id"` - Filename *string `db:"filename"` - Folderid *uuid.UUID `db:"folderid"` - Originalpath *string `db:"originalpath"` + Clientid string `db:"clientid"` + Hash string `db:"hash"` + BatchID *uuid.UUID `db:"batch_id"` + Filename *string `db:"filename"` + Folderid *uuid.UUID `db:"folderid"` + Originalpath *string `db:"originalpath"` + FileSizeBytes *int64 `db:"file_size_bytes"` } // CreateDocument // -// INSERT INTO documents (clientId, hash, batch_id, filename, folderId, originalPath) VALUES ($1, $2, $3, $4, $5, $6) RETURNING id +// INSERT INTO documents (clientId, hash, batch_id, filename, folderId, originalPath, file_size_bytes) VALUES ($1, $2, $3, $4, $5, $6, $7) RETURNING id func (q *Queries) CreateDocument(ctx context.Context, arg *CreateDocumentParams) (uuid.UUID, error) { row := q.db.QueryRow(ctx, createDocument, arg.Clientid, @@ -88,6 +89,7 @@ func (q *Queries) CreateDocument(ctx context.Context, arg *CreateDocumentParams) arg.Filename, arg.Folderid, arg.Originalpath, + arg.FileSizeBytes, ) var id uuid.UUID err := row.Scan(&id) @@ -101,18 +103,20 @@ SELECT d.hash, d.folderId, d.filename, - d.originalPath + d.originalPath, + d.file_size_bytes FROM documents d WHERE d.id = $1 ` type GetDocumentEnrichedRow struct { - ID uuid.UUID `db:"id"` - Clientid string `db:"clientid"` - Hash string `db:"hash"` - Folderid *uuid.UUID `db:"folderid"` - Filename *string `db:"filename"` - Originalpath *string `db:"originalpath"` + ID uuid.UUID `db:"id"` + Clientid string `db:"clientid"` + Hash string `db:"hash"` + Folderid *uuid.UUID `db:"folderid"` + Filename *string `db:"filename"` + Originalpath *string `db:"originalpath"` + FileSizeBytes *int64 `db:"file_size_bytes"` } // Retrieves a document with extended metadata including folder and filename @@ -123,7 +127,8 @@ type GetDocumentEnrichedRow struct { // d.hash, // d.folderId, // d.filename, -// d.originalPath +// d.originalPath, +// d.file_size_bytes // FROM documents d // WHERE d.id = $1 func (q *Queries) GetDocumentEnriched(ctx context.Context, id uuid.UUID) (*GetDocumentEnrichedRow, error) { @@ -136,6 +141,7 @@ func (q *Queries) GetDocumentEnriched(ctx context.Context, id uuid.UUID) (*GetDo &i.Folderid, &i.Filename, &i.Originalpath, + &i.FileSizeBytes, ) return &i, err } diff --git a/internal/database/repository/folders.sql.go b/internal/database/repository/folders.sql.go index ada1e085..a344f861 100644 --- a/internal/database/repository/folders.sql.go +++ b/internal/database/repository/folders.sql.go @@ -89,14 +89,14 @@ func (q *Queries) GetClientRootFolder(ctx context.Context, clientid string) (*Fo } const getDocumentsByFolder = `-- name: GetDocumentsByFolder :many -SELECT id, clientid, hash, batch_id, filename, folderid, originalpath FROM documents +SELECT id, clientid, hash, batch_id, filename, folderid, originalpath, file_size_bytes FROM documents WHERE folderId = $1 ORDER BY id ` // GetDocumentsByFolder // -// SELECT id, clientid, hash, batch_id, filename, folderid, originalpath FROM documents +// SELECT id, clientid, hash, batch_id, filename, folderid, originalpath, file_size_bytes FROM documents // WHERE folderId = $1 // ORDER BY id func (q *Queries) GetDocumentsByFolder(ctx context.Context, folderid *uuid.UUID) ([]*Document, error) { @@ -116,6 +116,7 @@ func (q *Queries) GetDocumentsByFolder(ctx context.Context, folderid *uuid.UUID) &i.Filename, &i.Folderid, &i.Originalpath, + &i.FileSizeBytes, ); err != nil { return nil, err } @@ -133,18 +134,20 @@ SELECT d.hash, d.folderId, d.filename, - d.originalPath + d.originalPath, + d.file_size_bytes FROM documents d WHERE d.folderId = $1 ORDER BY d.id ` type GetDocumentsByFolderEnrichedRow struct { - ID uuid.UUID `db:"id"` - Hash string `db:"hash"` - Folderid *uuid.UUID `db:"folderid"` - Filename *string `db:"filename"` - Originalpath *string `db:"originalpath"` + ID uuid.UUID `db:"id"` + Hash string `db:"hash"` + Folderid *uuid.UUID `db:"folderid"` + Filename *string `db:"filename"` + Originalpath *string `db:"originalpath"` + FileSizeBytes *int64 `db:"file_size_bytes"` } // Gets documents in a folder with all enriched metadata fields @@ -154,7 +157,8 @@ type GetDocumentsByFolderEnrichedRow struct { // d.hash, // d.folderId, // d.filename, -// d.originalPath +// d.originalPath, +// d.file_size_bytes // FROM documents d // WHERE d.folderId = $1 // ORDER BY d.id @@ -173,6 +177,7 @@ func (q *Queries) GetDocumentsByFolderEnriched(ctx context.Context, folderid *uu &i.Folderid, &i.Filename, &i.Originalpath, + &i.FileSizeBytes, ); err != nil { return nil, err } diff --git a/internal/database/repository/labels.sql.go b/internal/database/repository/labels.sql.go index f541780d..7c5c42d2 100644 --- a/internal/database/repository/labels.sql.go +++ b/internal/database/repository/labels.sql.go @@ -191,7 +191,7 @@ func (q *Queries) GetDocumentLabels(ctx context.Context, documentid uuid.UUID) ( } const getDocumentsByLabel = `-- name: GetDocumentsByLabel :many -SELECT DISTINCT d.id, d.clientid, d.hash, d.batch_id, d.filename, d.folderid, d.originalpath +SELECT DISTINCT d.id, d.clientid, d.hash, d.batch_id, d.filename, d.folderid, d.originalpath, d.file_size_bytes FROM documents d INNER JOIN documentLabels dl ON d.id = dl.documentId WHERE d.clientId = $1 AND dl.label = $2 @@ -205,7 +205,7 @@ type GetDocumentsByLabelParams struct { // GetDocumentsByLabel // -// SELECT DISTINCT d.id, d.clientid, d.hash, d.batch_id, d.filename, d.folderid, d.originalpath +// SELECT DISTINCT d.id, d.clientid, d.hash, d.batch_id, d.filename, d.folderid, d.originalpath, d.file_size_bytes // FROM documents d // INNER JOIN documentLabels dl ON d.id = dl.documentId // WHERE d.clientId = $1 AND dl.label = $2 @@ -227,6 +227,7 @@ func (q *Queries) GetDocumentsByLabel(ctx context.Context, arg *GetDocumentsByLa &i.Filename, &i.Folderid, &i.Originalpath, + &i.FileSizeBytes, ); err != nil { return nil, err } @@ -239,7 +240,7 @@ func (q *Queries) GetDocumentsByLabel(ctx context.Context, arg *GetDocumentsByLa } const getDocumentsByLabelAndFolder = `-- name: GetDocumentsByLabelAndFolder :many -SELECT DISTINCT d.id, d.clientid, d.hash, d.batch_id, d.filename, d.folderid, d.originalpath +SELECT DISTINCT d.id, d.clientid, d.hash, d.batch_id, d.filename, d.folderid, d.originalpath, d.file_size_bytes FROM documents d INNER JOIN documentLabels dl ON d.id = dl.documentId WHERE d.folderId = $1 AND dl.label = $2 @@ -253,7 +254,7 @@ type GetDocumentsByLabelAndFolderParams struct { // GetDocumentsByLabelAndFolder // -// SELECT DISTINCT d.id, d.clientid, d.hash, d.batch_id, d.filename, d.folderid, d.originalpath +// SELECT DISTINCT d.id, d.clientid, d.hash, d.batch_id, d.filename, d.folderid, d.originalpath, d.file_size_bytes // FROM documents d // INNER JOIN documentLabels dl ON d.id = dl.documentId // WHERE d.folderId = $1 AND dl.label = $2 @@ -275,6 +276,7 @@ func (q *Queries) GetDocumentsByLabelAndFolder(ctx context.Context, arg *GetDocu &i.Filename, &i.Folderid, &i.Originalpath, + &i.FileSizeBytes, ); err != nil { return nil, err } diff --git a/internal/database/repository/models.go b/internal/database/repository/models.go index 17ac58dc..d0b0c27c 100644 --- a/internal/database/repository/models.go +++ b/internal/database/repository/models.go @@ -425,7 +425,8 @@ type Document struct { Filename *string `db:"filename"` Folderid *uuid.UUID `db:"folderid"` // Original path provided during upload. IMMUTABLE after creation - never modify this value. - Originalpath *string `db:"originalpath"` + Originalpath *string `db:"originalpath"` + FileSizeBytes *int64 `db:"file_size_bytes"` } type Documentclean struct { diff --git a/internal/document/get.go b/internal/document/get.go index 844e0bf4..48bda745 100644 --- a/internal/document/get.go +++ b/internal/document/get.go @@ -103,6 +103,7 @@ func (s *Service) GetEnriched(ctx context.Context, id uuid.UUID, includeTextReco Filename: docEnriched.Filename, OriginalPath: docEnriched.Originalpath, Labels: labels, + FileSizeBytes: docEnriched.FileSizeBytes, } // 5. Optionally fetch full text record if requested and exists diff --git a/internal/document/init/create.go b/internal/document/init/create.go index 502083e1..6f473acb 100644 --- a/internal/document/init/create.go +++ b/internal/document/init/create.go @@ -4,6 +4,7 @@ import ( "context" "database/sql" "errors" + "fmt" "log/slog" docsyncrunner "queryorchestration/api/docSyncRunner" @@ -11,6 +12,8 @@ import ( "queryorchestration/internal/serviceconfig/objectstore" "queryorchestration/internal/serviceconfig/queue" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/google/uuid" ) @@ -85,16 +88,24 @@ func (s *Service) getCreateParams(ctx context.Context, doc *Create) (*createDocu func (s *Service) submitCreate(ctx context.Context, params *createDocumentParams) (uuid.UUID, error) { var id uuid.UUID - err := s.cfg.ExecuteDBTransaction(ctx, func(ctx context.Context, q *repository.Queries) error { + // Measure file size from S3 (trusted source - we measure it ourselves) + // The S3 client must be initialized - file size tracking is required + fileSizeBytes, err := s.measureFileSize(ctx, params.Bucket, params.Key.String()) + if err != nil { + return uuid.Nil, fmt.Errorf("failed to measure file size: %w", err) + } + + err = s.cfg.ExecuteDBTransaction(ctx, func(ctx context.Context, q *repository.Queries) error { var dbid uuid.UUID if params.ID == nil { createid, err := s.cfg.GetDBQueries().CreateDocument(ctx, &repository.CreateDocumentParams{ - Clientid: params.Key.ClientID, - Hash: params.Hash, - BatchID: params.Key.BatchID, - Filename: params.Filename, - Folderid: params.FolderID, - Originalpath: params.OriginalPath, + Clientid: params.Key.ClientID, + Hash: params.Hash, + BatchID: params.Key.BatchID, + Filename: params.Filename, + Folderid: params.FolderID, + Originalpath: params.OriginalPath, + FileSizeBytes: fileSizeBytes, }) if err != nil { return err @@ -127,3 +138,38 @@ func (s *Service) submitCreate(ctx context.Context, params *createDocumentParams return id, nil } + +// measureFileSize retrieves the file size from S3 using HeadObject. +// Returns the file size in bytes, or an error if the S3 client is not configured +// or if the HeadObject call fails. +// +// Parameters: +// - ctx: Context for the S3 operation +// - bucket: S3 bucket name +// - key: S3 object key +// +// Returns: +// - *int64: Pointer to file size in bytes +// - error: Error if S3 client is nil or HeadObject fails +// +// Note: This operation requires s3:GetObject permission on the bucket/key. +func (s *Service) measureFileSize(ctx context.Context, bucket, key string) (*int64, error) { + storeClient := s.cfg.GetStoreClient() + if storeClient == nil { + return nil, errors.New("S3 client not initialized - required for file size measurement") + } + + headOutput, err := storeClient.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(key), + }) + if err != nil { + return nil, fmt.Errorf("HeadObject failed for s3://%s/%s: %w", bucket, key, err) + } + + if headOutput.ContentLength == nil { + return nil, fmt.Errorf("HeadObject returned nil ContentLength for s3://%s/%s", bucket, key) + } + + return headOutput.ContentLength, nil +} diff --git a/internal/document/init/create_test.go b/internal/document/init/create_test.go index 8c1eee6f..1824b76b 100644 --- a/internal/document/init/create_test.go +++ b/internal/document/init/create_test.go @@ -1,6 +1,7 @@ package documentinit import ( + "bytes" "errors" "fmt" "testing" @@ -9,10 +10,14 @@ import ( "queryorchestration/internal/database/repository" "queryorchestration/internal/document" "queryorchestration/internal/serviceconfig/objectstore" + "queryorchestration/internal/test" + objectstoremock "queryorchestration/mocks/objectstore" queuemock "queryorchestration/mocks/queue" "github.com/stretchr/testify/require" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/sqs" "github.com/google/uuid" "github.com/pashagolub/pgxmock/v3" @@ -30,6 +35,11 @@ func TestCreate(t *testing.T) { cfg.DBQueries = repository.New(pool) mockSQS := queuemock.NewMockSQSClient(t) cfg.QueueClient = mockSQS + + // Set up mock S3 client for file size measurement + mockS3 := objectstoremock.NewMockS3Client(t) + cfg.StoreClient = mockS3 + cfg.DocumentSyncURL = "/i/am/here" svc := New(cfg) @@ -48,11 +58,19 @@ func TestCreate(t *testing.T) { Location: objectstore.Import, } + // Mock HeadObject to return file size + fileSize := int64(1024) + mockS3.EXPECT(). + HeadObject(mock.Anything, mock.MatchedBy(func(in *s3.HeadObjectInput) bool { + return *in.Bucket == bucket && *in.Key == key.String() + })). + Return(&s3.HeadObjectOutput{ContentLength: &fileSize}, nil) + pool.ExpectQuery("name: GetDocumentIDByHash :one").WithArgs(doc.Hash, doc.ClientID).WillReturnRows( pgxmock.NewRows([]string{"id"}), ) pool.ExpectBegin() - pool.ExpectQuery("name: CreateDocument :one").WithArgs(doc.ClientID, doc.Hash, (*uuid.UUID)(nil), (*string)(nil), (*uuid.UUID)(nil), (*string)(nil)). + pool.ExpectQuery("name: CreateDocument :one").WithArgs(doc.ClientID, doc.Hash, (*uuid.UUID)(nil), (*string)(nil), (*uuid.UUID)(nil), (*string)(nil), &fileSize). WillReturnRows( pgxmock.NewRows([]string{"id"}). AddRow(doc.ID), @@ -220,6 +238,10 @@ func TestSubmitCreate(t *testing.T) { cfg.DBPool = pool cfg.DBQueries = repository.New(pool) + // Set up mock S3 client for file size measurement + mockS3 := objectstoremock.NewMockS3Client(t) + cfg.StoreClient = mockS3 + svc := New(cfg) doc := document.DocumentSummary{ @@ -235,8 +257,16 @@ func TestSubmitCreate(t *testing.T) { Location: objectstore.Import, } + // Mock HeadObject to return file size + fileSize := int64(2048) + mockS3.EXPECT(). + HeadObject(mock.Anything, mock.MatchedBy(func(in *s3.HeadObjectInput) bool { + return *in.Bucket == bucket && *in.Key == key.String() + })). + Return(&s3.HeadObjectOutput{ContentLength: &fileSize}, nil) + pool.ExpectBegin() - pool.ExpectQuery("name: CreateDocument :one").WithArgs(doc.ClientID, doc.Hash, (*uuid.UUID)(nil), (*string)(nil), (*uuid.UUID)(nil), (*string)(nil)). + pool.ExpectQuery("name: CreateDocument :one").WithArgs(doc.ClientID, doc.Hash, (*uuid.UUID)(nil), (*string)(nil), (*uuid.UUID)(nil), (*string)(nil), &fileSize). WillReturnRows( pgxmock.NewRows([]string{"id"}). AddRow(doc.ID), @@ -264,6 +294,10 @@ func TestSubmitCreateExists(t *testing.T) { cfg.DBPool = pool cfg.DBQueries = repository.New(pool) + // Set up mock S3 client for file size measurement + mockS3 := objectstoremock.NewMockS3Client(t) + cfg.StoreClient = mockS3 + svc := New(cfg) doc := document.DocumentSummary{ @@ -279,6 +313,14 @@ func TestSubmitCreateExists(t *testing.T) { Location: objectstore.Import, } + // Mock HeadObject to return file size (still called even for existing docs) + fileSize := int64(4096) + mockS3.EXPECT(). + HeadObject(mock.Anything, mock.MatchedBy(func(in *s3.HeadObjectInput) bool { + return *in.Bucket == bucket && *in.Key == key.String() + })). + Return(&s3.HeadObjectOutput{ContentLength: &fileSize}, nil) + pool.ExpectBegin() pool.ExpectExec("name: AddDocumentEntry :exec").WithArgs(doc.ID, bucket, key.String()). WillReturnResult(pgxmock.NewResult("", 1)) @@ -294,3 +336,200 @@ func TestSubmitCreateExists(t *testing.T) { require.NoError(t, err) assert.Equal(t, doc.ID, id) } + +// TestSubmitCreate_HeadObjectError verifies that HeadObject errors are properly propagated. +func TestSubmitCreate_HeadObjectError(t *testing.T) { + ctx := t.Context() + + pool, err := pgxmock.NewPool() + require.NoError(t, err) + cfg := &DocInitConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) + + // Set up mock S3 client that returns an error + mockS3 := objectstoremock.NewMockS3Client(t) + cfg.StoreClient = mockS3 + + svc := New(cfg) + + doc := document.DocumentSummary{ + ID: uuid.New(), + ClientID: "hello", + Hash: "example_hash", + } + bucket := "example_bucket" + key := objectstore.BucketKey{ + ClientID: doc.ClientID, + CreatedAt: time.Now().UTC(), + EntityID: uuid.New(), + Location: objectstore.Import, + } + + // Mock HeadObject to return an error + mockS3.EXPECT(). + HeadObject(mock.Anything, mock.Anything). + Return(nil, errors.New("access denied")) + + _, err = svc.submitCreate(ctx, &createDocumentParams{ + Hash: doc.Hash, + Bucket: bucket, + Key: key, + }) + assert.Error(t, err) + assert.Contains(t, err.Error(), "failed to measure file size") + assert.Contains(t, err.Error(), "access denied") +} + +// TestSubmitCreate_NoS3Client verifies that missing S3 client returns an error. +func TestSubmitCreate_NoS3Client(t *testing.T) { + ctx := t.Context() + + pool, err := pgxmock.NewPool() + require.NoError(t, err) + cfg := &DocInitConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) + // Intentionally not setting StoreClient + + svc := New(cfg) + + doc := document.DocumentSummary{ + ID: uuid.New(), + ClientID: "hello", + Hash: "example_hash", + } + bucket := "example_bucket" + key := objectstore.BucketKey{ + ClientID: doc.ClientID, + CreatedAt: time.Now().UTC(), + EntityID: uuid.New(), + Location: objectstore.Import, + } + + _, err = svc.submitCreate(ctx, &createDocumentParams{ + Hash: doc.Hash, + Bucket: bucket, + Key: key, + }) + assert.Error(t, err) + assert.Contains(t, err.Error(), "S3 client not initialized") +} + +// TestMeasureFileSize_NilContentLength verifies handling of nil ContentLength. +func TestMeasureFileSize_NilContentLength(t *testing.T) { + ctx := t.Context() + + pool, err := pgxmock.NewPool() + require.NoError(t, err) + cfg := &DocInitConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) + + // Set up mock S3 client that returns nil ContentLength + mockS3 := objectstoremock.NewMockS3Client(t) + cfg.StoreClient = mockS3 + + svc := New(cfg) + + bucket := "example_bucket" + key := "example_key" + + mockS3.EXPECT(). + HeadObject(mock.Anything, mock.Anything). + Return(&s3.HeadObjectOutput{ContentLength: nil}, nil) + + _, err = svc.measureFileSize(ctx, bucket, key) + assert.Error(t, err) + assert.Contains(t, err.Error(), "nil ContentLength") +} + +// TestMeasureFileSize_Integration tests the measureFileSize function using a real +// S3 client (localstack) to verify that HeadObject correctly returns file sizes. +// This is an integration test that requires Docker. +func TestMeasureFileSize_Integration(t *testing.T) { + ctx := t.Context() + + // Set up AWS container (localstack) and S3 client + cfg := &DocInitConfig{} + test.CreateAWSResources(t, cfg) + + svc := New(cfg) + + // Upload a test file with known content + bucket := cfg.GetBucket() + testKey := fmt.Sprintf("test/filesize/%s.txt", uuid.New().String()) + testContent := []byte("Hello, World! This is a test file for measuring file size.") + expectedSize := int64(len(testContent)) + + // Upload the test file to S3 + _, err := cfg.GetStoreClient().PutObject(ctx, &s3.PutObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(testKey), + Body: bytes.NewReader(testContent), + }) + require.NoError(t, err, "failed to upload test file to S3") + + // Measure the file size using our function + measuredSize, err := svc.measureFileSize(ctx, bucket, testKey) + require.NoError(t, err, "measureFileSize should not return an error") + require.NotNil(t, measuredSize, "measured size should not be nil") + + // Verify the measured size matches expected + assert.Equal(t, expectedSize, *measuredSize, "measured file size should match uploaded content size") +} + +// TestMeasureFileSize_Integration_LargeFile tests file size measurement for larger files. +func TestMeasureFileSize_Integration_LargeFile(t *testing.T) { + ctx := t.Context() + + // Set up AWS container (localstack) and S3 client + cfg := &DocInitConfig{} + test.CreateAWSResources(t, cfg) + + svc := New(cfg) + + // Upload a larger test file (1MB) + bucket := cfg.GetBucket() + testKey := fmt.Sprintf("test/filesize/large/%s.bin", uuid.New().String()) + expectedSize := int64(1024 * 1024) // 1MB + testContent := make([]byte, expectedSize) + for i := range testContent { + testContent[i] = byte(i % 256) + } + + // Upload the test file to S3 + _, err := cfg.GetStoreClient().PutObject(ctx, &s3.PutObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(testKey), + Body: bytes.NewReader(testContent), + }) + require.NoError(t, err, "failed to upload large test file to S3") + + // Measure the file size using our function + measuredSize, err := svc.measureFileSize(ctx, bucket, testKey) + require.NoError(t, err, "measureFileSize should not return an error for large files") + require.NotNil(t, measuredSize, "measured size should not be nil") + + // Verify the measured size matches expected + assert.Equal(t, expectedSize, *measuredSize, "measured file size should match 1MB") +} + +// TestMeasureFileSize_Integration_NonExistentKey tests error handling for non-existent S3 keys. +func TestMeasureFileSize_Integration_NonExistentKey(t *testing.T) { + ctx := t.Context() + + // Set up AWS container (localstack) and S3 client + cfg := &DocInitConfig{} + test.CreateAWSResources(t, cfg) + + svc := New(cfg) + + bucket := cfg.GetBucket() + nonExistentKey := fmt.Sprintf("test/nonexistent/%s.txt", uuid.New().String()) + + // Try to measure size of non-existent file + _, err := svc.measureFileSize(ctx, bucket, nonExistentKey) + assert.Error(t, err, "measureFileSize should return an error for non-existent key") + assert.Contains(t, err.Error(), "HeadObject failed", "error should indicate HeadObject failure") +} diff --git a/internal/document/init/service.go b/internal/document/init/service.go index 9a0c5d0d..294aeb88 100644 --- a/internal/document/init/service.go +++ b/internal/document/init/service.go @@ -2,12 +2,14 @@ package documentinit import ( "queryorchestration/internal/serviceconfig" + "queryorchestration/internal/serviceconfig/objectstore" documentsync "queryorchestration/internal/serviceconfig/queue/documentsync" ) type ConfigProvider interface { serviceconfig.ConfigProvider documentsync.ConfigProvider + objectstore.ConfigProvider // Provides GetStoreClient() for S3 access to measure file size } type Service struct { diff --git a/internal/document/init/service_test.go b/internal/document/init/service_test.go index e600e0f7..2e7048f1 100644 --- a/internal/document/init/service_test.go +++ b/internal/document/init/service_test.go @@ -4,6 +4,7 @@ import ( "testing" "queryorchestration/internal/serviceconfig" + "queryorchestration/internal/serviceconfig/objectstore" "queryorchestration/internal/serviceconfig/queue/documentsync" "github.com/stretchr/testify/assert" @@ -12,6 +13,7 @@ import ( type DocInitConfig struct { serviceconfig.BaseConfig documentsync.DocSyncConfig + objectstore.ObjectStoreConfig } func TestNewDocumentInitService(t *testing.T) { diff --git a/internal/document/service.go b/internal/document/service.go index bb04cda7..56b481e5 100644 --- a/internal/document/service.go +++ b/internal/document/service.go @@ -36,6 +36,7 @@ type DocumentEnriched struct { OriginalPath *string Labels []LabelRecord TextRecord *TextRecord // Only populated when includeTextRecord=true + FileSizeBytes *int64 // File size in bytes (nil for legacy documents) } // LabelRecord represents a label applied to a document. diff --git a/internal/folder/service.go b/internal/folder/service.go index 3f41a883..159ca522 100644 --- a/internal/folder/service.go +++ b/internal/folder/service.go @@ -248,6 +248,7 @@ type DocumentInFolder struct { OriginalPath *string Labels []LabelRecord TextRecord *TextRecord // Only populated when includeTextRecord=true + FileSizeBytes *int64 // File size in bytes (nil for legacy documents) } // LabelRecord represents a label applied to a document. @@ -368,6 +369,7 @@ func (s *Service) GetDocumentsEnriched(ctx context.Context, folderID uuid.UUID, OriginalPath: doc.Originalpath, Labels: labels, TextRecord: textRecordsMap[doc.ID], + FileSizeBytes: doc.FileSizeBytes, } } diff --git a/internal/test/config.go b/internal/test/config.go new file mode 100644 index 00000000..479f1baa --- /dev/null +++ b/internal/test/config.go @@ -0,0 +1,152 @@ +// Package test provides configuration utilities for test infrastructure. +// This file centralizes timeout configuration for CI/CD stability. +package test + +import ( + "fmt" + "log/slog" + "net/http" + "os" + "time" +) + +// GetHTTPTimeout returns the HTTP client timeout for API calls. +// Default is 30 seconds, configurable via E2E_HTTP_TIMEOUT environment variable. +// In CI environments, this timeout ensures that stalled HTTP calls don't block +// indefinitely, which was a root cause of test hangs. +func GetHTTPTimeout() time.Duration { + if t := os.Getenv("E2E_HTTP_TIMEOUT"); t != "" { + if d, err := time.ParseDuration(t); err == nil { + return d + } + } + return 30 * time.Second +} + +// GetPollingTimeout returns the timeout for polling operations like waiting +// for client status or mock endpoint verification. +// Default is 60 seconds locally, 180 seconds in CI (detected via CI or +// BITBUCKET_PIPELINE_UUID environment variables). +// Configurable via E2E_WAIT_TIMEOUT environment variable. +func GetPollingTimeout() time.Duration { + if t := os.Getenv("E2E_WAIT_TIMEOUT"); t != "" { + if d, err := time.ParseDuration(t); err == nil { + return d + } + } + if isCI() { + return 180 * time.Second // 3 minutes in CI + } + return 60 * time.Second // 1 minute locally +} + +// GetPollInterval returns the interval between polling attempts. +// Default is 500ms, configurable via E2E_POLL_INTERVAL environment variable. +func GetPollInterval() time.Duration { + if t := os.Getenv("E2E_POLL_INTERVAL"); t != "" { + if d, err := time.ParseDuration(t); err == nil { + return d + } + } + return 500 * time.Millisecond +} + +// GetHTTPClient returns an HTTP client with the configured timeout. +// This should be used instead of &http.Client{} to ensure all HTTP operations +// have bounded timeouts and won't hang indefinitely. +func GetHTTPClient() *http.Client { + return &http.Client{ + Timeout: GetHTTPTimeout(), + } +} + +// isCI returns true if running in a CI environment. +// Detects CI via the CI or BITBUCKET_PIPELINE_UUID environment variables. +func isCI() bool { + return os.Getenv("CI") != "" || os.Getenv("BITBUCKET_PIPELINE_UUID") != "" +} + +// WaitForExternalReadiness waits for a service to be reachable from outside Docker. +// This addresses the race condition where a container's internal health check passes +// but the external port mapping isn't yet ready. This is particularly important in +// Docker-in-Docker (DinD) environments like Bitbucket Pipelines. +// +// Parameters: +// - uri: The base URI of the service (e.g., "http://localhost:32768") +// - healthPath: The health check endpoint path (e.g., "/health" or "/liveness/probe") +// - timeout: Maximum time to wait for the service to become reachable +// +// Returns an error if the service is not reachable within the timeout period. +func WaitForExternalReadiness(uri string, healthPath string, timeout time.Duration) error { + client := &http.Client{Timeout: 2 * time.Second} + deadline := time.Now().Add(timeout) + + fullURL := fmt.Sprintf("%s%s", uri, healthPath) + slog.Info("Waiting for external readiness", "url", fullURL, "timeout", timeout) + + var lastErr error + attempts := 0 + for time.Now().Before(deadline) { + attempts++ + resp, err := client.Get(fullURL) + if err == nil && resp.StatusCode == http.StatusOK { + resp.Body.Close() + slog.Info("Service is externally reachable", "url", fullURL, "attempts", attempts) + return nil + } + if err != nil { + lastErr = err + } else { + lastErr = fmt.Errorf("unexpected status code: %d", resp.StatusCode) + resp.Body.Close() + } + time.Sleep(500 * time.Millisecond) + } + + return fmt.Errorf("service not reachable from test client after %v (%d attempts): %w", timeout, attempts, lastErr) +} + +// WaitForAPIRoutes waits for API routes to be fully registered. +// This addresses a race condition where the health endpoint returns 200 but other +// API routes haven't been registered yet, causing "404 page not found" errors. +// This is particularly important in slow CI environments where route registration +// may take longer after the health endpoint becomes available. +// +// Parameters: +// - uri: The base URI of the service (e.g., "http://localhost:32768") +// - timeout: Maximum time to wait for routes to be registered +// +// Returns an error if routes are not available within the timeout period. +func WaitForAPIRoutes(uri string, timeout time.Duration) error { + client := &http.Client{Timeout: 2 * time.Second} + deadline := time.Now().Add(timeout) + + // Check /query endpoint - should return 200 with empty array when routes are ready + // A "404 page not found" means routes aren't registered yet + routeURL := fmt.Sprintf("%s/query", uri) + slog.Info("Waiting for API routes to be registered", "url", routeURL, "timeout", timeout) + + var lastErr error + attempts := 0 + for time.Now().Before(deadline) { + attempts++ + resp, err := client.Get(routeURL) + if err == nil { + // Any response other than connection error means routes are registered + // 200 = success, 401 = auth required (but route exists), etc. + // Only "404 page not found" indicates routes not yet registered + if resp.StatusCode != http.StatusNotFound { + resp.Body.Close() + slog.Info("API routes are registered", "url", routeURL, "attempts", attempts, "status", resp.StatusCode) + return nil + } + lastErr = fmt.Errorf("routes not registered yet: %d", resp.StatusCode) + resp.Body.Close() + } else { + lastErr = err + } + time.Sleep(500 * time.Millisecond) + } + + return fmt.Errorf("API routes not registered after %v (%d attempts): %w", timeout, attempts, lastErr) +} diff --git a/internal/test/config_test.go b/internal/test/config_test.go new file mode 100644 index 00000000..aed76f3c --- /dev/null +++ b/internal/test/config_test.go @@ -0,0 +1,219 @@ +package test + +import ( + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestGetHTTPTimeout(t *testing.T) { + // Test default value + timeout := GetHTTPTimeout() + assert.Equal(t, 30*time.Second, timeout, "default HTTP timeout should be 30 seconds") + + // Test with environment variable + t.Setenv("E2E_HTTP_TIMEOUT", "45s") + timeout = GetHTTPTimeout() + assert.Equal(t, 45*time.Second, timeout, "HTTP timeout should respect E2E_HTTP_TIMEOUT env var") + + // Test with invalid environment variable (should fall back to default) + t.Setenv("E2E_HTTP_TIMEOUT", "invalid") + timeout = GetHTTPTimeout() + assert.Equal(t, 30*time.Second, timeout, "invalid E2E_HTTP_TIMEOUT should fall back to default") +} + +func TestGetPollingTimeout(t *testing.T) { + // Clear CI env vars to ensure we test non-CI default first + t.Setenv("CI", "") + t.Setenv("BITBUCKET_PIPELINE_UUID", "") + + // Test default value (non-CI) + timeout := GetPollingTimeout() + assert.Equal(t, 60*time.Second, timeout, "default polling timeout should be 60 seconds") + + // Test with environment variable override + t.Setenv("E2E_WAIT_TIMEOUT", "120s") + timeout = GetPollingTimeout() + assert.Equal(t, 120*time.Second, timeout, "polling timeout should respect E2E_WAIT_TIMEOUT env var") + + // Test CI detection + t.Setenv("E2E_WAIT_TIMEOUT", "") + t.Setenv("CI", "true") + timeout = GetPollingTimeout() + assert.Equal(t, 180*time.Second, timeout, "polling timeout in CI should be 180 seconds") + + // Test BITBUCKET_PIPELINE_UUID detection + t.Setenv("CI", "") + t.Setenv("BITBUCKET_PIPELINE_UUID", "some-uuid") + timeout = GetPollingTimeout() + assert.Equal(t, 180*time.Second, timeout, "polling timeout with BITBUCKET_PIPELINE_UUID should be 180 seconds") +} + +func TestGetPollInterval(t *testing.T) { + // Test default value + interval := GetPollInterval() + assert.Equal(t, 500*time.Millisecond, interval, "default poll interval should be 500ms") + + // Test with environment variable + t.Setenv("E2E_POLL_INTERVAL", "1s") + interval = GetPollInterval() + assert.Equal(t, 1*time.Second, interval, "poll interval should respect E2E_POLL_INTERVAL env var") + + // Test with invalid environment variable (should fall back to default) + t.Setenv("E2E_POLL_INTERVAL", "invalid") + interval = GetPollInterval() + assert.Equal(t, 500*time.Millisecond, interval, "invalid E2E_POLL_INTERVAL should fall back to default") +} + +func TestGetHTTPClient(t *testing.T) { + client := GetHTTPClient() + assert.NotNil(t, client, "GetHTTPClient should return a non-nil client") + assert.Equal(t, GetHTTPTimeout(), client.Timeout, "HTTP client timeout should match GetHTTPTimeout()") +} + +func TestIsCI(t *testing.T) { + // Clear CI env vars to ensure consistent test behavior + t.Setenv("CI", "") + t.Setenv("BITBUCKET_PIPELINE_UUID", "") + + // Test non-CI environment + assert.False(t, isCI(), "isCI should return false when CI env vars are not set") + + // Test with CI env var + t.Setenv("CI", "true") + assert.True(t, isCI(), "isCI should return true when CI is set") + + // Test with BITBUCKET_PIPELINE_UUID + t.Setenv("CI", "") + t.Setenv("BITBUCKET_PIPELINE_UUID", "some-uuid") + assert.True(t, isCI(), "isCI should return true when BITBUCKET_PIPELINE_UUID is set") +} + +func TestWaitForExternalReadiness(t *testing.T) { + t.Run("success on first attempt", func(t *testing.T) { + // Create a test server that returns 200 OK + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/health" { + w.WriteHeader(http.StatusOK) + return + } + w.WriteHeader(http.StatusNotFound) + })) + defer server.Close() + + err := WaitForExternalReadiness(server.URL, "/health", 5*time.Second) + assert.NoError(t, err, "should succeed when server is immediately ready") + }) + + t.Run("success after delay", func(t *testing.T) { + attempts := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/health" { + attempts++ + if attempts < 3 { + w.WriteHeader(http.StatusServiceUnavailable) + return + } + w.WriteHeader(http.StatusOK) + return + } + w.WriteHeader(http.StatusNotFound) + })) + defer server.Close() + + err := WaitForExternalReadiness(server.URL, "/health", 5*time.Second) + assert.NoError(t, err, "should succeed after retries") + assert.GreaterOrEqual(t, attempts, 3, "should have made multiple attempts") + }) + + t.Run("timeout on non-responsive server", func(t *testing.T) { + // Create a server that always returns 503 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusServiceUnavailable) + })) + defer server.Close() + + err := WaitForExternalReadiness(server.URL, "/health", 2*time.Second) + assert.Error(t, err, "should fail when server never becomes ready") + assert.Contains(t, err.Error(), "not reachable", "error should indicate service not reachable") + }) + + t.Run("timeout on unreachable server", func(t *testing.T) { + // Use an invalid URL that will fail to connect + err := WaitForExternalReadiness("http://127.0.0.1:1", "/health", 2*time.Second) + assert.Error(t, err, "should fail when server is unreachable") + assert.Contains(t, err.Error(), "not reachable", "error should indicate service not reachable") + }) +} + +func TestWaitForAPIRoutes(t *testing.T) { + t.Run("success when routes return 200", func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/query" { + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("[]")) + return + } + w.WriteHeader(http.StatusNotFound) + })) + defer server.Close() + + err := WaitForAPIRoutes(server.URL, 5*time.Second) + assert.NoError(t, err, "should succeed when routes return 200") + }) + + t.Run("success after routes become available", func(t *testing.T) { + attempts := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/query" { + attempts++ + if attempts < 3 { + // Simulate "404 page not found" before routes are registered + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte("404 page not found")) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("[]")) + return + } + w.WriteHeader(http.StatusNotFound) + })) + defer server.Close() + + err := WaitForAPIRoutes(server.URL, 5*time.Second) + assert.NoError(t, err, "should succeed after routes become available") + assert.GreaterOrEqual(t, attempts, 3, "should have made multiple attempts") + }) + + t.Run("success when routes return 401", func(t *testing.T) { + // 401 means the route exists but requires auth - route is registered + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/query" { + w.WriteHeader(http.StatusUnauthorized) + return + } + w.WriteHeader(http.StatusNotFound) + })) + defer server.Close() + + err := WaitForAPIRoutes(server.URL, 5*time.Second) + assert.NoError(t, err, "should succeed when routes return 401 (route exists)") + }) + + t.Run("timeout when routes never register", func(t *testing.T) { + // Server always returns 404 - routes never registered + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte("404 page not found")) + })) + defer server.Close() + + err := WaitForAPIRoutes(server.URL, 2*time.Second) + assert.Error(t, err, "should fail when routes never register") + assert.Contains(t, err.Error(), "not registered", "error should indicate routes not registered") + }) +} diff --git a/internal/test/ecosystem.go b/internal/test/ecosystem.go index 606c791a..62cb09e4 100644 --- a/internal/test/ecosystem.go +++ b/internal/test/ecosystem.go @@ -70,7 +70,24 @@ func CreateFullNetwork(t testing.TB, ctx context.Context, cfg FullDependenciesCo wg.Wait() - qService, err := queryapi.NewClientWithResponses(apiContainers[QueryAPIName].URI) + // Wait for external readiness before creating the client. + // This addresses the race condition where containers report healthy internally + // but the external port mapping isn't fully ready, causing 404 errors in CI. + err := WaitForExternalReadiness(apiContainers[QueryAPIName].URI, "/health", GetPollingTimeout()) + require.NoError(t, err, "QueryAPI not externally reachable") + + // Wait for API routes to be registered. + // The health endpoint may return 200 before all routes are registered, + // causing "404 page not found" errors on actual API calls. + err = WaitForAPIRoutes(apiContainers[QueryAPIName].URI, GetPollingTimeout()) + require.NoError(t, err, "QueryAPI routes not registered") + + // Create API client with bounded HTTP timeout to prevent indefinite hangs + // that were causing CI failures (goroutines blocking until 15-minute global timeout) + qService, err := queryapi.NewClientWithResponses( + apiContainers[QueryAPIName].URI, + queryapi.WithHTTPClient(GetHTTPClient()), + ) require.NoError(t, err) return Network{ diff --git a/internal/test/mockserver.go b/internal/test/mockserver.go index 99a98f7b..87c90d29 100644 --- a/internal/test/mockserver.go +++ b/internal/test/mockserver.go @@ -103,7 +103,7 @@ func CreateMockServer(t testing.TB) *MockServer { require.NoError(t, err) return &MockServer{ - Client: &http.Client{}, + Client: GetHTTPClient(), Internal: Address(fmt.Sprintf("http://%s:%d", mockServerAlias, port.Int())), External: Address(fmt.Sprintf("http://%s:%d", host, externalPort.Int())), Container: container, @@ -193,11 +193,12 @@ func WaitForMockEndpoint(t testing.TB, server *MockServer, request MockRequest) verifyURL := fmt.Sprintf("%s/mockserver/verify", server.External) - clientTimeout := 500 * time.Millisecond - client := &http.Client{} + // Use configured HTTP client with timeout to prevent indefinite hangs + client := GetHTTPClient() - timeout := time.After(60 * time.Second) - ticker := time.NewTicker(clientTimeout) + // Use configurable polling timeout (longer in CI environments) + timeout := time.After(GetPollingTimeout()) + ticker := time.NewTicker(GetPollInterval()) defer ticker.Stop() slog.Info("Attempting to process request", "body", jsonData) @@ -238,7 +239,8 @@ func retrieveMatchingRequest(t testing.TB, server *MockServer, request MockReque req, err := http.NewRequest("PUT", retrieveURL, nil) require.NoError(t, err) - client := &http.Client{} + // Use configured HTTP client with timeout to prevent indefinite hangs + client := GetHTTPClient() resp, err := client.Do(req) require.NoError(t, err) defer resp.Body.Close() diff --git a/internal/test/queryAPI/service.go b/internal/test/queryAPI/service.go index 323e1283..a78f60ad 100644 --- a/internal/test/queryAPI/service.go +++ b/internal/test/queryAPI/service.go @@ -88,8 +88,10 @@ func SetQueryForClient(t testing.TB, client queryapi.ClientWithResponsesInterfac func WaitForClientStatus(t testing.TB, ctx context.Context, service queryapi.ClientWithResponsesInterface, id string, status queryapi.ClientStatus) { t.Helper() - timeout := time.After(60 * time.Second) - ticker := time.NewTicker(500 * time.Millisecond) // Increased from 100ms to avoid rate limiting (2 req/s < 5 req/s limit) + // Use configurable polling timeout (longer in CI environments to handle slower infrastructure) + // and configurable poll interval to balance responsiveness with rate limiting concerns + timeout := time.After(test.GetPollingTimeout()) + ticker := time.NewTicker(test.GetPollInterval()) defer ticker.Stop() for { diff --git a/pkg/queryAPI/api.gen.go b/pkg/queryAPI/api.gen.go index d9b896ac..49e8610a 100644 --- a/pkg/queryAPI/api.gen.go +++ b/pkg/queryAPI/api.gen.go @@ -582,6 +582,9 @@ type DocumentEnriched struct { // Fields The fields and the value for the document Fields map[string]interface{} `json:"fields"` + // FileSizeBytes File size in bytes (null for legacy documents) + FileSizeBytes *int64 `json:"fileSizeBytes,omitempty"` + // Filename Original filename of the document Filename *string `json:"filename,omitempty"` @@ -612,6 +615,9 @@ type DocumentID = openapi_types.UUID // DocumentInFolder Document within a folder with enriched metadata (lighter format, excludes client_id and fields) type DocumentInFolder struct { + // FileSizeBytes File size in bytes (null for legacy documents) + FileSizeBytes *int64 `json:"fileSizeBytes,omitempty"` + // Filename Original filename of the document Filename *string `json:"filename,omitempty"` diff --git a/scripts/tests.yml b/scripts/tests.yml index 49b9326d..68a7e45a 100644 --- a/scripts/tests.yml +++ b/scripts/tests.yml @@ -43,11 +43,27 @@ tasks: cmds: - rm -rf mocks/* - go tool mockery --log-level="" + cleanup: + desc: "Remove stale test containers from previous runs" + cmds: + - | + echo "Cleaning up stale test containers..." + # Remove specific test containers if they exist (running or stopped) + for container in postgres_test_queryorchestration localstack_test_queryorchestration mockserver_test_queryorchestration; do + if docker ps -a --format '{{.Names}}' | grep -q "^${container}$"; then + echo "Removing container: $container" + docker rm -f "$container" 2>/dev/null || true + fi + done + # Also remove any queryorchestration API/runner containers + docker ps -a --format '{{.Names}}' | grep -E '_test_queryorchestration' | xargs -r docker rm -f 2>/dev/null || true + echo "Cleanup complete" functional: vars: INCREMENTAL: "{{.INCREMENTAL | default \"true\"}}" cmds: - mkdir -p {{.OUT_DIR}} + - task: cleanup - task: wait - | # Determine test mode first diff --git a/serviceAPIs/queryAPI.yaml b/serviceAPIs/queryAPI.yaml index 4f7205f4..13e44328 100644 --- a/serviceAPIs/queryAPI.yaml +++ b/serviceAPIs/queryAPI.yaml @@ -2492,6 +2492,12 @@ components: description: "All labels applied to this document" textRecord: $ref: "#/components/schemas/FieldExtractionResponse" + fileSizeBytes: + type: integer + format: int64 + minimum: 0 + maximum: 4294967295 + description: "File size in bytes (null for legacy documents)" DocumentInFolder: description: Document within a folder with enriched metadata (lighter format, excludes client_id and fields) @@ -2530,6 +2536,12 @@ components: description: "All labels applied to this document" textRecord: $ref: "#/components/schemas/FieldExtractionResponse" + fileSizeBytes: + type: integer + format: int64 + minimum: 0 + maximum: 4294967295 + description: "File size in bytes (null for legacy documents)" ClientCreate: description: The properties for creation. diff --git a/test/process_test.go b/test/process_test.go index d979ccc5..8378e725 100644 --- a/test/process_test.go +++ b/test/process_test.go @@ -1,7 +1,6 @@ package endtoend_test import ( - "sync" "testing" "queryorchestration/internal/serviceconfig" @@ -14,6 +13,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "golang.org/x/sync/errgroup" ) type Config struct { @@ -29,46 +29,54 @@ func TestProcess(t *testing.T) { net, clean := test.CreateFullNetwork(t, t.Context(), cfg) defer clean() - var wg sync.WaitGroup var clientId queryapi.ClientID var jsonId queryapi.QueryID var textractExpectation test.MockExpectation textractBody := "Hello World" - wg.Add(2) - go func(t testing.TB) { + + // Use errgroup to properly propagate errors and avoid indefinite hangs + // If any goroutine fails, g.Wait() returns the error immediately + g := new(errgroup.Group) + + g.Go(func() error { _, jsonId = queryapitest.CreateDependentQueries(t, net.Client) - wg.Done() - }(t) - go func(t testing.TB) { + return nil + }) + g.Go(func() error { clientId = queryapitest.CreateClientWithSync(t, net.Client) - wg.Done() - }(t) + return nil + }) - wg.Wait() + if err := g.Wait(); err != nil { + t.Fatalf("setup phase 1 failed: %v", err) + } - wg.Add(3) - go func(t testing.TB) { + g = new(errgroup.Group) + + g.Go(func() error { textractExpectation = test.CreateDetectDocumentTextExpectation( t, net.Dependencies.MockServer, textractBody, ) - wg.Done() - }(t) - go func(t testing.TB) { + return nil + }) + g.Go(func() error { queryapitest.SetQueryForClient(t, net.Client, clientId, jsonId) - wg.Done() - }(t) - go func(t testing.TB) { + return nil + }) + g.Go(func() error { queryapitest.CreateFile(t, net.Client, queryapitest.File{ Filename: "helloworld.pdf", Content: []byte(pdfHelloWorld), ClientID: clientId, }) - wg.Done() - }(t) + return nil + }) - wg.Wait() + if err := g.Wait(); err != nil { + t.Fatalf("setup phase 2 failed: %v", err) + } test.WaitForMockEndpoint(t, net.Dependencies.MockServer, textractExpectation.Request) queryapitest.WaitForClientStatus(t, t.Context(), net.Client, clientId, queryapi.INSYNC) @@ -112,46 +120,52 @@ func TestProcess(t *testing.T) { net, clean := test.CreateFullNetwork(t, t.Context(), cfg) defer clean() - var wg sync.WaitGroup var clientId queryapi.ClientID var jsonId queryapi.QueryID var textractExpectation test.MockExpectation textractBody := "Hello World" - wg.Add(2) - go func(t testing.TB) { + + g := new(errgroup.Group) + + g.Go(func() error { _, jsonId = queryapitest.CreateDependentQueries(t, net.Client) - wg.Done() - }(t) - go func(t testing.TB) { + return nil + }) + g.Go(func() error { clientId = queryapitest.CreateClientWithSync(t, net.Client) - wg.Done() - }(t) + return nil + }) - wg.Wait() + if err := g.Wait(); err != nil { + t.Fatalf("setup phase 1 failed: %v", err) + } - wg.Add(3) - go func(t testing.TB) { + g = new(errgroup.Group) + + g.Go(func() error { queryapitest.SetQueryForClient(t, net.Client, clientId, jsonId) - wg.Done() - }(t) - go func(t testing.TB) { + return nil + }) + g.Go(func() error { queryapitest.CreateFile(t, net.Client, queryapitest.File{ Filename: "helloworld.pdf", Content: []byte(pdfHelloWorld), ClientID: clientId, }) - wg.Done() - }(t) - go func(t testing.TB) { + return nil + }) + g.Go(func() error { textractExpectation = test.CreateDetectDocumentTextExpectation( t, net.Dependencies.MockServer, textractBody, ) - wg.Done() - }(t) + return nil + }) - wg.Wait() + if err := g.Wait(); err != nil { + t.Fatalf("setup phase 2 failed: %v", err) + } test.WaitForMockEndpoint(t, net.Dependencies.MockServer, textractExpectation.Request) queryapitest.WaitForClientStatus(t, t.Context(), net.Client, clientId, queryapi.INSYNC) @@ -194,13 +208,14 @@ func TestProcess(t *testing.T) { net, clean := test.CreateFullNetwork(t, t.Context(), cfg) defer clean() - var wg sync.WaitGroup var clientId queryapi.ClientID var jsonId queryapi.QueryID var textractExpectation test.MockExpectation textractBody := "Hello World" - wg.Add(2) - go func(t testing.TB) { + + g := new(errgroup.Group) + + g.Go(func() error { _, jsonId = queryapitest.CreateDependentQueries(t, net.Client) jcfg := `{"path":"keytwo"}` @@ -208,39 +223,43 @@ func TestProcess(t *testing.T) { Config: &jcfg, }) require.NoError(t, err) - - wg.Done() - }(t) - go func(t testing.TB) { + return nil + }) + g.Go(func() error { clientId = queryapitest.CreateClientWithSync(t, net.Client) - wg.Done() - }(t) + return nil + }) - wg.Wait() + if err := g.Wait(); err != nil { + t.Fatalf("setup phase 1 failed: %v", err) + } - wg.Add(3) - go func(t testing.TB) { + g = new(errgroup.Group) + + g.Go(func() error { textractExpectation = test.CreateDetectDocumentTextExpectation( t, net.Dependencies.MockServer, textractBody, ) - wg.Done() - }(t) - go func(t testing.TB) { + return nil + }) + g.Go(func() error { queryapitest.SetQueryForClient(t, net.Client, clientId, jsonId) - wg.Done() - }(t) - go func(t testing.TB) { + return nil + }) + g.Go(func() error { queryapitest.CreateFile(t, net.Client, queryapitest.File{ Filename: "helloworld.pdf", Content: []byte(pdfHelloWorld), ClientID: clientId, }) - wg.Done() - }(t) + return nil + }) - wg.Wait() + if err := g.Wait(); err != nil { + t.Fatalf("setup phase 2 failed: %v", err) + } test.WaitForMockEndpoint(t, net.Dependencies.MockServer, textractExpectation.Request) queryapitest.WaitForClientStatus(t, t.Context(), net.Client, clientId, queryapi.INSYNC) @@ -250,28 +269,33 @@ func TestProcess(t *testing.T) { assert.Len(t, *docs.JSON200, 1) doc := (*docs.JSON200)[0] - wg.Add(2) - go func(t testing.TB) { + g = new(errgroup.Group) + + // Capture doc.Id to avoid closure issues + docId := doc.Id + g.Go(func() error { testRes, err := net.Client.TestQueryWithResponse(t.Context(), jsonId, queryapi.QueryTestRequest{ QueryVersion: 1, - DocumentId: doc.Id, + DocumentId: docId, }) require.NoError(t, err) assert.Equal(t, "valueone", testRes.JSON200.Value) - wg.Done() - }(t) + return nil + }) - go func(t testing.TB) { + g.Go(func() error { testRes, err := net.Client.TestQueryWithResponse(t.Context(), jsonId, queryapi.QueryTestRequest{ QueryVersion: 2, - DocumentId: doc.Id, + DocumentId: docId, }) require.NoError(t, err) assert.Equal(t, "valuetwo", testRes.JSON200.Value) - wg.Done() - }(t) + return nil + }) - wg.Wait() + if err := g.Wait(); err != nil { + t.Fatalf("test phase failed: %v", err) + } }) }