diff --git a/README.md b/README.md index 60626fb1..08dafa0d 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,11 @@ This repository contains the DoczyAI code. Using the following project as a baseline: `https://github.com/golang-standards/project-layout`. +## Documentation + +- [Queue Pipeline Flow](./queue_flow.md) - Detailed documentation of the SQS queue-based processing pipeline +- [Deployment Environment Variables](./deployment_env_variables.md) - Configuration guide for environment variables across all services + ## Installation and Usage - Install devbox: `https://www.jetify.com/docs/devbox/installing_devbox` diff --git a/api/queryAPI/api.gen.go b/api/queryAPI/api.gen.go index f034b5b2..458d9cab 100644 --- a/api/queryAPI/api.gen.go +++ b/api/queryAPI/api.gen.go @@ -1092,97 +1092,97 @@ func RegisterHandlersWithBaseURL(router EchoRouter, si ServerInterface, baseURL // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/+xdfW/bOJP/KoTuAQ541u/O++Fwlybtbhbdtpukt889bc6gpbHNrUy6IpXULfLdD3yT", - "KImyZddO93AB+kcS8WU4M/zNkJyZfgtCNl8wClTw4OxbMAMcQaJ+vMYCXpM5EfKXCHiYkIUgjAZn6hOK", - "5TdE6IQlcyw/IEKR/gV9DNTXf3sgNGIP/y7IHNr6549B0ArgC54vYgjOgn6vZxv1e0Er4OEM5ljO6G1z", - "JNvM8ZfXQKdiFpwNB61ggYWARJL1Px967dO7n2xj/dvfglYglgs5EBcJodPg8fFR9krwHIRZ6wsswtnV", - "ZXWltzNAY/kRpYuY4QhdXXaCVkDktwUWs6AVUDyXg6tWIxIFrSCBzylJIArORJKCu6i/JTAJzoJ/6eZc", - "7+qvvGtpkNRdxASoqCMoVF/rSfkOIrKJJRWXLEznK+iIzPe9UOJMLml5+WXBklpKQH3dCx3ZxJKK31NI", - "lnVEXF0iNkFiBuizbLZ7UuzsSoET4AtGOSj9vaJyC+D4ZZKwRP4hZFQAVVsXLxYxCdUO7f7JJbXfmi5d", - "jvYbcI6noCctLtrOijgk95AgkO3lsutQxDeZadvNG6qZrug9jkl0DZ9T4OIJl6SmRYmeF41ZtNzRit4w", - "8YqlNHqytVwDZ2kSAqJMoImcekcruWXsN0yXRjb8KRekVB6lC0aRYAzNMV1aWfEdrK4VXINIlu3ziYCk", - "usvfpPMxJHKXcwgZjTgaw4QlgESyJHSK8BQT2nENXH/Qc3e3No8SBagYDrQtI/N0HpydHB30eq1gTqj+", - "vZdZLUIFTCGRDHlsBe8pTsWMJeQrRE/P+IcZUJQ6JOxEox4tiza0xSQq8Dro9U8PT3rRpA2To8P28dHx", - "UfskgtP26cHB8BCfDofDYxy0ciGkqQJj1584qjgLLU3QjcAi5X6iuPom1QIXCJTEUSnMD8EiYSFwLkds", - "KZcrBgGywQSTWP0QYhpCLH++q6PhvRr1EgQmsSIFx/HbSXD2oYFjofvepPM5TpbBY+ubJGkBiSDagmg6", - "RhMSgzRUnpW+JlzINWZNkJhhgXRHVFhgJpIPQcTCYWcRTYKW/LF/rH6+awVEwFxN4nD/oHd6VPDnOj7v", - "TQnsSnfv93q9fKfgJMFLZafNH9j4TwhF8HhXVmjNQ4gK7ises1SURIgIDeM0kpvbLDTnUFEskj31THPH", - "5NJLRti4cUGrJAnVVP+YMWlj8RZ5VOZQKxBM4HgUspR6iL6VHxHNwM5QpMgWM8JzyjNBDw5bK8Bt0D84", - "PjgZHh0cr0E41zP6kHGiSO5dRbwFOVis8pxaLIolINKEQqThrCxwjnAYwkJIWbOkqNgeSUmXv6ln3wp4", - "BiNrOxjEyTqN0iSuLur99WskGApnEH4yCzFzuMjY1RLrnp+fd63X3lWtuxuDZkLaCUwgARpCET0P+4Pq", - "WcsjUn1Kysh0lrdGtla/K1wwH9Zu512IULOyQZ/8LOVA/gh7NtwfUg9F2bxlfYo6mEkiwgLa8mhdkkKv", - "FdA0jvFYil6fLyoQGiaAN6PmAXNkejUkojKpsTFWAfkqBytr1MDKDHaBPfKwpnz/kfzSjEjKaPvd5Stl", - "E7h2sRGhCCfhjNxDwQ/cDYksIVNCcZzZ6Sp1b02TzFDZQ6mWI0Ton1fvfCQG2ZI7X8miKM6mltnIZ3Mh", - "Zx0RZ2iCE5ewgx3Jd5GwaQKcjxaQhOAzfO8y9UK2MTKNpWfcmKb+Old+S0OgzeAK1pYtd85g4mzpgmJK", - "UvdjuTXM53DpU1/HDJQX59cmD4rU7lyPyAvI57M2GrQvML1Z0rDK4Cu9l8wNnPQV4pg9qLNfKMg9IL6k", - "Ic+3xpixGDB1RlbT+88RuWlSnoeilDB1oCxarc1sj8WJ9e3fyJZlWSrJqTHq+bXmphK+mIsiNVaOOOfn", - "59Xz13qUsXO+YNFy5bz6gLg976qcqGfBGy8aO8QYfXdXn4Ao2c3B4QYcqDuT3iwgJBOiTmjF06kmpeMc", - "TK/ejG7++81F0ArevL1VP768dH65evOz9zzqEuAXw7ldt5nfdc0076qyaQaKhcWXZWSGqJfT+0XUZAMq", - "r2OOl2gsLafs4tGlENMRNzixnmILKtttyepyWAT/BQkn+qbHcz0PXHIFhSwCdK9bdoIi2h8duGh/OhgM", - "h8eD3vDo5PDg+PioV7BjfZ8du2BxDKFgntuy7BOaswjiKvs0ZI7u80Ws4odd67Yu+IRAHK1XLkv0K938", - "sRXEWAAXW5BpODcKpQmApOkIrmCdUQR8EVsNUdohrkE2PKkntWb6ClNaZWF6N2CBt36lVRSVLaEyubZz", - "VY8a7abC3HpXtQL1UNNAj/KXlyIvDaZnw6xfdb2ZYKlYpMIwQA7c2c40lBS4nsuZU1hgbaM7p5IkC/dN", - "is7yfVPW4QZ8fjdeqjOmlLUCW+VQoYt6kW8NHVujwF9nM1c07JKFGujWWrWSE7Aja/aD3NFWTvGdnyvq", - "HLDm7TrngYcj32dnavcdppHaePc4TiFDOEuS6yRm1/TLfrbLdbec2OVAn0ftr0N762P/cBicfei3Bq3h", - "nU97ZpjP1q3vF9mmkaRLz/YVybnWR82cMWyVDNdGINQ+BY2Hk5P28fDgpH2Kx/12Hw4Px8NJ/wSi4y2e", - "giw9tReRBaK4uZVcpWJPy301m4/RhRfAyqou898khKh3/qKWzm3XYMlSNE+5MC8ngDCyHx/Li5/XTWgo", - "QfqvY6n0ylDhlENR0Nl0U3n6ngOaMlZ41tnCiJY4Z6n08k1FhzhPcvWGzcSoiIRMp5AgG7+xI9TR3sMo", - "ZvoB2K+Z9qs0/A8zEs4UVw1hX8lC3RmiBxLH8tiThQzk3ObDs24Xd8+7uk930Bsc9gb9YVfun07I70vc", - "7h2cFNit+nd+kv/MCCpI69vJY7fz08ePcgSvU9PsYKiFUXMwdIFnxSGxUZxRDdpA+yA67LeP+8OwfXoC", - "0D44OoCT4Um/P+kPt0Cbwnr8pp1xTsZxRphcmEYZe75335oJHdkLsfzl+a524lutqau12qizctioIaOq", - "0griRxMS24C74oCv1Ad9WxGyBaAx5hAhRo03bFxjZfd4YydV+W566AYeKqFT4JKebcjMOqM8rrDKBaDR", - "qP7yIcZcIPlZQmw2YKfBY8tw4N8yiaidjnA0IckOJ6x6Fj5fwxVJDXu16x+yOJ3TKjQyGhHRwGd2JrrI", - "+ljnbLT9gVGroF+A+htKOUzSGAmmFEUrU0FnNz/TrXzML+Gcs8KWw6+M9LvVYrlwObwCcPTGnGQiy2Yq", - "oE8MnI/EDMv5p+rmO7G/hjHjEI0IFZDc4zhoBWwB1P09hokYVZslZDrz/d34GwqY9U8+aPvF+Fsr3Dbj", - "lq6yYzVyuopqXahzxAmdxoCUpay5lS497VPyOQVEIqCCTAgk2o2ggohlZ2Nz0uwu+zXh4nLF+5LDp/z6", - "IDtPNoLlsg+9HpolUb+nkBDfzjtHHFSIzWfdosrbz3VdbXCO07PRCtRtUJFuN/TBvy8tFT6u6wE9S4vZ", - "lIQopUTRCV8gTP0vQttfpTI6IdNGK77QTRsdRLILs++4PrXcGzkCXNX12rQ3c+sXU8XqBrTeyobeM5Ma", - "onK/WVlXrVwvMg6Xr8fl39NEe+T2EiALHV8FPx8/fuv8/ePHRy8I6UlXPTNmLgqyi1UhRKoLwjkJFdu7", - "uab8eBmq3rXCqXP0FQ+eKsBUEw9cONHua1+HjZyQVEIbAl0VmgXr0WZ3B/ZKe9NNW+K9O3t5yFqRaEbU", - "BfFJTtijM5okbF5gRJUB+q7MFwuYxqKQr5ENsLHhL61aT1m/PqPOq95sZc9KMol1qn69eftm9PIft9fn", - "F7dvr4NWcPH2ze3Lf9yOXr1//drr9Kh5t3/4dPXtR5ue7wcU37mk0qrWV8gQU+vM1eWGboPeXWscnkbP", - "ujUvulvH73AI04SI5Y0k10L+lBLBzlPh8ZnlX6Uzaq6UUhU2dT7HXxlFF7ojurp810FXEj21x3j96uLk", - "eHCIfv3jFo0lcC0SqT6hwTRLglpTzB60fplkAzXNBYug8sf3SRycBTMhFvys241Y+HXZkQ06KW8D5qLd", - "72BFl1lPJ2TzLpMtBt0slUElH7KFOanPMZFjnochcC6tY8oh+VeO9Ad9VpGQGvwMAr29urxAgn0Cc2Cd", - "EB3fUe5sP6lIrk/wXXTr6VxlVn+XY//5IKzExoATSF5Z9fj1j9ugHAovRaEGQ2wsMKEQITwRkGQyxAU5", - "bytOtQlURJQiKQdRuXydc0LohNmUFhwKVw7zeThNCaXA+X9inIAAyYs8qe43Es4wxOi38GfTLGgFKl46", - "0GyVrStpLefvrrLTTNELU1sVvU3CGXBhPDQOyb1cmlxNTEIw5slQkFLztyib+eHhoWN8Bzu/IEKphm/8", - "83dX8pRut37Q6/Q6fXW1uwCKFyQ4C4adXmcYKIM0U3pq4rrVOxHzOQ7aD+QIIwoPNiLngQh987tI2D2J", - "IEKRvsDuaM3WBF1FWf8LG+9v/Awb8bOT5KNCXJwn+UhjnqLP76+GLnV5VmU5R3LQ6++YZhOB5qFZf7fx", - "0oinCgomaRzvKpvwoNer65GtultKolTd+uu7FXLMZKfB6fpO5aTAx1Zw2IxGN3XVNUQqsSkDsw93j3et", - "gNuHN6OZBcWWoIKnXPpIJkZMb9ngTg5rcyC+kehRkjX1RSJcg0gI3Ksdw7VLFtptM14iIrhJMi5uk59B", - "OHukoHS9nSld/sxfr3HORn5Wsu9XMmndsSP/q8sVOlYsKlCTlJc3cZ7v7hSkhx4nSzvtXL+tEK4CY1wU", - "l8pfC9668xOAtzlarAFv6QjphuuhuucxZXrZ9kzyjKm7V3ctR9ekNsTTbuiGgW6ArFmIqAHXQmmLKsba", - "5i+WZvNE+wTcPLbVB7gZ6c+Yuw/MdTUj0wpXI/OIwn0C8A0IO31O0nrovalX1j2AsBtcuR6Ex6CeTLyB", - "tetw+WBVmLcc9RmWd78jboo7YvUmKCNz5IRBrgFmqQ2xuefKX/nGSzflyYfMhUfDp4Hm4jvlCn84W4Zc", - "2LNC7kYh1WWoKLwFl0oaWAXNZLQrlPbecujscOlY6Dd2FcZmAjBMbSIF2Ux1wDGag8ARFtjjMqsSG3kk", - "bj1ez9NYkAVORHfCknlbDaeCfEIWETrVYU86OtL0ujXXvI7W68oY8GURq4vNCY45tAIuluqaSA4c5OVR", - "nHDgLK4qhrpIY82CLO89uyMeEyqlWHjn6PeOh8cH/ZPBQc+buF2fbWw5GqZcsDlyEjuricWmDsiKvOLO", - "T2tfV9SSq48rVQQweeuKEWMpvq1OHVYVsvTpZwzZ1VnD7Fkl0JWIUWfSdAWLRieOuLYOi8II8/gW5QBW", - "b+FeZFVJSjhWCmHWTzBOMrZ6JJJ7UtcfseXh1ENSfpWtCicWKiFGMMFpLIKzQa950nnf98ZTn4mf0cY/", - "qdx/H2VsMuFQQ9puksjv9ugxlEv1eBDDW6znebvvzmVYWQXpB7kMTjUMpF6eCCV0irR1jwG9u3xVdnP4", - "koZmLYWCJKtciRem8MJ3+RO2asdKl0IX79japXDmqPcqxk45EcyLXGzua6wz9XbAJta+RowF6TXwAAb7", - "wJssmsVD9wu3xk594alnCNqZx1GzszEvKfHWDkn3my1/8qh3UQy+uJsLVfCPl+uPqfgbwlGYJglQES9X", - "YYweo4ox6+5sClqXVR4sXN78H1Q6vdLVnbIqrH/Zd00ljWrdspXmcY3/G9lyh55aHNoKZzfypVkrV/Br", - "dG0v/ppNr1sHn+aO81lzf9zFfQHK8tJO+3DtWmvbZkX77sqArfPUlPOza8fyihJB1Jutk1tpILx6dWpy", - "7HTC3Z6eB4rZfDXvA4ZQ4x/pMnBPGlKT5854KHxZyJ19fv3dy/41GpLncTo716Si1npAeYJugzcGdX4y", - "pqjRu6+pM/U0j77lylb1jwtmCc/vvzs3I7Z0WK4o+4m7kWqcOe8NA8OE85qq6qlkiYNGl3XWSK3ntOcQ", - "MT2HR2mzm+xnfd25vkYl3uaqsFvvx83RUcprqiisVN2LGYSftN6aQ6WuE6C0183ZL6psXn0A9qmyxQoa", - "9bZ/hjkaA9C8LPKzAu/oxKlqh7sVJDorLP+mKpv//zlKYWdMPyPWgWyaUK2qv9z+9hrNgaZogae6SJKT", - "iQCRSqrgXqT9hc3hnS7Yu0ZtBXwR3ZmYx0V9LVKlKDGdsicjuQ5NmaTRvfNUw5VuPA9ODo9Lmf5/r/mP", - "qUpzZ9NIKp4VfmeI7ZWh1XnZrejrxmxKaK3eFs98qi2axOxBmoEEIpJAqCJ3BXMzo0xDSYAnpkZNWFLg", - "ob6fLm8aPYE7ej5yUWVe15YHMv+PgKTfO4jn/7BJE7ImW9KmNnX+7n3Wf1bglQr82Cpl4H2waWetwOah", - "2ZSy8mubEp2Rp2Ob1+t4O8RxPMbhp1pl/wXTKDaq/laOMkC2D4rSxNbqMtpjLj7QNYRArPtcyN7T9WlV", - "Kq+7OTCNEHwJZ5hOgSNikJd9AsprdsuFpXzNy/z56tlrXr9lu5X/c9uqjfDhvP1P3P7aa59+/Nge+f9T", - "wsrrvHK98mx9xYCLm+tXkqlCIgqriyHgwhdlX0Nsvzc42JjYu02RyVFCk1+YX+FoXdkQqSyu1o+2PWT9", - "Rx1m/X9CnwKglDc60GjBSOHd3gsoLK0PPH3NphoOpCeHWKoP0JpxugSsjt/jXCkNjepsqZ4no8kLD5KQ", - "77CmcnzlKUg31FHlzIX4Pu2NLYHPOrun8BMtQoXzm9jEz7YyT+MIM1OXwC1hp6rGhQkRkBDsD5+25Y32", - "HDJtp/GcNM7LK3g+Xu8w8ulzJl+rayoFvHCsbpDArYtObJy//btxD/bxuuSWG9ome/uzQ9tf4KVJZ+Y/", - "J24/UeK2lX7NrsggeIusbb1XVidt5xtjT6BrSsXV6dnzVfweUgczwRfyBat4u9E1Zla8Z7NcbQewG6Rq", - "7x2nVydq69RqW8c0K0tv6g1l5Z+2yKLQ2v6cuv0EqdsbIWpXmHJz37EZvG7LS1WqUgGyLlCXUv1YWufB", - "FOsml4JkgIu9bw23/l7N/igV3NskTqa3D2Lro4pvNY08jZ9zHncVHSNZun57qfGSe//N47uERWmYVZNS", - "ZbBSTxGwkM279321vcw81bw7s0GkVx0rVJVOtYps4PlFYDFUonrFWDOMzSp2RypnGjcdzI2+N2NVHsKb", - "jpWfosxIBe43HUW/dDqjFJ84mw5TrIuWj+ZeIjzePf5vAAAA//8uQJ8AYocAAA==", + "H4sIAAAAAAAC/+xdfW/bOJP/KoTuAQ541u92Xg+HOzdpd73ott0kvX3uaXMGLdE2tzLpklRSt8h3P/BN", + "oiTKll073cMZ6B9JxJfhzPA3Q3Jm+i0I6WJJCSKCB5ffgjmCEWLqxxso0Gu8wEL+EiEeMrwUmJLgUn0C", + "sfwGMJlStoDyA8AE6F/Ax0B9/bdHTCL6+O8CL1BT//wxCBoB+gIXyxgFl0G307GNup2gEfBwjhZQzuht", + "cyrbLOCX14jMxDy47PcawRIKgZgk638+dJoX9z/Zxvq3vwWNQKyWciAuGCaz4OnpSfZicIGEWesLKML5", + "6Lq80rs5AhP5ESTLmMIIjK5bQSPA8tsSinnQCAhcyMFVqzGOgkbA0OcEMxQFl4IlyF3U3xiaBpfBv7Qz", + "rrf1V962NEjqrmKMiKgiKFRfq0n5DiLSiSUV1zRMFmvoiMz3g1DiTC5pefllSVklJUh9PQgd6cSSit8T", + "xFZVRIyuAZ0CMUfgs2y2f1Ls7EqBGeJLSjhS+jsicgvA+CVjlMk/hJQIRNTWhctljEO1Q9t/ckntt7pL", + "l6P9hjiHM6QnzS/azgo4Yg+IASTby2VXoYhvMtO2nTVUM43IA4xxdIM+J4iLZ1ySmhYwPS+Y0Gi1pxW9", + "oeIVTUj0bGu5QZwmLESAUAGmcuo9reSO0t8gWRnZ8OdckFJ5kCwpAYJSsIBkZWXF97C6RnCDBFs1h1OB", + "WHmXv0kWE8TkLucopCTiYIKmlCEg2AqTGYAziEnLNXDdXsfd3do8ShQgot/TtgwvkkVweX466HQawQIT", + "/XsntVqYCDRDTDLkqRG8JzARc8rwVxQ9P+Mf54iAxCFhLxr1ZFm0pS3GUY7XQad7cXLeiaZNND09aZ6d", + "np02zyN00bwYDPon8KLf75/BoJEJIUkUGLv+xGnJWWhogm4FFAn3E8XVN6kWMEegJI5IYX4IloyGiHM5", + "YkO5XDESSDaYQhyrH0JIQhTLn++raHivRr1GAuJYkQLj+O00uPxQw7HQfW+TxQKyVfDU+CZJWiImsLYg", + "mo7xFMdIGirPSl9jLuQa0yZAzKEAuiPILTAVyYcgomG/tYymQUP+2D1TP983AizQQk3icH/QuTjN+XMt", + "n/emBDbS3budTifbKZAxuFJ22vyBTv5EoQie7osKrXmIopz7Cic0EQURAkzCOInk5jYLzTiUF4tkTzXT", + "3DG59JIBNG5c0ChIQjXVP6ZM2lq8eR4VOdQIBBUwHoc0IR6i7+RHQFKwMxQpssUc84zyVNC9k8YacOt1", + "B2eD8/7p4GwDwrme0YeUE3ly70vizcnBYpXn1GJRjCGRMIIiDWdFgXMAwxAthZQ1ZXnF9khKuvx1PftG", + "wFMY2djBIE7aaZywuLyo9zevgaAgnKPwk1mImcNFxraWWHs4HLat195WrdtbgybDTYamiCESojx6nnR7", + "5bOWR6T6lJSS6Sxvg2ytfpe4YD5s3M77EKFmZY0+2VnKgfwx9Gy4P6QeiqJ5S/vkdTCVRAQFasqjdUEK", + "nUZAkjiGEyl6fb4oQWjIENyOmkfIgelVk4jSpMbGWAXk6xystFENK9PbB/bIw5ry/cfySz0iCSXNd9ev", + "lE3g2sUGmADIwjl+QDk/cD8kUoZnmMA4tdNl6t6aJqmhsodSLUcUgX+O3vlIDNIlt77iZV6cdS2zkc/2", + "Qk47Ak7BFDKXsMGe5LtkdMYQ5+MlYiHyGb53qXoB2xiYxtIzrk1Td5Mrv6Mh0GZwDWuLljtjMHa2dE4x", + "JamHsdwa5jO49KmvYwaKi/NrkwdFKneuR+Q55PNZGw3aV5DcrkhYZvBI7yVzAyd9hTimj+rsFwr8gABf", + "kZBnW2NCaYwgcUZW0/vPEZlpUp6HohRTdaDMW63tbI/Fic3t38iWRVkqyakxqvm14aYSfTEXRWqsDHGG", + "w2H5/LUZZeycL2i0WjuvPiDuzrsyJ6pZ8MaLxg4xRt/d1TMkCnazd7IFB6rOpLdLFOIpVie0/OlUk9Jy", + "DqajN+Pb/35zFTSCN2/v1I8vr51fRm9+9p5HXQL8YhjadZv5XddM864sm3qgmFt8UUZmiGo5vV9GdTag", + "8joWcAUm0nLKLh5dCiEZc4MTmym2oLLbliwvh0bovxDjWN/0eK7nEZdcASGNEHjQLVtBHu1PBy7aX/R6", + "/f5Zr9M/PT8ZnJ2ddnJ2rOuzY1c0jlEoqOe2LP0EFjRCcZl9GjLHD9ki1vHDrnVXF3yKURxtVi5L9Cvd", + "/KkRxFAgLnYg03BuHEoTgFjdEVzBOqMI9EXsNERhh7gG2fCkmtSK6UtMaRSF6d2AOd76lVZRVLSEyuTa", + "zmU9qrWbcnPrXdUI1ENNDT3KXl7yvDSYng6zedXVZoImYpkIwwA5cGs301BQ4Goup05hjrW17pwKkszd", + "Nyk6i/dNaYdb5PO74UqdMaWsFdgqhwpcVYt8Z+jYGQX+Opu5pGHXNNRAt9GqFZyAPVmzH+SONjKK7/1c", + "UeeADW/XGQ88HPk+O1O57yCJ1MZ7gHGCUoSzJLlOYnpNv+qmu1x3y4hd9fR51P7at7c+9g8nweWHbqPX", + "6N/7tGcO+XzT+n6RbWpJuvBsX5Kca33UzCnD1slwYwRC5VPQpD89b571B+fNCzjpNrvo5GTSn3bPUXS2", + "w1OQpafyIjJHFDe3kutU7Hm5r2bzMTr3Alha1XX2m4QQ9c6f19KF7RqsaAIWCRfm5QQBCOzHp+LiF1UT", + "GkqA/utEKr0yVDDhKC/odLqZPH0vEJhRmnvW2cGIFjhnqfTyTUWHOE9y1YbNxKgIhmczxICN39gT6mjv", + "YRxT/QDs10z7VRr+xzkO54qrhrCveKnuDMEjjmN57ElDBjJu8/5luw3bw7bu0+51eiedXrfflvunFfKH", + "Arc7g/Mcu1X/1k/ynxlBBWl9O39qt376+FGO4HVq6h0MtTAqDoYu8Kw5JNaKM6pAG9QcRCfd5lm3HzYv", + "zhFqDk4H6Lx/3u1Ou/0d0Ca3Hr9pp5zjSZwSJhemUcae7923ZkzG9kIse3m+r5z4Tmvqeq026qwcNmLI", + "KKu0gvjxFMc24C4/4Cv1Qd9WhHSJwARyFAFKjDdsXGNl93htJ1X5bnroGh4qJjPEJT27kJl2BllcYZkL", + "iETj6suHGHIB5GcJsemArRqPLf2ef8swUTkd5mCK2R4nLHsWPl/DFUkFe7XrH9I4WZAyNFISYVHDZ3Ym", + "ukr7WOdsvPuBUaugX4D6G0g4miYxEFQpilamnM5uf6Zb+5hfwDlnhQ2HXynp9+vFcuVyeA3g6I05TUWW", + "zpRDnxhxPhZzKOefqZtvZn8NY8pRNMZEIPYA46AR0CUi7u8xmopxuRnDs7nv78bfUMCsf/JB2y/G31rj", + "thm3dJ0dq5DTKKp0oYaAYzKLEVCWsuJWuvC0T/DnBAEcISLwFCOm3QgisFi1tjYn9e6yX2Murte8Lzl8", + "yq4P0vNkLVgu+tCboVkS9XuCGPbtvCHgSIXYfNYtyrz9XNXVBuc4PWutQN0G5el2Qx/8+9JS4eO6HtCz", + "tJjOcAgSghWd6AsKE/+L0O5XqZRM8azWiq9001oHkfTC7DuuTy33xo4A13W9Me3N3PrFVLG6Bq13sqH3", + "zKSGKN1vltZVKderlMPF63H594Rpj9xeAqSh4+vg5+PHb62/f/z45AUhPem6Z8bURQF2sSqESHUBMCOh", + "ZHu315QfL0PVu1I4VY6+4sFzBZhq4hEXTrT7xtdhIycgldCGQJeFZsF6vN3dgb3S3nbTFnjvzl4cslIk", + "mhFVQXySE/boDKaMLnKMKDNA35X5YgGTWOTyNdIBtjb8hVXrKavXZ9R53Zut7FlKJrFO1a+3b9+MX/7j", + "7mZ4dff2JmgEV2/f3L38x9341fvXr71Oj5p394dPV99+tOn5fkDxnUtKrSp9hRQxtc6Mrrd0G/Tu2uDw", + "1HrWrXjR3Tl+h6MwYVisbiW5FvJnBAs6TITHZ5Z/lc6ouVJKVNjUcAG/UgKudEcwun7XAiOJntpjvHl1", + "dX7WOwG//nEHJhK4lkyqT2gwzZKg1hTTR61fJtlATXNFI1T643sWB5fBXIglv2y3Ixp+XbVkg1bCmwhy", + "0ey2oKLLrKcV0kWbyha9dprKoJIP6dKc1BcQyzGHYYg4l9Yx4Yj9Kwf6gz6rSEgNfkYCvB1dXwFBPyFz", + "YJ1iHd9R7Gw/qUiuT+i76NbTucqs/i7H/vNRWIlNEGSIvbLq8esfd0ExFF6KQg0G6ERATFAE4FQglsoQ", + "5uS8qzjVJlARUYqkDETl8nXOCSZTalNaYChcOSwxgzP4ABlcJv8JIUMCSWZkWXVD9TcwIqGcLMkx1mlf", + "ymwZvhulB5q8I6Z2K3jLwjniwjhpHLEHuTo5R4xDZCxUmQZsM+MSjjbQI7BQ2uKbb/huJA/uFg2CTqvT", + "6qrb3iUicImDy6Df6rT6gbJRc6W6JtRbPR1Rny+hXUMOICDo0QbpPGKhL4OXjD7gCEUg0nfaLa3smqBR", + "lPa/sikAxvWwQUB7yUfKhcp58pE0DCr6/C5s6FKXJVoW0yZ7ne6eaTZBaR6a9XcbQg14otBhmsTxvhIM", + "B51OVY901e1CXqXq1t3cLZd2Jjv1LjZ3KuYJPjWCk3o0utmsrm1SuU4pvn24f7pvBNy+xRnNzCm2xBk4", + "49JtMmFjegsH93JYmxbxDUdPkqyZLzjhBgmG0YPaMVx7aaHdNpMVwIKbvOP8NvkZCWeP5JSuszely17+", + "qzXO2chHJft+JZMGHzryH12v0bF8nYGKPL2sifOid68gPfT4XdqP5/q5BXMVK+OiuFT+SvDWnZ8BvM1p", + "YwN4S99IN9wM1R2PKdPLtseUI6buX921HF2TWhNP26EbGboFsqZRowZcc9Uuyhhrm79Ymc0THRJws3BX", + "H+CmpB8x9xCY62pGqhWuRmZBhocE4Fsk7PQZSZuh97ZaWQ8Awm685WYQniD1iuKNtd2Ey4N1kd9y1CMs", + "739H3OZ3xPpNUETmyImM3ADMUhtic/WVPfxNVm4WlA+Zc++IzwPN+afLNf5wugy5sKNC7kch1f2oyD0P", + "F6ocWAVNZbQvlPbecuiEcelY6Gd3FdlmYjJMuSIF2VR1gDFYIAEjKKDHZVZVN7Lg3Gq8XiSxwEvIRHtK", + "2aKphlNxPyGNMJnpSCgdMGl63ZmbX0frdbEM9GUZq7vOKYw5agRcrNQ1kRw4yCqmOBHCaahVjKqCjzUL", + "0lT49Np4gomUYu7po9s5658Nuue9Qceby12dgGw5GiZc0AVwcj3LucamNMiaVOPWTxsfXNSSy+8tZQQw", + "qeyKERMpvp1OHVYV0ozqI4bs66xh9qwS6FrEqDJpuqhFrRNHXFmaRWGEeY+LMgCrtnAv0kIlBRwrRDXr", + "VxknP1u9G8k9qUuS2Ipx6m0pu9xWtRRzxREjNIVJLILLXqd+HnrX9+xTnZyf0sY/qXIAPsrodMpRBWn7", + "ySu/P6DHUKze40EMb/2e43bfn8uwtjDSD3IZnAIZQD1GYYLJDGjrHiPw7vpV0c3hKxKateRqlKxzJV6Y", + "Wgzf5U/YQh5rXQpdz2Nnl8KZo9qrmDgVRiDPc7G+r7HJ1NsB61j7CjHmpFfDA+gdAm/SABcP3S/csjvV", + "taiOELQ3j6NiZ0NeUOKdHZL2N1sR5Unvohj5QnGuVA1AXixJpkJyMAdhwhgiIl6twxg9RhljNt3Z5LQu", + "LUaYu7z5P6h0eqXrO6WFWf+y75pKGuVSZmvN4wb/N7IVED3lObQVTm/kC7OWruA36NpB/DWbcbcJPs0d", + "51Fzf9zFfQ7KsmpPh3DtGhvbpnX87ouArVPXlPOzb8dyRLDA6s3WSbc0EF6+OjVpdzoH70DPA/kEv4r3", + "AUOo8Y90ZbhnDanJ0mk8FL7MpdMeX38Psn+NhmSpnc7ONdmplR5QlrNb441BnZ+MKar17mtKTz3Po2+x", + "2FX144JZwvH9d+9mxFYTyxTlMHE3Uo1T571mYJhwXlNViZU0l9Dosk4kqfScDhwipufwKG16k33U173r", + "a1TgbaYK+/V+3LQdpbymsMJa1b2ao/CT1ltzqNSlA5T2umn8eZXNChKgQ6psvqhGte2fQw4mCJGsUvJR", + "gfd04lTlxN2iEq01ln9blc3+Sx2lsHOqnxGrQDZhRKvqL3e/vQYLRBKwhDNdN8lJTkCRyrPgXqT9hS7Q", + "O13Dd4PaCvRFtOdiEef1NU+VosR0Sp+M5Do0ZZJG985TDVe48Rycn5wVkv//XvF/VRXmTqeRVBwVfm+I", + "7ZWh1XnZLe/rxnSGSaXe5s98qi2YxvRRmgGGIsxQqCJ3BXWTpUxDSYAnpkZNWFDgvr6fLm4aPYE7ejZy", + "XmVeV1YMMv+1gKTfO4jnv7VJGN6QQGmTYFp/9z7rHxV4rQI/NQpJeR9sJlojsKlpNsus+NqmRGfk6djm", + "zTreDGEcT2D4qVLZf4Ekio2qv5Wj9IDtA6KE2fJdRnvMxQe4QSHC1n3OJfTpkrUqu9fdHJBEAH0J55DM", + "EAfYIC/9hAiv2C1XlvINL/PD9bNXvH7Ldmv/M7d1G+HDsPlP2PzaaV58/Ngc+/+fwtLrvHK9sgR+xYCr", + "25tXkqlCIgqtiiHgwhdlX0Fst9MbbE3s/bbI5CihSTnMrnC0rmyJVBZXq0fbHbL+owqz/j+hTw5Qihsd", + "kWhJce7d3gsoNKkOPH1NZxoOpCcHaKIP0Jpxuiqsjt/jXCkNiapsqZ4npckLD5KQ77CmcnzlKUg31FHl", + "1IX4Pu2NLYFHnT1Q+IkWocL5bWziZ1usp3aEmSlV4Fa1U4XkQoYFYhj6w6dtxaMDh0zbaTwnjWFxBcfj", + "9R4jnz6n8rW6plLAc8fqGgncug7F1vnbvxv34BCvS24Fol2ytz87tP0FXpp0Zv4xcfuZEret9Ct2RQrB", + "O2Rt672yPmk72xgHAl1TPa5Kz45X8QdIHUwFn8sXLOPtVteYaT2f7XK1HcCukap9cJxen6itU6ttadO0", + "Ur0pQZRWhNohi0Jr+zF1+xlSt7dC1LYwFei+YzN43ZaXqnqlAmRdsy4h+rG0yoPJl1IuBMkgLg6+NdyS", + "fBX7o1CDb5s4mc4hiK2OKr7TNPIkPuY87is6RrJ08/ZS47EH/83jO0ajJEyrS6nKWImnLlhIF+2Hrtpe", + "Zp5y3p3ZINKrjhWqSqdaRTbw7CIwHypRvmKsGMZmFbsjFTON6w7mRt+bsUoP4XXHyk5RZqQc9+uOol86", + "nVHyT5x1h8mXSnOqkzmXCE/3T/8bAAD//69cLjR1hwAA", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/deployment_env_variables.md b/deployment_env_variables.md new file mode 100644 index 00000000..94d3846a --- /dev/null +++ b/deployment_env_variables.md @@ -0,0 +1,256 @@ +# Deployment Environment Variables Configuration Guide + +## Overview + +This guide provides rules for configuring environment variables when deploying all runners and the queryAPI service across different environments (dev, stage, prod). + +## Variable Categories + +### 1. Service-Specific Variables (UNIQUE per service) + +These variables **MUST be different** for each runner service: + +| Variable | Description | Unique To | +|----------|-------------|-----------| +| `QUEUE_URL` | Input queue the service reads from | Each runner (different value per runner) | + +### 2. Pipeline Queue URLs (SHARED across all services in same environment) + +These variables should be **IDENTICAL** for all services within the same environment: + +| Variable | Example Value (dev) | Purpose | +|----------|---------------------|---------| +| `DOCUMENT_INIT_URL` | `https://sqs.us-east-2.amazonaws.com/476114113988/document_init` | Queue for document initialization | +| `DOCUMENT_SYNC_URL` | `https://sqs.us-east-2.amazonaws.com/476114113988/document_sync` | Queue for document sync | +| `DOCUMENT_CLEAN_URL` | `https://sqs.us-east-2.amazonaws.com/476114113988/document_clean` | Queue for document cleaning | +| `DOCUMENT_TEXT_URL` | `https://sqs.us-east-2.amazonaws.com/476114113988/document_text` | Queue for text extraction | +| `QUERY_SYNC_URL` | `https://sqs.us-east-2.amazonaws.com/476114113988/query_sync` | Queue for query sync | +| `QUERY_URL` | `https://sqs.us-east-2.amazonaws.com/476114113988/query` | Queue for query execution | +| `CLIENT_SYNC_URL` | `https://sqs.us-east-2.amazonaws.com/476114113988/client_sync` | Queue for client sync | +| `QUERY_VERSION_SYNC_URL` | `https://sqs.us-east-2.amazonaws.com/476114113988/query_version_sync` | Queue for query version sync | + +### 3. AWS Configuration (SHARED across all services) + +These should be **IDENTICAL** for all services: + +| Variable | Example Value | Notes | +|----------|---------------|-------------------------------------------------------| +| `AWS_ACCESS_KEY_ID` | `AKIAXXXXXXXXXXXXXXXX` | AWS credentials override - should be blank | +| `AWS_SECRET_ACCESS_KEY` | `secret_key_here` | AWS credentials - override - should be blank | +| `AWS_SESSION_TOKEN` | `token_if_using_temporary_creds` | Optional, for temporary credentials - should be blank | +| `AWS_REGION` | `us-east-2` | AWS region for all services | + +⚠️ **Important for AWS deployment**: +- **DO NOT SET** `AWS_ENDPOINT_URL` (only for LocalStack) +- **DO NOT SET** `AWS_S3_USE_PATH_STYLE` (only for LocalStack) + +### 4. Database Configuration (SHARED across all services) + +These should be **IDENTICAL** for all services: + +| Variable | Example Value | Notes | +|----------|---------------|------------------------------------------------------| +| `PGHOST` | `database-hostname.rds.amazonaws.com` | Database hostname | +| `PGPORT` | `5432` | Database port | +| `PGDATABASE` | `queryorchestration_dev` | Database name | +| `PGUSER` | `dbuser` | Database username (whatever we use) | +| `PGPASSWORD` | `db_password_here` | Database password (keep secure) | +| `DB_NOSSL` | `false` | Set to `false` for AWS RDS (should be blank for now) | + +### 5. Authentication Configuration (SHARED across all services) + +These should be **IDENTICAL** for all services: + +| Variable | Example Value | Notes | +|----------|---------------|----------------------------------------------------| +| `DISABLE_AUTH` | `false` | Set to `false` for production/staging true for dev | +| `COGNITO_USER_POOL_ID` | `us-east-2_XXXXXXXXX` | AWS Cognito User Pool ID | +| `COGNITO_CLIENT_ID` | `client_id_here` | AWS Cognito Client ID | +| `COGNITO_CLIENT_SECRET` | `client_secret_here` | AWS Cognito Client Secret | +| `COGNITO_DOMAIN` | `your-domain.auth.us-east-2.amazoncognito.com` | AWS Cognito Domain | + +### 6. Application Configuration (SHARED across all services) + +| Variable | Example Value | Notes | +|----------|---------------|-------| +| `LOG_LEVEL` | `INFO` or `DEBUG` | Use `INFO` for prod, `DEBUG` for dev | +| `BUCKET` | `s3-bucket-name-dev` | S3 bucket name (only needed by some services) | + +### 7. Service-Specific Optional Variables + +| Service | Variable | Purpose | +|---------|----------|---------| +| `docCleanRunner` | `BUCKET` | S3 bucket for document processing | +| `docTextRunner` | `BUCKET` | S3 bucket for document processing | +| `queryAPI` | `BUCKET` | S3 bucket for document uploads | +| `queryAPI` | `DEBUG` | Set to `true` for debugging | + +## Per-Service Configuration Matrix + +### storeEventRunner +**Unique Variables:** +- `QUEUE_URL` = `https://sqs.us-east-2.amazonaws.com/476114113988/store_event` + +**Also Needs:** +- `DOCUMENT_INIT_URL` +- AWS config (shared) +- Database config (shared) +- Auth config (shared) + +### docInitRunner +**Unique Variables:** +- `QUEUE_URL` = `https://sqs.us-east-2.amazonaws.com/476114113988/document_init` + +**Also Needs:** +- `DOCUMENT_SYNC_URL` +- AWS config (shared) +- Database config (shared) +- Auth config (shared) + +### docSyncRunner +**Unique Variables:** +- `QUEUE_URL` = `https://sqs.us-east-2.amazonaws.com/476114113988/document_sync` + +**Also Needs:** +- `DOCUMENT_CLEAN_URL` +- AWS config (shared) +- Database config (shared) +- Auth config (shared) + +### docCleanRunner +**Unique Variables:** +- `QUEUE_URL` = `https://sqs.us-east-2.amazonaws.com/476114113988/document_clean` + +**Also Needs:** +- `DOCUMENT_TEXT_URL` +- `BUCKET` (S3 bucket name) +- AWS config (shared) +- Database config (shared) +- Auth config (shared) + +### docTextRunner +**Unique Variables:** +- `QUEUE_URL` = `https://sqs.us-east-2.amazonaws.com/476114113988/document_text` + +**Also Needs:** +- `QUERY_SYNC_URL` +- `BUCKET` (S3 bucket name) +- AWS config (shared) +- Database config (shared) +- Auth config (shared) + +### querySyncRunner +**Unique Variables:** +- `QUEUE_URL` = `https://sqs.us-east-2.amazonaws.com/476114113988/query_sync` + +**Also Needs:** +- `QUERY_URL` +- AWS config (shared) +- Database config (shared) +- Auth config (shared) + +### queryRunner +**Unique Variables:** +- `QUEUE_URL` = `https://sqs.us-east-2.amazonaws.com/476114113988/query` + +**Also Needs:** +- `QUERY_URL` (same as QUEUE_URL - self-referential) +- AWS config (shared) +- Database config (shared) +- Auth config (shared) + +### clientSyncRunner +**Unique Variables:** +- `QUEUE_URL` = `https://sqs.us-east-2.amazonaws.com/476114113988/client_sync` + +**Also Needs:** +- `DOCUMENT_SYNC_URL` +- AWS config (shared) +- Database config (shared) +- Auth config (shared) + +### queryVersionSyncRunner +**Unique Variables:** +- `QUEUE_URL` = `https://sqs.us-east-2.amazonaws.com/476114113988/query_version_sync` + +**Also Needs:** +- `CLIENT_SYNC_URL` +- AWS config (shared) +- Database config (shared) +- Auth config (shared) + +### queryAPI (REST API Service) +**Does NOT need `QUEUE_URL`** + +**Needs:** +- `CLIENT_SYNC_URL` +- `QUERY_VERSION_SYNC_URL` +- `BUCKET` (S3 bucket name) +- `DEBUG` (optional, set to `true` for development) +- AWS config (shared) +- Database config (shared) +- Auth config (shared) + +## Deployment Strategy + +### Option 1: Shared Base Configuration + Service Overrides + +1. **Create a base environment configuration** with all shared variables (database, AWS, auth, all queue URLs) +2. **For each service**, add only the unique `QUEUE_URL` value + +**Example:** +```bash +# base-dev.env (shared by all services) +AWS_REGION=us-east-2 +AWS_ACCESS_KEY_ID=AKIAXXXXXXXX +AWS_SECRET_ACCESS_KEY=secret_here +PGHOST=db-dev.example.com +PGDATABASE=queryorchestration_dev +# ... all queue URLs ... +DOCUMENT_INIT_URL=https://sqs.us-east-2.amazonaws.com/476114113988/document_init +DOCUMENT_SYNC_URL=https://sqs.us-east-2.amazonaws.com/476114113988/document_sync +# ... etc + +# storeEventRunner-override.env +QUEUE_URL=https://sqs.us-east-2.amazonaws.com/476114113988/store_event + +# docInitRunner-override.env +QUEUE_URL=https://sqs.us-east-2.amazonaws.com/476114113988/document_init +``` + +### Option 2: Complete Configuration Per Service + +Copy the full environment configuration to each service and only change `QUEUE_URL`. + +## Quick Checklist for New Environment + +When deploying to a new environment (e.g., moving from dev to stage): + +- [ ] Update AWS account ID in all queue URLs (e.g., `476114113988` → `123456789012`) +- [ ] Update environment suffix in queue names if using suffixes (e.g., `store_event-dev` → `store_event-stage`) +- [ ] Update database connection (PGHOST, PGDATABASE, PGUSER, PGPASSWORD) +- [ ] Update AWS credentials (AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY) +- [ ] Update Cognito configuration (if different per environment) +- [ ] Update S3 bucket name (BUCKET) +- [ ] Set LOG_LEVEL appropriately (DEBUG for dev, INFO for prod) +- [ ] Set DISABLE_AUTH appropriately (true for dev, false for prod) +- [ ] Ensure `QUEUE_URL` is unique for each runner service +- [ ] Remove `AWS_ENDPOINT_URL` and `AWS_S3_USE_PATH_STYLE` (LocalStack only) + +## Common Mistakes to Avoid + +❌ **Setting a global `QUEUE_URL`** - Each runner needs its own specific input queue +❌ **Using different queue URLs across services** - All services in same environment should see same queue URLs (except their own `QUEUE_URL`) +❌ **Forgetting to remove LocalStack variables** - `AWS_ENDPOINT_URL` and `AWS_S3_USE_PATH_STYLE` should not be set for AWS +❌ **Using empty string instead of unsetting** - Variables like `AWS_S3_USE_PATH_STYLE` should be completely absent, not empty +❌ **Mixing environment credentials** - Don't use dev database with prod queues + +## Validation + +Before deploying, verify: +1. Each runner has a unique `QUEUE_URL` matching its input queue +2. All queue URLs within an environment are consistent across all services +3. Database credentials are correct for the environment +4. AWS credentials have appropriate permissions for SQS, S3, and Textract +5. Cognito configuration matches the environment +6. LocalStack-specific variables are removed for AWS deployments \ No newline at end of file diff --git a/queue_flow.md b/queue_flow.md new file mode 100644 index 00000000..979f84d6 --- /dev/null +++ b/queue_flow.md @@ -0,0 +1,299 @@ +# Queue Pipeline Flow Documentation + +## Overview + +This document describes the SQS queue-based processing pipeline that orchestrates document processing through various stages. The system uses AWS SQS queues to connect microservices (runners) in a sequential processing pipeline. + +## Visual Pipeline Flow +[mermaid link](https://mermaid.live/edit#pako:eNqdlWtv2zYUhv8KwaDABiiZJOqOoUDjdFg_dGvktAVmFwVDHV0QmdQoqY1n57-PF8nxXBcYKoCASL08z7mR2mEmCsAZLlvxldVUDujues2Rel68QEuCXn8BPqClGCUDu74kq5_Uh-uRPcDw67385eWNYONGy953raDFz5_Q5eXL_WHzH2JoyobRoRF8j5a33pofCG9pw9G7poO24YBuRxihR5QXKB85B9lPyFtv1Q9CwmfQFg3UaA0J5d7OfDU4u9FIssRNvCdrIveMUwsJdFCIQjAkgQlZaI_8VTGF8LnhzbH92VWlsSh_p6RvlOiE488c33JqYA-oGLtWBw690d1v0es7WqGa9rXGkmdsv-XsLJZYLNHYpRKdYMmMNbr9B9o2heIh1jZzohjlxjwqW1ppbPCMZS1QfpYbWG6guQutOgEHMziw8WoJ6iYBE20LTJUE9YMqJpVFr8HhM3iAx_NpDi031Nw7JTrBhjPW6PavPi6RVknKrLk_Fzma5lO3Rau_R5Db72c4ssRoZ3RnchzPzMj2NauhGFtAWt9MpS1VrHNwmhpb6llgbIGxBZ7AkhlmVPvfVfomlDVWQAe8AM4U2YCODtPCFB3pENBv6jzbD6_evfFWt9qAfj2A3E9T4cyesdNtow0mK9s73ybMep9Y75OdlZ3JVzqHYKT7azqwGnVSMOhtsmjbTg2qc2bCIEdhWF8_qNOvSnguGmPWqp79Tqc6f7H7vud-at1PbfInyLdBpO4chNmwv5NNVYFEtCxVY0Nh29zEYPxPjvxfDlt1m1V2zlra9zdQ6hKOgMqmbbOLsixi13X6QYoHyC4IIdP75demGOrM7x4ddYaEzC5c1z0xJI2jk6UkZgD3P2hJX5q0mp1Kk_I-TX7QFO2aQ2zUjen_N3NkSN_zjrpp1SBqBGqEakRqxGokji6fyePxptxzct_JiZMHTh46eeTksZMnjqqcTdV_CGSO-njVNJUKwa5hB29AbmhTqP_iTq-t8VDDBtY4U6_qOntY4zV_Ujo6DkI3D84GOYKDpRirGmclbXs1s81509BK0s0s6Sj_S4jDFIpG-fPW_oTNv9jBldToyaI-7XIhRj7gzPdiz1jA2Q4_4ozEwVVCUt91Yz8KQ8938BZnl0mcXBEv8RMSxF7ok-DJwf8YpneVem4QpUkUpcSPI-_pXxutlHc) +```mermaid +flowchart TB + %% S3 Event Source + S3[(S3 Bucket
Document Upload)] -->|S3 Event Notification| SQ1 + + %% Main Pipeline Queues and Runners + SQ1[store_event
Queue] --> R1{storeEventRunner
:8081} + R1 -->|Creates doc record| SQ2[document_init
Queue] + + SQ2 --> R2{docInitRunner
:8082} + R2 -->|Check duplicates
by ETag hash| SQ3[document_sync
Queue] + + SQ3 --> R3{docSyncRunner
:8083} + R3 -->|Validate client
can_sync flag| SQ4[document_clean
Queue] + + SQ4 --> R4{docCleanRunner
:8084} + R4 -->|Clean per
collector standards| SQ5[document_text
Queue] + + SQ5 --> R5{docTextRunner
:8085} + R5 -->|AWS Textract
OCR extraction| SQ6[query_sync
Queue] + + SQ6 --> R6{querySyncRunner
:8087} + R6 -->|Schedule queries
for document| SQ7[query
Queue] + + SQ7 --> R7{queryRunner
:8088} + R7 -->|Handle query
dependencies| SQ7 + + %% Client Sync Flow + API1[Query API
:8080] -->|Client update| SQ8[client_sync
Queue] + SQ8 --> R8{clientSyncRunner
:8089} + R8 -->|Batch process
all client docs| SQ3 + + %% Query Version Sync Flow + API1 -->|Query update| SQ9[query_version_sync
Queue] + SQ9 --> R9{queryVersionSyncRunner
:8090} + R9 -->|Trigger affected
clients| SQ8 + + %% Styling + classDef queue fill:#ffd700,stroke:#333,stroke-width:2px,color:#000 + classDef runner fill:#87ceeb,stroke:#333,stroke-width:2px,color:#000 + classDef storage fill:#98fb98,stroke:#333,stroke-width:2px,color:#000 + classDef api fill:#ffa07a,stroke:#333,stroke-width:2px,color:#000 + + class SQ1,SQ2,SQ3,SQ4,SQ5,SQ6,SQ7,SQ8,SQ9 queue + class R1,R2,R3,R4,R5,R6,R7,R8,R9 runner + class S3 storage + class API1 api +``` + +### Flow Legend +- 🟨 **Yellow boxes**: SQS Queues +- 🟦 **Blue diamonds**: Runner Services (with port numbers) +- 🟩 **Green cylinder**: S3 Storage +- 🟧 **Orange box**: REST API endpoint +- **Solid arrows**: Message flow direction +- **Self-loop on queryRunner**: Handles query dependencies by re-queueing + +## Detailed Process Flow +[mermaid link](https://mermaid.live/edit#pako:eNp9lW9v2jAQxr-K5VfblFaEtCvkRaU1MAlp0xoSadKEVBnnAI_EoY4zlSK--84OgQYSkJAc--fnnjv_21GeJ0B9OpMFvJYgOYwEWyqWzSTB34YpLbjYMKlJ5LX0jQkrSKFzBS_wD7AnLKGENnB6JMcGnJZSgroERxPDJTkvM6RehBSdmqPJ9MBOkOoUjBqCxVbyTsGoFoyQ6hQMGoI8BSY7FYNaMTBYp2TckNTw1p10XCvGSHUJhjZpXFC1vZpxWGVswWs5h0eqU-kk1Jnlk0Ge80IvFUThj5msmMi7eXyMxj42iN0b5NOv-V_gOlDANCT-l88HcGzBqY8iaUoyKAq2hHpsioOjJ58EK-BrIhZEZJtcabSgVw1mgpFAJqaGRMhFTnbzkq9BO2QNW4esWLHa195GEzujNSR2N0Im5SYVHB2T-daqXGA2n-M6EwU8V0mDig7edjU0GZ28RJZo9xI1vfBUmAicSbv-DShoiWHqBalYinkKx3iBpdvjBRflZimml2yJPRDn1DOoRa6yalDIZWM8vpZ0bIl2E_GFCXt08K8Yx51zwmqRgKHIt98RiQ_QuVJkbqiTghX8yITXViiMLNFqNjyu0HeB09nGbhYstz00AoqPWHgZxKkOl1krrCQBxlfHsGZGe9CzHQobVMVb_hSuBsZvwEtdmdmejVU1UVCUqT7Nq2xO4ebV3Aim-B_liWniQlOHZqAyJhJ8ZXZm8ozqFWQwoz42E6bWM3x99sixUufmEqK-ViU4VOXlckX9BUsL_Co3CR6ew-t07MWL5U-eZ_UUSAR6_Vm9afZpc-hSmdgHSeNPBXkpNfX77p0VoP6OvlHf7bm3D3f9njcY9j13eN936Nb09m693sD1BgPPux_2eoO9Q99tyN7tYOC6_Ydhf-h-7WPjYf8f7gRf3A) +```mermaid +sequenceDiagram + participant S3 + participant SE as store_event Queue + participant SER as storeEventRunner + participant DI as document_init Queue + participant DIR as docInitRunner + participant DS as document_sync Queue + participant DSR as docSyncRunner + participant DC as document_clean Queue + participant DCR as docCleanRunner + participant DT as document_text Queue + participant DTR as docTextRunner + participant QS as query_sync Queue + participant QSR as querySyncRunner + participant Q as query Queue + participant QR as queryRunner + participant DB as PostgreSQL + + S3->>SE: S3 Event (ObjectCreated:*) + SE->>SER: Poll message + SER->>DB: Check if import path + SER->>DI: Send doc info {bucket, key, hash} + + DI->>DIR: Poll message + DIR->>DB: Check duplicate by hash + DIR->>DB: Create document record + DIR->>DS: Send {documentID} + + DS->>DSR: Poll message + DSR->>DB: Check client can_sync + DSR->>DC: Send {documentID} if eligible + + DC->>DCR: Poll message + DCR->>DB: Check if already clean + DCR->>DB: Perform cleaning + DCR->>DT: Send {documentID} + + DT->>DTR: Poll message + DTR->>DB: Check if text extracted + DTR-->>DTR: Call AWS Textract + DTR->>DB: Store extracted text + DTR->>QS: Send {documentID} + + QS->>QSR: Poll message + QSR->>DB: Find applicable queries + QSR->>Q: Send {documentID, queryID} for each + + Q->>QR: Poll message + QR->>DB: Check dependencies + QR->>DB: Execute query + QR->>DB: Store results + QR-->>Q: Re-queue if dependencies pending +``` + +## S3 Event Trigger Configuration + +### AWS Service Type +**S3 Event Notifications** (also known as S3 Bucket Notifications) + +### Configuration Requirements + +The pipeline is initiated by S3 Event Notifications configured to send messages directly to an SQS queue: + +```json +{ + "QueueConfigurations": [{ + "QueueArn": "arn:aws:sqs:{region}:{account-id}:store_event", + "Events": ["s3:ObjectCreated:*"], + "Filter": { + "Key": { + "FilterRules": [ + {"Name": "prefix", "Value": "documents/"} + ] + } + } + }] +} +``` + +**Important**: Files under the `batches/` path should be excluded from triggering the pipeline as they are temporary files. This can be accomplished through: +1. Prefix filters in S3 configuration (specify allowed prefixes) +2. Application-level filtering in `storeEventRunner` (recommended) + +### AWS CLI Setup Command +```bash +aws s3api put-bucket-notification-configuration \ + --bucket {BUCKET_NAME} \ + --notification-configuration '{ + "QueueConfigurations": [{ + "QueueArn": "arn:aws:sqs:{region}:{account-id}:store_event", + "Events": ["s3:ObjectCreated:*"] + }] + }' +``` + +## Queue Pipeline Flow Tables + +### Main Document Processing Pipeline + +| Order | Runner Service | Reads From Queue | Writes To Queue | Description | +|-------|---------------|------------------|-----------------|-------------| +| **1** | `storeEventRunner` | `store_event` | `document_init` | Receives S3 event notifications when files are uploaded to the bucket | +| **2** | `docInitRunner` | `document_init` | `document_sync` | Creates document records, checks for duplicates based on ETag hash | +| **3** | `docSyncRunner` | `document_sync` | `document_clean` | Validates client eligibility for document processing (can_sync flag) | +| **4** | `docCleanRunner` | `document_clean` | `document_text` | Cleans document content according to collector standards | +| **5** | `docTextRunner` | `document_text` | `query_sync` | Extracts text using AWS Textract OCR service | +| **6** | `querySyncRunner` | `query_sync` | `query` | Syncs and schedules queries that need to run for documents | +| **7** | `queryRunner` | `query` | `query` (self-referential) | Executes queries with dependency chains, can re-queue for dependencies | + +### Additional Async Runners + +These runners are triggered by API calls or other events outside the main pipeline: + +| Runner Service | Reads From Queue | Writes To Queue | Trigger | +|---------------|------------------|-----------------|---------| +| `clientSyncRunner` | `client_sync` | `document_sync` | Triggered by API calls or when client configuration changes | +| `queryVersionSyncRunner` | `query_version_sync` | `client_sync` | Triggered when query versions are updated via API | + +## Pipeline Flow Diagrams + +### 1. Document Upload Flow (Main Pipeline) +``` +S3 Upload Event + ↓ +[store_event queue] + ↓ +storeEventRunner + ↓ +[document_init queue] + ↓ +docInitRunner + ↓ +[document_sync queue] + ↓ +docSyncRunner + ↓ +[document_clean queue] + ↓ +docCleanRunner + ↓ +[document_text queue] + ↓ +docTextRunner + ↓ +[query_sync queue] + ↓ +querySyncRunner + ↓ +[query queue] + ↓ +queryRunner +``` + +### 2. Client Sync Flow (Batch Processing) +``` +Client Update (API) + ↓ +[client_sync queue] + ↓ +clientSyncRunner + ↓ +[document_sync queue] → (feeds into main pipeline) +``` + +### 3. Query Version Update Flow +``` +Query Update (API) + ↓ +[query_version_sync queue] + ↓ +queryVersionSyncRunner + ↓ +[client_sync queue] + ↓ +clientSyncRunner + ↓ +[document_sync queue] → (feeds into main pipeline) +``` + +## Queue Names and Environment Variables + +| Queue Purpose | Queue Name Variable | Queue URL Variable | +|--------------|-------------------|-------------------| +| S3 Events | `QNAME_STORE_EVENT` | `STORE_EVENT_URL` | +| Document Init | `QNAME_DOCUMENT_INIT` | `DOCUMENT_INIT_URL` | +| Document Sync | `QNAME_DOCUMENT_SYNC` | `DOCUMENT_SYNC_URL` | +| Document Clean | `QNAME_DOCUMENT_CLEAN` | `DOCUMENT_CLEAN_URL` | +| Document Text | `QNAME_DOCUMENT_TEXT` | `DOCUMENT_TEXT_URL` | +| Query Sync | `QNAME_QUERY_SYNC` | `QUERY_SYNC_URL` | +| Query Runner | `QNAME_QUERY_RUNNER` | `QUERY_URL` | +| Client Sync | `QNAME_CLIENT_SYNC` | `CLIENT_SYNC_URL` | +| Query Version Sync | `QNAME_QUERY_VERSION_SYNC` | `QUERY_VERSION_SYNC_URL` | + +## Key Design Principles + +1. **Single Responsibility**: Each runner has one specific task in the pipeline +2. **Sequential Processing**: Documents flow through defined stages in order +3. **Idempotent Operations**: Duplicate detection prevents reprocessing +4. **Async Reprocessing**: Client/query updates trigger document reprocessing +5. **Self-Referential Queuing**: Query runner can re-queue items for dependency handling +6. **Scalability**: Each runner can be scaled independently based on workload + +## IAM Permissions Required + +For the S3-to-SQS integration: +- S3 bucket must have permission to send messages to the SQS queue +- SQS queue policy must allow the S3 service principal to send messages +- Each runner service needs permissions to: + - Receive and delete messages from its input queue + - Send messages to its output queue(s) + - Access S3 buckets for document operations + - Access AWS Textract (for docTextRunner) + +## Monitoring and Debugging + +Each runner exposes Prometheus metrics on port 8080 and includes: +- Queue processing metrics +- Error rates +- Processing duration +- Health check endpoints + +Runner ports for local development: +- `storeEventRunner`: 8081 +- `docInitRunner`: 8082 +- `docSyncRunner`: 8083 +- `docCleanRunner`: 8084 +- `docTextRunner`: 8085 +- `querySyncRunner`: 8087 +- `queryRunner`: 8088 +- `clientSyncRunner`: 8089 +- `queryVersionSyncRunner`: 8090 \ No newline at end of file diff --git a/serviceAPIs/queryAPI.yaml b/serviceAPIs/queryAPI.yaml index 0bbfb866..3039a454 100644 --- a/serviceAPIs/queryAPI.yaml +++ b/serviceAPIs/queryAPI.yaml @@ -5,12 +5,12 @@ info: description: API documentation for the Query Orchestration services. version: 0.0.1 contact: - email: mmcguinness@aarete.com - name: Michael McGuinness - url: doczy.com + email: piragavarapu@aarete.com + name: Aarete Inc. + url: https://aarete.com license: - name: unlicensed - url: "www.example.com" + name: Aarete Inc.internal use + url: https://aarete.com servers: - url: https://doczy.com/v1 description: Production server