Merged in feature/deployment-docs (pull request #187)
deployment docs * deployment docs * branding on api page
This commit is contained in:
Jay Brown
@@ -5,6 +5,11 @@ This repository contains the DoczyAI code.
|
||||
|
||||
Using the following project as a baseline: `https://github.com/golang-standards/project-layout`.
|
||||
|
||||
## Documentation
|
||||
|
||||
- [Queue Pipeline Flow](./queue_flow.md) - Detailed documentation of the SQS queue-based processing pipeline
|
||||
- [Deployment Environment Variables](./deployment_env_variables.md) - Configuration guide for environment variables across all services
|
||||
|
||||
## Installation and Usage
|
||||
|
||||
- Install devbox: `https://www.jetify.com/docs/devbox/installing_devbox`
|
||||
|
||||
+91
-91
@@ -1092,97 +1092,97 @@ func RegisterHandlersWithBaseURL(router EchoRouter, si ServerInterface, baseURL
|
||||
// Base64 encoded, gzipped, json marshaled Swagger object
|
||||
var swaggerSpec = []string{
|
||||
|
||||
"H4sIAAAAAAAC/+xdfW/bOJP/KoTuAQ541u/O++Fwlybtbhbdtpukt889bc6gpbHNrUy6IpXULfLdD3yT",
|
||||
"KImyZddO93AB+kcS8WU4M/zNkJyZfgtCNl8wClTw4OxbMAMcQaJ+vMYCXpM5EfKXCHiYkIUgjAZn6hOK",
|
||||
"5TdE6IQlcyw/IEKR/gV9DNTXf3sgNGIP/y7IHNr6549B0ArgC54vYgjOgn6vZxv1e0Er4OEM5ljO6G1z",
|
||||
"JNvM8ZfXQKdiFpwNB61ggYWARJL1Px967dO7n2xj/dvfglYglgs5EBcJodPg8fFR9krwHIRZ6wsswtnV",
|
||||
"ZXWltzNAY/kRpYuY4QhdXXaCVkDktwUWs6AVUDyXg6tWIxIFrSCBzylJIArORJKCu6i/JTAJzoJ/6eZc",
|
||||
"7+qvvGtpkNRdxASoqCMoVF/rSfkOIrKJJRWXLEznK+iIzPe9UOJMLml5+WXBklpKQH3dCx3ZxJKK31NI",
|
||||
"lnVEXF0iNkFiBuizbLZ7UuzsSoET4AtGOSj9vaJyC+D4ZZKwRP4hZFQAVVsXLxYxCdUO7f7JJbXfmi5d",
|
||||
"jvYbcI6noCctLtrOijgk95AgkO3lsutQxDeZadvNG6qZrug9jkl0DZ9T4OIJl6SmRYmeF41ZtNzRit4w",
|
||||
"8YqlNHqytVwDZ2kSAqJMoImcekcruWXsN0yXRjb8KRekVB6lC0aRYAzNMV1aWfEdrK4VXINIlu3ziYCk",
|
||||
"usvfpPMxJHKXcwgZjTgaw4QlgESyJHSK8BQT2nENXH/Qc3e3No8SBagYDrQtI/N0HpydHB30eq1gTqj+",
|
||||
"vZdZLUIFTCGRDHlsBe8pTsWMJeQrRE/P+IcZUJQ6JOxEox4tiza0xSQq8Dro9U8PT3rRpA2To8P28dHx",
|
||||
"UfskgtP26cHB8BCfDofDYxy0ciGkqQJj1584qjgLLU3QjcAi5X6iuPom1QIXCJTEUSnMD8EiYSFwLkds",
|
||||
"KZcrBgGywQSTWP0QYhpCLH++q6PhvRr1EgQmsSIFx/HbSXD2oYFjofvepPM5TpbBY+ubJGkBiSDagmg6",
|
||||
"RhMSgzRUnpW+JlzINWZNkJhhgXRHVFhgJpIPQcTCYWcRTYKW/LF/rH6+awVEwFxN4nD/oHd6VPDnOj7v",
|
||||
"TQnsSnfv93q9fKfgJMFLZafNH9j4TwhF8HhXVmjNQ4gK7ises1SURIgIDeM0kpvbLDTnUFEskj31THPH",
|
||||
"5NJLRti4cUGrJAnVVP+YMWlj8RZ5VOZQKxBM4HgUspR6iL6VHxHNwM5QpMgWM8JzyjNBDw5bK8Bt0D84",
|
||||
"PjgZHh0cr0E41zP6kHGiSO5dRbwFOVis8pxaLIolINKEQqThrCxwjnAYwkJIWbOkqNgeSUmXv6ln3wp4",
|
||||
"BiNrOxjEyTqN0iSuLur99WskGApnEH4yCzFzuMjY1RLrnp+fd63X3lWtuxuDZkLaCUwgARpCET0P+4Pq",
|
||||
"WcsjUn1Kysh0lrdGtla/K1wwH9Zu512IULOyQZ/8LOVA/gh7NtwfUg9F2bxlfYo6mEkiwgLa8mhdkkKv",
|
||||
"FdA0jvFYil6fLyoQGiaAN6PmAXNkejUkojKpsTFWAfkqBytr1MDKDHaBPfKwpnz/kfzSjEjKaPvd5Stl",
|
||||
"E7h2sRGhCCfhjNxDwQ/cDYksIVNCcZzZ6Sp1b02TzFDZQ6mWI0Ton1fvfCQG2ZI7X8miKM6mltnIZ3Mh",
|
||||
"Zx0RZ2iCE5ewgx3Jd5GwaQKcjxaQhOAzfO8y9UK2MTKNpWfcmKb+Old+S0OgzeAK1pYtd85g4mzpgmJK",
|
||||
"UvdjuTXM53DpU1/HDJQX59cmD4rU7lyPyAvI57M2GrQvML1Z0rDK4Cu9l8wNnPQV4pg9qLNfKMg9IL6k",
|
||||
"Ic+3xpixGDB1RlbT+88RuWlSnoeilDB1oCxarc1sj8WJ9e3fyJZlWSrJqTHq+bXmphK+mIsiNVaOOOfn",
|
||||
"59Xz13qUsXO+YNFy5bz6gLg976qcqGfBGy8aO8QYfXdXn4Ao2c3B4QYcqDuT3iwgJBOiTmjF06kmpeMc",
|
||||
"TK/ejG7++81F0ArevL1VP768dH65evOz9zzqEuAXw7ldt5nfdc0076qyaQaKhcWXZWSGqJfT+0XUZAMq",
|
||||
"r2OOl2gsLafs4tGlENMRNzixnmILKtttyepyWAT/BQkn+qbHcz0PXHIFhSwCdK9bdoIi2h8duGh/OhgM",
|
||||
"h8eD3vDo5PDg+PioV7BjfZ8du2BxDKFgntuy7BOaswjiKvs0ZI7u80Ws4odd67Yu+IRAHK1XLkv0K938",
|
||||
"sRXEWAAXW5BpODcKpQmApOkIrmCdUQR8EVsNUdohrkE2PKkntWb6ClNaZWF6N2CBt36lVRSVLaEyubZz",
|
||||
"VY8a7abC3HpXtQL1UNNAj/KXlyIvDaZnw6xfdb2ZYKlYpMIwQA7c2c40lBS4nsuZU1hgbaM7p5IkC/dN",
|
||||
"is7yfVPW4QZ8fjdeqjOmlLUCW+VQoYt6kW8NHVujwF9nM1c07JKFGujWWrWSE7Aja/aD3NFWTvGdnyvq",
|
||||
"HLDm7TrngYcj32dnavcdppHaePc4TiFDOEuS6yRm1/TLfrbLdbec2OVAn0ftr0N762P/cBicfei3Bq3h",
|
||||
"nU97ZpjP1q3vF9mmkaRLz/YVybnWR82cMWyVDNdGINQ+BY2Hk5P28fDgpH2Kx/12Hw4Px8NJ/wSi4y2e",
|
||||
"giw9tReRBaK4uZVcpWJPy301m4/RhRfAyqou898khKh3/qKWzm3XYMlSNE+5MC8ngDCyHx/Li5/XTWgo",
|
||||
"QfqvY6n0ylDhlENR0Nl0U3n6ngOaMlZ41tnCiJY4Z6n08k1FhzhPcvWGzcSoiIRMp5AgG7+xI9TR3sMo",
|
||||
"ZvoB2K+Z9qs0/A8zEs4UVw1hX8lC3RmiBxLH8tiThQzk3ObDs24Xd8+7uk930Bsc9gb9YVfun07I70vc",
|
||||
"7h2cFNit+nd+kv/MCCpI69vJY7fz08ePcgSvU9PsYKiFUXMwdIFnxSGxUZxRDdpA+yA67LeP+8OwfXoC",
|
||||
"0D44OoCT4Um/P+kPt0Cbwnr8pp1xTsZxRphcmEYZe75335oJHdkLsfzl+a524lutqau12qizctioIaOq",
|
||||
"0griRxMS24C74oCv1Ad9WxGyBaAx5hAhRo03bFxjZfd4YydV+W566AYeKqFT4JKebcjMOqM8rrDKBaDR",
|
||||
"qP7yIcZcIPlZQmw2YKfBY8tw4N8yiaidjnA0IckOJ6x6Fj5fwxVJDXu16x+yOJ3TKjQyGhHRwGd2JrrI",
|
||||
"+ljnbLT9gVGroF+A+htKOUzSGAmmFEUrU0FnNz/TrXzML+Gcs8KWw6+M9LvVYrlwObwCcPTGnGQiy2Yq",
|
||||
"oE8MnI/EDMv5p+rmO7G/hjHjEI0IFZDc4zhoBWwB1P09hokYVZslZDrz/d34GwqY9U8+aPvF+Fsr3Dbj",
|
||||
"lq6yYzVyuopqXahzxAmdxoCUpay5lS497VPyOQVEIqCCTAgk2o2ggohlZ2Nz0uwu+zXh4nLF+5LDp/z6",
|
||||
"IDtPNoLlsg+9HpolUb+nkBDfzjtHHFSIzWfdosrbz3VdbXCO07PRCtRtUJFuN/TBvy8tFT6u6wE9S4vZ",
|
||||
"lIQopUTRCV8gTP0vQttfpTI6IdNGK77QTRsdRLILs++4PrXcGzkCXNX12rQ3c+sXU8XqBrTeyobeM5Ma",
|
||||
"onK/WVlXrVwvMg6Xr8fl39NEe+T2EiALHV8FPx8/fuv8/ePHRy8I6UlXPTNmLgqyi1UhRKoLwjkJFdu7",
|
||||
"uab8eBmq3rXCqXP0FQ+eKsBUEw9cONHua1+HjZyQVEIbAl0VmgXr0WZ3B/ZKe9NNW+K9O3t5yFqRaEbU",
|
||||
"BfFJTtijM5okbF5gRJUB+q7MFwuYxqKQr5ENsLHhL61aT1m/PqPOq95sZc9KMol1qn69eftm9PIft9fn",
|
||||
"F7dvr4NWcPH2ze3Lf9yOXr1//drr9Kh5t3/4dPXtR5ue7wcU37mk0qrWV8gQU+vM1eWGboPeXWscnkbP",
|
||||
"ujUvulvH73AI04SI5Y0k10L+lBLBzlPh8ZnlX6Uzaq6UUhU2dT7HXxlFF7ojurp810FXEj21x3j96uLk",
|
||||
"eHCIfv3jFo0lcC0SqT6hwTRLglpTzB60fplkAzXNBYug8sf3SRycBTMhFvys241Y+HXZkQ06KW8D5qLd",
|
||||
"72BFl1lPJ2TzLpMtBt0slUElH7KFOanPMZFjnochcC6tY8oh+VeO9Ad9VpGQGvwMAr29urxAgn0Cc2Cd",
|
||||
"EB3fUe5sP6lIrk/wXXTr6VxlVn+XY//5IKzExoATSF5Z9fj1j9ugHAovRaEGQ2wsMKEQITwRkGQyxAU5",
|
||||
"bytOtQlURJQiKQdRuXydc0LohNmUFhwKVw7zeThNCaXA+X9inIAAyYs8qe43Es4wxOi38GfTLGgFKl46",
|
||||
"0GyVrStpLefvrrLTTNELU1sVvU3CGXBhPDQOyb1cmlxNTEIw5slQkFLztyib+eHhoWN8Bzu/IEKphm/8",
|
||||
"83dX8pRut37Q6/Q6fXW1uwCKFyQ4C4adXmcYKIM0U3pq4rrVOxHzOQ7aD+QIIwoPNiLngQh987tI2D2J",
|
||||
"IEKRvsDuaM3WBF1FWf8LG+9v/Awb8bOT5KNCXJwn+UhjnqLP76+GLnV5VmU5R3LQ6++YZhOB5qFZf7fx",
|
||||
"0oinCgomaRzvKpvwoNer65GtultKolTd+uu7FXLMZKfB6fpO5aTAx1Zw2IxGN3XVNUQqsSkDsw93j3et",
|
||||
"gNuHN6OZBcWWoIKnXPpIJkZMb9ngTg5rcyC+kehRkjX1RSJcg0gI3Ksdw7VLFtptM14iIrhJMi5uk59B",
|
||||
"OHukoHS9nSld/sxfr3HORn5Wsu9XMmndsSP/q8sVOlYsKlCTlJc3cZ7v7hSkhx4nSzvtXL+tEK4CY1wU",
|
||||
"l8pfC9668xOAtzlarAFv6QjphuuhuucxZXrZ9kzyjKm7V3ctR9ekNsTTbuiGgW6ArFmIqAHXQmmLKsba",
|
||||
"5i+WZvNE+wTcPLbVB7gZ6c+Yuw/MdTUj0wpXI/OIwn0C8A0IO31O0nrovalX1j2AsBtcuR6Ex6CeTLyB",
|
||||
"tetw+WBVmLcc9RmWd78jboo7YvUmKCNz5IRBrgFmqQ2xuefKX/nGSzflyYfMhUfDp4Hm4jvlCn84W4Zc",
|
||||
"2LNC7kYh1WWoKLwFl0oaWAXNZLQrlPbecujscOlY6Dd2FcZmAjBMbSIF2Ux1wDGag8ARFtjjMqsSG3kk",
|
||||
"bj1ez9NYkAVORHfCknlbDaeCfEIWETrVYU86OtL0ujXXvI7W68oY8GURq4vNCY45tAIuluqaSA4c5OVR",
|
||||
"nHDgLK4qhrpIY82CLO89uyMeEyqlWHjn6PeOh8cH/ZPBQc+buF2fbWw5GqZcsDlyEjuricWmDsiKvOLO",
|
||||
"T2tfV9SSq48rVQQweeuKEWMpvq1OHVYVsvTpZwzZ1VnD7Fkl0JWIUWfSdAWLRieOuLYOi8II8/gW5QBW",
|
||||
"b+FeZFVJSjhWCmHWTzBOMrZ6JJJ7UtcfseXh1ENSfpWtCicWKiFGMMFpLIKzQa950nnf98ZTn4mf0cY/",
|
||||
"qdx/H2VsMuFQQ9puksjv9ugxlEv1eBDDW6znebvvzmVYWQXpB7kMTjUMpF6eCCV0irR1jwG9u3xVdnP4",
|
||||
"koZmLYWCJKtciRem8MJ3+RO2asdKl0IX79japXDmqPcqxk45EcyLXGzua6wz9XbAJta+RowF6TXwAAb7",
|
||||
"wJssmsVD9wu3xk594alnCNqZx1GzszEvKfHWDkn3my1/8qh3UQy+uJsLVfCPl+uPqfgbwlGYJglQES9X",
|
||||
"YYweo4ox6+5sClqXVR4sXN78H1Q6vdLVnbIqrH/Zd00ljWrdspXmcY3/G9lyh55aHNoKZzfypVkrV/Br",
|
||||
"dG0v/ppNr1sHn+aO81lzf9zFfQHK8tJO+3DtWmvbZkX77sqArfPUlPOza8fyihJB1Jutk1tpILx6dWpy",
|
||||
"7HTC3Z6eB4rZfDXvA4ZQ4x/pMnBPGlKT5854KHxZyJ19fv3dy/41GpLncTo716Si1npAeYJugzcGdX4y",
|
||||
"pqjRu6+pM/U0j77lylb1jwtmCc/vvzs3I7Z0WK4o+4m7kWqcOe8NA8OE85qq6qlkiYNGl3XWSK3ntOcQ",
|
||||
"MT2HR2mzm+xnfd25vkYl3uaqsFvvx83RUcprqiisVN2LGYSftN6aQ6WuE6C0183ZL6psXn0A9qmyxQoa",
|
||||
"9bZ/hjkaA9C8LPKzAu/oxKlqh7sVJDorLP+mKpv//zlKYWdMPyPWgWyaUK2qv9z+9hrNgaZogae6SJKT",
|
||||
"iQCRSqrgXqT9hc3hnS7Yu0ZtBXwR3ZmYx0V9LVKlKDGdsicjuQ5NmaTRvfNUw5VuPA9ODo9Lmf5/r/mP",
|
||||
"qUpzZ9NIKp4VfmeI7ZWh1XnZrejrxmxKaK3eFs98qi2axOxBmoEEIpJAqCJ3BXMzo0xDSYAnpkZNWFLg",
|
||||
"ob6fLm8aPYE7ej5yUWVe15YHMv+PgKTfO4jn/7BJE7ImW9KmNnX+7n3Wf1bglQr82Cpl4H2waWetwOah",
|
||||
"2ZSy8mubEp2Rp2Ob1+t4O8RxPMbhp1pl/wXTKDaq/laOMkC2D4rSxNbqMtpjLj7QNYRArPtcyN7T9WlV",
|
||||
"Kq+7OTCNEHwJZ5hOgSNikJd9AsprdsuFpXzNy/z56tlrXr9lu5X/c9uqjfDhvP1P3P7aa59+/Nge+f9T",
|
||||
"wsrrvHK98mx9xYCLm+tXkqlCIgqriyHgwhdlX0Nsvzc42JjYu02RyVFCk1+YX+FoXdkQqSyu1o+2PWT9",
|
||||
"Rx1m/X9CnwKglDc60GjBSOHd3gsoLK0PPH3NphoOpCeHWKoP0JpxugSsjt/jXCkNjepsqZ4no8kLD5KQ",
|
||||
"77CmcnzlKUg31FHlzIX4Pu2NLYHPOrun8BMtQoXzm9jEz7YyT+MIM1OXwC1hp6rGhQkRkBDsD5+25Y32",
|
||||
"HDJtp/GcNM7LK3g+Xu8w8ulzJl+rayoFvHCsbpDArYtObJy//btxD/bxuuSWG9ome/uzQ9tf4KVJZ+Y/",
|
||||
"J24/UeK2lX7NrsggeIusbb1XVidt5xtjT6BrSsXV6dnzVfweUgczwRfyBat4u9E1Zla8Z7NcbQewG6Rq",
|
||||
"7x2nVydq69RqW8c0K0tv6g1l5Z+2yKLQ2v6cuv0EqdsbIWpXmHJz37EZvG7LS1WqUgGyLlCXUv1YWufB",
|
||||
"FOsml4JkgIu9bw23/l7N/igV3NskTqa3D2Lro4pvNY08jZ9zHncVHSNZun57qfGSe//N47uERWmYVZNS",
|
||||
"ZbBSTxGwkM279321vcw81bw7s0GkVx0rVJVOtYps4PlFYDFUonrFWDOMzSp2RypnGjcdzI2+N2NVHsKb",
|
||||
"jpWfosxIBe43HUW/dDqjFJ84mw5TrIuWj+ZeIjzePf5vAAAA//8uQJ8AYocAAA==",
|
||||
"H4sIAAAAAAAC/+xdfW/bOJP/KoTuAQ541u92Xg+HOzdpd73ott0kvX3uaXMGLdE2tzLpklRSt8h3P/BN",
|
||||
"oiTKll073cMZ6B9JxJfhzPA3Q3Jm+i0I6WJJCSKCB5ffgjmCEWLqxxso0Gu8wEL+EiEeMrwUmJLgUn0C",
|
||||
"sfwGMJlStoDyA8AE6F/Ax0B9/bdHTCL6+O8CL1BT//wxCBoB+gIXyxgFl0G307GNup2gEfBwjhZQzuht",
|
||||
"cyrbLOCX14jMxDy47PcawRIKgZgk638+dJoX9z/Zxvq3vwWNQKyWciAuGCaz4OnpSfZicIGEWesLKML5",
|
||||
"6Lq80rs5AhP5ESTLmMIIjK5bQSPA8tsSinnQCAhcyMFVqzGOgkbA0OcEMxQFl4IlyF3U3xiaBpfBv7Qz",
|
||||
"rrf1V962NEjqrmKMiKgiKFRfq0n5DiLSiSUV1zRMFmvoiMz3g1DiTC5pefllSVklJUh9PQgd6cSSit8T",
|
||||
"xFZVRIyuAZ0CMUfgs2y2f1Ls7EqBGeJLSjhS+jsicgvA+CVjlMk/hJQIRNTWhctljEO1Q9t/ckntt7pL",
|
||||
"l6P9hjiHM6QnzS/azgo4Yg+IASTby2VXoYhvMtO2nTVUM43IA4xxdIM+J4iLZ1ySmhYwPS+Y0Gi1pxW9",
|
||||
"oeIVTUj0bGu5QZwmLESAUAGmcuo9reSO0t8gWRnZ8OdckFJ5kCwpAYJSsIBkZWXF97C6RnCDBFs1h1OB",
|
||||
"WHmXv0kWE8TkLucopCTiYIKmlCEg2AqTGYAziEnLNXDdXsfd3do8ShQgot/TtgwvkkVweX466HQawQIT",
|
||||
"/XsntVqYCDRDTDLkqRG8JzARc8rwVxQ9P+Mf54iAxCFhLxr1ZFm0pS3GUY7XQad7cXLeiaZNND09aZ6d",
|
||||
"np02zyN00bwYDPon8KLf75/BoJEJIUkUGLv+xGnJWWhogm4FFAn3E8XVN6kWMEegJI5IYX4IloyGiHM5",
|
||||
"YkO5XDESSDaYQhyrH0JIQhTLn++raHivRr1GAuJYkQLj+O00uPxQw7HQfW+TxQKyVfDU+CZJWiImsLYg",
|
||||
"mo7xFMdIGirPSl9jLuQa0yZAzKEAuiPILTAVyYcgomG/tYymQUP+2D1TP983AizQQk3icH/QuTjN+XMt",
|
||||
"n/emBDbS3budTifbKZAxuFJ22vyBTv5EoQie7osKrXmIopz7Cic0EQURAkzCOInk5jYLzTiUF4tkTzXT",
|
||||
"3DG59JIBNG5c0ChIQjXVP6ZM2lq8eR4VOdQIBBUwHoc0IR6i7+RHQFKwMxQpssUc84zyVNC9k8YacOt1",
|
||||
"B2eD8/7p4GwDwrme0YeUE3ly70vizcnBYpXn1GJRjCGRMIIiDWdFgXMAwxAthZQ1ZXnF9khKuvx1PftG",
|
||||
"wFMY2djBIE7aaZywuLyo9zevgaAgnKPwk1mImcNFxraWWHs4HLat195WrdtbgybDTYamiCESojx6nnR7",
|
||||
"5bOWR6T6lJSS6Sxvg2ytfpe4YD5s3M77EKFmZY0+2VnKgfwx9Gy4P6QeiqJ5S/vkdTCVRAQFasqjdUEK",
|
||||
"nUZAkjiGEyl6fb4oQWjIENyOmkfIgelVk4jSpMbGWAXk6xystFENK9PbB/bIw5ry/cfySz0iCSXNd9ev",
|
||||
"lE3g2sUGmADIwjl+QDk/cD8kUoZnmMA4tdNl6t6aJqmhsodSLUcUgX+O3vlIDNIlt77iZV6cdS2zkc/2",
|
||||
"Qk47Ak7BFDKXsMGe5LtkdMYQ5+MlYiHyGb53qXoB2xiYxtIzrk1Td5Mrv6Mh0GZwDWuLljtjMHa2dE4x",
|
||||
"JamHsdwa5jO49KmvYwaKi/NrkwdFKneuR+Q55PNZGw3aV5DcrkhYZvBI7yVzAyd9hTimj+rsFwr8gABf",
|
||||
"kZBnW2NCaYwgcUZW0/vPEZlpUp6HohRTdaDMW63tbI/Fic3t38iWRVkqyakxqvm14aYSfTEXRWqsDHGG",
|
||||
"w2H5/LUZZeycL2i0WjuvPiDuzrsyJ6pZ8MaLxg4xRt/d1TMkCnazd7IFB6rOpLdLFOIpVie0/OlUk9Jy",
|
||||
"DqajN+Pb/35zFTSCN2/v1I8vr51fRm9+9p5HXQL8YhjadZv5XddM864sm3qgmFt8UUZmiGo5vV9GdTag",
|
||||
"8joWcAUm0nLKLh5dCiEZc4MTmym2oLLbliwvh0bovxDjWN/0eK7nEZdcASGNEHjQLVtBHu1PBy7aX/R6",
|
||||
"/f5Zr9M/PT8ZnJ2ddnJ2rOuzY1c0jlEoqOe2LP0EFjRCcZl9GjLHD9ki1vHDrnVXF3yKURxtVi5L9Cvd",
|
||||
"/KkRxFAgLnYg03BuHEoTgFjdEVzBOqMI9EXsNERhh7gG2fCkmtSK6UtMaRSF6d2AOd76lVZRVLSEyuTa",
|
||||
"zmU9qrWbcnPrXdUI1ENNDT3KXl7yvDSYng6zedXVZoImYpkIwwA5cGs301BQ4Goup05hjrW17pwKkszd",
|
||||
"Nyk6i/dNaYdb5PO74UqdMaWsFdgqhwpcVYt8Z+jYGQX+Opu5pGHXNNRAt9GqFZyAPVmzH+SONjKK7/1c",
|
||||
"UeeADW/XGQ88HPk+O1O57yCJ1MZ7gHGCUoSzJLlOYnpNv+qmu1x3y4hd9fR51P7at7c+9g8nweWHbqPX",
|
||||
"6N/7tGcO+XzT+n6RbWpJuvBsX5Kca33UzCnD1slwYwRC5VPQpD89b571B+fNCzjpNrvo5GTSn3bPUXS2",
|
||||
"w1OQpafyIjJHFDe3kutU7Hm5r2bzMTr3Alha1XX2m4QQ9c6f19KF7RqsaAIWCRfm5QQBCOzHp+LiF1UT",
|
||||
"GkqA/utEKr0yVDDhKC/odLqZPH0vEJhRmnvW2cGIFjhnqfTyTUWHOE9y1YbNxKgIhmczxICN39gT6mjv",
|
||||
"YRxT/QDs10z7VRr+xzkO54qrhrCveKnuDMEjjmN57ElDBjJu8/5luw3bw7bu0+51eiedXrfflvunFfKH",
|
||||
"Arc7g/Mcu1X/1k/ynxlBBWl9O39qt376+FGO4HVq6h0MtTAqDoYu8Kw5JNaKM6pAG9QcRCfd5lm3HzYv",
|
||||
"zhFqDk4H6Lx/3u1Ou/0d0Ca3Hr9pp5zjSZwSJhemUcae7923ZkzG9kIse3m+r5z4Tmvqeq026qwcNmLI",
|
||||
"KKu0gvjxFMc24C4/4Cv1Qd9WhHSJwARyFAFKjDdsXGNl93htJ1X5bnroGh4qJjPEJT27kJl2BllcYZkL",
|
||||
"iETj6suHGHIB5GcJsemArRqPLf2ef8swUTkd5mCK2R4nLHsWPl/DFUkFe7XrH9I4WZAyNFISYVHDZ3Ym",
|
||||
"ukr7WOdsvPuBUaugX4D6G0g4miYxEFQpilamnM5uf6Zb+5hfwDlnhQ2HXynp9+vFcuVyeA3g6I05TUWW",
|
||||
"zpRDnxhxPhZzKOefqZtvZn8NY8pRNMZEIPYA46AR0CUi7u8xmopxuRnDs7nv78bfUMCsf/JB2y/G31rj",
|
||||
"thm3dJ0dq5DTKKp0oYaAYzKLEVCWsuJWuvC0T/DnBAEcISLwFCOm3QgisFi1tjYn9e6yX2Murte8Lzl8",
|
||||
"yq4P0vNkLVgu+tCboVkS9XuCGPbtvCHgSIXYfNYtyrz9XNXVBuc4PWutQN0G5el2Qx/8+9JS4eO6HtCz",
|
||||
"tJjOcAgSghWd6AsKE/+L0O5XqZRM8azWiq9001oHkfTC7DuuTy33xo4A13W9Me3N3PrFVLG6Bq13sqH3",
|
||||
"zKSGKN1vltZVKderlMPF63H594Rpj9xeAqSh4+vg5+PHb62/f/z45AUhPem6Z8bURQF2sSqESHUBMCOh",
|
||||
"ZHu315QfL0PVu1I4VY6+4sFzBZhq4hEXTrT7xtdhIycgldCGQJeFZsF6vN3dgb3S3nbTFnjvzl4cslIk",
|
||||
"mhFVQXySE/boDKaMLnKMKDNA35X5YgGTWOTyNdIBtjb8hVXrKavXZ9R53Zut7FlKJrFO1a+3b9+MX/7j",
|
||||
"7mZ4dff2JmgEV2/f3L38x9341fvXr71Oj5p394dPV99+tOn5fkDxnUtKrSp9hRQxtc6Mrrd0G/Tu2uDw",
|
||||
"1HrWrXjR3Tl+h6MwYVisbiW5FvJnBAs6TITHZ5Z/lc6ouVJKVNjUcAG/UgKudEcwun7XAiOJntpjvHl1",
|
||||
"dX7WOwG//nEHJhK4lkyqT2gwzZKg1hTTR61fJtlATXNFI1T643sWB5fBXIglv2y3Ixp+XbVkg1bCmwhy",
|
||||
"0ey2oKLLrKcV0kWbyha9dprKoJIP6dKc1BcQyzGHYYg4l9Yx4Yj9Kwf6gz6rSEgNfkYCvB1dXwFBPyFz",
|
||||
"YJ1iHd9R7Gw/qUiuT+i76NbTucqs/i7H/vNRWIlNEGSIvbLq8esfd0ExFF6KQg0G6ERATFAE4FQglsoQ",
|
||||
"5uS8qzjVJlARUYqkDETl8nXOCSZTalNaYChcOSwxgzP4ABlcJv8JIUMCSWZkWXVD9TcwIqGcLMkx1mlf",
|
||||
"ymwZvhulB5q8I6Z2K3jLwjniwjhpHLEHuTo5R4xDZCxUmQZsM+MSjjbQI7BQ2uKbb/huJA/uFg2CTqvT",
|
||||
"6qrb3iUicImDy6Df6rT6gbJRc6W6JtRbPR1Rny+hXUMOICDo0QbpPGKhL4OXjD7gCEUg0nfaLa3smqBR",
|
||||
"lPa/sikAxvWwQUB7yUfKhcp58pE0DCr6/C5s6FKXJVoW0yZ7ne6eaTZBaR6a9XcbQg14otBhmsTxvhIM",
|
||||
"B51OVY901e1CXqXq1t3cLZd2Jjv1LjZ3KuYJPjWCk3o0utmsrm1SuU4pvn24f7pvBNy+xRnNzCm2xBk4",
|
||||
"49JtMmFjegsH93JYmxbxDUdPkqyZLzjhBgmG0YPaMVx7aaHdNpMVwIKbvOP8NvkZCWeP5JSuszely17+",
|
||||
"qzXO2chHJft+JZMGHzryH12v0bF8nYGKPL2sifOid68gPfT4XdqP5/q5BXMVK+OiuFT+SvDWnZ8BvM1p",
|
||||
"YwN4S99IN9wM1R2PKdPLtseUI6buX921HF2TWhNP26EbGboFsqZRowZcc9Uuyhhrm79Ymc0THRJws3BX",
|
||||
"H+CmpB8x9xCY62pGqhWuRmZBhocE4Fsk7PQZSZuh97ZaWQ8Awm685WYQniD1iuKNtd2Ey4N1kd9y1CMs",
|
||||
"739H3OZ3xPpNUETmyImM3ADMUhtic/WVPfxNVm4WlA+Zc++IzwPN+afLNf5wugy5sKNC7kch1f2oyD0P",
|
||||
"F6ocWAVNZbQvlPbecuiEcelY6Gd3FdlmYjJMuSIF2VR1gDFYIAEjKKDHZVZVN7Lg3Gq8XiSxwEvIRHtK",
|
||||
"2aKphlNxPyGNMJnpSCgdMGl63ZmbX0frdbEM9GUZq7vOKYw5agRcrNQ1kRw4yCqmOBHCaahVjKqCjzUL",
|
||||
"0lT49Np4gomUYu7po9s5658Nuue9Qceby12dgGw5GiZc0AVwcj3LucamNMiaVOPWTxsfXNSSy+8tZQQw",
|
||||
"qeyKERMpvp1OHVYV0ozqI4bs66xh9qwS6FrEqDJpuqhFrRNHXFmaRWGEeY+LMgCrtnAv0kIlBRwrRDXr",
|
||||
"VxknP1u9G8k9qUuS2Ipx6m0pu9xWtRRzxREjNIVJLILLXqd+HnrX9+xTnZyf0sY/qXIAPsrodMpRBWn7",
|
||||
"ySu/P6DHUKze40EMb/2e43bfn8uwtjDSD3IZnAIZQD1GYYLJDGjrHiPw7vpV0c3hKxKateRqlKxzJV6Y",
|
||||
"Wgzf5U/YQh5rXQpdz2Nnl8KZo9qrmDgVRiDPc7G+r7HJ1NsB61j7CjHmpFfDA+gdAm/SABcP3S/csjvV",
|
||||
"taiOELQ3j6NiZ0NeUOKdHZL2N1sR5Unvohj5QnGuVA1AXixJpkJyMAdhwhgiIl6twxg9RhljNt3Z5LQu",
|
||||
"LUaYu7z5P6h0eqXrO6WFWf+y75pKGuVSZmvN4wb/N7IVED3lObQVTm/kC7OWruA36NpB/DWbcbcJPs0d",
|
||||
"51Fzf9zFfQ7KsmpPh3DtGhvbpnX87ouArVPXlPOzb8dyRLDA6s3WSbc0EF6+OjVpdzoH70DPA/kEv4r3",
|
||||
"AUOo8Y90ZbhnDanJ0mk8FL7MpdMeX38Psn+NhmSpnc7ONdmplR5QlrNb441BnZ+MKar17mtKTz3Po2+x",
|
||||
"2FX144JZwvH9d+9mxFYTyxTlMHE3Uo1T571mYJhwXlNViZU0l9Dosk4kqfScDhwipufwKG16k33U173r",
|
||||
"a1TgbaYK+/V+3LQdpbymsMJa1b2ao/CT1ltzqNSlA5T2umn8eZXNChKgQ6psvqhGte2fQw4mCJGsUvJR",
|
||||
"gfd04lTlxN2iEq01ln9blc3+Sx2lsHOqnxGrQDZhRKvqL3e/vQYLRBKwhDNdN8lJTkCRyrPgXqT9hS7Q",
|
||||
"O13Dd4PaCvRFtOdiEef1NU+VosR0Sp+M5Do0ZZJG985TDVe48Rycn5wVkv//XvF/VRXmTqeRVBwVfm+I",
|
||||
"7ZWh1XnZLe/rxnSGSaXe5s98qi2YxvRRmgGGIsxQqCJ3BXWTpUxDSYAnpkZNWFDgvr6fLm4aPYE7ejZy",
|
||||
"XmVeV1YMMv+1gKTfO4jnv7VJGN6QQGmTYFp/9z7rHxV4rQI/NQpJeR9sJlojsKlpNsus+NqmRGfk6djm",
|
||||
"zTreDGEcT2D4qVLZf4Ekio2qv5Wj9IDtA6KE2fJdRnvMxQe4QSHC1n3OJfTpkrUqu9fdHJBEAH0J55DM",
|
||||
"EAfYIC/9hAiv2C1XlvINL/PD9bNXvH7Ldmv/M7d1G+HDsPlP2PzaaV58/Ngc+/+fwtLrvHK9sgR+xYCr",
|
||||
"25tXkqlCIgqtiiHgwhdlX0Fst9MbbE3s/bbI5CihSTnMrnC0rmyJVBZXq0fbHbL+owqz/j+hTw5Qihsd",
|
||||
"kWhJce7d3gsoNKkOPH1NZxoOpCcHaKIP0Jpxuiqsjt/jXCkNiapsqZ4npckLD5KQ77CmcnzlKUg31FHl",
|
||||
"1IX4Pu2NLYFHnT1Q+IkWocL5bWziZ1usp3aEmSlV4Fa1U4XkQoYFYhj6w6dtxaMDh0zbaTwnjWFxBcfj",
|
||||
"9R4jnz6n8rW6plLAc8fqGgncug7F1vnbvxv34BCvS24Fol2ytz87tP0FXpp0Zv4xcfuZEret9Ct2RQrB",
|
||||
"O2Rt672yPmk72xgHAl1TPa5Kz45X8QdIHUwFn8sXLOPtVteYaT2f7XK1HcCukap9cJxen6itU6ttadO0",
|
||||
"Ur0pQZRWhNohi0Jr+zF1+xlSt7dC1LYwFei+YzN43ZaXqnqlAmRdsy4h+rG0yoPJl1IuBMkgLg6+NdyS",
|
||||
"fBX7o1CDb5s4mc4hiK2OKr7TNPIkPuY87is6RrJ08/ZS47EH/83jO0ajJEyrS6nKWImnLlhIF+2Hrtpe",
|
||||
"Zp5y3p3ZINKrjhWqSqdaRTbw7CIwHypRvmKsGMZmFbsjFTON6w7mRt+bsUoP4XXHyk5RZqQc9+uOol86",
|
||||
"nVHyT5x1h8mXSnOqkzmXCE/3T/8bAAD//69cLjR1hwAA",
|
||||
}
|
||||
|
||||
// GetSwagger returns the content of the embedded swagger specification file
|
||||
|
||||
@@ -0,0 +1,256 @@
|
||||
# Deployment Environment Variables Configuration Guide
|
||||
|
||||
## Overview
|
||||
|
||||
This guide provides rules for configuring environment variables when deploying all runners and the queryAPI service across different environments (dev, stage, prod).
|
||||
|
||||
## Variable Categories
|
||||
|
||||
### 1. Service-Specific Variables (UNIQUE per service)
|
||||
|
||||
These variables **MUST be different** for each runner service:
|
||||
|
||||
| Variable | Description | Unique To |
|
||||
|----------|-------------|-----------|
|
||||
| `QUEUE_URL` | Input queue the service reads from | Each runner (different value per runner) |
|
||||
|
||||
### 2. Pipeline Queue URLs (SHARED across all services in same environment)
|
||||
|
||||
These variables should be **IDENTICAL** for all services within the same environment:
|
||||
|
||||
| Variable | Example Value (dev) | Purpose |
|
||||
|----------|---------------------|---------|
|
||||
| `DOCUMENT_INIT_URL` | `https://sqs.us-east-2.amazonaws.com/476114113988/document_init` | Queue for document initialization |
|
||||
| `DOCUMENT_SYNC_URL` | `https://sqs.us-east-2.amazonaws.com/476114113988/document_sync` | Queue for document sync |
|
||||
| `DOCUMENT_CLEAN_URL` | `https://sqs.us-east-2.amazonaws.com/476114113988/document_clean` | Queue for document cleaning |
|
||||
| `DOCUMENT_TEXT_URL` | `https://sqs.us-east-2.amazonaws.com/476114113988/document_text` | Queue for text extraction |
|
||||
| `QUERY_SYNC_URL` | `https://sqs.us-east-2.amazonaws.com/476114113988/query_sync` | Queue for query sync |
|
||||
| `QUERY_URL` | `https://sqs.us-east-2.amazonaws.com/476114113988/query` | Queue for query execution |
|
||||
| `CLIENT_SYNC_URL` | `https://sqs.us-east-2.amazonaws.com/476114113988/client_sync` | Queue for client sync |
|
||||
| `QUERY_VERSION_SYNC_URL` | `https://sqs.us-east-2.amazonaws.com/476114113988/query_version_sync` | Queue for query version sync |
|
||||
|
||||
### 3. AWS Configuration (SHARED across all services)
|
||||
|
||||
These should be **IDENTICAL** for all services:
|
||||
|
||||
| Variable | Example Value | Notes |
|
||||
|----------|---------------|-------------------------------------------------------|
|
||||
| `AWS_ACCESS_KEY_ID` | `AKIAXXXXXXXXXXXXXXXX` | AWS credentials override - should be blank |
|
||||
| `AWS_SECRET_ACCESS_KEY` | `secret_key_here` | AWS credentials - override - should be blank |
|
||||
| `AWS_SESSION_TOKEN` | `token_if_using_temporary_creds` | Optional, for temporary credentials - should be blank |
|
||||
| `AWS_REGION` | `us-east-2` | AWS region for all services |
|
||||
|
||||
⚠️ **Important for AWS deployment**:
|
||||
- **DO NOT SET** `AWS_ENDPOINT_URL` (only for LocalStack)
|
||||
- **DO NOT SET** `AWS_S3_USE_PATH_STYLE` (only for LocalStack)
|
||||
|
||||
### 4. Database Configuration (SHARED across all services)
|
||||
|
||||
These should be **IDENTICAL** for all services:
|
||||
|
||||
| Variable | Example Value | Notes |
|
||||
|----------|---------------|------------------------------------------------------|
|
||||
| `PGHOST` | `database-hostname.rds.amazonaws.com` | Database hostname |
|
||||
| `PGPORT` | `5432` | Database port |
|
||||
| `PGDATABASE` | `queryorchestration_dev` | Database name |
|
||||
| `PGUSER` | `dbuser` | Database username (whatever we use) |
|
||||
| `PGPASSWORD` | `db_password_here` | Database password (keep secure) |
|
||||
| `DB_NOSSL` | `false` | Set to `false` for AWS RDS (should be blank for now) |
|
||||
|
||||
### 5. Authentication Configuration (SHARED across all services)
|
||||
|
||||
These should be **IDENTICAL** for all services:
|
||||
|
||||
| Variable | Example Value | Notes |
|
||||
|----------|---------------|----------------------------------------------------|
|
||||
| `DISABLE_AUTH` | `false` | Set to `false` for production/staging true for dev |
|
||||
| `COGNITO_USER_POOL_ID` | `us-east-2_XXXXXXXXX` | AWS Cognito User Pool ID |
|
||||
| `COGNITO_CLIENT_ID` | `client_id_here` | AWS Cognito Client ID |
|
||||
| `COGNITO_CLIENT_SECRET` | `client_secret_here` | AWS Cognito Client Secret |
|
||||
| `COGNITO_DOMAIN` | `your-domain.auth.us-east-2.amazoncognito.com` | AWS Cognito Domain |
|
||||
|
||||
### 6. Application Configuration (SHARED across all services)
|
||||
|
||||
| Variable | Example Value | Notes |
|
||||
|----------|---------------|-------|
|
||||
| `LOG_LEVEL` | `INFO` or `DEBUG` | Use `INFO` for prod, `DEBUG` for dev |
|
||||
| `BUCKET` | `s3-bucket-name-dev` | S3 bucket name (only needed by some services) |
|
||||
|
||||
### 7. Service-Specific Optional Variables
|
||||
|
||||
| Service | Variable | Purpose |
|
||||
|---------|----------|---------|
|
||||
| `docCleanRunner` | `BUCKET` | S3 bucket for document processing |
|
||||
| `docTextRunner` | `BUCKET` | S3 bucket for document processing |
|
||||
| `queryAPI` | `BUCKET` | S3 bucket for document uploads |
|
||||
| `queryAPI` | `DEBUG` | Set to `true` for debugging |
|
||||
|
||||
## Per-Service Configuration Matrix
|
||||
|
||||
### storeEventRunner
|
||||
**Unique Variables:**
|
||||
- `QUEUE_URL` = `https://sqs.us-east-2.amazonaws.com/476114113988/store_event`
|
||||
|
||||
**Also Needs:**
|
||||
- `DOCUMENT_INIT_URL`
|
||||
- AWS config (shared)
|
||||
- Database config (shared)
|
||||
- Auth config (shared)
|
||||
|
||||
### docInitRunner
|
||||
**Unique Variables:**
|
||||
- `QUEUE_URL` = `https://sqs.us-east-2.amazonaws.com/476114113988/document_init`
|
||||
|
||||
**Also Needs:**
|
||||
- `DOCUMENT_SYNC_URL`
|
||||
- AWS config (shared)
|
||||
- Database config (shared)
|
||||
- Auth config (shared)
|
||||
|
||||
### docSyncRunner
|
||||
**Unique Variables:**
|
||||
- `QUEUE_URL` = `https://sqs.us-east-2.amazonaws.com/476114113988/document_sync`
|
||||
|
||||
**Also Needs:**
|
||||
- `DOCUMENT_CLEAN_URL`
|
||||
- AWS config (shared)
|
||||
- Database config (shared)
|
||||
- Auth config (shared)
|
||||
|
||||
### docCleanRunner
|
||||
**Unique Variables:**
|
||||
- `QUEUE_URL` = `https://sqs.us-east-2.amazonaws.com/476114113988/document_clean`
|
||||
|
||||
**Also Needs:**
|
||||
- `DOCUMENT_TEXT_URL`
|
||||
- `BUCKET` (S3 bucket name)
|
||||
- AWS config (shared)
|
||||
- Database config (shared)
|
||||
- Auth config (shared)
|
||||
|
||||
### docTextRunner
|
||||
**Unique Variables:**
|
||||
- `QUEUE_URL` = `https://sqs.us-east-2.amazonaws.com/476114113988/document_text`
|
||||
|
||||
**Also Needs:**
|
||||
- `QUERY_SYNC_URL`
|
||||
- `BUCKET` (S3 bucket name)
|
||||
- AWS config (shared)
|
||||
- Database config (shared)
|
||||
- Auth config (shared)
|
||||
|
||||
### querySyncRunner
|
||||
**Unique Variables:**
|
||||
- `QUEUE_URL` = `https://sqs.us-east-2.amazonaws.com/476114113988/query_sync`
|
||||
|
||||
**Also Needs:**
|
||||
- `QUERY_URL`
|
||||
- AWS config (shared)
|
||||
- Database config (shared)
|
||||
- Auth config (shared)
|
||||
|
||||
### queryRunner
|
||||
**Unique Variables:**
|
||||
- `QUEUE_URL` = `https://sqs.us-east-2.amazonaws.com/476114113988/query`
|
||||
|
||||
**Also Needs:**
|
||||
- `QUERY_URL` (same as QUEUE_URL - self-referential)
|
||||
- AWS config (shared)
|
||||
- Database config (shared)
|
||||
- Auth config (shared)
|
||||
|
||||
### clientSyncRunner
|
||||
**Unique Variables:**
|
||||
- `QUEUE_URL` = `https://sqs.us-east-2.amazonaws.com/476114113988/client_sync`
|
||||
|
||||
**Also Needs:**
|
||||
- `DOCUMENT_SYNC_URL`
|
||||
- AWS config (shared)
|
||||
- Database config (shared)
|
||||
- Auth config (shared)
|
||||
|
||||
### queryVersionSyncRunner
|
||||
**Unique Variables:**
|
||||
- `QUEUE_URL` = `https://sqs.us-east-2.amazonaws.com/476114113988/query_version_sync`
|
||||
|
||||
**Also Needs:**
|
||||
- `CLIENT_SYNC_URL`
|
||||
- AWS config (shared)
|
||||
- Database config (shared)
|
||||
- Auth config (shared)
|
||||
|
||||
### queryAPI (REST API Service)
|
||||
**Does NOT need `QUEUE_URL`**
|
||||
|
||||
**Needs:**
|
||||
- `CLIENT_SYNC_URL`
|
||||
- `QUERY_VERSION_SYNC_URL`
|
||||
- `BUCKET` (S3 bucket name)
|
||||
- `DEBUG` (optional, set to `true` for development)
|
||||
- AWS config (shared)
|
||||
- Database config (shared)
|
||||
- Auth config (shared)
|
||||
|
||||
## Deployment Strategy
|
||||
|
||||
### Option 1: Shared Base Configuration + Service Overrides
|
||||
|
||||
1. **Create a base environment configuration** with all shared variables (database, AWS, auth, all queue URLs)
|
||||
2. **For each service**, add only the unique `QUEUE_URL` value
|
||||
|
||||
**Example:**
|
||||
```bash
|
||||
# base-dev.env (shared by all services)
|
||||
AWS_REGION=us-east-2
|
||||
AWS_ACCESS_KEY_ID=AKIAXXXXXXXX
|
||||
AWS_SECRET_ACCESS_KEY=secret_here
|
||||
PGHOST=db-dev.example.com
|
||||
PGDATABASE=queryorchestration_dev
|
||||
# ... all queue URLs ...
|
||||
DOCUMENT_INIT_URL=https://sqs.us-east-2.amazonaws.com/476114113988/document_init
|
||||
DOCUMENT_SYNC_URL=https://sqs.us-east-2.amazonaws.com/476114113988/document_sync
|
||||
# ... etc
|
||||
|
||||
# storeEventRunner-override.env
|
||||
QUEUE_URL=https://sqs.us-east-2.amazonaws.com/476114113988/store_event
|
||||
|
||||
# docInitRunner-override.env
|
||||
QUEUE_URL=https://sqs.us-east-2.amazonaws.com/476114113988/document_init
|
||||
```
|
||||
|
||||
### Option 2: Complete Configuration Per Service
|
||||
|
||||
Copy the full environment configuration to each service and only change `QUEUE_URL`.
|
||||
|
||||
## Quick Checklist for New Environment
|
||||
|
||||
When deploying to a new environment (e.g., moving from dev to stage):
|
||||
|
||||
- [ ] Update AWS account ID in all queue URLs (e.g., `476114113988` → `123456789012`)
|
||||
- [ ] Update environment suffix in queue names if using suffixes (e.g., `store_event-dev` → `store_event-stage`)
|
||||
- [ ] Update database connection (PGHOST, PGDATABASE, PGUSER, PGPASSWORD)
|
||||
- [ ] Update AWS credentials (AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY)
|
||||
- [ ] Update Cognito configuration (if different per environment)
|
||||
- [ ] Update S3 bucket name (BUCKET)
|
||||
- [ ] Set LOG_LEVEL appropriately (DEBUG for dev, INFO for prod)
|
||||
- [ ] Set DISABLE_AUTH appropriately (true for dev, false for prod)
|
||||
- [ ] Ensure `QUEUE_URL` is unique for each runner service
|
||||
- [ ] Remove `AWS_ENDPOINT_URL` and `AWS_S3_USE_PATH_STYLE` (LocalStack only)
|
||||
|
||||
## Common Mistakes to Avoid
|
||||
|
||||
❌ **Setting a global `QUEUE_URL`** - Each runner needs its own specific input queue
|
||||
❌ **Using different queue URLs across services** - All services in same environment should see same queue URLs (except their own `QUEUE_URL`)
|
||||
❌ **Forgetting to remove LocalStack variables** - `AWS_ENDPOINT_URL` and `AWS_S3_USE_PATH_STYLE` should not be set for AWS
|
||||
❌ **Using empty string instead of unsetting** - Variables like `AWS_S3_USE_PATH_STYLE` should be completely absent, not empty
|
||||
❌ **Mixing environment credentials** - Don't use dev database with prod queues
|
||||
|
||||
## Validation
|
||||
|
||||
Before deploying, verify:
|
||||
1. Each runner has a unique `QUEUE_URL` matching its input queue
|
||||
2. All queue URLs within an environment are consistent across all services
|
||||
3. Database credentials are correct for the environment
|
||||
4. AWS credentials have appropriate permissions for SQS, S3, and Textract
|
||||
5. Cognito configuration matches the environment
|
||||
6. LocalStack-specific variables are removed for AWS deployments
|
||||
+299
@@ -0,0 +1,299 @@
|
||||
# Queue Pipeline Flow Documentation
|
||||
|
||||
## Overview
|
||||
|
||||
This document describes the SQS queue-based processing pipeline that orchestrates document processing through various stages. The system uses AWS SQS queues to connect microservices (runners) in a sequential processing pipeline.
|
||||
|
||||
## Visual Pipeline Flow
|
||||
[mermaid link](https://mermaid.live/edit#pako:eNqdlWtv2zYUhv8KwaDABiiZJOqOoUDjdFg_dGvktAVmFwVDHV0QmdQoqY1n57-PF8nxXBcYKoCASL08z7mR2mEmCsAZLlvxldVUDujues2Rel68QEuCXn8BPqClGCUDu74kq5_Uh-uRPcDw67385eWNYONGy953raDFz5_Q5eXL_WHzH2JoyobRoRF8j5a33pofCG9pw9G7poO24YBuRxihR5QXKB85B9lPyFtv1Q9CwmfQFg3UaA0J5d7OfDU4u9FIssRNvCdrIveMUwsJdFCIQjAkgQlZaI_8VTGF8LnhzbH92VWlsSh_p6RvlOiE488c33JqYA-oGLtWBw690d1v0es7WqGa9rXGkmdsv-XsLJZYLNHYpRKdYMmMNbr9B9o2heIh1jZzohjlxjwqW1ppbPCMZS1QfpYbWG6guQutOgEHMziw8WoJ6iYBE20LTJUE9YMqJpVFr8HhM3iAx_NpDi031Nw7JTrBhjPW6PavPi6RVknKrLk_Fzma5lO3Rau_R5Db72c4ssRoZ3RnchzPzMj2NauhGFtAWt9MpS1VrHNwmhpb6llgbIGxBZ7AkhlmVPvfVfomlDVWQAe8AM4U2YCODtPCFB3pENBv6jzbD6_evfFWt9qAfj2A3E9T4cyesdNtow0mK9s73ybMep9Y75OdlZ3JVzqHYKT7azqwGnVSMOhtsmjbTg2qc2bCIEdhWF8_qNOvSnguGmPWqp79Tqc6f7H7vud-at1PbfInyLdBpO4chNmwv5NNVYFEtCxVY0Nh29zEYPxPjvxfDlt1m1V2zlra9zdQ6hKOgMqmbbOLsixi13X6QYoHyC4IIdP75demGOrM7x4ddYaEzC5c1z0xJI2jk6UkZgD3P2hJX5q0mp1Kk_I-TX7QFO2aQ2zUjen_N3NkSN_zjrpp1SBqBGqEakRqxGokji6fyePxptxzct_JiZMHTh46eeTksZMnjqqcTdV_CGSO-njVNJUKwa5hB29AbmhTqP_iTq-t8VDDBtY4U6_qOntY4zV_Ujo6DkI3D84GOYKDpRirGmclbXs1s81509BK0s0s6Sj_S4jDFIpG-fPW_oTNv9jBldToyaI-7XIhRj7gzPdiz1jA2Q4_4ozEwVVCUt91Yz8KQ8938BZnl0mcXBEv8RMSxF7ok-DJwf8YpneVem4QpUkUpcSPI-_pXxutlHc)
|
||||
```mermaid
|
||||
flowchart TB
|
||||
%% S3 Event Source
|
||||
S3[(S3 Bucket<br/>Document Upload)] -->|S3 Event Notification| SQ1
|
||||
|
||||
%% Main Pipeline Queues and Runners
|
||||
SQ1[store_event<br/>Queue] --> R1{storeEventRunner<br/>:8081}
|
||||
R1 -->|Creates doc record| SQ2[document_init<br/>Queue]
|
||||
|
||||
SQ2 --> R2{docInitRunner<br/>:8082}
|
||||
R2 -->|Check duplicates<br/>by ETag hash| SQ3[document_sync<br/>Queue]
|
||||
|
||||
SQ3 --> R3{docSyncRunner<br/>:8083}
|
||||
R3 -->|Validate client<br/>can_sync flag| SQ4[document_clean<br/>Queue]
|
||||
|
||||
SQ4 --> R4{docCleanRunner<br/>:8084}
|
||||
R4 -->|Clean per<br/>collector standards| SQ5[document_text<br/>Queue]
|
||||
|
||||
SQ5 --> R5{docTextRunner<br/>:8085}
|
||||
R5 -->|AWS Textract<br/>OCR extraction| SQ6[query_sync<br/>Queue]
|
||||
|
||||
SQ6 --> R6{querySyncRunner<br/>:8087}
|
||||
R6 -->|Schedule queries<br/>for document| SQ7[query<br/>Queue]
|
||||
|
||||
SQ7 --> R7{queryRunner<br/>:8088}
|
||||
R7 -->|Handle query<br/>dependencies| SQ7
|
||||
|
||||
%% Client Sync Flow
|
||||
API1[Query API<br/>:8080] -->|Client update| SQ8[client_sync<br/>Queue]
|
||||
SQ8 --> R8{clientSyncRunner<br/>:8089}
|
||||
R8 -->|Batch process<br/>all client docs| SQ3
|
||||
|
||||
%% Query Version Sync Flow
|
||||
API1 -->|Query update| SQ9[query_version_sync<br/>Queue]
|
||||
SQ9 --> R9{queryVersionSyncRunner<br/>:8090}
|
||||
R9 -->|Trigger affected<br/>clients| SQ8
|
||||
|
||||
%% Styling
|
||||
classDef queue fill:#ffd700,stroke:#333,stroke-width:2px,color:#000
|
||||
classDef runner fill:#87ceeb,stroke:#333,stroke-width:2px,color:#000
|
||||
classDef storage fill:#98fb98,stroke:#333,stroke-width:2px,color:#000
|
||||
classDef api fill:#ffa07a,stroke:#333,stroke-width:2px,color:#000
|
||||
|
||||
class SQ1,SQ2,SQ3,SQ4,SQ5,SQ6,SQ7,SQ8,SQ9 queue
|
||||
class R1,R2,R3,R4,R5,R6,R7,R8,R9 runner
|
||||
class S3 storage
|
||||
class API1 api
|
||||
```
|
||||
|
||||
### Flow Legend
|
||||
- 🟨 **Yellow boxes**: SQS Queues
|
||||
- 🟦 **Blue diamonds**: Runner Services (with port numbers)
|
||||
- 🟩 **Green cylinder**: S3 Storage
|
||||
- 🟧 **Orange box**: REST API endpoint
|
||||
- **Solid arrows**: Message flow direction
|
||||
- **Self-loop on queryRunner**: Handles query dependencies by re-queueing
|
||||
|
||||
## Detailed Process Flow
|
||||
[mermaid link](https://mermaid.live/edit#pako:eNp9lW9v2jAQxr-K5VfblFaEtCvkRaU1MAlp0xoSadKEVBnnAI_EoY4zlSK--84OgQYSkJAc--fnnjv_21GeJ0B9OpMFvJYgOYwEWyqWzSTB34YpLbjYMKlJ5LX0jQkrSKFzBS_wD7AnLKGENnB6JMcGnJZSgroERxPDJTkvM6RehBSdmqPJ9MBOkOoUjBqCxVbyTsGoFoyQ6hQMGoI8BSY7FYNaMTBYp2TckNTw1p10XCvGSHUJhjZpXFC1vZpxWGVswWs5h0eqU-kk1Jnlk0Ge80IvFUThj5msmMi7eXyMxj42iN0b5NOv-V_gOlDANCT-l88HcGzBqY8iaUoyKAq2hHpsioOjJ58EK-BrIhZEZJtcabSgVw1mgpFAJqaGRMhFTnbzkq9BO2QNW4esWLHa195GEzujNSR2N0Im5SYVHB2T-daqXGA2n-M6EwU8V0mDig7edjU0GZ28RJZo9xI1vfBUmAicSbv-DShoiWHqBalYinkKx3iBpdvjBRflZimml2yJPRDn1DOoRa6yalDIZWM8vpZ0bIl2E_GFCXt08K8Yx51zwmqRgKHIt98RiQ_QuVJkbqiTghX8yITXViiMLNFqNjyu0HeB09nGbhYstz00AoqPWHgZxKkOl1krrCQBxlfHsGZGe9CzHQobVMVb_hSuBsZvwEtdmdmejVU1UVCUqT7Nq2xO4ebV3Aim-B_liWniQlOHZqAyJhJ8ZXZm8ozqFWQwoz42E6bWM3x99sixUufmEqK-ViU4VOXlckX9BUsL_Co3CR6ew-t07MWL5U-eZ_UUSAR6_Vm9afZpc-hSmdgHSeNPBXkpNfX77p0VoP6OvlHf7bm3D3f9njcY9j13eN936Nb09m693sD1BgPPux_2eoO9Q99tyN7tYOC6_Ydhf-h-7WPjYf8f7gRf3A)
|
||||
```mermaid
|
||||
sequenceDiagram
|
||||
participant S3
|
||||
participant SE as store_event Queue
|
||||
participant SER as storeEventRunner
|
||||
participant DI as document_init Queue
|
||||
participant DIR as docInitRunner
|
||||
participant DS as document_sync Queue
|
||||
participant DSR as docSyncRunner
|
||||
participant DC as document_clean Queue
|
||||
participant DCR as docCleanRunner
|
||||
participant DT as document_text Queue
|
||||
participant DTR as docTextRunner
|
||||
participant QS as query_sync Queue
|
||||
participant QSR as querySyncRunner
|
||||
participant Q as query Queue
|
||||
participant QR as queryRunner
|
||||
participant DB as PostgreSQL
|
||||
|
||||
S3->>SE: S3 Event (ObjectCreated:*)
|
||||
SE->>SER: Poll message
|
||||
SER->>DB: Check if import path
|
||||
SER->>DI: Send doc info {bucket, key, hash}
|
||||
|
||||
DI->>DIR: Poll message
|
||||
DIR->>DB: Check duplicate by hash
|
||||
DIR->>DB: Create document record
|
||||
DIR->>DS: Send {documentID}
|
||||
|
||||
DS->>DSR: Poll message
|
||||
DSR->>DB: Check client can_sync
|
||||
DSR->>DC: Send {documentID} if eligible
|
||||
|
||||
DC->>DCR: Poll message
|
||||
DCR->>DB: Check if already clean
|
||||
DCR->>DB: Perform cleaning
|
||||
DCR->>DT: Send {documentID}
|
||||
|
||||
DT->>DTR: Poll message
|
||||
DTR->>DB: Check if text extracted
|
||||
DTR-->>DTR: Call AWS Textract
|
||||
DTR->>DB: Store extracted text
|
||||
DTR->>QS: Send {documentID}
|
||||
|
||||
QS->>QSR: Poll message
|
||||
QSR->>DB: Find applicable queries
|
||||
QSR->>Q: Send {documentID, queryID} for each
|
||||
|
||||
Q->>QR: Poll message
|
||||
QR->>DB: Check dependencies
|
||||
QR->>DB: Execute query
|
||||
QR->>DB: Store results
|
||||
QR-->>Q: Re-queue if dependencies pending
|
||||
```
|
||||
|
||||
## S3 Event Trigger Configuration
|
||||
|
||||
### AWS Service Type
|
||||
**S3 Event Notifications** (also known as S3 Bucket Notifications)
|
||||
|
||||
### Configuration Requirements
|
||||
|
||||
The pipeline is initiated by S3 Event Notifications configured to send messages directly to an SQS queue:
|
||||
|
||||
```json
|
||||
{
|
||||
"QueueConfigurations": [{
|
||||
"QueueArn": "arn:aws:sqs:{region}:{account-id}:store_event",
|
||||
"Events": ["s3:ObjectCreated:*"],
|
||||
"Filter": {
|
||||
"Key": {
|
||||
"FilterRules": [
|
||||
{"Name": "prefix", "Value": "documents/"}
|
||||
]
|
||||
}
|
||||
}
|
||||
}]
|
||||
}
|
||||
```
|
||||
|
||||
**Important**: Files under the `batches/` path should be excluded from triggering the pipeline as they are temporary files. This can be accomplished through:
|
||||
1. Prefix filters in S3 configuration (specify allowed prefixes)
|
||||
2. Application-level filtering in `storeEventRunner` (recommended)
|
||||
|
||||
### AWS CLI Setup Command
|
||||
```bash
|
||||
aws s3api put-bucket-notification-configuration \
|
||||
--bucket {BUCKET_NAME} \
|
||||
--notification-configuration '{
|
||||
"QueueConfigurations": [{
|
||||
"QueueArn": "arn:aws:sqs:{region}:{account-id}:store_event",
|
||||
"Events": ["s3:ObjectCreated:*"]
|
||||
}]
|
||||
}'
|
||||
```
|
||||
|
||||
## Queue Pipeline Flow Tables
|
||||
|
||||
### Main Document Processing Pipeline
|
||||
|
||||
| Order | Runner Service | Reads From Queue | Writes To Queue | Description |
|
||||
|-------|---------------|------------------|-----------------|-------------|
|
||||
| **1** | `storeEventRunner` | `store_event` | `document_init` | Receives S3 event notifications when files are uploaded to the bucket |
|
||||
| **2** | `docInitRunner` | `document_init` | `document_sync` | Creates document records, checks for duplicates based on ETag hash |
|
||||
| **3** | `docSyncRunner` | `document_sync` | `document_clean` | Validates client eligibility for document processing (can_sync flag) |
|
||||
| **4** | `docCleanRunner` | `document_clean` | `document_text` | Cleans document content according to collector standards |
|
||||
| **5** | `docTextRunner` | `document_text` | `query_sync` | Extracts text using AWS Textract OCR service |
|
||||
| **6** | `querySyncRunner` | `query_sync` | `query` | Syncs and schedules queries that need to run for documents |
|
||||
| **7** | `queryRunner` | `query` | `query` (self-referential) | Executes queries with dependency chains, can re-queue for dependencies |
|
||||
|
||||
### Additional Async Runners
|
||||
|
||||
These runners are triggered by API calls or other events outside the main pipeline:
|
||||
|
||||
| Runner Service | Reads From Queue | Writes To Queue | Trigger |
|
||||
|---------------|------------------|-----------------|---------|
|
||||
| `clientSyncRunner` | `client_sync` | `document_sync` | Triggered by API calls or when client configuration changes |
|
||||
| `queryVersionSyncRunner` | `query_version_sync` | `client_sync` | Triggered when query versions are updated via API |
|
||||
|
||||
## Pipeline Flow Diagrams
|
||||
|
||||
### 1. Document Upload Flow (Main Pipeline)
|
||||
```
|
||||
S3 Upload Event
|
||||
↓
|
||||
[store_event queue]
|
||||
↓
|
||||
storeEventRunner
|
||||
↓
|
||||
[document_init queue]
|
||||
↓
|
||||
docInitRunner
|
||||
↓
|
||||
[document_sync queue]
|
||||
↓
|
||||
docSyncRunner
|
||||
↓
|
||||
[document_clean queue]
|
||||
↓
|
||||
docCleanRunner
|
||||
↓
|
||||
[document_text queue]
|
||||
↓
|
||||
docTextRunner
|
||||
↓
|
||||
[query_sync queue]
|
||||
↓
|
||||
querySyncRunner
|
||||
↓
|
||||
[query queue]
|
||||
↓
|
||||
queryRunner
|
||||
```
|
||||
|
||||
### 2. Client Sync Flow (Batch Processing)
|
||||
```
|
||||
Client Update (API)
|
||||
↓
|
||||
[client_sync queue]
|
||||
↓
|
||||
clientSyncRunner
|
||||
↓
|
||||
[document_sync queue] → (feeds into main pipeline)
|
||||
```
|
||||
|
||||
### 3. Query Version Update Flow
|
||||
```
|
||||
Query Update (API)
|
||||
↓
|
||||
[query_version_sync queue]
|
||||
↓
|
||||
queryVersionSyncRunner
|
||||
↓
|
||||
[client_sync queue]
|
||||
↓
|
||||
clientSyncRunner
|
||||
↓
|
||||
[document_sync queue] → (feeds into main pipeline)
|
||||
```
|
||||
|
||||
## Queue Names and Environment Variables
|
||||
|
||||
| Queue Purpose | Queue Name Variable | Queue URL Variable |
|
||||
|--------------|-------------------|-------------------|
|
||||
| S3 Events | `QNAME_STORE_EVENT` | `STORE_EVENT_URL` |
|
||||
| Document Init | `QNAME_DOCUMENT_INIT` | `DOCUMENT_INIT_URL` |
|
||||
| Document Sync | `QNAME_DOCUMENT_SYNC` | `DOCUMENT_SYNC_URL` |
|
||||
| Document Clean | `QNAME_DOCUMENT_CLEAN` | `DOCUMENT_CLEAN_URL` |
|
||||
| Document Text | `QNAME_DOCUMENT_TEXT` | `DOCUMENT_TEXT_URL` |
|
||||
| Query Sync | `QNAME_QUERY_SYNC` | `QUERY_SYNC_URL` |
|
||||
| Query Runner | `QNAME_QUERY_RUNNER` | `QUERY_URL` |
|
||||
| Client Sync | `QNAME_CLIENT_SYNC` | `CLIENT_SYNC_URL` |
|
||||
| Query Version Sync | `QNAME_QUERY_VERSION_SYNC` | `QUERY_VERSION_SYNC_URL` |
|
||||
|
||||
## Key Design Principles
|
||||
|
||||
1. **Single Responsibility**: Each runner has one specific task in the pipeline
|
||||
2. **Sequential Processing**: Documents flow through defined stages in order
|
||||
3. **Idempotent Operations**: Duplicate detection prevents reprocessing
|
||||
4. **Async Reprocessing**: Client/query updates trigger document reprocessing
|
||||
5. **Self-Referential Queuing**: Query runner can re-queue items for dependency handling
|
||||
6. **Scalability**: Each runner can be scaled independently based on workload
|
||||
|
||||
## IAM Permissions Required
|
||||
|
||||
For the S3-to-SQS integration:
|
||||
- S3 bucket must have permission to send messages to the SQS queue
|
||||
- SQS queue policy must allow the S3 service principal to send messages
|
||||
- Each runner service needs permissions to:
|
||||
- Receive and delete messages from its input queue
|
||||
- Send messages to its output queue(s)
|
||||
- Access S3 buckets for document operations
|
||||
- Access AWS Textract (for docTextRunner)
|
||||
|
||||
## Monitoring and Debugging
|
||||
|
||||
Each runner exposes Prometheus metrics on port 8080 and includes:
|
||||
- Queue processing metrics
|
||||
- Error rates
|
||||
- Processing duration
|
||||
- Health check endpoints
|
||||
|
||||
Runner ports for local development:
|
||||
- `storeEventRunner`: 8081
|
||||
- `docInitRunner`: 8082
|
||||
- `docSyncRunner`: 8083
|
||||
- `docCleanRunner`: 8084
|
||||
- `docTextRunner`: 8085
|
||||
- `querySyncRunner`: 8087
|
||||
- `queryRunner`: 8088
|
||||
- `clientSyncRunner`: 8089
|
||||
- `queryVersionSyncRunner`: 8090
|
||||
@@ -5,12 +5,12 @@ info:
|
||||
description: API documentation for the Query Orchestration services.
|
||||
version: 0.0.1
|
||||
contact:
|
||||
email: mmcguinness@aarete.com
|
||||
name: Michael McGuinness
|
||||
url: doczy.com
|
||||
email: piragavarapu@aarete.com
|
||||
name: Aarete Inc.
|
||||
url: https://aarete.com
|
||||
license:
|
||||
name: unlicensed
|
||||
url: "www.example.com"
|
||||
name: Aarete Inc.internal use
|
||||
url: https://aarete.com
|
||||
servers:
|
||||
- url: https://doczy.com/v1
|
||||
description: Production server
|
||||
|
||||
Reference in New Issue
Block a user