diff --git a/cmd/docCleanRunner/main.go b/cmd/docCleanRunner/main.go index f64cab34..1edb66d2 100644 --- a/cmd/docCleanRunner/main.go +++ b/cmd/docCleanRunner/main.go @@ -1,3 +1,10 @@ +// **Listens For**: Body Containing Document ID. +// +// **Action**. +// +// If the document is not already clean according to the job collector standards, a clean is triggered. +// +// An event with the document id is pushed to the DOCTEXT queue. package main import ( diff --git a/cmd/docInitRunner/main.go b/cmd/docInitRunner/main.go index 9b20b4f8..2f2e43f7 100644 --- a/cmd/docInitRunner/main.go +++ b/cmd/docInitRunner/main.go @@ -1,3 +1,18 @@ +// **Listens For**: S3 Event notifications - limited to PUT notifications. +// +// **Action**. +// +// Checks if the uploaded document is a duplicate. +// +// Currently whether the document content is a duplicate or not is determines using the document ETag, and check if it already exists for the job. +// +// Find reference entity. +// +// If NOT a duplicate - Create a document reference entity. +// +// Create a document entry to log the successful upload of a document. +// +// Add an event to the DOC SYNC queue. package main import ( diff --git a/cmd/docSyncRunner/main.go b/cmd/docSyncRunner/main.go index 305f1cad..8b156ee3 100644 --- a/cmd/docSyncRunner/main.go +++ b/cmd/docSyncRunner/main.go @@ -1,3 +1,12 @@ +// **Listens For**: Body Containing Document ID. +// +// **Action**. +// +// Checks if the document is eligible for syncing. +// +// By validating: the client can_sync parameter, the job can_sync parameter. +// +// If eligible, send an event to the DOCCLEAN queue. package main import ( diff --git a/cmd/docTextRunner/main.go b/cmd/docTextRunner/main.go index 6ad17191..d8dd3d01 100644 --- a/cmd/docTextRunner/main.go +++ b/cmd/docTextRunner/main.go @@ -1,3 +1,10 @@ +// **Listens For**: Body Containing Document ID +// +// **Action**. +// +// If the document text is not already extracted according to the job collector standards, the text is extracted. +// +// An event with the document id is pushed to the QUERYSYNC queue. package main import ( diff --git a/cmd/jobSyncRunner/main.go b/cmd/jobSyncRunner/main.go index 3e26e152..19f1c2d2 100644 --- a/cmd/jobSyncRunner/main.go +++ b/cmd/jobSyncRunner/main.go @@ -1,3 +1,8 @@ +// **Listens For**: Body Containing Job ID +// +// **Action**. +// +// Adds an event for each document into the DOCSYNC queue for the given Job. package main import ( diff --git a/cmd/queryRunner/main.go b/cmd/queryRunner/main.go index d0672cd0..fa1b0e47 100644 --- a/cmd/queryRunner/main.go +++ b/cmd/queryRunner/main.go @@ -1,3 +1,10 @@ +// **Listens For**: Body Containing Document ID and Query ID +// +// **Action**. +// +// Run the query for the given document and store the result. +// +// Events with the document id and downstream query ids are pushed to the QUERY queue. package main import ( diff --git a/cmd/querySyncRunner/main.go b/cmd/querySyncRunner/main.go index 90569629..f773b42f 100644 --- a/cmd/querySyncRunner/main.go +++ b/cmd/querySyncRunner/main.go @@ -1,3 +1,18 @@ +// **Listens For**: Body Containing Document ID +// +// **Action**. +// +// Find the queries with no dependencies that do not have a valid result for a document. +// +// The reasoning for only extracting these is to start the query extraction process only for necessary queries, and the query runner will manage updating downstream dependencies. +// +// There are 2 situations that cause a query to show up in this list: +// +// The query has no required dependencies. +// +// All required dependencies have a valid result. +// +// An event with the document id is pushed to the DOCTEXT queue. package main import ( diff --git a/cmd/queryVersionSyncRunner/main.go b/cmd/queryVersionSyncRunner/main.go index 0f07bb90..f775d523 100644 --- a/cmd/queryVersionSyncRunner/main.go +++ b/cmd/queryVersionSyncRunner/main.go @@ -1,3 +1,8 @@ +// **Listens For**: Body Containing Query ID +// +// **Action**. +// +// Adds an event for each job into the JOBSYNC queue for the given query, i.e. each job that contains the query in its dependency tree. package main import ( diff --git a/docs/README.md b/docs/README.md index 68d5ee2f..fa2c5bcd 100644 --- a/docs/README.md +++ b/docs/README.md @@ -2,14 +2,62 @@ ## docCleanRunner +**Listens For**: Body Containing Document ID. + +**Action**. + +If the document is not already clean according to the job collector standards, a clean is triggered. + +An event with the document id is pushed to the DOCTEXT queue. + ## docInitRunner +**Listens For**: S3 Event notifications - limited to PUT notifications. + +**Action**. + +Checks if the uploaded document is a duplicate. + +Currently whether the document content is a duplicate or not is determines using the document ETag, and check if it already exists for the job. + +Find reference entity. + +If NOT a duplicate - Create a document reference entity. + +Create a document entry to log the successful upload of a document. + +Add an event to the DOC SYNC queue. + ## docSyncRunner +**Listens For**: Body Containing Document ID. + +**Action**. + +Checks if the document is eligible for syncing. + +By validating: the client can_sync parameter, the job can_sync parameter. + +If eligible, send an event to the DOCCLEAN queue. + ## docTextRunner +**Listens For**: Body Containing Document ID + +**Action**. + +If the document text is not already extracted according to the job collector standards, the text is extracted. + +An event with the document id is pushed to the QUERYSYNC queue. + ## jobSyncRunner +**Listens For**: Body Containing Job ID + +**Action**. + +Adds an event for each document into the DOCSYNC queue for the given Job. + ## metricsExample\_test This main.go serves as an example of how to use the custom Prometheus metrics package. @@ -28,6 +76,14 @@ It demonstrates: ## queryRunner +**Listens For**: Body Containing Document ID and Query ID + +**Action**. + +Run the query for the given document and store the result. + +Events with the document id and downstream query ids are pushed to the QUERY queue. + ## queryService This API currently manages all user entities to be managed. This allows users of the service to change required entities and processes. @@ -48,6 +104,28 @@ The entities in question: ## querySyncRunner +**Listens For**: Body Containing Document ID + +**Action**. + +Find the queries with no dependencies that do not have a valid result for a document. + +The reasoning for only extracting these is to start the query extraction process only for necessary queries, and the query runner will manage updating downstream dependencies. + +There are 2 situations that cause a query to show up in this list: + +The query has no required dependencies. + +All required dependencies have a valid result. + +An event with the document id is pushed to the DOCTEXT queue. + ## queryVersionSyncRunner +**Listens For**: Body Containing Query ID + +**Action**. + +Adds an event for each job into the JOBSYNC queue for the given query, i.e. each job that contains the query in its dependency tree. + Generated by Doczy