diff --git a/.gomarkdoc.yaml b/.gomarkdoc.yaml new file mode 100644 index 00000000..fd33a6f0 --- /dev/null +++ b/.gomarkdoc.yaml @@ -0,0 +1,12 @@ +--- +output: "docs/README.md" +repository: + url: "https://bitbucket.org/aarete/query-orchestration" + defaultBranch: "main" + path: "/" +includeUnexported: false +excludeDirs: + - "./vendor/..." + - "./mocks/..." + - "./test/..." + - "./docs/..." diff --git a/cmd/metricsExample_test/main.go b/cmd/metricsExample_test/main.go index 74a31141..346756a8 100644 --- a/cmd/metricsExample_test/main.go +++ b/cmd/metricsExample_test/main.go @@ -1,9 +1,15 @@ // This main.go serves as an example of how to use the custom Prometheus metrics package. +// // It demonstrates: +// // 1. Setting up metrics with proper context handling +// // 2. Configuring different types of metrics (Counter, Gauge, Histogram, Summary) +// // 3. Recording metrics safely in a concurrent environment +// // 4. Implementing graceful shutdown +// // 5. Proper error handling package main diff --git a/cmd/queryService/main.go b/cmd/queryService/main.go index 422ad23b..e5c1f6ca 100644 --- a/cmd/queryService/main.go +++ b/cmd/queryService/main.go @@ -1,3 +1,18 @@ +// This API currently manages all user entities to be managed. This allows users of the service to change required entities and processes. +// +// The entities in question: +// +// - Client - A client is a single client of the Doczy project, all configurations and jobs for a client will be in reference to the respective client. +// +// - Job - A job is a single instance of requirements for a client. This maintains a set of queries and processes in relation to itself. +// +// - Collector - A collector specifies the required queries for export for a given job. +// +// - Sample - A sample is a subset of documents to be used for testing. When it is active a job will only process documents in the sample. +// +// - Query - A query is a block of processing of data which takes an input and produces an output. +// +// - Export - An export is a manually started process which outputs the results for a job to a given location. package main import ( diff --git a/devbox.json b/devbox.json index 9acc17a8..76f0f2e6 100644 --- a/devbox.json +++ b/devbox.json @@ -1,36 +1,36 @@ { "$schema": "https://raw.githubusercontent.com/jetify-com/devbox/0.13.7/.schema/devbox.schema.json", "env": { - "APP_ENV": "development", - "AWS_ACCESS_KEY_ID": "test", - "AWS_ENDPOINT_URL": "http://localhost:4566", - "AWS_REGION": "us-east-1", - "AWS_SECRET_ACCESS_KEY": "test", - "AWS_SESSION_TOKEN": "", - "BUCKET_IN": "documentin", - "DB_NOSSL": "true", - "DOCUMENT_CLEAN_URL": "http://localstack:4566/queue/us-east-1/000000000000/document_clean", - "DOCUMENT_INIT_URL": "http://localstack:4566/queue/us-east-1/000000000000/document_init", - "DOCUMENT_SYNC_URL": "http://localstack:4566/queue/us-east-1/000000000000/document_sync", - "DOCUMENT_TEXT_URL": "http://localstack:4566/queue/us-east-1/000000000000/document_text", - "JOB_SYNC_URL": "http://localstack:4566/queue/us-east-1/000000000000/job_sync", - "PGDATABASE": "query_orchestration", - "PGHOST": "localhost", - "PGPASSWORD": "pass", - "PGPORT": "5432", - "PGPORT_GENERATE": "5431", - "PGUSER": "postgres", - "QNAME_DOCUMENT_CLEAN": "document_clean", - "QNAME_DOCUMENT_INIT": "document_init", - "QNAME_DOCUMENT_SYNC": "document_sync", - "QNAME_DOCUMENT_TEXT": "document_text", - "QNAME_JOB_SYNC": "job_sync", - "QNAME_QUERY_RUNNER": "query_runner", - "QNAME_QUERY_SYNC": "query_sync", + "APP_ENV": "development", + "AWS_ACCESS_KEY_ID": "test", + "AWS_ENDPOINT_URL": "http://localhost:4566", + "AWS_REGION": "us-east-1", + "AWS_SECRET_ACCESS_KEY": "test", + "AWS_SESSION_TOKEN": "", + "BUCKET_IN": "documentin", + "DB_NOSSL": "true", + "DOCUMENT_CLEAN_URL": "http://localstack:4566/queue/us-east-1/000000000000/document_clean", + "DOCUMENT_INIT_URL": "http://localstack:4566/queue/us-east-1/000000000000/document_init", + "DOCUMENT_SYNC_URL": "http://localstack:4566/queue/us-east-1/000000000000/document_sync", + "DOCUMENT_TEXT_URL": "http://localstack:4566/queue/us-east-1/000000000000/document_text", + "JOB_SYNC_URL": "http://localstack:4566/queue/us-east-1/000000000000/job_sync", + "PGDATABASE": "query_orchestration", + "PGHOST": "localhost", + "PGPASSWORD": "pass", + "PGPORT": "5432", + "PGPORT_GENERATE": "5431", + "PGUSER": "postgres", + "QNAME_DOCUMENT_CLEAN": "document_clean", + "QNAME_DOCUMENT_INIT": "document_init", + "QNAME_DOCUMENT_SYNC": "document_sync", + "QNAME_DOCUMENT_TEXT": "document_text", + "QNAME_JOB_SYNC": "job_sync", + "QNAME_QUERY_RUNNER": "query_runner", + "QNAME_QUERY_SYNC": "query_sync", "QNAME_QUERY_VERSION_SYNC": "query_version_sync", - "QUERY_SYNC_URL": "http://localstack:4566/queue/us-east-1/000000000000/query_sync", - "QUERY_URL": "http://localstack:4566/queue/us-east-1/000000000000/query_runner", - "QUERY_VERSION_SYNC_URL": "http://localstack:4566/queue/us-east-1/000000000000/query_version_sync" + "QUERY_SYNC_URL": "http://localstack:4566/queue/us-east-1/000000000000/query_sync", + "QUERY_URL": "http://localstack:4566/queue/us-east-1/000000000000/query_runner", + "QUERY_VERSION_SYNC_URL": "http://localstack:4566/queue/us-east-1/000000000000/query_version_sync" }, "env_from": ".env", "packages": [ @@ -52,7 +52,8 @@ "vacuum-go@0.14.1", "awscli2@2.19.0", "postgresql@17.2", - "bc@1.07.1" + "bc@1.07.1", + "gomarkdoc@1.1.0" ], "shell": { "init_hook": [ @@ -61,4 +62,4 @@ "echo 'Welcome to the DoczyAI devbox!'" ] } -} \ No newline at end of file +} diff --git a/devbox.lock b/devbox.lock index fccc6521..9057d7db 100644 --- a/devbox.lock +++ b/devbox.lock @@ -353,6 +353,54 @@ } } }, + "gomarkdoc@1.1.0": { + "last_modified": "2025-02-23T09:42:26Z", + "resolved": "github:NixOS/nixpkgs/2d068ae5c6516b2d04562de50a58c682540de9bf#gomarkdoc", + "source": "devbox-search", + "version": "1.1.0", + "systems": { + "aarch64-darwin": { + "outputs": [ + { + "name": "out", + "path": "/nix/store/6144v1asaj7sflv02h5n1250q91b1zzc-gomarkdoc-1.1.0", + "default": true + } + ], + "store_path": "/nix/store/6144v1asaj7sflv02h5n1250q91b1zzc-gomarkdoc-1.1.0" + }, + "aarch64-linux": { + "outputs": [ + { + "name": "out", + "path": "/nix/store/qvnkraphy556zc9w0x7xkji9i8ban0wv-gomarkdoc-1.1.0", + "default": true + } + ], + "store_path": "/nix/store/qvnkraphy556zc9w0x7xkji9i8ban0wv-gomarkdoc-1.1.0" + }, + "x86_64-darwin": { + "outputs": [ + { + "name": "out", + "path": "/nix/store/9yaqabqln9fq39win7gf50yqqpq5144z-gomarkdoc-1.1.0", + "default": true + } + ], + "store_path": "/nix/store/9yaqabqln9fq39win7gf50yqqpq5144z-gomarkdoc-1.1.0" + }, + "x86_64-linux": { + "outputs": [ + { + "name": "out", + "path": "/nix/store/irxxjyw5pid830n5xvd1rf9fajgq15rl-gomarkdoc-1.1.0", + "default": true + } + ], + "store_path": "/nix/store/irxxjyw5pid830n5xvd1rf9fajgq15rl-gomarkdoc-1.1.0" + } + } + }, "gotools@0.25.0": { "last_modified": "2024-12-23T21:10:33Z", "resolved": "github:NixOS/nixpkgs/de1864217bfa9b5845f465e771e0ecb48b30e02d#gotools", diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 00000000..68d5ee2f --- /dev/null +++ b/docs/README.md @@ -0,0 +1,53 @@ +# Query Orchestration + +## docCleanRunner + +## docInitRunner + +## docSyncRunner + +## docTextRunner + +## jobSyncRunner + +## metricsExample\_test + +This main.go serves as an example of how to use the custom Prometheus metrics package. + +It demonstrates: + +1. Setting up metrics with proper context handling + +2. Configuring different types of metrics (Counter, Gauge, Histogram, Summary) + +3. Recording metrics safely in a concurrent environment + +4. Implementing graceful shutdown + +5. Proper error handling + +## queryRunner + +## queryService + +This API currently manages all user entities to be managed. This allows users of the service to change required entities and processes. + +The entities in question: + +- Client - A client is a single client of the Doczy project, all configurations and jobs for a client will be in reference to the respective client. + +- Job - A job is a single instance of requirements for a client. This maintains a set of queries and processes in relation to itself. + +- Collector - A collector specifies the required queries for export for a given job. + +- Sample - A sample is a subset of documents to be used for testing. When it is active a job will only process documents in the sample. + +- Query - A query is a block of processing of data which takes an input and produces an output. + +- Export - An export is a manually started process which outputs the results for a job to a given location. + +## querySyncRunner + +## queryVersionSyncRunner + +Generated by Doczy diff --git a/docs/templates/root.gotxt b/docs/templates/root.gotxt new file mode 100644 index 00000000..4c369d29 --- /dev/null +++ b/docs/templates/root.gotxt @@ -0,0 +1,40 @@ +{{if .Header -}} + {{- .Header -}} + {{- spacer -}} +{{- end -}} + +{{- header 1 "Query Orchestration" -}} +{{- spacer -}} + +{{- range .Packages -}} + {{- if eq .Name "main" -}} + {{- header 2 .Dirname -}} + {{- else -}} + {{- header 2 .Name -}} + {{- end -}} + {{- spacer -}} + + {{- range (iter .Doc.Blocks) -}} + {{- if eq .Entry.Kind "paragraph" -}} + {{- range .Entry.Spans -}} + {{- if eq .Kind "text" -}} + {{- .Text -}} + {{- else if eq .Kind "rawText" -}} + {{- .Text -}} + {{- else if eq .Kind "autolink" -}} + {{- .Text -}} + {{- else if eq .Kind "link" -}} + {{- link (escape .Text) .URL -}} + {{- end -}} + {{- end -}} + {{- end -}} + {{- spacer -}} + {{- end -}} +{{- end -}} + +{{- if .Footer -}} + {{- .Footer -}} + {{- spacer -}} +{{- end -}} + +Generated by Doczy diff --git a/scripts/Taskfile.yml b/scripts/Taskfile.yml index d7356692..61e5d350 100644 --- a/scripts/Taskfile.yml +++ b/scripts/Taskfile.yml @@ -53,11 +53,13 @@ tasks: - db:generate:clean - openapi:generate - test:mocks:generate + - docs:generate generate: deps: - db:generate - openapi:generate - test:mocks:generate + - docs:generate lint: deps: - generate @@ -96,6 +98,6 @@ tasks: json:lint:fix: cmds: - jsonlint devbox.json -q -s -i - docs: + docs:generate: cmds: - - godoc -http=:6060 + - gomarkdoc --template-file file=docs/templates/root.gotxt ./cmd/...