Merged in feature/jobsync (pull request #63)

Document Sync and Job Sync

* docsyncfunc

* startedQueryWork

* morefixes

* jobsyncsvcstart

* save

* jobsynctext

* save

* save

* docsync

* shorttest

* jobsync

* jobsyncupdateoncollectorupdate

* passlivetest

* collectortest

* lint

internal/database/repository/job.sql.go

@@ -79,3 +79,41 @@ func (q *Queries) GetJob(ctx context.Context, jobid pgtype.UUID) (*GetJobRow, er
 	err := row.Scan(&i.ID, &i.Clientid, &i.Cansync)
 	return &i, err
 }
+
+const listJobDocumentIDsBatch = `-- name: ListJobDocumentIDsBatch :many
+SELECT id, totalCount FROM listJobDocumentIDs($1, $2, $3)
+`
+
+type ListJobDocumentIDsBatchParams struct {
+	Jobid      pgtype.UUID `db:"jobid"`
+	Batchsize  int32       `db:"batchsize"`
+	Pageoffset int32       `db:"pageoffset"`
+}
+
+type ListJobDocumentIDsBatchRow struct {
+	ID         pgtype.UUID `db:"id"`
+	Totalcount *int64      `db:"totalcount"`
+}
+
+// ListJobDocumentIDsBatch
+//
+//	SELECT id, totalCount FROM listJobDocumentIDs($1, $2, $3)
+func (q *Queries) ListJobDocumentIDsBatch(ctx context.Context, arg *ListJobDocumentIDsBatchParams) ([]*ListJobDocumentIDsBatchRow, error) {
+	rows, err := q.db.Query(ctx, listJobDocumentIDsBatch, arg.Jobid, arg.Batchsize, arg.Pageoffset)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	items := []*ListJobDocumentIDsBatchRow{}
+	for rows.Next() {
+		var i ListJobDocumentIDsBatchRow
+		if err := rows.Scan(&i.ID, &i.Totalcount); err != nil {
+			return nil, err
+		}
+		items = append(items, &i)
+	}
+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+	return items, nil
+}

internal/database/repository/job_test.go

@@ -58,3 +58,109 @@ func TestJob(t *testing.T) {
 		Cansync:  true,
 	}, job)
 }
+func TestListJobDocumentIDs(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping long test in short mode")
+	}
+	ctx := context.Background()
+
+	cfg := &serviceconfig.BaseConfig{}
+	test.SetCfgProvider(t, cfg)
+	cfg.SetBasePath(path.Join(os.Getenv("PWD"), "../../.."))
+	_, cleanup := test.CreateDB(t, ctx, &test.CreateDatabaseConfig{
+		Cfg:           cfg,
+		RunMigrations: true,
+	})
+	defer cleanup()
+
+	queries := cfg.GetDBQueries()
+
+	clientId, err := queries.CreateClient(ctx, "example_client")
+	assert.NoError(t, err)
+
+	id, err := queries.CreateJob(ctx, clientId)
+	assert.NoError(t, err)
+	assert.NotEmpty(t, id)
+
+	ids, err := queries.ListJobDocumentIDsBatch(ctx, &repository.ListJobDocumentIDsBatchParams{
+		Jobid:      id,
+		Batchsize:  1,
+		Pageoffset: 0,
+	})
+	assert.NoError(t, err)
+	assert.Len(t, ids, 0)
+
+	docOne, err := queries.CreateDocument(ctx, &repository.CreateDocumentParams{
+		Jobid: id,
+		Hash:  "example_hash",
+	})
+	assert.NoError(t, err)
+
+	ids, err = queries.ListJobDocumentIDsBatch(ctx, &repository.ListJobDocumentIDsBatchParams{
+		Jobid:      id,
+		Batchsize:  1,
+		Pageoffset: 0,
+	})
+	assert.NoError(t, err)
+	assert.Len(t, ids, 1)
+	total := int64(1)
+	assert.ElementsMatch(t, []*repository.ListJobDocumentIDsBatchRow{
+		{
+			ID:         docOne,
+			Totalcount: &total,
+		},
+	}, ids)
+
+	docTwo, err := queries.CreateDocument(ctx, &repository.CreateDocumentParams{
+		Jobid: id,
+		Hash:  "example_hash_two",
+	})
+	assert.NoError(t, err)
+
+	ids, err = queries.ListJobDocumentIDsBatch(ctx, &repository.ListJobDocumentIDsBatchParams{
+		Jobid:      id,
+		Batchsize:  1,
+		Pageoffset: 0,
+	})
+	assert.NoError(t, err)
+	assert.Len(t, ids, 1)
+	total = int64(2)
+	assert.ElementsMatch(t, []*repository.ListJobDocumentIDsBatchRow{
+		{
+			ID:         docOne,
+			Totalcount: &total,
+		},
+	}, ids)
+
+	ids, err = queries.ListJobDocumentIDsBatch(ctx, &repository.ListJobDocumentIDsBatchParams{
+		Jobid:      id,
+		Batchsize:  1,
+		Pageoffset: 1,
+	})
+	assert.NoError(t, err)
+	assert.Len(t, ids, 1)
+	assert.ElementsMatch(t, []*repository.ListJobDocumentIDsBatchRow{
+		{
+			ID:         docTwo,
+			Totalcount: &total,
+		},
+	}, ids)
+
+	ids, err = queries.ListJobDocumentIDsBatch(ctx, &repository.ListJobDocumentIDsBatchParams{
+		Jobid:      id,
+		Batchsize:  2,
+		Pageoffset: 0,
+	})
+	assert.NoError(t, err)
+	assert.Len(t, ids, 2)
+	assert.ElementsMatch(t, []*repository.ListJobDocumentIDsBatchRow{
+		{
+			ID:         docOne,
+			Totalcount: &total,
+		},
+		{
+			ID:         docTwo,
+			Totalcount: &total,
+		},
+	}, ids)
+}

internal/database/repository/result_test.go

@@ -2,7 +2,6 @@ package repository_test
 
 import (
 	"context"
-	"log"
 	"os"
 	"path"
 	"queryorchestration/internal/database/repository"
@@ -235,9 +234,6 @@ func TestResultValues(t *testing.T) {
 		Documentid: documentID,
 		Version:    jsonVersion,
 	})
-	for _, r := range qResults {
-		log.Print(r)
-	}
 	assert.NoError(t, err)
 	assert.Len(t, qResults, 1)
 	assert.EqualExportedValues(t, []*repository.ListQueryRequirementValuesRow{