Merged in feature/testquery (pull request #39)

Test Query * depstextandclean * startedcleaningresult * resulttidyup * roundone * cleaning * unsyncedquery * startedtestsandsimplification * api * querytests * resultprocessortests * unittests * cleanup
2025-01-29 11:52:37 +00:00
parent c36b0cdcf8
commit 0ac5ff9e15
109 changed files with 2804 additions and 2100 deletions
@@ -10,7 +10,6 @@ import (
 	"queryorchestration/internal/test"
 	"testing"

-	"github.com/google/uuid"
 	"github.com/jackc/pgx/v5/pgtype"
 	"github.com/stretchr/testify/assert"
 )
@@ -51,8 +50,8 @@ func TestCollector(t *testing.T) {
 	assert.EqualExportedValues(t, &repository.Fullactivecollector{
 		ID:              collId,
 		Jobid:           jobId,
-		Mincleanversion: nil,
-		Mintextversion:  nil,
+		Mincleanversion: 0,
+		Mintextversion:  0,
 		Activeversion:   1,
 		Latestversion:   1,
 	}, coll)
@@ -62,8 +61,8 @@ func TestCollector(t *testing.T) {
 	assert.EqualExportedValues(t, &repository.Fullactivecollector{
 		ID:              collId,
 		Jobid:           jobId,
-		Mincleanversion: nil,
-		Mintextversion:  nil,
+		Mincleanversion: 0,
+		Mintextversion:  0,
 		Activeversion:   1,
 		Latestversion:   1,
 	}, coll)
@@ -89,8 +88,8 @@ func TestCollector(t *testing.T) {
 	assert.EqualExportedValues(t, &repository.Fullactivecollector{
 		ID:              collId,
 		Jobid:           jobId,
-		Mincleanversion: &minCleanVersion,
-		Mintextversion:  &minTextVersion,
+		Mincleanversion: minCleanVersion,
+		Mintextversion:  minTextVersion,
 		Activeversion:   1,
 		Latestversion:   1,
 		Fields:          []byte(fmt.Sprintf("{\"example_key\": \"%s\"}", database.MustToUUID(jsonId).String())),
@@ -112,7 +111,7 @@ func TestCollector(t *testing.T) {
 			Queryid:      contextId,
 			Queryversion: 1,
 			Type:         repository.QuerytypeContextFull,
-			Requiredids:  []pgtype.UUID{database.MustToDBUUID(uuid.Nil)},
+			Requiredids:  []pgtype.UUID{},
 		},
 	}, qs)

@@ -142,8 +141,8 @@ func TestCollector(t *testing.T) {
 	assert.EqualExportedValues(t, &repository.Fullactivecollector{
 		ID:              collId,
 		Jobid:           jobId,
-		Mincleanversion: nil,
-		Mintextversion:  nil,
+		Mincleanversion: 0,
+		Mintextversion:  0,
 		Activeversion:   2,
 		Latestversion:   2,
 		Fields:          []byte(nil),
@@ -111,8 +111,8 @@ type Document struct {
 type Fullactivecollector struct {
 	ID              pgtype.UUID `db:"id"`
 	Jobid           pgtype.UUID `db:"jobid"`
-	Mincleanversion *int32      `db:"mincleanversion"`
-	Mintextversion  *int32      `db:"mintextversion"`
+	Mincleanversion int32       `db:"mincleanversion"`
+	Mintextversion  int32       `db:"mintextversion"`
 	Activeversion   int32       `db:"activeversion"`
 	Latestversion   int32       `db:"latestversion"`
 	Fields          []byte      `db:"fields"`
@@ -124,6 +124,59 @@ func (q *Queries) GetQueryConfig(ctx context.Context, arg *GetQueryConfigParams)
 	return &i, err
 }

+const getQueryWithVersion = `-- name: GetQueryWithVersion :one
+SELECT DISTINCT q.id, q.type, q.activeVersion, q.latestVersion, coalesce(c.config, null) as config, ARRAY_AGG(DISTINCT r.requiredQueryId)::uuid[] as requiredIds
+    FROM queries AS q
+    LEFT JOIN queryConfigs AS c ON q.id = c.queryId
+        and $2 >= c.addedVersion
+        and $2 < COALESCE(c.removedVersion, $2 + 1)
+    LEFT JOIN requiredQueries AS r ON q.id = r.queryId
+        and $2 >= r.addedVersion
+        and $2 < COALESCE(r.removedVersion, $2 + 1)
+    WHERE q.id = $1
+    GROUP BY q.id, q.type, q.activeversion, q.latestversion, c.config
+`
+
+type GetQueryWithVersionParams struct {
+	ID           pgtype.UUID `db:"id"`
+	Addedversion int32       `db:"addedversion"`
+}
+
+type GetQueryWithVersionRow struct {
+	ID            pgtype.UUID   `db:"id"`
+	Type          Querytype     `db:"type"`
+	Activeversion int32         `db:"activeversion"`
+	Latestversion int32         `db:"latestversion"`
+	Config        []byte        `db:"config"`
+	Requiredids   []pgtype.UUID `db:"requiredids"`
+}
+
+// GetQueryWithVersion
+//
+//	SELECT DISTINCT q.id, q.type, q.activeVersion, q.latestVersion, coalesce(c.config, null) as config, ARRAY_AGG(DISTINCT r.requiredQueryId)::uuid[] as requiredIds
+//	    FROM queries AS q
+//	    LEFT JOIN queryConfigs AS c ON q.id = c.queryId
+//	        and $2 >= c.addedVersion
+//	        and $2 < COALESCE(c.removedVersion, $2 + 1)
+//	    LEFT JOIN requiredQueries AS r ON q.id = r.queryId
+//	        and $2 >= r.addedVersion
+//	        and $2 < COALESCE(r.removedVersion, $2 + 1)
+//	    WHERE q.id = $1
+//	    GROUP BY q.id, q.type, q.activeversion, q.latestversion, c.config
+func (q *Queries) GetQueryWithVersion(ctx context.Context, arg *GetQueryWithVersionParams) (*GetQueryWithVersionRow, error) {
+	row := q.db.QueryRow(ctx, getQueryWithVersion, arg.ID, arg.Addedversion)
+	var i GetQueryWithVersionRow
+	err := row.Scan(
+		&i.ID,
+		&i.Type,
+		&i.Activeversion,
+		&i.Latestversion,
+		&i.Config,
+		&i.Requiredids,
+	)
+	return &i, err
+}
+
 const isQueryInDependencyTree = `-- name: IsQueryInDependencyTree :one
 SELECT EXISTS (
    SELECT 1 FROM queryActiveDependencies WHERE id = any($1) and requiredQueryId = $2 or $2 = any($1)
@@ -181,6 +234,40 @@ func (q *Queries) ListQueries(ctx context.Context) ([]*Fullactivequery, error) {
 	return items, nil
 }

+const listQueriesById = `-- name: ListQueriesById :many
+SELECT id, type, activeversion, latestversion, config, requiredids FROM fullActiveQueries WHERE id = any($1)
+`
+
+// ListQueriesById
+//
+//	SELECT id, type, activeversion, latestversion, config, requiredids FROM fullActiveQueries WHERE id = any($1)
+func (q *Queries) ListQueriesById(ctx context.Context, id []pgtype.UUID) ([]*Fullactivequery, error) {
+	rows, err := q.db.Query(ctx, listQueriesById, id)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	items := []*Fullactivequery{}
+	for rows.Next() {
+		var i Fullactivequery
+		if err := rows.Scan(
+			&i.ID,
+			&i.Type,
+			&i.Activeversion,
+			&i.Latestversion,
+			&i.Config,
+			&i.Requiredids,
+		); err != nil {
+			return nil, err
+		}
+		items = append(items, &i)
+	}
+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+	return items, nil
+}
+
 const removeQueryConfig = `-- name: RemoveQueryConfig :exec
 UPDATE queryConfigs SET removedVersion = $1 WHERE queryId = $2 and removedVersion is null
 `
@@ -41,7 +41,7 @@ func TestQueries(t *testing.T) {
 		Activeversion: 1,
 		Latestversion: 1,
 		Config:        nil,
-		Requiredids:   []pgtype.UUID{database.MustToDBUUID(uuid.Nil)},
+		Requiredids:   []pgtype.UUID{},
 	}, jsonQuery)

 	err = queries.UpdateQuery(ctx, &repository.UpdateQueryParams{
@@ -121,9 +121,23 @@ func TestQueries(t *testing.T) {
 		Activeversion: 2,
 		Latestversion: 2,
 		Config:        nil,
-		Requiredids:   []pgtype.UUID{database.MustToDBUUID(uuid.Nil)},
+		Requiredids:   []pgtype.UUID{},
 	}, jsonQuery)

+	versionedQuery, err := queries.GetQueryWithVersion(ctx, &repository.GetQueryWithVersionParams{
+		ID:           jsonQueryID,
+		Addedversion: 1,
+	})
+	assert.Nil(t, err)
+	assert.EqualExportedValues(t, &repository.GetQueryWithVersionRow{
+		ID:            jsonQueryID,
+		Type:          repository.QuerytypeJsonExtractor,
+		Activeversion: 2,
+		Latestversion: 2,
+		Config:        jsonConfig,
+		Requiredids:   []pgtype.UUID{contextQueryID},
+	}, versionedQuery)
+
 	all_exist, err := queries.AllQueriesExist(ctx, []pgtype.UUID{})
 	assert.Nil(t, err)
 	assert.True(t, all_exist)
@@ -240,7 +254,7 @@ func TestQueriesList(t *testing.T) {
 			Activeversion: 1,
 			Latestversion: 1,
 			Config:        nil,
-			Requiredids:   []pgtype.UUID{{}},
+			Requiredids:   []pgtype.UUID{},
 		},
 		{
 			ID:            contextQueryID,
@@ -248,7 +262,21 @@ func TestQueriesList(t *testing.T) {
 			Activeversion: 1,
 			Latestversion: 1,
 			Config:        nil,
-			Requiredids:   []pgtype.UUID{{}},
+			Requiredids:   []pgtype.UUID{},
+		},
+	}, qs)
+
+	qs, err = queries.ListQueriesById(ctx, []pgtype.UUID{jsonQueryID})
+	assert.Nil(t, err)
+	assert.Len(t, qs, 1)
+	assert.ElementsMatch(t, []*repository.Fullactivequery{
+		{
+			ID:            jsonQueryID,
+			Type:          repository.QuerytypeJsonExtractor,
+			Activeversion: 1,
+			Latestversion: 1,
+			Config:        nil,
+			Requiredids:   []pgtype.UUID{},
 		},
 	}, qs)
 }
@@ -11,29 +11,118 @@ import (
 	"github.com/jackc/pgx/v5/pgtype"
 )

-const listResultValuesByID = `-- name: ListResultValuesByID :many
-SELECT id, queryId, value FROM results where id = ANY($1)
+const getResultValueWithVersion = `-- name: GetResultValueWithVersion :one
+SELECT id, value FROM results WHERE queryId = $1 and queryVersion = $2 and documentId = $3 and cleanVersion >= $4 and textVersion >= $5
 `

-type ListResultValuesByIDRow struct {
-	ID      pgtype.UUID `db:"id"`
-	Queryid pgtype.UUID `db:"queryid"`
-	Value   string      `db:"value"`
+type GetResultValueWithVersionParams struct {
+	Queryid      pgtype.UUID `db:"queryid"`
+	Queryversion int32       `db:"queryversion"`
+	Documentid   pgtype.UUID `db:"documentid"`
+	Cleanversion int32       `db:"cleanversion"`
+	Textversion  int32       `db:"textversion"`
 }

-// ListResultValuesByID
+type GetResultValueWithVersionRow struct {
+	ID    pgtype.UUID `db:"id"`
+	Value string      `db:"value"`
+}
+
+// GetResultValueWithVersion
 //
-//	SELECT id, queryId, value FROM results where id = ANY($1)
-func (q *Queries) ListResultValuesByID(ctx context.Context, id []pgtype.UUID) ([]*ListResultValuesByIDRow, error) {
-	rows, err := q.db.Query(ctx, listResultValuesByID, id)
+//	SELECT id, value FROM results WHERE queryId = $1 and queryVersion = $2 and documentId = $3 and cleanVersion >= $4 and textVersion >= $5
+func (q *Queries) GetResultValueWithVersion(ctx context.Context, arg *GetResultValueWithVersionParams) (*GetResultValueWithVersionRow, error) {
+	row := q.db.QueryRow(ctx, getResultValueWithVersion,
+		arg.Queryid,
+		arg.Queryversion,
+		arg.Documentid,
+		arg.Cleanversion,
+		arg.Textversion,
+	)
+	var i GetResultValueWithVersionRow
+	err := row.Scan(&i.ID, &i.Value)
+	return &i, err
+}
+
+const listQueryRequirementValues = `-- name: ListQueryRequirementValues :many
+WITH latest_versions AS (
+    SELECT 
+        r.queryId,
+        MAX(r.cleanVersion) as max_clean_version,
+        MAX(r.textVersion) as max_text_version
+    FROM results r
+    WHERE r.documentId = $3
+        AND r.queryVersion = $2
+    GROUP BY r.queryId
+)
+SELECT rq.requiredQueryId as queryId, r.value, q.type
+    FROM requiredQueries as rq
+    JOIN results as r on r.queryId = rq.requiredQueryId
+    JOIN queries as q on q.id = rq.requiredQueryId
+    JOIN latest_versions lv ON lv.queryId = r.queryId
+    WHERE rq.queryId = $1 and r.documentId = $3
+        and $2 >= rq.addedVersion
+        and $2 < COALESCE(rq.removedVersion, $2 + 1)
+        and r.queryVersion = $2
+        and r.cleanVersion >= $4 and r.textVersion >= $5
+        AND r.cleanVersion = lv.max_clean_version 
+        AND r.textVersion = lv.max_text_version
+`
+
+type ListQueryRequirementValuesParams struct {
+	Queryid      pgtype.UUID `db:"queryid"`
+	Addedversion int32       `db:"addedversion"`
+	Documentid   pgtype.UUID `db:"documentid"`
+	Cleanversion int32       `db:"cleanversion"`
+	Textversion  int32       `db:"textversion"`
+}
+
+type ListQueryRequirementValuesRow struct {
+	Queryid pgtype.UUID `db:"queryid"`
+	Value   string      `db:"value"`
+	Type    Querytype   `db:"type"`
+}
+
+// ListQueryRequirementValues
+//
+//	WITH latest_versions AS (
+//	    SELECT
+//	        r.queryId,
+//	        MAX(r.cleanVersion) as max_clean_version,
+//	        MAX(r.textVersion) as max_text_version
+//	    FROM results r
+//	    WHERE r.documentId = $3
+//	        AND r.queryVersion = $2
+//	    GROUP BY r.queryId
+//	)
+//	SELECT rq.requiredQueryId as queryId, r.value, q.type
+//	    FROM requiredQueries as rq
+//	    JOIN results as r on r.queryId = rq.requiredQueryId
+//	    JOIN queries as q on q.id = rq.requiredQueryId
+//	    JOIN latest_versions lv ON lv.queryId = r.queryId
+//	    WHERE rq.queryId = $1 and r.documentId = $3
+//	        and $2 >= rq.addedVersion
+//	        and $2 < COALESCE(rq.removedVersion, $2 + 1)
+//	        and r.queryVersion = $2
+//	        and r.cleanVersion >= $4 and r.textVersion >= $5
+//	        AND r.cleanVersion = lv.max_clean_version
+//	        AND r.textVersion = lv.max_text_version
+func (q *Queries) ListQueryRequirementValues(ctx context.Context, arg *ListQueryRequirementValuesParams) ([]*ListQueryRequirementValuesRow, error) {
+	rows, err := q.db.Query(ctx, listQueryRequirementValues,
+		arg.Queryid,
+		arg.Addedversion,
+		arg.Documentid,
+		arg.Cleanversion,
+		arg.Textversion,
+	)
 	if err != nil {
 		return nil, err
 	}
 	defer rows.Close()
-	items := []*ListResultValuesByIDRow{}
+	items := []*ListQueryRequirementValuesRow{}
 	for rows.Next() {
-		var i ListResultValuesByIDRow
-		if err := rows.Scan(&i.ID, &i.Queryid, &i.Value); err != nil {
+		var i ListQueryRequirementValuesRow
+		if err := rows.Scan(&i.Queryid, &i.Value, &i.Type); err != nil {
 			return nil, err
 		}
 		items = append(items, &i)
@@ -44,37 +133,64 @@ func (q *Queries) ListResultValuesByID(ctx context.Context, id []pgtype.UUID) ([
 	return items, nil
 }

-const listResultsByDocumentID = `-- name: ListResultsByDocumentID :many
-SELECT id, queryId, queryVersion FROM results
-    where documentId = $1 and cleanVersion >= $2 and textVersion >= $3
+const listUnsyncedQueriesByDocId = `-- name: ListUnsyncedQueriesByDocId :many
+WITH RECURSIVE unsyncedQueries AS (
+    SELECT dt.queryId, dt.requiredIds
+        from documents as d
+        JOIN fullActiveCollectors as c on d.jobId = c.jobId
+        JOIN collectorQueryDependencyTree as dt on c.id = dt.collectorId
+        LEFT JOIN results as r on r.queryId = dt.queryId
+            and r.queryVersion = dt.queryVersion
+            and r.cleanVersion >= c.minCleanVersion and r.textVersion >= c.minTextVersion
+        where d.id = $1 and r.value is null
+
+    UNION
+
+    SELECT DISTINCT dt.queryId, dt.requiredIds
+        FROM unsyncedQueries as u
+        JOIN collectorQueryDependencyTree as dt ON u.queryId = any(dt.requiredIds)
+)
+SELECT id, type, activeversion, latestversion, config, requiredids FROM fullActiveQueries 
+WHERE id in (SELECT queryId FROM unsyncedQueries)
 `

-type ListResultsByDocumentIDParams struct {
-	Documentid   pgtype.UUID `db:"documentid"`
-	Cleanversion int32       `db:"cleanversion"`
-	Textversion  int32       `db:"textversion"`
-}
-
-type ListResultsByDocumentIDRow struct {
-	ID           pgtype.UUID `db:"id"`
-	Queryid      pgtype.UUID `db:"queryid"`
-	Queryversion int32       `db:"queryversion"`
-}
-
-// ListResultsByDocumentID
+// ListUnsyncedQueriesByDocId
 //
-//	SELECT id, queryId, queryVersion FROM results
-//	    where documentId = $1 and cleanVersion >= $2 and textVersion >= $3
-func (q *Queries) ListResultsByDocumentID(ctx context.Context, arg *ListResultsByDocumentIDParams) ([]*ListResultsByDocumentIDRow, error) {
-	rows, err := q.db.Query(ctx, listResultsByDocumentID, arg.Documentid, arg.Cleanversion, arg.Textversion)
+//	WITH RECURSIVE unsyncedQueries AS (
+//	    SELECT dt.queryId, dt.requiredIds
+//	        from documents as d
+//	        JOIN fullActiveCollectors as c on d.jobId = c.jobId
+//	        JOIN collectorQueryDependencyTree as dt on c.id = dt.collectorId
+//	        LEFT JOIN results as r on r.queryId = dt.queryId
+//	            and r.queryVersion = dt.queryVersion
+//	            and r.cleanVersion >= c.minCleanVersion and r.textVersion >= c.minTextVersion
+//	        where d.id = $1 and r.value is null
+//
+//	    UNION
+//
+//	    SELECT DISTINCT dt.queryId, dt.requiredIds
+//	        FROM unsyncedQueries as u
+//	        JOIN collectorQueryDependencyTree as dt ON u.queryId = any(dt.requiredIds)
+//	)
+//	SELECT id, type, activeversion, latestversion, config, requiredids FROM fullActiveQueries
+//	WHERE id in (SELECT queryId FROM unsyncedQueries)
+func (q *Queries) ListUnsyncedQueriesByDocId(ctx context.Context, id pgtype.UUID) ([]*Fullactivequery, error) {
+	rows, err := q.db.Query(ctx, listUnsyncedQueriesByDocId, id)
 	if err != nil {
 		return nil, err
 	}
 	defer rows.Close()
-	items := []*ListResultsByDocumentIDRow{}
+	items := []*Fullactivequery{}
 	for rows.Next() {
-		var i ListResultsByDocumentIDRow
-		if err := rows.Scan(&i.ID, &i.Queryid, &i.Queryversion); err != nil {
+		var i Fullactivequery
+		if err := rows.Scan(
+			&i.ID,
+			&i.Type,
+			&i.Activeversion,
+			&i.Latestversion,
+			&i.Config,
+			&i.Requiredids,
+		); err != nil {
 			return nil, err
 		}
 		items = append(items, &i)
@@ -55,29 +55,316 @@ func TestResults(t *testing.T) {
 	assert.Nil(t, err)
 	assert.True(t, jsonResultID.Valid)

-	resultsByDoc, err := queries.ListResultsByDocumentID(ctx, &repository.ListResultsByDocumentIDParams{
+	res, err := queries.GetResultValueWithVersion(ctx, &repository.GetResultValueWithVersionParams{
+		Queryid:      jsonQueryID,
+		Queryversion: jsonQuery.Activeversion,
 		Documentid:   documentID,
 		Cleanversion: cleanVersion,
 		Textversion:  textVersion,
 	})
 	assert.Nil(t, err)
-	assert.Len(t, resultsByDoc, 1)
-	assert.ElementsMatch(t, []*repository.ListResultsByDocumentIDRow{
-		{
-			ID:           jsonResultID,
-			Queryid:      jsonQueryID,
-			Queryversion: jsonQuery.Activeversion,
-		},
-	}, resultsByDoc)
-
-	results, err := queries.ListResultValuesByID(ctx, []pgtype.UUID{jsonResultID})
-	assert.Nil(t, err)
-	assert.Len(t, results, 1)
-	assert.ElementsMatch(t, []*repository.ListResultValuesByIDRow{
-		{
-			ID:      jsonResultID,
-			Queryid: jsonQueryID,
-			Value:   jsonResultValue,
-		},
-	}, results)
+	assert.EqualExportedValues(t, &repository.GetResultValueWithVersionRow{
+		ID:    jsonResultID,
+		Value: jsonResultValue,
+	}, res)
+}
+
+func TestResultValues(t *testing.T) {
+	ctx := context.Background()
+
+	db, cleanup := test.CreateDB(t, ctx, &test.CreateDatabaseConfig{
+		Migrations: &database.MigrationConfig{
+			BasePath: path.Join(os.Getenv("PWD"), "../../.."),
+		}})
+	defer cleanup()
+
+	queries := repository.New(db.Pool)
+
+	jsonQueryID, err := queries.CreateQuery(ctx, repository.Querytype(repository.QuerytypeJsonExtractor))
+	assert.Nil(t, err)
+
+	clientId, err := queries.CreateClient(ctx, "example_client")
+	assert.Nil(t, err)
+	jobId, err := queries.CreateJob(ctx, clientId)
+	assert.Nil(t, err)
+	documentID, err := queries.CreateDocument(ctx, &repository.CreateDocumentParams{
+		Jobid: jobId,
+		Hash:  "example_hash",
+	})
+	assert.Nil(t, err)
+
+	contextQueryID, err := queries.CreateQuery(ctx, repository.Querytype(repository.QuerytypeContextFull))
+	assert.Nil(t, err)
+	err = queries.AddRequiredQuery(ctx, &repository.AddRequiredQueryParams{
+		Queryid:         jsonQueryID,
+		Requiredqueryid: contextQueryID,
+		Addedversion:    1,
+	})
+	assert.Nil(t, err)
+
+	contextQuery, err := queries.GetQuery(ctx, contextQueryID)
+	assert.Nil(t, err)
+
+	result := repository.SetResultParams{
+		Queryid:      contextQueryID,
+		Documentid:   documentID,
+		Value:        "context_value",
+		Cleanversion: 1,
+		Textversion:  2,
+		Queryversion: contextQuery.Activeversion,
+	}
+
+	_, err = queries.SetResult(ctx, &result)
+	assert.Nil(t, err)
+
+	_, err = queries.SetResult(ctx, &repository.SetResultParams{
+		Queryid:      contextQueryID,
+		Documentid:   documentID,
+		Value:        "context_value",
+		Cleanversion: 1,
+		Textversion:  2,
+		Queryversion: contextQuery.Activeversion - 1,
+	})
+	assert.Nil(t, err)
+
+	_, err = queries.SetResult(ctx, &repository.SetResultParams{
+		Queryid:      contextQueryID,
+		Documentid:   documentID,
+		Value:        "context_value",
+		Cleanversion: 2,
+		Textversion:  2,
+		Queryversion: contextQuery.Activeversion,
+	})
+	assert.Nil(t, err)
+
+	_, err = queries.SetResult(ctx, &repository.SetResultParams{
+		Queryid:      jsonQueryID,
+		Documentid:   documentID,
+		Value:        "context_value",
+		Cleanversion: 1,
+		Textversion:  2,
+		Queryversion: contextQuery.Activeversion,
+	})
+	assert.Nil(t, err)
+
+	qResults, err := queries.ListQueryRequirementValues(ctx, &repository.ListQueryRequirementValuesParams{
+		Queryid:      jsonQueryID,
+		Documentid:   documentID,
+		Addedversion: 1,
+		Cleanversion: result.Cleanversion,
+		Textversion:  result.Textversion,
+	})
+	assert.Nil(t, err)
+	assert.EqualExportedValues(t, []*repository.ListQueryRequirementValuesRow{
+		{
+			Queryid: contextQueryID,
+			Type:    repository.QuerytypeContextFull,
+			Value:   result.Value,
+		},
+	}, qResults)
+}
+
+func TestUnsyncedQueries(t *testing.T) {
+	ctx := context.Background()
+
+	db, cleanup := test.CreateDB(t, ctx, &test.CreateDatabaseConfig{
+		Migrations: &database.MigrationConfig{
+			BasePath: path.Join(os.Getenv("PWD"), "../../.."),
+		}})
+	defer cleanup()
+
+	queries := repository.New(db.Pool)
+
+	clientId, err := queries.CreateClient(ctx, "example_client")
+	assert.Nil(t, err)
+	jobId, err := queries.CreateJob(ctx, clientId)
+	assert.Nil(t, err)
+	collectorId, err := queries.CreateCollector(ctx, jobId)
+	assert.Nil(t, err)
+	documentID, err := queries.CreateDocument(ctx, &repository.CreateDocumentParams{
+		Jobid: jobId,
+		Hash:  "example_hash",
+	})
+	assert.Nil(t, err)
+
+	contextQueryID, err := queries.CreateQuery(ctx, repository.Querytype(repository.QuerytypeContextFull))
+	assert.Nil(t, err)
+	jsonQueryID, err := queries.CreateQuery(ctx, repository.Querytype(repository.QuerytypeJsonExtractor))
+	assert.Nil(t, err)
+	err = queries.AddRequiredQuery(ctx, &repository.AddRequiredQueryParams{
+		Queryid:         jsonQueryID,
+		Requiredqueryid: contextQueryID,
+		Addedversion:    1,
+	})
+	assert.Nil(t, err)
+
+	err = queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{
+		Collectorid:  collectorId,
+		Name:         "example_name",
+		Queryid:      jsonQueryID,
+		Addedversion: 1,
+	})
+	assert.Nil(t, err)
+
+	qs, err := queries.ListUnsyncedQueriesByDocId(ctx, documentID)
+	assert.Nil(t, err)
+	assert.Len(t, qs, 2)
+	assert.ElementsMatch(t, []*repository.Fullactivequery{
+		{
+			ID:            jsonQueryID,
+			Type:          repository.QuerytypeJsonExtractor,
+			Activeversion: 1,
+			Latestversion: 1,
+			Config:        nil,
+			Requiredids:   []pgtype.UUID{contextQueryID},
+		},
+		{
+			ID:            contextQueryID,
+			Type:          repository.QuerytypeContextFull,
+			Activeversion: 1,
+			Latestversion: 1,
+			Config:        nil,
+			Requiredids:   []pgtype.UUID{},
+		},
+	}, qs)
+
+	_, err = queries.SetResult(ctx, &repository.SetResultParams{
+		Queryid:      contextQueryID,
+		Documentid:   documentID,
+		Value:        "context_value",
+		Cleanversion: 1,
+		Textversion:  2,
+		Queryversion: 1,
+	})
+	assert.Nil(t, err)
+
+	qs, err = queries.ListUnsyncedQueriesByDocId(ctx, documentID)
+	assert.Nil(t, err)
+	assert.Len(t, qs, 1)
+	assert.ElementsMatch(t, []*repository.Fullactivequery{
+		{
+			ID:            jsonQueryID,
+			Type:          repository.QuerytypeJsonExtractor,
+			Activeversion: 1,
+			Latestversion: 1,
+			Config:        nil,
+			Requiredids:   []pgtype.UUID{contextQueryID},
+		},
+	}, qs)
+
+	err = queries.UpdateQuery(ctx, &repository.UpdateQueryParams{
+		Latestversion: 2,
+		Activeversion: 2,
+		ID:            contextQueryID,
+	})
+	assert.Nil(t, err)
+
+	qs, err = queries.ListUnsyncedQueriesByDocId(ctx, documentID)
+	assert.Nil(t, err)
+	assert.Len(t, qs, 2)
+	assert.ElementsMatch(t, []*repository.Fullactivequery{
+		{
+			ID:            jsonQueryID,
+			Type:          repository.QuerytypeJsonExtractor,
+			Activeversion: 1,
+			Latestversion: 1,
+			Config:        nil,
+			Requiredids:   []pgtype.UUID{contextQueryID},
+		},
+		{
+			ID:            contextQueryID,
+			Type:          repository.QuerytypeContextFull,
+			Activeversion: 2,
+			Latestversion: 2,
+			Config:        nil,
+			Requiredids:   []pgtype.UUID{},
+		},
+	}, qs)
+
+	_, err = queries.SetResult(ctx, &repository.SetResultParams{
+		Queryid:      contextQueryID,
+		Documentid:   documentID,
+		Value:        "context_value",
+		Cleanversion: 1,
+		Textversion:  2,
+		Queryversion: 2,
+	})
+	assert.Nil(t, err)
+	_, err = queries.SetResult(ctx, &repository.SetResultParams{
+		Queryid:      jsonQueryID,
+		Documentid:   documentID,
+		Value:        "context_value",
+		Cleanversion: 1,
+		Textversion:  2,
+		Queryversion: 1,
+	})
+	assert.Nil(t, err)
+
+	qs, err = queries.ListUnsyncedQueriesByDocId(ctx, documentID)
+	assert.Nil(t, err)
+	assert.Len(t, qs, 0)
+
+	err = queries.UpdateQuery(ctx, &repository.UpdateQueryParams{
+		Latestversion: 2,
+		Activeversion: 2,
+		ID:            jsonQueryID,
+	})
+	assert.Nil(t, err)
+
+	qs, err = queries.ListUnsyncedQueriesByDocId(ctx, documentID)
+	assert.Nil(t, err)
+	assert.Len(t, qs, 1)
+	assert.ElementsMatch(t, []*repository.Fullactivequery{
+		{
+			ID:            jsonQueryID,
+			Type:          repository.QuerytypeJsonExtractor,
+			Activeversion: 2,
+			Latestversion: 2,
+			Config:        nil,
+			Requiredids:   []pgtype.UUID{contextQueryID},
+		},
+	}, qs)
+
+	_, err = queries.SetResult(ctx, &repository.SetResultParams{
+		Queryid:      jsonQueryID,
+		Documentid:   documentID,
+		Value:        "context_value",
+		Cleanversion: 1,
+		Textversion:  2,
+		Queryversion: 2,
+	})
+	assert.Nil(t, err)
+
+	qs, err = queries.ListUnsyncedQueriesByDocId(ctx, documentID)
+	assert.Nil(t, err)
+	assert.Len(t, qs, 0)
+
+	err = queries.UpdateQuery(ctx, &repository.UpdateQueryParams{
+		Latestversion: 3,
+		Activeversion: 3,
+		ID:            contextQueryID,
+	})
+	assert.Nil(t, err)
+
+	qs, err = queries.ListUnsyncedQueriesByDocId(ctx, documentID)
+	assert.Nil(t, err)
+	assert.Len(t, qs, 2)
+	assert.ElementsMatch(t, []*repository.Fullactivequery{
+		{
+			ID:            jsonQueryID,
+			Type:          repository.QuerytypeJsonExtractor,
+			Activeversion: 2,
+			Latestversion: 2,
+			Config:        nil,
+			Requiredids:   []pgtype.UUID{contextQueryID},
+		},
+		{
+			ID:            contextQueryID,
+			Type:          repository.QuerytypeContextFull,
+			Activeversion: 3,
+			Latestversion: 3,
+			Config:        nil,
+			Requiredids:   []pgtype.UUID{},
+		},
+	}, qs)
 }