Files
query-orchestration/internal/database/repository/query_test.go
T
Michael McGuinness 3d434eedb8 Merged in feature/jobsynced (pull request #72)
Job Status Get and DB tidy up

* initalquery

* tests

* shorttests

* testing queries

* job

* solvedthequery

* updatingdb

* fixingtests

* repotests

* shorttests

* docker

* testspassed
2025-02-20 19:02:44 +00:00

628 lines
20 KiB
Go

package repository_test
import (
"context"
"os"
"path"
"queryorchestration/internal/database"
"queryorchestration/internal/database/repository"
"queryorchestration/internal/serviceconfig"
"queryorchestration/internal/test"
"testing"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgtype"
"github.com/stretchr/testify/assert"
)
func TestQueries(t *testing.T) {
if testing.Short() {
t.Skip("Skipping long test in short mode")
}
ctx := context.Background()
cfg := &serviceconfig.BaseConfig{}
test.SetCfgProvider(t, cfg)
cfg.SetBasePath(path.Join(os.Getenv("PWD"), "../../.."))
_, cleanup := test.CreateDB(t, ctx, &test.CreateDatabaseConfig{
Cfg: cfg,
RunMigrations: true,
})
defer cleanup()
queries := cfg.GetDBQueries()
contextQueryID, err := queries.CreateQuery(ctx, repository.Querytype(repository.QuerytypeContextFull))
assert.NoError(t, err)
assert.True(t, contextQueryID.Valid)
contextQuery, err := queries.GetQuery(ctx, contextQueryID)
assert.NoError(t, err)
assert.EqualExportedValues(t, &repository.Fullactivequery{
ID: contextQueryID,
Type: repository.QuerytypeContextFull,
Activeversion: 0,
Latestversion: 0,
Config: nil,
Requiredids: []pgtype.UUID{},
}, contextQuery)
ctxVersion, err := queries.AddLatestQueryVersion(ctx, contextQueryID)
assert.NoError(t, err)
assert.Equal(t, int32(1), ctxVersion)
contextQuery, err = queries.GetQuery(ctx, contextQueryID)
assert.NoError(t, err)
assert.EqualExportedValues(t, &repository.Fullactivequery{
ID: contextQueryID,
Type: repository.QuerytypeContextFull,
Activeversion: 0,
Latestversion: 1,
Config: nil,
Requiredids: []pgtype.UUID{},
}, contextQuery)
jsonQueryID, err := queries.CreateQuery(ctx, repository.Querytype(repository.QuerytypeJsonExtractor))
assert.NoError(t, err)
assert.True(t, jsonQueryID.Valid)
jsonQuery, err := queries.GetQuery(ctx, jsonQueryID)
assert.NoError(t, err)
assert.EqualExportedValues(t, &repository.Fullactivequery{
ID: jsonQueryID,
Type: repository.QuerytypeJsonExtractor,
Activeversion: 0,
Latestversion: 0,
Config: nil,
Requiredids: []pgtype.UUID{},
}, jsonQuery)
jsonVersion, err := queries.AddLatestQueryVersion(ctx, jsonQueryID)
assert.NoError(t, err)
assert.Equal(t, int32(1), jsonVersion)
jsonQuery, err = queries.GetQuery(ctx, jsonQueryID)
assert.NoError(t, err)
assert.EqualExportedValues(t, &repository.Fullactivequery{
ID: jsonQueryID,
Type: repository.QuerytypeJsonExtractor,
Activeversion: 0,
Latestversion: 1,
Config: nil,
Requiredids: []pgtype.UUID{},
}, jsonQuery)
err = queries.AddActiveQueryVersion(ctx, &repository.AddActiveQueryVersionParams{
Versionid: 1,
Queryid: jsonQueryID,
})
assert.NoError(t, err)
jsonQuery, err = queries.GetQuery(ctx, jsonQueryID)
assert.NoError(t, err)
assert.EqualExportedValues(t, &repository.Fullactivequery{
ID: jsonQueryID,
Type: repository.QuerytypeJsonExtractor,
Activeversion: 1,
Latestversion: 1,
Config: nil,
Requiredids: []pgtype.UUID{},
}, jsonQuery)
jsonConfig := []byte("{\"path\": \"example_path\"}")
err = queries.AddRequiredQuery(ctx, &repository.AddRequiredQueryParams{
Queryid: jsonQueryID,
Requiredqueryid: contextQueryID,
Addedversion: 1,
})
assert.NoError(t, err)
jsonQuery, err = queries.GetQuery(ctx, jsonQueryID)
assert.NoError(t, err)
assert.EqualExportedValues(t, &repository.Fullactivequery{
ID: jsonQueryID,
Type: repository.QuerytypeJsonExtractor,
Activeversion: 1,
Latestversion: 1,
Config: nil,
Requiredids: []pgtype.UUID{contextQueryID},
}, jsonQuery)
jsonVersion, err = queries.AddLatestQueryVersion(ctx, jsonQueryID)
assert.NoError(t, err)
assert.Equal(t, int32(2), jsonVersion)
jsonQuery, err = queries.GetQuery(ctx, jsonQueryID)
assert.NoError(t, err)
assert.EqualExportedValues(t, &repository.Fullactivequery{
ID: jsonQueryID,
Type: repository.QuerytypeJsonExtractor,
Activeversion: 1,
Latestversion: 2,
Config: nil,
Requiredids: []pgtype.UUID{contextQueryID},
}, jsonQuery)
removeV := int32(2)
err = queries.RemoveRequiredQuery(ctx, &repository.RemoveRequiredQueryParams{
Queryid: jsonQueryID,
Requiredqueryid: contextQueryID,
Removedversion: &removeV,
})
assert.NoError(t, err)
err = queries.SetQueryConfig(ctx, &repository.SetQueryConfigParams{
Queryid: jsonQueryID,
Config: jsonConfig,
Addedversion: 1,
})
assert.NoError(t, err)
jsonQueryConfig, err := queries.GetActiveQueryConfig(ctx, jsonQueryID)
assert.NoError(t, err)
assert.Equal(t, jsonConfig, jsonQueryConfig)
err = queries.SetQueryConfig(ctx, &repository.SetQueryConfigParams{
Queryid: jsonQueryID,
Config: []byte(`{"second":"key"}`),
Addedversion: 2,
})
assert.NoError(t, err)
jsonQueryConfig, err = queries.GetActiveQueryConfig(ctx, jsonQueryID)
assert.NoError(t, err)
assert.Equal(t, jsonConfig, jsonQueryConfig)
err = queries.AddActiveQueryVersion(ctx, &repository.AddActiveQueryVersionParams{
Queryid: jsonQueryID,
Versionid: 2,
})
assert.NoError(t, err)
jsonQueryConfig, err = queries.GetActiveQueryConfig(ctx, jsonQueryID)
assert.NoError(t, err)
assert.Equal(t, []byte(`{"second": "key"}`), jsonQueryConfig)
jsonQuery, err = queries.GetQuery(ctx, jsonQueryID)
assert.NoError(t, err)
assert.EqualExportedValues(t, &repository.Fullactivequery{
ID: jsonQueryID,
Type: repository.QuerytypeJsonExtractor,
Activeversion: 2,
Latestversion: 2,
Config: jsonQueryConfig,
Requiredids: []pgtype.UUID{},
}, jsonQuery)
v := int32(1)
versionedQuery, err := queries.GetQueryWithVersion(ctx, &repository.GetQueryWithVersionParams{
ID: jsonQueryID,
Version: &v,
})
assert.NoError(t, err)
assert.EqualExportedValues(t, &repository.GetQueryWithVersionRow{
ID: jsonQueryID,
Type: repository.QuerytypeJsonExtractor,
Activeversion: 2,
Latestversion: 2,
Config: jsonConfig,
Requiredids: []pgtype.UUID{contextQueryID},
}, versionedQuery)
all_exist, err := queries.AllQueriesExist(ctx, []pgtype.UUID{})
assert.NoError(t, err)
assert.True(t, all_exist)
all_exist, err = queries.AllQueriesExist(ctx, []pgtype.UUID{database.MustToDBUUID(uuid.New())})
assert.NoError(t, err)
assert.False(t, all_exist)
all_exist, err = queries.AllQueriesExist(ctx, []pgtype.UUID{jsonQueryID})
assert.NoError(t, err)
assert.True(t, all_exist)
all_exist, err = queries.AllQueriesExist(ctx, []pgtype.UUID{jsonQueryID, contextQueryID})
assert.NoError(t, err)
assert.True(t, all_exist)
all_exist, err = queries.AllQueriesExist(ctx, []pgtype.UUID{jsonQueryID, database.MustToDBUUID(uuid.New())})
assert.NoError(t, err)
assert.False(t, all_exist)
}
func TestQueryDependencyTree(t *testing.T) {
if testing.Short() {
t.Skip("Skipping long test in short mode")
}
ctx := context.Background()
cfg := &serviceconfig.BaseConfig{}
test.SetCfgProvider(t, cfg)
cfg.SetBasePath(path.Join(os.Getenv("PWD"), "../../.."))
_, cleanup := test.CreateDB(t, ctx, &test.CreateDatabaseConfig{
Cfg: cfg,
RunMigrations: true,
})
defer cleanup()
queries := cfg.GetDBQueries()
clientID, err := queries.CreateClient(ctx, "example_client")
assert.NoError(t, err)
jobID, err := queries.CreateJob(ctx, clientID)
assert.NoError(t, err)
version, err := queries.AddLatestCollectorVersion(ctx, jobID)
assert.NoError(t, err)
err = queries.SetActiveCollectorVersion(ctx, &repository.SetActiveCollectorVersionParams{
Jobid: jobID,
Versionid: version,
})
assert.NoError(t, err)
docID, err := queries.CreateDocument(ctx, &repository.CreateDocumentParams{
Jobid: jobID,
Hash: "sample",
})
assert.NoError(t, err)
contextQueryID, err := queries.CreateQuery(ctx, repository.Querytype(repository.QuerytypeContextFull))
assert.NoError(t, err)
_, err = queries.AddLatestQueryVersion(ctx, contextQueryID)
assert.NoError(t, err)
err = queries.AddActiveQueryVersion(ctx, &repository.AddActiveQueryVersionParams{
Queryid: contextQueryID,
Versionid: 1,
})
assert.NoError(t, err)
dependents, err := queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{
Documentid: docID,
Queryid: contextQueryID,
})
assert.NoError(t, err)
assert.ElementsMatch(t, []pgtype.UUID{}, dependents)
jsonQueryID, err := queries.CreateQuery(ctx, repository.Querytype(repository.QuerytypeJsonExtractor))
assert.NoError(t, err)
_, err = queries.AddLatestQueryVersion(ctx, jsonQueryID)
assert.NoError(t, err)
err = queries.AddActiveQueryVersion(ctx, &repository.AddActiveQueryVersionParams{
Queryid: jsonQueryID,
Versionid: 1,
})
assert.NoError(t, err)
dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{
Documentid: docID,
Queryid: jsonQueryID,
})
assert.NoError(t, err)
assert.ElementsMatch(t, []pgtype.UUID{}, dependents)
err = queries.AddRequiredQuery(ctx, &repository.AddRequiredQueryParams{
Queryid: jsonQueryID,
Requiredqueryid: contextQueryID,
Addedversion: 1,
})
assert.NoError(t, err)
dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{
Documentid: docID,
Queryid: jsonQueryID,
})
assert.NoError(t, err)
assert.ElementsMatch(t, []pgtype.UUID{}, dependents)
dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{
Documentid: docID,
Queryid: contextQueryID,
})
assert.NoError(t, err)
assert.ElementsMatch(t, []pgtype.UUID{}, dependents)
err = queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{
Jobid: jobID,
Name: "example_name'",
Queryid: jsonQueryID,
Addedversion: 1,
})
assert.NoError(t, err)
dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{
Documentid: docID,
Queryid: jsonQueryID,
})
assert.NoError(t, err)
assert.ElementsMatch(t, []pgtype.UUID{}, dependents)
dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{
Documentid: docID,
Queryid: contextQueryID,
})
assert.NoError(t, err)
assert.ElementsMatch(t, []pgtype.UUID{jsonQueryID}, dependents)
secondJsonQueryID, err := queries.CreateQuery(ctx, repository.Querytype(repository.QuerytypeJsonExtractor))
assert.NoError(t, err)
_, err = queries.AddLatestQueryVersion(ctx, secondJsonQueryID)
assert.NoError(t, err)
err = queries.AddActiveQueryVersion(ctx, &repository.AddActiveQueryVersionParams{
Queryid: secondJsonQueryID,
Versionid: 1,
})
assert.NoError(t, err)
dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{
Documentid: docID,
Queryid: jsonQueryID,
})
assert.NoError(t, err)
assert.ElementsMatch(t, []pgtype.UUID{}, dependents)
dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{
Documentid: docID,
Queryid: contextQueryID,
})
assert.NoError(t, err)
assert.ElementsMatch(t, []pgtype.UUID{jsonQueryID}, dependents)
err = queries.AddRequiredQuery(ctx, &repository.AddRequiredQueryParams{
Queryid: secondJsonQueryID,
Requiredqueryid: jsonQueryID,
Addedversion: 1,
})
assert.NoError(t, err)
dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{
Documentid: docID,
Queryid: secondJsonQueryID,
})
assert.NoError(t, err)
assert.ElementsMatch(t, []pgtype.UUID{}, dependents)
dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{
Documentid: docID,
Queryid: jsonQueryID,
})
assert.NoError(t, err)
assert.ElementsMatch(t, []pgtype.UUID{}, dependents)
dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{
Documentid: docID,
Queryid: contextQueryID,
})
assert.NoError(t, err)
assert.ElementsMatch(t, []pgtype.UUID{jsonQueryID}, dependents)
err = queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{
Jobid: jobID,
Name: "example_name'",
Queryid: secondJsonQueryID,
Addedversion: 1,
})
assert.NoError(t, err)
dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{
Documentid: docID,
Queryid: secondJsonQueryID,
})
assert.NoError(t, err)
assert.ElementsMatch(t, []pgtype.UUID{}, dependents)
dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{
Documentid: docID,
Queryid: jsonQueryID,
})
assert.NoError(t, err)
assert.ElementsMatch(t, []pgtype.UUID{secondJsonQueryID}, dependents)
dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{
Documentid: docID,
Queryid: contextQueryID,
})
assert.NoError(t, err)
assert.ElementsMatch(t, []pgtype.UUID{jsonQueryID}, dependents)
isdependent, err := queries.IsQueryInDependencyTree(ctx, &repository.IsQueryInDependencyTreeParams{
Queryid: jsonQueryID,
Requiredqueryids: []pgtype.UUID{contextQueryID},
})
assert.NoError(t, err)
assert.False(t, isdependent)
isdependent, err = queries.IsQueryInDependencyTree(ctx, &repository.IsQueryInDependencyTreeParams{
Queryid: jsonQueryID,
Requiredqueryids: []pgtype.UUID{secondJsonQueryID},
})
assert.NoError(t, err)
assert.True(t, isdependent)
isdependent, err = queries.IsQueryInDependencyTree(ctx, &repository.IsQueryInDependencyTreeParams{
Queryid: jsonQueryID,
Requiredqueryids: []pgtype.UUID{jsonQueryID},
})
assert.NoError(t, err)
assert.True(t, isdependent)
isdependent, err = queries.IsQueryInDependencyTree(ctx, &repository.IsQueryInDependencyTreeParams{
Queryid: secondJsonQueryID,
Requiredqueryids: []pgtype.UUID{jsonQueryID, contextQueryID},
})
assert.NoError(t, err)
assert.False(t, isdependent)
isdependent, err = queries.IsQueryInDependencyTree(ctx, &repository.IsQueryInDependencyTreeParams{
Queryid: contextQueryID,
Requiredqueryids: []pgtype.UUID{jsonQueryID, secondJsonQueryID},
})
assert.NoError(t, err)
assert.True(t, isdependent)
}
func TestQueriesList(t *testing.T) {
if testing.Short() {
t.Skip("Skipping long test in short mode")
}
ctx := context.Background()
cfg := &serviceconfig.BaseConfig{}
test.SetCfgProvider(t, cfg)
cfg.SetBasePath(path.Join(os.Getenv("PWD"), "../../.."))
_, cleanup := test.CreateDB(t, ctx, &test.CreateDatabaseConfig{
Cfg: cfg,
RunMigrations: true,
})
defer cleanup()
queries := cfg.GetDBQueries()
contextQueryID, err := queries.CreateQuery(ctx, repository.Querytype(repository.QuerytypeContextFull))
assert.NoError(t, err)
jsonQueryID, err := queries.CreateQuery(ctx, repository.Querytype(repository.QuerytypeJsonExtractor))
assert.NoError(t, err)
qs, err := queries.ListQueries(ctx)
assert.NoError(t, err)
assert.Len(t, qs, 2)
assert.ElementsMatch(t, []*repository.Fullactivequery{
{
ID: jsonQueryID,
Type: repository.QuerytypeJsonExtractor,
Activeversion: 0,
Latestversion: 0,
Config: nil,
Requiredids: []pgtype.UUID{},
},
{
ID: contextQueryID,
Type: repository.QuerytypeContextFull,
Activeversion: 0,
Latestversion: 0,
Config: nil,
Requiredids: []pgtype.UUID{},
},
}, qs)
qs, err = queries.ListQueriesById(ctx, []pgtype.UUID{jsonQueryID})
assert.NoError(t, err)
assert.Len(t, qs, 1)
assert.ElementsMatch(t, []*repository.Fullactivequery{
{
ID: jsonQueryID,
Type: repository.QuerytypeJsonExtractor,
Activeversion: 0,
Latestversion: 0,
Config: nil,
Requiredids: []pgtype.UUID{},
},
}, qs)
}
func TestListQueryJobs(t *testing.T) {
if testing.Short() {
t.Skip("Skipping long test in short mode")
}
ctx := context.Background()
cfg := &serviceconfig.BaseConfig{}
test.SetCfgProvider(t, cfg)
cfg.SetBasePath(path.Join(os.Getenv("PWD"), "../../.."))
_, cleanup := test.CreateDB(t, ctx, &test.CreateDatabaseConfig{
Cfg: cfg,
RunMigrations: true,
})
defer cleanup()
queries := cfg.GetDBQueries()
contextID, err := queries.CreateQuery(ctx, repository.Querytype(repository.QuerytypeContextFull))
assert.NoError(t, err)
jobs, err := queries.ListQueryJobIDs(ctx, contextID)
assert.NoError(t, err)
assert.ElementsMatch(t, []pgtype.UUID{}, jobs)
clientOneID, err := queries.CreateClient(ctx, "example_client")
assert.NoError(t, err)
jobOneID, err := queries.CreateJob(ctx, clientOneID)
assert.NoError(t, err)
versionOne, err := queries.AddLatestCollectorVersion(ctx, jobOneID)
assert.NoError(t, err)
err = queries.SetActiveCollectorVersion(ctx, &repository.SetActiveCollectorVersionParams{
Jobid: jobOneID,
Versionid: versionOne,
})
assert.NoError(t, err)
err = queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{
Jobid: jobOneID,
Queryid: contextID,
Addedversion: 1,
Name: "example_key",
})
assert.NoError(t, err)
jobs, err = queries.ListQueryJobIDs(ctx, contextID)
assert.NoError(t, err)
assert.ElementsMatch(t, []pgtype.UUID{jobOneID}, jobs)
jobTwoID, err := queries.CreateJob(ctx, clientOneID)
assert.NoError(t, err)
versionTwo, err := queries.AddLatestCollectorVersion(ctx, jobTwoID)
assert.NoError(t, err)
err = queries.SetActiveCollectorVersion(ctx, &repository.SetActiveCollectorVersionParams{
Jobid: jobTwoID,
Versionid: versionTwo,
})
assert.NoError(t, err)
err = queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{
Jobid: jobTwoID,
Queryid: contextID,
Addedversion: 1,
Name: "example_key",
})
assert.NoError(t, err)
jobs, err = queries.ListQueryJobIDs(ctx, contextID)
assert.NoError(t, err)
assert.ElementsMatch(t, []pgtype.UUID{jobOneID, jobTwoID}, jobs)
clientTwoID, err := queries.CreateClient(ctx, "example_client_two")
assert.NoError(t, err)
jobThreeID, err := queries.CreateJob(ctx, clientTwoID)
assert.NoError(t, err)
versionThree, err := queries.AddLatestCollectorVersion(ctx, jobThreeID)
assert.NoError(t, err)
err = queries.SetActiveCollectorVersion(ctx, &repository.SetActiveCollectorVersionParams{
Jobid: jobThreeID,
Versionid: versionThree,
})
assert.NoError(t, err)
err = queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{
Jobid: jobThreeID,
Queryid: contextID,
Addedversion: 1,
Name: "example_key",
})
assert.NoError(t, err)
jobs, err = queries.ListQueryJobIDs(ctx, contextID)
assert.NoError(t, err)
assert.ElementsMatch(t, []pgtype.UUID{jobOneID, jobTwoID, jobThreeID}, jobs)
jsonID, err := queries.CreateQuery(ctx, repository.Querytype(repository.QuerytypeJsonExtractor))
assert.NoError(t, err)
_, err = queries.AddLatestQueryVersion(ctx, jsonID)
assert.NoError(t, err)
err = queries.AddRequiredQuery(ctx, &repository.AddRequiredQueryParams{
Queryid: jsonID,
Requiredqueryid: contextID,
Addedversion: 1,
})
assert.NoError(t, err)
err = queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{
Jobid: jobOneID,
Queryid: jsonID,
Addedversion: 1,
Name: "example_key",
})
assert.NoError(t, err)
jobs, err = queries.ListQueryJobIDs(ctx, jsonID)
assert.NoError(t, err)
assert.ElementsMatch(t, []pgtype.UUID{jobOneID}, jobs)
}