package repository_test import ( "context" "os" "path" "testing" "queryorchestration/internal/database/repository" "queryorchestration/internal/serviceconfig" "queryorchestration/internal/test" "github.com/stretchr/testify/assert" ) func TestJob(t *testing.T) { if testing.Short() { t.Skip("Skipping long test in short mode") } ctx := context.Background() cfg := &serviceconfig.BaseConfig{} test.SetCfgProvider(t, cfg) cfg.SetBasePath(path.Join(os.Getenv("PWD"), "../../..")) _, cleanup := test.CreateDB(t, ctx, &test.CreateDatabaseConfig{ Cfg: cfg, RunMigrations: true, }) defer cleanup() queries := cfg.GetDBQueries() clientId, err := queries.CreateClient(ctx, "example_client") assert.NoError(t, err) id, err := queries.CreateJob(ctx, clientId) assert.NoError(t, err) assert.NotEmpty(t, id) job, err := queries.GetJob(ctx, id) assert.NoError(t, err) assert.EqualExportedValues(t, &repository.GetJobRow{ ID: id, Clientid: clientId, Cansync: false, }, job) err = queries.AddJobCanSync(ctx, &repository.AddJobCanSyncParams{ Cansync: true, Jobid: id, }) assert.NoError(t, err) job, err = queries.GetJob(ctx, id) assert.NoError(t, err) assert.EqualExportedValues(t, &repository.GetJobRow{ ID: id, Clientid: clientId, Cansync: true, }, job) } func TestListJobDocumentIDs(t *testing.T) { if testing.Short() { t.Skip("Skipping long test in short mode") } ctx := context.Background() cfg := &serviceconfig.BaseConfig{} test.SetCfgProvider(t, cfg) cfg.SetBasePath(path.Join(os.Getenv("PWD"), "../../..")) _, cleanup := test.CreateDB(t, ctx, &test.CreateDatabaseConfig{ Cfg: cfg, RunMigrations: true, }) defer cleanup() queries := cfg.GetDBQueries() clientId, err := queries.CreateClient(ctx, "example_client") assert.NoError(t, err) id, err := queries.CreateJob(ctx, clientId) assert.NoError(t, err) assert.NotEmpty(t, id) ids, err := queries.ListJobDocumentIDsBatch(ctx, &repository.ListJobDocumentIDsBatchParams{ Jobid: id, Batchsize: 1, Pageoffset: 0, }) assert.NoError(t, err) assert.Len(t, ids, 0) docOne, err := queries.CreateDocument(ctx, &repository.CreateDocumentParams{ Jobid: id, Hash: "example_hash", }) assert.NoError(t, err) ids, err = queries.ListJobDocumentIDsBatch(ctx, &repository.ListJobDocumentIDsBatchParams{ Jobid: id, Batchsize: 1, Pageoffset: 0, }) assert.NoError(t, err) assert.Len(t, ids, 1) total := int64(1) assert.ElementsMatch(t, []*repository.ListJobDocumentIDsBatchRow{ { ID: docOne, Totalcount: &total, }, }, ids) docTwo, err := queries.CreateDocument(ctx, &repository.CreateDocumentParams{ Jobid: id, Hash: "example_hash_two", }) assert.NoError(t, err) ids, err = queries.ListJobDocumentIDsBatch(ctx, &repository.ListJobDocumentIDsBatchParams{ Jobid: id, Batchsize: 1, Pageoffset: 0, }) assert.NoError(t, err) assert.Len(t, ids, 1) total = int64(2) assert.ElementsMatch(t, []*repository.ListJobDocumentIDsBatchRow{ { ID: docOne, Totalcount: &total, }, }, ids) ids, err = queries.ListJobDocumentIDsBatch(ctx, &repository.ListJobDocumentIDsBatchParams{ Jobid: id, Batchsize: 1, Pageoffset: 1, }) assert.NoError(t, err) assert.Len(t, ids, 1) assert.ElementsMatch(t, []*repository.ListJobDocumentIDsBatchRow{ { ID: docTwo, Totalcount: &total, }, }, ids) ids, err = queries.ListJobDocumentIDsBatch(ctx, &repository.ListJobDocumentIDsBatchParams{ Jobid: id, Batchsize: 2, Pageoffset: 0, }) assert.NoError(t, err) assert.Len(t, ids, 2) assert.ElementsMatch(t, []*repository.ListJobDocumentIDsBatchRow{ { ID: docOne, Totalcount: &total, }, { ID: docTwo, Totalcount: &total, }, }, ids) } func TestJobSync(t *testing.T) { if testing.Short() { t.Skip("Skipping long test in short mode") } ctx := context.Background() cfg := &serviceconfig.BaseConfig{} test.SetCfgProvider(t, cfg) cfg.SetBasePath(path.Join(os.Getenv("PWD"), "../../..")) _, cleanup := test.CreateDB(t, ctx, &test.CreateDatabaseConfig{ Cfg: cfg, RunMigrations: true, }) defer cleanup() queries := cfg.GetDBQueries() contextQueryID, err := queries.CreateQuery(ctx, repository.Querytype(repository.QuerytypeContextFull)) assert.NoError(t, err) _, err = queries.AddLatestQueryVersion(ctx, contextQueryID) assert.NoError(t, err) err = queries.AddActiveQueryVersion(ctx, &repository.AddActiveQueryVersionParams{ Queryid: contextQueryID, Versionid: 1, }) assert.NoError(t, err) jsonQueryID, err := queries.CreateQuery(ctx, repository.Querytype(repository.QuerytypeJsonExtractor)) assert.NoError(t, err) _, err = queries.AddLatestQueryVersion(ctx, jsonQueryID) assert.NoError(t, err) err = queries.AddActiveQueryVersion(ctx, &repository.AddActiveQueryVersionParams{ Queryid: jsonQueryID, Versionid: 1, }) assert.NoError(t, err) err = queries.AddRequiredQuery(ctx, &repository.AddRequiredQueryParams{ Queryid: jsonQueryID, Requiredqueryid: contextQueryID, Addedversion: 1, }) assert.NoError(t, err) clientId, err := queries.CreateClient(ctx, "example_client") assert.NoError(t, err) jobId, err := queries.CreateJob(ctx, clientId) assert.NoError(t, err) _, err = queries.AddLatestCollectorVersion(ctx, jobId) assert.NoError(t, err) err = queries.SetActiveCollectorVersion(ctx, &repository.SetActiveCollectorVersionParams{ Versionid: 1, Jobid: jobId, }) assert.NoError(t, err) err = queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{ Jobid: jobId, Queryid: jsonQueryID, Addedversion: 1, Name: "example_key", }) assert.NoError(t, err) isSynced, err := queries.IsJobSynced(ctx, jobId) assert.NoError(t, err) assert.True(t, isSynced) documentNoCleanID, err := queries.CreateDocument(ctx, &repository.CreateDocumentParams{ Jobid: jobId, Hash: "example_noclean", }) assert.NoError(t, err) isSynced, err = queries.IsJobSynced(ctx, jobId) assert.NoError(t, err) assert.False(t, isSynced) bucket := "example_bucket" key := "example_key" cleanid, err := queries.AddDocumentClean(ctx, &repository.AddDocumentCleanParams{ Documentid: documentNoCleanID, Fail: repository.NullCleanfailtype{ Valid: true, Cleanfailtype: repository.CleanfailtypeInvalidMimetype, }, }) assert.NoError(t, err) err = queries.AddDocumentCleanEntry(ctx, &repository.AddDocumentCleanEntryParams{ Cleanid: cleanid, Version: 1, }) assert.NoError(t, err) isSynced, err = queries.IsJobSynced(ctx, jobId) assert.NoError(t, err) assert.True(t, isSynced) documentID, err := queries.CreateDocument(ctx, &repository.CreateDocumentParams{ Jobid: jobId, Hash: "example_hash", }) assert.NoError(t, err) isSynced, err = queries.IsJobSynced(ctx, jobId) assert.NoError(t, err) assert.False(t, isSynced) keytwo := "example_key_2" cleantwoid, err := queries.AddDocumentClean(ctx, &repository.AddDocumentCleanParams{ Documentid: documentID, Bucket: &bucket, Key: &keytwo, Mimetype: repository.NullCleanmimetype{ Valid: true, Cleanmimetype: repository.CleanmimetypeApplicationPdf, }, }) assert.NoError(t, err) err = queries.AddDocumentCleanEntry(ctx, &repository.AddDocumentCleanEntryParams{ Cleanid: cleantwoid, Version: 1, }) assert.NoError(t, err) isSynced, err = queries.IsJobSynced(ctx, jobId) assert.NoError(t, err) assert.False(t, isSynced) cleanentry, err := queries.GetDocumentCleanEntry(ctx, documentID) assert.NoError(t, err) err = queries.AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{ Version: 1, Bucket: "hi", Key: "hello", Cleanentryid: cleanentry.ID, }) assert.NoError(t, err) isSynced, err = queries.IsJobSynced(ctx, jobId) assert.NoError(t, err) assert.False(t, isSynced) textentry, err := queries.GetDocumentTextEntry(ctx, documentID) assert.NoError(t, err) depresultid, err := queries.AddResult(ctx, &repository.AddResultParams{ Queryid: contextQueryID, Value: "example_value", Textentryid: textentry.ID, Queryversion: 1, }) assert.NoError(t, err) isSynced, err = queries.IsJobSynced(ctx, jobId) assert.NoError(t, err) assert.False(t, isSynced) resultid, err := queries.AddResult(ctx, &repository.AddResultParams{ Queryid: jsonQueryID, Value: "example_value", Textentryid: textentry.ID, Queryversion: 1, }) assert.NoError(t, err) isSynced, err = queries.IsJobSynced(ctx, jobId) assert.NoError(t, err) assert.False(t, isSynced) err = queries.AddResultDependency(ctx, &repository.AddResultDependencyParams{ Resultid: resultid, Requiredresultid: depresultid, }) assert.NoError(t, err) isSynced, err = queries.IsJobSynced(ctx, jobId) assert.NoError(t, err) assert.True(t, isSynced) _, err = queries.AddLatestQueryVersion(ctx, contextQueryID) assert.NoError(t, err) err = queries.AddActiveQueryVersion(ctx, &repository.AddActiveQueryVersionParams{ Queryid: contextQueryID, Versionid: 2, }) assert.NoError(t, err) isSynced, err = queries.IsJobSynced(ctx, jobId) assert.NoError(t, err) assert.False(t, isSynced) depresultid, err = queries.AddResult(ctx, &repository.AddResultParams{ Queryid: contextQueryID, Value: "example_value", Textentryid: textentry.ID, Queryversion: 2, }) assert.NoError(t, err) isSynced, err = queries.IsJobSynced(ctx, jobId) assert.NoError(t, err) assert.False(t, isSynced) resultid, err = queries.AddResult(ctx, &repository.AddResultParams{ Queryid: jsonQueryID, Value: "example_value", Textentryid: textentry.ID, Queryversion: 1, }) assert.NoError(t, err) isSynced, err = queries.IsJobSynced(ctx, jobId) assert.NoError(t, err) assert.False(t, isSynced) err = queries.AddResultDependency(ctx, &repository.AddResultDependencyParams{ Resultid: resultid, Requiredresultid: depresultid, }) assert.NoError(t, err) isSynced, err = queries.IsJobSynced(ctx, jobId) assert.NoError(t, err) assert.True(t, isSynced) err = queries.AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{ Version: 1, Bucket: "hi", Key: "hello", Cleanentryid: cleanentry.ID, }) assert.NoError(t, err) isSynced, err = queries.IsJobSynced(ctx, jobId) assert.NoError(t, err) assert.False(t, isSynced) textentry, err = queries.GetDocumentTextEntry(ctx, documentID) assert.NoError(t, err) depresultid, err = queries.AddResult(ctx, &repository.AddResultParams{ Queryid: contextQueryID, Value: "example_value", Textentryid: textentry.ID, Queryversion: 2, }) assert.NoError(t, err) isSynced, err = queries.IsJobSynced(ctx, jobId) assert.NoError(t, err) assert.False(t, isSynced) resultid, err = queries.AddResult(ctx, &repository.AddResultParams{ Queryid: jsonQueryID, Value: "example_value", Textentryid: textentry.ID, Queryversion: 1, }) assert.NoError(t, err) isSynced, err = queries.IsJobSynced(ctx, jobId) assert.NoError(t, err) assert.False(t, isSynced) err = queries.AddResultDependency(ctx, &repository.AddResultDependencyParams{ Resultid: resultid, Requiredresultid: depresultid, }) assert.NoError(t, err) isSynced, err = queries.IsJobSynced(ctx, jobId) assert.NoError(t, err) assert.True(t, isSynced) cleanthreeid, err := queries.AddDocumentClean(ctx, &repository.AddDocumentCleanParams{ Documentid: documentID, Bucket: &bucket, Key: &key, Mimetype: repository.NullCleanmimetype{ Valid: true, Cleanmimetype: repository.CleanmimetypeApplicationPdf, }, }) assert.NoError(t, err) err = queries.AddDocumentCleanEntry(ctx, &repository.AddDocumentCleanEntryParams{ Cleanid: cleanthreeid, Version: 1, }) assert.NoError(t, err) isSynced, err = queries.IsJobSynced(ctx, jobId) assert.NoError(t, err) assert.False(t, isSynced) cleanentry, err = queries.GetDocumentCleanEntry(ctx, documentID) assert.NoError(t, err) err = queries.AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{ Version: 1, Bucket: "hi", Key: "hello", Cleanentryid: cleanentry.ID, }) assert.NoError(t, err) isSynced, err = queries.IsJobSynced(ctx, jobId) assert.NoError(t, err) assert.False(t, isSynced) textentry, err = queries.GetDocumentTextEntry(ctx, documentID) assert.NoError(t, err) depresultid, err = queries.AddResult(ctx, &repository.AddResultParams{ Queryid: contextQueryID, Value: "example_value", Textentryid: textentry.ID, Queryversion: 2, }) assert.NoError(t, err) isSynced, err = queries.IsJobSynced(ctx, jobId) assert.NoError(t, err) assert.False(t, isSynced) resultid, err = queries.AddResult(ctx, &repository.AddResultParams{ Queryid: jsonQueryID, Value: "example_value", Textentryid: textentry.ID, Queryversion: 1, }) assert.NoError(t, err) isSynced, err = queries.IsJobSynced(ctx, jobId) assert.NoError(t, err) assert.False(t, isSynced) err = queries.AddResultDependency(ctx, &repository.AddResultDependencyParams{ Resultid: resultid, Requiredresultid: depresultid, }) assert.NoError(t, err) isSynced, err = queries.IsJobSynced(ctx, jobId) assert.NoError(t, err) assert.True(t, isSynced) _, err = queries.AddLatestCollectorVersion(ctx, jobId) assert.NoError(t, err) isSynced, err = queries.IsJobSynced(ctx, jobId) assert.NoError(t, err) assert.True(t, isSynced) collversion := int32(2) err = queries.SetActiveCollectorVersion(ctx, &repository.SetActiveCollectorVersionParams{ Versionid: 2, Jobid: jobId, }) assert.NoError(t, err) isSynced, err = queries.IsJobSynced(ctx, jobId) assert.NoError(t, err) assert.True(t, isSynced) err = queries.RemoveCollectorQuery(ctx, &repository.RemoveCollectorQueryParams{ Jobid: jobId, Queryid: jsonQueryID, Removedversion: &collversion, }) assert.NoError(t, err) isSynced, err = queries.IsJobSynced(ctx, jobId) assert.NoError(t, err) assert.True(t, isSynced) err = queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{ Jobid: jobId, Queryid: jsonQueryID, Addedversion: 2, Name: "second_key", }) assert.NoError(t, err) isSynced, err = queries.IsJobSynced(ctx, jobId) assert.NoError(t, err) assert.True(t, isSynced) err = queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{ Jobid: jobId, Queryid: contextQueryID, Addedversion: 2, Name: "example_key", }) assert.NoError(t, err) isSynced, err = queries.IsJobSynced(ctx, jobId) assert.NoError(t, err) assert.True(t, isSynced) superQueryID, err := queries.CreateQuery(ctx, repository.Querytype(repository.QuerytypeJsonExtractor)) assert.NoError(t, err) _, err = queries.AddLatestQueryVersion(ctx, superQueryID) assert.NoError(t, err) err = queries.AddActiveQueryVersion(ctx, &repository.AddActiveQueryVersionParams{ Queryid: superQueryID, Versionid: 1, }) assert.NoError(t, err) err = queries.AddRequiredQuery(ctx, &repository.AddRequiredQueryParams{ Queryid: contextQueryID, Requiredqueryid: superQueryID, Addedversion: 1, }) assert.NoError(t, err) superresultid, err := queries.AddResult(ctx, &repository.AddResultParams{ Queryid: superQueryID, Value: "example_value", Textentryid: textentry.ID, Queryversion: 1, }) assert.NoError(t, err) isSynced, err = queries.IsJobSynced(ctx, jobId) assert.NoError(t, err) assert.False(t, isSynced) depresultid, err = queries.AddResult(ctx, &repository.AddResultParams{ Queryid: contextQueryID, Value: "example_value", Textentryid: textentry.ID, Queryversion: 2, }) assert.NoError(t, err) isSynced, err = queries.IsJobSynced(ctx, jobId) assert.NoError(t, err) assert.False(t, isSynced) resultid, err = queries.AddResult(ctx, &repository.AddResultParams{ Queryid: jsonQueryID, Value: "example_value", Textentryid: textentry.ID, Queryversion: 1, }) assert.NoError(t, err) isSynced, err = queries.IsJobSynced(ctx, jobId) assert.NoError(t, err) assert.False(t, isSynced) err = queries.AddResultDependency(ctx, &repository.AddResultDependencyParams{ Resultid: resultid, Requiredresultid: depresultid, }) assert.NoError(t, err) isSynced, err = queries.IsJobSynced(ctx, jobId) assert.NoError(t, err) assert.False(t, isSynced) err = queries.AddResultDependency(ctx, &repository.AddResultDependencyParams{ Resultid: depresultid, Requiredresultid: superresultid, }) assert.NoError(t, err) isSynced, err = queries.IsJobSynced(ctx, jobId) assert.NoError(t, err) assert.True(t, isSynced) }