query-orchestration/internal/database/repository/job_test.go

package repository_test

import (
	"context"
	"os"
	"path"
	"testing"

	"queryorchestration/internal/database/repository"
	"queryorchestration/internal/serviceconfig"
	"queryorchestration/internal/test"

	"github.com/stretchr/testify/assert"
)

func TestJob(t *testing.T) {
	if testing.Short() {
		t.Skip("Skipping long test in short mode")
	}
	ctx := context.Background()

	cfg := &serviceconfig.BaseConfig{}
	test.SetCfgProvider(t, cfg)
	cfg.SetBasePath(path.Join(os.Getenv("PWD"), "../../.."))
	_, cleanup := test.CreateDB(t, ctx, &test.CreateDatabaseConfig{
		Cfg:           cfg,
		RunMigrations: true,
	})
	defer cleanup()

	queries := cfg.GetDBQueries()

	clientId, err := queries.CreateClient(ctx, "example_client")
	assert.NoError(t, err)

	id, err := queries.CreateJob(ctx, clientId)
	assert.NoError(t, err)
	assert.NotEmpty(t, id)

	job, err := queries.GetJob(ctx, id)
	assert.NoError(t, err)
	assert.EqualExportedValues(t, &repository.GetJobRow{
		ID:       id,
		Clientid: clientId,
		Cansync:  false,
	}, job)

	err = queries.AddJobCanSync(ctx, &repository.AddJobCanSyncParams{
		Cansync: true,
		Jobid:   id,
	})
	assert.NoError(t, err)

	job, err = queries.GetJob(ctx, id)
	assert.NoError(t, err)
	assert.EqualExportedValues(t, &repository.GetJobRow{
		ID:       id,
		Clientid: clientId,
		Cansync:  true,
	}, job)
}
func TestListJobDocumentIDs(t *testing.T) {
	if testing.Short() {
		t.Skip("Skipping long test in short mode")
	}
	ctx := context.Background()

	cfg := &serviceconfig.BaseConfig{}
	test.SetCfgProvider(t, cfg)
	cfg.SetBasePath(path.Join(os.Getenv("PWD"), "../../.."))
	_, cleanup := test.CreateDB(t, ctx, &test.CreateDatabaseConfig{
		Cfg:           cfg,
		RunMigrations: true,
	})
	defer cleanup()

	queries := cfg.GetDBQueries()

	clientId, err := queries.CreateClient(ctx, "example_client")
	assert.NoError(t, err)

	id, err := queries.CreateJob(ctx, clientId)
	assert.NoError(t, err)
	assert.NotEmpty(t, id)

	ids, err := queries.ListJobDocumentIDsBatch(ctx, &repository.ListJobDocumentIDsBatchParams{
		Jobid:      id,
		Batchsize:  1,
		Pageoffset: 0,
	})
	assert.NoError(t, err)
	assert.Len(t, ids, 0)

	docOne, err := queries.CreateDocument(ctx, &repository.CreateDocumentParams{
		Jobid: id,
		Hash:  "example_hash",
	})
	assert.NoError(t, err)

	ids, err = queries.ListJobDocumentIDsBatch(ctx, &repository.ListJobDocumentIDsBatchParams{
		Jobid:      id,
		Batchsize:  1,
		Pageoffset: 0,
	})
	assert.NoError(t, err)
	assert.Len(t, ids, 1)
	total := int64(1)
	assert.ElementsMatch(t, []*repository.ListJobDocumentIDsBatchRow{
		{
			ID:         docOne,
			Totalcount: &total,
		},
	}, ids)

	docTwo, err := queries.CreateDocument(ctx, &repository.CreateDocumentParams{
		Jobid: id,
		Hash:  "example_hash_two",
	})
	assert.NoError(t, err)

	ids, err = queries.ListJobDocumentIDsBatch(ctx, &repository.ListJobDocumentIDsBatchParams{
		Jobid:      id,
		Batchsize:  1,
		Pageoffset: 0,
	})
	assert.NoError(t, err)
	assert.Len(t, ids, 1)
	total = int64(2)
	assert.ElementsMatch(t, []*repository.ListJobDocumentIDsBatchRow{
		{
			ID:         docOne,
			Totalcount: &total,
		},
	}, ids)

	ids, err = queries.ListJobDocumentIDsBatch(ctx, &repository.ListJobDocumentIDsBatchParams{
		Jobid:      id,
		Batchsize:  1,
		Pageoffset: 1,
	})
	assert.NoError(t, err)
	assert.Len(t, ids, 1)
	assert.ElementsMatch(t, []*repository.ListJobDocumentIDsBatchRow{
		{
			ID:         docTwo,
			Totalcount: &total,
		},
	}, ids)

	ids, err = queries.ListJobDocumentIDsBatch(ctx, &repository.ListJobDocumentIDsBatchParams{
		Jobid:      id,
		Batchsize:  2,
		Pageoffset: 0,
	})
	assert.NoError(t, err)
	assert.Len(t, ids, 2)
	assert.ElementsMatch(t, []*repository.ListJobDocumentIDsBatchRow{
		{
			ID:         docOne,
			Totalcount: &total,
		},
		{
			ID:         docTwo,
			Totalcount: &total,
		},
	}, ids)
}

func TestJobSync(t *testing.T) {
	if testing.Short() {
		t.Skip("Skipping long test in short mode")
	}
	ctx := context.Background()

	cfg := &serviceconfig.BaseConfig{}
	test.SetCfgProvider(t, cfg)
	cfg.SetBasePath(path.Join(os.Getenv("PWD"), "../../.."))
	_, cleanup := test.CreateDB(t, ctx, &test.CreateDatabaseConfig{
		Cfg:           cfg,
		RunMigrations: true,
	})
	defer cleanup()

	queries := cfg.GetDBQueries()

	contextQueryID, err := queries.CreateQuery(ctx, repository.Querytype(repository.QuerytypeContextFull))
	assert.NoError(t, err)
	_, err = queries.AddLatestQueryVersion(ctx, contextQueryID)
	assert.NoError(t, err)
	err = queries.AddActiveQueryVersion(ctx, &repository.AddActiveQueryVersionParams{
		Queryid:   contextQueryID,
		Versionid: 1,
	})
	assert.NoError(t, err)
	jsonQueryID, err := queries.CreateQuery(ctx, repository.Querytype(repository.QuerytypeJsonExtractor))
	assert.NoError(t, err)
	_, err = queries.AddLatestQueryVersion(ctx, jsonQueryID)
	assert.NoError(t, err)
	err = queries.AddActiveQueryVersion(ctx, &repository.AddActiveQueryVersionParams{
		Queryid:   jsonQueryID,
		Versionid: 1,
	})
	assert.NoError(t, err)
	err = queries.AddRequiredQuery(ctx, &repository.AddRequiredQueryParams{
		Queryid:         jsonQueryID,
		Requiredqueryid: contextQueryID,
		Addedversion:    1,
	})
	assert.NoError(t, err)

	clientId, err := queries.CreateClient(ctx, "example_client")
	assert.NoError(t, err)
	jobId, err := queries.CreateJob(ctx, clientId)
	assert.NoError(t, err)

	_, err = queries.AddLatestCollectorVersion(ctx, jobId)
	assert.NoError(t, err)
	err = queries.SetActiveCollectorVersion(ctx, &repository.SetActiveCollectorVersionParams{
		Versionid: 1,
		Jobid:     jobId,
	})
	assert.NoError(t, err)
	err = queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{
		Jobid:        jobId,
		Queryid:      jsonQueryID,
		Addedversion: 1,
		Name:         "example_key",
	})
	assert.NoError(t, err)

	isSynced, err := queries.IsJobSynced(ctx, jobId)
	assert.NoError(t, err)
	assert.True(t, isSynced)

	documentNoCleanID, err := queries.CreateDocument(ctx, &repository.CreateDocumentParams{
		Jobid: jobId,
		Hash:  "example_noclean",
	})
	assert.NoError(t, err)

	isSynced, err = queries.IsJobSynced(ctx, jobId)
	assert.NoError(t, err)
	assert.False(t, isSynced)

	bucket := "example_bucket"
	key := "example_key"
	cleanid, err := queries.AddDocumentClean(ctx, &repository.AddDocumentCleanParams{
		Documentid: documentNoCleanID,
		Fail: repository.NullCleanfailtype{
			Valid:         true,
			Cleanfailtype: repository.CleanfailtypeInvalidMimetype,
		},
	})
	assert.NoError(t, err)
	err = queries.AddDocumentCleanEntry(ctx, &repository.AddDocumentCleanEntryParams{
		Cleanid: cleanid,
		Version: 1,
	})
	assert.NoError(t, err)

	isSynced, err = queries.IsJobSynced(ctx, jobId)
	assert.NoError(t, err)
	assert.True(t, isSynced)

	documentID, err := queries.CreateDocument(ctx, &repository.CreateDocumentParams{
		Jobid: jobId,
		Hash:  "example_hash",
	})
	assert.NoError(t, err)

	isSynced, err = queries.IsJobSynced(ctx, jobId)
	assert.NoError(t, err)
	assert.False(t, isSynced)

	keytwo := "example_key_2"
	cleantwoid, err := queries.AddDocumentClean(ctx, &repository.AddDocumentCleanParams{
		Documentid: documentID,
		Bucket:     &bucket,
		Key:        &keytwo,
		Mimetype: repository.NullCleanmimetype{
			Valid:         true,
			Cleanmimetype: repository.CleanmimetypeApplicationPdf,
		},
	})
	assert.NoError(t, err)
	err = queries.AddDocumentCleanEntry(ctx, &repository.AddDocumentCleanEntryParams{
		Cleanid: cleantwoid,
		Version: 1,
	})
	assert.NoError(t, err)

	isSynced, err = queries.IsJobSynced(ctx, jobId)
	assert.NoError(t, err)
	assert.False(t, isSynced)

	cleanentry, err := queries.GetDocumentCleanEntry(ctx, documentID)
	assert.NoError(t, err)

	err = queries.AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{
		Version:      1,
		Bucket:       "hi",
		Key:          "hello",
		Cleanentryid: cleanentry.ID,
	})
	assert.NoError(t, err)

	isSynced, err = queries.IsJobSynced(ctx, jobId)
	assert.NoError(t, err)
	assert.False(t, isSynced)

	textentry, err := queries.GetDocumentTextEntry(ctx, documentID)
	assert.NoError(t, err)

	depresultid, err := queries.AddResult(ctx, &repository.AddResultParams{
		Queryid:      contextQueryID,
		Value:        "example_value",
		Textentryid:  textentry.ID,
		Queryversion: 1,
	})
	assert.NoError(t, err)

	isSynced, err = queries.IsJobSynced(ctx, jobId)
	assert.NoError(t, err)
	assert.False(t, isSynced)

	resultid, err := queries.AddResult(ctx, &repository.AddResultParams{
		Queryid:      jsonQueryID,
		Value:        "example_value",
		Textentryid:  textentry.ID,
		Queryversion: 1,
	})
	assert.NoError(t, err)

	isSynced, err = queries.IsJobSynced(ctx, jobId)
	assert.NoError(t, err)
	assert.False(t, isSynced)

	err = queries.AddResultDependency(ctx, &repository.AddResultDependencyParams{
		Resultid:         resultid,
		Requiredresultid: depresultid,
	})
	assert.NoError(t, err)

	isSynced, err = queries.IsJobSynced(ctx, jobId)
	assert.NoError(t, err)
	assert.True(t, isSynced)

	_, err = queries.AddLatestQueryVersion(ctx, contextQueryID)
	assert.NoError(t, err)
	err = queries.AddActiveQueryVersion(ctx, &repository.AddActiveQueryVersionParams{
		Queryid:   contextQueryID,
		Versionid: 2,
	})
	assert.NoError(t, err)

	isSynced, err = queries.IsJobSynced(ctx, jobId)
	assert.NoError(t, err)
	assert.False(t, isSynced)

	depresultid, err = queries.AddResult(ctx, &repository.AddResultParams{
		Queryid:      contextQueryID,
		Value:        "example_value",
		Textentryid:  textentry.ID,
		Queryversion: 2,
	})
	assert.NoError(t, err)

	isSynced, err = queries.IsJobSynced(ctx, jobId)
	assert.NoError(t, err)
	assert.False(t, isSynced)

	resultid, err = queries.AddResult(ctx, &repository.AddResultParams{
		Queryid:      jsonQueryID,
		Value:        "example_value",
		Textentryid:  textentry.ID,
		Queryversion: 1,
	})
	assert.NoError(t, err)

	isSynced, err = queries.IsJobSynced(ctx, jobId)
	assert.NoError(t, err)
	assert.False(t, isSynced)

	err = queries.AddResultDependency(ctx, &repository.AddResultDependencyParams{
		Resultid:         resultid,
		Requiredresultid: depresultid,
	})
	assert.NoError(t, err)

	isSynced, err = queries.IsJobSynced(ctx, jobId)
	assert.NoError(t, err)
	assert.True(t, isSynced)

	err = queries.AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{
		Version:      1,
		Bucket:       "hi",
		Key:          "hello",
		Cleanentryid: cleanentry.ID,
	})
	assert.NoError(t, err)

	isSynced, err = queries.IsJobSynced(ctx, jobId)
	assert.NoError(t, err)
	assert.False(t, isSynced)

	textentry, err = queries.GetDocumentTextEntry(ctx, documentID)
	assert.NoError(t, err)

	depresultid, err = queries.AddResult(ctx, &repository.AddResultParams{
		Queryid:      contextQueryID,
		Value:        "example_value",
		Textentryid:  textentry.ID,
		Queryversion: 2,
	})
	assert.NoError(t, err)

	isSynced, err = queries.IsJobSynced(ctx, jobId)
	assert.NoError(t, err)
	assert.False(t, isSynced)

	resultid, err = queries.AddResult(ctx, &repository.AddResultParams{
		Queryid:      jsonQueryID,
		Value:        "example_value",
		Textentryid:  textentry.ID,
		Queryversion: 1,
	})
	assert.NoError(t, err)

	isSynced, err = queries.IsJobSynced(ctx, jobId)
	assert.NoError(t, err)
	assert.False(t, isSynced)

	err = queries.AddResultDependency(ctx, &repository.AddResultDependencyParams{
		Resultid:         resultid,
		Requiredresultid: depresultid,
	})
	assert.NoError(t, err)

	isSynced, err = queries.IsJobSynced(ctx, jobId)
	assert.NoError(t, err)
	assert.True(t, isSynced)

	cleanthreeid, err := queries.AddDocumentClean(ctx, &repository.AddDocumentCleanParams{
		Documentid: documentID,
		Bucket:     &bucket,
		Key:        &key,
		Mimetype: repository.NullCleanmimetype{
			Valid:         true,
			Cleanmimetype: repository.CleanmimetypeApplicationPdf,
		},
	})
	assert.NoError(t, err)
	err = queries.AddDocumentCleanEntry(ctx, &repository.AddDocumentCleanEntryParams{
		Cleanid: cleanthreeid,
		Version: 1,
	})
	assert.NoError(t, err)

	isSynced, err = queries.IsJobSynced(ctx, jobId)
	assert.NoError(t, err)
	assert.False(t, isSynced)

	cleanentry, err = queries.GetDocumentCleanEntry(ctx, documentID)
	assert.NoError(t, err)

	err = queries.AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{
		Version:      1,
		Bucket:       "hi",
		Key:          "hello",
		Cleanentryid: cleanentry.ID,
	})
	assert.NoError(t, err)

	isSynced, err = queries.IsJobSynced(ctx, jobId)
	assert.NoError(t, err)
	assert.False(t, isSynced)

	textentry, err = queries.GetDocumentTextEntry(ctx, documentID)
	assert.NoError(t, err)

	depresultid, err = queries.AddResult(ctx, &repository.AddResultParams{
		Queryid:      contextQueryID,
		Value:        "example_value",
		Textentryid:  textentry.ID,
		Queryversion: 2,
	})
	assert.NoError(t, err)

	isSynced, err = queries.IsJobSynced(ctx, jobId)
	assert.NoError(t, err)
	assert.False(t, isSynced)

	resultid, err = queries.AddResult(ctx, &repository.AddResultParams{
		Queryid:      jsonQueryID,
		Value:        "example_value",
		Textentryid:  textentry.ID,
		Queryversion: 1,
	})
	assert.NoError(t, err)

	isSynced, err = queries.IsJobSynced(ctx, jobId)
	assert.NoError(t, err)
	assert.False(t, isSynced)

	err = queries.AddResultDependency(ctx, &repository.AddResultDependencyParams{
		Resultid:         resultid,
		Requiredresultid: depresultid,
	})
	assert.NoError(t, err)

	isSynced, err = queries.IsJobSynced(ctx, jobId)
	assert.NoError(t, err)
	assert.True(t, isSynced)

	_, err = queries.AddLatestCollectorVersion(ctx, jobId)
	assert.NoError(t, err)

	isSynced, err = queries.IsJobSynced(ctx, jobId)
	assert.NoError(t, err)
	assert.True(t, isSynced)

	collversion := int32(2)
	err = queries.SetActiveCollectorVersion(ctx, &repository.SetActiveCollectorVersionParams{
		Versionid: 2,
		Jobid:     jobId,
	})
	assert.NoError(t, err)

	isSynced, err = queries.IsJobSynced(ctx, jobId)
	assert.NoError(t, err)
	assert.True(t, isSynced)

	err = queries.RemoveCollectorQuery(ctx, &repository.RemoveCollectorQueryParams{
		Jobid:          jobId,
		Queryid:        jsonQueryID,
		Removedversion: &collversion,
	})
	assert.NoError(t, err)

	isSynced, err = queries.IsJobSynced(ctx, jobId)
	assert.NoError(t, err)
	assert.True(t, isSynced)

	err = queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{
		Jobid:        jobId,
		Queryid:      jsonQueryID,
		Addedversion: 2,
		Name:         "second_key",
	})
	assert.NoError(t, err)

	isSynced, err = queries.IsJobSynced(ctx, jobId)
	assert.NoError(t, err)
	assert.True(t, isSynced)

	err = queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{
		Jobid:        jobId,
		Queryid:      contextQueryID,
		Addedversion: 2,
		Name:         "example_key",
	})
	assert.NoError(t, err)

	isSynced, err = queries.IsJobSynced(ctx, jobId)
	assert.NoError(t, err)
	assert.True(t, isSynced)

	superQueryID, err := queries.CreateQuery(ctx, repository.Querytype(repository.QuerytypeJsonExtractor))
	assert.NoError(t, err)
	_, err = queries.AddLatestQueryVersion(ctx, superQueryID)
	assert.NoError(t, err)
	err = queries.AddActiveQueryVersion(ctx, &repository.AddActiveQueryVersionParams{
		Queryid:   superQueryID,
		Versionid: 1,
	})
	assert.NoError(t, err)
	err = queries.AddRequiredQuery(ctx, &repository.AddRequiredQueryParams{
		Queryid:         contextQueryID,
		Requiredqueryid: superQueryID,
		Addedversion:    1,
	})
	assert.NoError(t, err)

	superresultid, err := queries.AddResult(ctx, &repository.AddResultParams{
		Queryid:      superQueryID,
		Value:        "example_value",
		Textentryid:  textentry.ID,
		Queryversion: 1,
	})
	assert.NoError(t, err)

	isSynced, err = queries.IsJobSynced(ctx, jobId)
	assert.NoError(t, err)
	assert.False(t, isSynced)

	depresultid, err = queries.AddResult(ctx, &repository.AddResultParams{
		Queryid:      contextQueryID,
		Value:        "example_value",
		Textentryid:  textentry.ID,
		Queryversion: 2,
	})
	assert.NoError(t, err)

	isSynced, err = queries.IsJobSynced(ctx, jobId)
	assert.NoError(t, err)
	assert.False(t, isSynced)

	resultid, err = queries.AddResult(ctx, &repository.AddResultParams{
		Queryid:      jsonQueryID,
		Value:        "example_value",
		Textentryid:  textentry.ID,
		Queryversion: 1,
	})
	assert.NoError(t, err)

	isSynced, err = queries.IsJobSynced(ctx, jobId)
	assert.NoError(t, err)
	assert.False(t, isSynced)

	err = queries.AddResultDependency(ctx, &repository.AddResultDependencyParams{
		Resultid:         resultid,
		Requiredresultid: depresultid,
	})
	assert.NoError(t, err)

	isSynced, err = queries.IsJobSynced(ctx, jobId)
	assert.NoError(t, err)
	assert.False(t, isSynced)

	err = queries.AddResultDependency(ctx, &repository.AddResultDependencyParams{
		Resultid:         depresultid,
		Requiredresultid: superresultid,
	})
	assert.NoError(t, err)

	isSynced, err = queries.IsJobSynced(ctx, jobId)
	assert.NoError(t, err)
	assert.True(t, isSynced)
}