Merged in feature/collectorset (pull request #96)

Set Collector

* set

internal/collector/set/service.go

@@ -0,0 +1,32 @@
+package collectorset
+
+import (
+	collector "queryorchestration/internal/collector"
+	cleanversion "queryorchestration/internal/document/clean/version"
+	textversion "queryorchestration/internal/document/text/version"
+	"queryorchestration/internal/serviceconfig"
+	"queryorchestration/internal/serviceconfig/queue/clientsync"
+)
+
+type Services struct {
+	CleanVersion *cleanversion.Service
+	TextVersion  *textversion.Service
+	Collector    *collector.Service
+}
+
+type ConfigProvider interface {
+	serviceconfig.ConfigProvider
+	clientsync.ConfigProvider
+}
+
+type Service struct {
+	cfg ConfigProvider
+	svc *Services
+}
+
+func New(cfg ConfigProvider, svc *Services) *Service {
+	return &Service{
+		cfg,
+		svc,
+	}
+}

internal/collector/set/service_test.go

@@ -0,0 +1,23 @@
+package collectorset_test
+
+import (
+	"testing"
+
+	collectorset "queryorchestration/internal/collector/set"
+	"queryorchestration/internal/database/repository"
+
+	"github.com/pashagolub/pgxmock/v3"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestService(t *testing.T) {
+	pool, err := pgxmock.NewPool()
+	require.NoError(t, err)
+	cfg := &CollectorSetConfig{}
+	cfg.DBPool = pool
+	cfg.DBQueries = repository.New(pool)
+
+	svc := collectorset.New(cfg, &collectorset.Services{})
+	assert.NotNil(t, svc)
+}

internal/collector/set/set.go

@@ -0,0 +1,339 @@
+package collectorset
+
+import (
+	"context"
+	"errors"
+	"log/slog"
+
+	clientsyncrunner "queryorchestration/api/clientSyncRunner"
+	"queryorchestration/internal/collector"
+	"queryorchestration/internal/database"
+	"queryorchestration/internal/database/repository"
+	"queryorchestration/internal/serviceconfig/queue"
+	"queryorchestration/internal/validation"
+
+	"github.com/google/uuid"
+	"github.com/jackc/pgx/v5/pgtype"
+)
+
+type SetParams struct {
+	ClientID        uuid.UUID
+	ActiveVersion   *int32
+	MinCleanVersion *int32
+	MinTextVersion  *int32
+	Fields          *map[string]uuid.UUID
+}
+
+func (s *Service) SetByClientId(ctx context.Context, params *SetParams) error {
+	current, err := s.svc.Collector.GetByClientID(ctx, params.ClientID)
+	if err != nil {
+		return err
+	}
+
+	dbparams, err := s.getSetParams(ctx, current, params)
+	if err != nil {
+		return err
+	}
+
+	err = s.submitSet(ctx, current, dbparams)
+	if err != nil {
+		return err
+	}
+
+	err = s.informSet(ctx, dbparams)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (s *Service) informSet(ctx context.Context, update *dbSetParams) error {
+	if update.ActiveVersion == nil {
+		return nil
+	}
+
+	return s.cfg.SendToQueue(ctx, &queue.SendParams{
+		QueueURL: s.cfg.GetClientSyncURL(),
+		Body: clientsyncrunner.Body{
+			ID: database.MustToUUID(update.ClientID),
+		},
+	})
+}
+
+type dbSetParams struct {
+	ClientID        pgtype.UUID
+	ActiveVersion   *int32
+	MinCleanVersion *int32
+	MinTextVersion  *int32
+	Fields          *map[string]pgtype.UUID
+}
+
+func (s *Service) getSetParams(ctx context.Context, current *collector.Collector, params *SetParams) (*dbSetParams, error) {
+	err := s.normalizeCodeVersions(current, params)
+	if err != nil {
+		return nil, err
+	}
+
+	fields, err := s.normalizeSetFieldsToDB(ctx, current.Fields, params.Fields)
+	if err != nil {
+		return nil, err
+	}
+
+	err = s.normalizeActiveVersion(current, params)
+	if err != nil {
+		return nil, err
+	}
+
+	activeVersionName, err := validation.GetFieldName(params, params.ActiveVersion)
+	if err != nil {
+		return nil, err
+	}
+	fieldsName, err := validation.GetFieldName(params, params.Fields)
+	if err != nil {
+		return nil, err
+	}
+
+	if (params.ActiveVersion == nil || *params.ActiveVersion == current.LatestVersion+1) &&
+		validation.AreAllPointersNilExcept(params, activeVersionName, fieldsName) &&
+		fields == nil {
+		return nil, errors.New("no changes")
+	}
+
+	return &dbSetParams{
+		ClientID:        database.MustToDBUUID(params.ClientID),
+		ActiveVersion:   params.ActiveVersion,
+		MinCleanVersion: params.MinCleanVersion,
+		MinTextVersion:  params.MinTextVersion,
+		Fields:          fields,
+	}, nil
+}
+
+func (s *Service) normalizeCodeVersions(current *collector.Collector, params *SetParams) error {
+	if current == nil {
+		return errors.New("current collector required")
+	}
+
+	if params == nil {
+		return nil
+	}
+
+	if params.MinCleanVersion != nil && *params.MinCleanVersion == current.MinCleanVersion {
+		params.MinCleanVersion = nil
+	}
+
+	if params.MinCleanVersion != nil {
+		err := s.svc.CleanVersion.IsValidVersion(*params.MinCleanVersion)
+		if err != nil {
+			return err
+		}
+	}
+
+	if params.MinTextVersion != nil && *params.MinTextVersion == current.MinTextVersion {
+		params.MinTextVersion = nil
+	}
+
+	if params.MinTextVersion != nil {
+		err := s.svc.TextVersion.IsValidVersion(*params.MinTextVersion)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (s *Service) normalizeActiveVersion(current *collector.Collector, params *SetParams) error {
+	if current == nil {
+		return errors.New("current collector required")
+	}
+
+	if params == nil || params.ActiveVersion == nil {
+		return nil
+	}
+
+	err := validation.NormalizeInClosedInterval(&params.ActiveVersion, current.ActiveVersion, 1, current.LatestVersion+1)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (s *Service) normalizeSetFieldsToDB(ctx context.Context, current map[string]uuid.UUID, ofields *map[string]uuid.UUID) (*map[string]pgtype.UUID, error) {
+	if ofields == nil || *ofields == nil {
+		return nil, nil
+	} else if len(*ofields) == 0 {
+		return nil, nil
+	}
+
+	dbm := map[string]pgtype.UUID{}
+	for name, id := range *ofields {
+		dbm[name] = database.MustToDBUUID(id)
+	}
+
+	removeIDs := getRemoveFields(current, &dbm)
+	addIDs := getAddFields(current, &dbm)
+
+	if len(addIDs) == 0 && len(removeIDs) == 0 {
+		return nil, nil
+	}
+
+	return s.normalizeFieldsToDB(ctx, ofields)
+}
+
+func (s *Service) submitSet(ctx context.Context, current *collector.Collector, params *dbSetParams) error {
+	err := s.cfg.ExecuteDBTransaction(ctx, func(ctx context.Context, qtx *repository.Queries) error {
+		activeName, err := validation.GetFieldName(params, params.ActiveVersion)
+		if err != nil {
+			return err
+		}
+		onlyactive := validation.AreAllPointersNilExcept(params, activeName)
+
+		var latestVersion int32
+		if !onlyactive {
+			latestVersion, err = qtx.AddLatestCollectorVersion(ctx, params.ClientID)
+			if err != nil {
+				return err
+			}
+		}
+
+		if params.ActiveVersion != nil {
+			err := qtx.SetActiveCollectorVersion(ctx, &repository.SetActiveCollectorVersionParams{
+				Clientid:  params.ClientID,
+				Versionid: *params.ActiveVersion,
+			})
+			if err != nil {
+				return err
+			}
+		}
+
+		if params.MinCleanVersion != nil {
+			err = qtx.SetCollectorCleanVersion(ctx, &repository.SetCollectorCleanVersionParams{
+				Clientid:     params.ClientID,
+				Versionid:    *params.MinCleanVersion,
+				Addedversion: latestVersion,
+			})
+			if err != nil {
+				return err
+			}
+		}
+
+		if params.MinTextVersion != nil {
+			err = qtx.SetCollectorTextVersion(ctx, &repository.SetCollectorTextVersionParams{
+				Clientid:     params.ClientID,
+				Versionid:    *params.MinTextVersion,
+				Addedversion: latestVersion,
+			})
+			if err != nil {
+				return err
+			}
+		}
+
+		if params.Fields != nil {
+			removeIDs := getRemoveFields(current.Fields, params.Fields)
+			for _, field := range removeIDs {
+				err := qtx.RemoveCollectorQuery(ctx, &repository.RemoveCollectorQueryParams{
+					Clientid:       params.ClientID,
+					Queryid:        field,
+					Removedversion: &latestVersion,
+				})
+				if err != nil {
+					return err
+				}
+			}
+
+			addIDs := getAddFields(current.Fields, params.Fields)
+			for key, field := range addIDs {
+				err := qtx.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{
+					Clientid:     params.ClientID,
+					Name:         key,
+					Queryid:      field,
+					Addedversion: latestVersion,
+				})
+				if err != nil {
+					return err
+				}
+			}
+		}
+
+		slog.Debug("client collector updated", "update", *params)
+
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func getRemoveFields(current map[string]uuid.UUID, update *map[string]pgtype.UUID) []pgtype.UUID {
+	diff := []pgtype.UUID{}
+	if update == nil {
+		return diff
+	}
+
+	for ckey, cid := range current {
+		found := false
+		for ukey, uid := range *update {
+			if cid == database.MustToUUID(uid) &&
+				ckey == ukey {
+				found = true
+			}
+		}
+		if !found {
+			diff = append(diff, database.MustToDBUUID(cid))
+		}
+	}
+
+	return diff
+}
+
+func getAddFields(current map[string]uuid.UUID, update *map[string]pgtype.UUID) map[string]pgtype.UUID {
+	diff := map[string]pgtype.UUID{}
+	if update == nil {
+		return diff
+	}
+
+	for ukey, uid := range *update {
+		if current[ukey] != database.MustToUUID(uid) {
+			diff[ukey] = uid
+		}
+	}
+
+	return diff
+}
+
+func (s *Service) normalizeFieldsToDB(ctx context.Context, ofields *map[string]uuid.UUID) (*map[string]pgtype.UUID, error) {
+	if ofields == nil || *ofields == nil {
+		return nil, nil
+	} else if len(*ofields) == 0 {
+		return nil, nil
+	}
+
+	dbm := map[string]pgtype.UUID{}
+	for name, id := range *ofields {
+		dbm[name] = database.MustToDBUUID(id)
+	}
+
+	dbids := []pgtype.UUID{}
+	for _, id := range dbm {
+		dbids = append(dbids, id)
+	}
+
+	dedup := validation.DeduplicateArray(dbids)
+
+	if len(dedup) != len(dbids) {
+		return nil, errors.New("duplicate output fields")
+	}
+
+	exist, err := s.cfg.GetDBQueries().AllQueriesExist(ctx, dbids)
+	if err != nil {
+		return nil, err
+	} else if !exist {
+		return nil, errors.New("not all required ids are present")
+	}
+
+	return &dbm, nil
+}

internal/collector/set/set_test.go

@@ -0,0 +1,132 @@
+package collectorset_test
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	"queryorchestration/internal/collector"
+	collectorset "queryorchestration/internal/collector/set"
+	"queryorchestration/internal/database"
+	"queryorchestration/internal/database/repository"
+	"queryorchestration/internal/serviceconfig"
+	"queryorchestration/internal/serviceconfig/queue/clientsync"
+	queuemock "queryorchestration/mocks/queue"
+
+	"github.com/aws/aws-sdk-go-v2/service/sqs"
+	"github.com/google/uuid"
+	"github.com/jackc/pgx/v5"
+	"github.com/pashagolub/pgxmock/v3"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/mock"
+	"github.com/stretchr/testify/require"
+)
+
+type CollectorSetConfig struct {
+	serviceconfig.BaseConfig
+	clientsync.ClientSyncConfig
+}
+
+func TestSet(t *testing.T) {
+	ctx := context.Background()
+
+	pool, err := pgxmock.NewPool()
+	require.NoError(t, err)
+	cfg := &CollectorSetConfig{}
+	cfg.DBPool = pool
+	cfg.DBQueries = repository.New(pool)
+	mockSQS := queuemock.NewMockSQSClient(t)
+	cfg.QueueClient = mockSQS
+	cfg.ClientSyncURL = "here"
+
+	svc := collectorset.New(cfg, &collectorset.Services{
+		Collector: collector.New(cfg),
+	})
+
+	t.Run("valid", func(t *testing.T) {
+		current := collector.Collector{
+			ClientID:      uuid.New(),
+			ActiveVersion: 1,
+			LatestVersion: 4,
+		}
+		av := int32(2)
+		mv := int32(1)
+		update := collectorset.SetParams{
+			ClientID:        current.ClientID,
+			ActiveVersion:   &av,
+			MinCleanVersion: &mv,
+		}
+
+		pool.ExpectQuery("name: GetCollectorByClientID :one").WithArgs(database.MustToDBUUID(update.ClientID)).
+			WillReturnRows(
+				pgxmock.NewRows([]string{"clientId", "minCleanVersion", "minTextVersion", "activeVersion", "latestVersion", "fields"}).
+					AddRow(database.MustToDBUUID(current.ClientID), current.MinCleanVersion, current.MinTextVersion, current.ActiveVersion, current.LatestVersion, []byte("")),
+			)
+		pool.ExpectBeginTx(pgx.TxOptions{})
+		rv := current.LatestVersion + 1
+		pool.ExpectQuery("name: AddLatestCollectorVersion :one").WithArgs(database.MustToDBUUID(current.ClientID)).WillReturnRows(
+			pgxmock.NewRows([]string{"version"}).
+				AddRow(rv),
+		)
+		pool.ExpectExec("name: SetActiveCollectorVersion :exec").WithArgs(database.MustToDBUUID(current.ClientID), int32(2)).
+			WillReturnResult(pgxmock.NewResult("", 1))
+		pool.ExpectExec("name: SetCollectorCleanVersion :exec").WithArgs(database.MustToDBUUID(current.ClientID), rv, *update.MinCleanVersion).
+			WillReturnResult(pgxmock.NewResult("", 1))
+		pool.ExpectCommit()
+
+		mockSQS.EXPECT().
+			SendMessage(
+				mock.Anything,
+				mock.MatchedBy(func(in *sqs.SendMessageInput) bool {
+					return *in.QueueUrl == cfg.ClientSyncURL && *in.MessageBody == fmt.Sprintf("{\"id\":\"%s\"}", update.ClientID.String())
+				}),
+				mock.Anything,
+			).
+			Return(&sqs.SendMessageOutput{}, nil)
+
+		err = svc.SetByClientId(ctx, &update)
+		assert.NoError(t, err)
+	})
+	t.Run("no collector", func(t *testing.T) {
+		current := collector.Collector{
+			ClientID: uuid.New(),
+		}
+		av := int32(1)
+		mv := int32(1)
+		update := collectorset.SetParams{
+			ClientID:        current.ClientID,
+			ActiveVersion:   &av,
+			MinCleanVersion: &mv,
+		}
+
+		pool.ExpectQuery("name: GetCollectorByClientID :one").WithArgs(database.MustToDBUUID(update.ClientID)).
+			WillReturnRows(
+				pgxmock.NewRows([]string{"clientId", "minCleanVersion", "minTextVersion", "activeVersion", "latestVersion", "fields"}).
+					AddRow(database.MustToDBUUID(current.ClientID), current.MinCleanVersion, current.MinTextVersion, current.ActiveVersion, current.LatestVersion, []byte("")),
+			)
+		pool.ExpectBeginTx(pgx.TxOptions{})
+		rv := current.LatestVersion + 1
+		pool.ExpectQuery("name: AddLatestCollectorVersion :one").WithArgs(database.MustToDBUUID(current.ClientID)).WillReturnRows(
+			pgxmock.NewRows([]string{"version"}).
+				AddRow(rv),
+		)
+		pool.ExpectExec("name: SetActiveCollectorVersion :exec").WithArgs(database.MustToDBUUID(current.ClientID), int32(1)).
+			WillReturnResult(pgxmock.NewResult("", 1))
+		pool.ExpectExec("name: SetCollectorCleanVersion :exec").WithArgs(database.MustToDBUUID(current.ClientID), rv, *update.MinCleanVersion).
+			WillReturnResult(pgxmock.NewResult("", 1))
+		pool.ExpectCommit()
+
+		mockSQS.EXPECT().
+			SendMessage(
+				mock.Anything,
+				mock.MatchedBy(func(in *sqs.SendMessageInput) bool {
+					return *in.QueueUrl == cfg.ClientSyncURL && *in.MessageBody == fmt.Sprintf("{\"id\":\"%s\"}", update.ClientID.String())
+				}),
+				mock.Anything,
+			).
+			Return(&sqs.SendMessageOutput{}, nil)
+
+		err = svc.SetByClientId(ctx, &update)
+		assert.NoError(t, err)
+	})
+}

internal/collector/set/setprivate_test.go

@@ -0,0 +1,712 @@
+package collectorset
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	"queryorchestration/internal/collector"
+	"queryorchestration/internal/database"
+	"queryorchestration/internal/database/repository"
+	cleanversion "queryorchestration/internal/document/clean/version"
+	textversion "queryorchestration/internal/document/text/version"
+	"queryorchestration/internal/serviceconfig"
+	"queryorchestration/internal/serviceconfig/queue/clientsync"
+	queuemock "queryorchestration/mocks/queue"
+
+	"github.com/aws/aws-sdk-go-v2/service/sqs"
+	"github.com/google/uuid"
+	"github.com/jackc/pgx/v5"
+	"github.com/jackc/pgx/v5/pgtype"
+	"github.com/pashagolub/pgxmock/v3"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/mock"
+	"github.com/stretchr/testify/require"
+)
+
+type CollectorSetConfig struct {
+	serviceconfig.BaseConfig
+	clientsync.ClientSyncConfig
+}
+
+func TestGetSetParams(t *testing.T) {
+	t.Run("all params", func(t *testing.T) {
+		ctx := context.Background()
+
+		pool, err := pgxmock.NewPool()
+		require.NoError(t, err)
+		cfg := &CollectorSetConfig{}
+		cfg.DBPool = pool
+		cfg.DBQueries = repository.New(pool)
+
+		svc := Service{
+			cfg: cfg,
+			svc: &Services{
+				Collector: collector.New(cfg),
+			},
+		}
+
+		minCleanV := int32(1)
+		minTextV := int32(1)
+		aV := int32(3)
+		current := collector.Collector{
+			ActiveVersion: 1,
+			LatestVersion: 10,
+		}
+		params := SetParams{
+			ClientID:        uuid.New(),
+			ActiveVersion:   &aV,
+			MinCleanVersion: &minCleanV,
+			MinTextVersion:  &minTextV,
+			Fields: &map[string]uuid.UUID{
+				"example_key": uuid.New(),
+			},
+		}
+
+		pool.ExpectQuery("name: AllQueriesExist :one").WithArgs(database.MustToDBUUIDArray([]uuid.UUID{(*params.Fields)["example_key"]})).WillReturnRows(
+			pgxmock.NewRows([]string{"all_exist"}).
+				AddRow(true),
+		)
+
+		dbparams, err := svc.getSetParams(ctx, &current, &params)
+		assert.NoError(t, err)
+		assert.EqualExportedValues(t, &dbSetParams{
+			ClientID:        database.MustToDBUUID(params.ClientID),
+			ActiveVersion:   &aV,
+			MinCleanVersion: &minCleanV,
+			MinTextVersion:  &minTextV,
+			Fields: &map[string]pgtype.UUID{
+				"example_key": database.MustToDBUUID((*params.Fields)["example_key"]),
+			},
+		}, dbparams)
+
+		(*params.Fields)["second_key"] = (*params.Fields)["example_key"]
+		assert.Len(t, *params.Fields, 2)
+		_, err = svc.getSetParams(ctx, &current, &params)
+		assert.Error(t, err)
+	})
+
+	t.Run("no params", func(t *testing.T) {
+		ctx := context.Background()
+
+		pool, err := pgxmock.NewPool()
+		require.NoError(t, err)
+		cfg := &CollectorSetConfig{}
+		cfg.DBPool = pool
+		cfg.DBQueries = repository.New(pool)
+
+		svc := Service{
+			cfg: cfg,
+			svc: &Services{},
+		}
+
+		current := collector.Collector{}
+		params := SetParams{
+			ClientID: current.ClientID,
+		}
+		_, err = svc.getSetParams(ctx, &current, &params)
+		assert.EqualError(t, err, "no changes")
+	})
+
+	t.Run("ONLY active version to next latest", func(t *testing.T) {
+		ctx := context.Background()
+
+		pool, err := pgxmock.NewPool()
+		require.NoError(t, err)
+		cfg := &CollectorSetConfig{}
+		cfg.DBPool = pool
+		cfg.DBQueries = repository.New(pool)
+
+		svc := Service{
+			cfg: cfg,
+			svc: &Services{},
+		}
+
+		current := collector.Collector{
+			ActiveVersion: 2,
+			LatestVersion: 4,
+		}
+		version := int32(5)
+		params := SetParams{
+			ClientID:      current.ClientID,
+			ActiveVersion: &version,
+		}
+		_, err = svc.getSetParams(ctx, &current, &params)
+		assert.EqualError(t, err, "no changes")
+	})
+
+	t.Run("no fields change", func(t *testing.T) {
+		ctx := context.Background()
+
+		pool, err := pgxmock.NewPool()
+		require.NoError(t, err)
+		cfg := &CollectorSetConfig{}
+		cfg.DBPool = pool
+		cfg.DBQueries = repository.New(pool)
+
+		svc := Service{
+			cfg: cfg,
+			svc: &Services{},
+		}
+
+		current := collector.Collector{
+			Fields: map[string]uuid.UUID{
+				"example": uuid.New(),
+			},
+		}
+		params := SetParams{
+			ClientID: current.ClientID,
+			Fields: &map[string]uuid.UUID{
+				"example": current.Fields["example"],
+			},
+		}
+		_, err = svc.getSetParams(ctx, &current, &params)
+		assert.EqualError(t, err, "no changes")
+	})
+}
+
+func TestSubmitSet(t *testing.T) {
+	t.Run("all fields", func(t *testing.T) {
+
+		ctx := context.Background()
+
+		pool, err := pgxmock.NewPool()
+		require.NoError(t, err)
+		cfg := &CollectorSetConfig{}
+		cfg.DBPool = pool
+		cfg.DBQueries = repository.New(pool)
+
+		svc := Service{
+			cfg: cfg,
+			svc: &Services{},
+		}
+
+		minCleanV := int32(2)
+		minTextV := int32(4)
+		current := collector.Collector{
+			ClientID:        uuid.New(),
+			ActiveVersion:   1,
+			LatestVersion:   1,
+			MinCleanVersion: 1,
+			MinTextVersion:  2,
+			Fields: map[string]uuid.UUID{
+				"original_key": uuid.New(),
+				"og_key":       uuid.New(),
+			},
+		}
+		aV := int32(2)
+		params := dbSetParams{
+			ClientID:        database.MustToDBUUID(current.ClientID),
+			ActiveVersion:   &aV,
+			MinCleanVersion: &minCleanV,
+			MinTextVersion:  &minTextV,
+			Fields: &map[string]pgtype.UUID{
+				"example_key": database.MustToDBUUID(uuid.New()),
+				"second_key":  database.MustToDBUUID(uuid.New()),
+				"changed_key": database.MustToDBUUID(current.Fields["original_key"]),
+				"og_key":      database.MustToDBUUID(current.Fields["og_key"]),
+			},
+		}
+
+		pool.ExpectBeginTx(pgx.TxOptions{})
+		rv := int32(2)
+		pool.ExpectQuery("name: AddLatestCollectorVersion :one").WithArgs(database.MustToDBUUID(current.ClientID)).WillReturnRows(
+			pgxmock.NewRows([]string{"version"}).
+				AddRow(rv),
+		)
+		pool.ExpectExec("name: SetActiveCollectorVersion :exec").WithArgs(database.MustToDBUUID(current.ClientID), int32(2)).
+			WillReturnResult(pgxmock.NewResult("", 1))
+		pool.ExpectExec("name: SetCollectorCleanVersion :exec").WithArgs(database.MustToDBUUID(current.ClientID), rv, *params.MinCleanVersion).
+			WillReturnResult(pgxmock.NewResult("", 1))
+		pool.ExpectExec("name: SetCollectorTextVersion :exec").WithArgs(database.MustToDBUUID(current.ClientID), rv, *params.MinTextVersion).
+			WillReturnResult(pgxmock.NewResult("", 1))
+		pool.ExpectExec("name: RemoveCollectorQuery :exec").WithArgs(&rv, database.MustToDBUUID(current.Fields["original_key"]), database.MustToDBUUID(current.ClientID)).
+			WillReturnResult(pgxmock.NewResult("", 1))
+		pool.MatchExpectationsInOrder(false)
+		pool.ExpectExec("name: AddCollectorQuery :exec").WithArgs(database.MustToDBUUID(current.ClientID), "example_key", (*params.Fields)["example_key"], int32(2)).
+			WillReturnResult(pgxmock.NewResult("", 1))
+		pool.ExpectExec("name: AddCollectorQuery :exec").WithArgs(database.MustToDBUUID(current.ClientID), "second_key", (*params.Fields)["second_key"], int32(2)).
+			WillReturnResult(pgxmock.NewResult("", 1))
+		pool.ExpectExec("name: AddCollectorQuery :exec").WithArgs(database.MustToDBUUID(current.ClientID), "changed_key", (*params.Fields)["changed_key"], int32(2)).
+			WillReturnResult(pgxmock.NewResult("", 1))
+		pool.ExpectCommit()
+
+		err = svc.submitSet(ctx, &current, &params)
+		assert.NoError(t, err)
+	})
+	t.Run("only active version", func(t *testing.T) {
+
+		ctx := context.Background()
+
+		pool, err := pgxmock.NewPool()
+		require.NoError(t, err)
+		cfg := &CollectorSetConfig{}
+		cfg.DBPool = pool
+		cfg.DBQueries = repository.New(pool)
+
+		svc := Service{
+			cfg: cfg,
+			svc: &Services{},
+		}
+
+		current := collector.Collector{
+			ClientID:        uuid.New(),
+			ActiveVersion:   1,
+			LatestVersion:   1,
+			MinCleanVersion: 1,
+			MinTextVersion:  2,
+			Fields:          map[string]uuid.UUID{},
+		}
+		av := int32(2)
+		params := dbSetParams{
+			ClientID:      database.MustToDBUUID(current.ClientID),
+			ActiveVersion: &av,
+		}
+
+		pool.ExpectBeginTx(pgx.TxOptions{})
+		pool.ExpectExec("name: SetActiveCollectorVersion :exec").WithArgs(database.MustToDBUUID(current.ClientID), int32(2)).
+			WillReturnResult(pgxmock.NewResult("", 1))
+		pool.ExpectCommit()
+
+		err = svc.submitSet(ctx, &current, &params)
+		assert.NoError(t, err)
+	})
+}
+
+func TestNormalizeActiveVersion(t *testing.T) {
+	svc := Service{}
+
+	t.Run("all nil", func(t *testing.T) {
+		err := svc.normalizeActiveVersion(nil, nil)
+		assert.Error(t, err)
+	})
+
+	t.Run("nil update", func(t *testing.T) {
+		current := collector.Collector{
+			ActiveVersion: 1,
+			LatestVersion: 4,
+		}
+
+		err := svc.normalizeActiveVersion(&current, nil)
+		assert.NoError(t, err)
+	})
+
+	t.Run("no update", func(t *testing.T) {
+		current := collector.Collector{
+			ActiveVersion: 1,
+			LatestVersion: 4,
+		}
+		update := SetParams{}
+		err := svc.normalizeActiveVersion(&current, &update)
+		assert.NoError(t, err)
+		assert.Nil(t, update.ActiveVersion)
+	})
+
+	t.Run("same version", func(t *testing.T) {
+		version := int32(2)
+		current := collector.Collector{
+			ActiveVersion: version,
+			LatestVersion: 4,
+		}
+		update := SetParams{
+			ActiveVersion: &version,
+		}
+		err := svc.normalizeActiveVersion(&current, &update)
+		assert.NoError(t, err)
+		assert.Nil(t, update.ActiveVersion)
+	})
+
+	t.Run("latest version plus 1", func(t *testing.T) {
+		current := collector.Collector{
+			ActiveVersion: 1,
+			LatestVersion: 4,
+		}
+		version := int32(5)
+		update := SetParams{
+			ActiveVersion: &version,
+		}
+		err := svc.normalizeActiveVersion(&current, &update)
+		assert.NoError(t, err)
+		assert.Equal(t, version, *update.ActiveVersion)
+	})
+
+	t.Run("latest version plus 2", func(t *testing.T) {
+		current := collector.Collector{
+			ActiveVersion: 1,
+			LatestVersion: 4,
+		}
+		version := int32(6)
+		update := SetParams{
+			ActiveVersion: &version,
+		}
+		err := svc.normalizeActiveVersion(&current, &update)
+		assert.Error(t, err)
+	})
+}
+
+func TestNormalizeCodeVersions(t *testing.T) {
+	cfg := &CollectorSetConfig{}
+	svc := Service{
+		svc: &Services{
+			CleanVersion: cleanversion.New(cfg),
+			TextVersion:  textversion.New(cfg),
+		},
+	}
+
+	t.Run("all nil", func(t *testing.T) {
+		err := svc.normalizeCodeVersions(nil, nil)
+		assert.Error(t, err)
+	})
+
+	t.Run("nil update", func(t *testing.T) {
+		current := collector.Collector{}
+		err := svc.normalizeCodeVersions(&current, nil)
+		assert.NoError(t, err)
+	})
+
+	t.Run("empty", func(t *testing.T) {
+		current := collector.Collector{}
+		update := SetParams{}
+
+		err := svc.normalizeCodeVersions(&current, &update)
+		assert.NoError(t, err)
+		assert.Nil(t, update.MinCleanVersion)
+		assert.Nil(t, update.MinTextVersion)
+	})
+
+	t.Run("valid clean", func(t *testing.T) {
+		current := collector.Collector{}
+		update := SetParams{}
+
+		cv := int32(1)
+		update.MinCleanVersion = &cv
+		err := svc.normalizeCodeVersions(&current, &update)
+		assert.NoError(t, err)
+		assert.Equal(t, cv, *update.MinCleanVersion)
+		assert.Nil(t, update.MinTextVersion)
+	})
+
+	t.Run("valid text", func(t *testing.T) {
+		current := collector.Collector{}
+		update := SetParams{}
+
+		tv := int32(1)
+		update.MinTextVersion = &tv
+		err := svc.normalizeCodeVersions(&current, &update)
+		assert.NoError(t, err)
+		assert.Nil(t, update.MinCleanVersion)
+		assert.Equal(t, tv, *update.MinTextVersion)
+	})
+
+	t.Run("valid clean and text", func(t *testing.T) {
+		current := collector.Collector{}
+		update := SetParams{}
+
+		current.MinCleanVersion = 1
+		current.MinTextVersion = 1
+		err := svc.normalizeCodeVersions(&current, &update)
+		assert.NoError(t, err)
+		assert.Nil(t, update.MinCleanVersion)
+		assert.Nil(t, update.MinTextVersion)
+	})
+
+	t.Run("current clean", func(t *testing.T) {
+		current := collector.Collector{
+			MinCleanVersion: 1,
+		}
+		update := SetParams{}
+
+		update.MinCleanVersion = &current.MinCleanVersion
+		err := svc.normalizeCodeVersions(&current, &update)
+		assert.NoError(t, err)
+		assert.Nil(t, update.MinCleanVersion)
+		assert.Nil(t, update.MinTextVersion)
+	})
+
+	t.Run("current text", func(t *testing.T) {
+		current := collector.Collector{
+			MinTextVersion: 1,
+		}
+		update := SetParams{}
+
+		update.MinTextVersion = &current.MinTextVersion
+		err := svc.normalizeCodeVersions(&current, &update)
+		assert.NoError(t, err)
+		assert.Nil(t, update.MinCleanVersion)
+		assert.Nil(t, update.MinTextVersion)
+	})
+
+	t.Run("current clean and text", func(t *testing.T) {
+		current := collector.Collector{
+			MinCleanVersion: 1,
+			MinTextVersion:  1,
+		}
+		update := SetParams{}
+
+		update.MinCleanVersion = &current.MinCleanVersion
+		update.MinTextVersion = &current.MinTextVersion
+		err := svc.normalizeCodeVersions(&current, &update)
+		assert.NoError(t, err)
+		assert.Nil(t, update.MinCleanVersion)
+		assert.Nil(t, update.MinTextVersion)
+	})
+}
+
+func TestGetRemoveFields(t *testing.T) {
+	current := map[string]uuid.UUID{}
+	var update *map[string]pgtype.UUID
+
+	remove := getRemoveFields(current, update)
+	assert.Len(t, remove, 0)
+
+	current["a"] = uuid.New()
+	remove = getRemoveFields(current, update)
+	assert.Len(t, remove, 0)
+
+	update = &map[string]pgtype.UUID{
+		"b": database.MustToDBUUID(uuid.New()),
+	}
+
+	(*update)["a"] = database.MustToDBUUID(current["a"])
+	remove = getRemoveFields(current, update)
+	assert.Len(t, remove, 0)
+
+	(*update)["a"] = database.MustToDBUUID(uuid.New())
+	remove = getRemoveFields(current, update)
+	assert.Len(t, remove, 1)
+	assert.ElementsMatch(t, []pgtype.UUID{database.MustToDBUUID(current["a"])}, remove)
+
+	(*update)["a"] = database.MustToDBUUID(uuid.Nil)
+	(*update)["c"] = database.MustToDBUUID(current["a"])
+	remove = getRemoveFields(current, update)
+	assert.Len(t, remove, 1)
+	assert.ElementsMatch(t, []pgtype.UUID{database.MustToDBUUID(current["a"])}, remove)
+}
+
+func TestGetAddFields(t *testing.T) {
+	current := map[string]uuid.UUID{}
+	var update *map[string]pgtype.UUID
+
+	add := getAddFields(current, update)
+	assert.Len(t, add, 0)
+
+	current["a"] = uuid.New()
+	add = getAddFields(current, update)
+	assert.Len(t, add, 0)
+
+	update = &map[string]pgtype.UUID{}
+
+	add = getAddFields(current, update)
+	assert.Len(t, add, 0)
+
+	(*update)["a"] = database.MustToDBUUID(current["a"])
+	add = getAddFields(current, update)
+	assert.Len(t, add, 0)
+
+	(*update)["a"] = database.MustToDBUUID(uuid.New())
+	add = getAddFields(current, update)
+	assert.Len(t, add, 1)
+	assert.Equal(t, map[string]pgtype.UUID{
+		"a": (*update)["a"],
+	}, add)
+
+	(*update)["a"] = database.MustToDBUUID(current["a"])
+	(*update)["b"] = database.MustToDBUUID(uuid.New())
+	add = getAddFields(current, update)
+	assert.Len(t, add, 1)
+	assert.Equal(t, map[string]pgtype.UUID{
+		"b": (*update)["b"],
+	}, add)
+
+	current = map[string]uuid.UUID{}
+	add = getAddFields(current, update)
+	assert.Len(t, add, 2)
+	assert.Equal(t, map[string]pgtype.UUID{
+		"b": (*update)["b"],
+		"a": (*update)["a"],
+	}, add)
+}
+
+func TestNormalizeSetFieldsToDB(t *testing.T) {
+	t.Run("valid key", func(t *testing.T) {
+
+		ctx := context.Background()
+
+		pool, err := pgxmock.NewPool()
+		require.NoError(t, err)
+		cfg := &CollectorSetConfig{}
+		cfg.DBPool = pool
+		cfg.DBQueries = repository.New(pool)
+
+		svc := Service{
+			cfg: cfg,
+			svc: &Services{
+				Collector: collector.New(cfg),
+			},
+		}
+
+		current := map[string]uuid.UUID{}
+		fields := map[string]uuid.UUID{
+			"example_key": uuid.New(),
+		}
+
+		pool.ExpectQuery("name: AllQueriesExist :one").WithArgs(database.MustToDBUUIDArray([]uuid.UUID{fields["example_key"]})).WillReturnRows(
+			pgxmock.NewRows([]string{"all_exist"}).
+				AddRow(true),
+		)
+
+		dbparams, err := svc.normalizeSetFieldsToDB(ctx, current, &fields)
+		assert.NoError(t, err)
+		assert.EqualExportedValues(t, &map[string]pgtype.UUID{
+			"example_key": database.MustToDBUUID(fields["example_key"]),
+		}, dbparams)
+	})
+
+	t.Run("duplicate values", func(t *testing.T) {
+		ctx := context.Background()
+
+		pool, err := pgxmock.NewPool()
+		require.NoError(t, err)
+		cfg := &CollectorSetConfig{}
+		cfg.DBPool = pool
+		cfg.DBQueries = repository.New(pool)
+
+		svc := Service{
+			cfg: cfg,
+			svc: &Services{},
+		}
+
+		current := map[string]uuid.UUID{}
+		fields := map[string]uuid.UUID{
+			"example_key": uuid.New(),
+		}
+		fields["second_key"] = fields["example_key"]
+
+		pool.ExpectQuery("name: AllQueriesExist :one").WithArgs(database.MustToDBUUIDArray([]uuid.UUID{fields["example_key"]})).WillReturnRows(
+			pgxmock.NewRows([]string{"all_exist"}).
+				AddRow(true),
+		)
+
+		_, err = svc.normalizeSetFieldsToDB(ctx, current, &fields)
+		assert.Error(t, err)
+	})
+	t.Run("no changes", func(t *testing.T) {
+		ctx := context.Background()
+		cfg := &CollectorSetConfig{}
+
+		svc := Service{
+			cfg: cfg,
+			svc: &Services{
+				Collector: collector.New(cfg),
+			},
+		}
+
+		current := map[string]uuid.UUID{
+			"example_key": uuid.New(),
+		}
+		fields := map[string]uuid.UUID{
+			"example_key": current["example_key"],
+		}
+
+		val, err := svc.normalizeSetFieldsToDB(ctx, current, &fields)
+		assert.NoError(t, err)
+		assert.Nil(t, val)
+	})
+}
+
+func TestInformSet(t *testing.T) {
+	t.Run("with version update", func(t *testing.T) {
+		ctx := context.Background()
+
+		pool, err := pgxmock.NewPool()
+		require.NoError(t, err)
+		cfg := &CollectorSetConfig{}
+		cfg.DBPool = pool
+		cfg.DBQueries = repository.New(pool)
+		mockSQS := queuemock.NewMockSQSClient(t)
+		cfg.QueueClient = mockSQS
+		cfg.ClientSyncURL = "here"
+
+		svc := New(cfg, &Services{})
+
+		av := int32(2)
+		update := dbSetParams{
+			ClientID:      database.MustToDBUUID(uuid.New()),
+			ActiveVersion: &av,
+		}
+
+		mockSQS.EXPECT().
+			SendMessage(
+				mock.Anything,
+				mock.MatchedBy(func(in *sqs.SendMessageInput) bool {
+					return *in.QueueUrl == cfg.ClientSyncURL && *in.MessageBody == fmt.Sprintf("{\"id\":\"%s\"}", update.ClientID.String())
+				}),
+				mock.Anything,
+			).
+			Return(&sqs.SendMessageOutput{}, nil)
+
+		err = svc.informSet(ctx, &update)
+		assert.NoError(t, err)
+	})
+	t.Run("with no update", func(t *testing.T) {
+		ctx := context.Background()
+
+		pool, err := pgxmock.NewPool()
+		require.NoError(t, err)
+		cfg := &CollectorSetConfig{}
+		cfg.DBPool = pool
+		cfg.DBQueries = repository.New(pool)
+		mockSQS := queuemock.NewMockSQSClient(t)
+		cfg.QueueClient = mockSQS
+		cfg.ClientSyncURL = "here"
+
+		svc := New(cfg, &Services{})
+
+		update := dbSetParams{
+			ClientID: database.MustToDBUUID(uuid.New()),
+		}
+
+		err = svc.informSet(ctx, &update)
+		assert.NoError(t, err)
+	})
+}
+
+func TestNormalizeFieldsToDB(t *testing.T) {
+	ctx := context.Background()
+
+	pool, err := pgxmock.NewPool()
+	require.NoError(t, err)
+	cfg := &CollectorSetConfig{}
+	cfg.DBPool = pool
+	cfg.DBQueries = repository.New(pool)
+
+	svc := Service{
+		cfg: cfg,
+		svc: &Services{},
+	}
+
+	fields := map[string]uuid.UUID{
+		"example_key": uuid.New(),
+	}
+
+	pool.ExpectQuery("name: AllQueriesExist :one").WithArgs(database.MustToDBUUIDArray([]uuid.UUID{fields["example_key"]})).WillReturnRows(
+		pgxmock.NewRows([]string{"all_exist"}).
+			AddRow(true),
+	)
+
+	dbparams, err := svc.normalizeFieldsToDB(ctx, &fields)
+	assert.NoError(t, err)
+	assert.EqualExportedValues(t, &map[string]pgtype.UUID{
+		"example_key": database.MustToDBUUID(fields["example_key"]),
+	}, dbparams)
+
+	fields["second_key"] = fields["example_key"]
+
+	pool.ExpectQuery("name: AllQueriesExist :one").WithArgs(database.MustToDBUUIDArray([]uuid.UUID{fields["example_key"]})).WillReturnRows(
+		pgxmock.NewRows([]string{"all_exist"}).
+			AddRow(true),
+	)
+
+	_, err = svc.normalizeFieldsToDB(ctx, &fields)
+	assert.Error(t, err)
+}