Merged in feature/textextraction (pull request #110)

Text Extraction * bases * go * splitting * structure * movetoasync * movetoasync * settinguptrigger * reorder * storevent * standardisepollingvalidation * unittests * fixlint * fixlint * awscfg * generatesample * followthrough * tests * clena * store * externalidcleanup * clientid * local * baseunittests * putobjecttests * tests
2025-04-02 18:50:03 +00:00
parent e6a4cb3990
commit 81e7223560
226 changed files with 17283 additions and 2744 deletions
@@ -0,0 +1,90 @@
+package documentstore
+
+import (
+	"context"
+	"fmt"
+	"log/slog"
+	"regexp"
+
+	docinitrunner "queryorchestration/api/docInitRunner"
+	doctextprocessrunner "queryorchestration/api/docTextProcessRunner"
+	"queryorchestration/internal/client"
+	"queryorchestration/internal/serviceconfig/queue"
+)
+
+type Params struct {
+	Event  EventS3
+	Key    string
+	Bucket string
+	Hash   string
+}
+
+type EventS3 string
+
+const (
+	EventS3ObjectCreatedPut = "ObjectCreated:Put"
+)
+
+func (s *Service) Process(ctx context.Context, params Params) error {
+	if !s.isSupportedEvent(params.Event) {
+		slog.Warn("unsupported event", "name", params.Event)
+		return nil
+	}
+
+	task, clientId := getKeyMetadata(params.Key)
+
+	queueParams := &queue.SendParams{}
+	switch task {
+	case DocInit:
+		queueParams.QueueURL = s.cfg.GetDocInitURL()
+		queueParams.Body = docinitrunner.Body{
+			Bucket:   params.Bucket,
+			Key:      params.Key,
+			Hash:     params.Hash,
+			ClientID: clientId,
+		}
+	case DocTextProcess:
+		queueParams.QueueURL = s.cfg.GetDocumentTextProcessURL()
+		queueParams.Body = doctextprocessrunner.Body{
+			Bucket:   params.Bucket,
+			Key:      params.Key,
+			Hash:     params.Hash,
+			ClientID: clientId,
+		}
+	default:
+		slog.Info("unsupported key", "key", params.Key)
+		return nil
+	}
+
+	return s.cfg.SendToQueue(ctx, queueParams)
+}
+
+type Task string
+
+const (
+	DocInit        Task = "import"
+	DocTextProcess Task = "text/textract"
+	Invalid        Task = ""
+)
+
+func getKeyMetadata(key string) (Task, string) {
+	regexStr := fmt.Sprintf(`^(%s)/(import|text/textract)/.+$`, client.CLIENT_ID_REGEX)
+	re := regexp.MustCompile(regexStr)
+	match := re.FindStringSubmatch(key)
+	if len(match) < 3 {
+		return Invalid, ""
+	}
+
+	task := Task(match[2])
+	if task == Invalid {
+		return Invalid, ""
+	}
+
+	clientId := match[1]
+
+	return task, clientId
+}
+
+func (s *Service) isSupportedEvent(name EventS3) bool {
+	return name == EventS3ObjectCreatedPut
+}