Merged in feature/import (pull request #157)

Feature/import

* importstart

* pp

* tests

* importtests

* 100GB

* lint

* passtests

* utc

* awsprofile

* doublequotes

* host

internal/database/migrations/00000000000005_documents.up.sql

@@ -6,6 +6,16 @@ CREATE TABLE documents (
     unique(clientId, hash)
 );
 
+CREATE TABLE documentUploads (
+    id uuid primary key DEFAULT uuid_generate_v7(),
+    bucket text not null,
+    key text not null,
+    clientId varchar(255) not null,
+    part unsignedsmallint not null,
+    createdAt timestamp not null,
+    foreign key (clientId) references clients(clientId)
+);
+
 CREATE TABLE documentEntries (
     id uuid primary key DEFAULT uuid_generate_v7(),
     documentId uuid not null,

internal/database/queries/document.sql

@@ -44,3 +44,26 @@ SELECT id FROM documents WHERE hash = $1 and clientId = $2;
 
 -- name: ListDocumentIDsBatch :many
 SELECT id, totalCount FROM listDocumentIDs(@clientId, @batchSize, @pageOffset);
+
+-- name: AddDocumentUpload :exec
+INSERT INTO documentUploads (id, clientId, bucket, key, part, createdAt) VALUES ($1, $2, $3, $4, $5, $6);
+
+-- name: GetDocumentUploadCurrentPart :one
+WITH client as (
+    select clientId from clients where clientId = @clientId
+),
+parts as (
+    SELECT docs.part
+        FROM client as c
+        JOIN documentUploads as docs on docs.clientId = c.clientId
+        WHERE date(docs.createdAt) = date(@queryDate)
+),
+max_part as (
+    SELECT COALESCE(max(part), 0) as max_part_num FROM parts
+)
+SELECT
+    COALESCE(p.part, 0)::unsignedsmallint as part,
+    COUNT(p.part) as count
+FROM parts p
+RIGHT JOIN max_part mp ON p.part = mp.max_part_num
+GROUP BY p.part;

internal/database/repository/document.sql.go

@@ -9,6 +9,7 @@ import (
 	"context"
 
 	"github.com/google/uuid"
+	"github.com/jackc/pgx/v5/pgtype"
 )
 
 const addDocumentEntry = `-- name: AddDocumentEntry :exec
@@ -29,6 +30,34 @@ func (q *Queries) AddDocumentEntry(ctx context.Context, arg *AddDocumentEntryPar
 	return err
 }
 
+const addDocumentUpload = `-- name: AddDocumentUpload :exec
+INSERT INTO documentUploads (id, clientId, bucket, key, part, createdAt) VALUES ($1, $2, $3, $4, $5, $6)
+`
+
+type AddDocumentUploadParams struct {
+	ID        uuid.UUID        `db:"id"`
+	Clientid  string           `db:"clientid"`
+	Bucket    string           `db:"bucket"`
+	Key       string           `db:"key"`
+	Part      uint16           `db:"part"`
+	Createdat pgtype.Timestamp `db:"createdat"`
+}
+
+// AddDocumentUpload
+//
+//	INSERT INTO documentUploads (id, clientId, bucket, key, part, createdAt) VALUES ($1, $2, $3, $4, $5, $6)
+func (q *Queries) AddDocumentUpload(ctx context.Context, arg *AddDocumentUploadParams) error {
+	_, err := q.db.Exec(ctx, addDocumentUpload,
+		arg.ID,
+		arg.Clientid,
+		arg.Bucket,
+		arg.Key,
+		arg.Part,
+		arg.Createdat,
+	)
+	return err
+}
+
 const createDocument = `-- name: CreateDocument :one
 INSERT INTO documents (clientId, hash) VALUES ($1, $2) RETURNING id
 `
@@ -173,6 +202,64 @@ func (q *Queries) GetDocumentSummary(ctx context.Context, id uuid.UUID) (*Docume
 	return &i, err
 }
 
+const getDocumentUploadCurrentPart = `-- name: GetDocumentUploadCurrentPart :one
+WITH client as (
+    select clientId from clients where clientId = $1
+),
+parts as (
+    SELECT docs.part
+        FROM client as c
+        JOIN documentUploads as docs on docs.clientId = c.clientId
+        WHERE date(docs.createdAt) = date($2)
+),
+max_part as (
+    SELECT COALESCE(max(part), 0) as max_part_num FROM parts
+)
+SELECT
+    COALESCE(p.part, 0)::unsignedsmallint as part,
+    COUNT(p.part) as count
+FROM parts p
+RIGHT JOIN max_part mp ON p.part = mp.max_part_num
+GROUP BY p.part
+`
+
+type GetDocumentUploadCurrentPartParams struct {
+	Clientid  *string            `db:"clientid"`
+	Querydate pgtype.Timestamptz `db:"querydate"`
+}
+
+type GetDocumentUploadCurrentPartRow struct {
+	Part  uint16 `db:"part"`
+	Count int64  `db:"count"`
+}
+
+// GetDocumentUploadCurrentPart
+//
+//	WITH client as (
+//	    select clientId from clients where clientId = $1
+//	),
+//	parts as (
+//	    SELECT docs.part
+//	        FROM client as c
+//	        JOIN documentUploads as docs on docs.clientId = c.clientId
+//	        WHERE date(docs.createdAt) = date($2)
+//	),
+//	max_part as (
+//	    SELECT COALESCE(max(part), 0) as max_part_num FROM parts
+//	)
+//	SELECT
+//	    COALESCE(p.part, 0)::unsignedsmallint as part,
+//	    COUNT(p.part) as count
+//	FROM parts p
+//	RIGHT JOIN max_part mp ON p.part = mp.max_part_num
+//	GROUP BY p.part
+func (q *Queries) GetDocumentUploadCurrentPart(ctx context.Context, arg *GetDocumentUploadCurrentPartParams) (*GetDocumentUploadCurrentPartRow, error) {
+	row := q.db.QueryRow(ctx, getDocumentUploadCurrentPart, arg.Clientid, arg.Querydate)
+	var i GetDocumentUploadCurrentPartRow
+	err := row.Scan(&i.Part, &i.Count)
+	return &i, err
+}
+
 const listDocumentIDsBatch = `-- name: ListDocumentIDsBatch :many
 SELECT id, totalCount FROM listDocumentIDs($1, $2, $3)
 `

internal/database/repository/models.go

@@ -336,6 +336,15 @@ type Documenttextextractionentry struct {
 	Version int64     `db:"version"`
 }
 
+type Documentupload struct {
+	ID        uuid.UUID        `db:"id"`
+	Bucket    string           `db:"bucket"`
+	Key       string           `db:"key"`
+	Clientid  string           `db:"clientid"`
+	Part      uint16           `db:"part"`
+	Createdat pgtype.Timestamp `db:"createdat"`
+}
+
 type Fullactivecollector struct {
 	Clientid        string `db:"clientid"`
 	Mincleanversion int64  `db:"mincleanversion"`