jacob-mathison/query-orchestration
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Merged in feature/track-filesize (pull request #200)

Browse Source 
track file sizes for all documents in system

* feature complete

needs dev testing
This commit is contained in:
Jay Brown
2026-01-12 17:46:07 +00:00
parent 4ad8168f35
commit ebf47c6013
37 changed files with 1514 additions and 374 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/database/migrations/00000000000118_add_document_file_size.down.sql
+2
View File
@@ -0,0 +1,2 @@
-- Remove file_size_bytes column from documents table
ALTER TABLE documents DROP COLUMN file_size_bytes;
internal/database/migrations/00000000000118_add_document_file_size.up.sql
+4
View File
@@ -0,0 +1,4 @@
-- Add file_size_bytes column to documents table
-- Stores the size of the document in bytes as measured from S3 HeadObject
-- NULL for legacy documents (before this feature was added)
ALTER TABLE documents ADD COLUMN file_size_bytes BIGINT;
internal/database/queries/document.sql
+3 -2
View File
@@ -34,7 +34,7 @@ SELECT id, hash from documents where clientId = @clientId;
SELECT id, hash, filename from documents where batch_id = @batch_id;
-- name: CreateDocument :one
INSERT INTO documents (clientId, hash, batch_id, filename, folderId, originalPath) VALUES ($1, $2, $3, $4, $5, $6) RETURNING id;
INSERT INTO documents (clientId, hash, batch_id, filename, folderId, originalPath, file_size_bytes) VALUES ($1, $2, $3, $4, $5, $6, $7) RETURNING id;
-- name: AddDocumentEntry :exec
INSERT INTO documentEntries (documentId, bucket, key) VALUES ($1, $2, $3);
@@ -85,6 +85,7 @@ SELECT
d.hash,
d.folderId,
d.filename,
d.originalPath
d.originalPath,
d.file_size_bytes
FROM documents d
WHERE d.id = $1;
internal/database/queries/folders.sql
+2 -1
View File
@@ -95,7 +95,8 @@ SELECT
d.hash,
d.folderId,
d.filename,
d.originalPath
d.originalPath,
d.file_size_bytes
FROM documents d
WHERE d.folderId = $1
ORDER BY d.id;
internal/database/repository/document.sql.go
+22 -16
View File
@@ -65,21 +65,22 @@ func (q *Queries) AddDocumentUpload(ctx context.Context, arg *AddDocumentUploadP
}
const createDocument = `-- name: CreateDocument :one
INSERT INTO documents (clientId, hash, batch_id, filename, folderId, originalPath) VALUES ($1, $2, $3, $4, $5, $6) RETURNING id
INSERT INTO documents (clientId, hash, batch_id, filename, folderId, originalPath, file_size_bytes) VALUES ($1, $2, $3, $4, $5, $6, $7) RETURNING id
`
type CreateDocumentParams struct {
Clientid string `db:"clientid"`
Hash string `db:"hash"`
BatchID *uuid.UUID `db:"batch_id"`
Filename *string `db:"filename"`
Folderid *uuid.UUID `db:"folderid"`
Originalpath *string `db:"originalpath"`
Clientid string `db:"clientid"`
Hash string `db:"hash"`
BatchID *uuid.UUID `db:"batch_id"`
Filename *string `db:"filename"`
Folderid *uuid.UUID `db:"folderid"`
Originalpath *string `db:"originalpath"`
FileSizeBytes *int64 `db:"file_size_bytes"`
}
// CreateDocument
//
// INSERT INTO documents (clientId, hash, batch_id, filename, folderId, originalPath) VALUES ($1, $2, $3, $4, $5, $6) RETURNING id
// INSERT INTO documents (clientId, hash, batch_id, filename, folderId, originalPath, file_size_bytes) VALUES ($1, $2, $3, $4, $5, $6, $7) RETURNING id
func (q *Queries) CreateDocument(ctx context.Context, arg *CreateDocumentParams) (uuid.UUID, error) {
row := q.db.QueryRow(ctx, createDocument,
arg.Clientid,
@@ -88,6 +89,7 @@ func (q *Queries) CreateDocument(ctx context.Context, arg *CreateDocumentParams)
arg.Filename,
arg.Folderid,
arg.Originalpath,
arg.FileSizeBytes,
)
var id uuid.UUID
err := row.Scan(&id)
@@ -101,18 +103,20 @@ SELECT
d.hash,
d.folderId,
d.filename,
d.originalPath
d.originalPath,
d.file_size_bytes
FROM documents d
WHERE d.id = $1
`
type GetDocumentEnrichedRow struct {
ID uuid.UUID `db:"id"`
Clientid string `db:"clientid"`
Hash string `db:"hash"`
Folderid *uuid.UUID `db:"folderid"`
Filename *string `db:"filename"`
Originalpath *string `db:"originalpath"`
ID uuid.UUID `db:"id"`
Clientid string `db:"clientid"`
Hash string `db:"hash"`
Folderid *uuid.UUID `db:"folderid"`
Filename *string `db:"filename"`
Originalpath *string `db:"originalpath"`
FileSizeBytes *int64 `db:"file_size_bytes"`
}
// Retrieves a document with extended metadata including folder and filename
@@ -123,7 +127,8 @@ type GetDocumentEnrichedRow struct {
// d.hash,
// d.folderId,
// d.filename,
// d.originalPath
// d.originalPath,
// d.file_size_bytes
// FROM documents d
// WHERE d.id = $1
func (q *Queries) GetDocumentEnriched(ctx context.Context, id uuid.UUID) (*GetDocumentEnrichedRow, error) {
@@ -136,6 +141,7 @@ func (q *Queries) GetDocumentEnriched(ctx context.Context, id uuid.UUID) (*GetDo
&i.Folderid,
&i.Filename,
&i.Originalpath,
&i.FileSizeBytes,
)
return &i, err
}
internal/database/repository/folders.sql.go
+14 -9
View File
@@ -89,14 +89,14 @@ func (q *Queries) GetClientRootFolder(ctx context.Context, clientid string) (*Fo
}
const getDocumentsByFolder = `-- name: GetDocumentsByFolder :many
SELECT id, clientid, hash, batch_id, filename, folderid, originalpath FROM documents
SELECT id, clientid, hash, batch_id, filename, folderid, originalpath, file_size_bytes FROM documents
WHERE folderId = $1
ORDER BY id
`
// GetDocumentsByFolder
//
// SELECT id, clientid, hash, batch_id, filename, folderid, originalpath FROM documents
// SELECT id, clientid, hash, batch_id, filename, folderid, originalpath, file_size_bytes FROM documents
// WHERE folderId = $1
// ORDER BY id
func (q *Queries) GetDocumentsByFolder(ctx context.Context, folderid *uuid.UUID) ([]*Document, error) {
@@ -116,6 +116,7 @@ func (q *Queries) GetDocumentsByFolder(ctx context.Context, folderid *uuid.UUID)
&i.Filename,
&i.Folderid,
&i.Originalpath,
&i.FileSizeBytes,
); err != nil {
return nil, err
}
@@ -133,18 +134,20 @@ SELECT
d.hash,
d.folderId,
d.filename,
d.originalPath
d.originalPath,
d.file_size_bytes
FROM documents d
WHERE d.folderId = $1
ORDER BY d.id
`
type GetDocumentsByFolderEnrichedRow struct {
ID uuid.UUID `db:"id"`
Hash string `db:"hash"`
Folderid *uuid.UUID `db:"folderid"`
Filename *string `db:"filename"`
Originalpath *string `db:"originalpath"`
ID uuid.UUID `db:"id"`
Hash string `db:"hash"`
Folderid *uuid.UUID `db:"folderid"`
Filename *string `db:"filename"`
Originalpath *string `db:"originalpath"`
FileSizeBytes *int64 `db:"file_size_bytes"`
}
// Gets documents in a folder with all enriched metadata fields
@@ -154,7 +157,8 @@ type GetDocumentsByFolderEnrichedRow struct {
// d.hash,
// d.folderId,
// d.filename,
// d.originalPath
// d.originalPath,
// d.file_size_bytes
// FROM documents d
// WHERE d.folderId = $1
// ORDER BY d.id
@@ -173,6 +177,7 @@ func (q *Queries) GetDocumentsByFolderEnriched(ctx context.Context, folderid *uu
&i.Folderid,
&i.Filename,
&i.Originalpath,
&i.FileSizeBytes,
); err != nil {
return nil, err
}
internal/database/repository/labels.sql.go
+6 -4
View File
@@ -191,7 +191,7 @@ func (q *Queries) GetDocumentLabels(ctx context.Context, documentid uuid.UUID) (
}
const getDocumentsByLabel = `-- name: GetDocumentsByLabel :many
SELECT DISTINCT d.id, d.clientid, d.hash, d.batch_id, d.filename, d.folderid, d.originalpath
SELECT DISTINCT d.id, d.clientid, d.hash, d.batch_id, d.filename, d.folderid, d.originalpath, d.file_size_bytes
FROM documents d
INNER JOIN documentLabels dl ON d.id = dl.documentId
WHERE d.clientId = $1 AND dl.label = $2
@@ -205,7 +205,7 @@ type GetDocumentsByLabelParams struct {
// GetDocumentsByLabel
//
// SELECT DISTINCT d.id, d.clientid, d.hash, d.batch_id, d.filename, d.folderid, d.originalpath
// SELECT DISTINCT d.id, d.clientid, d.hash, d.batch_id, d.filename, d.folderid, d.originalpath, d.file_size_bytes
// FROM documents d
// INNER JOIN documentLabels dl ON d.id = dl.documentId
// WHERE d.clientId = $1 AND dl.label = $2
@@ -227,6 +227,7 @@ func (q *Queries) GetDocumentsByLabel(ctx context.Context, arg *GetDocumentsByLa
&i.Filename,
&i.Folderid,
&i.Originalpath,
&i.FileSizeBytes,
); err != nil {
return nil, err
}
@@ -239,7 +240,7 @@ func (q *Queries) GetDocumentsByLabel(ctx context.Context, arg *GetDocumentsByLa
}
const getDocumentsByLabelAndFolder = `-- name: GetDocumentsByLabelAndFolder :many
SELECT DISTINCT d.id, d.clientid, d.hash, d.batch_id, d.filename, d.folderid, d.originalpath
SELECT DISTINCT d.id, d.clientid, d.hash, d.batch_id, d.filename, d.folderid, d.originalpath, d.file_size_bytes
FROM documents d
INNER JOIN documentLabels dl ON d.id = dl.documentId
WHERE d.folderId = $1 AND dl.label = $2
@@ -253,7 +254,7 @@ type GetDocumentsByLabelAndFolderParams struct {
// GetDocumentsByLabelAndFolder
//
// SELECT DISTINCT d.id, d.clientid, d.hash, d.batch_id, d.filename, d.folderid, d.originalpath
// SELECT DISTINCT d.id, d.clientid, d.hash, d.batch_id, d.filename, d.folderid, d.originalpath, d.file_size_bytes
// FROM documents d
// INNER JOIN documentLabels dl ON d.id = dl.documentId
// WHERE d.folderId = $1 AND dl.label = $2
@@ -275,6 +276,7 @@ func (q *Queries) GetDocumentsByLabelAndFolder(ctx context.Context, arg *GetDocu
&i.Filename,
&i.Folderid,
&i.Originalpath,
&i.FileSizeBytes,
); err != nil {
return nil, err
}
internal/database/repository/models.go
+2 -1
View File
@@ -425,7 +425,8 @@ type Document struct {
Filename *string `db:"filename"`
Folderid *uuid.UUID `db:"folderid"`
// Original path provided during upload. IMMUTABLE after creation - never modify this value.
Originalpath *string `db:"originalpath"`
Originalpath *string `db:"originalpath"`
FileSizeBytes *int64 `db:"file_size_bytes"`
}
type Documentclean struct {