mirror of
https://github.com/PerpetualSoftware/pad.git
synced 2026-09-20 09:33:28 +00:00
0fd5d0cdfb
* fix(store): swap plainto_tsquery → websearch_to_tsquery for PG FTS (BUG-842)
`TestListItems_FTS_HyphenatedSearchTerm/task-five` has been failing on
every Go (PostgreSQL) CI run because `plainto_tsquery('english',
'task-five')` doesn't match the asciihword lexeme(s) the english parser
produces for an indexed `task-five-distinctive`. The result is that
every PG full-text search for hyphenated terms returns zero rows.
`websearch_to_tsquery` (Postgres 11+) is purpose-built for arbitrary
user input and tokenizes hyphenated terms the same way `to_tsvector`
does for the indexed document, so the query intersects the index
correctly. Swapped in three spots in the postgres dialect — FTSMatch,
FTSSnippet, FTSRank — and updated the caller-side comments that
referenced plainto_tsquery. SQLite path is unchanged: it goes through
items_fts MATCH with sanitizeFTSQuery, never through these methods.
* fix(server): drain background goroutines on Stop() (BUG-842)
`TestAdminBillingStats_SidecarSidecarError_DegradesToLocalOnly` (and
other server tests) have been flaking on the Go (PostgreSQL) CI runner
with `TempDir RemoveAll cleanup: directory not empty`. Root cause:
several request handlers spawned bare `go func() { ... }()` goroutines
that touched the SQLite WAL DB after the test function returned.
testServer's t.Cleanup closed the store but had no way to drain those
goroutines first, so a fire-and-forget WAL write could re-create the
`-wal`/`-shm` files between Close() and t.TempDir's RemoveAll.
Add a Server.bg sync.WaitGroup, a Server.goAsync helper that wraps a
WaitGroup-tracked goroutine, and a Server.Stop() that blocks until
every goAsync closure has finished. Convert the four known
fire-and-forget sites to goAsync:
- middleware_auth.go (TouchUserActivity)
- handlers_auth.go (password reset email)
- handlers_cloud.go (stripe_processed_events pruning)
- handlers_members.go (workspace invitation email)
Wire `srv.Stop()` into both testServer (server_test.go) and
newMetricsTestServer (metrics_auth_test.go) so cleanup order is
Stop → Close → TempDir RemoveAll. Add
TestServer_Stop_DrainsBackgroundGoroutines to pin the contract: a
goAsync goroutine must block Stop until it returns.
* fix(store): correct PG FTS hyphenation via OR-combined plainto_tsquery (BUG-842)
The previous attempt swapped plainto_tsquery → websearch_to_tsquery,
which was wrong: websearch_to_tsquery treats `-` as a NEGATION operator
(Google-style), so `task-five` becomes `task & !five` and the search
returns 0 rows for the same reason as before. This commit reverts the
swap and applies the actual fix.
PG's english parser indexes `task-five-distinctive` as
`{task-five-distinct, task, five, distinct}` — the asciihword AND its
parts. plainto_tsquery applied to the partial query `task-five`
produces `task-fiv & task & five`: the stemmed asciihword for the
PARTIAL query (`task-fiv`) is NOT in the vector, so the AND fails.
Replacing the hyphen with a space makes plainto emit `task & five`,
which DOES match — but doing that unconditionally breaks `BUG-842`-
style queries: PG indexes the `-842` suffix as a negative-number
lexeme, so `plainto_tsquery('BUG-842')` matches via `-842`, while
`plainto_tsquery('BUG 842')` searches for `842` and misses.
The fix ORs the two query variants together so the search vector is
matched against either the raw user query OR its hyphen-as-space form.
Both `task-five` (against `task-five-distinctive`) and `BUG-842`
(against `BUG-842 fix the cleanup race`) hit. Verified locally against
postgres:17-alpine via PAD_TEST_POSTGRES_URL — both 10x stress and
race-detector runs are green.
Surfaces:
- dialect.go: FTSMatch / FTSSnippet / FTSRank now consume TWO
placeholders each in the PG dialect.
- items.go: listItemsFTS PG branch + SearchItems PG branch update
args to pass (raw, sanitized) for every PG `?` placeholder.
- search.go: SearchItems main / count / facets PG branches updated
likewise. New sanitizePGFTSQuery helper alongside sanitizeFTSQuery.
- documents.go: ListDocuments PG branch updated.
Tests:
- TestListItems_FTS_HyphenatedSearchTerm extended with a `BUG-842`
case to pin the OR-combined logic — naive hyphen-stripping would
silently regress this.
- New TestSanitizePGFTSQuery unit test.
* chore: gofmt 11 files with import-order issues (BUG-842 PR cleanup)
The Go (SQLite) CI job has been failing on `main` (and every PR built
against it) because golangci-lint flags 11 files whose third-party
imports are intermixed with internal imports — the import-grouping
rule that gofmt enforces. None of these were introduced by the
BUG-842 PR; they're pre-existing on main. The PR can't go green
without this cleanup, though, so it's bundled here.
Pure mechanical change — `gofmt -w <files>` only re-orders import
groups; no logic changes. Files touched:
cmd/pad/configure.go
cmd/pad/main.go
internal/cli/format.go
internal/server/handlers_admin_invitations.go
internal/server/handlers_admin_users.go
internal/server/handlers_grants.go
internal/server/handlers_share_links.go
internal/server/handlers_stars.go
internal/server/middleware_auth.go
internal/store/store.go
internal/store/store_test.go
After this commit `gofmt -l ./cmd ./internal` returns clean.
449 lines
12 KiB
Go
449 lines
12 KiB
Go
package store
|
|
|
|
import (
|
|
"database/sql"
|
|
"fmt"
|
|
"strings"
|
|
|
|
"github.com/PerpetualSoftware/pad/internal/diff"
|
|
"github.com/PerpetualSoftware/pad/internal/links"
|
|
"github.com/PerpetualSoftware/pad/internal/models"
|
|
)
|
|
|
|
func (s *Store) ListDocuments(workspaceID string, params models.DocumentListParams) ([]models.Document, error) {
|
|
query := `
|
|
SELECT id, workspace_id, title, slug, content, doc_type, status, tags,
|
|
pinned, sort_order, created_by, last_modified_by, source,
|
|
created_at, updated_at
|
|
FROM documents
|
|
WHERE workspace_id = ? AND deleted_at IS NULL
|
|
`
|
|
args := []interface{}{workspaceID}
|
|
|
|
if params.Type != "" {
|
|
query += " AND doc_type = ?"
|
|
args = append(args, params.Type)
|
|
}
|
|
if params.Status != "" {
|
|
query += " AND status = ?"
|
|
args = append(args, params.Status)
|
|
}
|
|
if params.Tag != "" {
|
|
tagExpr, tagArg := s.dialect.JSONArrayContains("tags", params.Tag)
|
|
query += " AND " + tagExpr
|
|
args = append(args, tagArg)
|
|
}
|
|
if params.Pinned != nil {
|
|
if *params.Pinned {
|
|
query += " AND pinned = TRUE"
|
|
} else {
|
|
query += " AND pinned = FALSE"
|
|
}
|
|
}
|
|
// Whitespace-only queries collapse to empty after FTS sanitization, and
|
|
// SQLite FTS5 errors on `MATCH ''`. Treat them as "no search filter" to
|
|
// match the !="" semantics callers expect. See BUG-818.
|
|
hasSearch := strings.TrimSpace(params.Query) != ""
|
|
if hasSearch {
|
|
// Use FTS for search
|
|
if s.dialect.Driver() == DriverSQLite {
|
|
ftsMatch := s.dialect.FTSMatch("documents_fts", "search_vector")
|
|
query = fmt.Sprintf(`
|
|
SELECT d.id, d.workspace_id, d.title, d.slug, d.content, d.doc_type, d.status, d.tags,
|
|
d.pinned, d.sort_order, d.created_by, d.last_modified_by, d.source,
|
|
d.created_at, d.updated_at
|
|
FROM documents d
|
|
JOIN documents_fts fts ON d.rowid = fts.rowid
|
|
WHERE d.workspace_id = ? AND d.deleted_at IS NULL
|
|
AND %s
|
|
`, ftsMatch)
|
|
// Sanitize so FTS5 specials (hyphens, AND/OR/NOT, parens) are
|
|
// treated as literals rather than boolean operators — see BUG-818.
|
|
args = []interface{}{workspaceID, sanitizeFTSQuery(params.Query)}
|
|
} else {
|
|
// PostgreSQL: search_vector lives on the documents table (aliased as "d").
|
|
// PG FTSMatch consumes TWO args (raw + hyphen-sanitized) for the
|
|
// OR-combined plainto_tsquery — see dialect.go and BUG-842.
|
|
ftsMatch := s.dialect.FTSMatch("d", "search_vector")
|
|
query = fmt.Sprintf(`
|
|
SELECT d.id, d.workspace_id, d.title, d.slug, d.content, d.doc_type, d.status, d.tags,
|
|
d.pinned, d.sort_order, d.created_by, d.last_modified_by, d.source,
|
|
d.created_at, d.updated_at
|
|
FROM documents d
|
|
WHERE d.workspace_id = ? AND d.deleted_at IS NULL
|
|
AND %s
|
|
`, ftsMatch)
|
|
args = []interface{}{workspaceID, params.Query, sanitizePGFTSQuery(params.Query)}
|
|
}
|
|
|
|
if params.Type != "" {
|
|
query += " AND d.doc_type = ?"
|
|
args = append(args, params.Type)
|
|
}
|
|
if params.Status != "" {
|
|
query += " AND d.status = ?"
|
|
args = append(args, params.Status)
|
|
}
|
|
// Tag and Pinned filters were silently dropped by the FTS branch
|
|
// before this fix — see BUG-820 (documents analog of BUG-812).
|
|
if params.Tag != "" {
|
|
tagExpr, tagArg := s.dialect.JSONArrayContains("d.tags", params.Tag)
|
|
query += " AND " + tagExpr
|
|
args = append(args, tagArg)
|
|
}
|
|
if params.Pinned != nil {
|
|
if *params.Pinned {
|
|
query += " AND d.pinned = TRUE"
|
|
} else {
|
|
query += " AND d.pinned = FALSE"
|
|
}
|
|
}
|
|
}
|
|
|
|
// Sort
|
|
sortCol := "updated_at"
|
|
if params.Sort != "" {
|
|
switch params.Sort {
|
|
case "title":
|
|
sortCol = "title"
|
|
case "created_at":
|
|
sortCol = "created_at"
|
|
case "updated_at":
|
|
sortCol = "updated_at"
|
|
case "sort_order":
|
|
sortCol = "sort_order"
|
|
}
|
|
}
|
|
order := "DESC"
|
|
if params.Order == "asc" {
|
|
order = "ASC"
|
|
}
|
|
|
|
if hasSearch {
|
|
if s.dialect.Driver() == DriverPostgres {
|
|
// PostgreSQL ts_rank(): higher = more relevant → DESC.
|
|
// PG FTSRank consumes TWO args (raw + sanitized) — BUG-842.
|
|
ftsRank := s.dialect.FTSRank("d", "search_vector")
|
|
query += fmt.Sprintf(" ORDER BY %s DESC, d.%s %s", ftsRank, sortCol, order)
|
|
args = append(args, params.Query, sanitizePGFTSQuery(params.Query))
|
|
} else {
|
|
// SQLite FTS5: rank is a hidden column on the FTS JOIN (ascending = better)
|
|
query += fmt.Sprintf(" ORDER BY rank, d.%s %s", sortCol, order)
|
|
}
|
|
} else {
|
|
query += fmt.Sprintf(" ORDER BY pinned DESC, %s %s", sortCol, order)
|
|
}
|
|
|
|
rows, err := s.db.Query(s.q(query), args...)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("list documents: %w", err)
|
|
}
|
|
defer rows.Close()
|
|
|
|
return scanDocuments(rows)
|
|
}
|
|
|
|
func (s *Store) CreateDocument(workspaceID string, input models.DocumentCreate) (*models.Document, error) {
|
|
id := newID()
|
|
ts := now()
|
|
|
|
docType := input.DocType
|
|
if docType == "" {
|
|
docType = "notes"
|
|
}
|
|
status := input.Status
|
|
if status == "" {
|
|
status = "draft"
|
|
}
|
|
tags := input.Tags
|
|
if tags == "" {
|
|
tags = "[]"
|
|
}
|
|
createdBy := input.CreatedBy
|
|
if createdBy == "" {
|
|
createdBy = "user"
|
|
}
|
|
source := input.Source
|
|
if source == "" {
|
|
source = "web"
|
|
}
|
|
|
|
baseSlug := slugify(input.Title)
|
|
if baseSlug == "" {
|
|
baseSlug = "untitled"
|
|
}
|
|
slug, err := s.uniqueSlug("documents", "workspace_id", workspaceID, baseSlug)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
_, err = s.db.Exec(s.q(`
|
|
INSERT INTO documents (id, workspace_id, title, slug, content, doc_type, status, tags,
|
|
pinned, sort_order, created_by, last_modified_by, source, created_at, updated_at)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 0, ?, ?, ?, ?, ?)
|
|
`), id, workspaceID, input.Title, slug, input.Content, docType, status, tags,
|
|
s.dialect.BoolToInt(input.Pinned), createdBy, createdBy, source, ts, ts)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("insert document: %w", err)
|
|
}
|
|
|
|
return s.GetDocument(id)
|
|
}
|
|
|
|
func (s *Store) GetDocument(id string) (*models.Document, error) {
|
|
var d models.Document
|
|
var createdAt, updatedAt string
|
|
var deletedAt *string
|
|
var pinned bool
|
|
|
|
err := s.db.QueryRow(s.q(`
|
|
SELECT id, workspace_id, title, slug, content, doc_type, status, tags,
|
|
pinned, sort_order, created_by, last_modified_by, source,
|
|
created_at, updated_at, deleted_at
|
|
FROM documents
|
|
WHERE id = ? AND deleted_at IS NULL
|
|
`), id).Scan(
|
|
&d.ID, &d.WorkspaceID, &d.Title, &d.Slug, &d.Content, &d.DocType, &d.Status, &d.Tags,
|
|
&pinned, &d.SortOrder, &d.CreatedBy, &d.LastModifiedBy, &d.Source,
|
|
&createdAt, &updatedAt, &deletedAt,
|
|
)
|
|
if err == sql.ErrNoRows {
|
|
return nil, nil
|
|
}
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
d.Pinned = pinned
|
|
d.CreatedAt = parseTime(createdAt)
|
|
d.UpdatedAt = parseTime(updatedAt)
|
|
d.DeletedAt = parseTimePtr(deletedAt)
|
|
return &d, nil
|
|
}
|
|
|
|
func (s *Store) UpdateDocument(id string, input models.DocumentUpdate) (*models.Document, error) {
|
|
existing, err := s.GetDocument(id)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if existing == nil {
|
|
return nil, nil
|
|
}
|
|
|
|
tx, err := s.db.Begin()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer tx.Rollback()
|
|
|
|
ts := now()
|
|
|
|
// Create version if content is changing (throttled to avoid bloat from auto-save)
|
|
if input.Content != nil && *input.Content != existing.Content {
|
|
createdBy := input.LastModifiedBy
|
|
if createdBy == "" {
|
|
createdBy = "user"
|
|
}
|
|
source := input.Source
|
|
if source == "" {
|
|
source = "web"
|
|
}
|
|
|
|
// Title changes always get a version; content-only changes are throttled
|
|
forceVersion := input.Title != nil && *input.Title != existing.Title
|
|
shouldVersion := forceVersion
|
|
if !shouldVersion {
|
|
shouldVersion, err = s.ShouldCreateVersion(id, createdBy, source)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("check version throttle: %w", err)
|
|
}
|
|
}
|
|
|
|
if shouldVersion {
|
|
vid := newID()
|
|
|
|
// Store a reverse diff (patch from new → old) instead of full content.
|
|
// Falls back to full content if the diff isn't meaningfully smaller.
|
|
versionContent := existing.Content
|
|
isDiff := false
|
|
patch := diff.CreateReversePatch(existing.Content, *input.Content)
|
|
if diff.IsDiffSmaller(patch, existing.Content) {
|
|
versionContent = patch
|
|
isDiff = true
|
|
}
|
|
|
|
_, err = tx.Exec(s.q(`
|
|
INSERT INTO versions (id, document_id, content, change_summary, created_by, source, is_diff, created_at)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
`), vid, id, versionContent, input.ChangeSummary, createdBy, source, s.dialect.BoolToInt(isDiff), ts)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("create version: %w", err)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Update [[link]] references if title is changing
|
|
if input.Title != nil && *input.Title != existing.Title {
|
|
err = s.updateLinksInTx(tx, existing.WorkspaceID, existing.Title, *input.Title)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("update links: %w", err)
|
|
}
|
|
}
|
|
|
|
// Build update query
|
|
sets := []string{"updated_at = ?"}
|
|
args := []interface{}{ts}
|
|
|
|
if input.Title != nil {
|
|
sets = append(sets, "title = ?")
|
|
args = append(args, *input.Title)
|
|
// Update slug too, ensuring uniqueness within workspace
|
|
baseSlug := slugify(*input.Title)
|
|
if baseSlug == "" {
|
|
baseSlug = "untitled"
|
|
}
|
|
newSlug, err := s.uniqueSlugExcluding("documents", "workspace_id", existing.WorkspaceID, baseSlug, id)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("unique slug: %w", err)
|
|
}
|
|
sets = append(sets, "slug = ?")
|
|
args = append(args, newSlug)
|
|
}
|
|
if input.Content != nil {
|
|
sets = append(sets, "content = ?")
|
|
args = append(args, *input.Content)
|
|
}
|
|
if input.DocType != nil {
|
|
sets = append(sets, "doc_type = ?")
|
|
args = append(args, *input.DocType)
|
|
}
|
|
if input.Status != nil {
|
|
sets = append(sets, "status = ?")
|
|
args = append(args, *input.Status)
|
|
}
|
|
if input.Tags != nil {
|
|
sets = append(sets, "tags = ?")
|
|
args = append(args, *input.Tags)
|
|
}
|
|
if input.Pinned != nil {
|
|
sets = append(sets, "pinned = ?")
|
|
args = append(args, s.dialect.BoolToInt(*input.Pinned))
|
|
}
|
|
if input.SortOrder != nil {
|
|
sets = append(sets, "sort_order = ?")
|
|
args = append(args, *input.SortOrder)
|
|
}
|
|
if input.LastModifiedBy != "" {
|
|
sets = append(sets, "last_modified_by = ?")
|
|
args = append(args, input.LastModifiedBy)
|
|
}
|
|
if input.Source != "" {
|
|
sets = append(sets, "source = ?")
|
|
args = append(args, input.Source)
|
|
}
|
|
|
|
args = append(args, id)
|
|
query := fmt.Sprintf("UPDATE documents SET %s WHERE id = ?", strings.Join(sets, ", "))
|
|
_, err = tx.Exec(s.q(query), args...)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("update document: %w", err)
|
|
}
|
|
|
|
if err := tx.Commit(); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return s.GetDocument(id)
|
|
}
|
|
|
|
func (s *Store) updateLinksInTx(tx *sql.Tx, workspaceID, oldTitle, newTitle string) error {
|
|
// Find all documents in the workspace that contain [[oldTitle]]
|
|
searchTerm := "[[" + oldTitle + "]]"
|
|
rows, err := tx.Query(s.q(`
|
|
SELECT id, content FROM documents
|
|
WHERE workspace_id = ? AND deleted_at IS NULL AND content LIKE ?
|
|
`), workspaceID, "%"+searchTerm+"%")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer rows.Close()
|
|
|
|
type docUpdate struct {
|
|
id string
|
|
content string
|
|
}
|
|
var updates []docUpdate
|
|
for rows.Next() {
|
|
var du docUpdate
|
|
if err := rows.Scan(&du.id, &du.content); err != nil {
|
|
return err
|
|
}
|
|
du.content = links.ReplaceTitle(du.content, oldTitle, newTitle)
|
|
updates = append(updates, du)
|
|
}
|
|
if err := rows.Err(); err != nil {
|
|
return err
|
|
}
|
|
|
|
for _, du := range updates {
|
|
_, err = tx.Exec(s.q("UPDATE documents SET content = ? WHERE id = ?"), du.content, du.id)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (s *Store) DeleteDocument(id string) error {
|
|
ts := now()
|
|
result, err := s.db.Exec(s.q(`
|
|
UPDATE documents SET deleted_at = ?, updated_at = ?, status = 'archived'
|
|
WHERE id = ? AND deleted_at IS NULL
|
|
`), ts, ts, id)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
rows, _ := result.RowsAffected()
|
|
if rows == 0 {
|
|
return sql.ErrNoRows
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (s *Store) RestoreDocument(id string) (*models.Document, error) {
|
|
ts := now()
|
|
result, err := s.db.Exec(s.q(`
|
|
UPDATE documents SET deleted_at = NULL, updated_at = ?, status = 'draft'
|
|
WHERE id = ? AND deleted_at IS NOT NULL
|
|
`), ts, id)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
rows, _ := result.RowsAffected()
|
|
if rows == 0 {
|
|
return nil, sql.ErrNoRows
|
|
}
|
|
return s.GetDocument(id)
|
|
}
|
|
|
|
func scanDocuments(rows *sql.Rows) ([]models.Document, error) {
|
|
var docs []models.Document
|
|
for rows.Next() {
|
|
var d models.Document
|
|
var createdAt, updatedAt string
|
|
var pinned bool
|
|
if err := rows.Scan(
|
|
&d.ID, &d.WorkspaceID, &d.Title, &d.Slug, &d.Content, &d.DocType, &d.Status, &d.Tags,
|
|
&pinned, &d.SortOrder, &d.CreatedBy, &d.LastModifiedBy, &d.Source,
|
|
&createdAt, &updatedAt,
|
|
); err != nil {
|
|
return nil, err
|
|
}
|
|
d.Pinned = pinned
|
|
d.CreatedAt = parseTime(createdAt)
|
|
d.UpdatedAt = parseTime(updatedAt)
|
|
docs = append(docs, d)
|
|
}
|
|
return docs, rows.Err()
|
|
}
|