Fix accent-insensitive search for Postgres databases (#932)

This commit is contained in:
Matias Godoy
2025-08-05 02:35:22 +02:00
committed by GitHub
parent 0d3151ae5c
commit 362c0bb3e6
2 changed files with 39 additions and 55 deletions

View File

@@ -22,7 +22,7 @@ func AccentInsensitiveContains(field string, searchValue string) predicate.Item
return predicate.Item(func(s *sql.Selector) { return predicate.Item(func(s *sql.Selector) {
dialect := s.Dialect() dialect := s.Dialect()
switch dialect { switch dialect {
case "sqlite3": case "sqlite3":
// For SQLite, we'll create a custom normalization function using REPLACE // For SQLite, we'll create a custom normalization function using REPLACE
@@ -33,13 +33,15 @@ func AccentInsensitiveContains(field string, searchValue string) predicate.Item
"%"+normalizedSearch+"%", "%"+normalizedSearch+"%",
)) ))
case "postgres": case "postgres":
// For PostgreSQL, try to use unaccent extension if available // For PostgreSQL, use REPLACE-based normalization to avoid unaccent dependency
// Fall back to REPLACE-based normalization if not available normalizeFunc := buildGenericNormalizeExpression(s.C(field))
normalizeFunc := buildPostgreSQLNormalizeExpression(s.C(field)) // Use sql.P() for proper PostgreSQL parameter binding ($1, $2, etc.)
s.Where(sql.ExprP( s.Where(sql.P(func(b *sql.Builder) {
"LOWER("+normalizeFunc+") LIKE ?", b.WriteString("LOWER(")
"%"+normalizedSearch+"%", b.WriteString(normalizeFunc)
)) b.WriteString(") LIKE ")
b.Arg("%" + normalizedSearch + "%")
}))
default: default:
// Default fallback using REPLACE for common accented characters // Default fallback using REPLACE for common accented characters
normalizeFunc := buildGenericNormalizeExpression(s.C(field)) normalizeFunc := buildGenericNormalizeExpression(s.C(field))
@@ -56,22 +58,13 @@ func buildSQLiteNormalizeExpression(fieldExpr string) string {
return buildGenericNormalizeExpression(fieldExpr) return buildGenericNormalizeExpression(fieldExpr)
} }
// buildPostgreSQLNormalizeExpression creates a PostgreSQL expression to normalize accented characters
func buildPostgreSQLNormalizeExpression(fieldExpr string) string {
// Use a CASE statement to check if unaccent function exists before using it
// This prevents errors when the unaccent extension is not installed
return "CASE WHEN EXISTS (SELECT 1 FROM pg_proc WHERE proname = 'unaccent') " +
"THEN unaccent(" + fieldExpr + ") " +
"ELSE " + buildGenericNormalizeExpression(fieldExpr) + " END"
}
// buildGenericNormalizeExpression creates a database-agnostic expression to normalize common accented characters // buildGenericNormalizeExpression creates a database-agnostic expression to normalize common accented characters
func buildGenericNormalizeExpression(fieldExpr string) string { func buildGenericNormalizeExpression(fieldExpr string) string {
// Chain REPLACE functions to handle the most common accented characters // Chain REPLACE functions to handle the most common accented characters
// Focused on the most frequently used accents in Spanish, French, and Portuguese // Focused on the most frequently used accents in Spanish, French, and Portuguese
// Ordered by frequency of use for better performance // Ordered by frequency of use for better performance
normalized := fieldExpr normalized := fieldExpr
// Most common accented characters ordered by frequency // Most common accented characters ordered by frequency
commonAccents := []struct { commonAccents := []struct {
from, to string from, to string
@@ -88,11 +81,11 @@ func buildGenericNormalizeExpression(fieldExpr string) string {
{"ä", "a"}, {"ö", "o"}, {"ü", "u"}, {"ã", "a"}, {"õ", "o"}, {"ä", "a"}, {"ö", "o"}, {"ü", "u"}, {"ã", "a"}, {"õ", "o"},
{"Ä", "A"}, {"Ö", "O"}, {"Ü", "U"}, {"Ã", "A"}, {"Õ", "O"}, {"Ä", "A"}, {"Ö", "O"}, {"Ü", "U"}, {"Ã", "A"}, {"Õ", "O"},
} }
for _, accent := range commonAccents { for _, accent := range commonAccents {
normalized = "REPLACE(" + normalized + ", '" + accent.from + "', '" + accent.to + "')" normalized = "REPLACE(" + normalized + ", '" + accent.from + "', '" + accent.to + "')"
} }
return normalized return normalized
} }

View File

@@ -27,19 +27,19 @@ func TestBuildGenericNormalizeExpression(t *testing.T) {
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
result := buildGenericNormalizeExpression(tt.field) result := buildGenericNormalizeExpression(tt.field)
// Should contain the original field // Should contain the original field
assert.Contains(t, result, tt.field) assert.Contains(t, result, tt.field)
// Should contain REPLACE functions for accent normalization // Should contain REPLACE functions for accent normalization
assert.Contains(t, result, "REPLACE(") assert.Contains(t, result, "REPLACE(")
// Should handle common accented characters // Should handle common accented characters
assert.Contains(t, result, "'á'", "Should handle Spanish á") assert.Contains(t, result, "'á'", "Should handle Spanish á")
assert.Contains(t, result, "'é'", "Should handle Spanish é") assert.Contains(t, result, "'é'", "Should handle Spanish é")
assert.Contains(t, result, "'ñ'", "Should handle Spanish ñ") assert.Contains(t, result, "'ñ'", "Should handle Spanish ñ")
assert.Contains(t, result, "'ü'", "Should handle German ü") assert.Contains(t, result, "'ü'", "Should handle German ü")
// Should handle uppercase accents too // Should handle uppercase accents too
assert.Contains(t, result, "'Á'", "Should handle uppercase Spanish Á") assert.Contains(t, result, "'Á'", "Should handle uppercase Spanish Á")
assert.Contains(t, result, "'É'", "Should handle uppercase Spanish É") assert.Contains(t, result, "'É'", "Should handle uppercase Spanish É")
@@ -49,7 +49,7 @@ func TestBuildGenericNormalizeExpression(t *testing.T) {
func TestSQLiteNormalizeExpression(t *testing.T) { func TestSQLiteNormalizeExpression(t *testing.T) {
result := buildSQLiteNormalizeExpression("test_field") result := buildSQLiteNormalizeExpression("test_field")
// Should contain the field name and REPLACE functions // Should contain the field name and REPLACE functions
assert.Contains(t, result, "test_field") assert.Contains(t, result, "test_field")
assert.Contains(t, result, "REPLACE(") assert.Contains(t, result, "REPLACE(")
@@ -58,15 +58,6 @@ func TestSQLiteNormalizeExpression(t *testing.T) {
assert.Contains(t, result, "'ó'", "Should handle Spanish ó") assert.Contains(t, result, "'ó'", "Should handle Spanish ó")
} }
func TestPostgreSQLNormalizeExpression(t *testing.T) {
result := buildPostgreSQLNormalizeExpression("test_field")
// Should contain unaccent function and CASE WHEN logic
assert.Contains(t, result, "unaccent(")
assert.Contains(t, result, "CASE WHEN EXISTS")
assert.Contains(t, result, "test_field")
}
func TestAccentInsensitivePredicateCreation(t *testing.T) { func TestAccentInsensitivePredicateCreation(t *testing.T) {
tests := []struct { tests := []struct {
name string name string
@@ -104,46 +95,46 @@ func TestAccentInsensitivePredicateCreation(t *testing.T) {
func TestSpecificItemPredicates(t *testing.T) { func TestSpecificItemPredicates(t *testing.T) {
tests := []struct { tests := []struct {
name string name string
predicateFunc func(string) interface{} predicateFunc func(string) interface{}
searchValue string searchValue string
description string description string
}{ }{
{ {
name: "ItemNameAccentInsensitiveContains", name: "ItemNameAccentInsensitiveContains",
predicateFunc: func(val string) interface{} { return ItemNameAccentInsensitiveContains(val) }, predicateFunc: func(val string) interface{} { return ItemNameAccentInsensitiveContains(val) },
searchValue: "electronica", searchValue: "electronica",
description: "Should create accent-insensitive name search predicate", description: "Should create accent-insensitive name search predicate",
}, },
{ {
name: "ItemDescriptionAccentInsensitiveContains", name: "ItemDescriptionAccentInsensitiveContains",
predicateFunc: func(val string) interface{} { return ItemDescriptionAccentInsensitiveContains(val) }, predicateFunc: func(val string) interface{} { return ItemDescriptionAccentInsensitiveContains(val) },
searchValue: "descripcion", searchValue: "descripcion",
description: "Should create accent-insensitive description search predicate", description: "Should create accent-insensitive description search predicate",
}, },
{ {
name: "ItemManufacturerAccentInsensitiveContains", name: "ItemManufacturerAccentInsensitiveContains",
predicateFunc: func(val string) interface{} { return ItemManufacturerAccentInsensitiveContains(val) }, predicateFunc: func(val string) interface{} { return ItemManufacturerAccentInsensitiveContains(val) },
searchValue: "compañia", searchValue: "compañia",
description: "Should create accent-insensitive manufacturer search predicate", description: "Should create accent-insensitive manufacturer search predicate",
}, },
{ {
name: "ItemSerialNumberAccentInsensitiveContains", name: "ItemSerialNumberAccentInsensitiveContains",
predicateFunc: func(val string) interface{} { return ItemSerialNumberAccentInsensitiveContains(val) }, predicateFunc: func(val string) interface{} { return ItemSerialNumberAccentInsensitiveContains(val) },
searchValue: "sn123", searchValue: "sn123",
description: "Should create accent-insensitive serial number search predicate", description: "Should create accent-insensitive serial number search predicate",
}, },
{ {
name: "ItemModelNumberAccentInsensitiveContains", name: "ItemModelNumberAccentInsensitiveContains",
predicateFunc: func(val string) interface{} { return ItemModelNumberAccentInsensitiveContains(val) }, predicateFunc: func(val string) interface{} { return ItemModelNumberAccentInsensitiveContains(val) },
searchValue: "model456", searchValue: "model456",
description: "Should create accent-insensitive model number search predicate", description: "Should create accent-insensitive model number search predicate",
}, },
{ {
name: "ItemNotesAccentInsensitiveContains", name: "ItemNotesAccentInsensitiveContains",
predicateFunc: func(val string) interface{} { return ItemNotesAccentInsensitiveContains(val) }, predicateFunc: func(val string) interface{} { return ItemNotesAccentInsensitiveContains(val) },
searchValue: "notas importantes", searchValue: "notas importantes",
description: "Should create accent-insensitive notes search predicate", description: "Should create accent-insensitive notes search predicate",
}, },
} }