Normalize search field names case-insensitively in SearchScheme (#14154)

* Fix #14142: normalize cased search field names (ISBN/LCC) in WorkSearchScheme

* Normalize search field names case-insensitively in SearchScheme

Field names were validated case-insensitively but never rewritten, so
cased fields (ISBN:, LCC:, ID_GOODREADS:, Edition.ISBN:, NAME:) reached
Solr unchanged: works search 500'd and author/subject/edition search
silently returned nothing.

Add SearchScheme.canonical_field_name(), which matches a field
case-insensitively, resolves unprefixed aliases and applies the scheme's
own is_search_field. process_user_query now uses it both to validate
fields and to rename them after parsing, so every scheme sends Solr
canonical names. Alias resolution moves out of
WorkSearchScheme.transform_user_query, which supersedes the earlier
works-only normalization. EditionSearchScheme's publishers -> publisher
alias now applies on /search/editions.json.

---------

Co-authored-by: Raymond Berger <RayBB@users.noreply.github.com>

Author: wanjinhao1

openlibrary/plugins/worksearch/schemes/__init__.py

@@ -4,11 +4,13 @@
 from types import MappingProxyType
 
 from luqum.exceptions import ParseError
+from luqum.tree import SearchField
 
 from openlibrary.solr.query_utils import (
     escape_unknown_fields,
     fully_escape_query,
     luqum_parser,
+    luqum_traverse,
 )
 from openlibrary.utils.request_context import get_request_lang
 
@@ -47,6 +49,20 @@ def __init__(self, lang: str | None = None):
     def is_search_field(self, field: str):
         return field in self.all_fields or field in self.field_name_map
 
+    def canonical_field_name(self, field: str) -> str | None:
+        """
+        The schema field a user-typed search field refers to, or None if it
+        isn't searchable. Matching is case-insensitive and unprefixed aliases in
+        `field_name_map` are resolved, e.g. "ISBN" -> "isbn", "AUTHOR" -> "author_name".
+        Prefixed fields are only lowercased: "Work.Author" -> "work.author".
+        """
+        lowered = field.lower()
+        if lowered in self.field_name_map:
+            return self.field_name_map[lowered]
+        if self.is_search_field(lowered):
+            return lowered
+        return None
+
     def process_user_sort(self, user_sort: str) -> str:
         """
         Convert a user-provided sort to a solr sort
@@ -104,8 +120,8 @@ def process_user_query(self, q_param: str) -> str:
                     .replace("?", "\\?")
                     .replace("~", "\\~")
                 ),
-                self.is_search_field,
-                lower=True,
+                lambda field: self.canonical_field_name(field) is not None,
+                lower=False,
             )
             q_tree = luqum_parser(q_param)
         except ParseError:
@@ -114,6 +130,12 @@ def process_user_query(self, q_param: str) -> str:
             # Escape everything we can
             q_tree = luqum_parser(fully_escape_query(q_param))
 
+        # Rewrite the fields that passed validation above to their schema names,
+        # so Solr and transform_user_query only ever see canonical names.
+        for node, _ in luqum_traverse(q_tree):
+            if isinstance(node, SearchField) and (name := self.canonical_field_name(node.name)):
+                node.name = name
+
         q_tree = self.transform_user_query(q_param, q_tree)
         return str(q_tree)
 

openlibrary/plugins/worksearch/schemes/tests/test_scheme.py

@@ -0,0 +1,44 @@
+import pytest
+
+from openlibrary.plugins.worksearch.schemes.authors import AuthorSearchScheme
+from openlibrary.plugins.worksearch.schemes.editions import EditionSearchScheme
+from openlibrary.plugins.worksearch.schemes.lists import ListSearchScheme
+from openlibrary.plugins.worksearch.schemes.subjects import SubjectSearchScheme
+from openlibrary.plugins.worksearch.schemes.works import WorkSearchScheme
+
+
+@pytest.mark.parametrize(
+    ("scheme", "field", "expected"),
+    [
+        (WorkSearchScheme, "isbn", "isbn"),
+        (WorkSearchScheme, "ISBN", "isbn"),
+        (WorkSearchScheme, "author", "author_name"),
+        (WorkSearchScheme, "AUTHOR", "author_name"),
+        (WorkSearchScheme, "ID_GOODREADS", "id_goodreads"),
+        (WorkSearchScheme, "Edition.ISBN", "edition.isbn"),
+        (WorkSearchScheme, "foo", None),
+        (EditionSearchScheme, "Publishers", "publisher"),
+        (AuthorSearchScheme, "NAME", "name"),
+        (AuthorSearchScheme, "isbn", None),
+    ],
+)
+def test_canonical_field_name(scheme, field, expected):
+    assert scheme(lang="en").canonical_field_name(field) == expected
+
+
+@pytest.mark.parametrize(
+    ("scheme", "query", "expected"),
+    [
+        (AuthorSearchScheme, "NAME:Twain", "name:Twain"),
+        (AuthorSearchScheme, "Name:Mark Twain OR Top_Subjects:fiction", "name:(Mark Twain) OR top_subjects:fiction"),
+        (SubjectSearchScheme, "Work_Count:[10 TO *]", "work_count:[10 TO *]"),
+        (ListSearchScheme, "NAME:Fantasy", "name:Fantasy"),
+        (EditionSearchScheme, "ISBN:0060935464", "isbn:0060935464"),
+        (EditionSearchScheme, "ID_GOODREADS:123", "id_goodreads:123"),
+        (EditionSearchScheme, "publishers:Harper", "publisher:Harper"),
+        (EditionSearchScheme, "Publishers:Harper", "publisher:Harper"),
+        (AuthorSearchScheme, "FOO:bar", "FOO\\:bar"),
+    ],
+)
+def test_process_user_query_canonicalizes_fields(scheme, query, expected):
+    assert scheme(lang="en").process_user_query(query) == expected

openlibrary/plugins/worksearch/schemes/tests/test_works.py

@@ -71,6 +71,10 @@
         "isbn:978-0-06-093546-7",
         "isbn:9780060935467",
     ),
+    "Normalizes uppercase ISBN field": (
+        "ISBN:978-0-06-093546-7",
+        "isbn:9780060935467",
+    ),
     "Does not normalize ISBN stars": (
         "isbn:979*",
         "isbn:979*",
@@ -80,6 +84,10 @@
         "lcc:NC760 .B2813 2004",
         'lcc:"NC-0760.00000000.B2813 2004"',
     ),
+    "LCC: uppercase field normalized": (
+        "LCC:NC760 .B2813 2004",
+        'lcc:"NC-0760.00000000.B2813 2004"',
+    ),
     "LCC: star added if no space": (
         "lcc:NC760 .B2813",
         "lcc:NC-0760.00000000.B2813*",
@@ -121,6 +129,16 @@
     "DDC: wildcard integer": ("ddc:2*", "ddc:2*"),
     "DDC: ddc_sort": ("ddc_sort:65.8", "ddc_sort:065.8"),
     "DDC: quotes preserved": ('ddc:"65.8"', 'ddc:"065.8"'),
+    "DDC: uppercase field normalized": ("DDC:65.8", "ddc:065.8"),
+    "Normalizes cased id_ field": ("ID_GOODREADS:123", "id_goodreads:123"),
+    "Normalizes cased edition. prefixed field": ("Edition.Publisher:Harper", "edition.publisher:Harper"),
+    "Uppercase alias resolved": ("AUTHOR:Twain", "author_name:Twain"),
+    "Negated uppercase ISBN": ("-ISBN:978-0-06-093546-7", "-isbn:9780060935467"),
+    "Grouped mixed-case fields": ("(ISBN:9780060935467 OR LCC:PS1303)", "(isbn:9780060935467 OR lcc:PS-1303.00000000)"),
+    "Uppercase LCC range": ("LCC:[PS1 TO PS2]", "lcc:[PS-0001.00000000 TO PS-0002.00000000]"),
+    "Unknown cased field still escaped": ("Foo:bar", "Foo\\:bar"),
+    "Cased field after *:*": ("*:* AND ISBN:123", "*\\:* AND isbn:123"),
+    "Prefixed alias only lowercased": ("Work.Title:foo", "work.title:foo"),
 }
 
 

openlibrary/plugins/worksearch/schemes/works.py

@@ -300,8 +300,6 @@ def transform_user_query(self, user_query: str, q_tree: luqum.tree.Item) -> luqu
         for node, parents in luqum_traverse(q_tree):
             if isinstance(node, luqum.tree.SearchField):
                 has_search_fields = True
-                if node.name.lower() in self.field_name_map:
-                    node.name = self.field_name_map[node.name.lower()]
                 if node.name == "isbn":
                     isbn_transform(node)
                 if node.name in ("lcc", "lcc_sort"):