From 5a3a7b24affaafd154841d059e79fc1b93b596dd Mon Sep 17 00:00:00 2001
From: zzz <zzz@i2pmail.org>
Date: Fri, 9 Aug 2024 09:37:50 -0400
Subject: [PATCH] i2psnark, susimail: Normailze strings when searching (Gitlab
 #488)

to improve search for non-ASCII strings.
Normalize both search and searched strings to NFKD when searching.
NFKD is best because it is more efficient than NFKC and it removes
formatting distinctions. This results in the best fuzzy matching.
---
 .../java/src/org/klomp/snark/web/I2PSnarkServlet.java    | 5 +++--
 apps/susimail/src/src/i2p/susi/webmail/WebMail.java      | 9 +++++----
 2 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/apps/i2psnark/java/src/org/klomp/snark/web/I2PSnarkServlet.java b/apps/i2psnark/java/src/org/klomp/snark/web/I2PSnarkServlet.java
index 421fcd361..0f7251066 100644
--- a/apps/i2psnark/java/src/org/klomp/snark/web/I2PSnarkServlet.java
+++ b/apps/i2psnark/java/src/org/klomp/snark/web/I2PSnarkServlet.java
@@ -13,6 +13,7 @@ import java.net.URI;
 import java.net.URISyntaxException;
 import java.text.Collator;
 import java.text.DecimalFormat;
+import java.text.Normalizer;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
@@ -875,14 +876,14 @@ public class I2PSnarkServlet extends BasicServlet {
             if (term.length() > 0) {
                 if (searchList == null)
                     searchList = new ArrayList<String>(4);
-                searchList.add(term.toLowerCase(Locale.US));
+                searchList.add(Normalizer.normalize(term.toLowerCase(Locale.US), Normalizer.Form.NFKD));
             }
         }
         if (searchList == null)
             return null;
         List<Snark> matches = new ArrayList<Snark>(32);
         for (Snark snark : snarks) {
-            String lcname = snark.getBaseName().toLowerCase(Locale.US);
+            String lcname = Normalizer.normalize(snark.getBaseName().toLowerCase(Locale.US), Normalizer.Form.NFKD);
             // search for any term (OR)
             for (int j = 0; j < searchList.size(); j++) {
                 String term = searchList.get(j);
diff --git a/apps/susimail/src/src/i2p/susi/webmail/WebMail.java b/apps/susimail/src/src/i2p/susi/webmail/WebMail.java
index 6d5824d53..d038ec616 100644
--- a/apps/susimail/src/src/i2p/susi/webmail/WebMail.java
+++ b/apps/susimail/src/src/i2p/susi/webmail/WebMail.java
@@ -62,6 +62,7 @@ import java.net.URI;
 import java.net.URISyntaxException;
 import java.net.URLEncoder;
 import java.text.Collator;
+import java.text.Normalizer;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Comparator;
@@ -3851,7 +3852,7 @@ public class WebMail extends HttpServlet
 			terms = DataHelper.split(search, " ");
 			// decode
 			for (int i = 0; i < terms.length; i++) {
-				terms[i] = terms[i].toLowerCase(Locale.US);
+				terms[i] = Normalizer.normalize(terms[i].toLowerCase(Locale.US), Normalizer.Form.NFKD);
 			}
 		}
 
@@ -3863,10 +3864,10 @@ public class WebMail extends HttpServlet
 			Mail mail = mc.getMail(uidl, MailCache.FetchMode.HEADER_CACHE_ONLY);
 			if (mail == null)
 				return false;
-			String subj = mail.subject.toLowerCase(Locale.US);
+			String subj = Normalizer.normalize(mail.subject.toLowerCase(Locale.US), Normalizer.Form.NFKD);
 			String sender = isDrafts ? null : mail.sender;
 			if (sender != null)
-				sender = sender.toLowerCase(Locale.US);
+				sender = Normalizer.normalize(sender.toLowerCase(Locale.US), Normalizer.Form.NFKD);
 			String[] to = isDrafts ? mail.to : null;
 			for (String term : terms) {
 				if (subj.contains(term))
@@ -3875,7 +3876,7 @@ public class WebMail extends HttpServlet
 					return true;
 				if (to != null) {
 					for (int i = 0; i < to.length; i++) {
-						if (to[i].toLowerCase(Locale.US).contains(term))
+						if (Normalizer.normalize(to[i].toLowerCase(Locale.US), Normalizer.Form.NFKD).contains(term))
 							return true;
 					}
 				}