Avoid skip of non-ASCII chars in skip article code

The code used isalnum to skip leading white space. This could lead to
skip of whole string if that contained anything but 7-bit characters -
and as a result it would skip the whole string buffer (strstr
would find " ").

diff --git a/src/charsets.c b/src/charsets.c
index 7bbe102..d70c11d 100644 (file)
--- a/src/charsets.c
+++ b/src/charsets.c
@@ -184,8 +184,6 @@ void pp2_relevance_first(pp2_relevance_token_t prt,
         char *pout = firstword;
         char articles[] = "the den der die des an a "; // must end in space
         
-        while (*p && !isalnum(*(unsigned char *)p))
-            p++;
         for (; *p && *p != ' ' && pout - firstword < (sizeof(firstword)-2); p++)
             *pout++ = tolower(*(unsigned char *)p);
         *pout++ = ' ';