--- a/extensions/spellcheck/hunspell/src/Makefile.in
+++ b/extensions/spellcheck/hunspell/src/Makefile.in
@@ -56,19 +56,23 @@ REQUIRES = xpcom \
CPPSRCS = mozHunspell.cpp \
mozHunspellDirProvider.cpp \
$(NULL)
ifndef MOZ_NATIVE_HUNSPELL
CPPSRCS += affentry.cpp \
affixmgr.cpp \
csutil.cpp \
+ dictmgr.cpp \
+ filemgr.cpp \
hashmgr.cpp \
hunspell.cpp \
+ hunzip.cpp \
phonet.cpp \
+ replist.cpp \
suggestmgr.cpp \
$(NULL)
endif
EXTRA_DSO_LDOPTS = \
$(LIBS_DIR) \
$(XPCOM_LIBS) \
$(NSPR_LIBS) \
--- a/extensions/spellcheck/hunspell/src/README.hunspell
+++ b/extensions/spellcheck/hunspell/src/README.hunspell
@@ -29,17 +29,17 @@
* use your version of this file under the terms of the MPL, indicate your
* decision by deleting the provisions above and replace them with the notice
* and other provisions required by the GPL or the LGPL. If you do not delete
* the provisions above, a recipient may use your version of this file under
* the terms of any one of the MPL, the GPL or the LGPL.
*
******* END LICENSE BLOCK *******
-Hunspell Version: 1.1.12
+Hunspell Version: 1.2.8
Hunspell Author: László Németh
MySpell Author: Kevin Hendricks & David Einstein
Hunspell is a spell checker and morphological analyser library. Hunspell
is based on OpenOffice.org's Myspell. Documentation, tests, and examples
are available at http://hunspell.sourceforge.net.
--- a/extensions/spellcheck/hunspell/src/affentry.cpp
+++ b/extensions/spellcheck/hunspell/src/affentry.cpp
@@ -55,19 +55,19 @@
******* END LICENSE BLOCK *******/
#ifndef MOZILLA_CLIENT
#include <cstdlib>
#include <cstring>
#include <cctype>
#include <cstdio>
#else
-#include <stdlib.h>
+#include <stdlib.h>
#include <string.h>
-#include <stdio.h>
+#include <stdio.h>
#include <ctype.h>
#endif
#include "affentry.hxx"
#include "csutil.hxx"
#ifndef MOZILLA_CLIENT
#ifndef W32
@@ -77,132 +77,164 @@ using namespace std;
PfxEntry::PfxEntry(AffixMgr* pmgr, affentry* dp)
{
// register affix manager
pmyMgr = pmgr;
// set up its intial values
-
- aflag = dp->aflag; // flag
+
+ aflag = dp->aflag; // flag
strip = dp->strip; // string to strip
appnd = dp->appnd; // string to append
stripl = dp->stripl; // length of strip string
appndl = dp->appndl; // length of append string
- numconds = dp->numconds; // number of conditions to match
- opts = dp->opts; // cross product flag
+ numconds = dp->numconds; // length of the condition
+ opts = dp->opts; // cross product flag
// then copy over all of the conditions
- memcpy(&conds.base[0],&dp->conds.base[0],SETSIZE*sizeof(conds.base[0]));
+ if (opts & aeLONGCOND) {
+ memcpy(c.conds, dp->c.l.conds1, MAXCONDLEN_1);
+ c.l.conds2 = dp->c.l.conds2;
+ } else memcpy(c.conds, dp->c.conds, MAXCONDLEN);
next = NULL;
nextne = NULL;
nexteq = NULL;
-#ifdef HUNSPELL_EXPERIMENTAL
morphcode = dp->morphcode;
-#endif
contclass = dp->contclass;
contclasslen = dp->contclasslen;
}
PfxEntry::~PfxEntry()
{
aflag = 0;
if (appnd) free(appnd);
if (strip) free(strip);
pmyMgr = NULL;
appnd = NULL;
strip = NULL;
- if (opts & aeUTF8) {
- for (int i = 0; i < numconds; i++) {
- if (conds.utf8.wchars[i]) free(conds.utf8.wchars[i]);
- }
- }
-#ifdef HUNSPELL_EXPERIMENTAL
+ if (opts & aeLONGCOND) free(c.l.conds2);
if (morphcode && !(opts & aeALIASM)) free(morphcode);
-#endif
if (contclass && !(opts & aeALIASF)) free(contclass);
}
// add prefix to this word assuming conditions hold
char * PfxEntry::add(const char * word, int len)
{
char tword[MAXWORDUTF8LEN + 4];
- if ((len > stripl) && (len >= numconds) && test_condition(word) &&
- (!stripl || (strncmp(word, strip, stripl) == 0)) &&
+ if ((len > stripl || (len == 0 && pmyMgr->get_fullstrip())) &&
+ (len >= numconds) && test_condition(word) &&
+ (!stripl || (strncmp(word, strip, stripl) == 0)) &&
((MAXWORDUTF8LEN + 4) > (len + appndl - stripl))) {
/* we have a match so add prefix */
char * pp = tword;
if (appndl) {
strcpy(tword,appnd);
pp += appndl;
}
strcpy(pp, (word + stripl));
return mystrdup(tword);
}
- return NULL;
+ return NULL;
}
+inline char * PfxEntry::nextchar(char * p) {
+ if (p) {
+ p++;
+ if (opts & aeLONGCOND) {
+ // jump to the 2nd part of the condition
+ if (p == c.conds + MAXCONDLEN_1) return c.l.conds2;
+ // end of the MAXCONDLEN length condition
+ } else if (p == c.conds + MAXCONDLEN) return NULL;
+ return *p ? p : NULL;
+ }
+ return NULL;
+}
inline int PfxEntry::test_condition(const char * st)
{
- int cond;
- unsigned char * cp = (unsigned char *)st;
- if (!(opts & aeUTF8)) { // 256-character codepage
- for (cond = 0; cond < numconds; cond++) {
- if ((conds.base[*cp++] & (1 << cond)) == 0) return 0;
- }
- } else { // UTF-8 encoding
- unsigned short wc;
- for (cond = 0; cond < numconds; cond++) {
- // a simple 7-bit ASCII character in UTF-8
- if ((*cp >> 7) == 0) {
- // also check limit (end of word)
- if ((!*cp) || ((conds.utf8.ascii[*cp++] & (1 << cond)) == 0)) return 0;
- // UTF-8 multibyte character
- } else {
- // not dot wildcard in rule
- if (!conds.utf8.all[cond]) {
- if (conds.utf8.neg[cond]) {
- u8_u16((w_char *) &wc, 1, (char *) cp);
- if (conds.utf8.wchars[cond] &&
- flag_bsearch((unsigned short *)conds.utf8.wchars[cond],
- wc, (short) conds.utf8.wlen[cond])) return 0;
- } else {
- if (!conds.utf8.wchars[cond]) return 0;
- u8_u16((w_char *) &wc, 1, (char *) cp);
- if (!flag_bsearch((unsigned short *)conds.utf8.wchars[cond],
- wc, (short)conds.utf8.wlen[cond])) return 0;
- }
+ const char * pos = NULL; // group with pos input position
+ bool neg = false; // complementer
+ bool ingroup = false; // character in the group
+ if (numconds == 0) return 1;
+ char * p = c.conds;
+ while (1) {
+ switch (*p) {
+ case '\0': return 1;
+ case '[': {
+ neg = false;
+ ingroup = false;
+ p = nextchar(p);
+ pos = st; break;
+ }
+ case '^': { p = nextchar(p); neg = true; break; }
+ case ']': {
+ if ((neg && ingroup) || (!neg && !ingroup)) return 0;
+ pos = NULL;
+ p = nextchar(p);
+ // skip the next character
+ if (!ingroup) for (st++; (opts & aeUTF8) && (*st & 0xc0) == 0x80; st++);
+ if (*st == '\0' && p) return 0; // word <= condition
+ break;
}
- // jump to next UTF-8 character
- for(cp++; (*cp & 0xc0) == 0x80; cp++);
- }
+ case '.': if (!pos) { // dots are not metacharacters in groups: [.]
+ p = nextchar(p);
+ // skip the next character
+ for (st++; (opts & aeUTF8) && (*st & 0xc0) == 0x80; st++);
+ if (*st == '\0' && p) return 0; // word <= condition
+ break;
+ }
+ default: {
+ if (*st == *p) {
+ st++;
+ p = nextchar(p);
+ if ((opts & aeUTF8) && (*(st - 1) & 0x80)) { // multibyte
+ while (p && (*p & 0xc0) == 0x80) { // character
+ if (*p != *st) {
+ if (!pos) return 0;
+ st = pos;
+ break;
+ }
+ p = nextchar(p);
+ st++;
+ }
+ if (pos && st != pos) {
+ ingroup = true;
+ while (p && *p != ']' && (p = nextchar(p)));
+ }
+ } else if (pos) {
+ ingroup = true;
+ while (p && *p != ']' && (p = nextchar(p)));
+ }
+ } else if (pos) { // group
+ p = nextchar(p);
+ } else return 0;
+ }
}
+ if (!p) return 1;
}
- return 1;
}
-
-// check if this prefix entry matches
+// check if this prefix entry matches
struct hentry * PfxEntry::checkword(const char * word, int len, char in_compound, const FLAG needflag)
{
int tmpl; // length of tmpword
struct hentry * he; // hash entry of root word or NULL
char tmpword[MAXWORDUTF8LEN + 4];
// on entry prefix is 0 length or already matches the beginning of the word.
// So if the remaining root word has positive length
// and if there are enough chars in root word and added back strip chars
// to meet the number of characters conditions, then test it
tmpl = len - appndl;
- if ((tmpl > 0) && (tmpl + stripl >= numconds)) {
+ if (tmpl > 0 || (tmpl == 0 && pmyMgr->get_fullstrip())) {
// generate new root word by removing prefix and adding
// back any characters that would have been stripped
if (stripl) strcpy (tmpword, strip);
strcpy ((tmpword + stripl), (word + appndl));
// now make sure all of the conditions on characters
@@ -213,57 +245,58 @@ struct hentry * PfxEntry::checkword(cons
// if all conditions are met then check if resulting
// root word in the dictionary
if (test_condition(tmpword)) {
tmpl += stripl;
if ((he = pmyMgr->lookup(tmpword)) != NULL) {
do {
if (TESTAFF(he->astr, aflag, he->alen) &&
- // forbid single prefixes with pseudoroot flag
- ! TESTAFF(contclass, pmyMgr->get_pseudoroot(), contclasslen) &&
+ // forbid single prefixes with needaffix flag
+ ! TESTAFF(contclass, pmyMgr->get_needaffix(), contclasslen) &&
// needflag
((!needflag) || TESTAFF(he->astr, needflag, he->alen) ||
(contclass && TESTAFF(contclass, needflag, contclasslen))))
return he;
he = he->next_homonym; // check homonyms
} while (he);
}
-
- // prefix matched but no root word was found
- // if aeXPRODUCT is allowed, try again but now
+
+ // prefix matched but no root word was found
+ // if aeXPRODUCT is allowed, try again but now
// ross checked combined with a suffix
//if ((opts & aeXPRODUCT) && in_compound) {
if ((opts & aeXPRODUCT)) {
- he = pmyMgr->suffix_check(tmpword, tmpl, aeXPRODUCT, (AffEntry *)this, NULL,
+ he = pmyMgr->suffix_check(tmpword, tmpl, aeXPRODUCT, (AffEntry *)this, NULL,
0, NULL, FLAG_NULL, needflag, in_compound);
if (he) return he;
}
}
}
return NULL;
}
-// check if this prefix entry matches
+// check if this prefix entry matches
struct hentry * PfxEntry::check_twosfx(const char * word, int len,
char in_compound, const FLAG needflag)
{
int tmpl; // length of tmpword
struct hentry * he; // hash entry of root word or NULL
char tmpword[MAXWORDUTF8LEN + 4];
// on entry prefix is 0 length or already matches the beginning of the word.
// So if the remaining root word has positive length
// and if there are enough chars in root word and added back strip chars
// to meet the number of characters conditions, then test it
tmpl = len - appndl;
- if ((tmpl > 0) && (tmpl + stripl >= numconds)) {
+ if ((tmpl > 0 || (tmpl == 0 && pmyMgr->get_fullstrip())) &&
+ (tmpl + stripl >= numconds)) {
// generate new root word by removing prefix and adding
// back any characters that would have been stripped
if (stripl) strcpy (tmpword, strip);
strcpy ((tmpword + stripl), (word + appndl));
// now make sure all of the conditions on characters
@@ -272,45 +305,45 @@ struct hentry * PfxEntry::check_twosfx(c
// tested
// if all conditions are met then check if resulting
// root word in the dictionary
if (test_condition(tmpword)) {
tmpl += stripl;
- // prefix matched but no root word was found
- // if aeXPRODUCT is allowed, try again but now
+ // prefix matched but no root word was found
+ // if aeXPRODUCT is allowed, try again but now
// cross checked combined with a suffix
if ((opts & aeXPRODUCT) && (in_compound != IN_CPD_BEGIN)) {
he = pmyMgr->suffix_check_twosfx(tmpword, tmpl, aeXPRODUCT, (AffEntry *)this, needflag);
if (he) return he;
}
}
}
return NULL;
}
-#ifdef HUNSPELL_EXPERIMENTAL
-// check if this prefix entry matches
+// check if this prefix entry matches
char * PfxEntry::check_twosfx_morph(const char * word, int len,
char in_compound, const FLAG needflag)
{
int tmpl; // length of tmpword
char tmpword[MAXWORDUTF8LEN + 4];
// on entry prefix is 0 length or already matches the beginning of the word.
// So if the remaining root word has positive length
// and if there are enough chars in root word and added back strip chars
// to meet the number of characters conditions, then test it
tmpl = len - appndl;
- if ((tmpl > 0) && (tmpl + stripl >= numconds)) {
+ if ((tmpl > 0 || (tmpl == 0 && pmyMgr->get_fullstrip())) &&
+ (tmpl + stripl >= numconds)) {
// generate new root word by removing prefix and adding
// back any characters that would have been stripped
if (stripl) strcpy (tmpword, strip);
strcpy ((tmpword + stripl), (word + appndl));
// now make sure all of the conditions on characters
@@ -319,48 +352,49 @@ char * PfxEntry::check_twosfx_morph(cons
// tested
// if all conditions are met then check if resulting
// root word in the dictionary
if (test_condition(tmpword)) {
tmpl += stripl;
- // prefix matched but no root word was found
- // if aeXPRODUCT is allowed, try again but now
+ // prefix matched but no root word was found
+ // if aeXPRODUCT is allowed, try again but now
// ross checked combined with a suffix
if ((opts & aeXPRODUCT) && (in_compound != IN_CPD_BEGIN)) {
return pmyMgr->suffix_check_twosfx_morph(tmpword, tmpl,
aeXPRODUCT, (AffEntry *)this, needflag);
}
}
}
return NULL;
}
-// check if this prefix entry matches
+// check if this prefix entry matches
char * PfxEntry::check_morph(const char * word, int len, char in_compound, const FLAG needflag)
{
int tmpl; // length of tmpword
struct hentry * he; // hash entry of root word or NULL
char tmpword[MAXWORDUTF8LEN + 4];
char result[MAXLNLEN];
char * st;
-
+
*result = '\0';
// on entry prefix is 0 length or already matches the beginning of the word.
// So if the remaining root word has positive length
// and if there are enough chars in root word and added back strip chars
// to meet the number of characters conditions, then test it
tmpl = len - appndl;
- if ((tmpl > 0) && (tmpl + stripl >= numconds)) {
+ if ((tmpl > 0 || (tmpl == 0 && pmyMgr->get_fullstrip())) &&
+ (tmpl + stripl >= numconds)) {
// generate new root word by removing prefix and adding
// back any characters that would have been stripped
if (stripl) strcpy (tmpword, strip);
strcpy ((tmpword + stripl), (word + appndl));
// now make sure all of the conditions on characters
@@ -371,170 +405,238 @@ char * PfxEntry::check_morph(const char
// if all conditions are met then check if resulting
// root word in the dictionary
if (test_condition(tmpword)) {
tmpl += stripl;
if ((he = pmyMgr->lookup(tmpword)) != NULL) {
do {
if (TESTAFF(he->astr, aflag, he->alen) &&
- // forbid single prefixes with pseudoroot flag
- ! TESTAFF(contclass, pmyMgr->get_pseudoroot(), contclasslen) &&
+ // forbid single prefixes with needaffix flag
+ ! TESTAFF(contclass, pmyMgr->get_needaffix(), contclasslen) &&
// needflag
((!needflag) || TESTAFF(he->astr, needflag, he->alen) ||
(contclass && TESTAFF(contclass, needflag, contclasslen)))) {
- if (morphcode) strcat(result, morphcode); else strcat(result,getKey());
- if (he->description) {
- if ((*(he->description)=='[')||(*(he->description)=='<')) strcat(result,he->word);
- strcat(result,he->description);
+ if (morphcode) {
+ mystrcat(result, " ", MAXLNLEN);
+ mystrcat(result, morphcode, MAXLNLEN);
+ } else mystrcat(result,getKey(), MAXLNLEN);
+ if (!HENTRY_FIND(he, MORPH_STEM)) {
+ mystrcat(result, " ", MAXLNLEN);
+ mystrcat(result, MORPH_STEM, MAXLNLEN);
+ mystrcat(result, HENTRY_WORD(he), MAXLNLEN);
}
- strcat(result, "\n");
+ // store the pointer of the hash entry
+ if (HENTRY_DATA(he)) {
+ mystrcat(result, " ", MAXLNLEN);
+ mystrcat(result, HENTRY_DATA2(he), MAXLNLEN);
+ } else {
+ // return with debug information
+ char * flag = pmyMgr->encode_flag(getFlag());
+ mystrcat(result, " ", MAXLNLEN);
+ mystrcat(result, MORPH_FLAG, MAXLNLEN);
+ mystrcat(result, flag, MAXLNLEN);
+ free(flag);
+ }
+ mystrcat(result, "\n", MAXLNLEN);
}
he = he->next_homonym;
} while (he);
}
- // prefix matched but no root word was found
- // if aeXPRODUCT is allowed, try again but now
+ // prefix matched but no root word was found
+ // if aeXPRODUCT is allowed, try again but now
// ross checked combined with a suffix
if ((opts & aeXPRODUCT) && (in_compound != IN_CPD_BEGIN)) {
- st = pmyMgr->suffix_check_morph(tmpword, tmpl, aeXPRODUCT, (AffEntry *)this,
+ st = pmyMgr->suffix_check_morph(tmpword, tmpl, aeXPRODUCT, (AffEntry *)this,
FLAG_NULL, needflag);
if (st) {
- strcat(result, st);
+ mystrcat(result, st, MAXLNLEN);
free(st);
}
}
}
}
-
+
if (*result) return mystrdup(result);
return NULL;
}
-#endif // END OF HUNSPELL_EXPERIMENTAL CODE
SfxEntry::SfxEntry(AffixMgr * pmgr, affentry* dp)
{
// register affix manager
pmyMgr = pmgr;
// set up its intial values
- aflag = dp->aflag; // char flag
+ aflag = dp->aflag; // char flag
strip = dp->strip; // string to strip
appnd = dp->appnd; // string to append
stripl = dp->stripl; // length of strip string
appndl = dp->appndl; // length of append string
- numconds = dp->numconds; // number of conditions to match
- opts = dp->opts; // cross product flag
+ numconds = dp->numconds; // length of the condition
+ opts = dp->opts; // cross product flag
// then copy over all of the conditions
- memcpy(&conds.base[0],&dp->conds.base[0],SETSIZE*sizeof(conds.base[0]));
+ if (opts & aeLONGCOND) {
+ memcpy(c.l.conds1, dp->c.l.conds1, MAXCONDLEN_1);
+ c.l.conds2 = dp->c.l.conds2;
+ } else memcpy(c.conds, dp->c.conds, MAXCONDLEN);
rappnd = myrevstrdup(appnd);
-
-#ifdef HUNSPELL_EXPERIMENTAL
morphcode = dp->morphcode;
-#endif
contclass = dp->contclass;
contclasslen = dp->contclasslen;
}
SfxEntry::~SfxEntry()
{
aflag = 0;
if (appnd) free(appnd);
if (rappnd) free(rappnd);
if (strip) free(strip);
pmyMgr = NULL;
appnd = NULL;
- strip = NULL;
- if (opts & aeUTF8) {
- for (int i = 0; i < numconds; i++) {
- if (conds.utf8.wchars[i]) free(conds.utf8.wchars[i]);
- }
- }
-#ifdef HUNSPELL_EXPERIMENTAL
+ strip = NULL;
+ if (opts & aeLONGCOND) free(c.l.conds2);
if (morphcode && !(opts & aeALIASM)) free(morphcode);
-#endif
if (contclass && !(opts & aeALIASF)) free(contclass);
}
// add suffix to this word assuming conditions hold
char * SfxEntry::add(const char * word, int len)
{
char tword[MAXWORDUTF8LEN + 4];
/* make sure all conditions match */
- if ((len > stripl) && (len >= numconds) && test_condition(word + len, word) &&
+ if ((len > stripl || (len == 0 && pmyMgr->get_fullstrip())) &&
+ (len >= numconds) && test_condition(word + len, word) &&
(!stripl || (strcmp(word + len - stripl, strip) == 0)) &&
((MAXWORDUTF8LEN + 4) > (len + appndl - stripl))) {
/* we have a match so add suffix */
strcpy(tword,word);
if (appndl) {
strcpy(tword + len - stripl, appnd);
} else {
*(tword + len - stripl) = '\0';
}
return mystrdup(tword);
}
return NULL;
}
+inline char * SfxEntry::nextchar(char * p) {
+ if (p) {
+ p++;
+ if (opts & aeLONGCOND) {
+ // jump to the 2nd part of the condition
+ if (p == c.l.conds1 + MAXCONDLEN_1) return c.l.conds2;
+ // end of the MAXCONDLEN length condition
+ } else if (p == c.conds + MAXCONDLEN) return NULL;
+ return *p ? p : NULL;
+ }
+ return NULL;
+}
inline int SfxEntry::test_condition(const char * st, const char * beg)
{
- int cond;
- unsigned char * cp = (unsigned char *) st;
- if (!(opts & aeUTF8)) { // 256-character codepage
- // Domolki affix algorithm
- for (cond = numconds; --cond >= 0; ) {
- if ((conds.base[*--cp] & (1 << cond)) == 0) return 0;
- }
- } else { // UTF-8 encoding
- unsigned short wc;
- for (cond = numconds; --cond >= 0; ) {
- // go to next character position and check limit
- if ((char *) --cp < beg) return 0;
- // a simple 7-bit ASCII character in UTF-8
- if ((*cp >> 7) == 0) {
- if ((conds.utf8.ascii[*cp] & (1 << cond)) == 0) return 0;
- // UTF-8 multibyte character
- } else {
- // go to first character of UTF-8 multibyte character
- for (; (*cp & 0xc0) == 0x80; cp--);
- // not dot wildcard in rule
- if (!conds.utf8.all[cond]) {
- if (conds.utf8.neg[cond]) {
- u8_u16((w_char *) &wc, 1, (char *) cp);
- if (conds.utf8.wchars[cond] &&
- flag_bsearch((unsigned short *)conds.utf8.wchars[cond],
- wc, (short) conds.utf8.wlen[cond])) return 0;
- } else {
- if (!conds.utf8.wchars[cond]) return 0;
- u8_u16((w_char *) &wc, 1, (char *) cp);
- if (!flag_bsearch((unsigned short *)conds.utf8.wchars[cond],
- wc, (short)conds.utf8.wlen[cond])) return 0;
+ const char * pos = NULL; // group with pos input position
+ bool neg = false; // complementer
+ bool ingroup = false; // character in the group
+ if (numconds == 0) return 1;
+ char * p = c.conds;
+ st--;
+ int i = 1;
+ while (1) {
+ switch (*p) {
+ case '\0': return 1;
+ case '[': { p = nextchar(p); pos = st; break; }
+ case '^': { p = nextchar(p); neg = true; break; }
+ case ']': { if (!neg && !ingroup) return 0;
+ i++;
+ // skip the next character
+ if (!ingroup) {
+ for (; (opts & aeUTF8) && (st >= beg) && (*st & 0xc0) == 0x80; st--);
+ st--;
+ }
+ pos = NULL;
+ neg = false;
+ ingroup = false;
+ p = nextchar(p);
+ if (st < beg && p) return 0; // word <= condition
+ break;
+ }
+ case '.': if (!pos) { // dots are not metacharacters in groups: [.]
+ p = nextchar(p);
+ // skip the next character
+ for (st--; (opts & aeUTF8) && (st >= beg) && (*st & 0xc0) == 0x80; st--);
+ if (st < beg) { // word <= condition
+ if (p) return 0; else return 1;
+ }
+ if ((opts & aeUTF8) && (*st & 0x80)) { // head of the UTF-8 character
+ st--;
+ if (st < beg) { // word <= condition
+ if (p) return 0; else return 1;
+ }
}
+ break;
}
- }
+ default: {
+ if (*st == *p) {
+ p = nextchar(p);
+ if ((opts & aeUTF8) && (*st & 0x80)) {
+ st--;
+ while (p && (st >= beg)) {
+ if (*p != *st) {
+ if (!pos) return 0;
+ st = pos;
+ break;
+ }
+ // first byte of the UTF-8 multibyte character
+ if ((*p & 0xc0) != 0x80) break;
+ p = nextchar(p);
+ st--;
+ }
+ if (pos && st != pos) {
+ if (neg) return 0;
+ else if (i == numconds) return 1;
+ ingroup = true;
+ while (p && *p != ']' && (p = nextchar(p)));
+ st--;
+ }
+ if (p && *p != ']') p = nextchar(p);
+ } else if (pos) {
+ if (neg) return 0;
+ else if (i == numconds) return 1;
+ ingroup = true;
+ while (p && *p != ']' && (p = nextchar(p)));
+// if (p && *p != ']') p = nextchar(p);
+ st--;
+ }
+ if (!pos) {
+ i++;
+ st--;
+ }
+ if (st < beg && p && *p != ']') return 0; // word <= condition
+ } else if (pos) { // group
+ p = nextchar(p);
+ } else return 0;
+ }
}
+ if (!p) return 1;
}
- return 1;
}
-
-
-// see if this suffix is present in the word
+// see if this suffix is present in the word
struct hentry * SfxEntry::checkword(const char * word, int len, int optflags,
AffEntry* ppfx, char ** wlst, int maxSug, int * ns, const FLAG cclass, const FLAG needflag,
const FLAG badflag)
{
- int tmpl; // length of tmpword
+ int tmpl; // length of tmpword
struct hentry * he; // hash entry pointer
unsigned char * cp;
char tmpword[MAXWORDUTF8LEN + 4];
PfxEntry* ep = (PfxEntry *) ppfx;
// if this suffix is being cross checked with a prefix
// but it does not support cross products skip it
@@ -544,96 +646,98 @@ struct hentry * SfxEntry::checkword(cons
// upon entry suffix is 0 length or already matches the end of the word.
// So if the remaining root word has positive length
// and if there are enough chars in root word and added back strip chars
// to meet the number of characters conditions, then test it
tmpl = len - appndl;
// the second condition is not enough for UTF-8 strings
// it checked in test_condition()
-
- if ((tmpl > 0) && (tmpl + stripl >= numconds)) {
+
+ if ((tmpl > 0 || (tmpl == 0 && pmyMgr->get_fullstrip())) &&
+ (tmpl + stripl >= numconds)) {
// generate new root word by removing suffix and adding
// back any characters that would have been stripped or
// or null terminating the shorter string
strcpy (tmpword, word);
cp = (unsigned char *)(tmpword + tmpl);
if (stripl) {
strcpy ((char *)cp, strip);
tmpl += stripl;
cp = (unsigned char *)(tmpword + tmpl);
} else *cp = '\0';
// now make sure all of the conditions on characters
// are met. Please see the appendix at the end of
- // this file for more info on exactly what is being // tested
+ // this file for more info on exactly what is being
+ // tested
// if all conditions are met then check if resulting
// root word in the dictionary
if (test_condition((char *) cp, (char *) tmpword)) {
#ifdef SZOSZABLYA_POSSIBLE_ROOTS
fprintf(stdout,"%s %s %c\n", word, tmpword, aflag);
#endif
if ((he = pmyMgr->lookup(tmpword)) != NULL) {
do {
// check conditional suffix (enabled by prefix)
if ((TESTAFF(he->astr, aflag, he->alen) || (ep && ep->getCont() &&
- TESTAFF(ep->getCont(), aflag, ep->getContLen()))) &&
- (((optflags & aeXPRODUCT) == 0) ||
+ TESTAFF(ep->getCont(), aflag, ep->getContLen()))) &&
+ (((optflags & aeXPRODUCT) == 0) ||
TESTAFF(he->astr, ep->getFlag(), he->alen) ||
// enabled by prefix
((contclass) && TESTAFF(contclass, ep->getFlag(), contclasslen))
) &&
// handle cont. class
- ((!cclass) ||
+ ((!cclass) ||
((contclass) && TESTAFF(contclass, cclass, contclasslen))
) &&
// check only in compound homonyms (bad flags)
(!badflag || !TESTAFF(he->astr, badflag, he->alen)
- ) &&
+ ) &&
// handle required flag
- ((!needflag) ||
+ ((!needflag) ||
(TESTAFF(he->astr, needflag, he->alen) ||
((contclass) && TESTAFF(contclass, needflag, contclasslen)))
)
) return he;
he = he->next_homonym; // check homonyms
} while (he);
- // obsolote stemming code (used only by the
+ // obsolote stemming code (used only by the
// experimental SuffixMgr:suggest_pos_stems)
// store resulting root in wlst
} else if (wlst && (*ns < maxSug)) {
int cwrd = 1;
- for (int k=0; k < *ns; k++)
+ for (int k=0; k < *ns; k++)
if (strcmp(tmpword, wlst[k]) == 0) cwrd = 0;
if (cwrd) {
wlst[*ns] = mystrdup(tmpword);
if (wlst[*ns] == NULL) {
for (int j=0; j<*ns; j++) free(wlst[j]);
*ns = -1;
return NULL;
}
(*ns)++;
}
}
}
}
return NULL;
}
-// see if two-level suffix is present in the word
+// see if two-level suffix is present in the word
struct hentry * SfxEntry::check_twosfx(const char * word, int len, int optflags,
AffEntry* ppfx, const FLAG needflag)
{
- int tmpl; // length of tmpword
+ int tmpl; // length of tmpword
struct hentry * he; // hash entry pointer
unsigned char * cp;
char tmpword[MAXWORDUTF8LEN + 4];
PfxEntry* ep = (PfxEntry *) ppfx;
// if this suffix is being cross checked with a prefix
// but it does not support cross products skip it
@@ -643,17 +747,18 @@ struct hentry * SfxEntry::check_twosfx(c
// upon entry suffix is 0 length or already matches the end of the word.
// So if the remaining root word has positive length
// and if there are enough chars in root word and added back strip chars
// to meet the number of characters conditions, then test it
tmpl = len - appndl;
- if ((tmpl > 0) && (tmpl + stripl >= numconds)) {
+ if ((tmpl > 0 || (tmpl == 0 && pmyMgr->get_fullstrip())) &&
+ (tmpl + stripl >= numconds)) {
// generate new root word by removing suffix and adding
// back any characters that would have been stripped or
// or null terminating the shorter string
strcpy (tmpword, word);
cp = (unsigned char *)(tmpword + tmpl);
if (stripl) {
@@ -667,58 +772,58 @@ struct hentry * SfxEntry::check_twosfx(c
// this file for more info on exactly what is being
// tested
// if all conditions are met then recall suffix_check
if (test_condition((char *) cp, (char *) tmpword)) {
if (ppfx) {
// handle conditional suffix
- if ((contclass) && TESTAFF(contclass, ep->getFlag(), contclasslen))
+ if ((contclass) && TESTAFF(contclass, ep->getFlag(), contclasslen))
he = pmyMgr->suffix_check(tmpword, tmpl, 0, NULL, NULL, 0, NULL, (FLAG) aflag, needflag);
else
he = pmyMgr->suffix_check(tmpword, tmpl, optflags, ppfx, NULL, 0, NULL, (FLAG) aflag, needflag);
} else {
he = pmyMgr->suffix_check(tmpword, tmpl, 0, NULL, NULL, 0, NULL, (FLAG) aflag, needflag);
}
if (he) return he;
}
}
return NULL;
}
-#ifdef HUNSPELL_EXPERIMENTAL
-// see if two-level suffix is present in the word
+// see if two-level suffix is present in the word
char * SfxEntry::check_twosfx_morph(const char * word, int len, int optflags,
AffEntry* ppfx, const FLAG needflag)
{
- int tmpl; // length of tmpword
+ int tmpl; // length of tmpword
unsigned char * cp;
char tmpword[MAXWORDUTF8LEN + 4];
PfxEntry* ep = (PfxEntry *) ppfx;
char * st;
char result[MAXLNLEN];
-
+
*result = '\0';
// if this suffix is being cross checked with a prefix
// but it does not support cross products skip it
if ((optflags & aeXPRODUCT) != 0 && (opts & aeXPRODUCT) == 0)
return NULL;
// upon entry suffix is 0 length or already matches the end of the word.
// So if the remaining root word has positive length
// and if there are enough chars in root word and added back strip chars
// to meet the number of characters conditions, then test it
tmpl = len - appndl;
- if ((tmpl > 0) && (tmpl + stripl >= numconds)) {
+ if ((tmpl > 0 || (tmpl == 0 && pmyMgr->get_fullstrip())) &&
+ (tmpl + stripl >= numconds)) {
// generate new root word by removing suffix and adding
// back any characters that would have been stripped or
// or null terminating the shorter string
strcpy (tmpword, word);
cp = (unsigned char *)(tmpword + tmpl);
if (stripl) {
@@ -736,66 +841,66 @@ char * SfxEntry::check_twosfx_morph(cons
if (test_condition((char *) cp, (char *) tmpword)) {
if (ppfx) {
// handle conditional suffix
if ((contclass) && TESTAFF(contclass, ep->getFlag(), contclasslen)) {
st = pmyMgr->suffix_check_morph(tmpword, tmpl, 0, NULL, aflag, needflag);
if (st) {
if (((PfxEntry *) ppfx)->getMorph()) {
- strcat(result, ((PfxEntry *) ppfx)->getMorph());
+ mystrcat(result, ((PfxEntry *) ppfx)->getMorph(), MAXLNLEN);
+ mystrcat(result, " ", MAXLNLEN);
}
- strcat(result,st);
+ mystrcat(result,st, MAXLNLEN);
free(st);
mychomp(result);
}
} else {
st = pmyMgr->suffix_check_morph(tmpword, tmpl, optflags, ppfx, aflag, needflag);
if (st) {
- strcat(result, st);
+ mystrcat(result, st, MAXLNLEN);
free(st);
mychomp(result);
}
}
} else {
st = pmyMgr->suffix_check_morph(tmpword, tmpl, 0, NULL, aflag, needflag);
if (st) {
- strcat(result, st);
+ mystrcat(result, st, MAXLNLEN);
free(st);
mychomp(result);
}
}
if (*result) return mystrdup(result);
}
}
return NULL;
}
-#endif // END OF HUNSPELL_EXPERIMENTAL CODE
// get next homonym with same affix
-struct hentry * SfxEntry::get_next_homonym(struct hentry * he, int optflags, AffEntry* ppfx,
+struct hentry * SfxEntry::get_next_homonym(struct hentry * he, int optflags, AffEntry* ppfx,
const FLAG cclass, const FLAG needflag)
{
PfxEntry* ep = (PfxEntry *) ppfx;
FLAG eFlag = ep ? ep->getFlag() : FLAG_NULL;
while (he->next_homonym) {
he = he->next_homonym;
- if ((TESTAFF(he->astr, aflag, he->alen) || (ep && ep->getCont() && TESTAFF(ep->getCont(), aflag, ep->getContLen()))) &&
- ((optflags & aeXPRODUCT) == 0 ||
+ if ((TESTAFF(he->astr, aflag, he->alen) || (ep && ep->getCont() && TESTAFF(ep->getCont(), aflag, ep->getContLen()))) &&
+ ((optflags & aeXPRODUCT) == 0 ||
TESTAFF(he->astr, eFlag, he->alen) ||
// handle conditional suffix
((contclass) && TESTAFF(contclass, eFlag, contclasslen))
) &&
// handle cont. class
- ((!cclass) ||
+ ((!cclass) ||
((contclass) && TESTAFF(contclass, cclass, contclasslen))
) &&
// handle required flag
- ((!needflag) ||
+ ((!needflag) ||
(TESTAFF(he->astr, needflag, he->alen) ||
((contclass) && TESTAFF(contclass, needflag, contclasslen)))
)
) return he;
}
return NULL;
}
@@ -916,9 +1021,8 @@ first two affentries for the suffix D de
where X is all characters *but* a, e, i, o, or u
conds['y'] = (1 << 1) (the last char must be a y)
all other bits for all other entries in the conds array are zero
#endif
-
--- a/extensions/spellcheck/hunspell/src/affentry.hxx
+++ b/extensions/spellcheck/hunspell/src/affentry.hxx
@@ -52,19 +52,19 @@
* the provisions above, a recipient may use your version of this file under
* the terms of any one of the MPL, the GPL or the LGPL.
*
******* END LICENSE BLOCK *******/
#ifndef _AFFIX_HXX_
#define _AFFIX_HXX_
+#include "affixmgr.hxx"
#include "atypes.hxx"
#include "baseaffix.hxx"
-#include "affixmgr.hxx"
/* A Prefix Entry */
class PfxEntry : public AffEntry
{
AffixMgr* pmyMgr;
PfxEntry * next;
@@ -105,16 +105,17 @@ public:
inline PfxEntry * getNextEQ() { return nexteq; }
inline PfxEntry * getFlgNxt() { return flgnxt; }
inline void setNext(PfxEntry * ptr) { next = ptr; }
inline void setNextNE(PfxEntry * ptr) { nextne = ptr; }
inline void setNextEQ(PfxEntry * ptr) { nexteq = ptr; }
inline void setFlgNxt(PfxEntry * ptr) { flgnxt = ptr; }
+ inline char * nextchar(char * p);
inline int test_condition(const char * st);
};
/* A Suffix Entry */
@@ -174,14 +175,14 @@ public:
inline SfxEntry * getEQM() { return eq_morph; }
inline SfxEntry * getFlgNxt() { return flgnxt; }
inline void setNext(SfxEntry * ptr) { next = ptr; }
inline void setNextNE(SfxEntry * ptr) { nextne = ptr; }
inline void setNextEQ(SfxEntry * ptr) { nexteq = ptr; }
inline void setFlgNxt(SfxEntry * ptr) { flgnxt = ptr; }
+ inline char * nextchar(char * p);
inline int test_condition(const char * st, const char * begin);
+
};
#endif
-
-
--- a/extensions/spellcheck/hunspell/src/affixmgr.cpp
+++ b/extensions/spellcheck/hunspell/src/affixmgr.cpp
@@ -55,70 +55,76 @@
******* END LICENSE BLOCK *******/
#ifndef MOZILLA_CLIENT
#include <cstdlib>
#include <cstring>
#include <cctype>
#include <cstdio>
#else
-#include <stdlib.h>
+#include <stdlib.h>
#include <string.h>
-#include <stdio.h>
+#include <stdio.h>
#include <ctype.h>
#endif
+#include "affentry.hxx"
#include "affixmgr.hxx"
-#include "affentry.hxx"
+#include "csutil.hxx"
#include "langnum.hxx"
-#include "csutil.hxx"
-
#ifndef MOZILLA_CLIENT
#ifndef W32
using namespace std;
#endif
#endif
-AffixMgr::AffixMgr(const char * affpath, HashMgr* ptr)
+AffixMgr::AffixMgr(const char * affpath, HashMgr** ptr, int * md, const char * key)
{
// register hash manager and load affix data from aff file
- pHMgr = ptr;
+ pHMgr = ptr[0];
+ alldic = ptr;
+ maxdic = md;
keystring = NULL;
trystring = NULL;
encoding=NULL;
utf8 = 0;
complexprefixes = 0;
maptable = NULL;
nummap = 0;
breaktable = NULL;
numbreak = 0;
reptable = NULL;
numrep = 0;
+ iconvtable = NULL;
+ oconvtable = NULL;
checkcpdtable = NULL;
+ // allow simplified compound forms (see 3rd field of CHECKCOMPOUNDPATTERN)
+ simplifiedcpd = 0;
numcheckcpd = 0;
defcpdtable = NULL;
numdefcpd = 0;
phone = NULL;
compoundflag = FLAG_NULL; // permits word in compound forms
compoundbegin = FLAG_NULL; // may be first word in compound forms
compoundmiddle = FLAG_NULL; // may be middle word in compound forms
compoundend = FLAG_NULL; // may be last word in compound forms
compoundroot = FLAG_NULL; // compound word signing flag
compoundpermitflag = FLAG_NULL; // compound permitting flag for suffixed word
compoundforbidflag = FLAG_NULL; // compound fordidden flag for suffixed word
checkcompounddup = 0; // forbid double words in compounds
checkcompoundrep = 0; // forbid bad compounds (may be non compound word with a REP substitution)
checkcompoundcase = 0; // forbid upper and lowercase combinations at word bounds
checkcompoundtriple = 0; // forbid compounds with triple letters
- forbiddenword = FLAG_NULL; // forbidden word signing flag
+ simplifiedtriple = 0; // allow simplified triple letters in compounds (Schiff+fahrt -> Schiffahrt)
+ forbiddenword = FORBIDDENWORD; // forbidden word signing flag
nosuggest = FLAG_NULL; // don't suggest words signed with NOSUGGEST flag
lang = NULL; // language
langnum = 0; // language code (see http://l10n.openoffice.org/languages.html)
- pseudoroot = FLAG_NULL; // forbidden root, allowed only with suffixes
+ needaffix = FLAG_NULL; // forbidden root, allowed only with suffixes
cpdwordmax = -1; // default: unlimited wordcount in compound words
cpdmin = -1; // undefined
cpdmaxsyllable = 0; // default: unlimited syllablecount in compound words
cpdvowels=NULL; // vowels (for calculating of Hungarian compounding limit, O(n) search! XXX)
cpdvowels_utf16=NULL; // vowels for UTF-8 encoding (bsearch instead of O(n) search)
cpdvowels_utf16_len=0; // vowels
pfxappnd=NULL; // previous prefix for counting the syllables of prefix BUG
sfxappnd=NULL; // previous suffix for counting a special syllables BUG
@@ -132,39 +138,39 @@ AffixMgr::AffixMgr(const char * affpath,
ignorechars_utf16_len=0; // letters + spec. word characters
version=NULL; // affix and dictionary file version string
havecontclass=0; // flags of possible continuing classes (double affix)
// LEMMA_PRESENT: not put root into the morphological output. Lemma presents
// in morhological description in dictionary file. It's often combined with PSEUDOROOT.
lemma_present = FLAG_NULL;
circumfix = FLAG_NULL;
onlyincompound = FLAG_NULL;
- flag_mode = FLAG_CHAR; // default one-character flags in affix and dic file
maxngramsugs = -1; // undefined
nosplitsugs = 0;
sugswithdots = 0;
keepcase = 0;
checksharps = 0;
-
- derived = NULL; // XXX not threadsafe variable for experimental stemming
+ substandard = FLAG_NULL;
+ fullstrip = 0;
+
sfx = NULL;
pfx = NULL;
for (int i=0; i < SETSIZE; i++) {
pStart[i] = NULL;
sStart[i] = NULL;
pFlag[i] = NULL;
sFlag[i] = NULL;
}
for (int j=0; j < CONTSIZE; j++) {
contclasses[j] = 0;
}
- if (parse_file(affpath)) {
+ if (parse_file(affpath, key)) {
HUNSPELL_WARNING(stderr, "Failure loading aff file %s\n",affpath);
}
if (cpdmin == -1) cpdmin = MINCPDLEN;
}
@@ -227,16 +233,18 @@ AffixMgr::~AffixMgr()
if (reptable) {
for (int j=0; j < numrep; j++) {
free(reptable[j].pattern);
free(reptable[j].pattern2);
}
free(reptable);
reptable = NULL;
}
+ if (iconvtable) delete iconvtable;
+ if (oconvtable) delete oconvtable;
if (phone && phone->rules) {
for (int j=0; j < phone->num + 1; j++) {
free(phone->rules[j * 2]);
free(phone->rules[j * 2 + 1]);
}
free(phone->rules);
free(phone);
phone = NULL;
@@ -250,33 +258,35 @@ AffixMgr::~AffixMgr()
free(defcpdtable);
defcpdtable = NULL;
}
numrep = 0;
if (checkcpdtable) {
for (int j=0; j < numcheckcpd; j++) {
free(checkcpdtable[j].pattern);
free(checkcpdtable[j].pattern2);
+ free(checkcpdtable[j].pattern3);
checkcpdtable[j].pattern = NULL;
checkcpdtable[j].pattern2 = NULL;
+ checkcpdtable[j].pattern3 = NULL;
}
free(checkcpdtable);
checkcpdtable = NULL;
}
numcheckcpd = 0;
FREE_FLAG(compoundflag);
FREE_FLAG(compoundbegin);
FREE_FLAG(compoundmiddle);
FREE_FLAG(compoundend);
FREE_FLAG(compoundpermitflag);
FREE_FLAG(compoundforbidflag);
FREE_FLAG(compoundroot);
FREE_FLAG(forbiddenword);
FREE_FLAG(nosuggest);
- FREE_FLAG(pseudoroot);
+ FREE_FLAG(needaffix);
FREE_FLAG(lemma_present);
FREE_FLAG(circumfix);
FREE_FLAG(onlyincompound);
cpdwordmax = 0;
pHMgr = NULL;
cpdmin = 0;
cpdmaxsyllable = 0;
@@ -285,84 +295,77 @@ AffixMgr::~AffixMgr()
if (cpdsyllablenum) free(cpdsyllablenum);
free_utf_tbl();
if (lang) free(lang);
if (wordchars) free(wordchars);
if (wordchars_utf16) free(wordchars_utf16);
if (ignorechars) free(ignorechars);
if (ignorechars_utf16) free(ignorechars_utf16);
if (version) free(version);
- if (derived) free(derived);
checknum=0;
}
// read in aff file and build up prefix and suffix entry objects
-int AffixMgr::parse_file(const char * affpath)
+int AffixMgr::parse_file(const char * affpath, const char * key)
{
-
- // io buffers
- char line[MAXLNLEN+1];
-
- // affix type
- char ft;
+ char * line; // io buffers
+ char ft; // affix type
// checking flag duplication
char dupflags[CONTSIZE];
char dupflags_ini = 1;
// first line indicator for removing byte order mark
int firstline = 1;
// open the affix file
- FILE * afflst;
- afflst = fopen(affpath,"r");
+ FileMgr * afflst = new FileMgr(affpath, key);
if (!afflst) {
HUNSPELL_WARNING(stderr, "error: could not open affix description file %s\n",affpath);
return 1;
}
// step one is to parse the affix file building up the internal
// affix data structures
-
// read in each line ignoring any that do not
// start with a known line type indicator
- while (fgets(line,MAXLNLEN,afflst)) {
+ while ((line = afflst->getline())) {
mychomp(line);
/* remove byte order mark */
if (firstline) {
firstline = 0;
if (strncmp(line,"\xEF\xBB\xBF",3) == 0) {
memmove(line, line+3, strlen(line+3)+1);
HUNSPELL_WARNING(stderr, "warning: affix file begins with byte order mark: possible incompatibility with old Hunspell versions\n");
}
}
/* parse in the keyboard string */
if (strncmp(line,"KEY",3) == 0) {
- if (parse_string(line, &keystring, "KEY")) {
- fclose(afflst);
+ if (parse_string(line, &keystring, afflst->getlinenum())) {
+ delete afflst;
return 1;
}
}
/* parse in the try string */
if (strncmp(line,"TRY",3) == 0) {
- if (parse_string(line, &trystring, "TRY")) {
- fclose(afflst);
+ if (parse_string(line, &trystring, afflst->getlinenum())) {
+ delete afflst;
return 1;
}
}
/* parse in the name of the character set used by the .dict and .aff */
if (strncmp(line,"SET",3) == 0) {
- if (parse_string(line, &encoding, "SET")) {
- fclose(afflst);
+ if (parse_string(line, &encoding, afflst->getlinenum())) {
+ delete afflst;
return 1;
}
if (strcmp(encoding, "UTF-8") == 0) {
utf8 = 1;
#ifndef OPENOFFICEORG
#ifndef MOZILLA_CLIENT
if (initialize_utf_tbl()) return 1;
#endif
@@ -371,291 +374,321 @@ int AffixMgr::parse_file(const char * a
}
/* parse COMPLEXPREFIXES for agglutinative languages with right-to-left writing system */
if (strncmp(line,"COMPLEXPREFIXES",15) == 0)
complexprefixes = 1;
/* parse in the flag used by the controlled compound words */
if (strncmp(line,"COMPOUNDFLAG",12) == 0) {
- if (parse_flag(line, &compoundflag, "COMPOUNDFLAG")) {
- fclose(afflst);
+ if (parse_flag(line, &compoundflag, afflst)) {
+ delete afflst;
return 1;
}
}
/* parse in the flag used by compound words */
if (strncmp(line,"COMPOUNDBEGIN",13) == 0) {
if (complexprefixes) {
- if (parse_flag(line, &compoundend, "COMPOUNDBEGIN")) {
- fclose(afflst);
+ if (parse_flag(line, &compoundend, afflst)) {
+ delete afflst;
return 1;
}
} else {
- if (parse_flag(line, &compoundbegin, "COMPOUNDBEGIN")) {
- fclose(afflst);
+ if (parse_flag(line, &compoundbegin, afflst)) {
+ delete afflst;
return 1;
}
}
}
/* parse in the flag used by compound words */
if (strncmp(line,"COMPOUNDMIDDLE",14) == 0) {
- if (parse_flag(line, &compoundmiddle, "COMPOUNDMIDDLE")) {
- fclose(afflst);
+ if (parse_flag(line, &compoundmiddle, afflst)) {
+ delete afflst;
return 1;
}
}
/* parse in the flag used by compound words */
if (strncmp(line,"COMPOUNDEND",11) == 0) {
if (complexprefixes) {
- if (parse_flag(line, &compoundbegin, "COMPOUNDEND")) {
- fclose(afflst);
+ if (parse_flag(line, &compoundbegin, afflst)) {
+ delete afflst;
return 1;
}
} else {
- if (parse_flag(line, &compoundend, "COMPOUNDEND")) {
- fclose(afflst);
+ if (parse_flag(line, &compoundend, afflst)) {
+ delete afflst;
return 1;
}
}
}
/* parse in the data used by compound_check() method */
if (strncmp(line,"COMPOUNDWORDMAX",15) == 0) {
- if (parse_num(line, &cpdwordmax, "COMPOUNDWORDMAX")) {
- fclose(afflst);
+ if (parse_num(line, &cpdwordmax, afflst)) {
+ delete afflst;
return 1;
}
}
/* parse in the flag sign compounds in dictionary */
if (strncmp(line,"COMPOUNDROOT",12) == 0) {
- if (parse_flag(line, &compoundroot, "COMPOUNDROOT")) {
- fclose(afflst);
+ if (parse_flag(line, &compoundroot, afflst)) {
+ delete afflst;
return 1;
}
}
/* parse in the flag used by compound_check() method */
if (strncmp(line,"COMPOUNDPERMITFLAG",18) == 0) {
- if (parse_flag(line, &compoundpermitflag, "COMPOUNDPERMITFLAG")) {
- fclose(afflst);
+ if (parse_flag(line, &compoundpermitflag, afflst)) {
+ delete afflst;
return 1;
}
}
/* parse in the flag used by compound_check() method */
if (strncmp(line,"COMPOUNDFORBIDFLAG",18) == 0) {
- if (parse_flag(line, &compoundforbidflag, "COMPOUNDFORBIDFLAG")) {
- fclose(afflst);
+ if (parse_flag(line, &compoundforbidflag, afflst)) {
+ delete afflst;
return 1;
}
}
if (strncmp(line,"CHECKCOMPOUNDDUP",16) == 0) {
checkcompounddup = 1;
}
if (strncmp(line,"CHECKCOMPOUNDREP",16) == 0) {
checkcompoundrep = 1;
}
if (strncmp(line,"CHECKCOMPOUNDTRIPLE",19) == 0) {
checkcompoundtriple = 1;
}
+ if (strncmp(line,"SIMPLIFIEDTRIPLE",16) == 0) {
+ simplifiedtriple = 1;
+ }
+
if (strncmp(line,"CHECKCOMPOUNDCASE",17) == 0) {
checkcompoundcase = 1;
}
if (strncmp(line,"NOSUGGEST",9) == 0) {
- if (parse_flag(line, &nosuggest, "NOSUGGEST")) {
- fclose(afflst);
+ if (parse_flag(line, &nosuggest, afflst)) {
+ delete afflst;
return 1;
}
}
/* parse in the flag used by forbidden words */
if (strncmp(line,"FORBIDDENWORD",13) == 0) {
- if (parse_flag(line, &forbiddenword, "FORBIDDENWORD")) {
- fclose(afflst);
+ if (parse_flag(line, &forbiddenword, afflst)) {
+ delete afflst;
return 1;
}
}
/* parse in the flag used by forbidden words */
if (strncmp(line,"LEMMA_PRESENT",13) == 0) {
- if (parse_flag(line, &lemma_present, "LEMMA_PRESENT")) {
- fclose(afflst);
+ if (parse_flag(line, &lemma_present, afflst)) {
+ delete afflst;
return 1;
}
}
/* parse in the flag used by circumfixes */
if (strncmp(line,"CIRCUMFIX",9) == 0) {
- if (parse_flag(line, &circumfix, "CIRCUMFIX")) {
- fclose(afflst);
+ if (parse_flag(line, &circumfix, afflst)) {
+ delete afflst;
return 1;
}
}
/* parse in the flag used by fogemorphemes */
if (strncmp(line,"ONLYINCOMPOUND",14) == 0) {
- if (parse_flag(line, &onlyincompound, "ONLYINCOMPOUND")) {
- fclose(afflst);
+ if (parse_flag(line, &onlyincompound, afflst)) {
+ delete afflst;
return 1;
}
}
- /* parse in the flag used by `pseudoroots' */
+ /* parse in the flag used by `needaffixs' */
if (strncmp(line,"PSEUDOROOT",10) == 0) {
- if (parse_flag(line, &pseudoroot, "PSEUDOROOT")) {
- fclose(afflst);
+ if (parse_flag(line, &needaffix, afflst)) {
+ delete afflst;
return 1;
}
}
- /* parse in the flag used by `pseudoroots' */
+ /* parse in the flag used by `needaffixs' */
if (strncmp(line,"NEEDAFFIX",9) == 0) {
- if (parse_flag(line, &pseudoroot, "NEEDAFFIX")) {
- fclose(afflst);
+ if (parse_flag(line, &needaffix, afflst)) {
+ delete afflst;
return 1;
}
}
/* parse in the minimal length for words in compounds */
if (strncmp(line,"COMPOUNDMIN",11) == 0) {
- if (parse_num(line, &cpdmin, "COMPOUNDMIN")) {
- fclose(afflst);
+ if (parse_num(line, &cpdmin, afflst)) {
+ delete afflst;
return 1;
}
if (cpdmin < 1) cpdmin = 1;
}
/* parse in the max. words and syllables in compounds */
if (strncmp(line,"COMPOUNDSYLLABLE",16) == 0) {
- if (parse_cpdsyllable(line)) {
- fclose(afflst);
+ if (parse_cpdsyllable(line, afflst)) {
+ delete afflst;
return 1;
}
}
/* parse in the flag used by compound_check() method */
if (strncmp(line,"SYLLABLENUM",11) == 0) {
- if (parse_string(line, &cpdsyllablenum, "SYLLABLENUM")) {
- fclose(afflst);
+ if (parse_string(line, &cpdsyllablenum, afflst->getlinenum())) {
+ delete afflst;
return 1;
}
}
/* parse in the flag used by the controlled compound words */
if (strncmp(line,"CHECKNUM",8) == 0) {
checknum=1;
}
/* parse in the extra word characters */
if (strncmp(line,"WORDCHARS",9) == 0) {
- if (parse_array(line, &wordchars, &wordchars_utf16, &wordchars_utf16_len, "WORDCHARS", utf8)) {
- fclose(afflst);
+ if (parse_array(line, &wordchars, &wordchars_utf16, &wordchars_utf16_len, utf8, afflst->getlinenum())) {
+ delete afflst;
return 1;
}
}
/* parse in the ignored characters (for example, Arabic optional diacretics charachters */
if (strncmp(line,"IGNORE",6) == 0) {
- if (parse_array(line, &ignorechars, &ignorechars_utf16, &ignorechars_utf16_len, "IGNORE", utf8)) {
- fclose(afflst);
+ if (parse_array(line, &ignorechars, &ignorechars_utf16, &ignorechars_utf16_len, utf8, afflst->getlinenum())) {
+ delete afflst;
return 1;
}
}
/* parse in the typical fault correcting table */
if (strncmp(line,"REP",3) == 0) {
if (parse_reptable(line, afflst)) {
- fclose(afflst);
+ delete afflst;
+ return 1;
+ }
+ }
+
+ /* parse in the input conversion table */
+ if (strncmp(line,"ICONV",5) == 0) {
+ if (parse_convtable(line, afflst, &iconvtable, "ICONV")) {
+ delete afflst;
+ return 1;
+ }
+ }
+
+ /* parse in the input conversion table */
+ if (strncmp(line,"OCONV",5) == 0) {
+ if (parse_convtable(line, afflst, &oconvtable, "OCONV")) {
+ delete afflst;
return 1;
}
}
/* parse in the phonetic translation table */
if (strncmp(line,"PHONE",5) == 0) {
if (parse_phonetable(line, afflst)) {
- fclose(afflst);
+ delete afflst;
return 1;
}
}
/* parse in the checkcompoundpattern table */
if (strncmp(line,"CHECKCOMPOUNDPATTERN",20) == 0) {
if (parse_checkcpdtable(line, afflst)) {
- fclose(afflst);
+ delete afflst;
return 1;
}
}
/* parse in the defcompound table */
if (strncmp(line,"COMPOUNDRULE",12) == 0) {
if (parse_defcpdtable(line, afflst)) {
- fclose(afflst);
+ delete afflst;
return 1;
}
}
/* parse in the related character map table */
if (strncmp(line,"MAP",3) == 0) {
if (parse_maptable(line, afflst)) {
- fclose(afflst);
+ delete afflst;
return 1;
}
}
/* parse in the word breakpoints table */
if (strncmp(line,"BREAK",5) == 0) {
if (parse_breaktable(line, afflst)) {
- fclose(afflst);
+ delete afflst;
return 1;
}
}
/* parse in the language for language specific codes */
if (strncmp(line,"LANG",4) == 0) {
- if (parse_string(line, &lang, "LANG")) {
- fclose(afflst);
+ if (parse_string(line, &lang, afflst->getlinenum())) {
+ delete afflst;
return 1;
}
langnum = get_lang_num(lang);
}
if (strncmp(line,"VERSION",7) == 0) {
- if (parse_string(line, &version, "VERSION")) {
- fclose(afflst);
- return 1;
- }
+ for(line = line + 7; *line == ' ' || *line == '\t'; line++);
+ version = mystrdup(line);
}
if (strncmp(line,"MAXNGRAMSUGS",12) == 0) {
- if (parse_num(line, &maxngramsugs, "MAXNGRAMSUGS")) {
- fclose(afflst);
+ if (parse_num(line, &maxngramsugs, afflst)) {
+ delete afflst;
return 1;
}
}
if (strncmp(line,"NOSPLITSUGS",11) == 0) {
nosplitsugs=1;
}
+ if (strncmp(line,"FULLSTRIP",9) == 0) {
+ fullstrip=1;
+ }
+
if (strncmp(line,"SUGSWITHDOTS",12) == 0) {
sugswithdots=1;
}
/* parse in the flag used by forbidden words */
if (strncmp(line,"KEEPCASE",8) == 0) {
- if (parse_flag(line, &keepcase, "KEEPCASE")) {
- fclose(afflst);
+ if (parse_flag(line, &keepcase, afflst)) {
+ delete afflst;
+ return 1;
+ }
+ }
+
+ /* parse in the flag used by the affix generator */
+ if (strncmp(line,"SUBSTANDARD",11) == 0) {
+ if (parse_flag(line, &substandard, afflst)) {
+ delete afflst;
return 1;
}
}
if (strncmp(line,"CHECKSHARPS",11) == 0) {
checksharps=1;
}
@@ -664,25 +697,25 @@ int AffixMgr::parse_file(const char * a
if (strncmp(line,"PFX",3) == 0) ft = complexprefixes ? 'S' : 'P';
if (strncmp(line,"SFX",3) == 0) ft = complexprefixes ? 'P' : 'S';
if (ft != ' ') {
if (dupflags_ini) {
for (int i = 0; i < CONTSIZE; i++) dupflags[i] = 0;
dupflags_ini = 0;
}
if (parse_affix(line, ft, afflst, dupflags)) {
- fclose(afflst);
+ delete afflst;
process_pfx_tree_to_list();
process_sfx_tree_to_list();
return 1;
}
}
}
- fclose(afflst);
+ delete afflst;
// convert affix trees to sorted list
process_pfx_tree_to_list();
process_sfx_tree_to_list();
// now we can speed up performance greatly taking advantage of the
// relationship between the affixes and the idea of "subsets".
@@ -705,27 +738,47 @@ int AffixMgr::parse_file(const char * a
// Since we have built ordered lists, all that remains is to properly intialize
// the nextne and nexteq pointers that relate them
process_pfx_order();
process_sfx_order();
/* get encoding for CHECKCOMPOUNDCASE */
+ if (!utf8) {
char * enc = get_encoding();
csconv = get_current_cs(enc);
free(enc);
enc = NULL;
- // temporary BREAK definition for German dash handling (OOo issue 64400)
- if ((langnum == LANG_de) && (!breaktable)) {
- breaktable = (char **) malloc(sizeof(char *));
+ char expw[MAXLNLEN];
+ if (wordchars) {
+ strcpy(expw, wordchars);
+ free(wordchars);
+ } else *expw = '\0';
+
+ for (int i = 0; i <= 255; i++) {
+ if ( (csconv[i].cupper != csconv[i].clower) &&
+ (! strchr(expw, (char) i))) {
+ *(expw + strlen(expw) + 1) = '\0';
+ *(expw + strlen(expw)) = (char) i;
+ }
+ }
+
+ wordchars = mystrdup(expw);
+ }
+
+ // default BREAK definition
+ if (!breaktable) {
+ breaktable = (char **) malloc(sizeof(char *) * 3);
if (!breaktable) return 1;
breaktable[0] = mystrdup("-");
- numbreak = 1;
+ breaktable[1] = mystrdup("^-");
+ breaktable[2] = mystrdup("-$");
+ if (breaktable[0] && breaktable[1] && breaktable[2]) numbreak = 3;
}
return 0;
}
// we want to be able to quickly access prefix information
// both by prefix flag, and sorted by prefix string itself
// so we need to set up two indexes
@@ -989,201 +1042,62 @@ int AffixMgr::process_sfx_order()
mptr = nptr;
}
if (mptr) mptr->setNextNE(NULL);
}
}
return 0;
}
-
-
-// takes aff file condition string and creates the
-// conds array - please see the appendix at the end of the
-// file affentry.cpp which describes what is going on here
-// in much more detail
+// add flags to the result for dictionary debugging
+void AffixMgr::debugflag(char * result, unsigned short flag) {
+ char * st = encode_flag(flag);
+ mystrcat(result, " ", MAXLNLEN);
+ mystrcat(result, MORPH_FLAG, MAXLNLEN);
+ if (st) {
+ mystrcat(result, st, MAXLNLEN);
+ free(st);
+ }
+}
+
+// calculate the character length of the condition
+int AffixMgr::condlen(char * st)
+{
+ int l = 0;
+ bool group = false;
+ for(; *st; st++) {
+ if (*st == '[') {
+ group = true;
+ l++;
+ } else if (*st == ']') group = false;
+ else if (!group && (!utf8 ||
+ (!(*st & 0x80) || ((*st & 0xc0) == 0x80)))) l++;
+ }
+ return l;
+}
int AffixMgr::encodeit(struct affentry * ptr, char * cs)
{
- unsigned char c;
- int i, j, k;
- unsigned char mbr[MAXLNLEN];
- w_char wmbr[MAXLNLEN];
- w_char * wpos = wmbr;
-
- // now clear the conditions array */
- for (i=0;i<SETSIZE;i++) ptr->conds.base[i] = (unsigned char) 0;
-
- // now parse the string to create the conds array */
- int nc = strlen(cs);
- unsigned char neg = 0; // complement indicator
- int grp = 0; // group indicator
- unsigned char n = 0; // number of conditions
- int ec = 0; // end condition indicator
- int nm = 0; // number of member in group
-
- // if no condition just return
- if (strcmp(cs,".")==0) {
- ptr->numconds = 0;
- return 0;
- }
-
- i = 0;
- while (i < nc) {
- c = *((unsigned char *)(cs + i));
-
- // start group indicator
- if (c == '[') {
- grp = 1;
- c = 0;
- }
-
- // complement flag
- if ((grp == 1) && (c == '^')) {
- neg = 1;
- c = 0;
- }
-
- // end goup indicator
- if (c == ']') {
- ec = 1;
- c = 0;
- }
-
- // add character of group to list
- if ((grp == 1) && (c != 0)) {
- *(mbr + nm) = c;
- nm++;
- c = 0;
- }
-
- // end of condition
- if (c != 0) {
- ec = 1;
+ if (strcmp(cs,".") != 0) {
+ ptr->numconds = (char) condlen(cs);
+ strncpy(ptr->c.conds, cs, MAXCONDLEN);
+ // long condition (end of conds padded by strncpy)
+ if (ptr->c.conds[MAXCONDLEN - 1] && cs[MAXCONDLEN]) {
+ ptr->opts += aeLONGCOND;
+ ptr->c.l.conds2 = mystrdup(cs + MAXCONDLEN_1);
+ if (!ptr->c.l.conds2) return 1;
}
-
- if (ec) {
- if (!utf8) {
- if (grp == 1) {
- if (neg == 0) {
- // set the proper bits in the condition array vals for those chars
- for (j=0;j<nm;j++) {
- k = (unsigned int) mbr[j];
- ptr->conds.base[k] = ptr->conds.base[k] | ((unsigned char)1 << n);
- }
- } else {
- // complement so set all of them and then unset indicated ones
- for (j=0;j<SETSIZE;j++) ptr->conds.base[j] = ptr->conds.base[j] | ((unsigned char)1 << n);
- for (j=0;j<nm;j++) {
- k = (unsigned int) mbr[j];
- ptr->conds.base[k] = ptr->conds.base[k] & ~((unsigned char)1 << n);
- }
- }
- neg = 0;
- grp = 0;
- nm = 0;
- } else {
- // not a group so just set the proper bit for this char
- // but first handle special case of . inside condition
- if (c == '.') {
- // wild card character so set them all
- for (j=0;j<SETSIZE;j++) ptr->conds.base[j] = ptr->conds.base[j] | ((unsigned char)1 << n);
- } else {
- ptr->conds.base[(unsigned int) c] = ptr->conds.base[(unsigned int)c] | ((unsigned char)1 << n);
- }
- }
- n++;
- ec = 0;
- } else { // UTF-8 character set
- if (grp == 1) {
- ptr->conds.utf8.neg[n] = neg;
- if (neg == 0) {
- // set the proper bits in the condition array vals for those chars
- for (j=0;j<nm;j++) {
- k = (unsigned int) mbr[j];
- if (k >> 7) {
- u8_u16(wpos, 1, (char *) mbr + j);
- wpos++;
- if ((k & 0xe0) == 0xe0) j+=2; else j++; // 3-byte UTF-8 character
- } else {
- ptr->conds.utf8.ascii[k] = ptr->conds.utf8.ascii[k] | ((unsigned char)1 << n);
- }
- }
- } else { // neg == 1
- // complement so set all of them and then unset indicated ones
- for (j=0;j<(SETSIZE/2);j++) ptr->conds.utf8.ascii[j] = ptr->conds.utf8.ascii[j] | ((unsigned char)1 << n);
- for (j=0;j<nm;j++) {
- k = (unsigned int) mbr[j];
- if (k >> 7) {
- u8_u16(wpos, 1, (char *) mbr + j);
- wpos++;
- if ((k & 0xe0) == 0xe0) j+=2; else j++; // 3-byte UTF-8 character
- } else {
- ptr->conds.utf8.ascii[k] = ptr->conds.utf8.ascii[k] & ~((unsigned char)1 << n);
- }
- }
- }
- neg = 0;
- grp = 0;
- nm = 0;
- ptr->conds.utf8.wlen[n] = wpos - wmbr;
- if ((wpos - wmbr) != 0) {
- ptr->conds.utf8.wchars[n] = (w_char *) malloc(sizeof(w_char) * (wpos - wmbr));
- if (!ptr->conds.utf8.wchars[n]) return 1;
- memcpy(ptr->conds.utf8.wchars[n], wmbr, sizeof(w_char) * (wpos - wmbr));
- flag_qsort((unsigned short *) ptr->conds.utf8.wchars[n], 0, ptr->conds.utf8.wlen[n]);
- wpos = wmbr;
- }
- } else { // grp == 0
- // is UTF-8 character?
- if (c >> 7) {
- ptr->conds.utf8.wchars[n] = (w_char *) malloc(sizeof(w_char));
- if (!ptr->conds.utf8.wchars[n]) return 1;
- ptr->conds.utf8.wlen[n] = 1;
- u8_u16(ptr->conds.utf8.wchars[n], 1, cs + i);
- if ((c & 0xe0) == 0xe0) i+=2; else i++; // 3-byte UFT-8 character
- } else {
- ptr->conds.utf8.wchars[n] = NULL;
- // not a group so just set the proper bit for this char
- // but first handle special case of . inside condition
- if (c == '.') {
- ptr->conds.utf8.all[n] = 1;
- // wild card character so set them all
- for (j=0;j<(SETSIZE/2);j++) ptr->conds.utf8.ascii[j] = ptr->conds.utf8.ascii[j] | ((unsigned char)1 << n);
- } else {
- ptr->conds.utf8.all[n] = 0;
- ptr->conds.utf8.ascii[(unsigned int) c] = ptr->conds.utf8.ascii[(unsigned int)c] | ((unsigned char)1 << n);
- }
- }
- neg = 0;
- }
- n++;
- ec = 0;
- neg = 0;
- }
+ } else {
+ ptr->numconds = 0;
+ ptr->c.conds[0] = '\0';
}
-
- i++;
- }
- ptr->numconds = n;
return 0;
}
- // return 1 if s1 is a leading subset of s2
-/* inline int AffixMgr::isSubset(const char * s1, const char * s2)
- {
- while ((*s1 == *s2) && *s1) {
- s1++;
- s2++;
- }
- return (*s1 == '\0');
- }
-*/
-
- // return 1 if s1 is a leading subset of s2 (dots are for infixes)
+// return 1 if s1 is a leading subset of s2 (dots are for infixes)
inline int AffixMgr::isSubset(const char * s1, const char * s2)
{
while (((*s1 == *s2) || (*s1 == '.')) && (*s1 != '\0')) {
s1++;
s2++;
}
return (*s1 == '\0');
}
@@ -1283,17 +1197,16 @@ struct hentry * AffixMgr::prefix_check_t
} else {
pptr = pptr->getNextNE();
}
}
return NULL;
}
-#ifdef HUNSPELL_EXPERIMENTAL
// check word for prefixes
char * AffixMgr::prefix_check_morph(const char * word, int len, char in_compound,
const FLAG needflag)
{
char * st;
char result[MAXLNLEN];
result[0] = '\0';
@@ -1301,17 +1214,17 @@ char * AffixMgr::prefix_check_morph(cons
pfx = NULL;
sfxappnd = NULL;
// first handle the special case of 0 length prefixes
PfxEntry * pe = (PfxEntry *) pStart[0];
while (pe) {
st = pe->check_morph(word,len,in_compound, needflag);
if (st) {
- strcat(result, st);
+ mystrcat(result, st, MAXLNLEN);
free(st);
}
// if (rv) return rv;
pe = pe->getNext();
}
// now handle the general case
unsigned char sp = *((const unsigned char *)word);
@@ -1319,17 +1232,17 @@ char * AffixMgr::prefix_check_morph(cons
while (pptr) {
if (isSubset(pptr->getKey(),word)) {
st = pptr->check_morph(word,len,in_compound, needflag);
if (st) {
// fogemorpheme
if ((in_compound != IN_CPD_NOT) || !((pptr->getCont() &&
(TESTAFF(pptr->getCont(), onlyincompound, pptr->getContLen()))))) {
- strcat(result, st);
+ mystrcat(result, st, MAXLNLEN);
pfx = (AffEntry *)pptr;
}
free(st);
}
pptr = pptr->getNextEQ();
} else {
pptr = pptr->getNextNE();
}
@@ -1352,45 +1265,43 @@ char * AffixMgr::prefix_check_twosfx_mor
pfx = NULL;
sfxappnd = NULL;
// first handle the special case of 0 length prefixes
PfxEntry * pe = (PfxEntry *) pStart[0];
while (pe) {
st = pe->check_twosfx_morph(word,len,in_compound, needflag);
if (st) {
- strcat(result, st);
+ mystrcat(result, st, MAXLNLEN);
free(st);
}
pe = pe->getNext();
}
// now handle the general case
unsigned char sp = *((const unsigned char *)word);
PfxEntry * pptr = (PfxEntry *)pStart[sp];
while (pptr) {
if (isSubset(pptr->getKey(),word)) {
st = pptr->check_twosfx_morph(word, len, in_compound, needflag);
if (st) {
- strcat(result, st);
+ mystrcat(result, st, MAXLNLEN);
free(st);
pfx = (AffEntry *)pptr;
}
pptr = pptr->getNextEQ();
} else {
pptr = pptr->getNextNE();
}
}
if (*result) return mystrdup(result);
return NULL;
}
-#endif // END OF HUNSPELL_EXPERIMENTAL CODE
-
// Is word a non compound with a REP substitution (see checkcompoundrep)?
int AffixMgr::cpdrep_check(const char * word, int wl)
{
char candidate[MAXLNLEN];
const char * r;
int lenr, lenp;
@@ -1409,21 +1320,25 @@ int AffixMgr::cpdrep_check(const char *
if (candidate_check(candidate,strlen(candidate))) return 1;
r++; // search for the next letter
}
}
return 0;
}
// forbid compoundings when there are special patterns at word bound
-int AffixMgr::cpdpat_check(const char * word, int pos)
+int AffixMgr::cpdpat_check(const char * word, int pos, hentry * r1, hentry * r2)
{
int len;
for (int i = 0; i < numcheckcpd; i++) {
if (isSubset(checkcpdtable[i].pattern2, word + pos) &&
+ (!r1 || !checkcpdtable[i].cond ||
+ (r1->astr && TESTAFF(r1->astr, checkcpdtable[i].cond, r1->alen))) &&
+ (!r2 || !checkcpdtable[i].cond2 ||
+ (r2->astr && TESTAFF(r2->astr, checkcpdtable[i].cond2, r2->alen))) &&
(len = strlen(checkcpdtable[i].pattern)) && (pos > len) &&
(strncmp(word + pos - len, checkcpdtable[i].pattern, len) == 0)) return 1;
}
return 0;
}
// forbid compounding with neighbouring upper and lower case characters at word bounds
int AffixMgr::cpdcase_check(const char * word, int pos)
@@ -1431,41 +1346,62 @@ int AffixMgr::cpdcase_check(const char *
if (utf8) {
w_char u, w;
const char * p;
u8_u16(&u, 1, word + pos);
for (p = word + pos - 1; (*p & 0xc0) == 0x80; p--);
u8_u16(&w, 1, p);
unsigned short a = (u.h << 8) + u.l;
unsigned short b = (w.h << 8) + w.l;
- if (((unicodetoupper(a, langnum) == a) || (unicodetoupper(b, langnum) == b))) return 1;
+ if (((unicodetoupper(a, langnum) == a) || (unicodetoupper(b, langnum) == b)) &&
+ (a != '-') && (b != '-')) return 1;
} else {
unsigned char a = *(word + pos - 1);
unsigned char b = *(word + pos);
if ((csconv[a].ccase || csconv[b].ccase) && (a != '-') && (b != '-')) return 1;
}
return 0;
}
// check compound patterns
int AffixMgr::defcpd_check(hentry *** words, short wnum, hentry * rv, hentry ** def, char all)
{
signed short btpp[MAXWORDLEN]; // metacharacter (*, ?) positions for backtracking
signed short btwp[MAXWORDLEN]; // word positions for metacharacters
int btnum[MAXWORDLEN]; // number of matched characters in metacharacter positions
short bt = 0;
- int i;
+ int i, j;
int ok;
int w = 0;
+
if (!*words) {
w = 1;
*words = def;
}
(*words)[wnum] = rv;
+ // has the last word COMPOUNDRULE flag?
+ if (rv->alen == 0) {
+ (*words)[wnum] = NULL;
+ if (w) *words = NULL;
+ return 0;
+ }
+ ok = 0;
+ for (i = 0; i < numdefcpd; i++) {
+ for (j = 0; j < defcpdtable[i].len; j++) {
+ if (defcpdtable[i].def[j] != '*' && defcpdtable[i].def[j] != '?' &&
+ TESTAFF(rv->astr, defcpdtable[i].def[j], rv->alen)) ok = 1;
+ }
+ }
+ if (ok == 0) {
+ (*words)[wnum] = NULL;
+ if (w) *words = NULL;
+ return 0;
+ }
+
for (i = 0; i < numdefcpd; i++) {
signed short pp = 0; // pattern position
signed short wp = 0; // "words" position
int ok2;
ok = 1;
ok2 = 1;
do {
while ((pp < defcpdtable[i].len) && (wp <= wnum)) {
@@ -1500,27 +1436,28 @@ int AffixMgr::defcpd_check(hentry *** wo
if ((defcpdtable[i].len == pp) && !(wp > wnum)) ok = 0;
}
}
if (ok && ok2) {
int r = pp;
while ((defcpdtable[i].len > r) && ((r+1) < defcpdtable[i].len) &&
((defcpdtable[i].def[r+1] == '*') || (defcpdtable[i].def[r+1] == '?'))) r+=2;
if (defcpdtable[i].len <= r) return 1;
- }
+ }
// backtrack
if (bt) do {
ok = 1;
btnum[bt - 1]--;
pp = btpp[bt - 1];
- wp = btwp[bt - 1] + btnum[bt - 1];
+ wp = btwp[bt - 1] + (signed short) btnum[bt - 1];
} while ((btnum[bt - 1] < 0) && --bt);
} while (bt);
- if (ok && ok2 && (!all || (defcpdtable[i].len <= pp))) return 1;
+ if (ok && ok2 && (!all || (defcpdtable[i].len <= pp))) return 1;
+
// check zero ending
while (ok && ok2 && (defcpdtable[i].len > pp) && ((pp+1) < defcpdtable[i].len) &&
((defcpdtable[i].def[pp+1] == '*') || (defcpdtable[i].def[pp+1] == '?'))) pp+=2;
if (ok && ok2 && (defcpdtable[i].len <= pp)) return 1;
}
(*words)[wnum] = NULL;
if (w) *words = NULL;
return 0;
@@ -1558,122 +1495,147 @@ short AffixMgr::get_syllable(const char
for (; i > 0; i--) {
if (flag_bsearch((unsigned short *) cpdvowels_utf16,
((unsigned short *) w)[i - 1], cpdvowels_utf16_len)) num++;
}
}
return num;
}
+void AffixMgr::setcminmax(int * cmin, int * cmax, const char * word, int len) {
+ if (utf8) {
+ int i;
+ for (*cmin = 0, i = 0; (i < cpdmin) && word[*cmin]; i++) {
+ for ((*cmin)++; (word[*cmin] & 0xc0) == 0x80; (*cmin)++);
+ }
+ for (*cmax = len, i = 0; (i < (cpdmin - 1)) && *cmax; i++) {
+ for ((*cmax)--; (word[*cmax] & 0xc0) == 0x80; (*cmax)--);
+ }
+ } else {
+ *cmin = cpdmin;
+ *cmax = len - cpdmin + 1;
+ }
+}
+
// check if compound word is correctly spelled
// hu_mov_rule = spec. Hungarian rule (XXX)
struct hentry * AffixMgr::compound_check(const char * word, int len,
short wordnum, short numsyllable, short maxwordnum, short wnum, hentry ** words = NULL,
- char hu_mov_rule = 0, int * cmpdstemnum = NULL, int * cmpdstem = NULL, char is_sug = 0)
+ char hu_mov_rule = 0, char is_sug = 0)
{
int i;
short oldnumsyllable, oldnumsyllable2, oldwordnum, oldwordnum2;
- int oldcmpdstemnum = 0;
struct hentry * rv = NULL;
struct hentry * rv_first;
struct hentry * rwords[MAXWORDLEN]; // buffer for COMPOUND pattern checking
char st [MAXWORDUTF8LEN + 4];
char ch;
int cmin;
int cmax;
-
+ int striple = 0;
+ int scpd = 0;
+ int soldi = 0;
+ int oldcmin = 0;
+ int oldcmax = 0;
+ int oldlen = 0;
+ int checkedstriple = 0;
+
int checked_prefix;
-#ifdef HUNSTEM
- if (cmpdstemnum) {
- if (wordnum == 0) {
- *cmpdstemnum = 1;
- } else {
- (*cmpdstemnum)++;
- }
- }
-#endif
- if (utf8) {
- for (cmin = 0, i = 0; (i < cpdmin) && word[cmin]; i++) {
- cmin++;
- for (; (word[cmin] & 0xc0) == 0x80; cmin++);
- }
- for (cmax = len, i = 0; (i < (cpdmin - 1)) && cmax; i++) {
- cmax--;
- for (; (word[cmax] & 0xc0) == 0x80; cmax--);
- }
- } else {
- cmin = cpdmin;
- cmax = len - cpdmin + 1;
- }
+ setcminmax(&cmin, &cmax, word, len);
strcpy(st, word);
for (i = cmin; i < cmax; i++) {
oldnumsyllable = numsyllable;
oldwordnum = wordnum;
checked_prefix = 0;
// go to end of the UTF-8 character
if (utf8) {
for (; (st[i] & 0xc0) == 0x80; i++);
if (i >= cmax) return NULL;
}
-
+ do { // simplified checkcompoundpattern loop
+
+ if (scpd > 0) {
+ for (; scpd <= numcheckcpd && (!checkcpdtable[scpd-1].pattern3 ||
+ strncmp(word + i, checkcpdtable[scpd-1].pattern3, strlen(checkcpdtable[scpd-1].pattern3)) != 0); scpd++);
+
+ if (scpd > numcheckcpd) break; // break simplified checkcompoundpattern loop
+ strcpy(st + i, checkcpdtable[scpd-1].pattern);
+ soldi = i;
+ i += strlen(checkcpdtable[scpd-1].pattern);
+ strcpy(st + i, checkcpdtable[scpd-1].pattern2);
+ strcpy(st + i + strlen(checkcpdtable[scpd-1].pattern2), word + soldi + strlen(checkcpdtable[scpd-1].pattern3));
+
+ oldlen = len;
+ len += strlen(checkcpdtable[scpd-1].pattern) + strlen(checkcpdtable[scpd-1].pattern2) - strlen(checkcpdtable[scpd-1].pattern3);
+ oldcmin = cmin;
+ oldcmax = cmax;
+ setcminmax(&cmin, &cmax, st, len);
+
+ cmax = len - cpdmin + 1;
+ }
+
+
ch = st[i];
st[i] = '\0';
sfx = NULL;
pfx = NULL;
-
+
// FIRST WORD
-
+
rv = lookup(st); // perhaps without prefix
// search homonym with compound flag
while ((rv) && !hu_mov_rule &&
- ((pseudoroot && TESTAFF(rv->astr, pseudoroot, rv->alen)) ||
+ ((needaffix && TESTAFF(rv->astr, needaffix, rv->alen)) ||
!((compoundflag && !words && TESTAFF(rv->astr, compoundflag, rv->alen)) ||
(compoundbegin && !wordnum &&
TESTAFF(rv->astr, compoundbegin, rv->alen)) ||
(compoundmiddle && wordnum && !words &&
TESTAFF(rv->astr, compoundmiddle, rv->alen)) ||
(numdefcpd &&
((!words && !wordnum && defcpd_check(&words, wnum, rv, (hentry **) &rwords, 0)) ||
- (words && defcpd_check(&words, wnum, rv, (hentry **) &rwords, 0))))
- ))) {
+ (words && defcpd_check(&words, wnum, rv, (hentry **) &rwords, 0))))) ||
+ (scpd != 0 && checkcpdtable[scpd-1].cond != FLAG_NULL &&
+ !TESTAFF(rv->astr, checkcpdtable[scpd-1].cond, rv->alen)))
+ ) {
rv = rv->next_homonym;
}
if (!rv) {
if (compoundflag &&
!(rv = prefix_check(st, i, hu_mov_rule ? IN_CPD_OTHER : IN_CPD_BEGIN, compoundflag))) {
if ((rv = suffix_check(st, i, 0, NULL, NULL, 0, NULL,
FLAG_NULL, compoundflag, hu_mov_rule ? IN_CPD_OTHER : IN_CPD_BEGIN)) && !hu_mov_rule &&
((SfxEntry*)sfx)->getCont() &&
((compoundforbidflag && TESTAFF(((SfxEntry*)sfx)->getCont(), compoundforbidflag,
((SfxEntry*)sfx)->getContLen())) || (compoundend &&
TESTAFF(((SfxEntry*)sfx)->getCont(), compoundend,
((SfxEntry*)sfx)->getContLen())))) {
rv = NULL;
}
}
+
if (rv ||
(((wordnum == 0) && compoundbegin &&
((rv = suffix_check(st, i, 0, NULL, NULL, 0, NULL, FLAG_NULL, compoundbegin, hu_mov_rule ? IN_CPD_OTHER : IN_CPD_BEGIN)) ||
(rv = prefix_check(st, i, hu_mov_rule ? IN_CPD_OTHER : IN_CPD_BEGIN, compoundbegin)))) ||
((wordnum > 0) && compoundmiddle &&
((rv = suffix_check(st, i, 0, NULL, NULL, 0, NULL, FLAG_NULL, compoundmiddle, hu_mov_rule ? IN_CPD_OTHER : IN_CPD_BEGIN)) ||
(rv = prefix_check(st, i, hu_mov_rule ? IN_CPD_OTHER : IN_CPD_BEGIN, compoundmiddle)))))
) checked_prefix = 1;
- // else check forbiddenwords and pseudoroot
+ // else check forbiddenwords and needaffix
} else if (rv->astr && (TESTAFF(rv->astr, forbiddenword, rv->alen) ||
- TESTAFF(rv->astr, pseudoroot, rv->alen) ||
+ TESTAFF(rv->astr, needaffix, rv->alen) ||
(is_sug && nosuggest && TESTAFF(rv->astr, nosuggest, rv->alen))
)) {
st[i] = ch;
continue;
}
// check non_compound flag in suffix and prefix
if ((rv) && !hu_mov_rule &&
@@ -1691,27 +1653,27 @@ struct hentry * AffixMgr::compound_check
((pfx && ((PfxEntry*)pfx)->getCont() &&
TESTAFF(((PfxEntry*)pfx)->getCont(), compoundend,
((PfxEntry*)pfx)->getContLen())) ||
(sfx && ((SfxEntry*)sfx)->getCont() &&
TESTAFF(((SfxEntry*)sfx)->getCont(), compoundend,
((SfxEntry*)sfx)->getContLen())))) {
rv = NULL;
}
-
+
// check compoundmiddle flag in suffix and prefix
if ((rv) && !checked_prefix && (wordnum==0) && compoundmiddle && !hu_mov_rule &&
((pfx && ((PfxEntry*)pfx)->getCont() &&
TESTAFF(((PfxEntry*)pfx)->getCont(), compoundmiddle,
((PfxEntry*)pfx)->getContLen())) ||
(sfx && ((SfxEntry*)sfx)->getCont() &&
TESTAFF(((SfxEntry*)sfx)->getCont(), compoundmiddle,
((SfxEntry*)sfx)->getContLen())))) {
rv = NULL;
- }
+ }
// check forbiddenwords
if ((rv) && (rv->astr) && (TESTAFF(rv->astr, forbiddenword, rv->alen) ||
(is_sug && nosuggest && TESTAFF(rv->astr, nosuggest, rv->alen)))) {
return NULL;
}
// increment word number, if the second root has a compoundroot flag
@@ -1731,69 +1693,81 @@ struct hentry * AffixMgr::compound_check
// LANG_hu section: spec. Hungarian rule
|| ((langnum == LANG_hu) && hu_mov_rule && (
TESTAFF(rv->astr, 'F', rv->alen) || // XXX hardwired Hungarian dictionary codes
TESTAFF(rv->astr, 'G', rv->alen) ||
TESTAFF(rv->astr, 'H', rv->alen)
)
)
// END of LANG_hu section
+ ) &&
+ (
+ // test CHECKCOMPOUNDPATTERN conditions
+ scpd == 0 || checkcpdtable[scpd-1].cond == FLAG_NULL ||
+ TESTAFF(rv->astr, checkcpdtable[scpd-1].cond, rv->alen)
)
- && ! (( checkcompoundtriple && // test triple letters
+ && ! (( checkcompoundtriple && scpd == 0 && !words && // test triple letters
(word[i-1]==word[i]) && (
- ((i>1) && (word[i-1]==word[i-2])) ||
+ ((i>1) && (word[i-1]==word[i-2])) ||
((word[i-1]==word[i+1])) // may be word[i+1] == '\0'
)
) ||
- (
- // test CHECKCOMPOUNDPATTERN
- numcheckcpd && cpdpat_check(word, i)
- ) ||
- (
- checkcompoundcase && cpdcase_check(word, i)
+ (
+ checkcompoundcase && scpd == 0 && !words && cpdcase_check(word, i)
))
)
// LANG_hu section: spec. Hungarian rule
|| ((!rv) && (langnum == LANG_hu) && hu_mov_rule && (rv = affix_check(st,i)) &&
(sfx && ((SfxEntry*)sfx)->getCont() && ( // XXX hardwired Hungarian dic. codes
TESTAFF(((SfxEntry*)sfx)->getCont(), (unsigned short) 'x', ((SfxEntry*)sfx)->getContLen()) ||
TESTAFF(((SfxEntry*)sfx)->getCont(), (unsigned short) '%', ((SfxEntry*)sfx)->getContLen())
- )
+ )
)
)
-// END of LANG_hu section
- ) {
+ ) { // first word is ok condition
// LANG_hu section: spec. Hungarian rule
if (langnum == LANG_hu) {
- // calculate syllable number of the word
+ // calculate syllable number of the word
numsyllable += get_syllable(st, i);
// + 1 word, if syllable number of the prefix > 1 (hungarian convention)
if (pfx && (get_syllable(((PfxEntry *)pfx)->getKey(),strlen(((PfxEntry *)pfx)->getKey())) > 1)) wordnum++;
}
// END of LANG_hu section
-#ifdef HUNSTEM
- if (cmpdstem) cmpdstem[*cmpdstemnum - 1] = i;
-#endif
// NEXT WORD(S)
rv_first = rv;
- rv = lookup((word+i)); // perhaps without prefix
+ st[i] = ch;
+
+ do { // striple loop
+
+ // check simplifiedtriple
+ if (simplifiedtriple) {
+ if (striple) {
+ checkedstriple = 1;
+ i--; // check "fahrt" instead of "ahrt" in "Schiffahrt"
+ } else if (i > 2 && *(word+i - 1) == *(word + i - 2)) striple = 1;
+ }
+
+ rv = lookup((st+i)); // perhaps without prefix
// search homonym with compound flag
- while ((rv) && ((pseudoroot && TESTAFF(rv->astr, pseudoroot, rv->alen)) ||
+ while ((rv) && ((needaffix && TESTAFF(rv->astr, needaffix, rv->alen)) ||
!((compoundflag && !words && TESTAFF(rv->astr, compoundflag, rv->alen)) ||
(compoundend && !words && TESTAFF(rv->astr, compoundend, rv->alen)) ||
- (numdefcpd && words && defcpd_check(&words, wnum + 1, rv, NULL,1))))) {
+ (numdefcpd && words && defcpd_check(&words, wnum + 1, rv, NULL,1))) ||
+ (scpd != 0 && checkcpdtable[scpd-1].cond2 != FLAG_NULL &&
+ !TESTAFF(rv->astr, checkcpdtable[scpd-1].cond2, rv->alen))
+ )) {
rv = rv->next_homonym;
}
- if (rv && words && words[wnum + 1]) return rv;
+ if (rv && words && words[wnum + 1]) return rv_first;
oldnumsyllable2 = numsyllable;
oldwordnum2 = wordnum;
// LANG_hu section: spec. Hungarian rule, XXX hardwired dictionary code
if ((rv) && (langnum == LANG_hu) && (TESTAFF(rv->astr, 'I', rv->alen)) && !(TESTAFF(rv->astr, 'J', rv->alen))) {
numsyllable--;
}
@@ -1815,48 +1789,62 @@ struct hentry * AffixMgr::compound_check
// then the syllable number of root words must be 6, or lesser.
if ((rv) && (
(compoundflag && TESTAFF(rv->astr, compoundflag, rv->alen)) ||
(compoundend && TESTAFF(rv->astr, compoundend, rv->alen))
)
&& (
((cpdwordmax==-1) || (wordnum+1<cpdwordmax)) ||
- ((cpdmaxsyllable==0) ||
- (numsyllable + get_syllable(&(rv->word), rv->clen)<=cpdmaxsyllable))
- )
- && (
+ ((cpdmaxsyllable!=0) &&
+ (numsyllable + get_syllable(HENTRY_WORD(rv), rv->clen)<=cpdmaxsyllable))
+ ) &&
+ (
+ // test CHECKCOMPOUNDPATTERN
+ !numcheckcpd || scpd != 0 || !cpdpat_check(word, i, rv_first, rv)
+ ) &&
+ (
(!checkcompounddup || (rv != rv_first))
)
+ // test CHECKCOMPOUNDPATTERN conditions
+ && (scpd == 0 || checkcpdtable[scpd-1].cond2 == FLAG_NULL ||
+ TESTAFF(rv->astr, checkcpdtable[scpd-1].cond2, rv->alen))
)
{
// forbid compound word, if it is a non compound word with typical fault
if (checkcompoundrep && cpdrep_check(word,len)) return NULL;
- return rv;
+ return rv_first;
}
- numsyllable = oldnumsyllable2 ;
+ numsyllable = oldnumsyllable2;
wordnum = oldwordnum2;
// perhaps second word has prefix or/and suffix
sfx = NULL;
sfxflag = FLAG_NULL;
rv = (compoundflag) ? affix_check((word+i),strlen(word+i), compoundflag, IN_CPD_END) : NULL;
if (!rv && compoundend) {
sfx = NULL;
pfx = NULL;
rv = affix_check((word+i),strlen(word+i), compoundend, IN_CPD_END);
}
-
+
if (!rv && numdefcpd && words) {
rv = affix_check((word+i),strlen(word+i), 0, IN_CPD_END);
- if (rv && defcpd_check(&words, wnum + 1, rv, NULL, 1)) return rv;
+ if (rv && defcpd_check(&words, wnum + 1, rv, NULL, 1)) return rv_first;
rv = NULL;
}
+ // test CHECKCOMPOUNDPATTERN conditions (allowed forms)
+ if (rv && !(scpd == 0 || checkcpdtable[scpd-1].cond2 == FLAG_NULL ||
+ TESTAFF(rv->astr, checkcpdtable[scpd-1].cond2, rv->alen))) rv = NULL;
+
+ // test CHECKCOMPOUNDPATTERN conditions (forbidden compounds)
+ if (rv && numcheckcpd && scpd == 0 && cpdpat_check(word, i, rv_first, rv)) rv = NULL;
+
// check non_compound flag in suffix and prefix
if ((rv) &&
((pfx && ((PfxEntry*)pfx)->getCont() &&
TESTAFF(((PfxEntry*)pfx)->getCont(), compoundforbidflag,
((PfxEntry*)pfx)->getContLen())) ||
(sfx && ((SfxEntry*)sfx)->getCont() &&
TESTAFF(((SfxEntry*)sfx)->getCont(), compoundforbidflag,
((SfxEntry*)sfx)->getContLen())))) {
@@ -1870,229 +1858,233 @@ struct hentry * AffixMgr::compound_check
// pfxappnd = prefix of word+i, or NULL
// calculate syllable number of prefix.
// hungarian convention: when syllable number of prefix is more,
// than 1, the prefix+word counts as two words.
if (langnum == LANG_hu) {
// calculate syllable number of the word
numsyllable += get_syllable(word + i, strlen(word + i));
-
+
// - affix syllable num.
// XXX only second suffix (inflections, not derivations)
if (sfxappnd) {
char * tmp = myrevstrdup(sfxappnd);
numsyllable -= get_syllable(tmp, strlen(tmp));
free(tmp);
}
-
+
// + 1 word, if syllable number of the prefix > 1 (hungarian convention)
if (pfx && (get_syllable(((PfxEntry *)pfx)->getKey(),strlen(((PfxEntry *)pfx)->getKey())) > 1)) wordnum++;
// increment syllable num, if last word has a SYLLABLENUM flag
// and the suffix is beginning `s'
-
+
if (cpdsyllablenum) {
switch (sfxflag) {
case 'c': { numsyllable+=2; break; }
case 'J': { numsyllable += 1; break; }
case 'I': { if (TESTAFF(rv->astr, 'J', rv->alen)) numsyllable += 1; break; }
}
}
}
-
+
// increment word number, if the second word has a compoundroot flag
if ((rv) && (compoundroot) &&
(TESTAFF(rv->astr, compoundroot, rv->alen))) {
wordnum++;
}
// second word is acceptable, as a word with prefix or/and suffix?
// hungarian conventions: compounding is acceptable,
// when compound forms consist 2 word, otherwise
// the syllable number of root words is 6, or lesser.
if ((rv) &&
(
((cpdwordmax == -1) || (wordnum + 1 < cpdwordmax)) ||
- ((cpdmaxsyllable == 0) ||
+ ((cpdmaxsyllable != 0) &&
(numsyllable <= cpdmaxsyllable))
)
&& (
(!checkcompounddup || (rv != rv_first))
)) {
// forbid compound word, if it is a non compound word with typical fault
if (checkcompoundrep && cpdrep_check(word, len)) return NULL;
- return rv;
+ return rv_first;
}
numsyllable = oldnumsyllable2;
wordnum = oldwordnum2;
-#ifdef HUNSTEM
- if (cmpdstemnum) oldcmpdstemnum = *cmpdstemnum;
-#endif
+
// perhaps second word is a compound word (recursive call)
if (wordnum < maxwordnum) {
- rv = compound_check((word+i),strlen(word+i), wordnum+1,
- numsyllable, maxwordnum, wnum + 1, words,
- 0, cmpdstemnum, cmpdstem, is_sug);
+ rv = compound_check((st+i),strlen(st+i), wordnum+1,
+ numsyllable, maxwordnum, wnum + 1, words, 0, is_sug);
+ if (rv && numcheckcpd && (scpd == 0 && cpdpat_check(word, i, rv_first, rv) ||
+ scpd != 0 && !cpdpat_check(word, i, rv_first, rv))) rv = NULL;
} else {
rv=NULL;
}
if (rv) {
// forbid compound word, if it is a non compound word with typical fault
if (checkcompoundrep && cpdrep_check(word, len)) return NULL;
- return rv;
- } else {
-#ifdef HUNSTEM
- if (cmpdstemnum) *cmpdstemnum = oldcmpdstemnum;
-#endif
+ return rv_first;
}
+ } while (striple && !checkedstriple); // end of striple loop
+
+ if (checkedstriple) {
+ i++;
+ checkedstriple = 0;
+ striple = 0;
+ }
+
+ } // first word is ok condition
+
+ if (soldi != 0) {
+ i = soldi;
+ soldi = 0;
+ len = oldlen;
+ cmin = oldcmin;
+ cmax = oldcmax;
}
- st[i] = ch;
+ scpd++;
+
+ } while (simplifiedcpd && scpd <= numcheckcpd); // end of simplifiedcpd loop
+
+ if (soldi != 0) {
+ i = soldi;
+ strcpy(st, word); // XXX add more optim.
+ soldi = 0;
+ } else st[i] = ch;
+
+ scpd = 0;
wordnum = oldwordnum;
numsyllable = oldnumsyllable;
}
-
+
return NULL;
-}
-
-#ifdef HUNSPELL_EXPERIMENTAL
+}
+
// check if compound word is correctly spelled
// hu_mov_rule = spec. Hungarian rule (XXX)
int AffixMgr::compound_check_morph(const char * word, int len,
short wordnum, short numsyllable, short maxwordnum, short wnum, hentry ** words,
char hu_mov_rule = 0, char ** result = NULL, char * partresult = NULL)
{
int i;
short oldnumsyllable, oldnumsyllable2, oldwordnum, oldwordnum2;
int ok = 0;
struct hentry * rv = NULL;
struct hentry * rv_first;
struct hentry * rwords[MAXWORDLEN]; // buffer for COMPOUND pattern checking
char st [MAXWORDUTF8LEN + 4];
char ch;
-
+
int checked_prefix;
char presult[MAXLNLEN];
int cmin;
int cmax;
-
- if (utf8) {
- for (cmin = 0, i = 0; (i < cpdmin) && word[cmin]; i++) {
- cmin++;
- for (; (word[cmin] & 0xc0) == 0x80; cmin++);
- }
- for (cmax = len, i = 0; (i < (cpdmin - 1)) && cmax; i++) {
- cmax--;
- for (; (word[cmax] & 0xc0) == 0x80; cmax--);
- }
- } else {
- cmin = cpdmin;
- cmax = len - cpdmin + 1;
- }
+
+ setcminmax(&cmin, &cmax, word, len);
strcpy(st, word);
for (i = cmin; i < cmax; i++) {
oldnumsyllable = numsyllable;
oldwordnum = wordnum;
checked_prefix = 0;
// go to end of the UTF-8 character
if (utf8) {
for (; (st[i] & 0xc0) == 0x80; i++);
if (i >= cmax) return 0;
}
-
+
ch = st[i];
st[i] = '\0';
sfx = NULL;
// FIRST WORD
*presult = '\0';
if (partresult) strcat(presult, partresult);
-
+
rv = lookup(st); // perhaps without prefix
// search homonym with compound flag
while ((rv) && !hu_mov_rule &&
- ((pseudoroot && TESTAFF(rv->astr, pseudoroot, rv->alen)) ||
+ ((needaffix && TESTAFF(rv->astr, needaffix, rv->alen)) ||
!((compoundflag && !words && TESTAFF(rv->astr, compoundflag, rv->alen)) ||
(compoundbegin && !wordnum &&
TESTAFF(rv->astr, compoundbegin, rv->alen)) ||
(compoundmiddle && wordnum && !words &&
TESTAFF(rv->astr, compoundmiddle, rv->alen)) ||
(numdefcpd &&
((!words && !wordnum && defcpd_check(&words, wnum, rv, (hentry **) &rwords, 0)) ||
(words && defcpd_check(&words, wnum, rv, (hentry **) &rwords, 0))))
))) {
rv = rv->next_homonym;
}
if (rv) {
- if (rv->description) {
- if ((!rv->astr) || !TESTAFF(rv->astr, lemma_present, rv->alen))
- strcat(presult, st);
- strcat(presult, rv->description);
+ sprintf(presult + strlen(presult), "%c%s%s", MSEP_FLD, MORPH_PART, st);
+ if (!HENTRY_FIND(rv, MORPH_STEM)) {
+ sprintf(presult + strlen(presult), "%c%s%s", MSEP_FLD, MORPH_STEM, st);
}
- }
-
+ // store the pointer of the hash entry
+// sprintf(presult + strlen(presult), "%c%s%p", MSEP_FLD, MORPH_HENTRY, rv);
+ if (HENTRY_DATA(rv)) {
+ sprintf(presult + strlen(presult), "%c%s", MSEP_FLD, HENTRY_DATA2(rv));
+ }
+ }
if (!rv) {
if (compoundflag &&
!(rv = prefix_check(st, i, hu_mov_rule ? IN_CPD_OTHER : IN_CPD_BEGIN, compoundflag))) {
if ((rv = suffix_check(st, i, 0, NULL, NULL, 0, NULL,
FLAG_NULL, compoundflag, hu_mov_rule ? IN_CPD_OTHER : IN_CPD_BEGIN)) && !hu_mov_rule &&
((SfxEntry*)sfx)->getCont() &&
((compoundforbidflag && TESTAFF(((SfxEntry*)sfx)->getCont(), compoundforbidflag,
((SfxEntry*)sfx)->getContLen())) || (compoundend &&
TESTAFF(((SfxEntry*)sfx)->getCont(), compoundend,
((SfxEntry*)sfx)->getContLen())))) {
rv = NULL;
}
}
-
+
if (rv ||
(((wordnum == 0) && compoundbegin &&
((rv = suffix_check(st, i, 0, NULL, NULL, 0, NULL, FLAG_NULL, compoundbegin, hu_mov_rule ? IN_CPD_OTHER : IN_CPD_BEGIN)) ||
(rv = prefix_check(st, i, hu_mov_rule ? IN_CPD_OTHER : IN_CPD_BEGIN, compoundbegin)))) ||
((wordnum > 0) && compoundmiddle &&
((rv = suffix_check(st, i, 0, NULL, NULL, 0, NULL, FLAG_NULL, compoundmiddle, hu_mov_rule ? IN_CPD_OTHER : IN_CPD_BEGIN)) ||
(rv = prefix_check(st, i, hu_mov_rule ? IN_CPD_OTHER : IN_CPD_BEGIN, compoundmiddle)))))
) {
- //char * p = prefix_check_morph(st, i, 0, compound);
+ // char * p = prefix_check_morph(st, i, 0, compound);
char * p = NULL;
if (compoundflag) p = affix_check_morph(st, i, compoundflag);
if (!p || (*p == '\0')) {
+ if (p) free(p);
+ p = NULL;
if ((wordnum == 0) && compoundbegin) {
p = affix_check_morph(st, i, compoundbegin);
} else if ((wordnum > 0) && compoundmiddle) {
p = affix_check_morph(st, i, compoundmiddle);
}
}
- if (*p != '\0') {
- line_uniq(p);
- if (strchr(p, '\n')) {
- strcat(presult, "(");
- strcat(presult, line_join(p, '|'));
- strcat(presult, ")");
- } else {
- strcat(presult, p);
- }
+ if (p && (*p != '\0')) {
+ sprintf(presult + strlen(presult), "%c%s%s%s", MSEP_FLD,
+ MORPH_PART, st, line_uniq_app(&p, MSEP_REC));
}
- if (presult[strlen(presult) - 1] == '\n') {
- presult[strlen(presult) - 1] = '\0';
- }
+ if (p) free(p);
checked_prefix = 1;
- //strcat(presult, "+");
}
// else check forbiddenwords
} else if (rv->astr && (TESTAFF(rv->astr, forbiddenword, rv->alen) ||
- TESTAFF(rv->astr, pseudoroot, rv->alen))) {
+ TESTAFF(rv->astr, needaffix, rv->alen))) {
st[i] = ch;
continue;
}
// check non_compound flag in suffix and prefix
if ((rv) && !hu_mov_rule &&
((pfx && ((PfxEntry*)pfx)->getCont() &&
TESTAFF(((PfxEntry*)pfx)->getCont(), compoundforbidflag,
@@ -2145,28 +2137,28 @@ int AffixMgr::compound_check_morph(const
hu_mov_rule && (
TESTAFF(rv->astr, 'F', rv->alen) ||
TESTAFF(rv->astr, 'G', rv->alen) ||
TESTAFF(rv->astr, 'H', rv->alen)
)
)
// END of LANG_hu section
)
- && ! (( checkcompoundtriple && // test triple letters
+ && ! (( checkcompoundtriple && !words && // test triple letters
(word[i-1]==word[i]) && (
((i>1) && (word[i-1]==word[i-2])) ||
((word[i-1]==word[i+1])) // may be word[i+1] == '\0'
)
) ||
(
// test CHECKCOMPOUNDPATTERN
- numcheckcpd && cpdpat_check(word, i)
+ numcheckcpd && !words && cpdpat_check(word, i, rv, NULL)
) ||
(
- checkcompoundcase && cpdcase_check(word, i)
+ checkcompoundcase && !words && cpdcase_check(word, i)
))
)
// LANG_hu section: spec. Hungarian rule
|| ((!rv) && (langnum == LANG_hu) && hu_mov_rule && (rv = affix_check(st,i)) &&
(sfx && ((SfxEntry*)sfx)->getCont() && (
TESTAFF(((SfxEntry*)sfx)->getCont(), (unsigned short) 'x', ((SfxEntry*)sfx)->getContLen()) ||
TESTAFF(((SfxEntry*)sfx)->getCont(), (unsigned short) '%', ((SfxEntry*)sfx)->getContLen())
)
@@ -2185,30 +2177,40 @@ int AffixMgr::compound_check_morph(const
}
// END of LANG_hu section
// NEXT WORD(S)
rv_first = rv;
rv = lookup((word+i)); // perhaps without prefix
// search homonym with compound flag
- while ((rv) && ((pseudoroot && TESTAFF(rv->astr, pseudoroot, rv->alen)) ||
+ while ((rv) && ((needaffix && TESTAFF(rv->astr, needaffix, rv->alen)) ||
!((compoundflag && !words && TESTAFF(rv->astr, compoundflag, rv->alen)) ||
(compoundend && !words && TESTAFF(rv->astr, compoundend, rv->alen)) ||
(numdefcpd && defcpd_check(&words, wnum + 1, rv, NULL,1))))) {
rv = rv->next_homonym;
}
if (rv && words && words[wnum + 1]) {
strcat(*result, presult);
- if (complexprefixes && rv->description) strcat(*result, rv->description);
- if (rv->description && ((!rv->astr) ||
- !TESTAFF(rv->astr, lemma_present, rv->alen)))
- strcat(*result, &(rv->word));
- if (!complexprefixes && rv->description) strcat(*result, rv->description);
+ strcat(*result, " ");
+ strcat(*result, MORPH_PART);
+ strcat(*result, word+i);
+ if (complexprefixes && HENTRY_DATA(rv)) strcat(*result, HENTRY_DATA2(rv));
+ if (!HENTRY_FIND(rv, MORPH_STEM)) {
+ strcat(*result, " ");
+ strcat(*result, MORPH_STEM);
+ strcat(*result, HENTRY_WORD(rv));
+ }
+ // store the pointer of the hash entry
+// sprintf(*result + strlen(*result), " %s%p", MORPH_HENTRY, rv);
+ if (!complexprefixes && HENTRY_DATA(rv)) {
+ strcat(*result, " ");
+ strcat(*result, HENTRY_DATA2(rv));
+ }
strcat(*result, "\n");
ok = 1;
return 0;
}
oldnumsyllable2 = numsyllable;
oldwordnum2 = wordnum;
@@ -2223,43 +2225,54 @@ int AffixMgr::compound_check_morph(const
wordnum++;
}
// check forbiddenwords
if ((rv) && (rv->astr) && TESTAFF(rv->astr, forbiddenword, rv->alen)) {
st[i] = ch;
continue;
}
-
+
// second word is acceptable, as a root?
// hungarian conventions: compounding is acceptable,
// when compound forms consist of 2 words, or if more,
// then the syllable number of root words must be 6, or lesser.
if ((rv) && (
(compoundflag && TESTAFF(rv->astr, compoundflag, rv->alen)) ||
(compoundend && TESTAFF(rv->astr, compoundend, rv->alen))
)
&& (
((cpdwordmax==-1) || (wordnum+1<cpdwordmax)) ||
- ((cpdmaxsyllable==0) ||
- (numsyllable+get_syllable(&(rv->word),rv->wlen)<=cpdmaxsyllable))
+ ((cpdmaxsyllable!=0) &&
+ (numsyllable+get_syllable(HENTRY_WORD(rv),rv->blen)<=cpdmaxsyllable))
)
&& (
(!checkcompounddup || (rv != rv_first))
)
)
{
// bad compound word
strcat(*result, presult);
-
- if (rv->description) {
- if (complexprefixes) strcat(*result, rv->description);
- if ((!rv->astr) || !TESTAFF(rv->astr, lemma_present, rv->alen))
- strcat(*result, &(rv->word));
- if (!complexprefixes) strcat(*result, rv->description);
+ strcat(*result, " ");
+ strcat(*result, MORPH_PART);
+ strcat(*result, word+i);
+
+ if (HENTRY_DATA(rv)) {
+ if (complexprefixes) strcat(*result, HENTRY_DATA2(rv));
+ if (! HENTRY_FIND(rv, MORPH_STEM)) {
+ strcat(*result, " ");
+ strcat(*result, MORPH_STEM);
+ strcat(*result, HENTRY_WORD(rv));
+ }
+ // store the pointer of the hash entry
+// sprintf(*result + strlen(*result), " %s%p", MORPH_HENTRY, rv);
+ if (!complexprefixes) {
+ strcat(*result, " ");
+ strcat(*result, HENTRY_DATA2(rv));
+ }
}
strcat(*result, "\n");
ok = 1;
}
numsyllable = oldnumsyllable2 ;
wordnum = oldwordnum2;
@@ -2275,30 +2288,26 @@ int AffixMgr::compound_check_morph(const
rv = affix_check((word+i),strlen(word+i), compoundend);
}
if (!rv && numdefcpd && words) {
rv = affix_check((word+i),strlen(word+i), 0, IN_CPD_END);
if (rv && words && defcpd_check(&words, wnum + 1, rv, NULL, 1)) {
char * m = NULL;
if (compoundflag) m = affix_check_morph((word+i),strlen(word+i), compoundflag);
- if ((!m || *m == '\0') && compoundend)
+ if ((!m || *m == '\0') && compoundend) {
+ if (m) free(m);
m = affix_check_morph((word+i),strlen(word+i), compoundend);
+ }
strcat(*result, presult);
- if (m) {
- line_uniq(m);
- if (strchr(m, '\n')) {
- strcat(*result, "(");
- strcat(*result, line_join(m, '|'));
- strcat(*result, ")");
- } else {
- strcat(*result, m);
- }
- free(m);
+ if (m || (*m != '\0')) {
+ sprintf(*result + strlen(*result), "%c%s%s%s", MSEP_FLD,
+ MORPH_PART, word + i, line_uniq_app(&m, MSEP_REC));
}
+ if (m) free(m);
strcat(*result, "\n");
ok = 1;
}
}
// check non_compound flag in suffix and prefix
if ((rv) &&
((pfx && ((PfxEntry*)pfx)->getCont() &&
@@ -2307,17 +2316,17 @@ int AffixMgr::compound_check_morph(const
(sfx && ((SfxEntry*)sfx)->getCont() &&
TESTAFF(((SfxEntry*)sfx)->getCont(), compoundforbidflag,
((SfxEntry*)sfx)->getContLen())))) {
rv = NULL;
}
// check forbiddenwords
if ((rv) && (rv->astr) && (TESTAFF(rv->astr,forbiddenword,rv->alen))
- && (! TESTAFF(rv->astr, pseudoroot, rv->alen))) {
+ && (! TESTAFF(rv->astr, needaffix, rv->alen))) {
st[i] = ch;
continue;
}
if (langnum == LANG_hu) {
// calculate syllable number of the word
numsyllable += get_syllable(word + i, strlen(word + i));
@@ -2351,39 +2360,35 @@ int AffixMgr::compound_check_morph(const
}
// second word is acceptable, as a word with prefix or/and suffix?
// hungarian conventions: compounding is acceptable,
// when compound forms consist 2 word, otherwise
// the syllable number of root words is 6, or lesser.
if ((rv) &&
(
((cpdwordmax==-1) || (wordnum+1<cpdwordmax)) ||
- ((cpdmaxsyllable==0) ||
+ ((cpdmaxsyllable!=0) &&
(numsyllable <= cpdmaxsyllable))
)
&& (
(!checkcompounddup || (rv != rv_first))
)) {
char * m = NULL;
if (compoundflag) m = affix_check_morph((word+i),strlen(word+i), compoundflag);
- if ((!m || *m == '\0') && compoundend)
+ if ((!m || *m == '\0') && compoundend) {
+ if (m) free(m);
m = affix_check_morph((word+i),strlen(word+i), compoundend);
+ }
strcat(*result, presult);
- if (m) {
- line_uniq(m);
- if (strchr(m, '\n')) {
- strcat(*result, "(");
- strcat(*result, line_join(m, '|'));
- strcat(*result, ")");
- } else {
- strcat(*result, m);
- }
- free(m);
+ if (m && (*m != '\0')) {
+ sprintf(*result + strlen(*result), "%c%s%s%s", MSEP_FLD,
+ MORPH_PART, word + i, line_uniq_app(&m, MSEP_REC));
}
- strcat(*result, "\n");
+ if (m) free(m);
+ sprintf(*result + strlen(*result), "%c", MSEP_REC);
ok = 1;
}
numsyllable = oldnumsyllable2;
wordnum = oldwordnum2;
// perhaps second word is a compound word (recursive call)
if ((wordnum < maxwordnum) && (ok == 0)) {
@@ -2394,17 +2399,16 @@ int AffixMgr::compound_check_morph(const
}
}
st[i] = ch;
wordnum = oldwordnum;
numsyllable = oldnumsyllable;
}
return 0;
}
-#endif // END OF HUNSPELL_EXPERIMENTAL CODE
// return 1 if s1 (reversed) is a leading subset of end of s2
/* inline int AffixMgr::isRevSubset(const char * s1, const char * end_of_s2, int len)
{
while ((len > 0) && *s1 && (*s1 == *end_of_s2)) {
s1++;
end_of_s2--;
len--;
@@ -2425,18 +2429,16 @@ inline int AffixMgr::isRevSubset(const c
// check word for suffixes
struct hentry * AffixMgr::suffix_check (const char * word, int len,
int sfxopts, AffEntry * ppfx, char ** wlst, int maxSug, int * ns,
const FLAG cclass, const FLAG needflag, char in_compound)
{
struct hentry * rv = NULL;
- char result[MAXLNLEN];
-
PfxEntry* ep = (PfxEntry *) ppfx;
// first handle the special case of 0 length suffixes
SfxEntry * se = (SfxEntry *) sStart[0];
while (se) {
if (!cclass || se->getCont()) {
// suffixes are not allowed in beginning of compounds
@@ -2450,21 +2452,21 @@ struct hentry * AffixMgr::suffix_check (
(!se->getCont() || !(TESTAFF(se->getCont(),circumfix,se->getContLen())))) ||
// circumfix flag in prefix AND suffix
((ppfx && (ep->getCont()) && TESTAFF(ep->getCont(),
circumfix, ep->getContLen())) &&
(se->getCont() && (TESTAFF(se->getCont(),circumfix,se->getContLen()))))) &&
// fogemorpheme
(in_compound ||
!((se->getCont() && (TESTAFF(se->getCont(), onlyincompound, se->getContLen()))))) &&
- // pseudoroot on prefix or first suffix
+ // needaffix on prefix or first suffix
(cclass ||
- !(se->getCont() && TESTAFF(se->getCont(), pseudoroot, se->getContLen())) ||
+ !(se->getCont() && TESTAFF(se->getCont(), needaffix, se->getContLen())) ||
(ppfx && !((ep->getCont()) &&
- TESTAFF(ep->getCont(), pseudoroot,
+ TESTAFF(ep->getCont(), needaffix,
ep->getContLen())))
)
) {
rv = se->checkword(word,len, sfxopts, ppfx, wlst, maxSug, ns, (FLAG) cclass,
needflag, (in_compound ? 0 : onlyincompound));
if (rv) {
sfx=(AffEntry *)se; // BUG: sfx not stateless
return rv;
@@ -2492,41 +2494,30 @@ struct hentry * AffixMgr::suffix_check (
(!sptr->getCont() || !(TESTAFF(sptr->getCont(),circumfix,sptr->getContLen())))) ||
// circumfix flag in prefix AND suffix
((ppfx && (ep->getCont()) && TESTAFF(ep->getCont(),
circumfix, ep->getContLen())) &&
(sptr->getCont() && (TESTAFF(sptr->getCont(),circumfix,sptr->getContLen()))))) &&
// fogemorpheme
(in_compound ||
!((sptr->getCont() && (TESTAFF(sptr->getCont(), onlyincompound, sptr->getContLen()))))) &&
- // pseudoroot on prefix or first suffix
+ // needaffix on prefix or first suffix
(cclass ||
- !(sptr->getCont() && TESTAFF(sptr->getCont(), pseudoroot, sptr->getContLen())) ||
+ !(sptr->getCont() && TESTAFF(sptr->getCont(), needaffix, sptr->getContLen())) ||
(ppfx && !((ep->getCont()) &&
- TESTAFF(ep->getCont(), pseudoroot,
+ TESTAFF(ep->getCont(), needaffix,
ep->getContLen())))
)
) {
rv = sptr->checkword(word,len, sfxopts, ppfx, wlst,
maxSug, ns, cclass, needflag, (in_compound ? 0 : onlyincompound));
if (rv) {
sfx=(AffEntry *)sptr; // BUG: sfx not stateless
sfxflag = sptr->getFlag(); // BUG: sfxflag not stateless
if (!sptr->getCont()) sfxappnd=sptr->getKey(); // BUG: sfxappnd not stateless
- if (cclass || sptr->getCont()) {
- if (!derived) {
- derived = mystrdup(word);
- } else {
- strcpy(result, derived); // XXX check size
- strcat(result, "\n");
- strcat(result, word);
- free(derived);
- derived = mystrdup(result);
- }
- }
return rv;
}
}
sptr = sptr->getNextEQ();
} else {
sptr = sptr->getNextNE();
}
}
@@ -2571,17 +2562,16 @@ struct hentry * AffixMgr::suffix_check_t
} else {
sptr = sptr->getNextNE();
}
}
return NULL;
}
-#ifdef HUNSPELL_EXPERIMENTAL
char * AffixMgr::suffix_check_twosfx_morph(const char * word, int len,
int sfxopts, AffEntry * ppfx, const FLAG needflag)
{
char result[MAXLNLEN];
char result2[MAXLNLEN];
char result3[MAXLNLEN];
char * st;
@@ -2593,22 +2583,28 @@ char * AffixMgr::suffix_check_twosfx_mor
// first handle the special case of 0 length suffixes
SfxEntry * se = (SfxEntry *) sStart[0];
while (se) {
if (contclasses[se->getFlag()])
{
st = se->check_twosfx_morph(word,len, sfxopts, ppfx, needflag);
if (st) {
if (ppfx) {
- if (((PfxEntry *) ppfx)->getMorph()) strcat(result, ((PfxEntry *) ppfx)->getMorph());
+ if (((PfxEntry *) ppfx)->getMorph()) {
+ mystrcat(result, ((PfxEntry *) ppfx)->getMorph(), MAXLNLEN);
+ mystrcat(result, " ", MAXLNLEN);
+ } else debugflag(result, ((PfxEntry *) ppfx)->getFlag());
}
- strcat(result, st);
+ mystrcat(result, st, MAXLNLEN);
free(st);
- if (se->getMorph()) strcat(result, se->getMorph());
- strcat(result, "\n");
+ if (se->getMorph()) {
+ mystrcat(result, " ", MAXLNLEN);
+ mystrcat(result, se->getMorph(), MAXLNLEN);
+ } else debugflag(result, se->getFlag());
+ mystrcat(result, "\n", MAXLNLEN);
}
}
se = se->getNext();
}
// now handle the general case
unsigned char sp = *((const unsigned char *)(word + len - 1));
SfxEntry * sptr = (SfxEntry *) sStart[sp];
@@ -2620,37 +2616,32 @@ char * AffixMgr::suffix_check_twosfx_mor
st = sptr->check_twosfx_morph(word,len, sfxopts, ppfx, needflag);
if (st) {
sfxflag = sptr->getFlag(); // BUG: sfxflag not stateless
if (!sptr->getCont()) sfxappnd=sptr->getKey(); // BUG: sfxappnd not stateless
strcpy(result2, st);
free(st);
result3[0] = '\0';
-#ifdef DEBUG
- unsigned short flag = sptr->getFlag();
- if (flag_mode == FLAG_NUM) {
- sprintf(result3, "<%d>", sptr->getKey());
- } else if (flag_mode == FLAG_LONG) {
- sprintf(result3, "<%c%c>", flag >> 8, (flag << 8) >>8);
- } else sprintf(result3, "<%c>", flag);
- strcat(result3, ":");
-#endif
- if (sptr->getMorph()) strcat(result3, sptr->getMorph());
+
+ if (sptr->getMorph()) {
+ mystrcat(result3, " ", MAXLNLEN);
+ mystrcat(result3, sptr->getMorph(), MAXLNLEN);
+ } else debugflag(result3, sptr->getFlag());
strlinecat(result2, result3);
- strcat(result2, "\n");
- strcat(result, result2);
+ mystrcat(result2, "\n", MAXLNLEN);
+ mystrcat(result, result2, MAXLNLEN);
}
}
sptr = sptr->getNextEQ();
} else {
sptr = sptr->getNextNE();
}
}
- if (result) return mystrdup(result);
+ if (*result) return mystrdup(result);
return NULL;
}
char * AffixMgr::suffix_check_morph(const char * word, int len,
int sfxopts, AffEntry * ppfx, const FLAG cclass, const FLAG needflag, char in_compound)
{
char result[MAXLNLEN];
@@ -2675,36 +2666,50 @@ char * AffixMgr::suffix_check_morph(cons
(!se->getCont() || !(TESTAFF(se->getCont(),circumfix,se->getContLen())))) ||
// circumfix flag in prefix AND suffix
((ppfx && (ep->getCont()) && TESTAFF(ep->getCont(),
circumfix, ep->getContLen())) &&
(se->getCont() && (TESTAFF(se->getCont(),circumfix,se->getContLen()))))) &&
// fogemorpheme
(in_compound ||
!((se->getCont() && (TESTAFF(se->getCont(), onlyincompound, se->getContLen()))))) &&
- // pseudoroot on prefix or first suffix
+ // needaffix on prefix or first suffix
(cclass ||
- !(se->getCont() && TESTAFF(se->getCont(), pseudoroot, se->getContLen())) ||
+ !(se->getCont() && TESTAFF(se->getCont(), needaffix, se->getContLen())) ||
(ppfx && !((ep->getCont()) &&
- TESTAFF(ep->getCont(), pseudoroot,
+ TESTAFF(ep->getCont(), needaffix,
ep->getContLen())))
)
))
rv = se->checkword(word,len, sfxopts, ppfx, NULL, 0, 0, cclass, needflag);
while (rv) {
if (ppfx) {
- if (((PfxEntry *) ppfx)->getMorph()) strcat(result, ((PfxEntry *) ppfx)->getMorph());
+ if (((PfxEntry *) ppfx)->getMorph()) {
+ mystrcat(result, ((PfxEntry *) ppfx)->getMorph(), MAXLNLEN);
+ mystrcat(result, " ", MAXLNLEN);
+ } else debugflag(result, ((PfxEntry *) ppfx)->getFlag());
+ }
+ if (complexprefixes && HENTRY_DATA(rv)) mystrcat(result, HENTRY_DATA2(rv), MAXLNLEN);
+ if (! HENTRY_FIND(rv, MORPH_STEM)) {
+ mystrcat(result, " ", MAXLNLEN);
+ mystrcat(result, MORPH_STEM, MAXLNLEN);
+ mystrcat(result, HENTRY_WORD(rv), MAXLNLEN);
}
- if (complexprefixes && rv->description) strcat(result, rv->description);
- if (rv->description && ((!rv->astr) ||
- !TESTAFF(rv->astr, lemma_present, rv->alen)))
- strcat(result, &(rv->word));
- if (!complexprefixes && rv->description) strcat(result, rv->description);
- if (se->getMorph()) strcat(result, se->getMorph());
- strcat(result, "\n");
+ // store the pointer of the hash entry
+// sprintf(result + strlen(result), " %s%p", MORPH_HENTRY, rv);
+
+ if (!complexprefixes && HENTRY_DATA(rv)) {
+ mystrcat(result, " ", MAXLNLEN);
+ mystrcat(result, HENTRY_DATA2(rv), MAXLNLEN);
+ }
+ if (se->getMorph()) {
+ mystrcat(result, " ", MAXLNLEN);
+ mystrcat(result, se->getMorph(), MAXLNLEN);
+ } else debugflag(result, se->getFlag());
+ mystrcat(result, "\n", MAXLNLEN);
rv = se->get_next_homonym(rv, sfxopts, ppfx, cclass, needflag);
}
}
se = se->getNext();
}
// now handle the general case
unsigned char sp = *((const unsigned char *)(word + len - 1));
@@ -2724,60 +2729,62 @@ char * AffixMgr::suffix_check_morph(cons
(!sptr->getCont() || !(TESTAFF(sptr->getCont(),circumfix,sptr->getContLen())))) ||
// circumfix flag in prefix AND suffix
((ppfx && (ep->getCont()) && TESTAFF(ep->getCont(),
circumfix, ep->getContLen())) &&
(sptr->getCont() && (TESTAFF(sptr->getCont(),circumfix,sptr->getContLen()))))) &&
// fogemorpheme
(in_compound ||
!((sptr->getCont() && (TESTAFF(sptr->getCont(), onlyincompound, sptr->getContLen()))))) &&
- // pseudoroot on first suffix
+ // needaffix on first suffix
(cclass || !(sptr->getCont() &&
- TESTAFF(sptr->getCont(), pseudoroot, sptr->getContLen())))
+ TESTAFF(sptr->getCont(), needaffix, sptr->getContLen())))
)) rv = sptr->checkword(word,len, sfxopts, ppfx, NULL, 0, 0, cclass, needflag);
while (rv) {
if (ppfx) {
- if (((PfxEntry *) ppfx)->getMorph()) strcat(result, ((PfxEntry *) ppfx)->getMorph());
+ if (((PfxEntry *) ppfx)->getMorph()) {
+ mystrcat(result, ((PfxEntry *) ppfx)->getMorph(), MAXLNLEN);
+ mystrcat(result, " ", MAXLNLEN);
+ } else debugflag(result, ((PfxEntry *) ppfx)->getFlag());
}
- if (complexprefixes && rv->description) strcat(result, rv->description);
- if (rv->description && ((!rv->astr) ||
- !TESTAFF(rv->astr, lemma_present, rv->alen))) strcat(result, &(rv->word));
- if (!complexprefixes && rv->description) strcat(result, rv->description);
-#ifdef DEBUG
- unsigned short flag = sptr->getFlag();
- if (flag_mode == FLAG_NUM) {
- sprintf(result, "<%d>", sptr->getKey());
- } else if (flag_mode == FLAG_LONG) {
- sprintf(result, "<%c%c>", flag >> 8, (flag << 8) >>8);
- } else sprintf(result, "<%c>", flag);
- strcat(result, ":");
-#endif
-
- if (sptr->getMorph()) strcat(result, sptr->getMorph());
- strcat(result, "\n");
+ if (complexprefixes && HENTRY_DATA(rv)) mystrcat(result, HENTRY_DATA2(rv), MAXLNLEN);
+ if (! HENTRY_FIND(rv, MORPH_STEM)) {
+ mystrcat(result, " ", MAXLNLEN);
+ mystrcat(result, MORPH_STEM, MAXLNLEN);
+ mystrcat(result, HENTRY_WORD(rv), MAXLNLEN);
+ }
+ // store the pointer of the hash entry
+// sprintf(result + strlen(result), " %s%p", MORPH_HENTRY, rv);
+
+ if (!complexprefixes && HENTRY_DATA(rv)) {
+ mystrcat(result, " ", MAXLNLEN);
+ mystrcat(result, HENTRY_DATA2(rv), MAXLNLEN);
+ }
+
+ if (sptr->getMorph()) {
+ mystrcat(result, " ", MAXLNLEN);
+ mystrcat(result, sptr->getMorph(), MAXLNLEN);
+ } else debugflag(result, sptr->getFlag());
+ mystrcat(result, "\n", MAXLNLEN);
rv = sptr->get_next_homonym(rv, sfxopts, ppfx, cclass, needflag);
}
sptr = sptr->getNextEQ();
} else {
sptr = sptr->getNextNE();
}
}
if (*result) return mystrdup(result);
return NULL;
}
-#endif // END OF HUNSPELL_EXPERIMENTAL CODE
-
// check if word with affixes is correctly spelled
struct hentry * AffixMgr::affix_check (const char * word, int len, const FLAG needflag, char in_compound)
{
struct hentry * rv= NULL;
- if (derived) free(derived);
- derived = NULL;
// check all prefixes (also crossed with suffixes if allowed)
rv = prefix_check(word, len, in_compound, needflag);
if (rv) return rv;
// if still not found check all suffixes
rv = suffix_check(word, len, 0, NULL, NULL, 0, NULL, FLAG_NULL, needflag, in_compound);
@@ -2789,115 +2796,194 @@ struct hentry * AffixMgr::affix_check (c
rv = suffix_check_twosfx(word, len, 0, NULL, needflag);
if (rv) return rv;
// if still not found check all two-level suffixes
rv = prefix_check_twosfx(word, len, IN_CPD_NOT, needflag);
}
return rv;
}
-#ifdef HUNSPELL_EXPERIMENTAL
// check if word with affixes is correctly spelled
char * AffixMgr::affix_check_morph(const char * word, int len, const FLAG needflag, char in_compound)
{
char result[MAXLNLEN];
char * st = NULL;
*result = '\0';
// check all prefixes (also crossed with suffixes if allowed)
st = prefix_check_morph(word, len, in_compound);
if (st) {
- strcat(result, st);
+ mystrcat(result, st, MAXLNLEN);
free(st);
}
// if still not found check all suffixes
st = suffix_check_morph(word, len, 0, NULL, '\0', needflag, in_compound);
if (st) {
- strcat(result, st);
+ mystrcat(result, st, MAXLNLEN);
free(st);
}
if (havecontclass) {
sfx = NULL;
pfx = NULL;
// if still not found check all two-level suffixes
st = suffix_check_twosfx_morph(word, len, 0, NULL, needflag);
if (st) {
- strcat(result, st);
+ mystrcat(result, st, MAXLNLEN);
free(st);
}
// if still not found check all two-level suffixes
st = prefix_check_twosfx_morph(word, len, IN_CPD_NOT, needflag);
if (st) {
- strcat(result, st);
+ mystrcat(result, st, MAXLNLEN);
free(st);
}
}
-
+
return mystrdup(result);
}
-#endif // END OF HUNSPELL_EXPERIMENTAL CODE
+
+char * AffixMgr::morphgen(char * ts, int wl, const unsigned short * ap,
+ unsigned short al, char * morph, char * targetmorph, int level)
+{
+ // handle suffixes
+ char * stemmorph;
+ char * stemmorphcatpos;
+ char mymorph[MAXLNLEN];
+
+ if (!morph && !targetmorph) return NULL;
+
+ // check substandard flag
+ if (TESTAFF(ap, substandard, al)) return NULL;
+
+ if (morphcmp(morph, targetmorph) == 0) return mystrdup(ts);
+
+// int targetcount = get_sfxcount(targetmorph);
+
+ // use input suffix fields, if exist
+ if (strstr(morph, MORPH_INFL_SFX) || strstr(morph, MORPH_DERI_SFX)) {
+ stemmorph = mymorph;
+ strcpy(stemmorph, morph);
+ strcat(stemmorph, " ");
+ stemmorphcatpos = stemmorph + strlen(stemmorph);
+ } else {
+ stemmorph = morph;
+ stemmorphcatpos = NULL;
+ }
+
+ for (int i = 0; i < al; i++) {
+ const unsigned char c = (unsigned char) (ap[i] & 0x00FF);
+ SfxEntry * sptr = (SfxEntry *)sFlag[c];
+ while (sptr) {
+ if (sptr->getFlag() == ap[i] && sptr->getMorph() && ((sptr->getContLen() == 0) ||
+ // don't generate forms with substandard affixes
+ !TESTAFF(sptr->getCont(), substandard, sptr->getContLen()))) {
+
+ if (stemmorphcatpos) strcpy(stemmorphcatpos, sptr->getMorph());
+ else stemmorph = (char *) sptr->getMorph();
+
+ int cmp = morphcmp(stemmorph, targetmorph);
+
+ if (cmp == 0) {
+ char * newword = sptr->add(ts, wl);
+ if (newword) {
+ hentry * check = pHMgr->lookup(newword); // XXX extra dic
+ if (!check || !check->astr ||
+ !TESTAFF(check->astr, forbiddenword, check->alen)) {
+ return newword;
+ }
+ free(newword);
+ }
+ }
+
+ // recursive call for secondary suffixes
+ if ((level == 0) && (cmp == 1) && (sptr->getContLen() > 0) &&
+// (get_sfxcount(stemmorph) < targetcount) &&
+ !TESTAFF(sptr->getCont(), substandard, sptr->getContLen())) {
+ char * newword = sptr->add(ts, wl);
+ if (newword) {
+ char * newword2 = morphgen(newword, strlen(newword), sptr->getCont(),
+ sptr->getContLen(), stemmorph, targetmorph, 1);
+
+ if (newword2) {
+ free(newword);
+ return newword2;
+ }
+ free(newword);
+ newword = NULL;
+ }
+ }
+ }
+ sptr = (SfxEntry *)sptr ->getFlgNxt();
+ }
+ }
+ return NULL;
+}
int AffixMgr::expand_rootword(struct guessword * wlst, int maxn, const char * ts,
int wl, const unsigned short * ap, unsigned short al, char * bad, int badl,
- char * phone)
+ char * phon)
{
-
int nh=0;
// first add root word to list
- if ((nh < maxn) && !(al && ((pseudoroot && TESTAFF(ap, pseudoroot, al)) ||
+ if ((nh < maxn) && !(al && ((needaffix && TESTAFF(ap, needaffix, al)) ||
(onlyincompound && TESTAFF(ap, onlyincompound, al))))) {
wlst[nh].word = mystrdup(ts);
+ if (!wlst[nh].word) return 0;
wlst[nh].allow = (1 == 0);
wlst[nh].orig = NULL;
nh++;
// add special phonetic version
- if (phone && (nh < maxn)) {
- wlst[nh].word = mystrdup(phone);
+ if (phon && (nh < maxn)) {
+ wlst[nh].word = mystrdup(phon);
+ if (!wlst[nh].word) return nh - 1;
wlst[nh].allow = (1 == 0);
wlst[nh].orig = mystrdup(ts);
+ if (!wlst[nh].orig) return nh - 1;
nh++;
}
}
// handle suffixes
for (int i = 0; i < al; i++) {
const unsigned char c = (unsigned char) (ap[i] & 0x00FF);
SfxEntry * sptr = (SfxEntry *)sFlag[c];
while (sptr) {
if ((sptr->getFlag() == ap[i]) && (!sptr->getKeyLen() || ((badl > sptr->getKeyLen()) &&
(strcmp(sptr->getAffix(), bad + badl - sptr->getKeyLen()) == 0))) &&
- // check pseudoroot flag
- !(sptr->getCont() && ((pseudoroot &&
- TESTAFF(sptr->getCont(), pseudoroot, sptr->getContLen())) ||
+ // check needaffix flag
+ !(sptr->getCont() && ((needaffix &&
+ TESTAFF(sptr->getCont(), needaffix, sptr->getContLen())) ||
(circumfix &&
TESTAFF(sptr->getCont(), circumfix, sptr->getContLen())) ||
(onlyincompound &&
TESTAFF(sptr->getCont(), onlyincompound, sptr->getContLen()))))
) {
char * newword = sptr->add(ts, wl);
if (newword) {
if (nh < maxn) {
wlst[nh].word = newword;
wlst[nh].allow = sptr->allowCross();
wlst[nh].orig = NULL;
nh++;
// add special phonetic version
- if (phone && (nh < maxn)) {
+ if (phon && (nh < maxn)) {
char st[MAXWORDUTF8LEN];
- strcpy(st, phone);
+ strcpy(st, phon);
strcat(st, sptr->getKey());
- reverseword(st + strlen(phone));
+ reverseword(st + strlen(phon));
wlst[nh].word = mystrdup(st);
+ if (!wlst[nh].word) return nh - 1;
wlst[nh].allow = (1 == 0);
wlst[nh].orig = mystrdup(newword);
+ if (!wlst[nh].orig) return nh - 1;
nh++;
}
} else {
free(newword);
}
}
}
sptr = (SfxEntry *)sptr ->getFlgNxt();
@@ -2905,28 +2991,28 @@ int AffixMgr::expand_rootword(struct gue
}
int n = nh;
// handle cross products of prefixes and suffixes
for (int j=1;j<n ;j++)
if (wlst[j].allow) {
for (int k = 0; k < al; k++) {
- const unsigned char c = (unsigned char) (ap[k] & 0x00FF);
+ const unsigned char c = (unsigned char) (ap[k] & 0x00FF);
PfxEntry * cptr = (PfxEntry *) pFlag[c];
while (cptr) {
if ((cptr->getFlag() == ap[k]) && cptr->allowCross() && (!cptr->getKeyLen() || ((badl > cptr->getKeyLen()) &&
(strncmp(cptr->getKey(), bad, cptr->getKeyLen()) == 0)))) {
int l1 = strlen(wlst[j].word);
char * newword = cptr->add(wlst[j].word, l1);
if (newword) {
if (nh < maxn) {
wlst[nh].word = newword;
wlst[nh].allow = cptr->allowCross();
- wlst[nh].orig = NULL;
+ wlst[nh].orig = NULL;
nh++;
} else {
free(newword);
}
}
}
cptr = (PfxEntry *)cptr ->getFlgNxt();
}
@@ -2936,19 +3022,19 @@ int AffixMgr::expand_rootword(struct gue
// now handle pure prefixes
for (int m = 0; m < al; m ++) {
const unsigned char c = (unsigned char) (ap[m] & 0x00FF);
PfxEntry * ptr = (PfxEntry *) pFlag[c];
while (ptr) {
if ((ptr->getFlag() == ap[m]) && (!ptr->getKeyLen() || ((badl > ptr->getKeyLen()) &&
(strncmp(ptr->getKey(), bad, ptr->getKeyLen()) == 0))) &&
- // check pseudoroot flag
- !(ptr->getCont() && ((pseudoroot &&
- TESTAFF(ptr->getCont(), pseudoroot, ptr->getContLen())) ||
+ // check needaffix flag
+ !(ptr->getCont() && ((needaffix &&
+ TESTAFF(ptr->getCont(), needaffix, ptr->getContLen())) ||
(circumfix &&
TESTAFF(ptr->getCont(), circumfix, ptr->getContLen())) ||
(onlyincompound &&
TESTAFF(ptr->getCont(), onlyincompound, ptr->getContLen()))))
) {
char * newword = ptr->add(ts, wl);
if (newword) {
if (nh < maxn) {
@@ -2963,31 +3049,43 @@ int AffixMgr::expand_rootword(struct gue
}
ptr = (PfxEntry *)ptr ->getFlgNxt();
}
}
return nh;
}
-
-
// return length of replacing table
int AffixMgr::get_numrep()
{
return numrep;
}
// return replacing table
struct replentry * AffixMgr::get_reptable()
{
if (! reptable ) return NULL;
return reptable;
}
+// return iconv table
+RepList * AffixMgr::get_iconvtable()
+{
+ if (! iconvtable ) return NULL;
+ return iconvtable;
+}
+
+// return oconv table
+RepList * AffixMgr::get_oconvtable()
+{
+ if (! oconvtable ) return NULL;
+ return oconvtable;
+}
+
// return replacing table
struct phonetable * AffixMgr::get_phonetable()
{
if (! phone ) return NULL;
return phone;
}
// return length of character map table
@@ -3014,44 +3112,54 @@ char ** AffixMgr::get_breaktable()
{
if (! breaktable ) return NULL;
return breaktable;
}
// return text encoding of dictionary
char * AffixMgr::get_encoding()
{
- if (! encoding ) {
- encoding = mystrdup("ISO8859-1");
- }
+ if (! encoding ) encoding = mystrdup(SPELL_ENCODING);
return mystrdup(encoding);
}
// return text encoding of dictionary
int AffixMgr::get_langnum()
{
return langnum;
}
// return double prefix option
int AffixMgr::get_complexprefixes()
{
return complexprefixes;
}
+// return FULLSTRIP option
+int AffixMgr::get_fullstrip()
+{
+ return fullstrip;
+}
+
FLAG AffixMgr::get_keepcase()
{
return keepcase;
}
int AffixMgr::get_checksharps()
{
return checksharps;
}
+char * AffixMgr::encode_flag(unsigned short aflag)
+{
+ return pHMgr->encode_flag(aflag);
+}
+
+
// return the preferred ignore string for suggestions
char * AffixMgr::get_ignore()
{
if (!ignorechars) return NULL;
return ignorechars;
}
// return the preferred ignore string for suggestions
@@ -3059,17 +3167,17 @@ unsigned short * AffixMgr::get_ignore_ut
{
*len = ignorechars_utf16_len;
return ignorechars_utf16;
}
// return the keyboard string for suggestions
char * AffixMgr::get_key_string()
{
- if (! keystring ) return NULL;
+ if (! keystring ) keystring = mystrdup(SPELL_KEYSTRING);
return mystrdup(keystring);
}
// return the preferred try string for suggestions
char * AffixMgr::get_try_string()
{
if (! trystring ) return NULL;
return mystrdup(trystring);
@@ -3107,19 +3215,19 @@ FLAG AffixMgr::get_forbiddenword()
// return the forbidden words control flag
FLAG AffixMgr::get_nosuggest()
{
return nosuggest;
}
// return the forbidden words flag modify flag
-FLAG AffixMgr::get_pseudoroot()
+FLAG AffixMgr::get_needaffix()
{
- return pseudoroot;
+ return needaffix;
}
// return the onlyincompound flag
FLAG AffixMgr::get_onlyincompound()
{
return onlyincompound;
}
@@ -3149,39 +3257,37 @@ const char * AffixMgr::get_prefix()
}
// return the value of suffix
const char * AffixMgr::get_suffix()
{
return sfxappnd;
}
-// return the value of derived form (base word with first suffix).
-const char * AffixMgr::get_derived()
-{
- return derived;
-}
-
// return the value of suffix
const char * AffixMgr::get_version()
{
return version;
}
// return lemma_present flag
FLAG AffixMgr::get_lemma_present()
{
return lemma_present;
}
// utility method to look up root words in hash table
struct hentry * AffixMgr::lookup(const char * word)
{
- if (! pHMgr) return NULL;
- return pHMgr->lookup(word);
+ int i;
+ struct hentry * he = NULL;
+ for (i = 0; i < *maxdic && !he; i++) {
+ he = (alldic[i])->lookup(word);
+ }
+ return he;
}
// return the value of suffix
const int AffixMgr::have_contclass()
{
return havecontclass;
}
@@ -3205,43 +3311,43 @@ int AffixMgr::get_nosplitsugs(void)
// return sugswithdots
int AffixMgr::get_sugswithdots(void)
{
return sugswithdots;
}
/* parse flag */
-int AffixMgr::parse_flag(char * line, unsigned short * out, const char * name) {
+int AffixMgr::parse_flag(char * line, unsigned short * out, FileMgr * af) {
char * s = NULL;
- if (*out != FLAG_NULL) {
- HUNSPELL_WARNING(stderr, "error: duplicate %s line\n", name);
+ if (*out != FLAG_NULL && !(*out >= DEFAULTFLAGS)) {
+ HUNSPELL_WARNING(stderr, "error: line %d: multiple definitions of an affix file parameter\n", af->getlinenum());
return 1;
}
- if (parse_string(line, &s, name)) return 1;
+ if (parse_string(line, &s, af->getlinenum())) return 1;
*out = pHMgr->decode_flag(s);
free(s);
return 0;
}
/* parse num */
-int AffixMgr::parse_num(char * line, int * out, const char * name) {
+int AffixMgr::parse_num(char * line, int * out, FileMgr * af) {
char * s = NULL;
if (*out != -1) {
- HUNSPELL_WARNING(stderr, "error: duplicate %s line\n", name);
+ HUNSPELL_WARNING(stderr, "error: line %d: multiple definitions of an affix file parameter\n", af->getlinenum());
return 1;
}
- if (parse_string(line, &s, name)) return 1;
+ if (parse_string(line, &s, af->getlinenum())) return 1;
*out = atoi(s);
free(s);
return 0;
}
/* parse in the max syllablecount of compound words and */
-int AffixMgr::parse_cpdsyllable(char * line)
+int AffixMgr::parse_cpdsyllable(char * line, FileMgr * af)
{
char * tp = line;
char * piece;
int i = 0;
int np = 0;
w_char w[MAXWORDLEN];
piece = mystrsep(&tp, 0);
while (piece) {
@@ -3264,415 +3370,516 @@ int AffixMgr::parse_cpdsyllable(char *
}
np++;
break;
}
default: break;
}
i++;
}
- free(piece);
piece = mystrsep(&tp, 0);
}
if (np < 2) {
- HUNSPELL_WARNING(stderr, "error: missing compoundsyllable information\n");
+ HUNSPELL_WARNING(stderr, "error: line %d: missing compoundsyllable information\n", af->getlinenum());
return 1;
}
if (np == 2) cpdvowels = mystrdup("aeiouAEIOU");
return 0;
}
/* parse in the typical fault correcting table */
-int AffixMgr::parse_reptable(char * line, FILE * af)
+int AffixMgr::parse_reptable(char * line, FileMgr * af)
{
if (numrep != 0) {
- HUNSPELL_WARNING(stderr, "error: duplicate REP tables used\n");
+ HUNSPELL_WARNING(stderr, "error: line %d: multiple table definitions\n", af->getlinenum());
return 1;
}
char * tp = line;
char * piece;
int i = 0;
int np = 0;
piece = mystrsep(&tp, 0);
while (piece) {
if (*piece != '\0') {
switch(i) {
case 0: { np++; break; }
case 1: {
numrep = atoi(piece);
if (numrep < 1) {
- HUNSPELL_WARNING(stderr, "incorrect number of entries in replacement table\n");
- free(piece);
+ HUNSPELL_WARNING(stderr, "error: line %d: incorrect entry number\n", af->getlinenum());
return 1;
}
reptable = (replentry *) malloc(numrep * sizeof(struct replentry));
if (!reptable) return 1;
np++;
break;
}
default: break;
}
i++;
}
- free(piece);
piece = mystrsep(&tp, 0);
}
if (np != 2) {
- HUNSPELL_WARNING(stderr, "error: missing replacement table information\n");
+ HUNSPELL_WARNING(stderr, "error: line %d: missing data\n", af->getlinenum());
return 1;
}
/* now parse the numrep lines to read in the remainder of the table */
- char * nl = line;
+ char * nl;
for (int j=0; j < numrep; j++) {
- if (!fgets(nl,MAXLNLEN,af)) return 1;
+ if (!(nl = af->getline())) return 1;
mychomp(nl);
tp = nl;
i = 0;
reptable[j].pattern = NULL;
reptable[j].pattern2 = NULL;
piece = mystrsep(&tp, 0);
while (piece) {
if (*piece != '\0') {
switch(i) {
case 0: {
if (strncmp(piece,"REP",3) != 0) {
- HUNSPELL_WARNING(stderr, "error: replacement table is corrupt\n");
+ HUNSPELL_WARNING(stderr, "error: line %d: table is corrupt\n", af->getlinenum());
numrep = 0;
- free(piece);
return 1;
}
break;
}
case 1: { reptable[j].pattern = mystrrep(mystrdup(piece),"_"," "); break; }
case 2: { reptable[j].pattern2 = mystrrep(mystrdup(piece),"_"," "); break; }
default: break;
}
i++;
}
- free(piece);
piece = mystrsep(&tp, 0);
}
if ((!(reptable[j].pattern)) || (!(reptable[j].pattern2))) {
- HUNSPELL_WARNING(stderr, "error: replacement table is corrupt\n");
+ HUNSPELL_WARNING(stderr, "error: line %d: table is corrupt\n", af->getlinenum());
numrep = 0;
return 1;
}
}
return 0;
}
/* parse in the typical fault correcting table */
-int AffixMgr::parse_phonetable(char * line, FILE * af)
+int AffixMgr::parse_convtable(char * line, FileMgr * af, RepList ** rl, const char * keyword)
+{
+ if (*rl) {
+ HUNSPELL_WARNING(stderr, "error: line %d: multiple table definitions\n", af->getlinenum());
+ return 1;
+ }
+ char * tp = line;
+ char * piece;
+ int i = 0;
+ int np = 0;
+ int numrl = 0;
+ piece = mystrsep(&tp, 0);
+ while (piece) {
+ if (*piece != '\0') {
+ switch(i) {
+ case 0: { np++; break; }
+ case 1: {
+ numrl = atoi(piece);
+ if (numrl < 1) {
+ HUNSPELL_WARNING(stderr, "error: line %d: incorrect entry number\n", af->getlinenum());
+ return 1;
+ }
+ *rl = new RepList(numrl);
+ if (!rl) return 1;
+ np++;
+ break;
+ }
+ default: break;
+ }
+ i++;
+ }
+ piece = mystrsep(&tp, 0);
+ }
+ if (np != 2) {
+ HUNSPELL_WARNING(stderr, "error: line %d: missing data\n", af->getlinenum());
+ return 1;
+ }
+
+ /* now parse the num lines to read in the remainder of the table */
+ char * nl;
+ for (int j=0; j < numrl; j++) {
+ if (!(nl = af->getline())) return 1;
+ mychomp(nl);
+ tp = nl;
+ i = 0;
+ char * pattern = NULL;
+ char * pattern2 = NULL;
+ piece = mystrsep(&tp, 0);
+ while (piece) {
+ if (*piece != '\0') {
+ switch(i) {
+ case 0: {
+ if (strncmp(piece, keyword, sizeof(keyword)) != 0) {
+ HUNSPELL_WARNING(stderr, "error: line %d: table is corrupt\n", af->getlinenum());
+ delete *rl;
+ *rl = NULL;
+ return 1;
+ }
+ break;
+ }
+ case 1: { pattern = mystrrep(mystrdup(piece),"_"," "); break; }
+ case 2: {
+ pattern2 = mystrrep(mystrdup(piece),"_"," ");
+ break;
+ }
+ default: break;
+ }
+ i++;
+ }
+ piece = mystrsep(&tp, 0);
+ }
+ if (!pattern || !pattern2) {
+ HUNSPELL_WARNING(stderr, "error: line %d: table is corrupt\n", af->getlinenum());
+ return 1;
+ }
+ (*rl)->add(pattern, pattern2);
+ }
+ return 0;
+}
+
+
+/* parse in the typical fault correcting table */
+int AffixMgr::parse_phonetable(char * line, FileMgr * af)
{
if (phone) {
- HUNSPELL_WARNING(stderr, "error: duplicate PHONE tables used\n");
+ HUNSPELL_WARNING(stderr, "error: line %d: multiple table definitions\n", af->getlinenum());
return 1;
}
char * tp = line;
char * piece;
int i = 0;
int np = 0;
piece = mystrsep(&tp, 0);
while (piece) {
if (*piece != '\0') {
switch(i) {
case 0: { np++; break; }
case 1: {
- phone = (phonetable *) malloc(sizeof(struct phonetable));
- phone->num = atoi(piece);
- phone->rules = NULL;
- phone->utf8 = utf8;
+ phone = (phonetable *) malloc(sizeof(struct phonetable));
+ phone->num = atoi(piece);
+ phone->rules = NULL;
+ phone->utf8 = (char) utf8;
if (!phone) return 1;
if (phone->num < 1) {
- HUNSPELL_WARNING(stderr, "incorrect number of entries in phonelacement table\n");
- free(piece);
+ HUNSPELL_WARNING(stderr, "error: line %d: bad entry number\n", af->getlinenum());
return 1;
}
phone->rules = (char * *) malloc(2 * (phone->num + 1) * sizeof(char *));
if (!phone->rules) return 1;
np++;
break;
}
default: break;
}
i++;
}
- free(piece);
piece = mystrsep(&tp, 0);
}
if (np != 2) {
- HUNSPELL_WARNING(stderr, "error: missing PHONE table information\n");
+ HUNSPELL_WARNING(stderr, "error: line %d: missing data\n", af->getlinenum());
return 1;
}
/* now parse the phone->num lines to read in the remainder of the table */
- char * nl = line;
+ char * nl;
for (int j=0; j < phone->num; j++) {
- if (!fgets(nl,MAXLNLEN,af)) return 1;
+ if (!(nl = af->getline())) return 1;
mychomp(nl);
tp = nl;
i = 0;
phone->rules[j * 2] = NULL;
phone->rules[j * 2 + 1] = NULL;
piece = mystrsep(&tp, 0);
while (piece) {
if (*piece != '\0') {
switch(i) {
case 0: {
if (strncmp(piece,"PHONE",5) != 0) {
- HUNSPELL_WARNING(stderr, "error: PHONE table is corrupt\n");
+ HUNSPELL_WARNING(stderr, "error: line %d: table is corrupt\n", af->getlinenum());
phone->num = 0;
- free(piece);
return 1;
}
break;
}
case 1: { phone->rules[j * 2] = mystrrep(mystrdup(piece),"_",""); break; }
case 2: { phone->rules[j * 2 + 1] = mystrrep(mystrdup(piece),"_",""); break; }
default: break;
}
i++;
}
- free(piece);
piece = mystrsep(&tp, 0);
}
if ((!(phone->rules[j * 2])) || (!(phone->rules[j * 2 + 1]))) {
- HUNSPELL_WARNING(stderr, "error: PHONE table is corrupt\n");
+ HUNSPELL_WARNING(stderr, "error: line %d: table is corrupt\n", af->getlinenum());
phone->num = 0;
return 1;
}
}
phone->rules[phone->num * 2] = mystrdup("");
phone->rules[phone->num * 2 + 1] = mystrdup("");
init_phonet_hash(*phone);
return 0;
}
/* parse in the checkcompoundpattern table */
-int AffixMgr::parse_checkcpdtable(char * line, FILE * af)
+int AffixMgr::parse_checkcpdtable(char * line, FileMgr * af)
{
if (numcheckcpd != 0) {
- HUNSPELL_WARNING(stderr, "error: duplicate compound pattern tables used\n");
+ HUNSPELL_WARNING(stderr, "error: line %d: multiple table definitions\n", af->getlinenum());
return 1;
}
char * tp = line;
char * piece;
int i = 0;
int np = 0;
piece = mystrsep(&tp, 0);
while (piece) {
if (*piece != '\0') {
switch(i) {
case 0: { np++; break; }
case 1: {
numcheckcpd = atoi(piece);
if (numcheckcpd < 1) {
- HUNSPELL_WARNING(stderr, "incorrect number of entries in compound pattern table\n");
- free(piece);
+ HUNSPELL_WARNING(stderr, "error: line %d: bad entry number\n", af->getlinenum());
return 1;
}
- checkcpdtable = (replentry *) malloc(numcheckcpd * sizeof(struct replentry));
+ checkcpdtable = (patentry *) malloc(numcheckcpd * sizeof(struct patentry));
if (!checkcpdtable) return 1;
np++;
break;
}
default: break;
}
i++;
}
- free(piece);
piece = mystrsep(&tp, 0);
}
if (np != 2) {
- HUNSPELL_WARNING(stderr, "error: missing compound pattern table information\n");
+ HUNSPELL_WARNING(stderr, "error: line %d: missing data\n", af->getlinenum());
return 1;
- }
-
+ }
+
/* now parse the numcheckcpd lines to read in the remainder of the table */
- char * nl = line;
+ char * nl;
for (int j=0; j < numcheckcpd; j++) {
- if (!fgets(nl,MAXLNLEN,af)) return 1;
+ if (!(nl = af->getline())) return 1;
mychomp(nl);
tp = nl;
i = 0;
checkcpdtable[j].pattern = NULL;
checkcpdtable[j].pattern2 = NULL;
+ checkcpdtable[j].pattern3 = NULL;
+ checkcpdtable[j].cond = FLAG_NULL;
+ checkcpdtable[j].cond2 = FLAG_NULL;
piece = mystrsep(&tp, 0);
while (piece) {
if (*piece != '\0') {
switch(i) {
case 0: {
if (strncmp(piece,"CHECKCOMPOUNDPATTERN",20) != 0) {
- HUNSPELL_WARNING(stderr, "error: compound pattern table is corrupt\n");
+ HUNSPELL_WARNING(stderr, "error: line %d: table is corrupt\n", af->getlinenum());
numcheckcpd = 0;
- free(piece);
return 1;
}
break;
}
- case 1: { checkcpdtable[j].pattern = mystrdup(piece); break; }
- case 2: { checkcpdtable[j].pattern2 = mystrdup(piece); break; }
+ case 1: {
+ checkcpdtable[j].pattern = mystrdup(piece);
+ char * p = strchr(checkcpdtable[j].pattern, '/');
+ if (p) {
+ *p = '\0';
+ checkcpdtable[j].cond = pHMgr->decode_flag(p + 1);
+ }
+ break; }
+ case 2: {
+ checkcpdtable[j].pattern2 = mystrdup(piece);
+ char * p = strchr(checkcpdtable[j].pattern2, '/');
+ if (p) {
+ *p = '\0';
+ checkcpdtable[j].cond2 = pHMgr->decode_flag(p + 1);
+ }
+ break;
+ }
+ case 3: { checkcpdtable[j].pattern3 = mystrdup(piece); simplifiedcpd = 1; break; }
default: break;
}
i++;
}
- free(piece);
piece = mystrsep(&tp, 0);
}
if ((!(checkcpdtable[j].pattern)) || (!(checkcpdtable[j].pattern2))) {
- HUNSPELL_WARNING(stderr, "error: compound pattern table is corrupt\n");
- numcheckcpd = 0;
- return 1;
+ HUNSPELL_WARNING(stderr, "error: line %d: table is corrupt\n", af->getlinenum());
+ numcheckcpd = 0;
+ return 1;
}
}
return 0;
}
/* parse in the compound rule table */
-int AffixMgr::parse_defcpdtable(char * line, FILE * af)
+int AffixMgr::parse_defcpdtable(char * line, FileMgr * af)
{
if (numdefcpd != 0) {
- HUNSPELL_WARNING(stderr, "error: duplicate compound rule tables used\n");
+ HUNSPELL_WARNING(stderr, "error: line %d: multiple table definitions\n", af->getlinenum());
return 1;
}
char * tp = line;
char * piece;
int i = 0;
int np = 0;
piece = mystrsep(&tp, 0);
while (piece) {
if (*piece != '\0') {
switch(i) {
case 0: { np++; break; }
case 1: {
numdefcpd = atoi(piece);
if (numdefcpd < 1) {
- HUNSPELL_WARNING(stderr, "incorrect number of entries in compound rule table\n");
- free(piece);
+ HUNSPELL_WARNING(stderr, "error: line %d: bad entry number\n", af->getlinenum());
return 1;
}
defcpdtable = (flagentry *) malloc(numdefcpd * sizeof(flagentry));
if (!defcpdtable) return 1;
np++;
break;
}
default: break;
}
i++;
}
- free(piece);
piece = mystrsep(&tp, 0);
}
if (np != 2) {
- HUNSPELL_WARNING(stderr, "error: missing compound rule table information\n");
+ HUNSPELL_WARNING(stderr, "error: line %d: missing data\n", af->getlinenum());
return 1;
}
/* now parse the numdefcpd lines to read in the remainder of the table */
- char * nl = line;
+ char * nl;
for (int j=0; j < numdefcpd; j++) {
- if (!fgets(nl,MAXLNLEN,af)) return 1;
+ if (!(nl = af->getline())) return 1;
mychomp(nl);
tp = nl;
i = 0;
defcpdtable[j].def = NULL;
piece = mystrsep(&tp, 0);
while (piece) {
if (*piece != '\0') {
switch(i) {
case 0: {
if (strncmp(piece, "COMPOUNDRULE", 12) != 0) {
- HUNSPELL_WARNING(stderr, "error: compound rule table is corrupt\n");
- free(piece);
+ HUNSPELL_WARNING(stderr, "error: line %d: table is corrupt\n", af->getlinenum());
numdefcpd = 0;
return 1;
}
break;
}
- case 1: {
- defcpdtable[j].len =
- pHMgr->decode_flags(&(defcpdtable[j].def), piece);
+ case 1: { // handle parenthesized flags
+ if (strchr(piece, '(')) {
+ defcpdtable[j].def = (FLAG *) malloc(sizeof(piece) * sizeof(FLAG));
+ defcpdtable[j].len = 0;
+ int end = 0;
+ FLAG * conv;
+ while (!end) {
+ char * par = piece + 1;
+ while (*par != '(' && *par != ')' && *par != '\0') par++;
+ if (*par == '\0') end = 1; else *par = '\0';
+ if (*piece == '(') piece++;
+ if (*piece == '*' || *piece == '?') {
+ defcpdtable[j].def[defcpdtable[j].len++] = (FLAG) *piece;
+ } else if (*piece != '\0') {
+ int l = pHMgr->decode_flags(&conv, piece, af);
+ for (int k = 0; k < l; k++) defcpdtable[j].def[defcpdtable[j].len++] = conv[k];
+ free(conv);
+ }
+ piece = par + 1;
+ }
+ } else {
+ defcpdtable[j].len = pHMgr->decode_flags(&(defcpdtable[j].def), piece, af);
+ }
break;
}
default: break;
}
i++;
}
- free(piece);
piece = mystrsep(&tp, 0);
}
if (!defcpdtable[j].len) {
- HUNSPELL_WARNING(stderr, "error: compound rule table is corrupt\n");
+ HUNSPELL_WARNING(stderr, "error: line %d: table is corrupt\n", af->getlinenum());
numdefcpd = 0;
return 1;
}
}
return 0;
}
/* parse in the character map table */
-int AffixMgr::parse_maptable(char * line, FILE * af)
+int AffixMgr::parse_maptable(char * line, FileMgr * af)
{
if (nummap != 0) {
- HUNSPELL_WARNING(stderr, "error: duplicate MAP tables used\n");
+ HUNSPELL_WARNING(stderr, "error: line %d: multiple table definitions\n", af->getlinenum());
return 1;
}
char * tp = line;
char * piece;
int i = 0;
int np = 0;
piece = mystrsep(&tp, 0);
while (piece) {
if (*piece != '\0') {
switch(i) {
case 0: { np++; break; }
case 1: {
nummap = atoi(piece);
if (nummap < 1) {
- HUNSPELL_WARNING(stderr, "incorrect number of entries in map table\n");
- free(piece);
+ HUNSPELL_WARNING(stderr, "error: line %d: bad entry number\n", af->getlinenum());
return 1;
}
maptable = (mapentry *) malloc(nummap * sizeof(struct mapentry));
if (!maptable) return 1;
np++;
break;
}
default: break;
}
i++;
}
- free(piece);
piece = mystrsep(&tp, 0);
}
if (np != 2) {
- HUNSPELL_WARNING(stderr, "error: missing map table information\n");
+ HUNSPELL_WARNING(stderr, "error: line %d: missing data\n", af->getlinenum());
return 1;
}
/* now parse the nummap lines to read in the remainder of the table */
- char * nl = line;
+ char * nl;
for (int j=0; j < nummap; j++) {
- if (!fgets(nl,MAXLNLEN,af)) return 1;
+ if (!(nl = af->getline())) return 1;
mychomp(nl);
tp = nl;
i = 0;
maptable[j].set = NULL;
maptable[j].len = 0;
piece = mystrsep(&tp, 0);
while (piece) {
if (*piece != '\0') {
switch(i) {
case 0: {
if (strncmp(piece,"MAP",3) != 0) {
- HUNSPELL_WARNING(stderr, "error: map table is corrupt\n");
+ HUNSPELL_WARNING(stderr, "error: line %d: table is corrupt\n", af->getlinenum());
nummap = 0;
- free(piece);
return 1;
}
break;
}
case 1: {
maptable[j].len = 0;
maptable[j].set = NULL;
maptable[j].set_utf16 = NULL;
@@ -3690,109 +3897,129 @@ int AffixMgr::parse_maptable(char * lin
}
maptable[j].len = n;
}
break; }
default: break;
}
i++;
}
- free(piece);
piece = mystrsep(&tp, 0);
}
if ((!(maptable[j].set || maptable[j].set_utf16)) || (!(maptable[j].len))) {
- HUNSPELL_WARNING(stderr, "error: map table is corrupt\n");
+ HUNSPELL_WARNING(stderr, "error: line %d: table is corrupt\n", af->getlinenum());
nummap = 0;
return 1;
}
}
return 0;
}
/* parse in the word breakpoint table */
-int AffixMgr::parse_breaktable(char * line, FILE * af)
+int AffixMgr::parse_breaktable(char * line, FileMgr * af)
{
if (numbreak != 0) {
- HUNSPELL_WARNING(stderr, "error: duplicate word breakpoint tables used\n");
+ HUNSPELL_WARNING(stderr, "error: line %d: multiple table definitions\n", af->getlinenum());
return 1;
}
char * tp = line;
char * piece;
int i = 0;
int np = 0;
piece = mystrsep(&tp, 0);
while (piece) {
if (*piece != '\0') {
switch(i) {
case 0: { np++; break; }
case 1: {
numbreak = atoi(piece);
if (numbreak < 1) {
- HUNSPELL_WARNING(stderr, "incorrect number of entries in BREAK table\n");
- free(piece);
+ HUNSPELL_WARNING(stderr, "error: line %d: bad entry number\n", af->getlinenum());
return 1;
}
breaktable = (char **) malloc(numbreak * sizeof(char *));
if (!breaktable) return 1;
np++;
break;
}
default: break;
}
i++;
}
- free(piece);
piece = mystrsep(&tp, 0);
}
if (np != 2) {
- HUNSPELL_WARNING(stderr, "error: missing word breakpoint table information\n");
+ HUNSPELL_WARNING(stderr, "error: line %d: missing data\n", af->getlinenum());
return 1;
}
/* now parse the numbreak lines to read in the remainder of the table */
- char * nl = line;
+ char * nl;
for (int j=0; j < numbreak; j++) {
- if (!fgets(nl,MAXLNLEN,af)) return 1;
+ if (!(nl = af->getline())) return 1;
mychomp(nl);
tp = nl;
i = 0;
piece = mystrsep(&tp, 0);
while (piece) {
if (*piece != '\0') {
switch(i) {
case 0: {
if (strncmp(piece,"BREAK",5) != 0) {
- HUNSPELL_WARNING(stderr, "error: BREAK table is corrupt\n");
- free(piece);
+ HUNSPELL_WARNING(stderr, "error: line %d: table is corrupt\n", af->getlinenum());
numbreak = 0;
return 1;
}
break;
}
case 1: {
breaktable[j] = mystrdup(piece);
break;
}
default: break;
}
i++;
}
- free(piece);
piece = mystrsep(&tp, 0);
}
if (!breaktable) {
- HUNSPELL_WARNING(stderr, "error: BREAK table is corrupt\n");
+ HUNSPELL_WARNING(stderr, "error: line %d: table is corrupt\n", af->getlinenum());
numbreak = 0;
return 1;
}
}
return 0;
}
-int AffixMgr::parse_affix(char * line, const char at, FILE * af, char * dupflags)
+void AffixMgr::reverse_condition(char * piece) {
+ int neg = 0;
+ for (char * k = piece + strlen(piece) - 1; k >= piece; k--) {
+ switch(*k) {
+ case '[': {
+ if (neg) *(k+1) = '['; else *k = ']';
+ break;
+ }
+ case ']': {
+ *k = '[';
+ if (neg) *(k+1) = '^';
+ neg = 0;
+ break;
+ }
+ case '^': {
+ if (*(k+1) == ']') neg = 1; else *(k+1) = *k;
+ break;
+ }
+ default: {
+ if (neg) *(k+1) = *k;
+ }
+ }
+ }
+}
+
+int AffixMgr::parse_affix(char * line, const char at, FileMgr * af, char * dupflags)
{
int numents = 0; // number of affentry structures to parse
unsigned short aflag = 0; // affix char identifier
char ff=0;
struct affentry * ptr= NULL;
struct affentry * nptr= NULL;
@@ -3805,108 +4032,113 @@ int AffixMgr::parse_affix(char * line,
// checking lines with bad syntax
#ifdef DEBUG
int basefieldnum = 0;
#endif
// split affix header line into pieces
int np = 0;
+
piece = mystrsep(&tp, 0);
while (piece) {
if (*piece != '\0') {
switch(i) {
// piece 1 - is type of affix
case 0: { np++; break; }
// piece 2 - is affix char
case 1: {
np++;
aflag = pHMgr->decode_flag(piece);
if (((at == 'S') && (dupflags[aflag] & dupSFX)) ||
((at == 'P') && (dupflags[aflag] & dupPFX))) {
- HUNSPELL_WARNING(stderr, "error: duplicate affix flag %s in line %s\n", piece, nl);
+ HUNSPELL_WARNING(stderr, "error: line %d: multiple definitions of an affix flag\n",
+ af->getlinenum());
// return 1; XXX permissive mode for bad dictionaries
}
- dupflags[aflag] += ((at == 'S') ? dupSFX : dupPFX);
+ dupflags[aflag] += (char) ((at == 'S') ? dupSFX : dupPFX);
break;
}
// piece 3 - is cross product indicator
case 2: { np++; if (*piece == 'Y') ff = aeXPRODUCT; break; }
// piece 4 - is number of affentries
case 3: {
np++;
numents = atoi(piece);
if (numents == 0) {
char * err = pHMgr->encode_flag(aflag);
- HUNSPELL_WARNING(stderr, "error: affix %s header has incorrect entry count in line %s\n",
- err, nl);
- free(err);
+ if (err) {
+ HUNSPELL_WARNING(stderr, "error: line %d: bad entry number\n",
+ af->getlinenum());
+ free(err);
+ }
return 1;
}
ptr = (struct affentry *) malloc(numents * sizeof(struct affentry));
if (!ptr) return 1;
ptr->opts = ff;
if (utf8) ptr->opts += aeUTF8;
if (pHMgr->is_aliasf()) ptr->opts += aeALIASF;
-#ifdef HUNSPELL_EXPERIMENTAL
if (pHMgr->is_aliasm()) ptr->opts += aeALIASM;
-#endif
ptr->aflag = aflag;
}
default: break;
}
i++;
}
- free(piece);
piece = mystrsep(&tp, 0);
}
// check to make sure we parsed enough pieces
if (np != 4) {
- char * err = pHMgr->encode_flag(aflag);
- HUNSPELL_WARNING(stderr, "error: affix %s header has insufficient data in line %s\n", err, nl);
- free(err);
+ char * err = pHMgr->encode_flag(aflag);
+ if (err) {
+ HUNSPELL_WARNING(stderr, "error: line %d: missing data\n", af->getlinenum());
+ free(err);
+ }
free(ptr);
return 1;
}
// store away ptr to first affentry
nptr = ptr;
// now parse numents affentries for this affix
for (int j=0; j < numents; j++) {
- if (!fgets(nl,MAXLNLEN,af)) return 1;
+ if (!(nl = af->getline())) return 1;
mychomp(nl);
tp = nl;
i = 0;
np = 0;
// split line into pieces
piece = mystrsep(&tp, 0);
while (piece) {
if (*piece != '\0') {
switch(i) {
// piece 1 - is type
case 0: {
np++;
- if (nptr != ptr) nptr->opts = ptr->opts;
+ if (nptr != ptr) nptr->opts = ptr->opts &
+ (char) (aeXPRODUCT + aeUTF8 + aeALIASF + aeALIASM);
break;
}
// piece 2 - is affix char
case 1: {
np++;
if (pHMgr->decode_flag(piece) != aflag) {
char * err = pHMgr->encode_flag(aflag);
- HUNSPELL_WARNING(stderr, "error: affix %s is corrupt near line %s\n", err, nl);
- HUNSPELL_WARNING(stderr, "error: possible incorrect count\n");
- free(err);
- free(piece);
+ if (err) {
+ HUNSPELL_WARNING(stderr, "error: line %d: affix %s is corrupt\n",
+ af->getlinenum(), err);
+ free(err);
+ }
return 1;
}
if (nptr != ptr) nptr->aflag = ptr->aflag;
break;
}
// piece 3 - is string to strip or 0 for null
@@ -3923,44 +4155,43 @@ int AffixMgr::parse_affix(char * line,
nptr->stripl = 0;
}
break;
}
// piece 4 - is affix string or 0 for null
case 3: {
char * dash;
-#ifdef HUNSPELL_EXPERIMENTAL
nptr->morphcode = NULL;
-#endif
nptr->contclass = NULL;
nptr->contclasslen = 0;
np++;
dash = strchr(piece, '/');
if (dash) {
*dash = '\0';
if (ignorechars) {
if (utf8) {
remove_ignored_chars_utf(piece, ignorechars_utf16, ignorechars_utf16_len);
} else {
remove_ignored_chars(piece,ignorechars);
}
}
-
+
if (complexprefixes) {
if (utf8) reverseword_utf(piece); else reverseword(piece);
}
nptr->appnd = mystrdup(piece);
-
+
if (pHMgr->is_aliasf()) {
int index = atoi(dash + 1);
- nptr->contclasslen = (unsigned short) pHMgr->get_aliasf(index, &(nptr->contclass));
+ nptr->contclasslen = (unsigned short) pHMgr->get_aliasf(index, &(nptr->contclass), af);
+ if (!nptr->contclasslen) HUNSPELL_WARNING(stderr, "error: bad affix flag alias: \"%s\"\n", dash+1);
} else {
- nptr->contclasslen = (unsigned short) pHMgr->decode_flags(&(nptr->contclass), dash + 1);
+ nptr->contclasslen = (unsigned short) pHMgr->decode_flags(&(nptr->contclass), dash + 1, af);
flag_qsort(nptr->contclass, 0, nptr->contclasslen);
}
*dash = '/';
havecontclass = 1;
for (unsigned short _i = 0; _i < nptr->contclasslen; _i++) {
contclasses[(nptr->contclass)[_i]] = 1;
}
@@ -3971,185 +4202,172 @@ int AffixMgr::parse_affix(char * line,
} else {
remove_ignored_chars(piece,ignorechars);
}
}
if (complexprefixes) {
if (utf8) reverseword_utf(piece); else reverseword(piece);
}
- nptr->appnd = mystrdup(piece);
+ nptr->appnd = mystrdup(piece);
}
-
+
nptr->appndl = (unsigned char) strlen(nptr->appnd);
if (strcmp(nptr->appnd,"0") == 0) {
free(nptr->appnd);
nptr->appnd=mystrdup("");
nptr->appndl = 0;
}
break;
}
// piece 5 - is the conditions descriptions
case 4: {
np++;
if (complexprefixes) {
- int neg = 0;
if (utf8) reverseword_utf(piece); else reverseword(piece);
- // reverse condition
- for (char * k = piece + strlen(piece) - 1; k >= piece; k--) {
- switch(*k) {
- case '[': {
- if (neg) *(k+1) = '['; else *k = ']';
- break;
- }
- case ']': {
- *k = '[';
- if (neg) *(k+1) = '^';
- neg = 0;
- break;
- }
- case '^': {
- if (*(k+1) == ']') neg = 1; else *(k+1) = *k;
- break;
- }
- default: {
- if (neg) *(k+1) = *k;
- }
- }
- }
+ reverse_condition(piece);
}
if (nptr->stripl && (strcmp(piece, ".") != 0) &&
- redundant_condition(at, nptr->strip, nptr->stripl, piece, nl))
+ redundant_condition(at, nptr->strip, nptr->stripl, piece, af->getlinenum()))
strcpy(piece, ".");
- if (encodeit(nptr,piece)) return 1;
+ if (at == 'S') {
+ reverseword(piece);
+ reverse_condition(piece);
+ }
+ if (encodeit(nptr, piece)) return 1;
break;
}
-
-#ifdef HUNSPELL_EXPERIMENTAL
+
case 5: {
np++;
if (pHMgr->is_aliasm()) {
int index = atoi(piece);
nptr->morphcode = pHMgr->get_aliasm(index);
} else {
- if (complexprefixes) {
+ if (complexprefixes) { // XXX - fix me for morph. gen.
if (utf8) reverseword_utf(piece); else reverseword(piece);
}
+ // add the remaining of the line
+ if (*tp) {
+ *(tp - 1) = ' ';
+ tp = tp + strlen(tp);
+ }
nptr->morphcode = mystrdup(piece);
+ if (!nptr->morphcode) return 1;
}
break;
}
-#endif
-
default: break;
}
i++;
}
- free(piece);
piece = mystrsep(&tp, 0);
}
// check to make sure we parsed enough pieces
if (np < 4) {
char * err = pHMgr->encode_flag(aflag);
- HUNSPELL_WARNING(stderr, "error: affix %s is corrupt near line %s\n", err, nl);
- free(err);
+ if (err) {
+ HUNSPELL_WARNING(stderr, "error: line %d: affix %s is corrupt\n",
+ af->getlinenum(), err);
+ free(err);
+ }
free(ptr);
return 1;
}
#ifdef DEBUG
-#ifdef HUNSPELL_EXPERIMENTAL
// detect unnecessary fields, excepting comments
if (basefieldnum) {
int fieldnum = !(nptr->morphcode) ? 5 : ((*(nptr->morphcode)=='#') ? 5 : 6);
if (fieldnum != basefieldnum)
- HUNSPELL_WARNING(stderr, "warning: bad field number:\n%s\n", nl);
+ HUNSPELL_WARNING(stderr, "warning: line %d: bad field number\n", af->getlinenum());
} else {
basefieldnum = !(nptr->morphcode) ? 5 : ((*(nptr->morphcode)=='#') ? 5 : 6);
}
#endif
-#endif
nptr++;
}
// now create SfxEntry or PfxEntry objects and use links to
// build an ordered (sorted by affix string) list
nptr = ptr;
for (int k = 0; k < numents; k++) {
if (at == 'P') {
PfxEntry * pfxptr = new PfxEntry(this,nptr);
build_pfxtree((AffEntry *)pfxptr);
} else {
SfxEntry * sfxptr = new SfxEntry(this,nptr);
build_sfxtree((AffEntry *)sfxptr);
}
nptr++;
- }
+ }
free(ptr);
return 0;
}
-int AffixMgr::redundant_condition(char ft, char * strip, int stripl, const char * cond, char * warnvar) {
+int AffixMgr::redundant_condition(char ft, char * strip, int stripl, const char * cond, int linenum) {
int condl = strlen(cond);
int i;
int j;
int neg;
int in;
if (ft == 'P') { // prefix
if (strncmp(strip, cond, condl) == 0) return 1;
if (utf8) {
} else {
for (i = 0, j = 0; (i < stripl) && (j < condl); i++, j++) {
if (cond[j] != '[') {
if (cond[j] != strip[i]) {
- HUNSPELL_WARNING(stderr, "warning: incompatible stripping characters and condition:\n%s\n", warnvar);
+ HUNSPELL_WARNING(stderr, "warning: line %d: incompatible stripping characters and condition\n", linenum);
+ return 0;
}
} else {
neg = (cond[j+1] == '^') ? 1 : 0;
in = 0;
do {
j++;
if (strip[i] == cond[j]) in = 1;
} while ((j < (condl - 1)) && (cond[j] != ']'));
if (j == (condl - 1) && (cond[j] != ']')) {
- HUNSPELL_WARNING(stderr, "error: missing ] in condition:\n%s\n", warnvar);
+ HUNSPELL_WARNING(stderr, "error: line %d: missing ] in condition:\n%s\n", linenum);
return 0;
}
if ((!neg && !in) || (neg && in)) {
- HUNSPELL_WARNING(stderr, "warning: incompatible stripping characters and condition:\n%s\n", warnvar);
- return 0;
+ HUNSPELL_WARNING(stderr, "warning: line %d: incompatible stripping characters and condition\n", linenum);
+ return 0;
}
}
}
if (j >= condl) return 1;
}
} else { // suffix
if ((stripl >= condl) && strcmp(strip + stripl - condl, cond) == 0) return 1;
if (utf8) {
} else {
for (i = stripl - 1, j = condl - 1; (i >= 0) && (j >= 0); i--, j--) {
if (cond[j] != ']') {
if (cond[j] != strip[i]) {
- HUNSPELL_WARNING(stderr, "warning: incompatible stripping characters and condition:\n%s\n", warnvar);
+ HUNSPELL_WARNING(stderr, "warning: line %d: incompatible stripping characters and condition\n", linenum);
+ return 0;
}
} else {
in = 0;
do {
j--;
if (strip[i] == cond[j]) in = 1;
} while ((j > 0) && (cond[j] != '['));
if ((j == 0) && (cond[j] != '[')) {
- HUNSPELL_WARNING(stderr, "error: missing ] in condition:\n%s\n", warnvar);
+ HUNSPELL_WARNING(stderr, "error: error: %d: missing ] in condition:\n%s\n", linenum);
return 0;
}
neg = (cond[j+1] == '^') ? 1 : 0;
if ((!neg && !in) || (neg && in)) {
- HUNSPELL_WARNING(stderr, "warning: incompatible stripping characters and condition:\n%s\n", warnvar);
- return 0;
+ HUNSPELL_WARNING(stderr, "warning: line %d: incompatible stripping characters and condition\n", linenum);
+ return 0;
}
}
}
if (j < 0) return 1;
- }
+ }
}
return 0;
}
--- a/extensions/spellcheck/hunspell/src/affixmgr.hxx
+++ b/extensions/spellcheck/hunspell/src/affixmgr.hxx
@@ -65,29 +65,32 @@ using namespace std;
#else
#include <cstdio>
#endif
#include "atypes.hxx"
#include "baseaffix.hxx"
#include "hashmgr.hxx"
#include "phonet.hxx"
+#include "replist.hxx"
// check flag duplication
#define dupSFX (1 << 0)
#define dupPFX (1 << 1)
class AffixMgr
{
AffEntry * pStart[SETSIZE];
AffEntry * sStart[SETSIZE];
- AffEntry * pFlag[CONTSIZE];
- AffEntry * sFlag[CONTSIZE];
+ AffEntry * pFlag[SETSIZE];
+ AffEntry * sFlag[SETSIZE];
HashMgr * pHMgr;
+ HashMgr ** alldic;
+ int * maxdic;
char * keystring;
char * trystring;
char * encoding;
struct cs_info * csconv;
int utf8;
int complexprefixes;
FLAG compoundflag;
FLAG compoundbegin;
@@ -95,28 +98,32 @@ class AffixMgr
FLAG compoundend;
FLAG compoundroot;
FLAG compoundforbidflag;
FLAG compoundpermitflag;
int checkcompounddup;
int checkcompoundrep;
int checkcompoundcase;
int checkcompoundtriple;
+ int simplifiedtriple;
FLAG forbiddenword;
FLAG nosuggest;
- FLAG pseudoroot;
+ FLAG needaffix;
int cpdmin;
int numrep;
replentry * reptable;
+ RepList * iconvtable;
+ RepList * oconvtable;
int nummap;
mapentry * maptable;
int numbreak;
char ** breaktable;
int numcheckcpd;
- replentry * checkcpdtable;
+ patentry * checkcpdtable;
+ int simplifiedcpd;
int numdefcpd;
flagentry * defcpdtable;
phonetable * phone;
int maxngramsugs;
int nosplitsugs;
int sugswithdots;
int cpdwordmax;
int cpdmaxsyllable;
@@ -139,73 +146,85 @@ class AffixMgr
int ignorechars_utf16_len;
char * version;
char * lang;
int langnum;
FLAG lemma_present;
FLAG circumfix;
FLAG onlyincompound;
FLAG keepcase;
+ FLAG substandard;
int checksharps;
+ int fullstrip;
int havecontclass; // boolean variable
char contclasses[CONTSIZE]; // flags of possible continuing classes (twofold affix)
- flag flag_mode;
-
+
public:
-
- AffixMgr(const char * affpath, HashMgr * ptr);
+
+ AffixMgr(const char * affpath, HashMgr** ptr, int * md,
+ const char * key = NULL);
~AffixMgr();
struct hentry * affix_check(const char * word, int len,
- const unsigned short needflag = (unsigned short) 0, char in_compound = IN_CPD_NOT);
+ const unsigned short needflag = (unsigned short) 0,
+ char in_compound = IN_CPD_NOT);
struct hentry * prefix_check(const char * word, int len,
char in_compound, const FLAG needflag = FLAG_NULL);
inline int isSubset(const char * s1, const char * s2);
struct hentry * prefix_check_twosfx(const char * word, int len,
char in_compound, const FLAG needflag = FLAG_NULL);
inline int isRevSubset(const char * s1, const char * end_of_s2, int len);
- struct hentry * suffix_check(const char * word, int len, int sfxopts, AffEntry* ppfx,
- char ** wlst, int maxSug, int * ns, const FLAG cclass = FLAG_NULL,
- const FLAG needflag = FLAG_NULL, char in_compound = IN_CPD_NOT);
+ struct hentry * suffix_check(const char * word, int len, int sfxopts,
+ AffEntry* ppfx, char ** wlst, int maxSug, int * ns,
+ const FLAG cclass = FLAG_NULL, const FLAG needflag = FLAG_NULL,
+ char in_compound = IN_CPD_NOT);
struct hentry * suffix_check_twosfx(const char * word, int len,
int sfxopts, AffEntry* ppfx, const FLAG needflag = FLAG_NULL);
char * affix_check_morph(const char * word, int len,
- const FLAG needflag = FLAG_NULL, char in_compound = IN_CPD_NOT);
+ const FLAG needflag = FLAG_NULL, char in_compound = IN_CPD_NOT);
char * prefix_check_morph(const char * word, int len,
- char in_compound, const FLAG needflag = FLAG_NULL);
- char * suffix_check_morph (const char * word, int len, int sfxopts, AffEntry * ppfx,
- const FLAG cclass = FLAG_NULL, const FLAG needflag = FLAG_NULL, char in_compound = IN_CPD_NOT);
+ char in_compound, const FLAG needflag = FLAG_NULL);
+ char * suffix_check_morph (const char * word, int len, int sfxopts,
+ AffEntry * ppfx, const FLAG cclass = FLAG_NULL,
+ const FLAG needflag = FLAG_NULL, char in_compound = IN_CPD_NOT);
char * prefix_check_twosfx_morph(const char * word, int len,
char in_compound, const FLAG needflag = FLAG_NULL);
char * suffix_check_twosfx_morph(const char * word, int len,
int sfxopts, AffEntry * ppfx, const FLAG needflag = FLAG_NULL);
- int expand_rootword(struct guessword * wlst, int maxn, const char * ts,
- int wl, const unsigned short * ap, unsigned short al, char * bad, int,
- char *);
+ char * morphgen(char * ts, int wl, const unsigned short * ap,
+ unsigned short al, char * morph, char * targetmorph, int level);
+
+ int expand_rootword(struct guessword * wlst, int maxn, const char * ts,
+ int wl, const unsigned short * ap, unsigned short al, char * bad,
+ int, char *);
- short get_syllable (const char * word, int wlen);
- int cpdrep_check(const char * word, int len);
- int cpdpat_check(const char * word, int len);
- int defcpd_check(hentry *** words, short wnum, hentry * rv, hentry ** rwords, char all);
- int cpdcase_check(const char * word, int len);
- inline int candidate_check(const char * word, int len);
- struct hentry * compound_check(const char * word, int len,
- short wordnum, short numsyllable, short maxwordnum, short wnum, hentry ** words,
- char hu_mov_rule, int * cmpdstemnum, int * cmpdstem, char is_sug);
+ short get_syllable (const char * word, int wlen);
+ int cpdrep_check(const char * word, int len);
+ int cpdpat_check(const char * word, int len, hentry * r1, hentry * r2);
+ int defcpd_check(hentry *** words, short wnum, hentry * rv,
+ hentry ** rwords, char all);
+ int cpdcase_check(const char * word, int len);
+ inline int candidate_check(const char * word, int len);
+ void setcminmax(int * cmin, int * cmax, const char * word, int len);
+ struct hentry * compound_check(const char * word, int len, short wordnum,
+ short numsyllable, short maxwordnum, short wnum, hentry ** words,
+ char hu_mov_rule, char is_sug);
- int compound_check_morph(const char * word, int len,
- short wordnum, short numsyllable, short maxwordnum, short wnum, hentry ** words,
- char hu_mov_rule, char ** result, char * partresult);
+ int compound_check_morph(const char * word, int len, short wordnum,
+ short numsyllable, short maxwordnum, short wnum, hentry ** words,
+ char hu_mov_rule, char ** result, char * partresult);
- struct hentry * lookup(const char * word);
+ struct hentry * lookup(const char * word);
int get_numrep();
struct replentry * get_reptable();
+ RepList * get_iconvtable();
+ RepList * get_oconvtable();
struct phonetable * get_phonetable();
int get_nummap();
struct mapentry * get_maptable();
int get_numbreak();
char ** get_breaktable();
char * get_encoding();
int get_langnum();
char * get_key_string();
@@ -214,18 +233,17 @@ public:
unsigned short * get_wordchars_utf16(int * len);
char * get_ignore();
unsigned short * get_ignore_utf16(int * len);
int get_compound();
FLAG get_compoundflag();
FLAG get_compoundbegin();
FLAG get_forbiddenword();
FLAG get_nosuggest();
-// FLAG get_circumfix();
- FLAG get_pseudoroot();
+ FLAG get_needaffix();
FLAG get_onlyincompound();
FLAG get_compoundroot();
FLAG get_lemma_present();
int get_checknum();
char * get_possible_root();
const char * get_prefix();
const char * get_suffix();
const char * get_derived();
@@ -234,39 +252,42 @@ public:
int get_utf8();
int get_complexprefixes();
char * get_suffixed(char );
int get_maxngramsugs();
int get_nosplitsugs();
int get_sugswithdots(void);
FLAG get_keepcase(void);
int get_checksharps(void);
+ char * encode_flag(unsigned short aflag);
+ int get_fullstrip();
private:
- int parse_file(const char * affpath);
-// int parse_string(char * line, char ** out, const char * name);
- int parse_flag(char * line, unsigned short * out, const char * name);
- int parse_num(char * line, int * out, const char * name);
-// int parse_array(char * line, char ** out, unsigned short ** out_utf16,
-// int * out_utf16_len, const char * name);
- int parse_cpdsyllable(char * line);
- int parse_reptable(char * line, FILE * af);
- int parse_phonetable(char * line, FILE * af);
- int parse_maptable(char * line, FILE * af);
- int parse_breaktable(char * line, FILE * af);
- int parse_checkcpdtable(char * line, FILE * af);
- int parse_defcpdtable(char * line, FILE * af);
- int parse_affix(char * line, const char at, FILE * af, char * dupflags);
+ int parse_file(const char * affpath, const char * key);
+ int parse_flag(char * line, unsigned short * out, FileMgr * af);
+ int parse_num(char * line, int * out, FileMgr * af);
+ int parse_cpdsyllable(char * line, FileMgr * af);
+ int parse_reptable(char * line, FileMgr * af);
+ int parse_convtable(char * line, FileMgr * af, RepList ** rl, const char * keyword);
+ int parse_phonetable(char * line, FileMgr * af);
+ int parse_maptable(char * line, FileMgr * af);
+ int parse_breaktable(char * line, FileMgr * af);
+ int parse_checkcpdtable(char * line, FileMgr * af);
+ int parse_defcpdtable(char * line, FileMgr * af);
+ int parse_affix(char * line, const char at, FileMgr * af, char * dupflags);
+ void reverse_condition(char *);
+ void debugflag(char * result, unsigned short flag);
+ int condlen(char *);
int encodeit(struct affentry * ptr, char * cs);
int build_pfxtree(AffEntry* pfxptr);
int build_sfxtree(AffEntry* sfxptr);
int process_pfx_order();
int process_sfx_order();
AffEntry * process_pfx_in_order(AffEntry * ptr, AffEntry * nptr);
AffEntry * process_sfx_in_order(AffEntry * ptr, AffEntry * nptr);
int process_pfx_tree_to_list();
int process_sfx_tree_to_list();
- int redundant_condition(char, char * strip, int stripl, const char * cond, char *);
+ int redundant_condition(char, char * strip, int stripl,
+ const char * cond, int);
};
#endif
-
--- a/extensions/spellcheck/hunspell/src/atypes.hxx
+++ b/extensions/spellcheck/hunspell/src/atypes.hxx
@@ -64,41 +64,43 @@
// empty inline function to switch off warnings (instead of the C99 standard variadic macros)
static inline void HUNSPELL_WARNING(FILE *, const char *, ...) {}
#endif
#endif
// HUNSTEM def.
#define HUNSTEM
-#include "csutil.hxx"
#include "hashmgr.hxx"
+#include "w_char.hxx"
#define SETSIZE 256
#define CONTSIZE 65536
#define MAXWORDLEN 100
#define MAXWORDUTF8LEN 256
// affentry options
#define aeXPRODUCT (1 << 0)
#define aeUTF8 (1 << 1)
#define aeALIASF (1 << 2)
#define aeALIASM (1 << 3)
-#define aeINFIX (1 << 4)
+#define aeLONGCOND (1 << 4)
// compound options
#define IN_CPD_NOT 0
#define IN_CPD_BEGIN 1
#define IN_CPD_END 2
#define IN_CPD_OTHER 3
#define MAXLNLEN 8192
#define MINCPDLEN 3
#define MAXCOMPOUND 10
+#define MAXCONDLEN 20
+#define MAXCONDLEN_1 (MAXCONDLEN - sizeof(char *))
#define MAXACC 1000
#define FLAG unsigned short
#define FLAG_NULL 0x00
#define FREE_FLAG(a) a = 0
#define TESTAFF( a, b , c ) flag_bsearch((unsigned short *) a, (unsigned short) b, c)
@@ -107,48 +109,46 @@ struct affentry
{
char * strip;
char * appnd;
unsigned char stripl;
unsigned char appndl;
char numconds;
char opts;
unsigned short aflag;
- union {
- char base[SETSIZE];
- struct {
- char ascii[SETSIZE/2];
- char neg[8];
- char all[8];
- w_char * wchars[8];
- int wlen[8];
- } utf8;
- } conds;
-#ifdef HUNSPELL_EXPERIMENTAL
- char * morphcode;
-#endif
unsigned short * contclass;
short contclasslen;
+ union {
+ char conds[MAXCONDLEN];
+ struct {
+ char conds1[MAXCONDLEN_1];
+ char * conds2;
+ } l;
+ } c;
+ char * morphcode;
+};
+
+struct guessword {
+ char * word;
+ bool allow;
+ char * orig;
};
struct mapentry {
char * set;
w_char * set_utf16;
int len;
};
struct flagentry {
FLAG * def;
int len;
};
-struct guessword {
- char * word;
- bool allow;
- char * orig;
+struct patentry {
+ char * pattern;
+ char * pattern2;
+ char * pattern3;
+ FLAG cond;
+ FLAG cond2;
};
#endif
-
-
-
-
-
--- a/extensions/spellcheck/hunspell/src/baseaffix.hxx
+++ b/extensions/spellcheck/hunspell/src/baseaffix.hxx
@@ -57,31 +57,28 @@
#ifndef _BASEAFF_HXX_
#define _BASEAFF_HXX_
class AffEntry
{
public:
protected:
- char * appnd;
- char * strip;
- unsigned char appndl;
- unsigned char stripl;
- char numconds;
- char opts;
- unsigned short aflag;
- union {
- char base[SETSIZE];
- struct {
- char ascii[SETSIZE/2];
- char neg[8];
- char all[8];
- w_char * wchars[8];
- int wlen[8];
- } utf8;
- } conds;
- char * morphcode;
- unsigned short * contclass;
- short contclasslen;
+ char * appnd;
+ char * strip;
+ unsigned char appndl;
+ unsigned char stripl;
+ char numconds;
+ char opts;
+ unsigned short aflag;
+ union {
+ char conds[MAXCONDLEN];
+ struct {
+ char conds1[MAXCONDLEN_1];
+ char * conds2;
+ } l;
+ } c;
+ char * morphcode;
+ unsigned short * contclass;
+ short contclasslen;
};
#endif
--- a/extensions/spellcheck/hunspell/src/csutil.cpp
+++ b/extensions/spellcheck/hunspell/src/csutil.cpp
@@ -61,18 +61,18 @@
#include <cctype>
#else
#include <stdlib.h>
#include <string.h>
#include <stdio.h>
#include <ctype.h>
#endif
+#include "atypes.hxx"
#include "csutil.hxx"
-#include "atypes.hxx"
#include "langnum.hxx"
#ifdef OPENOFFICEORG
# include <unicode/uchar.h>
#else
# ifndef MOZILLA_CLIENT
# include "utf_info.cxx"
# define UTF_LST_LEN (sizeof(utf_lst) / (sizeof(unicode_info)))
@@ -103,18 +103,18 @@ using namespace std;
#endif
#endif
static struct unicode_info2 * utf_tbl = NULL;
static int utf_tbl_count = 0; // utf_tbl can be used by multiple Hunspell instances
/* only UTF-16 (BMP) implementation */
char * u16_u8(char * dest, int size, const w_char * src, int srclen) {
- char * u8 = dest;
- char * u8_max = u8 + size;
+ signed char * u8 = (signed char *)dest;
+ signed char * u8_max = (signed char *)(u8 + size);
const w_char * u2 = src;
const w_char * u2_max = src + srclen;
while ((u2 < u2_max) && (u8 < u8_max)) {
if (u2->h) { // > 0xFF
// XXX 4-byte haven't implemented yet.
if (u2->h >= 0x08) { // >= 0x800 (3-byte UTF-8 character)
*u8 = 0xe0 + (u2->h >> 4);
u8++;
@@ -151,17 +151,17 @@ char * u16_u8(char * dest, int size, con
}
*u8 = '\0';
return dest;
}
/* only UTF-16 (BMP) implementation */
int u8_u16(w_char * dest, int size, const char * src) {
- const char * u8 = src;
+ const signed char * u8 = (const signed char *)src;
w_char * u2 = dest;
w_char * u2_max = u2 + size;
while ((u2 < u2_max) && *u8) {
switch ((*u8) & 0xf0) {
case 0x00:
case 0x10:
case 0x20:
@@ -173,48 +173,48 @@ int u8_u16(w_char * dest, int size, cons
u2->h = 0;
u2->l = *u8;
break;
}
case 0x80:
case 0x90:
case 0xa0:
case 0xb0: {
- HUNSPELL_WARNING(stderr, "UTF-8 encoding error. Unexpected continuation bytes in %d. character position\n%s\n", u8 - src, src);
+ HUNSPELL_WARNING(stderr, "UTF-8 encoding error. Unexpected continuation bytes in %ld. character position\n%s\n", static_cast<long>(u8 - (signed char *)src), src);
u2->h = 0xff;
u2->l = 0xfd;
break;
}
case 0xc0:
case 0xd0: { // 2-byte UTF-8 codes
if ((*(u8+1) & 0xc0) == 0x80) {
u2->h = (*u8 & 0x1f) >> 2;
u2->l = (*u8 << 6) + (*(u8+1) & 0x3f);
u8++;
} else {
- HUNSPELL_WARNING(stderr, "UTF-8 encoding error. Missing continuation byte in %d. character position:\n%s\n", u8 - src, src);
+ HUNSPELL_WARNING(stderr, "UTF-8 encoding error. Missing continuation byte in %ld. character position:\n%s\n", static_cast<long>(u8 - (signed char *)src), src);
u2->h = 0xff;
u2->l = 0xfd;
}
break;
}
case 0xe0: { // 3-byte UTF-8 codes
if ((*(u8+1) & 0xc0) == 0x80) {
u2->h = ((*u8 & 0x0f) << 4) + ((*(u8+1) & 0x3f) >> 2);
u8++;
if ((*(u8+1) & 0xc0) == 0x80) {
u2->l = (*u8 << 6) + (*(u8+1) & 0x3f);
u8++;
} else {
- HUNSPELL_WARNING(stderr, "UTF-8 encoding error. Missing continuation byte in %d. character position:\n%s\n", u8 - src, src);
+ HUNSPELL_WARNING(stderr, "UTF-8 encoding error. Missing continuation byte in %ld. character position:\n%s\n", static_cast<long>(u8 - (signed char *)src), src);
u2->h = 0xff;
u2->l = 0xfd;
}
} else {
- HUNSPELL_WARNING(stderr, "UTF-8 encoding error. Missing continuation byte in %d. character position:\n%s\n", u8 - src, src);
+ HUNSPELL_WARNING(stderr, "UTF-8 encoding error. Missing continuation byte in %ld. character position:\n%s\n", static_cast<long>(u8 - (signed char *)src), src);
u2->h = 0xff;
u2->l = 0xfd;
}
break;
}
case 0xf0: { // 4 or more byte UTF-8 codes
HUNSPELL_WARNING(stderr, "This UTF-8 encoding can't convert to UTF-16:\n%s\n", src);
u2->h = 0xff;
@@ -269,65 +269,68 @@ int flag_bsearch(unsigned short flags[],
// strip strings into token based on single char delimiter
// acts like strsep() but only uses a delim char and not
// a delim string
// default delimiter: white space characters
char * mystrsep(char ** stringp, const char delim)
{
- char * rv = NULL;
char * mp = *stringp;
- int n = strlen(mp);
- if (n > 0) {
+ if (*mp != '\0') {
char * dp;
if (delim) {
- dp = (char *)memchr(mp,(int)((unsigned char)delim),n);
+ dp = strchr(mp, delim);
} else {
// don't use isspace() here, the string can be in some random charset
// that's way different than the locale's
for (dp = mp; (*dp && *dp != ' ' && *dp != '\t'); dp++);
if (!*dp) dp = NULL;
}
if (dp) {
*stringp = dp+1;
int nc = (int)((unsigned long)dp - (unsigned long)mp);
- rv = (char *) malloc(nc+1);
- if (rv) {
- memcpy(rv,mp,nc);
- *(rv+nc) = '\0';
- return rv;
- }
+ *(mp+nc) = '\0';
+ return mp;
} else {
- rv = (char *) malloc(n+1);
- if (rv) {
- memcpy(rv, mp, n);
- *(rv+n) = '\0';
- *stringp = mp + n;
- return rv;
- }
+ *stringp = mp + strlen(mp);
+ return mp;
}
}
return NULL;
}
-
// replaces strdup with ansi version
char * mystrdup(const char * s)
{
char * d = NULL;
if (s) {
int sl = strlen(s);
d = (char *) malloc(((sl+1) * sizeof(char)));
- if (d) memcpy(d,s,((sl+1)*sizeof(char)));
+ if (d) {
+ memcpy(d,s,((sl+1)*sizeof(char)));
+ return d;
+ }
+ HUNSPELL_WARNING(stderr, "Can't allocate memory.\n");
}
return d;
}
-
-
+
+ // strcat for limited length destination string
+ char * mystrcat(char * dest, const char * st, int max) {
+ int len;
+ int len2;
+ if (dest == NULL || st == NULL) return dest;
+ len = strlen(dest);
+ len2 = strlen(st);
+ if (len + len2 + 1 > max) return dest;
+ strcpy(dest + len, st);
+ return dest;
+ }
+
// remove cross-platform text line end characters
void mychomp(char * s)
{
int k = strlen(s);
if ((k > 0) && ((*(s+k-1)=='\r') || (*(s+k-1)=='\n'))) *(s+k-1) = '\0';
if ((k > 1) && (*(s+k-2) == '\r')) *(s+k-2) = '\0';
}
@@ -344,122 +347,268 @@ int flag_bsearch(unsigned short flags[],
char * q = d;
while (p >= s) *q++ = *p--;
*q = '\0';
}
}
return d;
}
-#ifdef HUNSPELL_EXPERIMENTAL
+// break text to lines
+// return number of lines
+int line_tok(const char * text, char *** lines, char breakchar) {
+ int linenum = 0;
+ char * dup = mystrdup(text);
+ char * p = strchr(dup, breakchar);
+ while (p) {
+ linenum++;
+ *p = '\0';
+ p++;
+ p = strchr(p, breakchar);
+ }
+ linenum++;
+// fprintf(stderr, "LINEN:%d %p %p\n", linenum, lines, *lines);
+ *lines = (char **) malloc(linenum * sizeof(char *));
+// fprintf(stderr, "hello\n");
+ if (!(*lines)) {
+ free(dup);
+ return 0;
+ }
+
+ p = dup;
+ int l = 0;
+ for (int i = 0; i < linenum; i++) {
+ if (*p != '\0') {
+ (*lines)[l] = mystrdup(p);
+ if (!(*lines)[l]) {
+ for (i = 0; i < l; i++) free((*lines)[i]);
+ free(dup);
+ return 0;
+ }
+ l++;
+ }
+ p += strlen(p) + 1;
+ }
+ free(dup);
+ if (!l) free(*lines);
+ return l;
+}
+
+// uniq line in place
+char * line_uniq(char * text, char breakchar) {
+ char ** lines;
+ int linenum = line_tok(text, &lines, breakchar);
+ int i;
+ strcpy(text, lines[0]);
+ for ( i = 1; i < linenum; i++ ) {
+ int dup = 0;
+ for (int j = 0; j < i; j++) {
+ if (strcmp(lines[i], lines[j]) == 0) dup = 1;
+ }
+ if (!dup) {
+ if ((i > 1) || (*(lines[0]) != '\0')) {
+ sprintf(text + strlen(text), "%c", breakchar);
+ }
+ strcat(text, lines[i]);
+ }
+ }
+ for ( i = 0; i < linenum; i++ ) {
+ if (lines[i]) free(lines[i]);
+ }
+ if (lines) free(lines);
+ return text;
+}
+
+// uniq and boundary for compound analysis: "1\n\2\n\1" -> " ( \1 | \2 ) "
+char * line_uniq_app(char ** text, char breakchar) {
+ if (!strchr(*text, breakchar)) {
+ return *text;
+ }
+
+ char ** lines;
+ int i;
+ int linenum = line_tok(*text, &lines, breakchar);
+ int dup = 0;
+ for (i = 0; i < linenum; i++) {
+ for (int j = 0; j < (i - 1); j++) {
+ if (strcmp(lines[i], lines[j]) == 0) {
+ *(lines[i]) = '\0';
+ dup++;
+ break;
+ }
+ }
+ }
+ if ((linenum - dup) == 1) {
+ strcpy(*text, lines[0]);
+ freelist(&lines, linenum);
+ return *text;
+ }
+ char * newtext = (char *) malloc(strlen(*text) + 2 * linenum + 3 + 1);
+ if (newtext) {
+ free(*text);
+ *text = newtext;
+ } else {
+ freelist(&lines, linenum);
+ return *text;
+ }
+ strcpy(*text," ( ");
+ for (i = 0; i < linenum; i++) if (*(lines[i])) {
+ sprintf(*text + strlen(*text), "%s%s", lines[i], " | ");
+ }
+ (*text)[strlen(*text) - 2] = ')'; // " ) "
+ freelist(&lines, linenum);
+ return *text;
+}
+
// append s to ends of every lines in text
void strlinecat(char * dest, const char * s)
{
char * dup = mystrdup(dest);
char * source = dup;
int len = strlen(s);
- while (*source) {
- if (*source == '\n') {
- strncpy(dest, s, len);
- dest += len;
+ if (dup) {
+ while (*source) {
+ if (*source == '\n') {
+ strncpy(dest, s, len);
+ dest += len;
+ }
+ *dest = *source;
+ source++; dest++;
}
- *dest = *source;
- source++; dest++;
+ strcpy(dest, s);
+ free(dup);
}
- strcpy(dest, s);
- free(dup);
}
-// break text to lines
-// return number of lines
-int line_tok(const char * text, char *** lines) {
- int linenum = 0;
- char * dup = mystrdup(text);
- char * p = strchr(dup, '\n');
- while (p) {
- linenum++;
- *p = '\0';
- p++;
- p = strchr(p, '\n');
- }
- *lines = (char **) calloc(linenum + 1, sizeof(char *));
- if (!(*lines)) return -1;
-
- p = dup;
- for (int i = 0; i < linenum + 1; i++) {
- (*lines)[i] = mystrdup(p);
- p += strlen(p) + 1;
- }
- free(dup);
- return linenum;
-}
-
-// uniq line in place
-char * line_uniq(char * text) {
- char ** lines;
- int linenum = line_tok(text, &lines);
- int i;
- strcpy(text, lines[0]);
- for ( i = 1; i<=linenum; i++ ) {
- int dup = 0;
- for (int j = 0; j < i; j++) {
- if (strcmp(lines[i], lines[j]) == 0) dup = 1;
- }
- if (!dup) {
- if ((i > 1) || (*(lines[0]) != '\0')) strcat(text, "\n");
- strcat(text, lines[i]);
- }
- }
- for ( i = 0; i<=linenum; i++ ) {
- if (lines[i]) free(lines[i]);
- }
- if (lines) free(lines);
+// change \n to char c
+char * tr(char * text, char oldc, char newc) {
+ char * p;
+ for (p = text; *p; p++) if (*p == oldc) *p = newc;
return text;
}
-// change \n to char c
-char * line_join(char * text, char c) {
- char * p;
- for (p = text; *p; p++) if (*p == '\n') *p = c;
- return text;
+// morphcmp(): compare MORPH_DERI_SFX, MORPH_INFL_SFX and MORPH_TERM_SFX fields
+// in the first line of the inputs
+// return 0, if inputs equal
+// return 1, if inputs may equal with a secondary suffix
+// otherwise return -1
+int morphcmp(const char * s, const char * t)
+{
+ int se = 0;
+ int te = 0;
+ const char * sl;
+ const char * tl;
+ const char * olds;
+ const char * oldt;
+ if (!s || !t) return 1;
+ olds = s;
+ sl = strchr(s, '\n');
+ s = strstr(s, MORPH_DERI_SFX);
+ if (!s || (sl && sl < s)) s = strstr(olds, MORPH_INFL_SFX);
+ if (!s || (sl && sl < s)) {
+ s= strstr(olds, MORPH_TERM_SFX);
+ olds = NULL;
+ }
+ oldt = t;
+ tl = strchr(t, '\n');
+ t = strstr(t, MORPH_DERI_SFX);
+ if (!t || (tl && tl < t)) t = strstr(oldt, MORPH_INFL_SFX);
+ if (!t || (tl && tl < t)) {
+ t = strstr(oldt, MORPH_TERM_SFX);
+ oldt = NULL;
+ }
+ while (s && t && (!sl || sl > s) && (!tl || tl > t)) {
+ s += MORPH_TAG_LEN;
+ t += MORPH_TAG_LEN;
+ se = 0;
+ te = 0;
+ while ((*s == *t) && !se && !te) {
+ s++;
+ t++;
+ switch(*s) {
+ case ' ':
+ case '\n':
+ case '\t':
+ case '\0': se = 1;
+ }
+ switch(*t) {
+ case ' ':
+ case '\n':
+ case '\t':
+ case '\0': te = 1;
+ }
+ }
+ if (!se || !te) {
+ // not terminal suffix difference
+ if (olds) return -1;
+ return 1;
+ }
+ olds = s;
+ s = strstr(s, MORPH_DERI_SFX);
+ if (!s || (sl && sl < s)) s = strstr(olds, MORPH_INFL_SFX);
+ if (!s || (sl && sl < s)) {
+ s = strstr(olds, MORPH_TERM_SFX);
+ olds = NULL;
+ }
+ oldt = t;
+ t = strstr(t, MORPH_DERI_SFX);
+ if (!t || (tl && tl < t)) t = strstr(oldt, MORPH_INFL_SFX);
+ if (!t || (tl && tl < t)) {
+ t = strstr(oldt, MORPH_TERM_SFX);
+ oldt = NULL;
+ }
+ }
+ if (!s && !t && se && te) return 0;
+ return 1;
}
-// leave only last {[^}]*} substring for handling zero morphemes
-char * delete_zeros(char * morphout) {
- char * p = morphout;
- char * q = p;
- char * q2 = NULL;
- int suffix = 0;
-
- for (;*p && *(p+1);) {
- switch (*p) {
- case '{':
- q2 = q;
- q--;
- break;
- case '}':
- if (q2) {
- suffix = 1;
- q--;
- }
- break;
- default:
- if (suffix) {
- q = q2;
- }
- suffix = 0;
- *q = *p;
- }
- p++;
- q++;
+int get_sfxcount(const char * morph)
+{
+ if (!morph || !*morph) return 0;
+ int n = 0;
+ const char * old = morph;
+ morph = strstr(morph, MORPH_DERI_SFX);
+ if (!morph) morph = strstr(old, MORPH_INFL_SFX);
+ if (!morph) morph = strstr(old, MORPH_TERM_SFX);
+ while (morph) {
+ n++;
+ old = morph;
+ morph = strstr(morph + 1, MORPH_DERI_SFX);
+ if (!morph) morph = strstr(old + 1, MORPH_INFL_SFX);
+ if (!morph) morph = strstr(old + 1, MORPH_TERM_SFX);
}
- *q = '\0';
- return morphout;
+ return n;
+}
+
+
+int fieldlen(const char * r)
+{
+ int n = 0;
+ while (r && *r != '\t' && *r != '\0' && *r != '\n' && *r != ' ') {
+ r++;
+ n++;
+ }
+ return n;
}
-#endif // END OF HUNSPELL_EXPERIMENTAL CODE
+
+char * copy_field(char * dest, const char * morph, const char * var)
+{
+ if (!morph) return NULL;
+ const char * beg = strstr(morph, var);
+ if (beg) {
+ char * d = dest;
+ for (beg += MORPH_TAG_LEN; *beg != ' ' && *beg != '\t' &&
+ *beg != '\n' && *beg != '\0'; d++, beg++) {
+ *d = *beg;
+ }
+ *d = '\0';
+ return dest;
+ }
+ return NULL;
+}
char * mystrrep(char * word, const char * pat, const char * rep) {
char * pos = strstr(word, pat);
if (pos) {
int replen = strlen(rep);
int patlen = strlen(pat);
if (replen < patlen) {
char * end = word + strlen(word);
@@ -500,16 +649,44 @@ char * mystrrep(char * word, const char
for (w_char * dest = w + l - 1; p < dest; p++, dest--) {
r=*p;
*p = *dest;
*dest = r;
}
u16_u8(word, MAXWORDUTF8LEN, w, l);
return 0;
}
+
+ int uniqlist(char ** list, int n) {
+ int i;
+ if (n < 2) return n;
+ for (i = 0; i < n; i++) {
+ for (int j = 0; j < i; j++) {
+ if (list[j] && list[i] && (strcmp(list[j], list[i]) == 0)) {
+ free(list[i]);
+ list[i] = NULL;
+ break;
+ }
+ }
+ }
+ int m = 1;
+ for (i = 1; i < n; i++) if (list[i]) {
+ list[m] = list[i];
+ m++;
+ }
+ return m;
+ }
+
+ void freelist(char *** list, int n) {
+ if (list && *list && n > 0) {
+ for (int i = 0; i < n; i++) if ((*list)[i]) free((*list)[i]);
+ free(*list);
+ *list = NULL;
+ }
+ }
// convert null terminated string to all caps
void mkallcap(char * p, const struct cs_info * csconv)
{
while (*p != '\0') {
*p = csconv[((unsigned char) *p)].cupper;
p++;
}
@@ -545,16 +722,30 @@ void mkallcap_utf(w_char * u, int nc, in
}
// convert null terminated string to have intial capital
void mkinitcap(char * p, const struct cs_info * csconv)
{
if (*p != '\0') *p = csconv[((unsigned char)*p)].cupper;
}
+ // conversion function for protected memory
+ void store_pointer(char * dest, char * source)
+ {
+ memcpy(dest, &source, sizeof(char *));
+ }
+
+ // conversion function for protected memory
+ char * get_stored_pointer(char * s)
+ {
+ char * p;
+ memcpy(&p, s, sizeof(char *));
+ return p;
+ }
+
#ifndef MOZILLA_CLIENT
// convert null terminated string to all caps using encoding
void enmkallcap(char * d, const char * p, const char * encoding)
{
struct cs_info * csconv = get_current_cs(encoding);
while (*p != '\0') {
*d++ = csconv[((unsigned char) *p)].cupper;
@@ -837,17 +1028,17 @@ struct cs_info iso1_tbl[] = {
{ 0x00, 0xf7, 0xf7 },
{ 0x00, 0xf8, 0xd8 },
{ 0x00, 0xf9, 0xd9 },
{ 0x00, 0xfa, 0xda },
{ 0x00, 0xfb, 0xdb },
{ 0x00, 0xfc, 0xdc },
{ 0x00, 0xfd, 0xdd },
{ 0x00, 0xfe, 0xde },
-{ 0x00, 0xff, 0xff },
+{ 0x00, 0xff, 0xff }
};
struct cs_info iso2_tbl[] = {
{ 0x00, 0x00, 0x00 },
{ 0x00, 0x01, 0x01 },
{ 0x00, 0x02, 0x02 },
{ 0x00, 0x03, 0x03 },
@@ -1097,17 +1288,17 @@ struct cs_info iso2_tbl[] = {
{ 0x00, 0xf7, 0xf7 },
{ 0x00, 0xf8, 0xd8 },
{ 0x00, 0xf9, 0xd9 },
{ 0x00, 0xfa, 0xda },
{ 0x00, 0xfb, 0xdb },
{ 0x00, 0xfc, 0xdc },
{ 0x00, 0xfd, 0xdd },
{ 0x00, 0xfe, 0xde },
-{ 0x00, 0xff, 0xff },
+{ 0x00, 0xff, 0xff }
};
struct cs_info iso3_tbl[] = {
{ 0x00, 0x00, 0x00 },
{ 0x00, 0x01, 0x01 },
{ 0x00, 0x02, 0x02 },
{ 0x00, 0x03, 0x03 },
@@ -1357,17 +1548,17 @@ struct cs_info iso3_tbl[] = {
{ 0x00, 0xf7, 0xf7 },
{ 0x00, 0xf8, 0xd8 },
{ 0x00, 0xf9, 0xd9 },
{ 0x00, 0xfa, 0xda },
{ 0x00, 0xfb, 0xdb },
{ 0x00, 0xfc, 0xdc },
{ 0x00, 0xfd, 0xdd },
{ 0x00, 0xfe, 0xde },
-{ 0x00, 0xff, 0xff },
+{ 0x00, 0xff, 0xff }
};
struct cs_info iso4_tbl[] = {
{ 0x00, 0x00, 0x00 },
{ 0x00, 0x01, 0x01 },
{ 0x00, 0x02, 0x02 },
{ 0x00, 0x03, 0x03 },
{ 0x00, 0x04, 0x04 },
@@ -1616,17 +1807,17 @@ struct cs_info iso4_tbl[] = {
{ 0x00, 0xf7, 0xf7 },
{ 0x00, 0xf8, 0xd8 },
{ 0x00, 0xf9, 0xd9 },
{ 0x00, 0xfa, 0xda },
{ 0x00, 0xfb, 0xdb },
{ 0x00, 0xfc, 0xdc },
{ 0x00, 0xfd, 0xdd },
{ 0x00, 0xfe, 0xde },
-{ 0x00, 0xff, 0xff },
+{ 0x00, 0xff, 0xff }
};
struct cs_info iso5_tbl[] = {
{ 0x00, 0x00, 0x00 },
{ 0x00, 0x01, 0x01 },
{ 0x00, 0x02, 0x02 },
{ 0x00, 0x03, 0x03 },
{ 0x00, 0x04, 0x04 },
@@ -1875,17 +2066,17 @@ struct cs_info iso5_tbl[] = {
{ 0x00, 0xf7, 0xa7 },
{ 0x00, 0xf8, 0xa8 },
{ 0x00, 0xf9, 0xa9 },
{ 0x00, 0xfa, 0xaa },
{ 0x00, 0xfb, 0xab },
{ 0x00, 0xfc, 0xac },
{ 0x00, 0xfd, 0xfd },
{ 0x00, 0xfe, 0xae },
-{ 0x00, 0xff, 0xaf },
+{ 0x00, 0xff, 0xaf }
};
struct cs_info iso6_tbl[] = {
{ 0x00, 0x00, 0x00 },
{ 0x00, 0x01, 0x01 },
{ 0x00, 0x02, 0x02 },
{ 0x00, 0x03, 0x03 },
{ 0x00, 0x04, 0x04 },
@@ -2134,17 +2325,17 @@ struct cs_info iso6_tbl[] = {
{ 0x00, 0xf7, 0xf7 },
{ 0x00, 0xf8, 0xf8 },
{ 0x00, 0xf9, 0xf9 },
{ 0x00, 0xfa, 0xfa },
{ 0x00, 0xfb, 0xfb },
{ 0x00, 0xfc, 0xfc },
{ 0x00, 0xfd, 0xfd },
{ 0x00, 0xfe, 0xfe },
-{ 0x00, 0xff, 0xff },
+{ 0x00, 0xff, 0xff }
};
struct cs_info iso7_tbl[] = {
{ 0x00, 0x00, 0x00 },
{ 0x00, 0x01, 0x01 },
{ 0x00, 0x02, 0x02 },
{ 0x00, 0x03, 0x03 },
{ 0x00, 0x04, 0x04 },
@@ -2393,17 +2584,17 @@ struct cs_info iso7_tbl[] = {
{ 0x00, 0xf7, 0xd7 },
{ 0x00, 0xf8, 0xd8 },
{ 0x00, 0xf9, 0xd9 },
{ 0x00, 0xfa, 0xda },
{ 0x00, 0xfb, 0xdb },
{ 0x00, 0xfc, 0xbc },
{ 0x00, 0xfd, 0xbe },
{ 0x00, 0xfe, 0xbf },
-{ 0x00, 0xff, 0xff },
+{ 0x00, 0xff, 0xff }
};
struct cs_info iso8_tbl[] = {
{ 0x00, 0x00, 0x00 },
{ 0x00, 0x01, 0x01 },
{ 0x00, 0x02, 0x02 },
{ 0x00, 0x03, 0x03 },
{ 0x00, 0x04, 0x04 },
@@ -2652,17 +2843,17 @@ struct cs_info iso8_tbl[] = {
{ 0x00, 0xf7, 0xf7 },
{ 0x00, 0xf8, 0xf8 },
{ 0x00, 0xf9, 0xf9 },
{ 0x00, 0xfa, 0xfa },
{ 0x00, 0xfb, 0xfb },
{ 0x00, 0xfc, 0xfc },
{ 0x00, 0xfd, 0xfd },
{ 0x00, 0xfe, 0xfe },
-{ 0x00, 0xff, 0xff },
+{ 0x00, 0xff, 0xff }
};
struct cs_info iso9_tbl[] = {
{ 0x00, 0x00, 0x00 },
{ 0x00, 0x01, 0x01 },
{ 0x00, 0x02, 0x02 },
{ 0x00, 0x03, 0x03 },
{ 0x00, 0x04, 0x04 },
@@ -2911,17 +3102,17 @@ struct cs_info iso9_tbl[] = {
{ 0x00, 0xf7, 0xf7 },
{ 0x00, 0xf8, 0xd8 },
{ 0x00, 0xf9, 0xd9 },
{ 0x00, 0xfa, 0xda },
{ 0x00, 0xfb, 0xdb },
{ 0x00, 0xfc, 0xdc },
{ 0x00, 0xfd, 0x49 },
{ 0x00, 0xfe, 0xde },
-{ 0x00, 0xff, 0xff },
+{ 0x00, 0xff, 0xff }
};
struct cs_info iso10_tbl[] = {
{ 0x00, 0x00, 0x00 },
{ 0x00, 0x01, 0x01 },
{ 0x00, 0x02, 0x02 },
{ 0x00, 0x03, 0x03 },
{ 0x00, 0x04, 0x04 },
@@ -3170,17 +3361,17 @@ struct cs_info iso10_tbl[] = {
{ 0x00, 0xf7, 0xf7 },
{ 0x00, 0xf8, 0xf8 },
{ 0x00, 0xf9, 0xf9 },
{ 0x00, 0xfa, 0xfa },
{ 0x00, 0xfb, 0xfb },
{ 0x00, 0xfc, 0xfc },
{ 0x00, 0xfd, 0xfd },
{ 0x00, 0xfe, 0xfe },
-{ 0x00, 0xff, 0xff },
+{ 0x00, 0xff, 0xff }
};
struct cs_info koi8r_tbl[] = {
{ 0x00, 0x00, 0x00 },
{ 0x00, 0x01, 0x01 },
{ 0x00, 0x02, 0x02 },
{ 0x00, 0x03, 0x03 },
{ 0x00, 0x04, 0x04 },
@@ -3429,17 +3620,17 @@ struct cs_info koi8r_tbl[] = {
{ 0x01, 0xd7, 0xf7 },
{ 0x01, 0xd8, 0xf8 },
{ 0x01, 0xd9, 0xf9 },
{ 0x01, 0xda, 0xfa },
{ 0x01, 0xdb, 0xfb },
{ 0x01, 0xdc, 0xfc },
{ 0x01, 0xdd, 0xfd },
{ 0x01, 0xde, 0xfe },
-{ 0x01, 0xdf, 0xff },
+{ 0x01, 0xdf, 0xff }
};
struct cs_info koi8u_tbl[] = {
{ 0x00, 0x00, 0x00 },
{ 0x00, 0x01, 0x01 },
{ 0x00, 0x02, 0x02 },
{ 0x00, 0x03, 0x03 },
{ 0x00, 0x04, 0x04 },
@@ -3688,17 +3879,17 @@ struct cs_info koi8u_tbl[] = {
{ 0x01, 0xd7, 0xf7 },
{ 0x01, 0xd8, 0xf8 },
{ 0x01, 0xd9, 0xf9 },
{ 0x01, 0xda, 0xfa },
{ 0x01, 0xdb, 0xfb },
{ 0x01, 0xdc, 0xfc },
{ 0x01, 0xdd, 0xfd },
{ 0x01, 0xde, 0xfe },
-{ 0x01, 0xdf, 0xff },
+{ 0x01, 0xdf, 0xff }
};
struct cs_info cp1251_tbl[] = {
{ 0x00, 0x00, 0x00 },
{ 0x00, 0x01, 0x01 },
{ 0x00, 0x02, 0x02 },
{ 0x00, 0x03, 0x03 },
{ 0x00, 0x04, 0x04 },
@@ -3947,17 +4138,17 @@ struct cs_info cp1251_tbl[] = {
{ 0x00, 0xf7, 0xd7 },
{ 0x00, 0xf8, 0xd8 },
{ 0x00, 0xf9, 0xd9 },
{ 0x00, 0xfa, 0xda },
{ 0x00, 0xfb, 0xdb },
{ 0x00, 0xfc, 0xdc },
{ 0x00, 0xfd, 0xdd },
{ 0x00, 0xfe, 0xde },
-{ 0x00, 0xff, 0xdf },
+{ 0x00, 0xff, 0xdf }
};
struct cs_info iso13_tbl[] = {
{ 0x00, 0x00, 0x00 },
{ 0x00, 0x01, 0x01 },
{ 0x00, 0x02, 0x02 },
{ 0x00, 0x03, 0x03 },
{ 0x00, 0x04, 0x04 },
@@ -4206,17 +4397,17 @@ struct cs_info iso13_tbl[] = {
{ 0x00, 0xF7, 0xF7 },
{ 0x00, 0xF8, 0xD8 },
{ 0x00, 0xF9, 0xD9 },
{ 0x00, 0xFA, 0xDA },
{ 0x00, 0xFB, 0xDB },
{ 0x00, 0xFC, 0xDC },
{ 0x00, 0xFD, 0xDD },
{ 0x00, 0xFE, 0xDE },
-{ 0x00, 0xFF, 0xFF },
+{ 0x00, 0xFF, 0xFF }
};
struct cs_info iso14_tbl[] = {
{ 0x00, 0x00, 0x00 },
{ 0x00, 0x01, 0x01 },
{ 0x00, 0x02, 0x02 },
{ 0x00, 0x03, 0x03 },
@@ -4466,17 +4657,17 @@ struct cs_info iso14_tbl[] = {
{ 0x00, 0xf7, 0xd7 },
{ 0x00, 0xf8, 0xd8 },
{ 0x00, 0xf9, 0xd9 },
{ 0x00, 0xfa, 0xda },
{ 0x00, 0xfb, 0xdb },
{ 0x00, 0xfc, 0xdc },
{ 0x00, 0xfd, 0xdd },
{ 0x00, 0xfe, 0xde },
-{ 0x00, 0xff, 0xff },
+{ 0x00, 0xff, 0xff }
};
struct cs_info iso15_tbl[] = {
{ 0x00, 0x00, 0x00 },
{ 0x00, 0x01, 0x01 },
{ 0x00, 0x02, 0x02 },
{ 0x00, 0x03, 0x03 },
{ 0x00, 0x04, 0x04 },
@@ -4725,17 +4916,17 @@ struct cs_info iso15_tbl[] = {
{ 0x00, 0xf7, 0xf7 },
{ 0x00, 0xf8, 0xd8 },
{ 0x00, 0xf9, 0xd9 },
{ 0x00, 0xfa, 0xda },
{ 0x00, 0xfb, 0xdb },
{ 0x00, 0xfc, 0xdc },
{ 0x00, 0xfd, 0xdd },
{ 0x00, 0xfe, 0xde },
-{ 0x00, 0xff, 0xbe },
+{ 0x00, 0xff, 0xbe }
};
struct cs_info iscii_devanagari_tbl[] = {
{ 0x00, 0x00, 0x00 },
{ 0x00, 0x01, 0x01 },
{ 0x00, 0x02, 0x02 },
{ 0x00, 0x03, 0x03 },
{ 0x00, 0x04, 0x04 },
@@ -4984,45 +5175,46 @@ struct cs_info iscii_devanagari_tbl[] =
{ 0x00, 0xf7, 0xf7 },
{ 0x00, 0xf8, 0xf8 },
{ 0x00, 0xf9, 0xf9 },
{ 0x00, 0xfa, 0xfa },
{ 0x00, 0xfb, 0xfb },
{ 0x00, 0xfc, 0xfc },
{ 0x00, 0xfd, 0xfd },
{ 0x00, 0xfe, 0xfe },
-{ 0x00, 0xff, 0xff },
+{ 0x00, 0xff, 0xff }
};
-struct enc_entry encds[] = {
+static struct enc_entry encds[] = {
{"ISO8859-1",iso1_tbl},
{"ISO8859-2",iso2_tbl},
{"ISO8859-3",iso3_tbl},
{"ISO8859-4",iso4_tbl},
{"ISO8859-5",iso5_tbl},
{"ISO8859-6",iso6_tbl},
{"ISO8859-7",iso7_tbl},
{"ISO8859-8",iso8_tbl},
{"ISO8859-9",iso9_tbl},
{"ISO8859-10",iso10_tbl},
{"KOI8-R",koi8r_tbl},
{"KOI8-U",koi8u_tbl},
{"microsoft-cp1251",cp1251_tbl},
{"ISO8859-13", iso13_tbl},
{"ISO8859-14", iso14_tbl},
{"ISO8859-15", iso15_tbl},
-{"ISCII-DEVANAGARI", iscii_devanagari_tbl},
+{"ISCII-DEVANAGARI", iscii_devanagari_tbl}
};
struct cs_info * get_current_cs(const char * es) {
struct cs_info * ccs = encds[0].cs_table;
int n = sizeof(encds) / sizeof(encds[0]);
for (int i = 0; i < n; i++) {
if (strcmp(es,encds[i].enc_name) == 0) {
ccs = encds[i].cs_table;
+ break;
}
}
return ccs;
}
#else
// XXX This function was rewritten for mozilla. Instead of storing the
// conversion tables static in this file, create them when needed
// with help the mozilla backend.
@@ -5258,24 +5450,24 @@ int unicodeisalpha(unsigned short c)
}
/* get type of capitalization */
int get_captype(char * word, int nl, cs_info * csconv) {
// now determine the capitalization type of the first nl letters
int ncap = 0;
int nneutral = 0;
int firstcap = 0;
-
- for (char * q = word; *q != '\0'; q++) {
- if (csconv[*((unsigned char *)q)].ccase) ncap++;
- if (csconv[*((unsigned char *)q)].cupper == csconv[*((unsigned char *)q)].clower) nneutral++;
- }
- if (ncap) {
- firstcap = csconv[*((unsigned char *) word)].ccase;
- }
+ if (csconv == NULL) return NOCAP;
+ for (char * q = word; *q != '\0'; q++) {
+ if (csconv[*((unsigned char *)q)].ccase) ncap++;
+ if (csconv[*((unsigned char *)q)].cupper == csconv[*((unsigned char *)q)].clower) nneutral++;
+ }
+ if (ncap) {
+ firstcap = csconv[*((unsigned char *) word)].ccase;
+ }
// now finally set the captype
if (ncap == 0) {
return NOCAP;
} else if ((ncap == 1) && firstcap) {
return INITCAP;
} else if ((ncap == nl) || ((ncap + nneutral) == nl)) {
return ALLCAP;
@@ -5343,53 +5535,54 @@ void remove_ignored_chars(char * word, c
if (!strchr(ignored_chars, *p)) {
*word = *p;
word++;
}
}
*word = '\0';
}
-int parse_string(char * line, char ** out, const char * warnvar)
+int parse_string(char * line, char ** out, int ln)
{
char * tp = line;
char * piece;
int i = 0;
int np = 0;
if (*out) {
- HUNSPELL_WARNING(stderr, "error: duplicate %s line\n", warnvar);
+ HUNSPELL_WARNING(stderr, "error: line %d: multiple definitions\n", ln);
return 1;
}
piece = mystrsep(&tp, 0);
while (piece) {
if (*piece != '\0') {
switch(i) {
case 0: { np++; break; }
case 1: {
*out = mystrdup(piece);
+ if (!*out) return 1;
np++;
break;
}
default: break;
}
i++;
}
- free(piece);
+ // free(piece);
piece = mystrsep(&tp, 0);
}
if (np != 2) {
- HUNSPELL_WARNING(stderr, "error: missing %s information\n", warnvar);
+ HUNSPELL_WARNING(stderr, "error: line %d: missing data\n", ln);
return 1;
}
return 0;
}
-int parse_array(char * line, char ** out,
- unsigned short ** out_utf16, int * out_utf16_len, const char * name, int utf8) {
- if (parse_string(line, out, name)) return 1;
+int parse_array(char * line, char ** out, unsigned short ** out_utf16,
+ int * out_utf16_len, int utf8, int ln) {
+ if (parse_string(line, out, ln)) return 1;
if (utf8) {
w_char w[MAXWORDLEN];
int n = u8_u16(w, MAXWORDLEN, *out);
if (n > 0) {
flag_qsort((unsigned short *) w, 0, n);
*out_utf16 = (unsigned short *) malloc(n * sizeof(unsigned short));
if (!*out_utf16) return 1;
memcpy(*out_utf16, w, n * sizeof(unsigned short));
--- a/extensions/spellcheck/hunspell/src/csutil.hxx
+++ b/extensions/spellcheck/hunspell/src/csutil.hxx
@@ -54,38 +54,64 @@
*
******* END LICENSE BLOCK *******/
#ifndef __CSUTILHXX__
#define __CSUTILHXX__
// First some base level utility routines
+#include "w_char.hxx"
+
+// casing
#define NOCAP 0
#define INITCAP 1
#define ALLCAP 2
#define HUHCAP 3
#define HUHINITCAP 4
-#define FIELD_STEM "st:"
-#define FIELD_POS "po:"
-#define FIELD_SUFF "su:"
-#define FIELD_PREF "pr:"
-#define FIELD_FREQ "fr:"
-#define FIELD_PHON "ph:"
-#define FIELD_HYPH "hy:"
-#define FIELD_COMP "co:"
+// default encoding and keystring
+#define SPELL_ENCODING "ISO8859-1"
+#define SPELL_KEYSTRING "qwertyuiop|asdfghjkl|zxcvbnm"
+
+// default morphological fields
+#define MORPH_STEM "st:"
+#define MORPH_ALLOMORPH "al:"
+#define MORPH_POS "po:"
+#define MORPH_DERI_PFX "dp:"
+#define MORPH_INFL_PFX "ip:"
+#define MORPH_TERM_PFX "tp:"
+#define MORPH_DERI_SFX "ds:"
+#define MORPH_INFL_SFX "is:"
+#define MORPH_TERM_SFX "ts:"
+#define MORPH_SURF_PFX "sp:"
+#define MORPH_FREQ "fr:"
+#define MORPH_PHON "ph:"
+#define MORPH_HYPH "hy:"
+#define MORPH_PART "pa:"
+#define MORPH_FLAG "fl:"
+#define MORPH_HENTRY "_H:"
+#define MORPH_TAG_LEN strlen(MORPH_STEM)
+
+#define MSEP_FLD ' '
+#define MSEP_REC '\n'
+#define MSEP_ALT '\v'
// default flags
-#define ONLYUPCASEFLAG 65535
+#define DEFAULTFLAGS 65510
+#define FORBIDDENWORD 65510
+#define ONLYUPCASEFLAG 65511
-typedef struct {
- unsigned char l;
- unsigned char h;
-} w_char;
+// hash entry macros
+#define HENTRY_DATA(h) (h->var ? ((h->var & H_OPT_ALIASM) ? \
+ get_stored_pointer(&(h->word) + h->blen + 1) : &(h->word) + h->blen + 1) : NULL)
+// NULL-free version for warning-free OOo build
+#define HENTRY_DATA2(h) (h->var ? ((h->var & H_OPT_ALIASM) ? \
+ get_stored_pointer(&(h->word) + h->blen + 1) : &(h->word) + h->blen + 1) : "")
+#define HENTRY_FIND(h,p) (HENTRY_DATA(h) ? strstr(HENTRY_DATA(h), p) : NULL)
#define w_char_eq(a,b) (((a).l == (b).l) && ((a).h == (b).h))
// convert UTF-16 characters to UTF-8
char * u16_u8(char * dest, int size, const w_char * src, int srclen);
// convert UTF-8 characters to UTF-16
int u8_u16(w_char * dest, int size, const char * src);
@@ -97,61 +123,62 @@ void flag_qsort(unsigned short flags[],
int flag_bsearch(unsigned short flags[], unsigned short flag, int right);
// remove end of line char(s)
void mychomp(char * s);
// duplicate string
char * mystrdup(const char * s);
+// strcat for limited length destination string
+char * mystrcat(char * dest, const char * st, int max);
+
// duplicate reverse of string
char * myrevstrdup(const char * s);
// parse into tokens with char delimiter
char * mystrsep(char ** sptr, const char delim);
// parse into tokens with char delimiter
char * mystrsep2(char ** sptr, const char delim);
// parse into tokens with char delimiter
char * mystrrep(char *, const char *, const char *);
// append s to ends of every lines in text
void strlinecat(char * lines, const char * s);
// tokenize into lines with new line
- int line_tok(const char * text, char *** lines);
+ int line_tok(const char * text, char *** lines, char breakchar);
// tokenize into lines with new line and uniq in place
- char * line_uniq(char * text);
+ char * line_uniq(char * text, char breakchar);
+ char * line_uniq_app(char ** text, char breakchar);
-// change \n to c in place
- char * line_join(char * text, char c);
-
-// leave only last {[^}]*} pattern in string
- char * delete_zeros(char * morphout);
+// change oldchar to newchar in place
+ char * tr(char * text, char oldc, char newc);
// reverse word
int reverseword(char *);
// reverse word
int reverseword_utf(char *);
+// remove duplicates
+ int uniqlist(char ** list, int n);
+
+// free character array list
+ void freelist(char *** list, int n);
+
// character encoding information
struct cs_info {
unsigned char ccase;
unsigned char clower;
unsigned char cupper;
};
-// two character arrays
-struct replentry {
- char * pattern;
- char * pattern2;
-};
-
// Unicode character encoding information
struct unicode_info {
unsigned short c;
unsigned short cupper;
unsigned short clower;
};
struct unicode_info2 {
@@ -220,14 +247,27 @@ int get_captype(char * q, int nl, cs_inf
int get_captype_utf8(w_char * q, int nl, int langnum);
// strip all ignored characters in the string
void remove_ignored_chars_utf(char * word, unsigned short ignored_chars[], int ignored_len);
// strip all ignored characters in the string
void remove_ignored_chars(char * word, char * ignored_chars);
-int parse_string(char * line, char ** out, const char * name);
+int parse_string(char * line, char ** out, int ln);
+
+int parse_array(char * line, char ** out, unsigned short ** out_utf16,
+ int * out_utf16_len, int utf8, int ln);
+
+int fieldlen(const char * r);
+char * copy_field(char * dest, const char * morph, const char * var);
-int parse_array(char * line, char ** out,
- unsigned short ** out_utf16, int * out_utf16_len, const char * name, int utf8);
+int morphcmp(const char * s, const char * t);
+
+int get_sfxcount(const char * morph);
+
+// conversion function for protected memory
+void store_pointer(char * dest, char * source);
+
+// conversion function for protected memory
+char * get_stored_pointer(char * s);
#endif
new file mode 100644
--- /dev/null
+++ b/extensions/spellcheck/hunspell/src/dictmgr.cpp
@@ -0,0 +1,219 @@
+/******* BEGIN LICENSE BLOCK *******
+ * Version: MPL 1.1/GPL 2.0/LGPL 2.1
+ *
+ * The contents of this file are subject to the Mozilla Public License Version
+ * 1.1 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * http://www.mozilla.org/MPL/
+ *
+ * Software distributed under the License is distributed on an "AS IS" basis,
+ * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+ * for the specific language governing rights and limitations under the
+ * License.
+ *
+ * The Initial Developers of the Original Code are Kevin Hendricks (MySpell)
+ * and László Németh (Hunspell). Portions created by the Initial Developers
+ * are Copyright (C) 2002-2005 the Initial Developers. All Rights Reserved.
+ *
+ * Contributor(s): László Németh (nemethl@gyorsposta.hu)
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * either the GNU General Public License Version 2 or later (the "GPL"), or
+ * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+ * in which case the provisions of the GPL or the LGPL are applicable instead
+ * of those above. If you wish to allow use of your version of this file only
+ * under the terms of either the GPL or the LGPL, and not to allow others to
+ * use your version of this file under the terms of the MPL, indicate your
+ * decision by deleting the provisions above and replace them with the notice
+ * and other provisions required by the GPL or the LGPL. If you do not delete
+ * the provisions above, a recipient may use your version of this file under
+ * the terms of any one of the MPL, the GPL or the LGPL.
+ *
+ ******* END LICENSE BLOCK *******/
+
+#ifndef MOZILLA_CLIENT
+#include <cstdlib>
+#include <cstring>
+#include <cstdio>
+#include <cctype>
+#else
+#include <stdlib.h>
+#include <string.h>
+#include <stdio.h>
+#include <ctype.h>
+#endif
+
+#include "dictmgr.hxx"
+
+#ifndef MOZILLA_CLIENT
+#ifndef W32
+using namespace std;
+#endif
+#endif
+
+DictMgr::DictMgr(const char * dictpath, const char * etype)
+{
+ // load list of etype entries
+ numdict = 0;
+ pdentry = (dictentry *)malloc(MAXDICTIONARIES*sizeof(struct dictentry));
+ if (pdentry) {
+ if (parse_file(dictpath, etype)) {
+ numdict = 0;
+ // no dictionary.lst found is okay
+ }
+ } else {
+ numdict = 0;
+ }
+}
+
+
+DictMgr::~DictMgr()
+{
+ dictentry * pdict = NULL;
+ if (pdentry) {
+ pdict = pdentry;
+ for (int i=0;i<numdict;i++) {
+ if (pdict->lang) {
+ free(pdict->lang);
+ pdict->lang = NULL;
+ }
+ if (pdict->region) {
+ free(pdict->region);
+ pdict->region=NULL;
+ }
+ if (pdict->filename) {
+ free(pdict->filename);
+ pdict->filename = NULL;
+ }
+ pdict++;
+ }
+ free(pdentry);
+ pdentry = NULL;
+ pdict = NULL;
+ }
+ numdict = 0;
+}
+
+
+// read in list of etype entries and build up structure to describe them
+int DictMgr::parse_file(const char * dictpath, const char * etype)
+{
+
+ int i;
+ char line[MAXDICTENTRYLEN+1];
+ dictentry * pdict = pdentry;
+
+ // open the dictionary list file
+ FILE * dictlst;
+ dictlst = fopen(dictpath,"r");
+ if (!dictlst) {
+ return 1;
+ }
+
+ // step one is to parse the dictionary list building up the
+ // descriptive structures
+
+ // read in each line ignoring any that dont start with etype
+ while (fgets(line,MAXDICTENTRYLEN,dictlst)) {
+ mychomp(line);
+
+ /* parse in a dictionary entry */
+ if (strncmp(line,etype,4) == 0) {
+ if (numdict < MAXDICTIONARIES) {
+ char * tp = line;
+ char * piece;
+ i = 0;
+ while ((piece=mystrsep(&tp,' '))) {
+ if (*piece != '\0') {
+ switch(i) {
+ case 0: break;
+ case 1: pdict->lang = mystrdup(piece); break;
+ case 2: if (strcmp (piece, "ANY") == 0)
+ pdict->region = mystrdup("");
+ else
+ pdict->region = mystrdup(piece);
+ break;
+ case 3: pdict->filename = mystrdup(piece); break;
+ default: break;
+ }
+ i++;
+ }
+ free(piece);
+ }
+ if (i == 4) {
+ numdict++;
+ pdict++;
+ } else {
+ fprintf(stderr,"dictionary list corruption in line \"%s\"\n",line);
+ fflush(stderr);
+ }
+ }
+ }
+ }
+ fclose(dictlst);
+ return 0;
+}
+
+// return text encoding of dictionary
+int DictMgr::get_list(dictentry ** ppentry)
+{
+ *ppentry = pdentry;
+ return numdict;
+}
+
+
+
+// strip strings into token based on single char delimiter
+// acts like strsep() but only uses a delim char and not
+// a delim string
+
+char * DictMgr::mystrsep(char ** stringp, const char delim)
+{
+ char * rv = NULL;
+ char * mp = *stringp;
+ int n = strlen(mp);
+ if (n > 0) {
+ char * dp = (char *)memchr(mp,(int)((unsigned char)delim),n);
+ if (dp) {
+ *stringp = dp+1;
+ int nc = (int)((unsigned long)dp - (unsigned long)mp);
+ rv = (char *) malloc(nc+1);
+ if (rv) {
+ memcpy(rv,mp,nc);
+ *(rv+nc) = '\0';
+ return rv;
+ }
+ } else {
+ rv = (char *) malloc(n+1);
+ if (rv) {
+ memcpy(rv, mp, n);
+ *(rv+n) = '\0';
+ *stringp = mp + n;
+ return rv;
+ }
+ }
+ }
+ return NULL;
+}
+
+
+// replaces strdup with ansi version
+char * DictMgr::mystrdup(const char * s)
+{
+ char * d = NULL;
+ if (s) {
+ int sl = strlen(s);
+ d = (char *) malloc(((sl+1) * sizeof(char)));
+ if (d) memcpy(d,s,((sl+1)*sizeof(char)));
+ }
+ return d;
+}
+
+
+// remove cross-platform text line end characters
+void DictMgr:: mychomp(char * s)
+{
+ int k = strlen(s);
+ if ((k > 0) && ((*(s+k-1)=='\r') || (*(s+k-1)=='\n'))) *(s+k-1) = '\0';
+ if ((k > 1) && (*(s+k-2) == '\r')) *(s+k-2) = '\0';
+}
new file mode 100644
--- /dev/null
+++ b/extensions/spellcheck/hunspell/src/dictmgr.hxx
@@ -0,0 +1,67 @@
+/******* BEGIN LICENSE BLOCK *******
+ * Version: MPL 1.1/GPL 2.0/LGPL 2.1
+ *
+ * The contents of this file are subject to the Mozilla Public License Version
+ * 1.1 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * http://www.mozilla.org/MPL/
+ *
+ * Software distributed under the License is distributed on an "AS IS" basis,
+ * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+ * for the specific language governing rights and limitations under the
+ * License.
+ *
+ * The Initial Developers of the Original Code are Kevin Hendricks (MySpell)
+ * and László Németh (Hunspell). Portions created by the Initial Developers
+ * are Copyright (C) 2002-2005 the Initial Developers. All Rights Reserved.
+ *
+ * Contributor(s): László Németh (nemethl@gyorsposta.hu)
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * either the GNU General Public License Version 2 or later (the "GPL"), or
+ * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+ * in which case the provisions of the GPL or the LGPL are applicable instead
+ * of those above. If you wish to allow use of your version of this file only
+ * under the terms of either the GPL or the LGPL, and not to allow others to
+ * use your version of this file under the terms of the MPL, indicate your
+ * decision by deleting the provisions above and replace them with the notice
+ * and other provisions required by the GPL or the LGPL. If you do not delete
+ * the provisions above, a recipient may use your version of this file under
+ * the terms of any one of the MPL, the GPL or the LGPL.
+ *
+ ******* END LICENSE BLOCK *******/
+
+#ifndef _DICTMGR_HXX_
+#define _DICTMGR_HXX_
+
+#define MAXDICTIONARIES 100
+#define MAXDICTENTRYLEN 1024
+
+struct dictentry {
+ char * filename;
+ char * lang;
+ char * region;
+};
+
+
+class DictMgr
+{
+
+ int numdict;
+ dictentry * pdentry;
+
+public:
+
+ DictMgr(const char * dictpath, const char * etype);
+ ~DictMgr();
+ int get_list(dictentry** ppentry);
+
+private:
+ int parse_file(const char * dictpath, const char * etype);
+ char * mystrsep(char ** stringp, const char delim);
+ char * mystrdup(const char * s);
+ void mychomp(char * s);
+
+};
+
+#endif
new file mode 100644
--- /dev/null
+++ b/extensions/spellcheck/hunspell/src/filemgr.cpp
@@ -0,0 +1,84 @@
+/******* BEGIN LICENSE BLOCK *******
+ * Version: MPL 1.1/GPL 2.0/LGPL 2.1
+ *
+ * The contents of this file are subject to the Mozilla Public License Version
+ * 1.1 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * http://www.mozilla.org/MPL/
+ *
+ * Software distributed under the License is distributed on an "AS IS" basis,
+ * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+ * for the specific language governing rights and limitations under the
+ * License.
+ *
+ * The Initial Developers of the Original Code are Kevin Hendricks (MySpell)
+ * and László Németh (Hunspell). Portions created by the Initial Developers
+ * are Copyright (C) 2002-2005 the Initial Developers. All Rights Reserved.
+ *
+ * Contributor(s): László Németh (nemethl@gyorsposta.hu)
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * either the GNU General Public License Version 2 or later (the "GPL"), or
+ * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+ * in which case the provisions of the GPL or the LGPL are applicable instead
+ * of those above. If you wish to allow use of your version of this file only
+ * under the terms of either the GPL or the LGPL, and not to allow others to
+ * use your version of this file under the terms of the MPL, indicate your
+ * decision by deleting the provisions above and replace them with the notice
+ * and other provisions required by the GPL or the LGPL. If you do not delete
+ * the provisions above, a recipient may use your version of this file under
+ * the terms of any one of the MPL, the GPL or the LGPL.
+ *
+ ******* END LICENSE BLOCK *******/
+
+#ifndef MOZILLA_CLIENT
+#include <cstdlib>
+#include <cstring>
+#include <cstdio>
+#else
+#include <stdlib.h>
+#include <string.h>
+#include <stdio.h>
+#endif
+
+#include "filemgr.hxx"
+
+int FileMgr::fail(const char * err, const char * par) {
+ fprintf(stderr, err, par);
+ return -1;
+}
+
+FileMgr::FileMgr(const char * file, const char * key) {
+ linenum = 0;
+ hin = NULL;
+ fin = fopen(file, "r");
+ if (!fin) {
+ // check hzipped file
+ char * st = (char *) malloc(strlen(file) + strlen(HZIP_EXTENSION));
+ if (st) {
+ strcpy(st, file);
+ strcat(st, HZIP_EXTENSION);
+ hin = new Hunzip(st, key);
+ }
+ }
+ if (!fin && !hin) fail(MSG_OPEN, file);
+}
+
+FileMgr::~FileMgr()
+{
+ if (fin) fclose(fin);
+ if (hin) delete hin;
+}
+
+char * FileMgr::getline() {
+ const char * l;
+ linenum++;
+ if (fin) return fgets(in, BUFSIZE - 1, fin);
+ if (hin && (l = hin->getline())) return strcpy(in, l);
+ linenum--;
+ return NULL;
+}
+
+int FileMgr::getlinenum() {
+ return linenum;
+}
new file mode 100644
--- /dev/null
+++ b/extensions/spellcheck/hunspell/src/filemgr.hxx
@@ -0,0 +1,54 @@
+/******* BEGIN LICENSE BLOCK *******
+ * Version: MPL 1.1/GPL 2.0/LGPL 2.1
+ *
+ * The contents of this file are subject to the Mozilla Public License Version
+ * 1.1 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * http://www.mozilla.org/MPL/
+ *
+ * Software distributed under the License is distributed on an "AS IS" basis,
+ * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+ * for the specific language governing rights and limitations under the
+ * License.
+ *
+ * The Initial Developers of the Original Code are Kevin Hendricks (MySpell)
+ * and László Németh (Hunspell). Portions created by the Initial Developers
+ * are Copyright (C) 2002-2005 the Initial Developers. All Rights Reserved.
+ *
+ * Contributor(s): László Németh (nemethl@gyorsposta.hu)
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * either the GNU General Public License Version 2 or later (the "GPL"), or
+ * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+ * in which case the provisions of the GPL or the LGPL are applicable instead
+ * of those above. If you wish to allow use of your version of this file only
+ * under the terms of either the GPL or the LGPL, and not to allow others to
+ * use your version of this file under the terms of the MPL, indicate your
+ * decision by deleting the provisions above and replace them with the notice
+ * and other provisions required by the GPL or the LGPL. If you do not delete
+ * the provisions above, a recipient may use your version of this file under
+ * the terms of any one of the MPL, the GPL or the LGPL.
+ *
+ ******* END LICENSE BLOCK *******/
+
+#ifndef _FILEMGR_HXX_
+#define _FILEMGR_HXX_
+
+#include "hunzip.hxx"
+
+class FileMgr
+{
+protected:
+ FILE * fin;
+ Hunzip * hin;
+ char in[BUFSIZE + 50]; // input buffer
+ int fail(const char * err, const char * par);
+ int linenum;
+
+public:
+ FileMgr(const char * filename, const char * key = NULL);
+ ~FileMgr();
+ char * getline();
+ int getlinenum();
+};
+#endif
--- a/extensions/spellcheck/hunspell/src/hashmgr.cpp
+++ b/extensions/spellcheck/hunspell/src/hashmgr.cpp
@@ -61,33 +61,33 @@
#include <cctype>
#else
#include <stdlib.h>
#include <string.h>
#include <stdio.h>
#include <ctype.h>
#endif
-#include "hashmgr.hxx"
+#include "atypes.hxx"
#include "csutil.hxx"
-#include "atypes.hxx"
+#include "hashmgr.hxx"
#ifdef MOZILLA_CLIENT
#ifdef __SUNPRO_CC // for SunONE Studio compiler
using namespace std;
#endif
#else
#ifndef W32
using namespace std;
#endif
#endif
// build a hash table from a munched word list
-HashMgr::HashMgr(const char * tpath, const char * apath)
+HashMgr::HashMgr(const char * tpath, const char * apath, const char * key)
{
tablesize = 0;
tableptr = NULL;
flag_mode = FLAG_CHAR;
complexprefixes = 0;
utf8 = 0;
langnum = 0;
lang = NULL;
@@ -95,19 +95,19 @@ HashMgr::HashMgr(const char * tpath, con
csconv = 0;
ignorechars = NULL;
ignorechars_utf16 = NULL;
ignorechars_utf16_len = 0;
numaliasf = 0;
aliasf = NULL;
numaliasm = 0;
aliasm = NULL;
- forbiddenword = FLAG_NULL; // forbidden word signing flag
- load_config(apath);
- int ec = load_tables(tpath);
+ forbiddenword = FORBIDDENWORD; // forbidden word signing flag
+ load_config(apath, key);
+ int ec = load_tables(tpath, key);
if (ec) {
/* error condition - what should we do here */
HUNSPELL_WARNING(stderr, "Hash Manager Error : %d\n",ec);
if (tableptr) {
free(tableptr);
tableptr = NULL;
}
tablesize = 0;
@@ -118,30 +118,19 @@ HashMgr::HashMgr(const char * tpath, con
HashMgr::~HashMgr()
{
if (tableptr) {
// now pass through hash table freeing up everything
// go through column by column of the table
for (int i=0; i < tablesize; i++) {
struct hentry * pt = tableptr[i];
struct hentry * nt = NULL;
-/* if (pt) {
- if (pt->astr && (!aliasf || TESTAFF(pt->astr, ONLYUPCASEFLAG, pt->alen))) free(pt->astr);
-#ifdef HUNSPELL_EXPERIMENTAL
- if (pt->description && !aliasm) free(pt->description);
-#endif
- pt = pt->next;
- }
-*/
while(pt) {
nt = pt->next;
if (pt->astr && (!aliasf || TESTAFF(pt->astr, ONLYUPCASEFLAG, pt->alen))) free(pt->astr);
-#ifdef HUNSPELL_EXPERIMENTAL
- if (pt->description && !aliasm) free(pt->description);
-#endif
free(pt);
pt = nt;
}
}
free(tableptr);
}
tablesize = 0;
@@ -188,66 +177,59 @@ struct hentry * HashMgr::lookup(const ch
return NULL;
}
// add a word to the hash table (private)
int HashMgr::add_word(const char * word, int wbl, int wcl, unsigned short * aff,
int al, const char * desc, bool onlyupcase)
{
bool upcasehomonym = false;
- int descl = (desc) ? strlen(desc) : 0;
+ int descl = desc ? (aliasm ? sizeof(short) : strlen(desc) + 1) : 0;
// variable-length hash record with word and optional fields
- // instead of mmap implementation temporarily
struct hentry* hp =
- (struct hentry *) malloc (sizeof(struct hentry) + wbl + descl + 1);
+ (struct hentry *) malloc (sizeof(struct hentry) + wbl + descl);
if (!hp) return 1;
char * hpw = &(hp->word);
strcpy(hpw, word);
- if (desc && strncmp(desc, FIELD_PHON, strlen(FIELD_PHON)) == 0) {
- strcpy(hpw + wbl + 1, desc + strlen(FIELD_PHON));
- hp->var = 1;
- } else {
- hp->var = 0;
- }
if (ignorechars != NULL) {
if (utf8) {
remove_ignored_chars_utf(hpw, ignorechars_utf16, ignorechars_utf16_len);
} else {
remove_ignored_chars(hpw, ignorechars);
}
}
if (complexprefixes) {
if (utf8) reverseword_utf(hpw); else reverseword(hpw);
}
int i = hash(hpw);
- hp->blen = (unsigned char) wbl;
- hp->clen = (unsigned char) wcl;
- hp->alen = (short) al;
- hp->astr = aff;
- hp->next = NULL;
- hp->next_homonym = NULL;
-#ifdef HUNSPELL_EXPERIMENTAL
- if (aliasm) {
- hp->description = (desc) ? get_aliasm(atoi(desc)) : mystrdup(desc);
- } else {
- hp->description = mystrdup(desc);
- if (desc && !hp->description)
- {
- free(hp->astr);
- free(hp);
- return 1;
+ hp->blen = (unsigned char) wbl;
+ hp->clen = (unsigned char) wcl;
+ hp->alen = (short) al;
+ hp->astr = aff;
+ hp->next = NULL;
+ hp->next_homonym = NULL;
+
+ // store the description string or its pointer
+ if (desc) {
+ hp->var = H_OPT;
+ if (aliasm) {
+ hp->var += H_OPT_ALIASM;
+ store_pointer(hpw + wbl + 1, get_aliasm(atoi(desc)));
+ } else {
+ strcpy(hpw + wbl + 1, desc);
+ if (complexprefixes) {
+ if (utf8) reverseword_utf(HENTRY_DATA(hp));
+ else reverseword(HENTRY_DATA(hp));
}
- if (hp->description && complexprefixes) {
- if (utf8) reverseword_utf(hp->description); else reverseword(hp->description);
- }
- }
-#endif
-
+ }
+ if (strstr(HENTRY_DATA(hp), MORPH_PHON)) hp->var += H_OPT_PHON;
+ } else hp->var = 0;
+
struct hentry * dp = tableptr[i];
if (!dp) {
tableptr[i] = hp;
return 0;
}
while (dp->next != NULL) {
if ((!dp->next_homonym) && (strcmp(&(hp->word), &(dp->word)) == 0)) {
// remove hidden onlyupcase homonym
@@ -302,69 +284,110 @@ int HashMgr::add_hidden_capitalized_word
if (((captype == HUHCAP) || (captype == HUHINITCAP) ||
((captype == ALLCAP) && (flags != NULL))) &&
!((flags != NULL) && TESTAFF(flags, forbiddenword, al))) {
unsigned short * flags2 = (unsigned short *) malloc (sizeof(unsigned short) * (al+1));
if (!flags2) return 1;
if (al) memcpy(flags2, flags, al * sizeof(unsigned short));
flags2[al] = ONLYUPCASEFLAG;
if (utf8) {
- char st[MAXDELEN];
- w_char w[MAXDELEN];
- int wlen = u8_u16(w, MAXDELEN, word);
+ char st[BUFSIZE];
+ w_char w[BUFSIZE];
+ int wlen = u8_u16(w, BUFSIZE, word);
mkallsmall_utf(w, wlen, langnum);
mkallcap_utf(w, 1, langnum);
- u16_u8(st, MAXDELEN, w, wlen);
+ u16_u8(st, BUFSIZE, w, wlen);
return add_word(st,wbl,wcl,flags2,al+1,dp, true);
} else {
mkallsmall(word, csconv);
mkinitcap(word, csconv);
return add_word(word,wbl,wcl,flags2,al+1,dp, true);
}
}
return 0;
}
// detect captype and modify word length for UTF-8 encoding
int HashMgr::get_clen_and_captype(const char * word, int wbl, int * captype) {
int len;
if (utf8) {
- w_char dest_utf[MAXDELEN];
- len = u8_u16(dest_utf, MAXDELEN, word);
+ w_char dest_utf[BUFSIZE];
+ len = u8_u16(dest_utf, BUFSIZE, word);
*captype = get_captype_utf8(dest_utf, len, langnum);
} else {
len = wbl;
*captype = get_captype((char *) word, len, csconv);
}
return len;
}
-// add a custom dic. word to the hash table (public)
-int HashMgr::put_word(const char * word, char * aff)
+// remove word (personal dictionary function for standalone applications)
+int HashMgr::remove(const char * word)
{
- unsigned short * flags;
- int al = 0;
- if (aff) {
- al = decode_flags(&flags, aff);
- flag_qsort(flags, 0, al);
- } else {
- flags = NULL;
+ struct hentry * dp = lookup(word);
+ while (dp) {
+ if (dp->alen == 0 || !TESTAFF(dp->astr, forbiddenword, dp->alen)) {
+ unsigned short * flags =
+ (unsigned short *) malloc(sizeof(short *) * (dp->alen + 1));
+ if (!flags) return 1;
+ for (int i = 0; i < dp->alen; i++) flags[i] = dp->astr[i];
+ flags[dp->alen] = forbiddenword;
+ dp->astr = flags;
+ dp->alen++;
+ flag_qsort(flags, 0, dp->alen);
+ }
+ dp = dp->next_homonym;
}
-
- int captype;
- int wbl = strlen(word);
- int wcl = get_clen_and_captype(word, wbl, &captype);
- add_word(word, wbl, wcl, flags, al, NULL, false);
- return add_hidden_capitalized_word((char *) word, wbl, wcl, flags, al, NULL, captype);
+ return 0;
}
-int HashMgr::put_word_pattern(const char * word, const char * pattern)
+/* remove forbidden flag to add a personal word to the hash */
+int HashMgr::remove_forbidden_flag(const char * word) {
+ struct hentry * dp = lookup(word);
+ if (!dp) return 1;
+ while (dp) {
+ if (dp->astr && TESTAFF(dp->astr, forbiddenword, dp->alen)) {
+ if (dp->alen == 1) dp->alen = 0; // XXX forbidden words of personal dic.
+ else {
+ unsigned short * flags2 =
+ (unsigned short *) malloc(sizeof(short *) * (dp->alen - 1));
+ if (!flags2) return 1;
+ int i, j = 0;
+ for (i = 0; i < dp->alen; i++) {
+ if (dp->astr[i] != forbiddenword) flags2[j++] = dp->astr[i];
+ }
+ dp->alen--;
+ dp->astr = flags2; // XXX allowed forbidden words
+ }
+ }
+ dp = dp->next_homonym;
+ }
+ return 0;
+}
+
+// add a custom dic. word to the hash table (public)
+int HashMgr::add(const char * word)
+{
+ unsigned short * flags = NULL;
+ int al = 0;
+ if (remove_forbidden_flag(word)) {
+ int captype;
+ int wbl = strlen(word);
+ int wcl = get_clen_and_captype(word, wbl, &captype);
+ add_word(word, wbl, wcl, flags, al, NULL, false);
+ return add_hidden_capitalized_word((char *) word, wbl, wcl, flags, al, NULL, captype);
+ }
+ return 0;
+}
+
+int HashMgr::add_with_affix(const char * word, const char * example)
{
// detect captype and modify word length for UTF-8 encoding
- struct hentry * dp = lookup(pattern);
+ struct hentry * dp = lookup(example);
+ remove_forbidden_flag(word);
if (dp && dp->astr) {
int captype;
int wbl = strlen(word);
int wcl = get_clen_and_captype(word, wbl, &captype);
if (aliasf) {
add_word(word, wbl, wcl, dp->astr, dp->alen, NULL, false);
} else {
unsigned short * flags = (unsigned short *) malloc (dp->alen * sizeof(short));
@@ -387,72 +410,86 @@ struct hentry * HashMgr::walk_hashtable(
if (tableptr[col]) return tableptr[col];
}
// null at end and reset to start
col = -1;
return NULL;
}
// load a munched word list and build a hash table on the fly
-int HashMgr::load_tables(const char * tpath)
+int HashMgr::load_tables(const char * tpath, const char * key)
{
int al;
char * ap;
char * dp;
+ char * dp2;
unsigned short * flags;
+ char * ts;
- // raw dictionary - munched file
- FILE * rawdict = fopen(tpath, "r");
- if (rawdict == NULL) return 1;
+ // open dictionary file
+ FileMgr * dict = new FileMgr(tpath, key);
+ if (dict == NULL) return 1;
// first read the first line of file to get hash table size */
- char ts[MAXDELEN];
- if (! fgets(ts, MAXDELEN-1,rawdict)) {
+ if (!(ts = dict->getline())) {
HUNSPELL_WARNING(stderr, "error: empty dic file\n");
- fclose(rawdict);
+ delete dict;
return 2;
}
mychomp(ts);
-
+
/* remove byte order mark */
if (strncmp(ts,"\xEF\xBB\xBF",3) == 0) {
memmove(ts, ts+3, strlen(ts+3)+1);
HUNSPELL_WARNING(stderr, "warning: dic file begins with byte order mark: possible incompatibility with old Hunspell versions\n");
}
-
- if ((*ts < '1') || (*ts > '9')) HUNSPELL_WARNING(stderr, "error - missing word count in dictionary file\n");
+
tablesize = atoi(ts);
- if (!tablesize) {
- fclose(rawdict);
+ if (tablesize == 0) {
+ HUNSPELL_WARNING(stderr, "error: line 1: missing or bad word count in the dic file\n");
+ delete dict;
return 4;
}
tablesize = tablesize + 5 + USERWORD;
if ((tablesize %2) == 0) tablesize++;
// allocate the hash table
tableptr = (struct hentry **) malloc(tablesize * sizeof(struct hentry *));
if (! tableptr) {
- fclose(rawdict);
+ delete dict;
return 3;
}
for (int i=0; i<tablesize; i++) tableptr[i] = NULL;
// loop through all words on much list and add to hash
// table and create word and affix strings
- while (fgets(ts,MAXDELEN-1,rawdict)) {
+ while ((ts = dict->getline())) {
mychomp(ts);
// split each line into word and morphological description
- dp = strchr(ts,'\t');
+ dp = ts;
+ while ((dp = strchr(dp, ':'))) {
+ if ((dp > ts + 3) && (*(dp - 3) == ' ' || *(dp - 3) == '\t')) {
+ for (dp -= 4; dp >= ts && (*dp == ' ' || *dp == '\t'); dp--);
+ if (dp < ts) { // missing word
+ dp = NULL;
+ } else {
+ *(dp + 1) = '\0';
+ dp = dp + 2;
+ }
+ break;
+ }
+ dp++;
+ }
- if (dp) {
- *dp = '\0';
- dp++;
- } else {
- dp = NULL;
+ // tabulator is the old morphological field separator
+ dp2 = strchr(ts, '\t');
+ if (dp2 && (!dp || dp2 < dp)) {
+ *dp2 = '\0';
+ dp = dp2 + 1;
}
// split each line into word and affix char strings
// "\/" signs slash in words (not affix separator)
// "/" at beginning of the line is word character (not affix separator)
ap = strchr(ts,'/');
while (ap) {
if (ap == ts) {
@@ -463,138 +500,147 @@ int HashMgr::load_tables(const char * tp
for (char * sp = ap - 1; *sp; *sp = *(sp + 1), sp++);
ap = strchr(ap,'/');
}
if (ap) {
*ap = '\0';
if (aliasf) {
int index = atoi(ap + 1);
- al = get_aliasf(index, &flags);
+ al = get_aliasf(index, &flags, dict);
if (!al) {
- HUNSPELL_WARNING(stderr, "error - bad flag vector alias: %s\n", ts);
+ HUNSPELL_WARNING(stderr, "error: line %d: bad flag vector alias\n", dict->getlinenum());
*ap = '\0';
}
} else {
- al = decode_flags(&flags, ap + 1);
+ al = decode_flags(&flags, ap + 1, dict);
flag_qsort(flags, 0, al);
}
} else {
al = 0;
ap = NULL;
flags = NULL;
}
int captype;
int wbl = strlen(ts);
int wcl = get_clen_and_captype(ts, wbl, &captype);
// add the word and its index plus its capitalized form optionally
if (add_word(ts,wbl,wcl,flags,al,dp, false) ||
add_hidden_capitalized_word(ts, wbl, wcl, flags, al, dp, captype)) {
- fclose(rawdict);
+ delete dict;
return 5;
}
}
- fclose(rawdict);
+ delete dict;
return 0;
}
-
// the hash function is a simple load and rotate
// algorithm borrowed
int HashMgr::hash(const char * word) const
{
long hv = 0;
for (int i=0; i < 4 && *word != 0; i++)
hv = (hv << 8) | (*word++);
while (*word != 0) {
ROTATE(hv,ROTATE_LEN);
hv ^= (*word++);
}
return (unsigned long) hv % tablesize;
}
-int HashMgr::decode_flags(unsigned short ** result, char * flags) {
+int HashMgr::decode_flags(unsigned short ** result, char * flags, FileMgr * af) {
int len;
switch (flag_mode) {
case FLAG_LONG: { // two-character flags (1x2yZz -> 1x 2y Zz)
len = strlen(flags);
- if (len%2 == 1) HUNSPELL_WARNING(stderr, "error: length of FLAG_LONG flagvector is odd: %s\n", flags);
+ if (len%2 == 1) HUNSPELL_WARNING(stderr, "error: line %d: bad flagvector\n", af->getlinenum());
len /= 2;
*result = (unsigned short *) malloc(len * sizeof(short));
if (!*result) return -1;
for (int i = 0; i < len; i++) {
(*result)[i] = (((unsigned short) flags[i * 2]) << 8) + (unsigned short) flags[i * 2 + 1];
}
break;
}
case FLAG_NUM: { // decimal numbers separated by comma (4521,23,233 -> 4521 23 233)
+ int i;
len = 1;
char * src = flags;
unsigned short * dest;
char * p;
for (p = flags; *p; p++) {
if (*p == ',') len++;
}
*result = (unsigned short *) malloc(len * sizeof(short));
if (!*result) return -1;
dest = *result;
for (p = flags; *p; p++) {
if (*p == ',') {
- *dest = (unsigned short) atoi(src);
- if (*dest == 0) HUNSPELL_WARNING(stderr, "error: 0 is wrong flag id\n");
+ i = atoi(src);
+ if (i >= DEFAULTFLAGS) HUNSPELL_WARNING(stderr, "error: line %d: flag id %d is too large (max: %d)\n",
+ af->getlinenum(), i, DEFAULTFLAGS - 1);
+ *dest = (unsigned short) i;
+ if (*dest == 0) HUNSPELL_WARNING(stderr, "error: line %d: 0 is wrong flag id\n", af->getlinenum());
src = p + 1;
dest++;
}
}
- *dest = (unsigned short) atoi(src);
- if (*dest == 0) HUNSPELL_WARNING(stderr, "error: 0 is wrong flag id\n");
+ i = atoi(src);
+ if (i >= DEFAULTFLAGS) HUNSPELL_WARNING(stderr, "error: line %d: flag id %d is too large (max: %d)\n",
+ af->getlinenum(), i, DEFAULTFLAGS - 1);
+ *dest = (unsigned short) i;
+ if (*dest == 0) HUNSPELL_WARNING(stderr, "error: line %d: 0 is wrong flag id\n", af->getlinenum());
break;
}
case FLAG_UNI: { // UTF-8 characters
- w_char w[MAXDELEN/2];
- len = u8_u16(w, MAXDELEN/2, flags);
+ w_char w[BUFSIZE/2];
+ len = u8_u16(w, BUFSIZE/2, flags);
*result = (unsigned short *) malloc(len * sizeof(short));
if (!*result) return -1;
memcpy(*result, w, len * sizeof(short));
break;
}
default: { // Ispell's one-character flags (erfg -> e r f g)
unsigned short * dest;
len = strlen(flags);
*result = (unsigned short *) malloc(len * sizeof(short));
if (!*result) return -1;
dest = *result;
for (unsigned char * p = (unsigned char *) flags; *p; p++) {
*dest = (unsigned short) *p;
dest++;
}
}
- }
+ }
return len;
}
unsigned short HashMgr::decode_flag(const char * f) {
unsigned short s = 0;
+ int i;
switch (flag_mode) {
case FLAG_LONG:
s = ((unsigned short) f[0] << 8) + (unsigned short) f[1];
break;
case FLAG_NUM:
- s = (unsigned short) atoi(f);
+ i = atoi(f);
+ if (i >= DEFAULTFLAGS) HUNSPELL_WARNING(stderr, "error: flag id %d is too large (max: %d)\n", i, DEFAULTFLAGS - 1);
+ s = (unsigned short) i;
break;
case FLAG_UNI:
u8_u16((w_char *) &s, 1, f);
break;
default:
s = (unsigned short) *((unsigned char *)f);
}
- if (!s) HUNSPELL_WARNING(stderr, "error: 0 is wrong flag id\n");
+ if (s == 0) HUNSPELL_WARNING(stderr, "error: 0 is wrong flag id\n");
return s;
}
char * HashMgr::encode_flag(unsigned short f) {
unsigned char ch[10];
if (f==0) return mystrdup("(NULL)");
if (flag_mode == FLAG_LONG) {
ch[0] = (unsigned char) (f >> 8);
@@ -607,122 +653,119 @@ char * HashMgr::encode_flag(unsigned sho
} else {
ch[0] = (unsigned char) (f);
ch[1] = '\0';
}
return mystrdup((char *) ch);
}
// read in aff file and set flag mode
-int HashMgr::load_config(const char * affpath)
+int HashMgr::load_config(const char * affpath, const char * key)
{
+ char * line; // io buffers
int firstline = 1;
-
- // io buffers
- char line[MAXDELEN+1];
// open the affix file
- FILE * afflst;
- afflst = fopen(affpath,"r");
+ FileMgr * afflst = new FileMgr(affpath, key);
if (!afflst) {
HUNSPELL_WARNING(stderr, "Error - could not open affix description file %s\n",affpath);
return 1;
}
// read in each line ignoring any that do not
// start with a known line type indicator
- while (fgets(line,MAXDELEN,afflst)) {
+ while ((line = afflst->getline())) {
mychomp(line);
/* remove byte order mark */
if (firstline) {
firstline = 0;
if (strncmp(line,"\xEF\xBB\xBF",3) == 0) memmove(line, line+3, strlen(line+3)+1);
}
/* parse in the try string */
if ((strncmp(line,"FLAG",4) == 0) && isspace(line[4])) {
if (flag_mode != FLAG_CHAR) {
- HUNSPELL_WARNING(stderr, "error: duplicate FLAG parameter\n");
+ HUNSPELL_WARNING(stderr, "error: line %d: multiple definitions of the FLAG affix file parameter\n", afflst->getlinenum());
}
if (strstr(line, "long")) flag_mode = FLAG_LONG;
if (strstr(line, "num")) flag_mode = FLAG_NUM;
if (strstr(line, "UTF-8")) flag_mode = FLAG_UNI;
if (flag_mode == FLAG_CHAR) {
- HUNSPELL_WARNING(stderr, "error: FLAG need `num', `long' or `UTF-8' parameter: %s\n", line);
+ HUNSPELL_WARNING(stderr, "error: line %d: FLAG needs `num', `long' or `UTF-8' parameter\n", afflst->getlinenum());
}
}
if (strncmp(line,"FORBIDDENWORD",13) == 0) {
char * st = NULL;
- if (parse_string(line, &st, "FORBIDDENWORD")) {
- fclose(afflst);
+ if (parse_string(line, &st, afflst->getlinenum())) {
+ delete afflst;
return 1;
}
forbiddenword = decode_flag(st);
free(st);
}
if (strncmp(line, "SET", 3) == 0) {
- if (parse_string(line, &enc, "SET")) {
- fclose(afflst);
+ if (parse_string(line, &enc, afflst->getlinenum())) {
+ delete afflst;
return 1;
}
if (strcmp(enc, "UTF-8") == 0) {
utf8 = 1;
#ifndef OPENOFFICEORG
#ifndef MOZILLA_CLIENT
initialize_utf_tbl();
#endif
#endif
} else csconv = get_current_cs(enc);
}
- if (strncmp(line, "LANG", 4) == 0) {
- if (parse_string(line, &lang, "LANG")) {
- fclose(afflst);
+ if (strncmp(line, "LANG", 4) == 0) {
+ if (parse_string(line, &lang, afflst->getlinenum())) {
+ delete afflst;
return 1;
}
langnum = get_lang_num(lang);
}
/* parse in the ignored characters (for example, Arabic optional diacritics characters */
if (strncmp(line,"IGNORE",6) == 0) {
- if (parse_array(line, &ignorechars, &ignorechars_utf16, &ignorechars_utf16_len, "IGNORE", utf8)) {
- fclose(afflst);
+ if (parse_array(line, &ignorechars, &ignorechars_utf16,
+ &ignorechars_utf16_len, utf8, afflst->getlinenum())) {
+ delete afflst;
return 1;
}
}
if ((strncmp(line,"AF",2) == 0) && isspace(line[2])) {
if (parse_aliasf(line, afflst)) {
- fclose(afflst);
+ delete afflst;
+ return 1;
+ }
+ }
+
+ if ((strncmp(line,"AM",2) == 0) && isspace(line[2])) {
+ if (parse_aliasm(line, afflst)) {
+ delete afflst;
return 1;
}
}
-#ifdef HUNSPELL_EXPERIMENTAL
- if ((strncmp(line,"AM",2) == 0) && isspace(line[2])) {
- if (parse_aliasm(line, afflst)) {
- fclose(afflst);
- return 1;
- }
- }
-#endif
- if (strncmp(line,"COMPLEXPREFIXES",15) == 0) complexprefixes = 1;
- if (((strncmp(line,"SFX",3) == 0) || (strncmp(line,"PFX",3) == 0)) && isspace(line[3])) break;
+ if (strncmp(line,"COMPLEXPREFIXES",15) == 0) complexprefixes = 1;
+ if (((strncmp(line,"SFX",3) == 0) || (strncmp(line,"PFX",3) == 0)) && isspace(line[3])) break;
}
- if (csconv == NULL) csconv = get_current_cs("ISO8859-1");
- fclose(afflst);
+ if (csconv == NULL) csconv = get_current_cs(SPELL_ENCODING);
+ delete afflst;
return 0;
}
/* parse in the ALIAS table */
-int HashMgr::parse_aliasf(char * line, FILE * af)
+int HashMgr::parse_aliasf(char * line, FileMgr * af)
{
if (numaliasf != 0) {
- HUNSPELL_WARNING(stderr, "error: duplicate AF (alias for flag vector) tables used\n");
+ HUNSPELL_WARNING(stderr, "error: line %d: multiple table definitions\n", af->getlinenum());
return 1;
}
char * tp = line;
char * piece;
int i = 0;
int np = 0;
piece = mystrsep(&tp, 0);
while (piece) {
@@ -730,18 +773,17 @@ int HashMgr::parse_aliasf(char * line,
switch(i) {
case 0: { np++; break; }
case 1: {
numaliasf = atoi(piece);
if (numaliasf < 1) {
numaliasf = 0;
aliasf = NULL;
aliasflen = NULL;
- HUNSPELL_WARNING(stderr, "incorrect number of entries in AF table\n");
- free(piece);
+ HUNSPELL_WARNING(stderr, "error: line %d: bad entry number\n", af->getlinenum());
return 1;
}
aliasf = (unsigned short **) malloc(numaliasf * sizeof(unsigned short *));
aliasflen = (unsigned short *) malloc(numaliasf * sizeof(short));
if (!aliasf || !aliasflen) {
numaliasf = 0;
if (aliasf) free(aliasf);
if (aliasflen) free(aliasflen);
@@ -751,191 +793,193 @@ int HashMgr::parse_aliasf(char * line,
}
np++;
break;
}
default: break;
}
i++;
}
- free(piece);
piece = mystrsep(&tp, 0);
}
if (np != 2) {
numaliasf = 0;
free(aliasf);
free(aliasflen);
aliasf = NULL;
aliasflen = NULL;
- HUNSPELL_WARNING(stderr, "error: missing AF table information\n");
+ HUNSPELL_WARNING(stderr, "error: line %d: missing data\n", af->getlinenum());
return 1;
}
/* now parse the numaliasf lines to read in the remainder of the table */
- char * nl = line;
+ char * nl;
for (int j=0; j < numaliasf; j++) {
- if (!fgets(nl,MAXDELEN,af)) return 1;
+ if (!(nl = af->getline())) return 1;
mychomp(nl);
tp = nl;
i = 0;
aliasf[j] = NULL;
aliasflen[j] = 0;
piece = mystrsep(&tp, 0);
while (piece) {
if (*piece != '\0') {
switch(i) {
case 0: {
if (strncmp(piece,"AF",2) != 0) {
numaliasf = 0;
free(aliasf);
free(aliasflen);
aliasf = NULL;
aliasflen = NULL;
- HUNSPELL_WARNING(stderr, "error: AF table is corrupt\n");
- free(piece);
+ HUNSPELL_WARNING(stderr, "error: line %d: table is corrupt\n", af->getlinenum());
return 1;
}
break;
}
case 1: {
- aliasflen[j] = (unsigned short) decode_flags(&(aliasf[j]), piece);
+ aliasflen[j] = (unsigned short) decode_flags(&(aliasf[j]), piece, af);
flag_qsort(aliasf[j], 0, aliasflen[j]);
break;
}
default: break;
}
i++;
}
- free(piece);
piece = mystrsep(&tp, 0);
}
if (!aliasf[j]) {
free(aliasf);
free(aliasflen);
aliasf = NULL;
aliasflen = NULL;
numaliasf = 0;
- HUNSPELL_WARNING(stderr, "error: AF table is corrupt\n");
+ HUNSPELL_WARNING(stderr, "error: line %d: table is corrupt\n", af->getlinenum());
return 1;
}
}
return 0;
}
int HashMgr::is_aliasf() {
return (aliasf != NULL);
}
-int HashMgr::get_aliasf(int index, unsigned short ** fvec) {
+int HashMgr::get_aliasf(int index, unsigned short ** fvec, FileMgr * af) {
if ((index > 0) && (index <= numaliasf)) {
*fvec = aliasf[index - 1];
return aliasflen[index - 1];
}
- HUNSPELL_WARNING(stderr, "error: bad flag alias index: %d\n", index);
+ HUNSPELL_WARNING(stderr, "error: line %d: bad flag alias index: %d\n", af->getlinenum(), index);
*fvec = NULL;
return 0;
}
-#ifdef HUNSPELL_EXPERIMENTAL
/* parse morph alias definitions */
-int HashMgr::parse_aliasm(char * line, FILE * af)
+int HashMgr::parse_aliasm(char * line, FileMgr * af)
{
if (numaliasm != 0) {
- HUNSPELL_WARNING(stderr, "error: duplicate AM (aliases for morphological descriptions) tables used\n");
+ HUNSPELL_WARNING(stderr, "error: line %d: multiple table definitions\n", af->getlinenum());
return 1;
}
char * tp = line;
char * piece;
int i = 0;
int np = 0;
piece = mystrsep(&tp, 0);
while (piece) {
if (*piece != '\0') {
switch(i) {
case 0: { np++; break; }
case 1: {
numaliasm = atoi(piece);
if (numaliasm < 1) {
- HUNSPELL_WARNING(stderr, "incorrect number of entries in AM table\n");
- free(piece);
+ HUNSPELL_WARNING(stderr, "error: line %d: bad entry number\n", af->getlinenum());
return 1;
}
aliasm = (char **) malloc(numaliasm * sizeof(char *));
if (!aliasm) {
numaliasm = 0;
return 1;
}
np++;
break;
}
default: break;
}
i++;
}
- free(piece);
piece = mystrsep(&tp, 0);
}
if (np != 2) {
numaliasm = 0;
free(aliasm);
aliasm = NULL;
- HUNSPELL_WARNING(stderr, "error: missing AM alias information\n");
+ HUNSPELL_WARNING(stderr, "error: line %d: missing data\n", af->getlinenum());
return 1;
}
/* now parse the numaliasm lines to read in the remainder of the table */
char * nl = line;
for (int j=0; j < numaliasm; j++) {
- if (!fgets(nl,MAXDELEN,af)) return 1;
+ if (!(nl = af->getline())) return 1;
mychomp(nl);
tp = nl;
i = 0;
aliasm[j] = NULL;
- piece = mystrsep(&tp, 0);
+ piece = mystrsep(&tp, ' ');
while (piece) {
if (*piece != '\0') {
switch(i) {
case 0: {
if (strncmp(piece,"AM",2) != 0) {
- HUNSPELL_WARNING(stderr, "error: AM table is corrupt\n");
- free(piece);
+ HUNSPELL_WARNING(stderr, "error: line %d: table is corrupt\n", af->getlinenum());
numaliasm = 0;
free(aliasm);
aliasm = NULL;
return 1;
}
break;
}
case 1: {
+ // add the remaining of the line
+ if (*tp) {
+ *(tp - 1) = ' ';
+ tp = tp + strlen(tp);
+ }
if (complexprefixes) {
if (utf8) reverseword_utf(piece);
else reverseword(piece);
}
aliasm[j] = mystrdup(piece);
+ if (!aliasm[j]) {
+ numaliasm = 0;
+ free(aliasm);
+ aliasm = NULL;
+ return 1;
+ }
break; }
default: break;
}
i++;
}
- free(piece);
- piece = mystrsep(&tp, 0);
+ piece = mystrsep(&tp, ' ');
}
if (!aliasm[j]) {
numaliasm = 0;
free(aliasm);
aliasm = NULL;
- HUNSPELL_WARNING(stderr, "error: map table is corrupt\n");
+ HUNSPELL_WARNING(stderr, "error: line %d: table is corrupt\n", af->getlinenum());
return 1;
}
}
return 0;
}
int HashMgr::is_aliasm() {
return (aliasm != NULL);
}
char * HashMgr::get_aliasm(int index) {
if ((index > 0) && (index <= numaliasm)) return aliasm[index - 1];
HUNSPELL_WARNING(stderr, "error: bad morph. alias index: %d\n", index);
return NULL;
}
-#endif
--- a/extensions/spellcheck/hunspell/src/hashmgr.hxx
+++ b/extensions/spellcheck/hunspell/src/hashmgr.hxx
@@ -58,72 +58,70 @@
#define _HASHMGR_HXX_
#ifndef MOZILLA_CLIENT
#include <cstdio>
#else
#include <stdio.h>
#endif
+#include "filemgr.hxx"
#include "htypes.hxx"
enum flag { FLAG_CHAR, FLAG_LONG, FLAG_NUM, FLAG_UNI };
class HashMgr
{
- int tablesize;
- struct hentry ** tableptr;
- int userword;
- flag flag_mode;
- int complexprefixes;
- int utf8;
- unsigned short forbiddenword;
- int langnum;
- char * enc;
- char * lang;
- struct cs_info * csconv;
- char * ignorechars;
- unsigned short * ignorechars_utf16;
- int ignorechars_utf16_len;
- int numaliasf; // flag vector `compression' with aliases
- unsigned short ** aliasf;
- unsigned short * aliasflen;
- int numaliasm; // morphological desciption `compression' with aliases
- char ** aliasm;
+ int tablesize;
+ struct hentry ** tableptr;
+ int userword;
+ flag flag_mode;
+ int complexprefixes;
+ int utf8;
+ unsigned short forbiddenword;
+ int langnum;
+ char * enc;
+ char * lang;
+ struct cs_info * csconv;
+ char * ignorechars;
+ unsigned short * ignorechars_utf16;
+ int ignorechars_utf16_len;
+ int numaliasf; // flag vector `compression' with aliases
+ unsigned short ** aliasf;
+ unsigned short * aliasflen;
+ int numaliasm; // morphological desciption `compression' with aliases
+ char ** aliasm;
public:
- HashMgr(const char * tpath, const char * apath);
+ HashMgr(const char * tpath, const char * apath, const char * key = NULL);
~HashMgr();
struct hentry * lookup(const char *) const;
int hash(const char *) const;
struct hentry * walk_hashtable(int & col, struct hentry * hp) const;
- int put_word(const char * word, char * ap);
- int put_word_pattern(const char * word, const char * pattern);
- int decode_flags(unsigned short ** result, char * flags);
+ int add(const char * word);
+ int add_with_affix(const char * word, const char * pattern);
+ int remove(const char * word);
+ int decode_flags(unsigned short ** result, char * flags, FileMgr * af);
unsigned short decode_flag(const char * flag);
char * encode_flag(unsigned short flag);
int is_aliasf();
- int get_aliasf(int index, unsigned short ** fvec);
-#ifdef HUNSPELL_EXPERIMENTAL
+ int get_aliasf(int index, unsigned short ** fvec, FileMgr * af);
int is_aliasm();
char * get_aliasm(int index);
-#endif
-
private:
int get_clen_and_captype(const char * word, int wbl, int * captype);
- int load_tables(const char * tpath);
+ int load_tables(const char * tpath, const char * key);
int add_word(const char * word, int wbl, int wcl, unsigned short * ap,
int al, const char * desc, bool onlyupcase);
- int load_config(const char * affpath);
- int parse_aliasf(char * line, FILE * af);
+ int load_config(const char * affpath, const char * key);
+ int parse_aliasf(char * line, FileMgr * af);
int add_hidden_capitalized_word(char * word, int wbl, int wcl,
unsigned short * flags, int al, char * dp, int captype);
-#ifdef HUNSPELL_EXPERIMENTAL
- int parse_aliasm(char * line, FILE * af);
-#endif
+ int parse_aliasm(char * line, FileMgr * af);
+ int remove_forbidden_flag(const char * word);
};
#endif
--- a/extensions/spellcheck/hunspell/src/htypes.hxx
+++ b/extensions/spellcheck/hunspell/src/htypes.hxx
@@ -52,34 +52,37 @@
* the provisions above, a recipient may use your version of this file under
* the terms of any one of the MPL, the GPL or the LGPL.
*
******* END LICENSE BLOCK *******/
#ifndef _HTYPES_HXX_
#define _HTYPES_HXX_
-#define MAXDELEN 8192
-
#define ROTATE_LEN 5
#define ROTATE(v,q) \
(v) = ((v) << (q)) | (((v) >> (32 - q)) & ((1 << (q))-1));
+// hentry options
+#define H_OPT (1 << 0)
+#define H_OPT_ALIASM (1 << 1)
+#define H_OPT_PHON (1 << 2)
+
+// see also csutil.hxx
+#define HENTRY_WORD(h) &(h->word)
+
// approx. number of user defined words
#define USERWORD 1000
struct hentry
{
unsigned char blen; // word length in bytes
unsigned char clen; // word length in characters (different for UTF-8 enc.)
short alen; // length of affix flag vector
unsigned short * astr; // affix flag vector
struct hentry * next; // next word with same hash code
struct hentry * next_homonym; // next homonym word (with same hash code)
-#ifdef HUNSPELL_EXPERIMENTAL
- char * description; // morphological data (optional)
-#endif
char var; // variable fields (only for special pronounciation yet)
char word; // variable-length word (8-bit or UTF-8 encoding)
};
#endif
--- a/extensions/spellcheck/hunspell/src/hunspell.cpp
+++ b/extensions/spellcheck/hunspell/src/hunspell.cpp
@@ -54,108 +54,120 @@
*
******* END LICENSE BLOCK *******/
#ifndef MOZILLA_CLIENT
#include <cstdlib>
#include <cstring>
#include <cstdio>
#else
-#include <stdlib.h>
+#include <stdlib.h>
#include <string.h>
-#include <stdio.h>
+#include <stdio.h>
#endif
+#include "csutil.hxx"
+#include "hunspell.h"
#include "hunspell.hxx"
-#include "hunspell.h"
#ifndef MOZILLA_CLIENT
#ifndef W32
using namespace std;
#endif
#endif
-Hunspell::Hunspell(const char * affpath, const char * dpath)
+Hunspell::Hunspell(const char * affpath, const char * dpath, const char * key)
{
encoding = NULL;
csconv = NULL;
utf8 = 0;
complexprefixes = 0;
+ affixpath = mystrdup(affpath);
+ maxdic = 0;
/* first set up the hash manager */
- pHMgr = new HashMgr(dpath, affpath);
+ pHMgr[0] = new HashMgr(dpath, affpath, key);
+ if (pHMgr[0]) maxdic = 1;
/* next set up the affix manager */
/* it needs access to the hash manager lookup methods */
- pAMgr = new AffixMgr(affpath,pHMgr);
+ pAMgr = new AffixMgr(affpath, pHMgr, &maxdic, key);
/* get the preferred try string and the dictionary */
/* encoding from the Affix Manager for that dictionary */
char * try_string = pAMgr->get_try_string();
encoding = pAMgr->get_encoding();
csconv = get_current_cs(encoding);
langnum = pAMgr->get_langnum();
utf8 = pAMgr->get_utf8();
complexprefixes = pAMgr->get_complexprefixes();
wordbreak = pAMgr->get_breaktable();
/* and finally set up the suggestion manager */
pSMgr = new SuggestMgr(try_string, MAXSUGGESTION, pAMgr);
if (try_string) free(try_string);
-
}
Hunspell::~Hunspell()
{
if (pSMgr) delete pSMgr;
if (pAMgr) delete pAMgr;
- if (pHMgr) delete pHMgr;
+ for (int i = 0; i < maxdic; i++) delete pHMgr[i];
+ maxdic = 0;
pSMgr = NULL;
pAMgr = NULL;
- pHMgr = NULL;
#ifdef MOZILLA_CLIENT
free(csconv);
#endif
csconv= NULL;
if (encoding) free(encoding);
encoding = NULL;
+ if (affixpath) free(affixpath);
+ affixpath = NULL;
}
+// load extra dictionaries
+int Hunspell::add_dic(const char * dpath, const char * key) {
+ if (maxdic == MAXDIC || !affixpath) return 1;
+ pHMgr[maxdic] = new HashMgr(dpath, affixpath, key);
+ if (pHMgr[maxdic]) maxdic++; else return 1;
+ return 0;
+}
// make a copy of src at destination while removing all leading
// blanks and removing any trailing periods after recording
// their presence with the abbreviation flag
-// also since already going through character by character,
+// also since already going through character by character,
// set the capitalization type
// return the length of the "cleaned" (and UTF-8 encoded) word
-int Hunspell::cleanword2(char * dest, const char * src,
+int Hunspell::cleanword2(char * dest, const char * src,
w_char * dest_utf, int * nc, int * pcaptype, int * pabbrev)
-{
+{
unsigned char * p = (unsigned char *) dest;
const unsigned char * q = (const unsigned char * ) src;
// first skip over any leading blanks
while ((*q != '\0') && (*q == ' ')) q++;
-
+
// now strip off any trailing periods (recording their presence)
*pabbrev = 0;
int nl = strlen((const char *)q);
while ((nl > 0) && (*(q+nl-1)=='.')) {
nl--;
(*pabbrev)++;
}
-
+
// if no characters are left it can't be capitalized
- if (nl <= 0) {
+ if (nl <= 0) {
*pcaptype = NOCAP;
*p = '\0';
return 0;
}
-
+
strncpy(dest, (char *) q, nl);
*(dest + nl) = '\0';
nl = strlen(dest);
if (utf8) {
*nc = u8_u16(dest_utf, MAXWORDLEN, dest);
// don't check too long words
if (*nc >= MAXWORDLEN) return 0;
if (*nc == -1) { // big Unicode character (non BMP area)
@@ -163,39 +175,38 @@ int Hunspell::cleanword2(char * dest, co
return nl;
}
*pcaptype = get_captype_utf8(dest_utf, *nc, langnum);
} else {
*pcaptype = get_captype(dest, nl, csconv);
*nc = nl;
}
return nl;
-}
+}
-#ifdef HUNSPELL_EXPERIMENTAL
-int Hunspell::cleanword(char * dest, const char * src,
+int Hunspell::cleanword(char * dest, const char * src,
int * pcaptype, int * pabbrev)
-{
+{
unsigned char * p = (unsigned char *) dest;
const unsigned char * q = (const unsigned char * ) src;
int firstcap = 0;
// first skip over any leading blanks
while ((*q != '\0') && (*q == ' ')) q++;
-
+
// now strip off any trailing periods (recording their presence)
*pabbrev = 0;
int nl = strlen((const char *)q);
while ((nl > 0) && (*(q+nl-1)=='.')) {
nl--;
(*pabbrev)++;
}
-
+
// if no characters are left it can't be capitalized
- if (nl <= 0) {
+ if (nl <= 0) {
*pcaptype = NOCAP;
*p = '\0';
return 0;
}
// now determine the capitalization type of the first nl letters
int ncap = 0;
int nneutral = 0;
@@ -237,35 +248,34 @@ int Hunspell::cleanword(char * dest, con
} else if ((ncap == nc) || ((ncap + nneutral) == nc)){
*pcaptype = ALLCAP;
} else if ((ncap > 1) && firstcap) {
*pcaptype = HUHINITCAP;
} else {
*pcaptype = HUHCAP;
}
return strlen(dest);
-}
-#endif
+}
void Hunspell::mkallcap(char * p)
{
if (utf8) {
w_char u[MAXWORDLEN];
int nc = u8_u16(u, MAXWORDLEN, p);
unsigned short idx;
for (int i = 0; i < nc; i++) {
idx = (u[i].h << 8) + u[i].l;
if (idx != unicodetoupper(idx, langnum)) {
u[i].h = (unsigned char) (unicodetoupper(idx, langnum) >> 8);
u[i].l = (unsigned char) (unicodetoupper(idx, langnum) & 0x00FF);
}
}
u16_u8(p, MAXWORDUTF8LEN, u, nc);
} else {
- while (*p != '\0') {
+ while (*p != '\0') {
*p = csconv[((unsigned char) *p)].cupper;
p++;
}
}
}
int Hunspell::mkallcap2(char * p, w_char * u, int nc)
{
@@ -275,30 +285,30 @@ int Hunspell::mkallcap2(char * p, w_char
idx = (u[i].h << 8) + u[i].l;
unsigned short up = unicodetoupper(idx, langnum);
if (idx != up) {
u[i].h = (unsigned char) (up >> 8);
u[i].l = (unsigned char) (up & 0x00FF);
}
}
u16_u8(p, MAXWORDUTF8LEN, u, nc);
- return strlen(p);
+ return strlen(p);
} else {
- while (*p != '\0') {
+ while (*p != '\0') {
*p = csconv[((unsigned char) *p)].cupper;
p++;
}
}
return nc;
}
void Hunspell::mkallsmall(char * p)
{
- while (*p != '\0') {
+ while (*p != '\0') {
*p = csconv[((unsigned char) *p)].clower;
p++;
}
}
int Hunspell::mkallsmall2(char * p, w_char * u, int nc)
{
if (utf8) {
@@ -309,17 +319,17 @@ int Hunspell::mkallsmall2(char * p, w_ch
if (idx != low) {
u[i].h = (unsigned char) (low >> 8);
u[i].l = (unsigned char) (low & 0x00FF);
}
}
u16_u8(p, MAXWORDUTF8LEN, u, nc);
return strlen(p);
} else {
- while (*p != '\0') {
+ while (*p != '\0') {
*p = csconv[((unsigned char) *p)].clower;
p++;
}
}
return nc;
}
// convert UTF-8 sharp S codes to latin 1
@@ -353,77 +363,79 @@ hentry * Hunspell::spellsharps(char * ba
return NULL;
}
int Hunspell::is_keepcase(const hentry * rv) {
return pAMgr && rv->astr && pAMgr->get_keepcase() &&
TESTAFF(rv->astr, pAMgr->get_keepcase(), rv->alen);
}
-/* insert a word to beginning of the suggestion array and return ns */
+/* insert a word to the beginning of the suggestion array and return ns */
int Hunspell::insert_sug(char ***slst, char * word, int ns) {
+ char * dup = mystrdup(word);
+ if (!dup) return ns;
if (ns == MAXSUGGESTION) {
ns--;
free((*slst)[ns]);
}
for (int k = ns; k > 0; k--) (*slst)[k] = (*slst)[k - 1];
- (*slst)[0] = mystrdup(word);
+ (*slst)[0] = dup;
return ns + 1;
}
int Hunspell::spell(const char * word, int * info, char ** root)
{
struct hentry * rv=NULL;
// need larger vector. For example, Turkish capital letter I converted a
// 2-byte UTF-8 character (dotless i) by mkallsmall.
char cw[MAXWORDUTF8LEN];
char wspace[MAXWORDUTF8LEN];
w_char unicw[MAXWORDLEN];
+ // Hunspell supports XML input of the simplified API (see manual)
+ if (strcmp(word, SPELL_XML) == 0) return 1;
int nc = strlen(word);
int wl2 = 0;
if (utf8) {
if (nc >= MAXWORDUTF8LEN) return 0;
} else {
if (nc >= MAXWORDLEN) return 0;
}
int captype = 0;
int abbv = 0;
- int wl = cleanword2(cw, word, unicw, &nc, &captype, &abbv);
+ int wl = 0;
+
+ // input conversion
+ RepList * rl = (pAMgr) ? pAMgr->get_iconvtable() : NULL;
+ if (rl && rl->conv(word, wspace)) wl = cleanword2(cw, wspace, unicw, &nc, &captype, &abbv);
+ else wl = cleanword2(cw, word, unicw, &nc, &captype, &abbv);
+
int info2 = 0;
- if (wl == 0) return 1;
+ if (wl == 0 || maxdic == 0) return 1;
if (root) *root = NULL;
- // allow numbers with dots and commas (but forbid double separators: "..", ",," etc.)
+ // allow numbers with dots, dashes and commas (but forbid double separators: "..", "--" etc.)
enum { NBEGIN, NNUM, NSEP };
int nstate = NBEGIN;
int i;
for (i = 0; (i < wl); i++) {
if ((cw[i] <= '9') && (cw[i] >= '0')) {
nstate = NNUM;
} else if ((cw[i] == ',') || (cw[i] == '.') || (cw[i] == '-')) {
if ((nstate == NSEP) || (i == 0)) break;
nstate = NSEP;
} else break;
}
if ((i == wl) && (nstate == NNUM)) return 1;
if (!info) info = &info2; else *info = 0;
- // LANG_hu section: number(s) + (percent or degree) with suffixes
- if (langnum == LANG_hu) {
- if ((nstate == NNUM) && ((cw[i] == '%') || ((!utf8 && (cw[i] == '\xB0')) ||
- (utf8 && (strncmp(cw + i, "\xC2\xB0", 2)==0))))
- && checkword(cw + i, info, root)) return 1;
- }
- // END of LANG_hu section
-
switch(captype) {
- case HUHCAP:
- case HUHINITCAP:
- case NOCAP: {
+ case HUHCAP:
+ case HUHINITCAP:
+ case NOCAP: {
rv = checkword(cw, info, root);
if ((abbv) && !(rv)) {
memcpy(wspace,cw,wl);
*(wspace+wl) = '.';
*(wspace+wl+1) = '\0';
rv = checkword(wspace, info, root);
}
break;
@@ -480,30 +492,30 @@ int Hunspell::spell(const char * word, i
*(wspace+wl2) = '.';
*(wspace+wl2+1) = '\0';
rv = spellsharps(wspace, wspace, 0, 0, tmpword, info, root);
}
}
if (rv) break;
}
}
- case INITCAP: {
+ case INITCAP: {
wl = mkallsmall2(cw, unicw, nc);
memcpy(wspace,cw,(wl+1));
wl2 = mkinitcap2(cw, unicw, nc);
if (captype == INITCAP) *info += SPELL_INITCAP;
rv = checkword(cw, info, root);
if (captype == INITCAP) *info -= SPELL_INITCAP;
// forbid bad capitalization
// (for example, ijs -> Ijs instead of IJs in Dutch)
// use explicit forms in dic: Ijs/F (F = FORBIDDENWORD flag)
if (*info & SPELL_FORBIDDEN) {
rv = NULL;
break;
- }
+ }
if (rv && is_keepcase(rv) && (captype == ALLCAP)) rv = NULL;
if (rv) break;
rv = checkword(wspace, info, root);
if (abbv && !rv) {
*(wspace+wl) = '.';
*(wspace+wl+1) = '\0';
@@ -520,105 +532,77 @@ int Hunspell::spell(const char * word, i
}
}
if (rv && is_keepcase(rv) &&
((captype == ALLCAP) ||
// if CHECKSHARPS: KEEPCASE words with \xDF are allowed
// in INITCAP form, too.
!(pAMgr->get_checksharps() &&
((utf8 && strstr(wspace, "\xC3\x9F")) ||
- (!utf8 && strchr(wspace, '\xDF')))))) rv = NULL;
+ (!utf8 && strchr(wspace, '\xDF')))))) rv = NULL;
break;
- }
+ }
}
-
+
if (rv) return 1;
- // recursive breaking at break points (not good for morphological analysis)
+ // recursive breaking at break points
if (wordbreak) {
char * s;
char r;
int corr = 0;
- // German words beginning with "-" are not accepted
- if (langnum == LANG_de) corr = 1;
+ wl = strlen(cw);
int numbreak = pAMgr ? pAMgr->get_numbreak() : 0;
+ // check boundary patterns (^begin and end$)
for (int j = 0; j < numbreak; j++) {
- s=(char *) strstr(cw + corr, wordbreak[j]);
- if (s) {
+ int plen = strlen(wordbreak[j]);
+ if (plen == 1 || plen > wl) continue;
+ if (wordbreak[j][0] == '^' && strncmp(cw, wordbreak[j] + 1, plen - 1) == 0
+ && spell(cw + plen - 1)) return 1;
+ if (wordbreak[j][plen - 1] == '$' &&
+ strncmp(cw + wl - plen + 1, wordbreak[j], plen - 1) == 0) {
+ r = cw[wl - plen + 1];
+ cw[wl - plen + 1] = '\0';
+ if (spell(cw)) return 1;
+ cw[wl - plen + 1] = r;
+ }
+ }
+ // other patterns
+ for (int j = 0; j < numbreak; j++) {
+ int result = 0;
+ int plen = strlen(wordbreak[j]);
+ s=(char *) strstr(cw, wordbreak[j]);
+ if (s && (s > cw) && (s < cw + wl - plen)) {
+ if (!spell(s + plen)) continue;
r = *s;
*s = '\0';
// examine 2 sides of the break point
- if (spell(cw) && spell(s + strlen(wordbreak[j]))) {
- *s = r;
- return 1;
- }
+ if (spell(cw)) return 1;
*s = r;
+
+ // LANG_hu: spec. dash rule
+ if (langnum == LANG_hu && strcmp(wordbreak[j], "-") == 0) {
+ r = s[1];
+ s[1] = '\0';
+ if (spell(cw)) return 1; // check the first part with dash
+ s[1] = r;
+ }
+ // end of LANG speficic region
+
}
}
}
- // LANG_hu: compoundings with dashes and n-dashes XXX deprecated!
- if (langnum == LANG_hu) {
- int n;
- // compound word with dash (HU) I18n
- char * dash;
- int result = 0;
- // n-dash
- dash = (char *) strstr(cw,"\xE2\x80\x93");
- if (dash && !wordbreak) {
- *dash = '\0';
- // examine 2 sides of the dash
- if (spell(cw) && spell(dash + 3)) {
- *dash = '\xE2';
- return 1;
- }
- *dash = '\xE2';
- }
- dash = (char *) strchr(cw,'-');
- if (dash) {
- *dash='\0';
- // examine 2 sides of the dash
- if (dash[1] == '\0') { // base word ending with dash
- if (spell(cw)) return 1;
- } else {
- // first word ending with dash: word-
- char r2 = *(dash + 1);
- dash[0]='-';
- dash[1]='\0';
- result = spell(cw);
- dash[1] = r2;
- dash[0]='\0';
- if (result && spell(dash+1) && ((strlen(dash+1) > 1) || (dash[1] == 'e') ||
- ((dash[1] > '0') && (dash[1] < '9')))) return 1;
- }
- // affixed number in correct word
- if (result && (dash > cw) && (((*(dash-1)<='9') && (*(dash-1)>='0')) || (*(dash-1)>='.'))) {
- *dash='-';
- n = 1;
- if (*(dash - n) == '.') n++;
- // search first not a number character to left from dash
- while (((dash - n)>=cw) && ((*(dash - n)=='0') || (n < 3)) && (n < 6)) {
- n++;
- }
- if ((dash - n) < cw) n--;
- // numbers: deprecated
- for(; n >= 1; n--) {
- if ((*(dash - n) >= '0') && (*(dash - n) <= '9') &&
- checkword(dash - n, info, root)) return 1;
- }
- }
- }
- }
return 0;
}
struct hentry * Hunspell::checkword(const char * w, int * info, char ** root)
{
struct hentry * he = NULL;
- int len;
+ int len, i;
char w2[MAXWORDUTF8LEN];
const char * word;
char * ignoredchars = pAMgr->get_ignore();
if (ignoredchars != NULL) {
strcpy(w2, w);
if (utf8) {
int ignoredchars_utf16_len;
@@ -635,81 +619,82 @@ struct hentry * Hunspell::checkword(cons
if (word != w2) {
strcpy(w2, word);
word = w2;
}
if (utf8) reverseword_utf(w2); else reverseword(w2);
}
// look word in hash table
- if (pHMgr) he = pHMgr->lookup(word);
+ for (i = 0; (i < maxdic) && !he; i ++) {
+ he = (pHMgr[i])->lookup(word);
// check forbidden and onlyincompound words
if ((he) && (he->astr) && (pAMgr) && TESTAFF(he->astr, pAMgr->get_forbiddenword(), he->alen)) {
if (info) *info += SPELL_FORBIDDEN;
// LANG_hu section: set dash information for suggestions
if (langnum == LANG_hu) {
if (pAMgr->get_compoundflag() &&
TESTAFF(he->astr, pAMgr->get_compoundflag(), he->alen)) {
if (info) *info += SPELL_COMPOUND;
}
}
return NULL;
}
- // he = next not pseudoroot, onlyincompound homonym or onlyupcase word
+ // he = next not needaffix, onlyincompound homonym or onlyupcase word
while (he && (he->astr) &&
- ((pAMgr->get_pseudoroot() && TESTAFF(he->astr, pAMgr->get_pseudoroot(), he->alen)) ||
+ ((pAMgr->get_needaffix() && TESTAFF(he->astr, pAMgr->get_needaffix(), he->alen)) ||
(pAMgr->get_onlyincompound() && TESTAFF(he->astr, pAMgr->get_onlyincompound(), he->alen)) ||
(info && (*info & SPELL_INITCAP) && TESTAFF(he->astr, ONLYUPCASEFLAG, he->alen))
)) he = he->next_homonym;
+ }
// check with affixes
if (!he && pAMgr) {
// try stripping off affixes */
len = strlen(word);
he = pAMgr->affix_check(word, len, 0);
// check compound restriction and onlyupcase
if (he && he->astr && (
- (pAMgr->get_onlyincompound() &&
- TESTAFF(he->astr, pAMgr->get_onlyincompound(), he->alen)) ||
+ (pAMgr->get_onlyincompound() &&
+ TESTAFF(he->astr, pAMgr->get_onlyincompound(), he->alen)) ||
(info && (*info & SPELL_INITCAP) &&
TESTAFF(he->astr, ONLYUPCASEFLAG, he->alen)))) {
he = NULL;
}
if (he) {
if ((he->astr) && (pAMgr) && TESTAFF(he->astr, pAMgr->get_forbiddenword(), he->alen)) {
if (info) *info += SPELL_FORBIDDEN;
return NULL;
}
if (root) {
*root = mystrdup(&(he->word));
- if (complexprefixes) {
+ if (*root && complexprefixes) {
if (utf8) reverseword_utf(*root); else reverseword(*root);
}
}
// try check compound word
} else if (pAMgr->get_compound()) {
- he = pAMgr->compound_check(word, len,
- 0,0,100,0,NULL,0,NULL,NULL,0);
+ he = pAMgr->compound_check(word, len, 0, 0, 100, 0, NULL, 0, 0);
// LANG_hu section: `moving rule' with last dash
- if ((!he) && (langnum == LANG_hu) && (word[len-1]=='-')) {
+ if ((!he) && (langnum == LANG_hu) && (word[len-1] == '-')) {
char * dup = mystrdup(word);
+ if (!dup) return NULL;
dup[len-1] = '\0';
- he = pAMgr->compound_check(dup, len-1,
- -5,0,100,0,NULL,1,NULL,NULL,0);
+ he = pAMgr->compound_check(dup, len-1, -5, 0, 100, 0, NULL, 1, 0);
free(dup);
}
- // end of LANG speficic region
+ // end of LANG speficic region
if (he) {
if (root) {
*root = mystrdup(&(he->word));
- if (complexprefixes) {
+ if (*root && complexprefixes) {
if (utf8) reverseword_utf(*root); else reverseword(*root);
}
}
if (info) *info += SPELL_COMPOUND;
}
}
}
@@ -717,65 +702,75 @@ struct hentry * Hunspell::checkword(cons
return he;
}
int Hunspell::suggest(char*** slst, const char * word)
{
int onlycmpdsug = 0;
char cw[MAXWORDUTF8LEN];
char wspace[MAXWORDUTF8LEN];
- if (! pSMgr) return 0;
+ if (!pSMgr || maxdic == 0) return 0;
w_char unicw[MAXWORDLEN];
+ *slst = NULL;
+ // process XML input of the simplified API (see manual)
+ if (strncmp(word, SPELL_XML, sizeof(SPELL_XML) - 3) == 0) {
+ return spellml(slst, word);
+ }
int nc = strlen(word);
if (utf8) {
if (nc >= MAXWORDUTF8LEN) return 0;
} else {
if (nc >= MAXWORDLEN) return 0;
}
int captype = 0;
int abbv = 0;
- int wl = cleanword2(cw, word, unicw, &nc, &captype, &abbv);
+ int wl = 0;
+
+ // input conversion
+ RepList * rl = (pAMgr) ? pAMgr->get_iconvtable() : NULL;
+ if (rl && rl->conv(word, wspace)) wl = cleanword2(cw, wspace, unicw, &nc, &captype, &abbv);
+ else wl = cleanword2(cw, word, unicw, &nc, &captype, &abbv);
+
if (wl == 0) return 0;
int ns = 0;
- *slst = NULL;
int capwords = 0;
switch(captype) {
- case NOCAP: {
+ case NOCAP: {
ns = pSMgr->suggest(slst, cw, ns, &onlycmpdsug);
break;
}
- case INITCAP: {
+ case INITCAP: {
capwords = 1;
ns = pSMgr->suggest(slst, cw, ns, &onlycmpdsug);
if (ns == -1) break;
memcpy(wspace,cw,(wl+1));
mkallsmall2(wspace, unicw, nc);
ns = pSMgr->suggest(slst, wspace, ns, &onlycmpdsug);
break;
}
case HUHINITCAP:
capwords = 1;
- case HUHCAP: {
+ case HUHCAP: {
ns = pSMgr->suggest(slst, cw, ns, &onlycmpdsug);
if (ns != -1) {
int prevns;
// something.The -> something. The
char * dot = strchr(cw, '.');
if (dot && (dot > cw)) {
int captype_;
if (utf8) {
w_char w_[MAXWORDLEN];
int wl_ = u8_u16(w_, MAXWORDLEN, dot + 1);
captype_ = get_captype_utf8(w_, wl_, langnum);
} else captype_ = get_captype(dot+1, strlen(dot+1), csconv);
if (captype_ == INITCAP) {
char * st = mystrdup(cw);
- st = (char *) realloc(st, wl + 2);
+ if (st) st = (char *) realloc(st, wl + 2);
if (st) {
st[(dot - cw) + 1] = ' ';
strcpy(st + (dot - cw) + 2, dot + 1);
ns = insert_sug(slst, st, ns);
free(st);
}
}
}
@@ -812,17 +807,17 @@ int Hunspell::suggest(char*** slst, cons
(*slst)[0] = r;
}
}
}
}
break;
}
- case ALLCAP: {
+ case ALLCAP: {
memcpy(wspace, cw, (wl+1));
mkallsmall2(wspace, unicw, nc);
ns = pSMgr->suggest(slst, wspace, ns, &onlycmpdsug);
if (ns == -1) break;
if (pAMgr && pAMgr->get_keepcase() && spell(wspace))
ns = insert_sug(slst, wspace, ns);
mkinitcap2(wspace, unicw, nc);
ns = pSMgr->suggest(slst, wspace, ns, &onlycmpdsug);
@@ -864,49 +859,88 @@ int Hunspell::suggest(char*** slst, cons
spell(w, &info, NULL);
if ((info & SPELL_COMPOUND) && (info & SPELL_FORBIDDEN)) {
*pos = ' ';
} else *pos = '-';
}
}
}
// END OF LANG_hu section
-
+
// try ngram approach since found nothing
if ((ns == 0 || onlycmpdsug) && pAMgr && (pAMgr->get_maxngramsugs() != 0)) {
switch(captype) {
case NOCAP: {
- ns = pSMgr->ngsuggest(*slst, cw, ns, pHMgr);
+ ns = pSMgr->ngsuggest(*slst, cw, ns, pHMgr, maxdic);
break;
}
+ case HUHINITCAP:
+ capwords = 1;
case HUHCAP: {
memcpy(wspace,cw,(wl+1));
mkallsmall2(wspace, unicw, nc);
- ns = pSMgr->ngsuggest(*slst, wspace, ns, pHMgr);
- break;
+ ns = pSMgr->ngsuggest(*slst, wspace, ns, pHMgr, maxdic);
+ break;
}
- case INITCAP: {
+ case INITCAP: {
capwords = 1;
memcpy(wspace,cw,(wl+1));
mkallsmall2(wspace, unicw, nc);
- ns = pSMgr->ngsuggest(*slst, wspace, ns, pHMgr);
+ ns = pSMgr->ngsuggest(*slst, wspace, ns, pHMgr, maxdic);
break;
}
case ALLCAP: {
memcpy(wspace,cw,(wl+1));
mkallsmall2(wspace, unicw, nc);
int oldns = ns;
- ns = pSMgr->ngsuggest(*slst, wspace, ns, pHMgr);
- for (int j = oldns; j < ns; j++)
+ ns = pSMgr->ngsuggest(*slst, wspace, ns, pHMgr, maxdic);
+ for (int j = oldns; j < ns; j++)
mkallcap((*slst)[j]);
break;
}
}
}
+ // try dash suggestion (Afo-American -> Afro-American)
+ if (strchr(cw, '-')) {
+ char * pos = strchr(cw, '-');
+ char * ppos = cw;
+ int nodashsug = 1;
+ char ** nlst = NULL;
+ int nn = 0;
+ int last = 0;
+ for (int j = 0; j < ns && nodashsug == 1; j++) {
+ if (strchr((*slst)[j], '-')) nodashsug = 0;
+ }
+ while (nodashsug && !last) {
+ if (*pos == '\0') last = 1; else *pos = '\0';
+ if (!spell(ppos)) {
+ nn = suggest(&nlst, ppos);
+ for (int j = nn - 1; j >= 0; j--) {
+ strncpy(wspace, cw, ppos - cw);
+ strcpy(wspace + (ppos - cw), nlst[j]);
+ if (!last) {
+ strcat(wspace, "-");
+ strcat(wspace, pos + 1);
+ }
+ ns = insert_sug(slst, wspace, ns);
+ free(nlst[j]);
+ }
+ if (nlst != NULL) free(nlst);
+ nodashsug = 0;
+ }
+ if (!last) {
+ *pos = '-';
+ ppos = pos + 1;
+ pos = strchr(ppos, '-');
+ }
+ if (!pos) pos = cw + strlen(cw);
+ }
+ }
+
// word reversing wrapper for complex prefixes
if (complexprefixes) {
for (int j = 0; j < ns; j++) {
if (utf8) reverseword_utf((*slst)[j]); else reverseword((*slst)[j]);
}
}
// capitalize
@@ -935,31 +969,31 @@ int Hunspell::suggest(char*** slst, cons
int len;
if (utf8) {
len = u8_u16(w, MAXSWL, (*slst)[j]);
} else {
strcpy(s, (*slst)[j]);
len = strlen(s);
}
mkallsmall2(s, w, len);
- free((*slst)[j]);
+ free((*slst)[j]);
if (spell(s)) {
(*slst)[l] = mystrdup(s);
- l++;
+ if ((*slst)[l]) l++;
} else {
mkinitcap2(s, w, len);
if (spell(s)) {
(*slst)[l] = mystrdup(s);
- l++;
+ if ((*slst)[l]) l++;
}
}
} else {
(*slst)[l] = (*slst)[j];
l++;
- }
+ }
}
ns = l;
}
}
}
// remove duplications
int l = 0;
@@ -968,73 +1002,93 @@ int Hunspell::suggest(char*** slst, cons
for (int k = 0; k < l; k++) {
if (strcmp((*slst)[k], (*slst)[j]) == 0) {
free((*slst)[j]);
l--;
}
}
l++;
}
+
+ // output conversion
+ rl = (pAMgr) ? pAMgr->get_oconvtable() : NULL;
+ for (int j = 0; rl && j < ns; j++) {
+ if (rl->conv((*slst)[j], wspace)) {
+ free((*slst)[j]);
+ (*slst)[j] = mystrdup(wspace);
+ }
+ }
+
+ // if suggestions removed by nosuggest, onlyincompound parameters
+ if (l == 0 && *slst) {
+ free(*slst);
+ *slst = NULL;
+ }
return l;
}
+void Hunspell::free_list(char *** slst, int n) {
+ freelist(slst, n);
+}
+
char * Hunspell::get_dic_encoding()
{
return encoding;
}
#ifdef HUNSPELL_EXPERIMENTAL
// XXX need UTF-8 support
int Hunspell::suggest_auto(char*** slst, const char * word)
{
char cw[MAXWORDUTF8LEN];
char wspace[MAXWORDUTF8LEN];
- if (! pSMgr) return 0;
+ if (!pSMgr || maxdic == 0) return 0;
int wl = strlen(word);
if (utf8) {
if (wl >= MAXWORDUTF8LEN) return 0;
} else {
if (wl >= MAXWORDLEN) return 0;
}
int captype = 0;
int abbv = 0;
wl = cleanword(cw, word, &captype, &abbv);
if (wl == 0) return 0;
int ns = 0;
*slst = NULL; // HU, nsug in pSMgr->suggest
-
+
switch(captype) {
- case NOCAP: {
+ case NOCAP: {
ns = pSMgr->suggest_auto(slst, cw, ns);
if (ns>0) break;
break;
}
- case INITCAP: {
+ case INITCAP: {
memcpy(wspace,cw,(wl+1));
mkallsmall(wspace);
ns = pSMgr->suggest_auto(slst, wspace, ns);
for (int j=0; j < ns; j++)
mkinitcap((*slst)[j]);
ns = pSMgr->suggest_auto(slst, cw, ns);
break;
-
+
}
- case HUHCAP: {
+ case HUHINITCAP:
+ case HUHCAP: {
ns = pSMgr->suggest_auto(slst, cw, ns);
if (ns == 0) {
memcpy(wspace,cw,(wl+1));
mkallsmall(wspace);
ns = pSMgr->suggest_auto(slst, wspace, ns);
}
break;
}
- case ALLCAP: {
+ case ALLCAP: {
memcpy(wspace,cw,(wl+1));
mkallsmall(wspace);
ns = pSMgr->suggest_auto(slst, wspace, ns);
mkinitcap(wspace);
ns = pSMgr->suggest_auto(slst, wspace, ns);
for (int j=0; j < ns; j++)
@@ -1070,161 +1124,147 @@ int Hunspell::suggest_auto(char*** slst,
strcat(w, pos + 1);
spell(w, &info, NULL);
if ((info & SPELL_COMPOUND) && (info & SPELL_FORBIDDEN)) {
*pos = ' ';
} else *pos = '-';
}
}
}
- // END OF LANG_hu section
+ // END OF LANG_hu section
return ns;
}
+#endif
-// XXX need UTF-8 support
+int Hunspell::stem(char*** slst, char ** desc, int n)
+{
+ char result[MAXLNLEN];
+ char result2[MAXLNLEN];
+ *slst = NULL;
+ if (n == 0) return 0;
+ *result2 = '\0';
+ for (int i = 0; i < n; i++) {
+ *result = '\0';
+ // add compound word parts (except the last one)
+ char * s = (char *) desc[i];
+ char * part = strstr(s, MORPH_PART);
+ if (part) {
+ char * nextpart = strstr(part + 1, MORPH_PART);
+ while (nextpart) {
+ copy_field(result + strlen(result), part, MORPH_PART);
+ part = nextpart;
+ nextpart = strstr(part + 1, MORPH_PART);
+ }
+ s = part;
+ }
+
+ char **pl;
+ char tok[MAXLNLEN];
+ strcpy(tok, s);
+ char * alt = strstr(tok, " | ");
+ while (alt) {
+ alt[1] = MSEP_ALT;
+ alt = strstr(alt, " | ");
+ }
+ int pln = line_tok(tok, &pl, MSEP_ALT);
+ for (int k = 0; k < pln; k++) {
+ // add derivational suffixes
+ if (strstr(pl[k], MORPH_DERI_SFX)) {
+ // remove inflectional suffixes
+ char * is = strstr(pl[k], MORPH_INFL_SFX);
+ if (is) *is = '\0';
+ char * sg = pSMgr->suggest_gen(&(pl[k]), 1, pl[k]);
+ if (sg) {
+ char ** gen;
+ int genl = line_tok(sg, &gen, MSEP_REC);
+ free(sg);
+ for (int j = 0; j < genl; j++) {
+ sprintf(result2 + strlen(result2), "%c%s%s",
+ MSEP_REC, result, gen[j]);
+ }
+ freelist(&gen, genl);
+ }
+ } else {
+ sprintf(result2 + strlen(result2), "%c%s", MSEP_REC, result);
+ if (strstr(pl[k], MORPH_SURF_PFX)) {
+ copy_field(result2 + strlen(result2), pl[k], MORPH_SURF_PFX);
+ }
+ copy_field(result2 + strlen(result2), pl[k], MORPH_STEM);
+ }
+ }
+ freelist(&pl, pln);
+ }
+ int sln = line_tok(result2, slst, MSEP_REC);
+ return uniqlist(*slst, sln);
+
+}
+
int Hunspell::stem(char*** slst, const char * word)
{
+ char ** pl;
+ int pln = analyze(&pl, word);
+ int pln2 = stem(slst, pl, pln);
+ freelist(&pl, pln);
+ return pln2;
+}
+
+#ifdef HUNSPELL_EXPERIMENTAL
+int Hunspell::suggest_pos_stems(char*** slst, const char * word)
+{
char cw[MAXWORDUTF8LEN];
char wspace[MAXWORDUTF8LEN];
- if (! pSMgr) return 0;
+ if (! pSMgr || maxdic == 0) return 0;
int wl = strlen(word);
if (utf8) {
if (wl >= MAXWORDUTF8LEN) return 0;
} else {
if (wl >= MAXWORDLEN) return 0;
}
int captype = 0;
int abbv = 0;
wl = cleanword(cw, word, &captype, &abbv);
if (wl == 0) return 0;
-
- int ns = 0;
- *slst = NULL; // HU, nsug in pSMgr->suggest
-
- switch(captype) {
- case HUHCAP:
- case NOCAP: {
- ns = pSMgr->suggest_stems(slst, cw, ns);
-
- if ((abbv) && (ns == 0)) {
- memcpy(wspace,cw,wl);
- *(wspace+wl) = '.';
- *(wspace+wl+1) = '\0';
- ns = pSMgr->suggest_stems(slst, wspace, ns);
- }
-
- break;
- }
-
- case INITCAP: {
-
- ns = pSMgr->suggest_stems(slst, cw, ns);
-
- if (ns == 0) {
- memcpy(wspace,cw,(wl+1));
- mkallsmall(wspace);
- ns = pSMgr->suggest_stems(slst, wspace, ns);
-
- }
-
- if ((abbv) && (ns == 0)) {
- memcpy(wspace,cw,wl);
- mkallsmall(wspace);
- *(wspace+wl) = '.';
- *(wspace+wl+1) = '\0';
- ns = pSMgr->suggest_stems(slst, wspace, ns);
- }
-
- break;
-
- }
-
- case ALLCAP: {
- ns = pSMgr->suggest_stems(slst, cw, ns);
- if (ns != 0) break;
-
- memcpy(wspace,cw,(wl+1));
- mkallsmall(wspace);
- ns = pSMgr->suggest_stems(slst, wspace, ns);
-
- if (ns == 0) {
- mkinitcap(wspace);
- ns = pSMgr->suggest_stems(slst, wspace, ns);
- }
-
- if ((abbv) && (ns == 0)) {
- memcpy(wspace,cw,wl);
- mkallsmall(wspace);
- *(wspace+wl) = '.';
- *(wspace+wl+1) = '\0';
- ns = pSMgr->suggest_stems(slst, wspace, ns);
- }
-
-
- break;
- }
- }
-
- return ns;
-}
-
-int Hunspell::suggest_pos_stems(char*** slst, const char * word)
-{
- char cw[MAXWORDUTF8LEN];
- char wspace[MAXWORDUTF8LEN];
- if (! pSMgr) return 0;
- int wl = strlen(word);
- if (utf8) {
- if (wl >= MAXWORDUTF8LEN) return 0;
- } else {
- if (wl >= MAXWORDLEN) return 0;
- }
- int captype = 0;
- int abbv = 0;
- wl = cleanword(cw, word, &captype, &abbv);
- if (wl == 0) return 0;
-
int ns = 0; // ns=0 = normalized input
*slst = NULL; // HU, nsug in pSMgr->suggest
-
+
switch(captype) {
case HUHCAP:
- case NOCAP: {
+ case NOCAP: {
ns = pSMgr->suggest_pos_stems(slst, cw, ns);
if ((abbv) && (ns == 0)) {
memcpy(wspace,cw,wl);
*(wspace+wl) = '.';
*(wspace+wl+1) = '\0';
ns = pSMgr->suggest_pos_stems(slst, wspace, ns);
}
break;
}
- case INITCAP: {
+ case INITCAP: {
ns = pSMgr->suggest_pos_stems(slst, cw, ns);
if (ns == 0 || ((*slst)[0][0] == '#')) {
memcpy(wspace,cw,(wl+1));
mkallsmall(wspace);
ns = pSMgr->suggest_pos_stems(slst, wspace, ns);
}
-
+
break;
-
+
}
- case ALLCAP: {
+ case ALLCAP: {
ns = pSMgr->suggest_pos_stems(slst, cw, ns);
if (ns != 0) break;
-
+
memcpy(wspace,cw,(wl+1));
mkallsmall(wspace);
ns = pSMgr->suggest_pos_stems(slst, wspace, ns);
if (ns == 0) {
mkinitcap(wspace);
ns = pSMgr->suggest_pos_stems(slst, wspace, ns);
}
@@ -1284,513 +1324,605 @@ int Hunspell::mkinitsmall2(char * p, w_c
u[0].h = (unsigned char) (i >> 8);
u[0].l = (unsigned char) (i & 0x00FF);
u16_u8(p, MAXWORDUTF8LEN, u, nc);
return strlen(p);
}
return nc;
}
-int Hunspell::put_word(const char * word)
+int Hunspell::add(const char * word)
{
- if (pHMgr) return pHMgr->put_word(word, NULL);
+ if (pHMgr[0]) return (pHMgr[0])->add(word);
return 0;
}
-int Hunspell::put_word_pattern(const char * word, const char * pattern)
+int Hunspell::add_with_affix(const char * word, const char * example)
{
- if (pHMgr) return pHMgr->put_word_pattern(word, pattern);
+ if (pHMgr[0]) return (pHMgr[0])->add_with_affix(word, example);
+ return 0;
+}
+
+int Hunspell::remove(const char * word)
+{
+ if (pHMgr[0]) return (pHMgr[0])->remove(word);
return 0;
}
const char * Hunspell::get_version()
{
return pAMgr->get_version();
}
struct cs_info * Hunspell::get_csconv()
{
return csconv;
}
-#ifdef HUNSPELL_EXPERIMENTAL
-// XXX need UTF-8 support
-char * Hunspell::morph(const char * word)
+void Hunspell::cat_result(char * result, char * st)
+{
+ if (st) {
+ if (*result) mystrcat(result, "\n", MAXLNLEN);
+ mystrcat(result, st, MAXLNLEN);
+ free(st);
+ }
+}
+
+int Hunspell::analyze(char*** slst, const char * word)
{
char cw[MAXWORDUTF8LEN];
char wspace[MAXWORDUTF8LEN];
- if (! pSMgr) return 0;
- int wl = strlen(word);
+ w_char unicw[MAXWORDLEN];
+ int wl2 = 0;
+ *slst = NULL;
+ if (! pSMgr || maxdic == 0) return 0;
+ int nc = strlen(word);
if (utf8) {
- if (wl >= MAXWORDUTF8LEN) return 0;
+ if (nc >= MAXWORDUTF8LEN) return 0;
} else {
- if (wl >= MAXWORDLEN) return 0;
+ if (nc >= MAXWORDLEN) return 0;
}
int captype = 0;
int abbv = 0;
- wl = cleanword(cw, word, &captype, &abbv);
+ int wl = 0;
+
+ // input conversion
+ RepList * rl = (pAMgr) ? pAMgr->get_iconvtable() : NULL;
+ if (rl && rl->conv(word, wspace)) wl = cleanword2(cw, wspace, unicw, &nc, &captype, &abbv);
+ else wl = cleanword2(cw, word, unicw, &nc, &captype, &abbv);
+
if (wl == 0) {
if (abbv) {
for (wl = 0; wl < abbv; wl++) cw[wl] = '.';
cw[wl] = '\0';
abbv = 0;
} else return 0;
}
char result[MAXLNLEN];
char * st = NULL;
-
+
*result = '\0';
int n = 0;
int n2 = 0;
int n3 = 0;
// test numbers
// LANG_hu section: set dash information for suggestions
if (langnum == LANG_hu) {
- while ((n < wl) &&
+ while ((n < wl) &&
(((cw[n] <= '9') && (cw[n] >= '0')) || (((cw[n] == '.') || (cw[n] == ',')) && (n > 0)))) {
n++;
if ((cw[n] == '.') || (cw[n] == ',')) {
- if (((n2 == 0) && (n > 3)) ||
+ if (((n2 == 0) && (n > 3)) ||
((n2 > 0) && ((cw[n-1] == '.') || (cw[n-1] == ',')))) break;
n2++;
n3 = n;
}
}
- if ((n == wl) && (n3 > 0) && (n - n3 > 3)) return NULL;
+ if ((n == wl) && (n3 > 0) && (n - n3 > 3)) return 0;
if ((n == wl) || ((n>0) && ((cw[n]=='%') || (cw[n]=='\xB0')) && checkword(cw+n, NULL, NULL))) {
- strcat(result, cw);
+ mystrcat(result, cw, MAXLNLEN);
result[n - 1] = '\0';
- if (n == wl) {
- st = pSMgr->suggest_morph(cw + n - 1);
- if (st) {
- strcat(result, st);
- free(st);
- }
- } else {
+ if (n == wl) cat_result(result, pSMgr->suggest_morph(cw + n - 1));
+ else {
char sign = cw[n];
cw[n] = '\0';
- st = pSMgr->suggest_morph(cw + n - 1);
- if (st) {
- strcat(result, st);
- free(st);
- }
- strcat(result, "+"); // XXX SPEC. MORPHCODE
+ cat_result(result, pSMgr->suggest_morph(cw + n - 1));
+ mystrcat(result, "+", MAXLNLEN); // XXX SPEC. MORPHCODE
cw[n] = sign;
- st = pSMgr->suggest_morph(cw + n);
- if (st) {
- strcat(result, st);
- free(st);
- }
+ cat_result(result, pSMgr->suggest_morph(cw + n));
}
- return mystrdup(result);
+ return line_tok(result, slst, MSEP_REC);
}
}
// END OF LANG_hu section
-
+
switch(captype) {
- case NOCAP: {
- st = pSMgr->suggest_morph(cw);
- if (st) {
- strcat(result, st);
- free(st);
- }
- if (abbv) {
- memcpy(wspace,cw,wl);
+ case HUHCAP:
+ case HUHINITCAP:
+ case NOCAP: {
+ cat_result(result, pSMgr->suggest_morph(cw));
+ if (abbv) {
+ memcpy(wspace,cw,wl);
+ *(wspace+wl) = '.';
+ *(wspace+wl+1) = '\0';
+ cat_result(result, pSMgr->suggest_morph(wspace));
+ }
+ break;
+ }
+ case INITCAP: {
+ wl = mkallsmall2(cw, unicw, nc);
+ memcpy(wspace,cw,(wl+1));
+ wl2 = mkinitcap2(cw, unicw, nc);
+ cat_result(result, pSMgr->suggest_morph(wspace));
+ cat_result(result, pSMgr->suggest_morph(cw));
+ if (abbv) {
*(wspace+wl) = '.';
*(wspace+wl+1) = '\0';
- st = pSMgr->suggest_morph(wspace);
- if (st) {
- if (*result) strcat(result, "\n");
- strcat(result, st);
- free(st);
- }
- }
- break;
- }
- case INITCAP: {
- memcpy(wspace,cw,(wl+1));
- mkallsmall(wspace);
- st = pSMgr->suggest_morph(wspace);
- if (st) {
- strcat(result, st);
- free(st);
- }
- st = pSMgr->suggest_morph(cw);
- if (st) {
- if (*result) strcat(result, "\n");
- strcat(result, st);
- free(st);
- }
- if (abbv) {
- memcpy(wspace,cw,wl);
- *(wspace+wl) = '.';
- *(wspace+wl+1) = '\0';
- mkallsmall(wspace);
- st = pSMgr->suggest_morph(wspace);
- if (st) {
- if (*result) strcat(result, "\n");
- strcat(result, st);
- free(st);
- }
- mkinitcap(wspace);
- st = pSMgr->suggest_morph(wspace);
- if (st) {
- if (*result) strcat(result, "\n");
- strcat(result, st);
- free(st);
- }
+ cat_result(result, pSMgr->suggest_morph(wspace));
+
+ memcpy(wspace, cw, wl2);
+ *(wspace+wl2) = '.';
+ *(wspace+wl2+1) = '\0';
+
+ cat_result(result, pSMgr->suggest_morph(wspace));
}
break;
}
- case HUHCAP: {
- st = pSMgr->suggest_morph(cw);
- if (st) {
- strcat(result, st);
- free(st);
- }
-#if 0
- memcpy(wspace,cw,(wl+1));
- mkallsmall(wspace);
- st = pSMgr->suggest_morph(wspace);
- if (st) {
- if (*result) strcat(result, "\n");
- strcat(result, st);
- free(st);
- }
-#endif
- break;
- }
- case ALLCAP: {
- memcpy(wspace,cw,(wl+1));
- st = pSMgr->suggest_morph(wspace);
- if (st) {
- strcat(result, st);
- free(st);
- }
- mkallsmall(wspace);
- st = pSMgr->suggest_morph(wspace);
- if (st) {
- if (*result) strcat(result, "\n");
- strcat(result, st);
- free(st);
+ case ALLCAP: {
+ cat_result(result, pSMgr->suggest_morph(cw));
+ if (abbv) {
+ memcpy(wspace,cw,wl);
+ *(wspace+wl) = '.';
+ *(wspace+wl+1) = '\0';
+ cat_result(result, pSMgr->suggest_morph(cw));
}
- mkinitcap(wspace);
- st = pSMgr->suggest_morph(wspace);
- if (st) {
- if (*result) strcat(result, "\n");
- strcat(result, st);
- free(st);
+ wl = mkallsmall2(cw, unicw, nc);
+ memcpy(wspace,cw,(wl+1));
+ wl2 = mkinitcap2(cw, unicw, nc);
+
+ cat_result(result, pSMgr->suggest_morph(wspace));
+ cat_result(result, pSMgr->suggest_morph(cw));
+ if (abbv) {
+ *(wspace+wl) = '.';
+ *(wspace+wl+1) = '\0';
+ cat_result(result, pSMgr->suggest_morph(wspace));
+
+ memcpy(wspace, cw, wl2);
+ *(wspace+wl2) = '.';
+ *(wspace+wl2+1) = '\0';
+
+ cat_result(result, pSMgr->suggest_morph(wspace));
}
- if (abbv) {
- memcpy(wspace,cw,(wl+1));
- *(wspace+wl) = '.';
- *(wspace+wl+1) = '\0';
- if (*result) strcat(result, "\n");
- st = pSMgr->suggest_morph(wspace);
- if (st) {
- strcat(result, st);
- free(st);
- }
- mkallsmall(wspace);
- st = pSMgr->suggest_morph(wspace);
- if (st) {
- if (*result) strcat(result, "\n");
- strcat(result, st);
- free(st);
- }
- mkinitcap(wspace);
- st = pSMgr->suggest_morph(wspace);
- if (st) {
- if (*result) strcat(result, "\n");
- strcat(result, st);
- free(st);
- }
- }
break;
}
}
- if (result && (*result)) {
+ if (*result) {
// word reversing wrapper for complex prefixes
if (complexprefixes) {
if (utf8) reverseword_utf(result); else reverseword(result);
}
- return mystrdup(result);
+ return line_tok(result, slst, MSEP_REC);
+
}
// compound word with dash (HU) I18n
char * dash = NULL;
int nresult = 0;
// LANG_hu section: set dash information for suggestions
if (langnum == LANG_hu) dash = (char *) strchr(cw,'-');
if ((langnum == LANG_hu) && dash) {
- *dash='\0';
+ *dash='\0';
// examine 2 sides of the dash
if (dash[1] == '\0') { // base word ending with dash
- if (spell(cw)) return pSMgr->suggest_morph(cw);
+ if (spell(cw)) return line_tok(pSMgr->suggest_morph(cw), slst, MSEP_REC);
} else if ((dash[1] == 'e') && (dash[2] == '\0')) { // XXX (HU) -e hat.
if (spell(cw) && (spell("-e"))) {
st = pSMgr->suggest_morph(cw);
if (st) {
- strcat(result, st);
+ mystrcat(result, st, MAXLNLEN);
free(st);
}
- strcat(result,"+"); // XXX spec. separator in MORPHCODE
+ mystrcat(result,"+", MAXLNLEN); // XXX spec. separator in MORPHCODE
st = pSMgr->suggest_morph("-e");
if (st) {
- strcat(result, st);
+ mystrcat(result, st, MAXLNLEN);
free(st);
}
- return mystrdup(result);
+ return line_tok(result, slst, MSEP_REC);
}
} else {
// first word ending with dash: word- XXX ???
char r2 = *(dash + 1);
dash[0]='-';
dash[1]='\0';
nresult = spell(cw);
dash[1] = r2;
dash[0]='\0';
if (nresult && spell(dash+1) && ((strlen(dash+1) > 1) ||
((dash[1] > '0') && (dash[1] < '9')))) {
- st = morph(cw);
+ st = pSMgr->suggest_morph(cw);
if (st) {
- strcat(result, st);
+ mystrcat(result, st, MAXLNLEN);
free(st);
- strcat(result,"+"); // XXX spec. separator in MORPHCODE
+ mystrcat(result,"+", MAXLNLEN); // XXX spec. separator in MORPHCODE
}
- st = morph(dash+1);
+ st = pSMgr->suggest_morph(dash+1);
if (st) {
- strcat(result, st);
+ mystrcat(result, st, MAXLNLEN);
free(st);
}
- return mystrdup(result);
+ return line_tok(result, slst, MSEP_REC);
}
}
// affixed number in correct word
- if (nresult && (dash > cw) && (((*(dash-1)<='9') &&
+ if (nresult && (dash > cw) && (((*(dash-1)<='9') &&
(*(dash-1)>='0')) || (*(dash-1)=='.'))) {
*dash='-';
n = 1;
if (*(dash - n) == '.') n++;
// search first not a number character to left from dash
while (((dash - n)>=cw) && ((*(dash - n)=='0') || (n < 3)) && (n < 6)) {
n++;
}
if ((dash - n) < cw) n--;
// numbers: valami1000000-hoz
// examine 100000-hoz, 10000-hoz 1000-hoz, 10-hoz,
// 56-hoz, 6-hoz
for(; n >= 1; n--) {
if ((*(dash - n) >= '0') && (*(dash - n) <= '9') && checkword(dash - n, NULL, NULL)) {
- strcat(result, cw);
+ mystrcat(result, cw, MAXLNLEN);
result[dash - cw - n] = '\0';
st = pSMgr->suggest_morph(dash - n);
if (st) {
- strcat(result, st);
+ mystrcat(result, st, MAXLNLEN);
free(st);
}
- return mystrdup(result);
+ return line_tok(result, slst, MSEP_REC);
}
}
}
}
- return NULL;
+ return 0;
+}
+
+int Hunspell::generate(char*** slst, const char * word, char ** pl, int pln)
+{
+ *slst = NULL;
+ if (!pSMgr || !pln) return 0;
+ char **pl2;
+ int pl2n = analyze(&pl2, word);
+ int captype = 0;
+ int abbv = 0;
+ char cw[MAXWORDUTF8LEN];
+ cleanword(cw, word, &captype, &abbv);
+ char result[MAXLNLEN];
+ *result = '\0';
+
+ for (int i = 0; i < pln; i++) {
+ cat_result(result, pSMgr->suggest_gen(pl2, pl2n, pl[i]));
+ }
+ freelist(&pl2, pl2n);
+
+ if (*result) {
+ // allcap
+ if (captype == ALLCAP) mkallcap(result);
+
+ // line split
+ int linenum = line_tok(result, slst, MSEP_REC);
+
+ // capitalize
+ if (captype == INITCAP || captype == HUHINITCAP) {
+ for (int j=0; j < linenum; j++) mkinitcap((*slst)[j]);
+ }
+
+ // temporary filtering of prefix related errors (eg.
+ // generate("undrinkable", "eats") --> "undrinkables" and "*undrinks")
+
+ int r = 0;
+ for (int j=0; j < linenum; j++) {
+ if (!spell((*slst)[j])) {
+ free((*slst)[j]);
+ (*slst)[j] = NULL;
+ } else {
+ if (r < j) (*slst)[r] = (*slst)[j];
+ r++;
+ }
+ }
+ if (r > 0) return r;
+ free(*slst);
+ *slst = NULL;
+ }
+ return 0;
+}
+
+int Hunspell::generate(char*** slst, const char * word, const char * pattern)
+{
+ char **pl;
+ int pln = analyze(&pl, pattern);
+ int n = generate(slst, word, pl, pln);
+ freelist(&pl, pln);
+ return uniqlist(*slst, n);
+}
+
+// minimal XML parser functions
+int Hunspell::get_xml_par(char * dest, const char * par, int max)
+{
+ char * d = dest;
+ if (!par) return 0;
+ char end = *par;
+ char * dmax = dest + max;
+ if (end == '>') end = '<';
+ else if (end != '\'' && end != '"') return 0; // bad XML
+ for (par++; d < dmax && *par != '\0' && *par != end; par++, d++) *d = *par;
+ *d = '\0';
+ mystrrep(dest, "<", "<");
+ mystrrep(dest, "&", "&");
+ return d - dest;
}
+// return the beginning of the element (attr == NULL) or the attribute
+const char * Hunspell::get_xml_pos(const char * s, const char * attr)
+{
+ const char * end = strchr(s, '>');
+ const char * p = s;
+ if (attr == NULL) return end;
+ do {
+ p = strstr(p, attr);
+ if (!p || p >= end) return 0;
+ } while (*(p-1) != ' ' && *(p-1) != '\n');
+ return p + strlen(attr);
+}
+
+int Hunspell::check_xml_par(const char * q, const char * attr, const char * value) {
+ char cw[MAXWORDUTF8LEN];
+ if (get_xml_par(cw, get_xml_pos(q, attr), MAXWORDUTF8LEN - 1) &&
+ strcmp(cw, value) == 0) return 1;
+ return 0;
+}
+
+int Hunspell::get_xml_list(char ***slst, char * list, const char * tag) {
+ int n = 0;
+ char * p;
+ if (!list) return 0;
+ for (p = list; (p = strstr(p, tag)); p++) n++;
+ if (n == 0) return 0;
+ *slst = (char **) malloc(sizeof(char *) * n);
+ if (!*slst) return 0;
+ for (p = list, n = 0; (p = strstr(p, tag)); p++, n++) {
+ int l = strlen(p);
+ (*slst)[n] = (char *) malloc(l);
+ if (!(*slst)[n]) return (n > 0 ? n - 1 : 0);
+ get_xml_par((*slst)[n], p + strlen(tag) - 1, l);
+ }
+ return n;
+}
+
+int Hunspell::spellml(char*** slst, const char * word)
+{
+ char *q, *q2;
+ char cw[MAXWORDUTF8LEN], cw2[MAXWORDUTF8LEN];
+ q = (char *) strstr(word, "<query");
+ if (!q) return 0; // bad XML input
+ q2 = strchr(q, '>');
+ if (!q2) return 0; // bad XML input
+ q2 = strstr(q2, "<word");
+ if (!q2) return 0; // bad XML input
+ if (check_xml_par(q, "type=", "analyze")) {
+ int n = 0, s = 0;
+ if (get_xml_par(cw, strchr(q2, '>'), MAXWORDUTF8LEN)) n = analyze(slst, cw);
+ if (n == 0) return 0;
+ // convert the result to <code><a>ana1</a><a>ana2</a></code> format
+ for (int i = 0; i < n; i++) s+= strlen((*slst)[i]);
+ char * r = (char *) malloc(6 + 5 * s + 7 * n + 7 + 1); // XXX 5*s->&->&
+ if (!r) return 0;
+ strcpy(r, "<code>");
+ for (int i = 0; i < n; i++) {
+ int l = strlen(r);
+ strcpy(r + l, "<a>");
+ strcpy(r + l + 3, (*slst)[i]);
+ mystrrep(r + l + 3, "\t", " ");
+ mystrrep(r + l + 3, "<", "<");
+ mystrrep(r + l + 3, "&", "&");
+ strcat(r, "</a>");
+ free((*slst)[i]);
+ }
+ strcat(r, "</code>");
+ (*slst)[0] = r;
+ return 1;
+ } else if (check_xml_par(q, "type=", "stem")) {
+ if (get_xml_par(cw, strchr(q2, '>'), MAXWORDUTF8LEN)) return stem(slst, cw);
+ } else if (check_xml_par(q, "type=", "generate")) {
+ int n = get_xml_par(cw, strchr(q2, '>'), MAXWORDUTF8LEN);
+ if (n == 0) return 0;
+ char * q3 = strstr(q2 + 1, "<word");
+ if (q3) {
+ if (get_xml_par(cw2, strchr(q3, '>'), MAXWORDUTF8LEN)) {
+ return generate(slst, cw, cw2);
+ }
+ } else {
+ char ** slst2;
+ if ((q2 = strstr(q2 + 1, "<code")) &&
+ (n = get_xml_list(&slst2, strchr(q2, '>'), "<a>"))) {
+ int n2 = generate(slst, cw, slst2, n);
+ freelist(&slst2, n);
+ return uniqlist(*slst, n2);
+ }
+ }
+ }
+ return 0;
+}
+
+
+#ifdef HUNSPELL_EXPERIMENTAL
// XXX need UTF-8 support
char * Hunspell::morph_with_correction(const char * word)
{
char cw[MAXWORDUTF8LEN];
char wspace[MAXWORDUTF8LEN];
- if (! pSMgr) return 0;
+ if (! pSMgr || maxdic == 0) return NULL;
int wl = strlen(word);
if (utf8) {
- if (wl >= MAXWORDUTF8LEN) return 0;
+ if (wl >= MAXWORDUTF8LEN) return NULL;
} else {
- if (wl >= MAXWORDLEN) return 0;
+ if (wl >= MAXWORDLEN) return NULL;
}
int captype = 0;
int abbv = 0;
wl = cleanword(cw, word, &captype, &abbv);
- if (wl == 0) return 0;
+ if (wl == 0) return NULL;
char result[MAXLNLEN];
char * st = NULL;
-
+
*result = '\0';
-
-
+
+
switch(captype) {
- case NOCAP: {
+ case NOCAP: {
st = pSMgr->suggest_morph_for_spelling_error(cw);
if (st) {
- strcat(result, st);
+ mystrcat(result, st, MAXLNLEN);
free(st);
}
- if (abbv) {
- memcpy(wspace,cw,wl);
+ if (abbv) {
+ memcpy(wspace,cw,wl);
*(wspace+wl) = '.';
*(wspace+wl+1) = '\0';
st = pSMgr->suggest_morph_for_spelling_error(wspace);
if (st) {
- if (*result) strcat(result, "\n");
- strcat(result, st);
+ if (*result) mystrcat(result, "\n", MAXLNLEN);
+ mystrcat(result, st, MAXLNLEN);
free(st);
}
}
break;
}
- case INITCAP: {
+ case INITCAP: {
memcpy(wspace,cw,(wl+1));
mkallsmall(wspace);
st = pSMgr->suggest_morph_for_spelling_error(wspace);
if (st) {
- strcat(result, st);
- free(st);
- }
- st = pSMgr->suggest_morph_for_spelling_error(cw);
- if (st) {
- if (*result) strcat(result, "\n");
- strcat(result, st);
+ mystrcat(result, st, MAXLNLEN);
free(st);
}
- if (abbv) {
- memcpy(wspace,cw,wl);
+ st = pSMgr->suggest_morph_for_spelling_error(cw);
+ if (st) {
+ if (*result) mystrcat(result, "\n", MAXLNLEN);
+ mystrcat(result, st, MAXLNLEN);
+ free(st);
+ }
+ if (abbv) {
+ memcpy(wspace,cw,wl);
*(wspace+wl) = '.';
*(wspace+wl+1) = '\0';
mkallsmall(wspace);
st = pSMgr->suggest_morph_for_spelling_error(wspace);
if (st) {
- if (*result) strcat(result, "\n");
- strcat(result, st);
+ if (*result) mystrcat(result, "\n", MAXLNLEN);
+ mystrcat(result, st, MAXLNLEN);
free(st);
- }
+ }
mkinitcap(wspace);
st = pSMgr->suggest_morph_for_spelling_error(wspace);
if (st) {
- if (*result) strcat(result, "\n");
- strcat(result, st);
+ if (*result) mystrcat(result, "\n", MAXLNLEN);
+ mystrcat(result, st, MAXLNLEN);
free(st);
- }
+ }
}
break;
}
- case HUHCAP: {
+ case HUHCAP: {
st = pSMgr->suggest_morph_for_spelling_error(cw);
if (st) {
- strcat(result, st);
+ mystrcat(result, st, MAXLNLEN);
free(st);
}
memcpy(wspace,cw,(wl+1));
mkallsmall(wspace);
st = pSMgr->suggest_morph_for_spelling_error(wspace);
if (st) {
- if (*result) strcat(result, "\n");
- strcat(result, st);
+ if (*result) mystrcat(result, "\n", MAXLNLEN);
+ mystrcat(result, st, MAXLNLEN);
free(st);
- }
+ }
break;
}
- case ALLCAP: {
+ case ALLCAP: {
memcpy(wspace,cw,(wl+1));
st = pSMgr->suggest_morph_for_spelling_error(wspace);
if (st) {
- strcat(result, st);
+ mystrcat(result, st, MAXLNLEN);
free(st);
- }
+ }
mkallsmall(wspace);
st = pSMgr->suggest_morph_for_spelling_error(wspace);
if (st) {
- if (*result) strcat(result, "\n");
- strcat(result, st);
+ if (*result) mystrcat(result, "\n", MAXLNLEN);
+ mystrcat(result, st, MAXLNLEN);
free(st);
}
- mkinitcap(wspace);
- st = pSMgr->suggest_morph_for_spelling_error(wspace);
+ mkinitcap(wspace);
+ st = pSMgr->suggest_morph_for_spelling_error(wspace);
if (st) {
- if (*result) strcat(result, "\n");
- strcat(result, st);
+ if (*result) mystrcat(result, "\n", MAXLNLEN);
+ mystrcat(result, st, MAXLNLEN);
free(st);
}
- if (abbv) {
+ if (abbv) {
memcpy(wspace,cw,(wl+1));
*(wspace+wl) = '.';
*(wspace+wl+1) = '\0';
- if (*result) strcat(result, "\n");
+ if (*result) mystrcat(result, "\n", MAXLNLEN);
st = pSMgr->suggest_morph_for_spelling_error(wspace);
if (st) {
- strcat(result, st);
- free(st);
- }
+ mystrcat(result, st, MAXLNLEN);
+ free(st);
+ }
mkallsmall(wspace);
st = pSMgr->suggest_morph_for_spelling_error(wspace);
if (st) {
- if (*result) strcat(result, "\n");
- strcat(result, st);
+ if (*result) mystrcat(result, "\n", MAXLNLEN);
+ mystrcat(result, st, MAXLNLEN);
free(st);
}
- mkinitcap(wspace);
- st = pSMgr->suggest_morph_for_spelling_error(wspace);
+ mkinitcap(wspace);
+ st = pSMgr->suggest_morph_for_spelling_error(wspace);
if (st) {
- if (*result) strcat(result, "\n");
- strcat(result, st);
+ if (*result) mystrcat(result, "\n", MAXLNLEN);
+ mystrcat(result, st, MAXLNLEN);
free(st);
}
- }
+ }
break;
}
}
- if (result) return mystrdup(result);
+ if (*result) return mystrdup(result);
return NULL;
}
-/* analyze word
- * return line count
- * XXX need a better data structure for morphological analysis */
-int Hunspell::analyze(char ***out, const char *word) {
- int n = 0;
- if (!word) return 0;
- char * m = morph(word);
- if(!m) return 0;
- if (!out)
- {
- n = line_tok(m, out);
- free(m);
- return n;
- }
-
- // without memory allocation
- /* BUG missing buffer size checking */
- int i, p;
- for(p = 0, i = 0; m[i]; i++) {
- if(m[i] == '\n' || !m[i+1]) {
- n++;
- strncpy((*out)[n++], m + p, i - p + 1);
- if (m[i] == '\n') (*out)[n++][i - p] = '\0';
- if(!m[i+1]) break;
- p = i + 1;
- }
- }
- free(m);
- return n;
-}
-
#endif // END OF HUNSPELL_EXPERIMENTAL CODE
Hunhandle *Hunspell_create(const char * affpath, const char * dpath)
{
return (Hunhandle*)(new Hunspell(affpath, dpath));
}
+Hunhandle *Hunspell_create_key(const char * affpath, const char * dpath,
+ const char * key)
+{
+ return (Hunhandle*)(new Hunspell(affpath, dpath, key));
+}
+
void Hunspell_destroy(Hunhandle *pHunspell)
