LibreOffice Module i18npool (master)  1
xdictionary.cxx
Go to the documentation of this file.
1 /* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
2 /*
3  * This file is part of the LibreOffice project.
4  *
5  * This Source Code Form is subject to the terms of the Mozilla Public
6  * License, v. 2.0. If a copy of the MPL was not distributed with this
7  * file, You can obtain one at http://mozilla.org/MPL/2.0/.
8  *
9  * This file incorporates work covered by the following license notice:
10  *
11  * Licensed to the Apache Software Foundation (ASF) under one or more
12  * contributor license agreements. See the NOTICE file distributed
13  * with this work for additional information regarding copyright
14  * ownership. The ASF licenses this file to you under the Apache
15  * License, Version 2.0 (the "License"); you may not use this file
16  * except in compliance with the License. You may obtain a copy of
17  * the License at http://www.apache.org/licenses/LICENSE-2.0 .
18  */
19 
20 #include <config_folders.h>
21 #include <o3tl/temporary.hxx>
22 #include <osl/file.h>
23 #include <osl/module.h>
24 #include <osl/mutex.hxx>
25 #include <rtl/bootstrap.hxx>
26 #include <com/sun/star/i18n/ScriptType.hpp>
27 #include <com/sun/star/i18n/WordType.hpp>
28 #include <xdictionary.hxx>
29 #include <unicode/uchar.h>
30 #include <string.h>
31 #include <breakiteratorImpl.hxx>
32 
33 using namespace com::sun::star::i18n;
34 
35 namespace i18npool {
36 
37 #ifdef DICT_JA_ZH_IN_DATAFILE
38 
39 #elif !defined DISABLE_DYNLOADING
40 
41 extern "C" { static void thisModule() {} }
42 
43 #else
44 
45 extern "C" {
46 
47 sal_uInt8* getExistMark_ja();
48 sal_Int16* getIndex1_ja();
49 sal_Int32* getIndex2_ja();
50 sal_Int32* getLenArray_ja();
51 sal_Unicode* getDataArea_ja();
52 
53 sal_uInt8* getExistMark_zh();
54 sal_Int16* getIndex1_zh();
55 sal_Int32* getIndex2_zh();
56 sal_Int32* getLenArray_zh();
57 sal_Unicode* getDataArea_zh();
58 
59 }
60 
61 #endif
62 
63 xdictionary::xdictionary(const char *lang) :
64  japaneseWordBreak( false )
65 {
66 
67 #ifdef DICT_JA_ZH_IN_DATAFILE
68 
69  if( strcmp( lang, "ja" ) == 0 || strcmp( lang, "zh" ) == 0 )
70  {
71  OUString sUrl( "$BRAND_BASE_DIR/" LIBO_SHARE_FOLDER "/dict_" );
72  rtl::Bootstrap::expandMacros(sUrl);
73 
74  if( strcmp( lang, "ja" ) == 0 )
75  sUrl += "ja.data";
76  else if( strcmp( lang, "zh" ) == 0 )
77  sUrl += "zh.data";
78 
79  oslFileHandle aFileHandle;
80  sal_uInt64 nFileSize;
81  char *pMapping;
82  if( osl_openFile( sUrl.pData, &aFileHandle, osl_File_OpenFlag_Read ) == osl_File_E_None &&
83  osl_getFileSize( aFileHandle, &nFileSize) == osl_File_E_None &&
84  osl_mapFile( aFileHandle, (void **) &pMapping, nFileSize, 0, osl_File_MapFlag_RandomAccess ) == osl_File_E_None )
85  {
86  // We have the offsets to the parts of the file at its end, see gendict.cxx
87  sal_Int64 *pEOF = (sal_Int64*)(pMapping + nFileSize);
88 
89  data.existMark = (sal_uInt8*) (pMapping + pEOF[-1]);
90  data.index2 = (sal_Int32*) (pMapping + pEOF[-2]);
91  data.index1 = (sal_Int16*) (pMapping + pEOF[-3]);
92  data.lenArray = (sal_Int32*) (pMapping + pEOF[-4]);
93  data.dataArea = (sal_Unicode*) (pMapping + pEOF[-5]);
94  }
95  }
96 
97 #elif !defined DISABLE_DYNLOADING
98 
99  initDictionaryData( lang );
100 
101 #else
102 
103  if( strcmp( lang, "ja" ) == 0 ) {
104  data.existMark = getExistMark_ja();
105  data.index1 = getIndex1_ja();
106  data.index2 = getIndex2_ja();
107  data.lenArray = getLenArray_ja();
108  data.dataArea = getDataArea_ja();
109  }
110  else if( strcmp( lang, "zh" ) == 0 ) {
111  data.existMark = getExistMark_zh();
112  data.index1 = getIndex1_zh();
113  data.index2 = getIndex2_zh();
114  data.lenArray = getLenArray_zh();
115  data.dataArea = getDataArea_zh();
116  }
117 
118 #endif
119 
120  for (WordBreakCache & i : cache)
121  i.size = 0;
122 
123  japaneseWordBreak = false;
124 }
125 
127 {
128  for (const WordBreakCache & i : cache) {
129  if (i.size > 0) {
130  delete [] i.contents;
131  delete [] i.wordboundary;
132  }
133  }
134 }
135 
136 namespace {
137  struct datacache {
138  oslModule mhModule;
139  OString maLang;
140  xdictionarydata maData;
141  };
142 }
143 
144 #if !defined(DICT_JA_ZH_IN_DATAFILE) && !defined(DISABLE_DYNLOADING)
145 
146 void xdictionary::initDictionaryData(const char *pLang)
147 {
148  // Global cache, never released for performance
149  static std::vector< datacache > aLoadedCache;
150 
151  osl::MutexGuard aGuard( osl::Mutex::getGlobalMutex() );
152  for(const datacache & i : aLoadedCache)
153  {
154  if( i.maLang == pLang )
155  {
156  data = i.maData;
157  return;
158  }
159  }
160 
161  // otherwise add to the cache, positive or negative.
162  datacache aEntry;
163  aEntry.maLang = OString( pLang, strlen( pLang ) );
164 
165 #ifdef SAL_DLLPREFIX
166  OString sModuleName = // mostly "lib*.so" (with * == dict_zh)
167  OString::Concat(SAL_DLLPREFIX "dict_") + pLang + SAL_DLLEXTENSION;
168 #else
169  OString sModuleName = // mostly "*.dll" (with * == dict_zh)
170  OString::Concat("dict_") + pLang + SAL_DLLEXTENSION;
171 #endif
172  aEntry.mhModule = osl_loadModuleRelativeAscii( &thisModule, sModuleName.getStr(), SAL_LOADMODULE_DEFAULT );
173  if( aEntry.mhModule ) {
174  oslGenericFunction func;
175  func = osl_getAsciiFunctionSymbol( aEntry.mhModule, "getExistMark" );
176  aEntry.maData.existMark = reinterpret_cast<sal_uInt8 const * (*)()>(func)();
177  func = osl_getAsciiFunctionSymbol( aEntry.mhModule, "getIndex1" );
178  aEntry.maData.index1 = reinterpret_cast<sal_Int16 const * (*)()>(func)();
179  func = osl_getAsciiFunctionSymbol( aEntry.mhModule, "getIndex2" );
180  aEntry.maData.index2 = reinterpret_cast<sal_Int32 const * (*)()>(func)();
181  func = osl_getAsciiFunctionSymbol( aEntry.mhModule, "getLenArray" );
182  aEntry.maData.lenArray = reinterpret_cast<sal_Int32 const * (*)()>(func)();
183  func = osl_getAsciiFunctionSymbol( aEntry.mhModule, "getDataArea" );
184  aEntry.maData.dataArea = reinterpret_cast<sal_Unicode const * (*)()>(func)();
185  }
186 
187  data = aEntry.maData;
188  aLoadedCache.push_back( aEntry );
189 }
190 
191 #endif
192 
194 {
195  japaneseWordBreak = true;
196 }
197 
198 bool xdictionary::exists(const sal_uInt32 c) const
199 {
200  // 0x1FFF is the hardcoded limit in gendict for data.existMarks
201  bool exist = data.existMark && (c>>3) < 0x1FFF && (data.existMark[c>>3] & (1<<(c&0x07))) != 0;
202  if (!exist && japaneseWordBreak)
203  return BreakIteratorImpl::getScriptClass(c) == css::i18n::ScriptType::ASIAN;
204  else
205  return exist;
206 }
207 
208 sal_Int32 xdictionary::getLongestMatch(const sal_Unicode* str, sal_Int32 sLen) const
209 {
210  if ( !data.index1 ) return 0;
211 
212  sal_Int16 idx = data.index1[str[0] >> 8];
213 
214  if (idx == 0xFF) return 0;
215 
216  idx = (idx<<8) | (str[0]&0xff);
217 
218  sal_uInt32 begin = data.index2[idx], end = data.index2[idx+1];
219 
220  if (begin == 0) return 0;
221 
222  str++; sLen--; // first character is not stored in the dictionary
223  for (sal_uInt32 i = end; i > begin; i--) {
224  sal_Int32 len = data.lenArray[i] - data.lenArray[i - 1];
225  if (sLen >= len) {
226  const sal_Unicode *dstr = data.dataArea + data.lenArray[i-1];
227  sal_Int32 pos = 0;
228 
229  while (pos < len && dstr[pos] == str[pos]) { pos++; }
230 
231  if (pos == len)
232  return len + 1;
233  }
234  }
235  return 0;
236 }
237 
238 
239 /*
240  * c-tor
241  */
242 
244  contents( nullptr ),
245  wordboundary( nullptr ),
246  length( 0 ),
247  size( 0 )
248 {
249 }
250 
251 /*
252  * Compare two unicode string,
253  */
254 
255 bool WordBreakCache::equals(const sal_Unicode* str, Boundary const & boundary) const
256 {
257  // Different length, different string.
258  if (length != boundary.endPos - boundary.startPos) return false;
259 
260  for (sal_Int32 i = 0; i < length; i++)
261  if (contents[i] != str[i + boundary.startPos]) return false;
262 
263  return true;
264 }
265 
266 
267 /*
268  * Retrieve the segment containing the character at pos.
269  * @param pos : Position of the given character.
270  * @return true if CJK.
271  */
272 bool xdictionary::seekSegment(const OUString &rText, sal_Int32 pos,
273  Boundary& segBoundary)
274 {
275  sal_Int32 indexUtf16;
276 
277  if (segmentCachedString.pData != rText.pData) {
278  // Cache the passed text so we can avoid regenerating the segment if it's the same
279  // (pData is refcounted and assigning the OUString references it, which ensures that
280  // the object is the same if we get the same pointer back later)
281  segmentCachedString = rText;
282  } else {
283  // If pos is within the cached boundary, use that boundary
284  if (pos >= segmentCachedBoundary.startPos && pos <= segmentCachedBoundary.endPos) {
285  segBoundary.startPos = segmentCachedBoundary.startPos;
286  segBoundary.endPos = segmentCachedBoundary.endPos;
287  indexUtf16 = segmentCachedBoundary.startPos;
288  rText.iterateCodePoints(&indexUtf16);
289  return segmentCachedBoundary.endPos > indexUtf16;
290  }
291  }
292 
293  segBoundary.endPos = segBoundary.startPos = pos;
294 
295  indexUtf16 = pos;
296  while (indexUtf16 > 0)
297  {
298  sal_uInt32 ch = rText.iterateCodePoints(&indexUtf16, -1);
299  if (u_isWhitespace(ch) || exists(ch))
300  segBoundary.startPos = indexUtf16;
301  else
302  break;
303  }
304 
305  indexUtf16 = pos;
306  while (indexUtf16 < rText.getLength())
307  {
308  sal_uInt32 ch = rText.iterateCodePoints(&indexUtf16);
309  if (u_isWhitespace(ch) || exists(ch))
310  segBoundary.endPos = indexUtf16;
311  else
312  break;
313  }
314 
315  // Cache the calculated boundary
316  segmentCachedBoundary.startPos = segBoundary.startPos;
317  segmentCachedBoundary.endPos = segBoundary.endPos;
318 
319  indexUtf16 = segBoundary.startPos;
320  rText.iterateCodePoints(&indexUtf16);
321  return segBoundary.endPos > indexUtf16;
322 }
323 
324 #define KANJA 1
325 #define KATAKANA 2
326 #define HIRAKANA 3
327 
328 static sal_Int16 JapaneseCharType(sal_Unicode c)
329 {
330  if (0x3041 <= c && c <= 0x309e)
331  return HIRAKANA;
332  if ((0x30a1 <= c && c <= 0x30fe) || (0xff65 <= c && c <= 0xff9f))
333  return KATAKANA;
334  return KANJA;
335 }
336 
337 WordBreakCache& xdictionary::getCache(const sal_Unicode *text, Boundary const & wordBoundary)
338 {
339  WordBreakCache& rCache = cache[text[0] & 0x1f];
340 
341  if (rCache.size != 0 && rCache.equals(text, wordBoundary))
342  return rCache;
343 
344  sal_Int32 len = wordBoundary.endPos - wordBoundary.startPos;
345 
346  if (rCache.size == 0 || len > rCache.size) {
347  if (rCache.size != 0) {
348  delete [] rCache.contents;
349  delete [] rCache.wordboundary;
350  rCache.size = len;
351  }
352  else
353  rCache.size = std::max<sal_Int32>(len, DEFAULT_SIZE);
354  rCache.contents = new sal_Unicode[rCache.size + 1];
355  rCache.wordboundary = new sal_Int32[rCache.size + 2];
356  }
357  rCache.length = len;
358  memcpy(rCache.contents, text + wordBoundary.startPos, len * sizeof(sal_Unicode));
359  *(rCache.contents + len) = 0x0000;
360  // reset the wordboundary in cache
361  memset(rCache.wordboundary, '\0', sizeof(sal_Int32)*(len + 2));
362 
363  sal_Int32 i = 0; // loop variable
364  while (rCache.wordboundary[i] < rCache.length) {
365  len = 0;
366  // look the continuous white space as one word and cache it
367  while (u_isWhitespace(static_cast<sal_uInt32>(text[wordBoundary.startPos + rCache.wordboundary[i] + len])))
368  len ++;
369 
370  if (len == 0) {
371  const sal_Unicode *str = text + wordBoundary.startPos + rCache.wordboundary[i];
372  sal_Int32 slen = rCache.length - rCache.wordboundary[i];
373  sal_Int16 type = 0, count = 0;
374  for (;len == 0 && slen > 0; str++, slen--) {
375  len = getLongestMatch(str, slen);
376  if (len == 0) {
377  if (!japaneseWordBreak) {
378  len = 1;
379  } else {
380  if (count == 0)
381  type = JapaneseCharType(*str);
382  else if (type != JapaneseCharType(*str))
383  break;
384  count++;
385  }
386  }
387  }
388  if (count)
389  {
390  rCache.wordboundary[i+1] = rCache.wordboundary[i] + count;
391  i++;
392  }
393  }
394 
395  if (len) {
396  rCache.wordboundary[i+1] = rCache.wordboundary[i] + len;
397  i++;
398  }
399  }
400  rCache.wordboundary[i + 1] = rCache.length + 1;
401 
402  return rCache;
403 }
404 
405 Boundary xdictionary::previousWord(const OUString& rText, sal_Int32 anyPos, sal_Int16 wordType)
406 {
407  // looking for the first non-whitespace character from anyPos
408  sal_uInt32 ch = 0;
409  if (anyPos > 0)
410  rText.iterateCodePoints(&anyPos, -1);
411 
412  while (anyPos > 0 && u_isWhitespace(ch)) ch = rText.iterateCodePoints(&anyPos, -1);
413 
414  return getWordBoundary(rText, anyPos, wordType, true);
415 }
416 
417 Boundary xdictionary::nextWord(const OUString& rText, sal_Int32 anyPos, sal_Int16 wordType)
418 {
419  boundary = getWordBoundary(rText, anyPos, wordType, true);
420  anyPos = boundary.endPos;
421  const sal_Int32 nLen = rText.getLength();
422  if (anyPos < nLen) {
423  // looking for the first non-whitespace character from anyPos
424  sal_uInt32 ch = rText.iterateCodePoints(&anyPos);
425  while (u_isWhitespace(ch) && (anyPos < nLen)) ch=rText.iterateCodePoints(&anyPos);
426  if (anyPos > 0)
427  rText.iterateCodePoints(&anyPos, -1);
428  }
429 
430  return getWordBoundary(rText, anyPos, wordType, true);
431 }
432 
433 Boundary const & xdictionary::getWordBoundary(const OUString& rText, sal_Int32 anyPos, sal_Int16 wordType, bool bDirection)
434 {
435  const sal_Unicode *text=rText.getStr();
436  sal_Int32 len=rText.getLength();
437  if (anyPos >= len || anyPos < 0) {
438  boundary.startPos = boundary.endPos = anyPos < 0 ? 0 : len;
439  } else if (seekSegment(rText, anyPos, boundary)) { // character in dict
440  WordBreakCache& aCache = getCache(text, boundary);
441  sal_Int32 i = 0;
442 
443  while (aCache.wordboundary[i] <= anyPos - boundary.startPos) i++;
444 
445  sal_Int32 startPos = aCache.wordboundary[i - 1];
446  // if bDirection is false
447  if (!bDirection && startPos > 0 && startPos == (anyPos - boundary.startPos))
448  {
449  sal_uInt32 ch = rText.iterateCodePoints(&o3tl::temporary(sal_Int32(anyPos-1)));
450  if (u_isWhitespace(ch))
451  i--;
452  }
453 
454  boundary.endPos = boundary.startPos;
455  boundary.endPos += aCache.wordboundary[i];
456  boundary.startPos += aCache.wordboundary[i-1];
457 
458  } else {
459  boundary.startPos = anyPos;
460  if (anyPos < len) rText.iterateCodePoints(&anyPos);
461  boundary.endPos = std::min(anyPos, len);
462  }
463  if (wordType == WordType::WORD_COUNT) {
464  // skip punctuation for word count.
465  while (boundary.endPos < len)
466  {
467  sal_Int32 indexUtf16 = boundary.endPos;
468  if (u_ispunct(rText.iterateCodePoints(&indexUtf16)))
469  boundary.endPos = indexUtf16;
470  else
471  break;
472  }
473  }
474 
475  return boundary;
476 }
477 
478 }
479 
480 /* vim:set shiftwidth=4 softtabstop=4 expandtab: */
xdictionarydata maData
xdictionarydata data
Definition: xdictionary.hxx:61
static sal_Int16 getScriptClass(sal_uInt32 currentChar)
#define SAL_DLLEXTENSION
#define KATAKANA
const sal_uInt8 * existMark
Definition: xdictionary.hxx:43
const sal_Int32 * lenArray
Definition: xdictionary.hxx:46
const sal_Int16 * index1
Definition: xdictionary.hxx:44
void initDictionaryData(const char *lang)
WordBreakCache & getCache(const sal_Unicode *text, css::i18n::Boundary const &boundary)
oslModule mhModule
const sal_Unicode * dataArea
Definition: xdictionary.hxx:47
uno_Mapping * pMapping
#define SAL_DLLPREFIX
sal_uInt16 sal_Unicode
enumrange< T >::Iterator begin(enumrange< T >)
size_t pos
static sal_Int16 JapaneseCharType(sal_Unicode c)
bool exists(const sal_uInt32 u) const
exports com.sun.star. text
bool seekSegment(const OUString &rText, sal_Int32 pos, css::i18n::Boundary &boundary)
int i
#define KANJA
size
Constant values shared between i18npool and, for example, the number formatter.
css::i18n::Boundary nextWord(const OUString &rText, sal_Int32 nPos, sal_Int16 wordType)
css::i18n::Boundary boundary
Definition: xdictionary.hxx:64
css::i18n::Boundary const & getWordBoundary(const OUString &rText, sal_Int32 nPos, sal_Int16 wordType, bool bDirection)
#define DEFAULT_SIZE
Definition: xdictionary.hxx:28
enumrange< T >::Iterator end(enumrange< T >)
constexpr T & temporary(T &&x)
const sal_uInt16 idx[]
sal_Int32 getLongestMatch(const sal_Unicode *text, sal_Int32 len) const
const sal_Int32 * index2
Definition: xdictionary.hxx:45
unsigned char sal_uInt8
WordBreakCache cache[CACHE_MAX]
Definition: xdictionary.hxx:76
OUString segmentCachedString
Definition: xdictionary.hxx:77
ResultType type
#define HIRAKANA
OString maLang
css::i18n::Boundary segmentCachedBoundary
Definition: xdictionary.hxx:78
css::i18n::Boundary previousWord(const OUString &rText, sal_Int32 nPos, sal_Int16 wordType)
static void thisModule()
Definition: xdictionary.cxx:41
bool equals(const sal_Unicode *str, css::i18n::Boundary const &boundary) const