LibreOffice Module sc (master)  1
dpcache.cxx
Go to the documentation of this file.
1 /* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
2 /*
3  * This file is part of the LibreOffice project.
4  *
5  * This Source Code Form is subject to the terms of the Mozilla Public
6  * License, v. 2.0. If a copy of the MPL was not distributed with this
7  * file, You can obtain one at http://mozilla.org/MPL/2.0/.
8  *
9  * This file incorporates work covered by the following license notice:
10  *
11  * Licensed to the Apache Software Foundation (ASF) under one or more
12  * contributor license agreements. See the NOTICE file distributed
13  * with this work for additional information regarding copyright
14  * ownership. The ASF licenses this file to you under the Apache
15  * License, Version 2.0 (the "License"); you may not use this file
16  * except in compliance with the License. You may obtain a copy of
17  * the License at http://www.apache.org/licenses/LICENSE-2.0 .
18  */
19 
20 #include <dpcache.hxx>
21 
22 #include <document.hxx>
23 #include <queryentry.hxx>
24 #include <queryparam.hxx>
25 #include <dpobject.hxx>
26 #include <globstr.hrc>
27 #include <scresid.hxx>
28 #include <docoptio.hxx>
29 #include <dpitemdata.hxx>
30 #include <dputil.hxx>
31 #include <dpnumgroupinfo.hxx>
32 #include <columniterator.hxx>
33 #include <cellvalue.hxx>
34 
36 #include <rtl/math.hxx>
37 #include <unotools/charclass.hxx>
38 #include <unotools/textsearch.hxx>
41 #include <svl/numformat.hxx>
42 #include <svl/zforlist.hxx>
43 #include <o3tl/safeint.hxx>
44 #include <osl/diagnose.h>
45 
46 #if DUMP_PIVOT_TABLE
47 #include <com/sun/star/sheet/DataPilotFieldGroupBy.hpp>
48 #endif
49 
50 // TODO : Threaded pivot cache operation is disabled until we can figure out
51 // ways to make the edit engine and number formatter codes thread-safe in a
52 // proper fashion.
53 #define ENABLE_THREADED_PIVOT_CACHE 0
54 
55 #if ENABLE_THREADED_PIVOT_CACHE
56 #include <thread>
57 #include <future>
58 #include <queue>
59 #endif
60 
61 using namespace ::com::sun::star;
62 
63 using ::com::sun::star::uno::Exception;
64 
65 ScDPCache::GroupItems::GroupItems() : mnGroupType(0) {}
66 
67 ScDPCache::GroupItems::GroupItems(const ScDPNumGroupInfo& rInfo, sal_Int32 nGroupType) :
68  maInfo(rInfo), mnGroupType(nGroupType) {}
69 
70 ScDPCache::Field::Field() : mnNumFormat(0) {}
71 
73  mrDoc( rDoc ),
74  mnColumnCount ( 0 ),
75  maEmptyRows(0, rDoc.GetMaxRowCount(), true),
76  mnDataSize(-1),
77  mnRowCount(0),
79 {
80 }
81 
82 namespace {
83 
84 struct ClearObjectSource
85 {
86  void operator() (ScDPObject* p) const
87  {
88  p->ClearTableData();
89  }
90 };
91 
92 }
93 
95 {
96  // Make sure no live ScDPObject instances hold reference to this cache any
97  // more.
98  mbDisposing = true;
99  std::for_each(maRefObjects.begin(), maRefObjects.end(), ClearObjectSource());
100 }
101 
102 namespace {
103 
108 class MacroInterpretIncrementer
109 {
110 public:
111  explicit MacroInterpretIncrementer(ScDocument& rDoc) :
112  mrDoc(rDoc)
113  {
114  mrDoc.IncMacroInterpretLevel();
115  }
116  ~MacroInterpretIncrementer()
117  {
118  mrDoc.DecMacroInterpretLevel();
119  }
120 private:
121  ScDocument& mrDoc;
122 };
123 
124 rtl_uString* internString( ScDPCache::StringSetType& rPool, const OUString& rStr )
125 {
126  return rPool.insert(rStr).first->pData;
127 }
128 
129 OUString createLabelString( const ScDocument& rDoc, SCCOL nCol, const ScRefCellValue& rCell )
130 {
131  OUString aDocStr = rCell.getRawString(rDoc);
132 
133  if (aDocStr.isEmpty())
134  {
135  // Replace an empty label string with column name.
136 
137  ScAddress aColAddr(nCol, 0, 0);
138  aDocStr = ScResId(STR_COLUMN) + " " + aColAddr.Format(ScRefFlags::COL_VALID);
139  }
140  return aDocStr;
141 }
142 
143 void initFromCell(
144  ScDPCache::StringSetType& rStrPool, const ScDocument& rDoc, const ScAddress& rPos,
145  const ScRefCellValue& rCell, ScDPItemData& rData, sal_uInt32& rNumFormat)
146 {
147  OUString aDocStr = rCell.getRawString(rDoc);
148  rNumFormat = 0;
149 
150  if (rCell.hasError())
151  {
152  rData.SetErrorStringInterned(internString(rStrPool, rDoc.GetString(rPos.Col(), rPos.Row(), rPos.Tab())));
153  }
154  else if (rCell.hasNumeric())
155  {
156  double fVal = rCell.getRawValue();
157  rNumFormat = rDoc.GetNumberFormat(rPos);
158  rData.SetValue(fVal);
159  }
160  else if (!rCell.isEmpty())
161  {
162  rData.SetStringInterned(internString(rStrPool, aDocStr));
163  }
164  else
165  rData.SetEmpty();
166 }
167 
168 struct Bucket
169 {
171  SCROW mnOrderIndex;
172  SCROW mnDataIndex;
173  Bucket() :
174  mnOrderIndex(0), mnDataIndex(0) {}
175  Bucket(const ScDPItemData& rValue, SCROW nData) :
176  maValue(rValue), mnOrderIndex(0), mnDataIndex(nData) {}
177 };
178 
179 #if DEBUG_PIVOT_TABLE
180 #include <iostream>
181 using std::cout;
182 using std::endl;
183 
184 struct PrintBucket
185 {
186  void operator() (const Bucket& v) const
187  {
188  cout << "value: " << v.maValue.GetValue() << " order index: " << v.mnOrderIndex << " data index: " << v.mnDataIndex << endl;
189  }
190 };
191 
192 #endif
193 
194 struct LessByValue
195 {
196  bool operator() (const Bucket& left, const Bucket& right) const
197  {
198  return left.maValue < right.maValue;
199  }
200 };
201 
202 struct LessByOrderIndex
203 {
204  bool operator() (const Bucket& left, const Bucket& right) const
205  {
206  return left.mnOrderIndex < right.mnOrderIndex;
207  }
208 };
209 
210 struct LessByDataIndex
211 {
212  bool operator() (const Bucket& left, const Bucket& right) const
213  {
214  return left.mnDataIndex < right.mnDataIndex;
215  }
216 };
217 
218 struct EqualByOrderIndex
219 {
220  bool operator() (const Bucket& left, const Bucket& right) const
221  {
222  return left.mnOrderIndex == right.mnOrderIndex;
223  }
224 };
225 
226 class PushBackValue
227 {
229 public:
230  explicit PushBackValue(ScDPCache::ScDPItemDataVec& _items) : mrItems(_items) {}
231  void operator() (const Bucket& v)
232  {
233  mrItems.push_back(v.maValue);
234  }
235 };
236 
237 class PushBackOrderIndex
238 {
240 public:
241  explicit PushBackOrderIndex(ScDPCache::IndexArrayType& _items) : mrData(_items) {}
242  void operator() (const Bucket& v)
243  {
244  mrData.push_back(v.mnOrderIndex);
245  }
246 };
247 
248 void processBuckets(std::vector<Bucket>& aBuckets, ScDPCache::Field& rField)
249 {
250  if (aBuckets.empty())
251  return;
252 
253  // Sort by the value.
254  comphelper::parallelSort(aBuckets.begin(), aBuckets.end(), LessByValue());
255 
256  {
257  // Set order index such that unique values have identical index value.
258  SCROW nCurIndex = 0;
259  std::vector<Bucket>::iterator it = aBuckets.begin(), itEnd = aBuckets.end();
260  ScDPItemData aPrev = it->maValue;
261  it->mnOrderIndex = nCurIndex;
262  for (++it; it != itEnd; ++it)
263  {
264  if (!aPrev.IsCaseInsEqual(it->maValue))
265  ++nCurIndex;
266 
267  it->mnOrderIndex = nCurIndex;
268  aPrev = it->maValue;
269  }
270  }
271 
272  // Re-sort the bucket this time by the data index.
273  comphelper::parallelSort(aBuckets.begin(), aBuckets.end(), LessByDataIndex());
274 
275  // Copy the order index series into the field object.
276  rField.maData.reserve(aBuckets.size());
277  std::for_each(aBuckets.begin(), aBuckets.end(), PushBackOrderIndex(rField.maData));
278 
279  // Sort by the value again.
280  comphelper::parallelSort(aBuckets.begin(), aBuckets.end(), LessByOrderIndex());
281 
282  // Unique by value.
283  std::vector<Bucket>::iterator itUniqueEnd =
284  std::unique(aBuckets.begin(), aBuckets.end(), EqualByOrderIndex());
285 
286  // Copy the unique values into items.
287  std::vector<Bucket>::iterator itBeg = aBuckets.begin();
288  size_t nLen = distance(itBeg, itUniqueEnd);
289  rField.maItems.reserve(nLen);
290  std::for_each(itBeg, itUniqueEnd, PushBackValue(rField.maItems));
291 }
292 
293 struct InitColumnData
294 {
295  ScDPCache::EmptyRowsType maEmptyRows;
296  OUString maLabel;
297 
298  ScDPCache::StringSetType* mpStrPool;
299  ScDPCache::Field* mpField;
300 
301  SCCOL mnCol;
302 
303  InitColumnData(ScSheetLimits const & rSheetLimits) :
304  maEmptyRows(0, rSheetLimits.GetMaxRowCount(), true),
305  mpStrPool(nullptr),
306  mpField(nullptr),
307  mnCol(-1) {}
308 
309  void init( SCCOL nCol, ScDPCache::StringSetType* pStrPool, ScDPCache::Field* pField )
310  {
311  mpStrPool = pStrPool;
312  mpField = pField;
313  mnCol = nCol;
314  }
315 };
316 
317 struct InitDocData
318 {
319  ScDocument& mrDoc;
320  SCTAB mnDocTab;
321  SCROW mnStartRow;
322  SCROW mnEndRow;
323  bool mbTailEmptyRows;
324 
325  InitDocData(ScDocument& rDoc) :
326  mrDoc(rDoc),
327  mnDocTab(-1),
328  mnStartRow(-1),
329  mnEndRow(-1),
330  mbTailEmptyRows(false) {}
331 };
332 
333 typedef std::unordered_set<OUString> LabelSet;
334 
335 void normalizeAddLabel(const OUString& rLabel, std::vector<OUString>& rLabels, LabelSet& rExistingNames)
336 {
337  const OUString aLabelLower = ScGlobal::getCharClass().lowercase(rLabel);
338  sal_Int32 nSuffix = 1;
339  OUString aNewLabel = rLabel;
340  OUString aNewLabelLower = aLabelLower;
341  while (true)
342  {
343  if (!rExistingNames.count(aNewLabelLower))
344  {
345  // this is a unique label.
346  rLabels.push_back(aNewLabel);
347  rExistingNames.insert(aNewLabelLower);
348  break;
349  }
350 
351  // This name already exists.
352  aNewLabel = rLabel + OUString::number(++nSuffix);
353  aNewLabelLower = aLabelLower + OUString::number(nSuffix);
354  }
355 }
356 
357 std::vector<OUString> normalizeLabels(const std::vector<InitColumnData>& rColData)
358 {
359  std::vector<OUString> aLabels;
360  aLabels.reserve(rColData.size() + 1);
361 
362  LabelSet aExistingNames;
363  normalizeAddLabel(ScResId(STR_PIVOT_DATA), aLabels, aExistingNames);
364 
365  for (const InitColumnData& rCol : rColData)
366  normalizeAddLabel(rCol.maLabel, aLabels, aExistingNames);
367 
368  return aLabels;
369 }
370 
371 std::vector<OUString> normalizeLabels(const ScDPCache::DBConnector& rDB, const sal_Int32 nLabelCount)
372 {
373  std::vector<OUString> aLabels;
374  aLabels.reserve(nLabelCount + 1);
375 
376  LabelSet aExistingNames;
377  normalizeAddLabel(ScResId(STR_PIVOT_DATA), aLabels, aExistingNames);
378 
379  for (sal_Int32 nCol = 0; nCol < nLabelCount; ++nCol)
380  {
381  OUString aColTitle = rDB.getColumnLabel(nCol);
382  normalizeAddLabel(aColTitle, aLabels, aExistingNames);
383  }
384 
385  return aLabels;
386 }
387 
388 void initColumnFromDoc( InitDocData& rDocData, InitColumnData &rColData )
389 {
390  ScDPCache::Field& rField = *rColData.mpField;
391  ScDocument& rDoc = rDocData.mrDoc;
392  SCTAB nDocTab = rDocData.mnDocTab;
393  SCCOL nCol = rColData.mnCol;
394  SCROW nStartRow = rDocData.mnStartRow;
395  SCROW nEndRow = rDocData.mnEndRow;
396  bool bTailEmptyRows = rDocData.mbTailEmptyRows;
397 
398  std::optional<sc::ColumnIterator> pIter =
399  rDoc.GetColumnIterator(nDocTab, nCol, nStartRow, nEndRow);
400  assert(pIter);
401  assert(pIter->hasCell());
402 
404 
405  rColData.maLabel = createLabelString(rDoc, nCol, pIter->getCell());
406  pIter->next();
407 
408  std::vector<Bucket> aBuckets;
409  aBuckets.reserve(nEndRow-nStartRow); // skip the topmost label cell.
410 
411  // Push back all original values.
412  for (SCROW i = 0, n = nEndRow-nStartRow; i < n; ++i, pIter->next())
413  {
414  assert(pIter->hasCell());
415 
416  sal_uInt32 nNumFormat = 0;
417  ScAddress aPos(nCol, pIter->getRow(), nDocTab);
418  initFromCell(*rColData.mpStrPool, rDoc, aPos, pIter->getCell(), aData, nNumFormat);
419 
420  aBuckets.emplace_back(aData, i);
421 
422  if (!aData.IsEmpty())
423  {
424  rColData.maEmptyRows.insert_back(i, i+1, false);
425  if (nNumFormat)
426  // Only take non-default number format.
427  rField.mnNumFormat = nNumFormat;
428  }
429  }
430 
431  processBuckets(aBuckets, rField);
432 
433  if (bTailEmptyRows)
434  {
435  // If the last item is not empty, append one. Note that the items
436  // are sorted, and empty item should come last when sorted.
437  if (rField.maItems.empty() || !rField.maItems.back().IsEmpty())
438  {
439  aData.SetEmpty();
440  rField.maItems.push_back(aData);
441  }
442  }
443 }
444 
445 #if ENABLE_THREADED_PIVOT_CACHE
446 
447 class ThreadQueue
448 {
449  using FutureType = std::future<void>;
450  std::queue<FutureType> maQueue;
451  std::mutex maMutex;
452  std::condition_variable maCond;
453 
454  size_t mnMaxQueue;
455 
456 public:
457  ThreadQueue( size_t nMaxQueue ) : mnMaxQueue(nMaxQueue) {}
458 
459  void push( std::function<void()> aFunc )
460  {
461  std::unique_lock<std::mutex> lock(maMutex);
462 
463  while (maQueue.size() >= mnMaxQueue)
464  maCond.wait(lock);
465 
466  FutureType f = std::async(std::launch::async, aFunc);
467  maQueue.push(std::move(f));
468  lock.unlock();
469 
470  maCond.notify_one();
471  }
472 
473  void waitForOne()
474  {
475  std::unique_lock<std::mutex> lock(maMutex);
476 
477  while (maQueue.empty())
478  maCond.wait(lock);
479 
480  FutureType ret = std::move(maQueue.front());
481  maQueue.pop();
482  lock.unlock();
483 
484  ret.get(); // This may throw if an exception was thrown on the async thread.
485 
486  maCond.notify_one();
487  }
488 };
489 
490 class ThreadScopedGuard
491 {
492  std::thread maThread;
493 public:
494  ThreadScopedGuard(std::thread thread) : maThread(std::move(thread)) {}
495  ThreadScopedGuard(ThreadScopedGuard&& other) : maThread(std::move(other.maThread)) {}
496 
497  ThreadScopedGuard(const ThreadScopedGuard&) = delete;
498  ThreadScopedGuard& operator= (const ThreadScopedGuard&) = delete;
499 
500  ~ThreadScopedGuard()
501  {
502  maThread.join();
503  }
504 };
505 
506 #endif
507 
508 }
509 
510 void ScDPCache::InitFromDoc(ScDocument& rDoc, const ScRange& rRange)
511 {
512  Clear();
513 
514  InitDocData aDocData(rDoc);
515 
516  // Make sure the formula cells within the data range are interpreted
517  // during this call, for this method may be called from the interpretation
518  // of GETPIVOTDATA, which disables nested formula interpretation without
519  // increasing the macro level.
520  MacroInterpretIncrementer aMacroInc(rDoc);
521 
522  aDocData.mnStartRow = rRange.aStart.Row(); // start of data
523  aDocData.mnEndRow = rRange.aEnd.Row();
524 
525  // Sanity check
526  if (!GetDoc().ValidRow(aDocData.mnStartRow) || !GetDoc().ValidRow(aDocData.mnEndRow) || aDocData.mnEndRow <= aDocData.mnStartRow)
527  return;
528 
529  SCCOL nStartCol = rRange.aStart.Col();
530  SCCOL nEndCol = rRange.aEnd.Col();
531  aDocData.mnDocTab = rRange.aStart.Tab();
532 
533  mnColumnCount = nEndCol - nStartCol + 1;
534 
535  // this row count must include the trailing empty rows.
536  mnRowCount = aDocData.mnEndRow - aDocData.mnStartRow; // skip the topmost label row.
537 
538  // Skip trailing empty rows if exists.
539  SCCOL nCol1 = nStartCol, nCol2 = nEndCol;
540  SCROW nRow1 = aDocData.mnStartRow, nRow2 = aDocData.mnEndRow;
541  rDoc.ShrinkToDataArea(aDocData.mnDocTab, nCol1, nRow1, nCol2, nRow2);
542  aDocData.mbTailEmptyRows = aDocData.mnEndRow > nRow2; // Trailing empty rows exist.
543  aDocData.mnEndRow = nRow2;
544 
545  if (aDocData.mnEndRow <= aDocData.mnStartRow)
546  {
547  // Check this again since the end row position has changed. It's
548  // possible that the new end row becomes lower than the start row
549  // after the shrinkage.
550  Clear();
551  return;
552  }
553 
555  std::vector<InitColumnData> aColData(mnColumnCount, InitColumnData(rDoc.GetSheetLimits()));
556  maFields.reserve(mnColumnCount);
557  for (SCCOL i = 0; i < mnColumnCount; ++i)
558  maFields.push_back(std::make_unique<Field>());
559 
560  maLabelNames.reserve(mnColumnCount+1);
561 
562  // Ensure that none of the formula cells in the data range are dirty.
563  rDoc.EnsureFormulaCellResults(rRange);
564 
565 #if ENABLE_THREADED_PIVOT_CACHE
566  ThreadQueue aQueue(std::thread::hardware_concurrency());
567 
568  auto aFuncLaunchFieldThreads = [&]()
569  {
570  for (sal_uInt16 nCol = nStartCol; nCol <= nEndCol; ++nCol)
571  {
572  size_t nDim = nCol - nStartCol;
573  InitColumnData& rColData = aColData[nDim];
574  rColData.init(nCol, &maStringPools[nDim], maFields[nDim].get());
575 
576  auto func = [&aDocData,&rColData]()
577  {
578  initColumnFromDoc(aDocData, rColData);
579  };
580 
581  aQueue.push(std::move(func));
582  }
583  };
584 
585  {
586  // Launch a separate thread that in turn spawns async threads to populate the fields.
587  std::thread t(aFuncLaunchFieldThreads);
588  ThreadScopedGuard sg(std::move(t));
589 
590  // Wait for all the async threads to complete on the main thread.
591  for (SCCOL i = 0; i < mnColumnCount; ++i)
592  aQueue.waitForOne();
593  }
594 
595 #else
596  for (sal_uInt16 nCol = nStartCol; nCol <= nEndCol; ++nCol)
597  {
598  size_t nDim = nCol - nStartCol;
599  InitColumnData& rColData = aColData[nDim];
600  rColData.init(nCol, &maStringPools[nDim], maFields[nDim].get());
601 
602  initColumnFromDoc(aDocData, rColData);
603  }
604 #endif
605 
606  maLabelNames = normalizeLabels(aColData);
607 
608  // Merge all non-empty rows data.
609  for (const InitColumnData& rCol : aColData)
610  {
611  EmptyRowsType::const_segment_iterator it = rCol.maEmptyRows.begin_segment();
612  EmptyRowsType::const_segment_iterator ite = rCol.maEmptyRows.end_segment();
613  EmptyRowsType::const_iterator pos = maEmptyRows.begin();
614 
615  for (; it != ite; ++it)
616  {
617  if (!it->value)
618  // Non-empty segment found. Record it.
619  pos = maEmptyRows.insert(pos, it->start, it->end, false).first;
620  }
621  }
622 
623  PostInit();
624 }
625 
627 {
628  Clear();
629 
630  try
631  {
634  maFields.clear();
635  maFields.reserve(mnColumnCount);
636  for (SCCOL i = 0; i < mnColumnCount; ++i)
637  maFields.push_back(std::make_unique<Field>());
638 
639  // Get column titles and types.
640  maLabelNames = normalizeLabels(rDB, mnColumnCount);
641 
642  std::vector<Bucket> aBuckets;
643  ScDPItemData aData;
644  for (sal_Int32 nCol = 0; nCol < mnColumnCount; ++nCol)
645  {
646  if (!rDB.first())
647  continue;
648 
649  aBuckets.clear();
650  Field& rField = *maFields[nCol];
651  SCROW nRow = 0;
652  do
653  {
654  SvNumFormatType nFormatType = SvNumFormatType::UNDEFINED;
655  aData.SetEmpty();
656  rDB.getValue(nCol, aData, nFormatType);
657  aBuckets.emplace_back(aData, nRow);
658  if (!aData.IsEmpty())
659  {
660  maEmptyRows.insert_back(nRow, nRow+1, false);
661  SvNumberFormatter* pFormatter = mrDoc.GetFormatTable();
662  rField.mnNumFormat = pFormatter ? pFormatter->GetStandardFormat(nFormatType) : 0;
663  }
664 
665  ++nRow;
666  }
667  while (rDB.next());
668 
669  processBuckets(aBuckets, rField);
670  }
671 
672  rDB.finish();
673 
674  if (!maFields.empty())
675  mnRowCount = maFields[0]->maData.size();
676 
677  PostInit();
678  return true;
679  }
680  catch (const Exception&)
681  {
682  return false;
683  }
684 }
685 
686 bool ScDPCache::ValidQuery( SCROW nRow, const ScQueryParam &rParam) const
687 {
688  if (!rParam.GetEntryCount())
689  return true;
690 
691  if (!rParam.GetEntry(0).bDoQuery)
692  return true;
693 
694  bool bMatchWholeCell = mrDoc.GetDocOptions().IsMatchWholeCell();
695 
696  SCSIZE nEntryCount = rParam.GetEntryCount();
697  std::vector<bool> aPassed(nEntryCount, false);
698 
699  tools::Long nPos = -1;
700  CollatorWrapper& rCollator = ScGlobal::GetCollator(rParam.bCaseSens);
702 
703  for (size_t i = 0; i < nEntryCount && rParam.GetEntry(i).bDoQuery; ++i)
704  {
705  const ScQueryEntry& rEntry = rParam.GetEntry(i);
706  const ScQueryEntry::Item& rItem = rEntry.GetQueryItem();
707  // we can only handle one single direct query
708  // #i115431# nField in QueryParam is the sheet column, not the field within the source range
709  SCCOL nQueryCol = static_cast<SCCOL>(rEntry.nField);
710  if ( nQueryCol < rParam.nCol1 )
711  nQueryCol = rParam.nCol1;
712  if ( nQueryCol > rParam.nCol2 )
713  nQueryCol = rParam.nCol2;
714  SCCOL nSourceField = nQueryCol - rParam.nCol1;
715  SCROW nId = GetItemDataId( nSourceField, nRow, false );
716  const ScDPItemData* pCellData = GetItemDataById( nSourceField, nId );
717 
718  bool bOk = false;
719 
720  if (rEntry.GetQueryItem().meType == ScQueryEntry::ByEmpty)
721  {
722  if (rEntry.IsQueryByEmpty())
723  bOk = pCellData->IsEmpty();
724  else
725  {
726  assert(rEntry.IsQueryByNonEmpty());
727  bOk = !pCellData->IsEmpty();
728  }
729  }
730  else if (rEntry.GetQueryItem().meType != ScQueryEntry::ByString && pCellData->IsValue())
731  { // by Value
732  double nCellVal = pCellData->GetValue();
733 
734  switch (rEntry.eOp)
735  {
736  case SC_EQUAL :
737  bOk = ::rtl::math::approxEqual(nCellVal, rItem.mfVal);
738  break;
739  case SC_LESS :
740  bOk = (nCellVal < rItem.mfVal) && !::rtl::math::approxEqual(nCellVal, rItem.mfVal);
741  break;
742  case SC_GREATER :
743  bOk = (nCellVal > rItem.mfVal) && !::rtl::math::approxEqual(nCellVal, rItem.mfVal);
744  break;
745  case SC_LESS_EQUAL :
746  bOk = (nCellVal < rItem.mfVal) || ::rtl::math::approxEqual(nCellVal, rItem.mfVal);
747  break;
748  case SC_GREATER_EQUAL :
749  bOk = (nCellVal > rItem.mfVal) || ::rtl::math::approxEqual(nCellVal, rItem.mfVal);
750  break;
751  case SC_NOT_EQUAL :
752  bOk = !::rtl::math::approxEqual(nCellVal, rItem.mfVal);
753  break;
754  default:
755  bOk= false;
756  break;
757  }
758  }
759  else if ((rEntry.eOp == SC_EQUAL || rEntry.eOp == SC_NOT_EQUAL)
761  && pCellData->HasStringData() )
762  )
763  { // by String
764  OUString aCellStr = pCellData->GetString();
765 
766  bool bRealWildOrRegExp = (rParam.eSearchType != utl::SearchParam::SearchType::Normal &&
767  ((rEntry.eOp == SC_EQUAL) || (rEntry.eOp == SC_NOT_EQUAL)));
768  if (bRealWildOrRegExp)
769  {
770  sal_Int32 nStart = 0;
771  sal_Int32 nEnd = aCellStr.getLength();
772 
773  bool bMatch = rEntry.GetSearchTextPtr( rParam.eSearchType, rParam.bCaseSens, bMatchWholeCell )
774  ->SearchForward( aCellStr, &nStart, &nEnd );
775  // from 614 on, nEnd is behind the found text
776  if (bMatch && bMatchWholeCell
777  && (nStart != 0 || nEnd != aCellStr.getLength()))
778  bMatch = false; // RegExp must match entire cell string
779 
780  bOk = ((rEntry.eOp == SC_NOT_EQUAL) ? !bMatch : bMatch);
781  }
782  else
783  {
784  if (rEntry.eOp == SC_EQUAL || rEntry.eOp == SC_NOT_EQUAL)
785  {
786  if (bMatchWholeCell)
787  {
788  // TODO: Use shared string for fast equality check.
789  OUString aStr = rEntry.GetQueryItem().maString.getString();
790  bOk = rTransliteration.isEqual(aCellStr, aStr);
791  bool bHasStar = false;
792  sal_Int32 nIndex;
793  if (( nIndex = aStr.indexOf('*') ) != -1)
794  bHasStar = true;
795  if (bHasStar && (nIndex>0))
796  {
797  for (sal_Int32 j=0;(j<nIndex) && (j< aCellStr.getLength()) ; j++)
798  {
799  if (aCellStr[j] == aStr[j])
800  {
801  bOk=true;
802  }
803  else
804  {
805  bOk=false;
806  break;
807  }
808  }
809  }
810  }
811  else
812  {
813  OUString aQueryStr = rEntry.GetQueryItem().maString.getString();
814  css::uno::Sequence< sal_Int32 > xOff;
815  const LanguageType nLang = ScGlobal::oSysLocale->GetLanguageTag().getLanguageType();
816  OUString aCell = rTransliteration.transliterate(
817  aCellStr, nLang, 0, aCellStr.getLength(), &xOff);
818  OUString aQuer = rTransliteration.transliterate(
819  aQueryStr, nLang, 0, aQueryStr.getLength(), &xOff);
820  bOk = (aCell.indexOf( aQuer ) != -1);
821  }
822  if (rEntry.eOp == SC_NOT_EQUAL)
823  bOk = !bOk;
824  }
825  else
826  { // use collator here because data was probably sorted
827  sal_Int32 nCompare = rCollator.compareString(
828  aCellStr, rEntry.GetQueryItem().maString.getString());
829  switch (rEntry.eOp)
830  {
831  case SC_LESS :
832  bOk = (nCompare < 0);
833  break;
834  case SC_GREATER :
835  bOk = (nCompare > 0);
836  break;
837  case SC_LESS_EQUAL :
838  bOk = (nCompare <= 0);
839  break;
840  case SC_GREATER_EQUAL :
841  bOk = (nCompare >= 0);
842  break;
843  case SC_NOT_EQUAL:
844  OSL_FAIL("SC_NOT_EQUAL");
845  break;
846  case SC_TOPVAL:
847  case SC_BOTVAL:
848  case SC_TOPPERC:
849  case SC_BOTPERC:
850  default:
851  break;
852  }
853  }
854  }
855  }
856 
857  if (nPos == -1)
858  {
859  nPos++;
860  aPassed[nPos] = bOk;
861  }
862  else
863  {
864  if (rEntry.eConnect == SC_AND)
865  {
866  aPassed[nPos] = aPassed[nPos] && bOk;
867  }
868  else
869  {
870  nPos++;
871  aPassed[nPos] = bOk;
872  }
873  }
874  }
875 
876  for (tools::Long j=1; j <= nPos; j++)
877  aPassed[0] = aPassed[0] || aPassed[j];
878 
879  bool bRet = aPassed[0];
880  return bRet;
881 }
882 
884 {
885  return mrDoc;
886 }
887 
889 {
890  return mnColumnCount;
891 }
892 
893 bool ScDPCache::IsRowEmpty(SCROW nRow) const
894 {
895  bool bEmpty = true;
896  maEmptyRows.search_tree(nRow, bEmpty);
897  return bEmpty;
898 }
899 
901 {
902  if (nDim < 0)
903  return nullptr;
904 
905  tools::Long nSourceCount = static_cast<tools::Long>(maFields.size());
906  if (nDim < nSourceCount)
907  return maFields[nDim]->mpGroup.get();
908 
909  nDim -= nSourceCount;
910  if (nDim < static_cast<tools::Long>(maGroupFields.size()))
911  return maGroupFields[nDim].get();
912 
913  return nullptr;
914 }
915 
916 OUString ScDPCache::GetDimensionName(std::vector<OUString>::size_type nDim) const
917 {
918  OSL_ENSURE(nDim < maLabelNames.size()-1 , "ScDPTableDataCache::GetDimensionName");
919  OSL_ENSURE(maLabelNames.size() == static_cast <sal_uInt16> (mnColumnCount+1), "ScDPTableDataCache::GetDimensionName");
920 
921  if ( nDim+1 < maLabelNames.size() )
922  {
923  return maLabelNames[nDim+1];
924  }
925  else
926  return OUString();
927 }
928 
930 {
931  OSL_ENSURE(!maFields.empty(), "Cache not initialized!");
932 
933  maEmptyRows.build_tree();
934  auto it = maEmptyRows.rbegin();
935  OSL_ENSURE(it != maEmptyRows.rend(), "corrupt flat_segment_tree instance!");
936  mnDataSize = maFields[0]->maData.size();
937  ++it; // Skip the first position.
938  OSL_ENSURE(it != maEmptyRows.rend(), "buggy version of flat_segment_tree is used.");
939  if (it->second)
940  {
941  SCROW nLastNonEmpty = it->first - 1;
942  if (nLastNonEmpty+1 < mnDataSize)
943  mnDataSize = nLastNonEmpty+1;
944  }
945 }
946 
948 {
949  mnColumnCount = 0;
950  mnRowCount = 0;
951  maFields.clear();
952  maLabelNames.clear();
953  maGroupFields.clear();
954  maEmptyRows.clear();
955  maStringPools.clear();
956 }
957 
958 SCROW ScDPCache::GetItemDataId(sal_uInt16 nDim, SCROW nRow, bool bRepeatIfEmpty) const
959 {
960  OSL_ENSURE(nDim < mnColumnCount, "ScDPTableDataCache::GetItemDataId ");
961 
962  const Field& rField = *maFields[nDim];
963  if (o3tl::make_unsigned(nRow) >= rField.maData.size())
964  {
965  // nRow is in the trailing empty rows area.
966  if (bRepeatIfEmpty)
967  nRow = rField.maData.size()-1; // Move to the last non-empty row.
968  else
969  // Return the last item, which should always be empty if the
970  // initialization has skipped trailing empty rows.
971  return rField.maItems.size()-1;
972 
973  }
974  else if (bRepeatIfEmpty)
975  {
976  while (nRow > 0 && rField.maItems[rField.maData[nRow]].IsEmpty())
977  --nRow;
978  }
979 
980  return rField.maData[nRow];
981 }
982 
984 {
985  if (nDim < 0 || nId < 0)
986  return nullptr;
987 
988  size_t nSourceCount = maFields.size();
989  size_t nDimPos = static_cast<size_t>(nDim);
990  size_t nItemId = static_cast<size_t>(nId);
991  if (nDimPos < nSourceCount)
992  {
993  // source field.
994  const Field& rField = *maFields[nDimPos];
995  if (nItemId < rField.maItems.size())
996  return &rField.maItems[nItemId];
997 
998  if (!rField.mpGroup)
999  return nullptr;
1000 
1001  nItemId -= rField.maItems.size();
1002  const ScDPItemDataVec& rGI = rField.mpGroup->maItems;
1003  if (nItemId >= rGI.size())
1004  return nullptr;
1005 
1006  return &rGI[nItemId];
1007  }
1008 
1009  // Try group fields.
1010  nDimPos -= nSourceCount;
1011  if (nDimPos >= maGroupFields.size())
1012  return nullptr;
1013 
1014  const ScDPItemDataVec& rGI = maGroupFields[nDimPos]->maItems;
1015  if (nItemId >= rGI.size())
1016  return nullptr;
1017 
1018  return &rGI[nItemId];
1019 }
1020 
1022 {
1023  return maFields.size();
1024 }
1025 
1027 {
1028  return maGroupFields.size();
1029 }
1030 
1032 {
1033  return mnRowCount;
1034 }
1035 
1037 {
1038  OSL_ENSURE(mnDataSize <= GetRowCount(), "Data size should never be larger than the row count.");
1039  return mnDataSize >= 0 ? mnDataSize : 0;
1040 }
1041 
1043 {
1044  if (nDim >= maFields.size())
1045  return nullptr;
1046 
1047  return &maFields[nDim]->maData;
1048 }
1049 
1051 {
1052  OSL_ENSURE( nDim>=0 && nDim < mnColumnCount ," nDim < mnColumnCount ");
1053  return maFields.at(nDim)->maItems;
1054 }
1055 
1056 sal_uInt32 ScDPCache::GetNumberFormat( tools::Long nDim ) const
1057 {
1058  if ( nDim >= mnColumnCount )
1059  return 0;
1060 
1061  // TODO: Find a way to determine the dominant number format in presence of
1062  // multiple number formats in the same field.
1063  return maFields[nDim]->mnNumFormat;
1064 }
1065 
1067 {
1068  if (nDim >= mnColumnCount)
1069  return false;
1070 
1071  SvNumberFormatter* pFormatter = mrDoc.GetFormatTable();
1072  if (!pFormatter)
1073  return false;
1074 
1075  SvNumFormatType eType = pFormatter->GetType(maFields[nDim]->mnNumFormat);
1076  return (eType == SvNumFormatType::DATE) || (eType == SvNumFormatType::DATETIME);
1077 }
1078 
1080 {
1081  OSL_ENSURE( nDim>=0 && nDim < mnColumnCount ," ScDPTableDataCache::GetDimMemberCount : out of bound ");
1082  return maFields[nDim]->maItems.size();
1083 }
1084 
1085 SCCOL ScDPCache::GetDimensionIndex(std::u16string_view sName) const
1086 {
1087  for (size_t i = 1; i < maLabelNames.size(); ++i)
1088  {
1089  if (maLabelNames[i] == sName)
1090  return static_cast<SCCOL>(i-1);
1091  }
1092  return -1;
1093 }
1094 
1095 rtl_uString* ScDPCache::InternString( size_t nDim, const OUString& rStr )
1096 {
1097  assert(nDim < maStringPools.size());
1098  return internString(maStringPools[nDim], rStr);
1099 }
1100 
1102 {
1103  maRefObjects.insert(pObj);
1104 }
1105 
1107 {
1108  if (mbDisposing)
1109  // Object being deleted.
1110  return;
1111 
1112  maRefObjects.erase(pObj);
1113  if (maRefObjects.empty())
1115 }
1116 
1118 {
1119  return maRefObjects;
1120 }
1121 
1123 {
1124  if (nDim < 0)
1125  return -1;
1126 
1127  if (nDim < mnColumnCount)
1128  {
1129  // source field.
1130  const ScDPItemDataVec& rItems = maFields[nDim]->maItems;
1131  for (size_t i = 0, n = rItems.size(); i < n; ++i)
1132  {
1133  if (rItems[i] == rItem)
1134  return i;
1135  }
1136 
1137  if (!maFields[nDim]->mpGroup)
1138  return -1;
1139 
1140  // grouped source field.
1141  const ScDPItemDataVec& rGI = maFields[nDim]->mpGroup->maItems;
1142  for (size_t i = 0, n = rGI.size(); i < n; ++i)
1143  {
1144  if (rGI[i] == rItem)
1145  return rItems.size() + i;
1146  }
1147  return -1;
1148  }
1149 
1150  // group field.
1151  nDim -= mnColumnCount;
1152  if (o3tl::make_unsigned(nDim) < maGroupFields.size())
1153  {
1154  const ScDPItemDataVec& rGI = maGroupFields[nDim]->maItems;
1155  for (size_t i = 0, n = rGI.size(); i < n; ++i)
1156  {
1157  if (rGI[i] == rItem)
1158  return i;
1159  }
1160  }
1161 
1162  return -1;
1163 }
1164 
1165 // static
1166 sal_uInt32 ScDPCache::GetLocaleIndependentFormat( SvNumberFormatter& rFormatter, sal_uInt32 nNumFormat )
1167 {
1168  // For a date or date+time format use ISO format so it works across locales
1169  // and can be matched against string based item queries. For time use 24h
1170  // format. All others use General format, no currency, percent, ...
1171  // Use en-US locale for all.
1172  switch (rFormatter.GetType( nNumFormat))
1173  {
1174  case SvNumFormatType::DATE:
1176  case SvNumFormatType::TIME:
1177  return rFormatter.GetFormatIndex( NF_TIME_HHMMSS, LANGUAGE_ENGLISH_US);
1178  case SvNumFormatType::DATETIME:
1180  default:
1182  }
1183 }
1184 
1185 // static
1187 {
1188  return rtl::math::doubleToUString( fValue, rtl_math_StringFormat_Automatic, rtl_math_DecimalPlaces_Max, '.', true);
1189 }
1190 
1191 // static
1193  SvNumberFormatter& rFormatter, sal_uInt32 nNumFormat )
1194 {
1195  nNumFormat = GetLocaleIndependentFormat( rFormatter, nNumFormat);
1196  if ((nNumFormat % SV_COUNTRY_LANGUAGE_OFFSET) == 0)
1198 
1199  OUString aStr;
1200  const Color* pColor = nullptr;
1201  rFormatter.GetOutputString( fValue, nNumFormat, aStr, &pColor);
1202  return aStr;
1203 }
1204 
1205 OUString ScDPCache::GetFormattedString(tools::Long nDim, const ScDPItemData& rItem, bool bLocaleIndependent) const
1206 {
1207  if (nDim < 0)
1208  return rItem.GetString();
1209 
1210  ScDPItemData::Type eType = rItem.GetType();
1211  if (eType == ScDPItemData::Value)
1212  {
1213  // Format value using the stored number format.
1214  SvNumberFormatter* pFormatter = mrDoc.GetFormatTable();
1215  if (pFormatter)
1216  {
1217  sal_uInt32 nNumFormat = GetNumberFormat(nDim);
1218  if (bLocaleIndependent)
1219  return GetLocaleIndependentFormattedString( rItem.GetValue(), *pFormatter, nNumFormat);
1220 
1221  OUString aStr;
1222  const Color* pColor = nullptr;
1223  pFormatter->GetOutputString(rItem.GetValue(), nNumFormat, aStr, &pColor);
1224  return aStr;
1225  }
1226 
1227  // Last resort...
1229  }
1230 
1231  if (eType == ScDPItemData::GroupValue)
1232  {
1234  double fStart = 0.0, fEnd = 0.0;
1235  const GroupItems* p = GetGroupItems(nDim);
1236  if (p)
1237  {
1238  fStart = p->maInfo.mfStart;
1239  fEnd = p->maInfo.mfEnd;
1240  }
1242  aAttr.mnGroupType, aAttr.mnValue, mrDoc.GetFormatTable(), fStart, fEnd);
1243  }
1244 
1245  if (eType == ScDPItemData::RangeStart)
1246  {
1247  double fVal = rItem.GetValue();
1248  const GroupItems* p = GetGroupItems(nDim);
1249  if (!p)
1250  return rItem.GetString();
1251 
1253  return ScDPUtil::getNumGroupName(fVal, p->maInfo, cDecSep, mrDoc.GetFormatTable());
1254  }
1255 
1256  return rItem.GetString();
1257 }
1258 
1260 {
1261  return mrDoc.GetFormatTable();
1262 }
1263 
1265 {
1266  maGroupFields.push_back(std::make_unique<GroupItems>());
1267  return static_cast<tools::Long>(maFields.size() + maGroupFields.size() - 1);
1268 }
1269 
1270 void ScDPCache::ResetGroupItems(tools::Long nDim, const ScDPNumGroupInfo& rNumInfo, sal_Int32 nGroupType)
1271 {
1272  if (nDim < 0)
1273  return;
1274 
1275  tools::Long nSourceCount = static_cast<tools::Long>(maFields.size());
1276  if (nDim < nSourceCount)
1277  {
1278  maFields.at(nDim)->mpGroup.reset(new GroupItems(rNumInfo, nGroupType));
1279  return;
1280  }
1281 
1282  nDim -= nSourceCount;
1283  if (nDim < static_cast<tools::Long>(maGroupFields.size()))
1284  {
1285  GroupItems& rGI = *maGroupFields[nDim];
1286  rGI.maItems.clear();
1287  rGI.maInfo = rNumInfo;
1288  rGI.mnGroupType = nGroupType;
1289  }
1290 }
1291 
1293 {
1294  if (nDim < 0)
1295  return -1;
1296 
1297  tools::Long nSourceCount = static_cast<tools::Long>(maFields.size());
1298  if (nDim < nSourceCount)
1299  {
1300  GroupItems& rGI = *maFields.at(nDim)->mpGroup;
1301  rGI.maItems.push_back(rData);
1302  SCROW nId = maFields[nDim]->maItems.size() + rGI.maItems.size() - 1;
1303  return nId;
1304  }
1305 
1306  nDim -= nSourceCount;
1307  if (nDim < static_cast<tools::Long>(maGroupFields.size()))
1308  {
1309  ScDPItemDataVec& rItems = maGroupFields.at(nDim)->maItems;
1310  rItems.push_back(rData);
1311  return rItems.size()-1;
1312  }
1313 
1314  return -1;
1315 }
1316 
1317 void ScDPCache::GetGroupDimMemberIds(tools::Long nDim, std::vector<SCROW>& rIds) const
1318 {
1319  if (nDim < 0)
1320  return;
1321 
1322  tools::Long nSourceCount = static_cast<tools::Long>(maFields.size());
1323  if (nDim < nSourceCount)
1324  {
1325  if (!maFields.at(nDim)->mpGroup)
1326  return;
1327 
1328  size_t nOffset = maFields[nDim]->maItems.size();
1329  const ScDPItemDataVec& rGI = maFields[nDim]->mpGroup->maItems;
1330  for (size_t i = 0, n = rGI.size(); i < n; ++i)
1331  rIds.push_back(static_cast<SCROW>(i + nOffset));
1332 
1333  return;
1334  }
1335 
1336  nDim -= nSourceCount;
1337  if (nDim < static_cast<tools::Long>(maGroupFields.size()))
1338  {
1339  const ScDPItemDataVec& rGI = maGroupFields.at(nDim)->maItems;
1340  for (size_t i = 0, n = rGI.size(); i < n; ++i)
1341  rIds.push_back(static_cast<SCROW>(i));
1342  }
1343 }
1344 
1345 namespace {
1346 
1347 struct ClearGroupItems
1348 {
1349  void operator() (const std::unique_ptr<ScDPCache::Field>& r) const
1350  {
1351  r->mpGroup.reset();
1352  }
1353 };
1354 
1355 }
1356 
1358 {
1359  maGroupFields.clear();
1360 }
1361 
1363 {
1364  ClearGroupFields();
1365  std::for_each(maFields.begin(), maFields.end(), ClearGroupItems());
1366 }
1367 
1369 {
1370  if (nDim < 0)
1371  return nullptr;
1372 
1373  tools::Long nSourceCount = static_cast<tools::Long>(maFields.size());
1374  if (nDim < nSourceCount)
1375  {
1376  if (!maFields.at(nDim)->mpGroup)
1377  return nullptr;
1378 
1379  return &maFields[nDim]->mpGroup->maInfo;
1380  }
1381 
1382  nDim -= nSourceCount;
1383  if (nDim < static_cast<tools::Long>(maGroupFields.size()))
1384  return &maGroupFields.at(nDim)->maInfo;
1385 
1386  return nullptr;
1387 }
1388 
1390 {
1391  if (nDim < 0)
1392  return 0;
1393 
1394  tools::Long nSourceCount = static_cast<tools::Long>(maFields.size());
1395  if (nDim < nSourceCount)
1396  {
1397  if (!maFields.at(nDim)->mpGroup)
1398  return 0;
1399 
1400  return maFields[nDim]->mpGroup->mnGroupType;
1401  }
1402 
1403  nDim -= nSourceCount;
1404  if (nDim < static_cast<tools::Long>(maGroupFields.size()))
1405  return maGroupFields.at(nDim)->mnGroupType;
1406 
1407  return 0;
1408 }
1409 
1410 #if DUMP_PIVOT_TABLE
1411 
1412 namespace {
1413 
1414 void dumpItems(const ScDPCache& rCache, tools::Long nDim, const ScDPCache::ScDPItemDataVec& rItems, size_t nOffset)
1415 {
1416  for (size_t i = 0; i < rItems.size(); ++i)
1417  cout << " " << (i+nOffset) << ": " << rCache.GetFormattedString(nDim, rItems[i], false) << endl;
1418 }
1419 
1420 void dumpSourceData(const ScDPCache& rCache, tools::Long nDim, const ScDPCache::ScDPItemDataVec& rItems, const ScDPCache::IndexArrayType& rArray)
1421 {
1422  for (const auto& rIndex : rArray)
1423  cout << " '" << rCache.GetFormattedString(nDim, rItems[rIndex], false) << "'" << endl;
1424 }
1425 
1426 const char* getGroupTypeName(sal_Int32 nType)
1427 {
1428  static const char* pNames[] = {
1429  "", "years", "quarters", "months", "days", "hours", "minutes", "seconds"
1430  };
1431 
1432  switch (nType)
1433  {
1434  case sheet::DataPilotFieldGroupBy::YEARS: return pNames[1];
1435  case sheet::DataPilotFieldGroupBy::QUARTERS: return pNames[2];
1436  case sheet::DataPilotFieldGroupBy::MONTHS: return pNames[3];
1437  case sheet::DataPilotFieldGroupBy::DAYS: return pNames[4];
1438  case sheet::DataPilotFieldGroupBy::HOURS: return pNames[5];
1439  case sheet::DataPilotFieldGroupBy::MINUTES: return pNames[6];
1440  case sheet::DataPilotFieldGroupBy::SECONDS: return pNames[7];
1441  default:
1442  ;
1443  }
1444 
1445  return pNames[0];
1446 }
1447 
1448 }
1449 
1450 void ScDPCache::Dump() const
1451 {
1452  // Change these flags to fit your debugging needs.
1453  bool bDumpItems = false;
1454  bool bDumpSourceData = false;
1455 
1456  cout << "--- pivot cache dump" << endl;
1457  {
1458  size_t i = 0;
1459  for (const auto& rxField : maFields)
1460  {
1461  const Field& fld = *rxField;
1462  cout << "* source dimension: " << GetDimensionName(i) << " (ID = " << i << ")" << endl;
1463  cout << " item count: " << fld.maItems.size() << endl;
1464  if (bDumpItems)
1465  dumpItems(*this, i, fld.maItems, 0);
1466  if (fld.mpGroup)
1467  {
1468  cout << " group item count: " << fld.mpGroup->maItems.size() << endl;
1469  cout << " group type: " << getGroupTypeName(fld.mpGroup->mnGroupType) << endl;
1470  if (bDumpItems)
1471  dumpItems(*this, i, fld.mpGroup->maItems, fld.maItems.size());
1472  }
1473 
1474  if (bDumpSourceData)
1475  {
1476  cout << " source data (re-constructed):" << endl;
1477  dumpSourceData(*this, i, fld.maItems, fld.maData);
1478  }
1479 
1480  ++i;
1481  }
1482  }
1483 
1484  {
1485  size_t i = maFields.size();
1486  for (const auto& rxGroupField : maGroupFields)
1487  {
1488  const GroupItems& gi = *rxGroupField;
1489  cout << "* group dimension: (unnamed) (ID = " << i << ")" << endl;
1490  cout << " item count: " << gi.maItems.size() << endl;
1491  cout << " group type: " << getGroupTypeName(gi.mnGroupType) << endl;
1492  if (bDumpItems)
1493  dumpItems(*this, i, gi.maItems, 0);
1494  ++i;
1495  }
1496  }
1497 
1498  {
1499  struct { SCROW start; SCROW end; bool empty; } aRange;
1500  cout << "* empty rows: " << endl;
1501  mdds::flat_segment_tree<SCROW, bool>::const_iterator it = maEmptyRows.begin(), itEnd = maEmptyRows.end();
1502  if (it != itEnd)
1503  {
1504  aRange.start = it->first;
1505  aRange.empty = it->second;
1506 
1507  for (++it; it != itEnd; ++it)
1508  {
1509  aRange.end = it->first-1;
1510  cout << " rows " << aRange.start << "-" << aRange.end << ": " << (aRange.empty ? "empty" : "not-empty") << endl;
1511  aRange.start = it->first;
1512  aRange.empty = it->second;
1513  }
1514  }
1515  }
1516 
1517  cout << "---" << endl;
1518 }
1519 
1520 #endif
1521 
1522 /* vim:set shiftwidth=4 softtabstop=4 expandtab: */
void SetValue(double fVal)
Definition: dpitemdata.cxx:153
ScDPNumGroupInfo maInfo
Definition: dpcache.hxx:59
size_t GetGroupFieldCount() const
Definition: dpcache.cxx:1026
static sal_uInt32 GetLocaleIndependentFormat(SvNumberFormatter &rFormatter, sal_uInt32 nNumFormat)
Definition: dpcache.cxx:1166
sal_Int32 nIndex
OUString getString() const
virtual bool first()=0
ScDPItemDataVec maItems
Definition: dpcache.hxx:58
ScAddress aStart
Definition: address.hxx:497
bool hasError() const
Definition: cellvalue.cxx:627
OUString GetFormattedString(tools::Long nDim, const ScDPItemData &rItem, bool bLocaleIndependent) const
Definition: dpcache.cxx:1205
EmptyRowsType maEmptyRows
Definition: dpcache.hxx:127
sal_Int32 compareString(const OUString &s1, const OUString &s2) const
OUString ScResId(TranslateId aId)
Definition: scdll.cxx:90
tools::Long GetDimMemberCount(tools::Long nDim) const
Definition: dpcache.cxx:1079
sal_Int32 mnGroupType
Definition: dpcache.hxx:60
OUString GetDimensionName(std::vector< OUString >::size_type nDim) const
Definition: dpcache.cxx:916
SCROW Row() const
Definition: address.hxx:274
static OUString GetLocaleIndependentFormattedString(double fValue, SvNumberFormatter &rFormatter, sal_uInt32 nNumFormat)
Definition: dpcache.cxx:1192
#define LANGUAGE_ENGLISH_US
OUString getRawString(const ScDocument &rDoc) const
Retrieve a string value without modifying the states of any objects in the referenced document store...
Definition: cellvalue.cxx:665
std::vector< StringSetType > maStringPools
Definition: dpcache.hxx:124
const Item & GetQueryItem() const
Definition: queryentry.hxx:85
virtual bool next()=0
size_t GetFieldCount() const
Definition: dpcache.cxx:1021
sal_uInt32 GetFormatIndex(NfIndexTableOffset, LanguageType eLnge=LANGUAGE_DONTKNOW)
bool isEmpty() const
Definition: cellvalue.cxx:670
std::optional< sc::ColumnIterator > GetColumnIterator(SCTAB nTab, SCCOL nCol, SCROW nRow1, SCROW nRow2) const
Definition: document10.cxx:983
SCROW SetGroupItem(tools::Long nDim, const ScDPItemData &rData)
Definition: dpcache.cxx:1292
ScDocument & GetDoc() const
Definition: dpcache.cxx:883
SCROW GetIdByItemData(tools::Long nDim, const ScDPItemData &rItem) const
Definition: dpcache.cxx:1122
ScDPCache(const ScDPCache &)=delete
long Long
const GroupItems * GetGroupItems(tools::Long nDim) const
Definition: dpcache.cxx:900
static OUString getNumGroupName(double fValue, const ScDPNumGroupInfo &rInfo, sal_Unicode cDecSep, SvNumberFormatter *pFormatter)
Definition: dputil.cxx:260
sal_Int64 n
SC_DLLPUBLIC ScDPCollection * GetDPCollection()
Definition: documen3.cxx:365
sal_Int32 mnCol
SvNumFormatType GetType(sal_uInt32 nFIndex) const
rtl_uString * InternString(size_t nDim, const OUString &rStr)
Definition: dpcache.cxx:1095
sal_Int16 nId
static std::optional< SvtSysLocale > oSysLocale
Definition: global.hxx:529
SCCOLROW nField
Definition: queryentry.hxx:61
ScAddress aEnd
Definition: address.hxx:498
std::mutex maMutex
bool IsQueryByEmpty() const
Definition: queryentry.cxx:87
const ScDPItemDataVec & GetDimMemberValues(SCCOL nDim) const
Definition: dpcache.cxx:1050
This is very similar to ScCellValue, except that it references the original value instead of copying ...
Definition: cellvalue.hxx:103
This class represents the cached data part of the datapilot cache table implementation.
Definition: dpcache.hxx:47
void InitFromDoc(ScDocument &rDoc, const ScRange &rRange)
Definition: dpcache.cxx:510
bool IsEmpty() const
Definition: dpitemdata.cxx:317
SCROW GetItemDataId(sal_uInt16 nDim, SCROW nRow, bool bRepeatIfEmpty) const
Definition: dpcache.cxx:958
SCCOL GetDimensionIndex(std::u16string_view sName) const
Definition: dpcache.cxx:1085
SCCOL mnColumnCount
Definition: dpcache.hxx:112
std::unordered_set< OUString > StringSetType
Definition: dpcache.hxx:50
NF_NUMBER_STANDARD
sal_uInt16 sal_Unicode
static SC_DLLPUBLIC const LocaleDataWrapper & getLocaleData()
Definition: global.cxx:1015
bool IsRowEmpty(SCROW nRow) const
Definition: dpcache.cxx:893
ObjectFormatterData & mrData
static SC_DLLPUBLIC OUString getDateGroupName(sal_Int32 nDatePart, sal_Int32 nValue, SvNumberFormatter *pFormatter, double fStart, double fEnd)
Definition: dputil.cxx:104
sal_uInt32 mnNumFormat
Definition: dpcache.hxx:87
size_t SCSIZE
size_t typedef to be able to find places where code was changed from USHORT to size_t and is used to ...
Definition: address.hxx:44
size_t pos
sal_Int32 GetGroupType(tools::Long nDim) const
Return a group type identifier.
Definition: dpcache.cxx:1389
tools::Long AppendGroupField()
Definition: dpcache.cxx:1264
SCROW GetRowCount() const
Row count is the number of records plus any trailing empty rows in case the source data is sheet and ...
Definition: dpcache.cxx:1031
SC_DLLPUBLIC SCSIZE GetEntryCount() const
Definition: queryparam.cxx:110
SC_DLLPUBLIC OUString GetString(SCCOL nCol, SCROW nRow, SCTAB nTab, const ScInterpreterContext *pContext=nullptr) const
Definition: document.cxx:3536
const ScDPItemData * GetItemDataById(tools::Long nDim, SCROW nId) const
Definition: dpcache.cxx:983
SC_DLLPUBLIC const ScQueryEntry & GetEntry(SCSIZE n) const
Definition: queryparam.cxx:115
void AddReference(ScDPObject *pObj) const
Definition: dpcache.cxx:1101
SCTAB Tab() const
Definition: address.hxx:283
NF_DATE_ISO_YYYYMMDD
virtual OUString getColumnLabel(tools::Long nCol) const =0
NF_TIME_HHMMSS
void SetStringInterned(rtl_uString *pS)
Definition: dpitemdata.cxx:145
const ScDPNumGroupInfo * GetNumGroupInfo(tools::Long nDim) const
Definition: dpcache.cxx:1368
void Clear()
Definition: dpcache.cxx:947
~ScDPCache()
Definition: dpcache.cxx:94
ScDPObjectSet maRefObjects
All pivot table objects that references this cache.
Definition: dpcache.hxx:117
utl::TextSearch * GetSearchTextPtr(utl::SearchParam::SearchType eSearchType, bool bCaseSens, bool bWildMatchSel) const
creates pSearchParam and pSearchText if necessary
Definition: queryentry.cxx:194
std::unique_ptr< GroupItems > mpGroup
Optional items for grouped field.
Definition: dpcache.hxx:73
constexpr OUStringLiteral aData
bool IsMatchWholeCell() const
Definition: docoptio.hxx:55
bool SearchForward(const OUString &rStr, sal_Int32 *pStart, sal_Int32 *pEnd, css::util::SearchResult *pRes=nullptr)
double distance
DocumentType eType
std::vector< OUString > maLabelNames
Definition: dpcache.hxx:126
SC_DLLPUBLIC const ScDocOptions & GetDocOptions() const
Definition: documen3.cxx:1946
Interface for connecting to database source.
Definition: dpcache.hxx:97
const OUString & getNumDecimalSep() const
double getRawValue() const
Retrieve a numeric value without modifying the states of any objects in the referenced document store...
Definition: cellvalue.cxx:646
void ClearGroupFields()
Definition: dpcache.cxx:1357
SC_DLLPUBLIC SvNumberFormatter * GetFormatTable() const
Definition: documen2.cxx:459
SCROW mnDataSize
Definition: dpcache.hxx:128
Type GetType() const
Definition: dpitemdata.hxx:67
When assigning a string value, you can also assign an interned string whose life-cycle is managed by ...
Definition: dpitemdata.hxx:28
int i
bool IsQueryByNonEmpty() const
Definition: queryentry.cxx:109
ScDocument & mrDoc
Definition: dpcache.hxx:111
sal_Int16 SCCOL
Definition: types.hxx:21
void ResetGroupItems(tools::Long nDim, const ScDPNumGroupInfo &rNumInfo, sal_Int32 nGroupType)
Definition: dpcache.cxx:1270
static SC_DLLPUBLIC const CharClass & getCharClass()
Definition: global.cxx:1024
SCROW GetDataSize() const
Data size is the number of records without any trailing empty rows for sheet source data...
Definition: dpcache.cxx:1036
ScSheetLimits & GetSheetLimits() const
Definition: document.hxx:896
SvNumberFormatter * GetNumberFormatter() const
Definition: dpcache.cxx:1259
constexpr std::enable_if_t< std::is_signed_v< T >, std::make_unsigned_t< T > > make_unsigned(T value)
void SetErrorStringInterned(rtl_uString *pS)
Definition: dpitemdata.cxx:181
std::vector< ScDPItemData > ScDPItemDataVec
Definition: dpcache.hxx:52
tools::Long GetColumnCount() const
Definition: dpcache.cxx:888
bool isEqual(const OUString &rStr1, const OUString &rStr2) const
GroupValueAttr GetGroupValue() const
Definition: dpitemdata.cxx:355
bool ShrinkToDataArea(SCTAB nTab, SCCOL &rStartCol, SCROW &rStartRow, SCCOL &rEndCol, SCROW &rEndRow) const
Shrink a range to only include data area.
Definition: document.cxx:1035
SvNumFormatType
void ClearTableData()
Definition: dpobject.cxx:800
void ClearAllFields()
Definition: dpcache.cxx:1362
const_iterator end() const
bool hasNumeric() const
Definition: cellvalue.cxx:622
#define SV_COUNTRY_LANGUAGE_OFFSET
bool empty() const
OUString lowercase(const OUString &rStr, sal_Int32 nPos, sal_Int32 nCount) const
svl::SharedString maString
Definition: queryentry.hxx:49
virtual void finish()=0
enumrange< T >::Iterator end(enumrange< T >)
SCCOL Col() const
Definition: address.hxx:279
XPropertyListType t
GroupFieldsType maGroupFields
Definition: dpcache.hxx:123
void RemoveCache(const ScDPCache *pCache)
Only to be called from ScDPCache::RemoveReference().
Definition: dpobject.cxx:3834
const_iterator begin() const
IndexArrayType maData
Original source data represented as indices to the unique value list.
Definition: dpcache.hxx:85
const IndexArrayType * GetFieldIndexArray(size_t nDim) const
Definition: dpcache.cxx:1042
void PostInit()
Definition: dpcache.cxx:929
SC_DLLPUBLIC bool EnsureFormulaCellResults(const ScRange &rRange, bool bSkipRunning=false)
Make sure all of the formula cells in the specified range have been fully calculated.
bool mbDisposing
Definition: dpcache.hxx:131
bool InitFromDataBase(DBConnector &rDB)
Definition: dpcache.cxx:626
sal_Int32 SCROW
Definition: types.hxx:17
ScDPItemDataVec maItems
Unique values in the field, stored in ascending order.
Definition: dpcache.hxx:78
static OUString GetLocaleIndependentFormattedNumberString(double fValue)
Definition: dpcache.cxx:1186
void RemoveReference(ScDPObject *pObj) const
Definition: dpcache.cxx:1106
static SC_DLLPUBLIC CollatorWrapper & GetCollator()
case-insensitive collator
Definition: global.cxx:1055
bool IsDateDimension(tools::Long nDim) const
Definition: dpcache.cxx:1066
SvStream & endl(SvStream &rStr)
OUString transliterate(const OUString &rStr, sal_Int32 nStart, sal_Int32 nLen) const
sal_uInt32 GetStandardFormat(SvNumFormatType eType, LanguageType eLnge=LANGUAGE_DONTKNOW)
if(aStr!=aBuf) UpdateName_Impl(m_xFollowLb.get()
const ScDPObjectSet & GetAllReferences() const
Definition: dpcache.cxx:1117
sal_uInt32 GetNumberFormat(tools::Long nDim) const
Definition: dpcache.cxx:1056
bool HasStringData() const
Definition: dpitemdata.cxx:366
FieldsType maFields
Definition: dpcache.hxx:122
NF_DATETIME_ISO_YYYYMMDD_HHMMSS
void * p
QPRO_FUNC_TYPE nType
Definition: qproform.cxx:398
double GetValue() const
Definition: dpitemdata.cxx:347
void Dump() const
ScQueryConnect eConnect
Definition: queryentry.hxx:63
virtual tools::Long getColumnCount() const =0
bool IsValue() const
Definition: dpitemdata.cxx:322
FILE * init(int, char **)
std::vector< SCROW > IndexArrayType
Definition: dpcache.hxx:54
bool ValidRow(SCROW nRow, SCROW nMaxRow)
Definition: address.hxx:105
void(* f)(TrueTypeTable *)
OUString GetString() const
Definition: dpitemdata.cxx:327
virtual void getValue(tools::Long nCol, ScDPItemData &rData, SvNumFormatType &rNumType) const =0
std::vector< NoInitInt8 > maData
static SC_DLLPUBLIC::utl::TransliterationWrapper & GetTransliteration()
Definition: global.cxx:986
bool IsCaseInsEqual(const ScDPItemData &r) const
Definition: dpitemdata.cxx:187
std::pair< const_iterator, bool > insert(Value &&x)
void GetGroupDimMemberIds(tools::Long nDim, std::vector< SCROW > &rIds) const
Definition: dpcache.cxx:1317
ScQueryOp eOp
Definition: queryentry.hxx:62
void parallelSort(const RandItr aBegin, const RandItr aEnd, Compare aComp=Compare())
aStr
mdds::flat_segment_tree< SCROW, bool > EmptyRowsType
Definition: dpcache.hxx:51
std::shared_ptr< osl::Mutex > const & lock()
Each instance of this struct represents a single filtering criteria.
Definition: queryentry.hxx:33
void GetOutputString(const double &fOutNumber, sal_uInt32 nFIndex, OUString &sOutString, const Color **ppColor, bool bUseStarFormat=false)
void SetEmpty()
Definition: dpitemdata.cxx:131
utl::SearchParam::SearchType eSearchType
Definition: queryparam.hxx:43
double maValue
sal_uInt16 nPos
sal_Int16 SCTAB
Definition: types.hxx:22
size_type erase(const Value &x)
SCROW mnRowCount
Definition: dpcache.hxx:129
SC_DLLPUBLIC sal_uInt32 GetNumberFormat(SCCOL nCol, SCROW nRow, SCTAB nTab) const
Definition: document.cxx:3677
bool ValidQuery(SCROW nRow, const ScQueryParam &rQueryParam) const
Definition: dpcache.cxx:686
bool m_bDetectedRangeSegmentation false