FARGOS/VISTA Object Management Environment Core  ..
FARGOS/VISTA Object Management Environment Core Table of Contents
OrderedInput.hpp
Go to the documentation of this file.
1 #ifndef _ORDERED_INPUT_HPP_
2 #define _ORDERED_INPUT_HPP_ "$Id: OrderedInput.hpp 455 2020-07-23 20:23:59Z geoff $"
4 
6 /* Copyright (C) 2015 - 2020 FARGOS Development, LLC
7  */
8 
9 #include <queue>
10 #include <vector>
11 #include <set>
12 
15 
16 /* typedef only present to create forward declaration to compensate
17  * for doxygen bugs
18  */
19 //class OrderedMultipleInputFilter;
20 
34 // subgroup
57 #ifndef FORCE_ORDERED_INPUT_BLOCK_AS_FINAL
58 
64 #define FORCE_ORDERED_INPUT_BLOCK_AS_FINAL 0
65 #endif
66 
67 #if FORCE_ORDERED_INPUT_BLOCK_AS_FINAL == 1
68 #define ORDERED_INPUT_BLOCK_FINAL final
69 #define ORDERED_INPUT_BLOCK_VIRTUAL
70 #else
71 #define ORDERED_INPUT_BLOCK_FINAL
73 #define ORDERED_INPUT_BLOCK_VIRTUAL virtual
75 #endif
76 
86 {
87 protected:
88  enum { BLOCK_REUSE_PADDING=128 };
89 public:
90 #ifdef _MSC_VER
91  typedef int64_t ssize_t; // Visual Studio lacks definition
92 #endif
93  typedef ssize_t BlockSize_t; // int32_t is probably more appropriate
94  uint64_t sortKey;
95  class OrderedInputSource *inputFile;
96  unsigned char *blockStart;
98 //protected: // blockLen currently used in mergeLogs
100  bool ownBlock;
101 
102 public:
113  OrderedInputBlock(uint64_t key, unsigned char *block, BlockSize_t len, OrderedInputSource *source, bool makeCopy=true) {
114  OME_PREFETCH(this, 1, 0); // going to modify, limited use
115  inputFile = source;
116  sortKey = key;
117  usedLen = len;
118  ownBlock = makeCopy;
119  if (makeCopy == false) {
120  blockStart = block;
121  blockLen = 0; // since we do not own the block, record available length as 0
122  } else { // we allocate a new block and copy
123  /* NOTE: no rounding up is done on the initial allocation,
124  * but reuseBlock() will attempt resize blocks in units of pages
125  */
126  OME_PREFETCH(block, 0, 0); // read-only, not used again from this location
127  unsigned char *newBlock = new unsigned char[len];
128  blockStart = newBlock;
129  blockLen = len;
130  memcpy(newBlock, block, len);
131  }
132  }
133 
146  inline void reuseBlock(uint64_t key, unsigned char *block, BlockSize_t len, OrderedInputSource *source, bool makeCopy=true) OME_ALWAYS_INLINE OME_ALWAYS_OPTIMIZE("-O3") {
147  inputFile = source;
148  sortKey = key;
149  if (makeCopy == false) { // no copy needed, just point at it
150  if (OME_EXPECT_FALSE(ownBlock == true)) { // delete prior held copy
151  delete[] blockStart;
152  ownBlock = false;
153  }
154  blockStart = block;
155  blockLen = 0;
156  usedLen = len;
157  return;
158  }
159  // must make copy
160  OME_PREFETCH(block, 0, 0); // read-only, not used again from this location
161  OME_PREFETCH(block + CACHE_LINE_LENGTH, 0, 0); // read-only, not used again from this location
162  /* NOTE: if ownBlock == false, then blockLen will be 0 */
163  if (OME_EXPECT_TRUE(len <= blockLen)) { // have enough space
164  memcpy(blockStart, block, len);
165  usedLen = len;
166  // blockLen > 0, so ownBlock was true
167  return;
168  }
169  // need more space
170  if (len <= (8192 + 4096)) { // small enough to double
171  // get most significant bit
172 #ifndef _MSC_VER
173  int b = __builtin_clz(len); // count leading zeroes
174  b = ((sizeof(len) * 8) - 1) - b;
175 #else
176  unsigned long b;
177  _BitScanReverse(&b, len);
178 #endif
179  blockLen = 2 << b;
180  } else { // just round up to 4K boundary
181  blockLen = (len & ~0xfff) + 0x1000;
182  }
183  if (OME_EXPECT_TRUE(ownBlock == true)) { // delete prior held copy
184  delete[] blockStart;
185  } else {
186  ownBlock = true;
187  }
188  unsigned char *newBlock = new unsigned char[blockLen];
189  memcpy(newBlock, block, len);
190  blockStart = newBlock;
191  usedLen = len;
192  }
193 
198  if (ownBlock) {
199  delete[] blockStart;
200  }
201  }
202 
205  inline uint64_t getSortKey() const OME_ALWAYS_INLINE {
206  return (sortKey);
207  }
208 
215  return (~static_cast<uint64_t>(0));
216  }
217 
240  /* NOTE: the test is reversed between the keys so that the priority
241  * queue returns elements in the desired order.
242  */
243  if (sortKey > arg.sortKey) {
244  return (true);
245  }
246 #if 0
247  if (sortKey < arg.sortKey) {
248  return (false);
249  }
250  /* keys match...compare file object pointer to
251  * generate a strict partial order
252  */
253  uint64_t a1 = (uint64_t) inputFile;
254  uint64_t a2 = (uint64_t) arg.inputFile;
255  if (a1 < a2) {
256  return (true);
257  }
258 #endif
259  return (false);
260  }
261 
262 }; // end class OrderedInputBlock
263 
272 class OrderedInputSource {
273 public:
274  static const struct timespec WAIT_YEAR_MAX;
275  static const struct timespec WAIT_NEVER;
276 #ifdef _MSC_VER
277  typedef int64_t ssize_t; // Visual Studio lacks definition
278 #endif
279 
288  ssize_t obtainedLen;
289  };
290 
291  virtual ~OrderedInputSource() {}
292 
314  /* NOTE: for now, maxWaitTime is passed as a pointer, since nothing
315  * yet makes use of it. This should probably evolve to being passed by
316  * value.
317  */
318  virtual OrderedInputSource::InputBlockReturn getNextInputBlock(bool forceCopy=false, const struct timespec *maxWaitTime=nullptr) = 0;
319 
327  virtual void recoverInputBlock(OrderedInputBlock *block) {
328  delete block;
329  }
330 
344  /* NOTE:
345  * Could be abstract, but only proxy objects need implement it.
346  * More appropriately, it could/should be defined only in
347  * the OrderedInputProxy subclass, but the cost of a dynamic_cast
348  * is prohibitive, so this this a virtual with a default uninteresting
349  * implementation.
350  */
351  virtual int forwardInputBlock(OrderedInputBlock *block) {
352  return (0);
353  }
354 
364  virtual const char *sourceLabel(char *outputBfr, uint_fast32_t bfrLen) const {
365  return ("AnonInputSource");
366  }
367 
370  virtual void noteEOFread() {}
371 
374  virtual void noteEOFprocessed() {}
375 
376 }; // end class OrderedInputSource
377 
401 #if __cplusplus >= 201100
402  final /* not intended to be used as a user-extensible base class */
403 #endif
404 {
405 protected:
406  typedef std::vector<OrderedInputBlock *> pQueueContainer_t;
407 
410  inline bool operator()(const OrderedInputBlock *arg1, const OrderedInputBlock *arg2) const OME_ALWAYS_INLINE OME_ALWAYS_OPTIMIZE("-O3") {
411  // dereference and evaluate
412  return (*arg1 < *arg2);
413  }
414  }; // end struct ltOrderedInputBlockPtr
415 
416  std::priority_queue<OrderedInputBlock *,pQueueContainer_t,ltOrderedInputBlockPtr> pQueue;
417  size_t pQueueSize; // pQueue.size(), but maintained separately to avoid overhead of function call
418 
419 public:
421  pQueueSize = 0;
422  }
423 
425  while (pQueue.empty() == false) { // there was some content left in the queue
426  OrderedInputBlock *block = popNextBlock();
427  OrderedInputSource *source = block->inputFile;
428  source->recoverInputBlock(block); // usually deletes it
429  }
430  }
431 
435  return (pQueueSize);
436  }
437 
449  OrderedInputBlock *retBlock = pQueue.top();
450  OME_PREFETCH(retBlock, 1, 0); // read/write, used immediately
451  pQueue.pop();
452  pQueueSize -= decrementBy;
453  return (retBlock);
454  }
455 
466  void removeNextBlock(uint_fast8_t decrementBy=1) OME_ALWAYS_INLINE OME_ALWAYS_OPTIMIZE("-O3") {
467  pQueue.pop();
468  pQueueSize -= decrementBy;
469  }
470 
477  OrderedInputBlock *retBlock = pQueue.top();
478  OME_PREFETCH(retBlock, 0, 0); // read-only, used immediately
479  return (retBlock);
480  }
481 
490  void pushBlock(OrderedInputBlock *block, uint_fast8_t incrementBy=1) OME_ALWAYS_INLINE OME_ALWAYS_OPTIMIZE("-O3") {
491  pQueue.push(block);
492  pQueueSize += incrementBy;
493  }
494 
495 }; // end class OrderedInputPriorityQueue
496 
515 class OrderedInputSourceProxy : public OrderedInputSource {
516 protected:
517  OrderedInputSource *proxyFor;
518 
519 public:
525  explicit OrderedInputSourceProxy(OrderedInputSource *interfaceFor) {
526  OME_PREFETCH(&proxyFor, 1, 3); // going to modify, want long-term
527  OME_PREFETCH(interfaceFor, 0, 3); // read-only, want long-term
528  proxyFor = interfaceFor;
529  }
530 
532  delete proxyFor;
533  }
534 
535  // block->inputFile is rewritten to be this
536  virtual OrderedInputSource::InputBlockReturn getNextInputBlock(bool forceCopy=false, const struct timespec *maxWaitTime=nullptr) VIRTUAL_OVERRIDE {
537  InputBlockReturn result = proxyFor->getNextInputBlock(forceCopy, maxWaitTime);
538  if (OME_EXPECT_TRUE(result.block != nullptr)) {
539  result.block->inputFile = this;
540  }
541  return (result);
542  }
543 
544  // block->inputFile is rewritten to be the proxyFor object
546  OME_PREFETCH(proxyFor, 0, 3); // read-only, want long-term
547  block->inputFile = proxyFor;
548  proxyFor->recoverInputBlock(block);
549  }
550 
551  // block->inputFile is rewritten to be the proxyFor object
553  OME_PREFETCH(proxyFor, 0, 3); // read-only, want long-term
554  block->inputFile = proxyFor;
555  int rc = proxyFor->forwardInputBlock(block);
556  return (rc);
557  }
558 
559  virtual const char *sourceLabel(char *outputBfr, uint_fast32_t bfrLen) const VIRTUAL_OVERRIDE {
560  return (proxyFor->sourceLabel(outputBfr, bfrLen));
561  }
562 
563  virtual void noteEOFread() VIRTUAL_OVERRIDE {
564  proxyFor->noteEOFread();
565  }
566 
568  proxyFor->noteEOFprocessed();
569  }
570 
571 }; // end class OrderedInputSourceProxy
572 
573 #pragma GCC diagnostic push
574 #pragma GCC diagnostic ignored "-Wsuggest-final-types"
575 
588 protected:
589  /* Only data from a single source is maintained by this priority queue.
590  * Consequently, an implementation that is heavily biased towards
591  * records arriving in the correct order will perform best.
592  * The number of records to be maintained simultaneously will be determined
593  * by the maxReorder parameter passed to the constructor.
594  */
596  bool sawEOF;
597 
598 public:
611  OrderedInputSourceProxyWithReordering(OrderedInputSource *interfaceFor, size_t maxReorder) :
612  OrderedInputSourceProxy(interfaceFor)
613  {
614 LOG_COMPONENT_CERR(io,trace) << "Create reorder proxy with amt=" << (uint32_t) maxReorder << LOG_ENDLINE;
615  sawEOF = false;
616  for(size_t i=0;i<=maxReorder;i+=1) {
617  InputBlockReturn result = interfaceFor->getNextInputBlock(true, &OrderedInputSource::WAIT_NEVER);
618  if (result.block == nullptr) { // EOF or error?
619  if (result.obtainedLen == 0) { // EOF
620  sawEOF = true;
621  interfaceFor->noteEOFread();
622  }
623  // stop reading from this source
624  break;
625  }
626  reorderQueue.pushBlock(result.block);
627  }
628  }
629 
630 #pragma GCC diagnostic push
631 #pragma GCC diagnostic ignored "-Wsuggest-final-methods"
633 #pragma GCC diagnostic pop
634 
637  void dropProxy() {
638  proxyFor = nullptr;
639  }
640 
646  inline size_t totalBlocksPending() const OME_ALWAYS_INLINE {
647  return (reorderQueue.getQueueLength());
648  }
649 
657  virtual OrderedInputSource::InputBlockReturn getNextInputBlock(bool forceCopy=false, const struct timespec *maxWaitTime=nullptr) VIRTUAL_OVERRIDE {
659  if (OME_EXPECT_TRUE(sawEOF == false)) {
660  result = proxyFor->getNextInputBlock(forceCopy, maxWaitTime);
661  if (OME_EXPECT_TRUE(result.block != nullptr)) {
662  reorderQueue.pushBlock(result.block);
663  } else if (OME_EXPECT_FALSE(result.obtainedLen == 0)) {
664 LOG_COMPONENT_CERR(io,trace) << "Reorder saw physical EOF pktsLeft=" << reorderQueue.getQueueLength() << LOG_ENDLINE;
665  sawEOF = true;
666  proxyFor->noteEOFread();
667  }
668  }
670 LOG_COMPONENT_CERR(io,trace) << "Reorder queue now empty" << LOG_ENDLINE;
671  result.block = nullptr;
672  result.obtainedLen = -2;
673  return (result);
674  }
676  retBlock->inputFile = this;
677  result.block = retBlock;
678  result.obtainedLen = retBlock->usedLen;
679  return (result);
680  }
681 
682 }; // end class OrderedInputSourceProxyWithReordering
683 
684 #pragma GCC diagnostic pop
685 
715 public:
716 #ifdef _MSC_VER
717  typedef int64_t ssize_t; // Visual Studio lacks definition
718 #endif
719 
720 
721 protected:
722  /* NOTE: this priority queue is expected to have a small
723  * number of elements: 1 per distinct input source.
724  * While entries obtained from a given source will
725  * tend to be in sequential order, they may frequently
726  * be before or after existing elements in the queue.
727  */
729  std::vector<OrderedInputSource *> sourceFiles;
730  std::set<OrderedInputSource *> activeInputSources;
732  uint_fast16_t activeSourceCount;
733  bool forceCopy;
735 
736 public:
737 
744  OrderedMultipleInputFilter(size_t maxReorderFromSource=0) {
745  activeSourceCount = 0;
746  maxReorderedElements = maxReorderFromSource;
747  forceCopy = (maxReorderedElements > 0) ? true : false;
748  threadingEnabled = false;
749  }
750 
752  purgeQueue(); // if anything was left, cleanup
753  }
754 
759  void setThreadingEnabled(bool enabled) {
760  threadingEnabled = enabled;
761  }
762 
765  return (threadingEnabled);
766  }
767 
768 #pragma GCC diagnostic push
769 #pragma GCC diagnostic ignored "-Wsuggest-final-methods"
770 
782  virtual void noteNextKey(uint64_t key, int64_t blockCount, uint64_t nextKeyFromAlternateSource) {}
783 #pragma GCC diagnostic pop
784 
796  /* NOTE: a default implementation could just output the block to
797  * standard out, but it would seem to have limited applicability.
798  */
799  virtual int processInputBlock(OrderedInputBlock *block) = 0;
800 
807  virtual void nowAtEndOfFile(OrderedInputSource *source) {
808  source->noteEOFread();
809  }
810 
813  inline uint_fast32_t inputSourceTotal() const OME_ALWAYS_INLINE OME_ALWAYS_OPTIMIZE("-O3") {
814  return (priorityQueue.getQueueLength());
815  }
816 
819  virtual uint_fast32_t getTotalBlocksPending() const {
820  /* TODO: would have to loop across sourceFiles and determine amount
821  * of pending work, but OrderedInputSource does not have an
822  * interface defined to report total queued blocks.
823  */
824  return (0);
825  }
826 
830  virtual uint_fast32_t getTotalBlocksInProgress() const {
831  return (0); // no background thread, so 0
832  }
833 
837  return (priorityQueue.getQueueLength() == 0);
838  }
839 
842  uint_fast32_t purgeQueue() {
843  uint_fast32_t count = 0;
844  while (priorityQueue.getQueueLength() != 0) { // there was some content left in the queue
845  count += 1;
847  OrderedInputSource *source = block->inputFile;
848  source->recoverInputBlock(block); // usually deletes it
849  }
850  // queue is now totally empty...
851  return (count);
852  }
853 
861  void addInputSource(OrderedInputSource *source, ssize_t preloadBy=-1) {
862  size_t preloadAmt = (preloadBy >= 0) ? preloadBy : maxReorderedElements;
863 LOG_COMPONENT_CERR(io,trace) << "Add inputSource preloadAmt=" << preloadAmt << LOG_ENDLINE;
864  if (preloadAmt > 0) { // use proxy capable of reordering
865  OrderedInputSourceProxyWithReordering *reorderSource = new OrderedInputSourceProxyWithReordering(source, preloadAmt);
866  size_t addedAmt = reorderSource->totalBlocksPending();
867 LOG_COMPONENT_CERR(io,trace) << "preloaded=" << addedAmt << LOG_ENDLINE;
868  if (addedAmt > 0) {
869  activeInputSources.insert(reorderSource);
870  sourceFiles.push_back(reorderSource);
871  } else {
872  reorderSource->dropProxy();
873  delete reorderSource;
874  sourceFiles.push_back(source);
875  }
876  } else {
877  sourceFiles.push_back(source);
878  }
879  }
880 
892  int preloadQueue() {
893  const uint_fast32_t n = (uint_fast32_t) sourceFiles.size();
894  int blocksLoaded = 0;
895  for(uint_fast32_t i=0;i<n;i+=1) { // for each file
896  OrderedInputSource *f = sourceFiles[i];
897  OrderedInputSource::InputBlockReturn retrievedBlock = f->getNextInputBlock(forceCopy, &OrderedInputSource::WAIT_NEVER);
898  if (OME_EXPECT_TRUE(retrievedBlock.obtainedLen > 0)) {
899  priorityQueue.pushBlock(retrievedBlock.block);
900  activeInputSources.insert(f);
901  blocksLoaded += 1;
902  } else { // EOF or error
903  if (retrievedBlock.obtainedLen == -2) { // temporary EOF
904  // invoke user exit to notify application
905  this->nowAtEndOfFile(f);
906  }
907  }
908  }
909  activeSourceCount = (uint_fast16_t) activeInputSources.size();
910  // all preloading done
911  return (blocksLoaded);
912  }
913 
949  int64_t processQueue(uint32_t *stopFlagPtr, const struct timespec *maxWaitTime=&OrderedInputSource::WAIT_YEAR_MAX) OME_ALWAYS_OPTIMIZE("-O3")
950  {
951  OME_PREFETCH(&priorityQueue, 1, 3); // going to be modified, long-lived
952  if (OME_EXPECT_FALSE(priorityQueue.getQueueLength() == 0)) { // nothing to do...
953  return (0);
954  }
955  // to get past here, there must have been at least 1 block queued
956  uint32_t fakeStopFlag = 0; // storage for stopFlag if none provided
957  uint32_t *flagPtr = (stopFlagPtr != nullptr) ? stopFlagPtr : &fakeStopFlag;
958  OME_PREFETCH(flagPtr, 0, 3); // read-only, used for a long time
959 
960  int64_t totalBlocks = 0;
961  // peek at initial block
963  uint64_t nextSortKey = block->sortKey; // local copy of block->sortKey
964  uint64_t maxTimeNS;
965  if (OME_EXPECT_FALSE(maxWaitTime == nullptr)) {
966  maxTimeNS = ~static_cast<uint64_t>(0U);
967  } else {
968  maxTimeNS = (static_cast<uint64_t>(maxWaitTime->tv_sec) * 1000000000U) +
969  maxWaitTime->tv_nsec;
970  if (maxTimeNS < (static_cast<uint64_t>(24 * 60 * 60) * 1000000000U)) { // treat relative to start of next block
971 LOG_COMPONENT_CERR(io,trace) << " Set interval of " << maxTimeNS << " to end at " << POSIXtimeInNanoseconds(0, nextSortKey + maxTimeNS) << LOG_ENDLINE;
972  maxTimeNS += nextSortKey;
973  } else { // was an absolute time
974  if (nextSortKey >= maxTimeNS) {
975  // return without doing anything
976 LOG_COMPONENT_CERR(io,trace) << "Already at end of interval " << nextSortKey << LOG_ENDLINE;
977  return (0); // do nothing
978  }
979  }
980  }
981  // from this point onwards, we are committed to at least handle block
982 #define _TRACK_RANGE_PROCESSED 1
983 #if _TRACK_RANGE_PROCESSED == 1
984 uint64_t firstSortKey = nextSortKey;
985 uint64_t lastSortKey = nextSortKey;
986 #endif
987  priorityQueue.removeNextBlock(); // already have top in block
988  const bool drainEverything = false; // TODO: currently no way to enable this feature
989 uint32_t queueModificationCount = 0; // this now counts queue modifications
990  do {
991  char label[256]; // storage for sourceLabel
992  /* NOTE: "block" is the just-popped first entry in the queue,
993  * while "secondBlock" is the next apparent entry
994  * in the queue.
995  * For extremely intentional performance reasons, only a handful
996  * of blocks are ever loaded into the priority queue: 1 + the
997  * number of reordering we want to tolerate per source.
998  */
999  OrderedInputSource *source = block->inputFile;
1000  OME_PREFETCH(source, 0, 3); // read-only, should be around for a while
1001  // peek ahead to next block in queue
1002  OrderedInputBlock *secondBlock;
1003  uint64_t secondSortKey; // local copy of secondBlock->sortKey
1005  secondBlock = priorityQueue.peekNextBlock();
1006  secondSortKey = secondBlock->sortKey;
1007  } else {
1008  secondBlock = nullptr;
1009  secondSortKey = ~static_cast<uint64_t>(0U); // max possible value
1010  }
1011  int rc = 0;
1012  do { // process what we can from this source
1013  totalBlocks += 1;
1014  // process block
1015  noteNextKey(nextSortKey, totalBlocks, secondSortKey);
1016  // invoke abstract function
1017  rc = processInputBlock(block);
1018 #if _TRACK_RANGE_PROCESSED == 1
1019  lastSortKey = nextSortKey; // last processed
1020 #endif
1021  if (OME_EXPECT_TRUE(rc <= 0)) { // recover block
1022  source->recoverInputBlock(block); // usually deletes it
1023  // block was handled, one way or the other
1024  if (OME_EXPECT_FALSE(rc < 0)) { // error or forced EOF
1025 LOG_COMPONENT_CERR(io,trace) << "Next block now null because rc=" << rc << LOG_ENDLINE;
1026  block = nullptr;
1027  break;
1028  }
1029  }
1030  // get next block from source
1031  OrderedInputSource::InputBlockReturn retrievedBlock = source->getNextInputBlock(forceCopy, maxWaitTime);
1032  block = retrievedBlock.block;
1033  if (OME_EXPECT_FALSE(block == nullptr)) { // normal EOF from this source
1034  break; // this is the exit from the main loop
1035  }
1036  nextSortKey = block->sortKey;
1037  // NOTE: if no second block, secondSortKey is ~0, so test cannot be true
1038  if (OME_EXPECT_FALSE(nextSortKey > secondSortKey)) {
1039  // was past earliest entries from other sources
1040  queueModificationCount += 1;
1041  if (OME_EXPECT_FALSE(secondSortKey >= maxTimeNS)) { // next one will be past interval
1042  LOG_COMPONENT_CERR(io,debug) <<
1043  "Second block would be past end of interval " << secondSortKey <<
1044  " queueSkips=" << totalBlocks - queueModificationCount <<
1045  " totalBlocks=" << totalBlocks <<
1046 #if _TRACK_RANGE_PROCESSED == 1
1047  " nsFromFirst=" << secondSortKey - firstSortKey <<
1048  " nsFromPrior=" << secondSortKey - lastSortKey <<
1049 #endif
1050  " nsToNext=" << secondSortKey - maxTimeNS <<
1051  " nextApparentTime=\"" << POSIXtimeInNanoseconds(0, secondSortKey) << "\"" <<
1052  LOG_ENDLINE;
1053  priorityQueue.pushBlock(block); // just save retrieved block
1054  goto cleanup_and_exit; // for performance, we cheat using a goto
1055  }
1056  /* Blocks were either out-of-order from a given source or we
1057  * need to change input sources.
1058  * We'll use "secondBlock" but need to save
1059  * retrieved block.
1060  */
1061  // This is where a bunch of push/pops are done when maxReorderedElements > 0
1062  source = secondBlock->inputFile; // shift over to new source
1063  OrderedInputBlock *newBlock = block; // save temporarily
1064  block = secondBlock; // already had pQueue.top()
1065  nextSortKey = secondSortKey;
1066  OME_PREFETCH(source, 0, 3); // read-only, should be around for a while
1067  /* NOTE: the order is intentional here to reduce minimize
1068  * number of elements in the priority queue for
1069  * performance, but requires the temporary "newBlock"
1070  * to hold a pointer; however, a simple register-to-
1071  * register copy is cheap.
1072  */
1073  priorityQueue.removeNextBlock(0); // drop first element, which we have now in block
1074  priorityQueue.pushBlock(newBlock, 0); // save for processing later
1075  secondBlock = priorityQueue.peekNextBlock(); // update after pop()/push()
1076  secondSortKey = secondBlock->sortKey;
1077  } // end if new block later than earliest from all other sources
1078  if (OME_EXPECT_FALSE(nextSortKey >= maxTimeNS)) { // beyond interval
1079  queueModificationCount += 1;
1080  LOG_COMPONENT_CERR(io,debug) <<
1081  "Reached end of interval " << nextSortKey <<
1082  " queueSkips=" << totalBlocks - queueModificationCount <<
1083  " totalBlocks=" << totalBlocks <<
1084 #if _TRACK_RANGE_PROCESSED == 1
1085  " nsFromFirst=" << nextSortKey - firstSortKey <<
1086  " nsFromPrior=" << nextSortKey - lastSortKey <<
1087 #endif
1088  " nsToNext=" << nextSortKey - maxTimeNS <<
1089  " nextApparentTime=\"" << POSIXtimeInNanoseconds(0, nextSortKey) << "\"" <<
1090  LOG_ENDLINE;
1091  priorityQueue.pushBlock(block);
1092  goto cleanup_and_exit; // for performance, we cheat using a goto
1093  } // end if beyond interval boundary
1094  } while (1);
1095  // block is now a nullptr, we have seen an EOF
1096  if (activeInputSources.erase(source) != 0) { // first time
1097  nowAtEndOfFile(source);
1098  // always stop processing this input
1099  activeSourceCount = (uint_fast16_t) activeInputSources.size();
1100  LOG_COMPONENT_CERR(io,info) << "EOF seen on source=\"" <<
1101  source->sourceLabel(label, sizeof(label)) <<
1102  "\", activeCount=" << (uint32_t) activeSourceCount <<
1103  LOG_ENDLINE;
1104  } else {
1105  LOG_COMPONENT_CERR(io,info) << "EOF on source=\"" <<
1106  source->sourceLabel(label, sizeof(label)) <<
1107  "\" that was already reaped" << LOG_ENDLINE;
1108  }
1109  if (OME_EXPECT_FALSE(rc == -2)) { // EOF being forced from source
1110  // stop processing further input from this source
1111  LOG_COMPONENT_CERR(io,warn) <<
1112  "Ceasing all input processing from source=\"" <<
1113  source->sourceLabel(label, sizeof(label)) << "\"" << LOG_ENDLINE;
1114  break;
1115  }
1116  if (priorityQueue.getQueueLength() == 0) { // no more data
1117  break;
1118  }
1119  if (maxWaitTime != nullptr) {
1120  if (secondSortKey >= maxTimeNS) { // next block to process is beyond interval
1121  LOG_COMPONENT_CERR(io,trace) <<
1122  "End of interval and reached EOF on source=\"" <<
1123  source->sourceLabel(label, sizeof(label)) << "\"" <<
1124  " totalBlocks=" << totalBlocks <<
1125  " queueSkips=" << totalBlocks - queueModificationCount <<
1126 #if _TRACK_RANGE_PROCESSED == 1
1127  " nsFromFirst=" << secondSortKey - firstSortKey <<
1128  " nsFromPrior=" << secondSortKey - lastSortKey <<
1129 #endif
1130  " nsToNext=" << secondSortKey - maxTimeNS <<
1131  " nextApparentTime=\"" << POSIXtimeInNanoseconds(0, secondSortKey) << "\"" <<
1132  LOG_ENDLINE;
1133  break;
1134  }
1135  }
1136  // normal EOF with more data remaining
1137  LOG_COMPONENT_CERR(io,info) << "Apparent EOF on source=\"" <<
1138  source->sourceLabel(label, sizeof(label)) << "\"" <<
1139  LOG_ENDLINE;
1140  LOG_COMPONENT_CERR(io,info) << "Next source=" << secondBlock->inputFile->sourceLabel(label, sizeof(label)) << LOG_ENDLINE;
1141  block = secondBlock; // get current top of queue
1142  nextSortKey = secondSortKey;
1143  OME_PREFETCH(block, 0, 0); // read-only, should be short term
1144 // reset at top of loop source = block->inputFile;
1145  priorityQueue.removeNextBlock(); // remove top entry from queue
1147  secondBlock = priorityQueue.peekNextBlock();
1148  secondSortKey = secondBlock->sortKey;
1149  } else {
1150  secondBlock = nullptr;
1151  secondSortKey = ~static_cast<uint64_t>(0U);
1152  }
1153  } while (OME_EXPECT_TRUE(*flagPtr == 0)); // loop as long as stop flag not set
1154 cleanup_and_exit:
1155  if (OME_EXPECT_FALSE(drainEverything == true)) { // currently no way to enable this
1156  purgeQueue();
1157  }
1158  LOG_COMPONENT_CERR(io,trace) <<
1159  "Queue skipCount=" << totalBlocks - queueModificationCount <<
1160  " totalBlocks=" << totalBlocks <<
1161  " pQueue=" << priorityQueue.getQueueLength() <<
1162 #if _TRACK_RANGE_PROCESSED == 1
1163  " duration=" << lastSortKey - firstSortKey <<
1164 #endif
1165  LOG_ENDLINE;
1166  return (totalBlocks);
1167 #undef _TRACK_RANGE_PROCESSED
1168  }
1169 
1194  int64_t processInputFiles(uint32_t *stopFlag, const struct timespec *maxWaitTime=&OrderedInputSource::WAIT_YEAR_MAX)
1195  {
1196  LOG_COMPONENT_CERR(io,trace) << "Do preload queue" << LOG_ENDLINE;
1197  preloadQueue();
1198 
1199  LOG_COMPONENT_CERR(io,trace) << "call processQueue with " << priorityQueue.getQueueLength() << " intial elements" << LOG_ENDLINE;
1200  int64_t result = processQueue(stopFlag, maxWaitTime);
1201  LOG_COMPONENT_CERR(io,trace) << "processQueue result=" << result << LOG_ENDLINE;
1202  return (result);
1203  }
1204 
1216  virtual int64_t processWorkInProgress(uint32_t *stopFlagPtr, const struct timespec *maxWaitTime=&OrderedInputSource::WAIT_YEAR_MAX) {
1217  return (0);
1218  }
1219 }; // end class OrderedMultipleInputFilter
1220 
1221 //typedef OrderedMultipleInputFilter *OrderedMultipleInputFilterP;
1222 
1226 #endif
1227 /* vim: set expandtab shiftwidth=4 tabstop=4: */
OMEcleanupGlobalData
void OMEcleanupGlobalData()
Definition: vista.cpp:36
OrderedInputPriorityQueue::removeNextBlock
void removeNextBlock(uint_fast8_t decrementBy=1) OME_ALWAYS_INLINE OME_ALWAYS_OPTIMIZE("-O3")
Specialized routine that does the equivalent of popNextBlock(), but does not intend to examine the bl...
Definition: OrderedInput.hpp:466
OME_EXT_TYPE
#define OME_EXT_TYPE(member)
Definition: OMEbaseType.h:19
OrderedInputPriorityQueue::getQueueLength
size_t getQueueLength() const OME_ALWAYS_INLINE OME_ALWAYS_OPTIMIZE("-O3")
Returns the number of entries in the queue.
Definition: OrderedInput.hpp:434
OrderedInputBlock::operator<
bool operator<(const OrderedInputBlock &arg) const OME_ALWAYS_INLINE OME_ALWAYS_OPTIMIZE("-O3")
Less-than comparison between OrderedInputBlock objects.
Definition: OrderedInput.hpp:239
OrderedInputSourceProxyWithReordering::OrderedInputSourceProxyWithReordering
OrderedInputSourceProxyWithReordering(OrderedInputSource *interfaceFor, size_t maxReorder)
Constructor for reordering proxy.
Definition: OrderedInput.hpp:611
OrderedMultipleInputFilter::setThreadingEnabled
void setThreadingEnabled(bool enabled)
Set threading enabled flag.
Definition: OrderedInput.hpp:759
OME_NLM
@ OME_NLM
Definition: OMEmanifests.h:90
l
Ïúíþ ð Ø ˜ ˜ __text __TEXT € __apple_names __DWARF __apple_objc __DWARF __apple_namespac__DWARF H X __apple_types __DWARF l
Definition: tmp3.o.cpp:1
operator!=
bool operator!=(const OMEtype &lArg, const OMEtype &rArg)
Definition: OMEtype_operators.cpp:5508
OrderedInputSourceProxyWithReordering::~OrderedInputSourceProxyWithReordering
virtual ~OrderedInputSourceProxyWithReordering()
Definition: OrderedInput.hpp:632
OrderedMultipleInputFilter::processWorkInProgress
virtual int64_t processWorkInProgress(uint32_t *stopFlagPtr, const struct timespec *maxWaitTime=&OrderedInputSource::WAIT_YEAR_MAX)
Interface to complete processing any background work that was previously initiated via processQueue()...
Definition: OrderedInput.hpp:1216
OME_FLOAT
@ OME_FLOAT
Definition: OMEmanifests.h:82
OrderedInputSourceProxyWithReordering::dropProxy
void dropProxy()
Disconnects proxy from its client.
Definition: OrderedInput.hpp:637
OMEtype::initializeAsType
void initializeAsType(const enum OMEtypes_t t)
The fundamental tagged data type used through the FARGOS/VISTA infrastructure.
Definition: OMEtype.cpp:95
OrderedMultipleInputFilter::sourceFiles
std::vector< OrderedInputSource * > sourceFiles
Definition: OrderedInput.hpp:729
OrderedMultipleInputFilter::processInputBlock
virtual int processInputBlock(OrderedInputBlock *block)=0
Abstract interface to process block.
OMEprofileCounter< uint32_t >
OrderedInputSourceProxyWithReordering::reorderQueue
OrderedInputPriorityQueue reorderQueue
only holds data from single source
Definition: OrderedInput.hpp:595
OrderedInputBlock::inputFile
class OrderedInputSource * inputFile
source/owner of storage block
Definition: OrderedInput.hpp:95
io_processor.hpp
FARGOS I/O Processing classes.
OMEmainLoop
int OMEmainLoop(uint_fast32_t methodLimit)
FARGOS/VISTA Object Mangement Environment main loop. Invokes OMEdoWork(). Upon return,...
Definition: OMEevent.cpp:243
OME_POINTER
@ OME_POINTER
Definition: OMEmanifests.h:92
OrderedMultipleInputFilter::priorityQueue
OrderedInputPriorityQueue priorityQueue
Definition: OrderedInput.hpp:728
OMEstartCriticalSection
void OMEstartCriticalSection(eOMEcriticalSectionLabel regionID)
Definition: OMEmutex.cpp:217
OrderedInputSourceProxyWithReordering
Class that extends OrderedInputSourceProxy with the ability to tolerate a certain number of out-of-or...
Definition: OrderedInput.hpp:587
operator/
OMEtype operator/(const OMEtype &lArg, const OMEtype &rArg)
Definition: OMEtype_operators.cpp:40694
OME_UINT16
@ OME_UINT16
Definition: OMEmanifests.h:98
OMEinitSystem
int OMEinitSystem(const char *rcFileName, int mainArgc, const char *mainArgv[], const char *envp[])
Definition: OMEinit.cpp:142
OMEruntime.h
OMEassoc
Implements associative array of OMEtype elements.
Definition: OMEassoc.h:112
OrderedMultipleInputFilter
Streaming filter that consumes input data from multiple sources and outputs blocks in sorted order.
Definition: OrderedInput.hpp:714
OMEstring
Implements text and binary string storage.
Definition: OMEstring.h:305
stderr
#define stderr
Definition: tmp.o.cpp:3115
OrderedInputSourceProxyWithReordering::getNextInputBlock
virtual OrderedInputSource::InputBlockReturn getNextInputBlock(bool forceCopy=false, const struct timespec *maxWaitTime=nullptr) VIRTUAL_OVERRIDE
Implementation of OrderedInputSource::getNextInputBlock() interface that inserts itself as a proxy be...
Definition: OrderedInput.hpp:657
OrderedInputSourceProxy::noteEOFprocessed
virtual void noteEOFprocessed() VIRTUAL_OVERRIDE
Definition: OrderedInput.hpp:567
OrderedMultipleInputFilter::isQueueEmpty
bool isQueueEmpty() const OME_ALWAYS_INLINE OME_ALWAYS_OPTIMIZE("-O3")
Return Boolean indication of queue being empty.
Definition: OrderedInput.hpp:836
operator/=
OMEtype & operator/=(OMEtype &lArg, const OMEtype &rArg)
Definition: OMEtype_operators.cpp:30582
OrderedInputBlock::BLOCK_REUSE_PADDING
@ BLOCK_REUSE_PADDING
Definition: OrderedInput.hpp:88
OrderedInputSourceProxy::getNextInputBlock
virtual OrderedInputSource::InputBlockReturn getNextInputBlock(bool forceCopy=false, const struct timespec *maxWaitTime=nullptr) VIRTUAL_OVERRIDE
Definition: OrderedInput.hpp:536
OMEtype::value
union OMEtype::@26 value
OrderedMultipleInputFilter::getThreadingEnabled
bool getThreadingEnabled() const OME_ALWAYS_INLINE
Retrieve threading enabled flag.
Definition: OrderedInput.hpp:764
OMEtype
Fundamental ANY type for FARGOS/VISTA Object Management Environment.
Definition: OMEbaseType.h:250
OrderedInputBlock::getMaximumPossibleSortKey
static CONSTEXPR uint64_t getMaximumPossibleSortKey() OME_ALWAYS_INLINE OME_CONST_FUNCTION
Return maximum possible value for a sort key.
Definition: OrderedInput.hpp:214
OME_DOUBLE
@ OME_DOUBLE
Definition: OMEmanifests.h:83
VIRTUAL_OVERRIDE
#define VIRTUAL_OVERRIDE
Generates override if the compiler supports it.
Definition: compiler_hints.h:435
OrderedInputPriorityQueue::~OrderedInputPriorityQueue
~OrderedInputPriorityQueue()
Definition: OrderedInput.hpp:424
OME_ALWAYS_OPTIMIZE
#define OME_ALWAYS_OPTIMIZE(level)
Mark a function to be compiled with a specific level of optimization.
Definition: compiler_hints.h:406
OrderedInputBlock::~OrderedInputBlock
~OrderedInputBlock()
Definition: OrderedInput.hpp:197
OrderedInputSourceProxy
Proxy class to allow state to be associated with an existing OrderedInputSource, while appearing to b...
Definition: OrderedInput.hpp:515
io
LogMaskType_t COMPONENT_LOG_MASK() io("io_logMask", &DEFAULT_sharedMemoryVariableManager, COMPONENT_LEVEL(io, warn)|COMPONENT_LEVEL(io, error)|COMPONENT_LEVEL(io, fatal))
OME_UINT32
@ OME_UINT32
Definition: OMEmanifests.h:96
OrderedInputSourceProxy::noteEOFread
virtual void noteEOFread() VIRTUAL_OVERRIDE
Definition: OrderedInput.hpp:563
ORDERED_INPUT_BLOCK_VIRTUAL
#define ORDERED_INPUT_BLOCK_VIRTUAL
Set to virtual if ORDERED_INPUT_BLOCK_AS_FINAL is set to 0.
Definition: OrderedInput.hpp:74
OMEbadType
void OMEbadType(const char *opName, const char *leftTypeName, const char *rightTypeName, uint32_t fromLine)
Definition: OMEtype_operators.cpp:11
OrderedInputBlock::ownBlock
bool ownBlock
Boolean flag indicating ownership.
Definition: OrderedInput.hpp:100
OrderedInputPriorityQueue::pQueueSize
size_t pQueueSize
Definition: OrderedInput.hpp:417
OrderedMultipleInputFilter::purgeQueue
uint_fast32_t purgeQueue()
Completely purge any pending input blocks.
Definition: OrderedInput.hpp:842
operator|
OMEtype operator|(const OMEtype &lArg, const OMEtype &rArg)
Definition: OMEtype_operators.cpp:43769
OrderedMultipleInputFilter::activeInputSources
std::set< OrderedInputSource * > activeInputSources
Definition: OrderedInput.hpp:730
OrderedInputSourceProxy::OrderedInputSourceProxy
OrderedInputSourceProxy(OrderedInputSource *interfaceFor)
Construct a proxy for an existing OrderedInputSource object.
Definition: OrderedInput.hpp:525
operator&
OMEtype operator&(const OMEtype &lArg, const OMEtype &rArg)
Definition: OMEtype_operators.cpp:42937
OrderedMultipleInputFilter::getTotalBlocksInProgress
virtual uint_fast32_t getTotalBlocksInProgress() const
Interface to return total amount of work currently being processed.
Definition: OrderedInput.hpp:830
operator-=
OMEtype & operator-=(OMEtype &lArg, const OMEtype &rArg)
Definition: OMEtype_operators.cpp:24618
OME_PREFETCH
#define OME_PREFETCH(addr, rw, locality)
Macro to request prefetch.
Definition: compiler_hints.h:362
OrderedInputBlock::blockLen
BlockSize_t blockLen
size of storage block
Definition: OrderedInput.hpp:99
OrderedMultipleInputFilter::inputSourceTotal
uint_fast32_t inputSourceTotal() const OME_ALWAYS_INLINE OME_ALWAYS_OPTIMIZE("-O3")
Return total number of registered input sources.
Definition: OrderedInput.hpp:813
makeUnique
int makeUnique(OMEthread *thread, OMEtype &result, const OMEtype &arg)
Definition: OILdebug.cpp:357
OMEnlm
Public interface to an OME Native Language Message.
Definition: OMEnlm.h:98
OrderedInput.hpp
Implements high-performance ordered input from multiple data streams.
OrderedMultipleInputFilter::threadingEnabled
bool threadingEnabled
Definition: OrderedInput.hpp:734
OME_MAX_CPUS_PERMITTED
#define OME_MAX_CPUS_PERMITTED
Definition: OMEmutex.h:79
srcID
const char srcID[]
Definition: catSym.c:17
OME_STRING
@ OME_STRING
Definition: OMEmanifests.h:85
OrderedInputPriorityQueue::pQueueContainer_t
std::vector< OrderedInputBlock * > pQueueContainer_t
Definition: OrderedInput.hpp:406
OrderedMultipleInputFilter::activeSourceCount
uint_fast16_t activeSourceCount
Definition: OrderedInput.hpp:732
OrderedMultipleInputFilter::addInputSource
void addInputSource(OrderedInputSource *source, ssize_t preloadBy=-1)
Register a new input source.
Definition: OrderedInput.hpp:861
OrderedInputBlock::reuseBlock
void reuseBlock(uint64_t key, unsigned char *block, BlockSize_t len, OrderedInputSource *source, bool makeCopy=true) OME_ALWAYS_INLINE OME_ALWAYS_OPTIMIZE("-O3")
Reinitialize an existing block. This routine is nearly identical to the OrderedInputBlock constructor...
Definition: OrderedInput.hpp:146
OrderedInputPriorityQueue::ltOrderedInputBlockPtr
Less-than operator for OrderedInputBlock pointers.
Definition: OrderedInput.hpp:409
OrderedInputSource::InputBlockReturn::obtainedLen
ssize_t obtainedLen
length available, note that this is signed
Definition: OrderedInput.hpp:288
CACHE_LINE_LENGTH
#define CACHE_LINE_LENGTH
Definition for target system's cache line length.
Definition: compiler_hints.h:576
OrderedInputPriorityQueue
Encapsulation class for a priority queue.
Definition: OrderedInput.hpp:400
OrderedInputPriorityQueue::pushBlock
void pushBlock(OrderedInputBlock *block, uint_fast8_t incrementBy=1) OME_ALWAYS_INLINE OME_ALWAYS_OPTIMIZE("-O3")
Add a new entry to the queue.
Definition: OrderedInput.hpp:490
OMEtype::pointer
OMEtype * pointer
Definition: OMEbaseType.h:302
OrderedInputPriorityQueue::ltOrderedInputBlockPtr::operator()
bool operator()(const OrderedInputBlock *arg1, const OrderedInputBlock *arg2) const OME_ALWAYS_INLINE OME_ALWAYS_OPTIMIZE("-O3")
Definition: OrderedInput.hpp:410
OrderedMultipleInputFilter::getTotalBlocksPending
virtual uint_fast32_t getTotalBlocksPending() const
Interface to return total amount of pending work.
Definition: OrderedInput.hpp:819
OME_NIL
@ OME_NIL
Definition: OMEmanifests.h:78
OME_EXPECT_TRUE
#define OME_EXPECT_TRUE(expr)
Annotation macro for conditional expression expected to be true.
Definition: compiler_hints.h:541
OrderedMultipleInputFilter::OrderedMultipleInputFilter
OrderedMultipleInputFilter(size_t maxReorderFromSource=0)
Create an OrderedMultipleInputFilter.
Definition: OrderedInput.hpp:744
OrderedInputBlock
Input block record for use with OrderedMultipleInputFilter.
Definition: OrderedInput.hpp:82
operator%=
OMEtype & operator%=(OMEtype &lArg, const OMEtype &rArg)
Definition: OMEtype_operators.cpp:31998
OrderedInputPriorityQueue::peekNextBlock
OrderedInputBlock * peekNextBlock() const OME_ALWAYS_INLINE OME_ALWAYS_OPTIMIZE("-O3") NONNULL_RETURN
Similar to popNextBlock() but does not remove the entry from the queue.
Definition: OrderedInput.hpp:476
ORDERED_INPUT_BLOCK_FINAL
#define ORDERED_INPUT_BLOCK_FINAL
Set to final if ORDERED_INPUT_BLOCK_AS_FINAL is set to 1.
Definition: OrderedInput.hpp:72
OrderedMultipleInputFilter::nowAtEndOfFile
virtual void nowAtEndOfFile(OrderedInputSource *source)
User-exit called when end-of-file is detected on an input source.
Definition: OrderedInput.hpp:807
operator*=
OMEtype & operator*=(OMEtype &lArg, const OMEtype &rArg)
Definition: OMEtype_operators.cpp:29166
OrderedInputPriorityQueue::popNextBlock
OrderedInputBlock * popNextBlock(uint_fast8_t decrementBy=1) OME_ALWAYS_INLINE OME_ALWAYS_OPTIMIZE("-O3") NONNULL_RETURN
Removes the first element from the queue and returns it.
Definition: OrderedInput.hpp:448
OME_UINT64
@ OME_UINT64
Definition: OMEmanifests.h:97
OrderedInputSourceProxy::proxyFor
OrderedInputSource * proxyFor
points at the original input source
Definition: OrderedInput.hpp:517
OMEassoc::ASSOC_HASH_KEY_t
OMEassocStorage::ASSOC_HASH_KEY_t ASSOC_HASH_KEY_t
Definition: OMEassoc.h:125
OrderedMultipleInputFilter::forceCopy
bool forceCopy
Definition: OrderedInput.hpp:733
OMEtype.h
OME fundamental type implementation.
OrderedInputPriorityQueue::OrderedInputPriorityQueue
OrderedInputPriorityQueue()
Definition: OrderedInput.hpp:420
OrderedInputSourceProxy::sourceLabel
virtual const char * sourceLabel(char *outputBfr, uint_fast32_t bfrLen) const VIRTUAL_OVERRIDE
Definition: OrderedInput.hpp:559
OrderedInputPriorityQueue::pQueue
std::priority_queue< OrderedInputBlock *, pQueueContainer_t, ltOrderedInputBlockPtr > pQueue
Definition: OrderedInput.hpp:416
__cplusplus
#define __cplusplus
Definition: tmp.o.cpp:2998
operator&=
OMEtype & operator&=(OMEtype &lArg, const OMEtype &rArg)
Definition: OMEtype_operators.cpp:27148
OrderedMultipleInputFilter::processInputFiles
int64_t processInputFiles(uint32_t *stopFlag, const struct timespec *maxWaitTime=&OrderedInputSource::WAIT_YEAR_MAX)
Process data from the registered collection of OrderedInputSource objects. This is a convenience cove...
Definition: OrderedInput.hpp:1194
OrderedInputBlock::blockStart
unsigned char * blockStart
base of storage block
Definition: OrderedInput.hpp:96
OrderedInputBlock::usedLen
BlockSize_t usedLen
actively used amount within storage block
Definition: OrderedInput.hpp:97
OME_USED
const char srcID[] OME_USED
Definition: tick_time.cpp:24
MAX_TEMPORARIES
#define MAX_TEMPORARIES
Definition: OMEtypeOps.cpp:22
operator+
OMEtype operator+(const OMEtype &lArg, const OMEtype &rArg)
Definition: OMEtype_operators.cpp:33691
operator-
OMEtype operator-(const OMEtype &lArg, const OMEtype &rArg)
Definition: OMEtype_operators.cpp:36699
OME_CONST_FUNCTION
#define OME_CONST_FUNCTION
Mark as an idempotent function that only accesses arguments – no global data.
Definition: compiler_hints.h:390
OME_SET
@ OME_SET
Definition: OMEmanifests.h:89
OrderedInputSourceProxyWithReordering::totalBlocksPending
size_t totalBlocksPending() const OME_ALWAYS_INLINE
Returns total blocks already pending.
Definition: OrderedInput.hpp:646
OME_CRITICAL_SECTION_OMETYPE_OP
@ OME_CRITICAL_SECTION_OMETYPE_OP
Definition: OMEmutex.h:45
OrderedInputBlock::getSortKey
uint64_t getSortKey() const OME_ALWAYS_INLINE
Get sort key for block.
Definition: OrderedInput.hpp:205
OrderedMultipleInputFilter::noteNextKey
virtual void noteNextKey(uint64_t key, int64_t blockCount, uint64_t nextKeyFromAlternateSource)
User exit to note next input sort key.
Definition: OrderedInput.hpp:782
OrderedInputSourceProxy::recoverInputBlock
virtual void recoverInputBlock(OrderedInputBlock *block) VIRTUAL_OVERRIDE
Definition: OrderedInput.hpp:545
OMEendCriticalSection
void OMEendCriticalSection(eOMEcriticalSectionLabel regionID)
Definition: OMEmutex.cpp:236
operator^=
OMEtype & operator^=(OMEtype &lArg, const OMEtype &rArg)
Definition: OMEtype_operators.cpp:32844
OrderedInputSource::InputBlockReturn
Return result from getNextInputBlock().
Definition: OrderedInput.hpp:286
OME_EXPECT_FALSE
#define OME_EXPECT_FALSE(expr)
Annotation macro for conditional expression expected to be false.
Definition: compiler_hints.h:540
LOG_COMPONENT_CERR
#define LOG_COMPONENT_CERR(component, lvl)
Convenience macro that uses LOG_COMPONENT_INTO to conditionally log a message to standard error.
Definition: logging_api.hpp:3030
OME_ALWAYS_INLINE
#define OME_ALWAYS_INLINE
Tell the compiler to alway inline a function, regardless of optimization level.
Definition: compiler_hints.h:364
OrderedMultipleInputFilter::preloadQueue
int preloadQueue()
Preload the priority queue with the initial set of blocks. Only the first block from each source is l...
Definition: OrderedInput.hpp:892
operator+=
OMEtype & operator+=(OMEtype &lArg, const OMEtype &rArg)
Definition: OMEtype_operators.cpp:21683
OME_UINT8
@ OME_UINT8
Definition: OMEmanifests.h:99
OME_FIXED
@ OME_FIXED
Definition: OMEmanifests.h:91
PROCESS_COMMANDLINE_LOG_FLAGS
#define PROCESS_COMMANDLINE_LOG_FLAGS(argc, argv)
Standard mechanism to process logging-related command line arguments.
Definition: logging_api.hpp:2595
OMEdebugInfo.h
OME debug and profiling interfaces.
OMEarray
Implements sparse array of OMEtype elements.
Definition: OMEarray.h:75
OrderedInputBlock::sortKey
uint64_t sortKey
sort key for priority queue, typically represents nanoseconds
Definition: OrderedInput.hpp:94
LOG_ENDLINE
#define LOG_ENDLINE
Closing clause for text line output using << operators.
Definition: logging_api.hpp:2956
VISTAOMEmain
int VISTAOMEmain(int argc, const char *argv[], const char *envp[])
Definition: OMEuserMain.cpp:16
operator*
OMEtype operator*(const OMEtype &lArg, const OMEtype &rArg)
Definition: OMEtype_operators.cpp:39283
OrderedInputSourceProxy::forwardInputBlock
virtual int forwardInputBlock(OrderedInputBlock *block) VIRTUAL_OVERRIDE
Definition: OrderedInput.hpp:552
READ_ONLY_OMEtype
const class OMEtype & READ_ONLY_OMEtype
A convenience typedef for performing read-only access to sparse and associative arrays....
Definition: OMEtype.h:58
OrderedInputBlock::OrderedInputBlock
OrderedInputBlock(uint64_t key, unsigned char *block, BlockSize_t len, OrderedInputSource *source, bool makeCopy=true)
Create a input block, potentially copying the source data.
Definition: OrderedInput.hpp:113
operator|=
OMEtype & operator|=(OMEtype &lArg, const OMEtype &rArg)
Definition: OMEtype_operators.cpp:28157
OrderedMultipleInputFilter::maxReorderedElements
size_t maxReorderedElements
Definition: OrderedInput.hpp:731
OME_ARRAY
@ OME_ARRAY
Definition: OMEmanifests.h:86
OMEarray::ARRAY_SUBSCRIPT_t
OMEarrayStorage::ARRAY_SUBSCRIPT_t ARRAY_SUBSCRIPT_t
Definition: OMEarray.h:90
OrderedMultipleInputFilter::~OrderedMultipleInputFilter
virtual ~OrderedMultipleInputFilter()
Definition: OrderedInput.hpp:751
OrderedInputSource::InputBlockReturn::block
OrderedInputBlock * block
points at retrieved record
Definition: OrderedInput.hpp:287
OrderedInputBlock::BlockSize_t
ssize_t BlockSize_t
Definition: OrderedInput.hpp:93
OrderedMultipleInputFilter::processQueue
int64_t processQueue(uint32_t *stopFlagPtr, const struct timespec *maxWaitTime=&OrderedInputSource::WAIT_YEAR_MAX) OME_ALWAYS_OPTIMIZE("-O3")
Process input blocks until all OrderedInputSource objects are drained of available data or a specifie...
Definition: OrderedInput.hpp:949
operator%
OMEtype operator%(const OMEtype &lArg, const OMEtype &rArg)
Definition: OMEtype_operators.cpp:42105
operator^
OMEtype operator^(const OMEtype &lArg, const OMEtype &rArg)
Definition: OMEtype_operators.cpp:44601
OMEabortOnError
int OMEabortOnError
If non-zero, abort() called on type error.
Definition: OMEtype_operators.cpp:9
CONSTEXPR
#define CONSTEXPR
Generates constexpr if the compiler supports it.
Definition: compiler_hints.h:432
NONNULL_RETURN
char NONNULL_RETURN
Definition: compiler_hints.h:745
OrderedInputSourceProxy::~OrderedInputSourceProxy
virtual ~OrderedInputSourceProxy()
Definition: OrderedInput.hpp:531
POSIXtimeInNanoseconds
POSIXtimeInUnits< 1000000000U > POSIXtimeInNanoseconds
Convenience typedef for nanosecond-resolution time.
Definition: time_point.hpp:580
OME_ASSOC
@ OME_ASSOC
Definition: OMEmanifests.h:87
OrderedInputSourceProxyWithReordering::sawEOF
bool sawEOF
indicates if end-of-file has been seen from this source
Definition: OrderedInput.hpp:596
operator==
bool operator==(const OMEtype &lArg, const OMEtype &rArg)
Definition: OMEtype_operators.cpp:33
OME_INT32
@ OME_INT32
Definition: OMEmanifests.h:79
OME_INT64
@ OME_INT64
Definition: OMEmanifests.h:81
logging_api.hpp
FARGOS Logging API.
Generated: Fri Jul 31 2020 18:19:15
Support Information