1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
|
// Copyright (C) 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef ICING_INDEX_ITERATOR_DOC_HIT_INFO_ITERATOR_TEST_UTIL_H_
#define ICING_INDEX_ITERATOR_DOC_HIT_INFO_ITERATOR_TEST_UTIL_H_
#include <cinttypes>
#include <string>
#include <utility>
#include <vector>
#include "icing/text_classifier/lib3/utils/base/status.h"
#include "icing/absl_ports/canonical_errors.h"
#include "icing/absl_ports/str_cat.h"
#include "icing/index/hit/doc-hit-info.h"
#include "icing/index/iterator/doc-hit-info-iterator.h"
#include "icing/legacy/core/icing-string-util.h"
#include "icing/schema/section.h"
#include "icing/store/document-id.h"
namespace icing {
namespace lib {
class DocHitInfoTermFrequencyPair {
public:
DocHitInfoTermFrequencyPair(
const DocHitInfo& doc_hit_info,
const Hit::TermFrequencyArray& hit_term_frequency = {})
: doc_hit_info_(doc_hit_info), hit_term_frequency_(hit_term_frequency) {}
void UpdateSection(SectionId section_id,
Hit::TermFrequency hit_term_frequency) {
doc_hit_info_.UpdateSection(section_id);
hit_term_frequency_[section_id] = hit_term_frequency;
}
void MergeSectionsFrom(const DocHitInfoTermFrequencyPair& other) {
SectionIdMask other_mask = other.doc_hit_info_.hit_section_ids_mask();
doc_hit_info_.MergeSectionsFrom(other_mask);
while (other_mask) {
SectionId section_id = __builtin_ctzll(other_mask);
hit_term_frequency_[section_id] = other.hit_term_frequency_[section_id];
other_mask &= ~(UINT64_C(1) << section_id);
}
}
DocHitInfo doc_hit_info() const { return doc_hit_info_; }
Hit::TermFrequency hit_term_frequency(SectionId section_id) const {
return hit_term_frequency_[section_id];
}
private:
DocHitInfo doc_hit_info_;
Hit::TermFrequencyArray hit_term_frequency_;
};
// Dummy class to help with testing. It starts with an kInvalidDocumentId doc
// hit info until an Advance is called (like normal DocHitInfoIterators). It
// will then proceed to return the doc_hit_infos in order as Advance's are
// called. After all doc_hit_infos are returned, Advance will return a NotFound
// error (also like normal DocHitInfoIterators).
class DocHitInfoIteratorDummy : public DocHitInfoIterator {
public:
DocHitInfoIteratorDummy() = default;
explicit DocHitInfoIteratorDummy(
std::vector<DocHitInfoTermFrequencyPair> doc_hit_infos,
std::string term = "")
: doc_hit_infos_(std::move(doc_hit_infos)), term_(std::move(term)) {}
explicit DocHitInfoIteratorDummy(const std::vector<DocHitInfo>& doc_hit_infos,
std::string term = "",
int term_start_index = 0,
int unnormalized_term_length = 0)
: term_(std::move(term)),
term_start_index_(term_start_index),
unnormalized_term_length_(unnormalized_term_length) {
for (auto& doc_hit_info : doc_hit_infos) {
doc_hit_infos_.push_back(DocHitInfoTermFrequencyPair(doc_hit_info));
}
}
libtextclassifier3::Status Advance() override {
++index_;
if (index_ < doc_hit_infos_.size()) {
doc_hit_info_ = doc_hit_infos_.at(index_).doc_hit_info();
return libtextclassifier3::Status::OK;
}
return absl_ports::ResourceExhaustedError(
"No more DocHitInfos in iterator");
}
libtextclassifier3::StatusOr<TrimmedNode> TrimRightMostNode() && override {
DocHitInfoIterator::TrimmedNode node = {nullptr, term_, term_start_index_,
unnormalized_term_length_};
return node;
}
// Imitates behavior of DocHitInfoIteratorTermMain/DocHitInfoIteratorTermLite
void PopulateMatchedTermsStats(
std::vector<TermMatchInfo>* matched_terms_stats,
SectionIdMask filtering_section_mask = kSectionIdMaskAll) const override {
if (index_ == -1 || index_ >= doc_hit_infos_.size()) {
// Current hit isn't valid, return.
return;
}
SectionIdMask section_mask =
doc_hit_info_.hit_section_ids_mask() & filtering_section_mask;
SectionIdMask section_mask_copy = section_mask;
std::array<Hit::TermFrequency, kTotalNumSections> section_term_frequencies =
{Hit::kNoTermFrequency};
while (section_mask_copy) {
SectionId section_id = __builtin_ctzll(section_mask_copy);
section_term_frequencies.at(section_id) =
doc_hit_infos_.at(index_).hit_term_frequency(section_id);
section_mask_copy &= ~(UINT64_C(1) << section_id);
}
TermMatchInfo term_stats(term_, section_mask,
std::move(section_term_frequencies));
for (auto& cur_term_stats : *matched_terms_stats) {
if (cur_term_stats.term == term_stats.term) {
// Same docId and same term, we don't need to add the term and the term
// frequency should always be the same
return;
}
}
matched_terms_stats->push_back(term_stats);
}
void set_hit_intersect_section_ids_mask(
SectionIdMask hit_intersect_section_ids_mask) {
hit_intersect_section_ids_mask_ = hit_intersect_section_ids_mask;
}
int32_t GetNumBlocksInspected() const override {
return num_blocks_inspected_;
}
void SetNumBlocksInspected(int32_t num_blocks_inspected) {
num_blocks_inspected_ = num_blocks_inspected;
}
int32_t GetNumLeafAdvanceCalls() const override {
return num_leaf_advance_calls_;
}
void SetNumLeafAdvanceCalls(int32_t num_leaf_advance_calls) {
num_leaf_advance_calls_ = num_leaf_advance_calls;
}
std::string ToString() const override {
std::string ret = "<";
for (auto& doc_hit_info_pair : doc_hit_infos_) {
absl_ports::StrAppend(
&ret, IcingStringUtil::StringPrintf(
"[%d,%" PRIu64 "]",
doc_hit_info_pair.doc_hit_info().document_id(),
doc_hit_info_pair.doc_hit_info().hit_section_ids_mask()));
}
absl_ports::StrAppend(&ret, ">");
return ret;
}
private:
int32_t index_ = -1;
int32_t num_blocks_inspected_ = 0;
int32_t num_leaf_advance_calls_ = 0;
std::vector<DocHitInfoTermFrequencyPair> doc_hit_infos_;
std::string term_;
int term_start_index_;
int unnormalized_term_length_;
};
inline std::vector<DocumentId> GetDocumentIds(DocHitInfoIterator* iterator) {
std::vector<DocumentId> ids;
while (iterator->Advance().ok()) {
ids.push_back(iterator->doc_hit_info().document_id());
}
return ids;
}
inline std::vector<DocHitInfo> GetDocHitInfos(DocHitInfoIterator* iterator) {
std::vector<DocHitInfo> doc_hit_infos;
while (iterator->Advance().ok()) {
doc_hit_infos.push_back(iterator->doc_hit_info());
}
return doc_hit_infos;
}
} // namespace lib
} // namespace icing
#endif // ICING_INDEX_ITERATOR_DOC_HIT_INFO_ITERATOR_TEST_UTIL_H_
|
