ModErn Text Analysis
META Enumerates Textual Applications
ir_eval.h
Go to the documentation of this file.
1 
9 #ifndef META_IR_EVAL_H_
10 #define META_IR_EVAL_H_
11 
12 #include <iostream>
13 #include <limits>
14 #include <stdexcept>
15 #include <string>
16 #include <unordered_map>
17 #include <utility>
18 #include <vector>
20 #include "meta/meta.h"
21 
22 namespace meta
23 {
24 namespace index
25 {
26 
31 class ir_eval
32 {
33  public:
34  using result_type = std::vector<search_result>;
35 
39  ir_eval(const cpptoml::table& config);
40 
48  double precision(const result_type& results, query_id q_id,
49  uint64_t num_docs
50  = std::numeric_limits<uint64_t>::max()) const;
51 
59  double recall(const result_type& results, query_id q_id,
60  uint64_t num_docs
61  = std::numeric_limits<uint64_t>::max()) const;
62 
72  double f1(const result_type& results, query_id q_id,
73  uint64_t num_docs = std::numeric_limits<uint64_t>::max(),
74  double beta = 1.0) const;
75 
85  double ndcg(const result_type& results, query_id q_id,
86  uint64_t num_docs = std::numeric_limits<uint64_t>::max()) const;
87 
91  double avg_p(const result_type& results, query_id q_id,
92  uint64_t num_docs = std::numeric_limits<uint64_t>::max());
93 
100  double map() const;
101 
108  double gmap() const;
109 
115  void print_stats(const result_type& results, query_id q_id,
116  std::ostream& out = std::cout);
117 
121  void reset_stats();
122 
123  private:
128  std::unordered_map<query_id, std::unordered_map<doc_id, uint64_t>> qrels_;
129 
131  std::vector<double> scores_;
132 
136  void init_index(const std::string& path);
137 
144  double relevant_retrieved(const result_type& results, query_id q_id,
145  uint64_t num_docs) const;
146 };
147 
151 class ir_eval_exception : public std::runtime_error
152 {
153  public:
154  using std::runtime_error::runtime_error;
155 };
156 }
157 }
158 
159 #endif
void print_stats(const result_type &results, query_id q_id, std::ostream &out=std::cout)
Definition: ir_eval.cpp:219
Contains top-level namespace documentation for the META toolkit.
double avg_p(const result_type &results, query_id q_id, uint64_t num_docs=std::numeric_limits< uint64_t >::max())
Computes the average precision for a query.
Definition: ir_eval.cpp:162
std::unordered_map< query_id, std::unordered_map< doc_id, uint64_t > > qrels_
query_id -> (doc_id -> relevance) mapping If the doc_id isn&#39;t in the map, it is non-relevant.
Definition: ir_eval.h:128
ir_eval(const cpptoml::table &config)
Definition: ir_eval.cpp:22
double ndcg(const result_type &results, query_id q_id, uint64_t num_docs=std::numeric_limits< uint64_t >::max()) const
Definition: ir_eval.cpp:125
double recall(const result_type &results, query_id q_id, uint64_t num_docs=std::numeric_limits< uint64_t >::max()) const
Definition: ir_eval.cpp:81
The ModErn Text Analysis toolkit is a suite of natural language processing, classification, information retrieval, data mining, and other applications of text processing.
Definition: analyzer.h:25
std::vector< double > scores_
Collection of scores used to calculate MAP and gMAP.
Definition: ir_eval.h:131
Evaluates lists of ranked documents returned from a search engine; can give stats per-query (e...
Definition: ir_eval.h:31
void init_index(const std::string &path)
Definition: ir_eval.cpp:31
void reset_stats()
Clears saved scores for MAP and gMAP.
Definition: ir_eval.cpp:239
Basic exception for ir_eval interactions.
Definition: ir_eval.h:151
double relevant_retrieved(const result_type &results, query_id q_id, uint64_t num_docs) const
Definition: ir_eval.cpp:94
double map() const
Definition: ir_eval.cpp:194
double gmap() const
Definition: ir_eval.cpp:203
double precision(const result_type &results, query_id q_id, uint64_t num_docs=std::numeric_limits< uint64_t >::max()) const
Definition: ir_eval.cpp:67
double f1(const result_type &results, query_id q_id, uint64_t num_docs=std::numeric_limits< uint64_t >::max(), double beta=1.0) const
Definition: ir_eval.cpp:111