/*++
Revision History:
Date: Jun 28, 2024.
Author: Rajas Chavadekar.
Desc: Created.
--*/
#include
#include
#include
#include "BaseVectorizer.h"
std::string preprocess_text(const std::string& text) {
std::string processed = text; //.substr(0, MAX_TEXT_LEN);
std::replace(processed.begin(), processed.end(), '\n', ' ');
std::string filtered = "";
for (char c : processed) {
if (std::isalnum(c) || std::isspace(c) || std::ispunct(c)) {
filtered += std::tolower(c);
}
else
{
filtered += ' ';
}
}
return filtered;
}
/**
* @brief Generate n-grams from a vector of words.
*
* @param tokens The vector of words (tokens) from which to generate n-grams.
* @param n The size of n-grams to generate.
* @return A vector of n-grams as strings.
*/
vector generateNGrams(const vector& tokens, int n) {
vector ngrams;
if (n <= 0 || tokens.size() < n) {
return ngrams; // Return empty if n is invalid or insufficient tokens
}
for (size_t i = 0; i <= tokens.size() - n; ++i) {
string ngram = tokens[i];
for (int j = 1; j < n; ++j) {
ngram += " " + tokens[i + j];
}
ngrams.push_back(ngram);
}
return ngrams;
}
/**
* @brief Split a sentence into a vector of words.
*
* @param sentence_ The sentence to split.
* @return Vector of words.
*/
vector BaseVectorizer::buildSentenceVector(string sentence_, bool preprocess)
{
GlobalData vars;
string new_word = "";
vector ret;
if (true == preprocess)
{
sentence_ = preprocess_text(sentence_);
}
for (char x : sentence_)
{
if (isupper(x) && !case_sensitive)
{
x = tolower(x);
}
if (x == ' ')
{
if (!include_stopwords && vars.stopWords.count(new_word))
{
new_word = "";
}
else
{
ret.push_back(new_word);
new_word = "";
}
}
else if (vars.punctuation.count(x))
{
ret.push_back(new_word);
new_word = x;
ret.push_back(new_word);
new_word = "";
}
else
{
new_word += x;
}
}
if (new_word != "")
{
ret.push_back(new_word);
}
vector fixed_ret;
for (const auto& s : ret)
{
if (!s.empty())
{
fixed_ret.push_back(s);
}
}
if (ngrams > 1) {
return generateNGrams(fixed_ret, ngrams);
}
return fixed_ret;
}
void BaseVectorizer::scanForSparseHistogram(std::string abs_filepath_to_features, int minfrequency)
{
ifstream in;
string feature;
vector features;
in.open(abs_filepath_to_features);
if (!in)
{
cout << "ERROR: Cannot open features file.\n";
return;
}
std::unordered_map<:string int> histogram2;
while (getline(in, feature))
{
features = buildSentenceVector(feature);
for (const auto& x : features)
{
if (histogram2.count(x) || x.length() == 1)
{
histogram2[x]++;
}
else
{
histogram2[x] = 1;
}
}
}
in.close();
for (const auto& entry : histogram2)
{
if (entry.second < minfrequency)
{
histogram[entry.first] = entry.second;
}
}
std::cout << "No of Rare Words = " << histogram.size() << std::endl;
}
void BaseVectorizer::setVersionInfo(char* vers_info_in)
{
memset(vers_info, 0, sizeof(vers_info));
strcpy(vers_info, vers_info_in);
}