Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions src/configuration.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -161,6 +161,7 @@ bool configure(int argc, char **argv)
std::make_tuple("plyrics", "rihanna", "umbrella"),
std::make_tuple("tekstowo", "rihanna", "umbrella"),
std::make_tuple("zeneszoveg", "rihanna", "umbrella"),
std::make_tuple("letras", "mago de oz", "al-mejandria"),
};
for (auto &data : fetcher_data)
{
Expand Down
201 changes: 197 additions & 4 deletions src/lyrics_fetcher.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -20,12 +20,17 @@

#include "config.h"

#include <algorithm>
#include <cctype>
#include <cstdlib>
#include <cstring>
#include <sstream>
#include <vector>
#include <boost/algorithm/string/join.hpp>
#include <boost/algorithm/string/replace.hpp>
#include <boost/algorithm/string/split.hpp>
#include <boost/algorithm/string/trim.hpp>
#include <boost/property_tree/json_parser.hpp>
#include <boost/regex.hpp>

#ifdef HAVE_TAGLIB_H
Expand All @@ -40,6 +45,121 @@
#include "utility/html.h"
#include "utility/string.h"

namespace
{
std::string letrasSlug(const std::string &input)
{
struct Accent { const char *from; char to; };
static const Accent accents[] = {
{ "á", 'a' }, { "à", 'a' }, { "ä", 'a' }, { "â", 'a' }, { "ã", 'a' },
{ "é", 'e' }, { "è", 'e' }, { "ë", 'e' }, { "ê", 'e' },
{ "í", 'i' }, { "ì", 'i' }, { "ï", 'i' }, { "î", 'i' },
{ "ó", 'o' }, { "ò", 'o' }, { "ö", 'o' }, { "ô", 'o' }, { "õ", 'o' },
{ "ú", 'u' }, { "ù", 'u' }, { "ü", 'u' }, { "û", 'u' },
{ "ñ", 'n' }, { "ç", 'c' }, { "ÿ", 'y' }, { "œ", 'o' }
};
std::string result;
result.reserve(input.size());
size_t i = 0;
while (i < input.size())
{
unsigned char c = input[i];
if (c < 0x80)
{
char lc = std::tolower(c);
if ((lc >= 'a' && lc <= 'z') || (lc >= '0' && lc <= '9'))
result += lc;
else if (c == '&')
result += "and";
else if (!result.empty() && result.back() != '-')
result += '-';
++i;
}
else
{
bool matched = false;
for (const Accent &a : accents)
{
size_t len = std::strlen(a.from);
if (i + len <= input.size() && input.compare(i, len, a.from) == 0)
{
result += a.to;
i += len;
matched = true;
break;
}
}
if (!matched)
{
if (!result.empty() && result.back() != '-')
result += '-';
++i;
}
}
}
while (!result.empty() && result.back() == '-')
result.pop_back();
return result;
}

const char *letrasPartTokens[] = { "pt", "pts", "part", "parts", "parte", "partes", "par" };

const char *letrasRomanNums[] = { "i", "ii", "iii", "iv", "v", "vi", "vii", "viii", "ix", "x", "xi", "xii" };

bool isLetrasPartToken(const std::string &s)
{
for (const char *t : letrasPartTokens)
if (s == t)
return true;
return false;
}

int letrasRomanValue(const std::string &s)
{
for (size_t i = 0; i < 12; ++i)
if (s == letrasRomanNums[i])
return static_cast<int>(i)+1;
return 0;
}

std::vector<std::string> letrasTokens(const std::string &s)
{
std::string slug = letrasSlug(s);
std::vector<std::string> tokens;
boost::split(tokens, slug, boost::is_any_of("-"), boost::token_compress_off);
return tokens;
}

// Normaliza el título para comparar canónico (letras.com) vs tag (biblioteca): "part 1" == "pt. 1" == "parte i".
std::string letrasNormTitle(const std::string &s)
{
std::vector<std::string> tokens = letrasTokens(s);
std::string result;
for (auto &t : tokens)
{
if (isLetrasPartToken(t))
t = "p";
else if (int v = letrasRomanValue(t))
t = std::to_string(v);
if (!result.empty())
result += '-';
result += t;
}
return result;
}

// Tokens a usar en la búsqueda Solr: descarta "part N"/"pt. II" (matan el ranking).
std::vector<std::string> letrasQueryTokens(const std::string &s)
{
std::vector<std::string> tokens = letrasTokens(s);
tokens.erase(std::remove_if(tokens.begin(), tokens.end(), [](const std::string &t) {
return isLetrasPartToken(t) || letrasRomanValue(t) != 0
|| (t.find_first_not_of("0123456789") == std::string::npos);
}), tokens.end());
return tokens;
}

}
std::istream &operator>>(std::istream &is, LyricsFetcher_ &fetcher)
{
std::string s;
Expand All @@ -54,6 +174,8 @@ std::istream &operator>>(std::istream &is, LyricsFetcher_ &fetcher)
fetcher = std::make_unique<TekstowoFetcher>();
else if (s == "zeneszoveg")
fetcher = std::make_unique<ZeneszovegFetcher>();
else if (s == "letras")
fetcher = std::make_unique<LetrasFetcher>();
else if (s == "internet")
fetcher = std::make_unique<InternetLyricsFetcher>();
#ifdef HAVE_TAGLIB_H
Expand All @@ -74,10 +196,13 @@ LyricsFetcher::Result LyricsFetcher::fetch(const std::string &artist,
Result result;
result.first = false;

std::string url = urlTemplate();
boost::replace_all(url, "%artist%", Curl::escape(artist));
boost::replace_all(url, "%title%", Curl::escape(title));

std::string url = buildURL(artist, title);
if (url.empty())
{
result.second = msgNotFound;
return result;
}

std::string data;
CURLcode code = Curl::perform(data, url, "", true);

Expand Down Expand Up @@ -134,6 +259,74 @@ std::vector<std::string> LyricsFetcher::getContent(const char *regex_,
return result;
}

std::string LyricsFetcher::buildURL(const std::string &artist, const std::string &title) const
{
std::string url = urlTemplate();
boost::replace_all(url, "%artist%", Curl::escape(artist));
boost::replace_all(url, "%title%", Curl::escape(title));
return url;
}

std::string LetrasFetcher::buildURL(const std::string &artist, const std::string &title) const
{
// Las URLs "bonitas" de letras.com mapean mal las canciones con varias
// partes (part-1 -> PT.2, part-2 -> PT.1). Se resuelve la página canónica
// a través de la búsqueda Solr del propio sitio y se empareja el título
// normalizado (part/pt/romanos) contra el tag de la biblioteca.
std::vector<std::string> q = letrasTokens(artist);
std::vector<std::string> tt = letrasQueryTokens(title);
q.insert(q.end(), tt.begin(), tt.end());
if (q.empty())
return "";

std::string searchUrl = "https://solr.sscdn.co/letras/m1/?q=" + boost::algorithm::join(q, "+") + "&wt=json";
std::string data;
if (Curl::perform(data, searchUrl, "") != CURLE_OK)
return "";

size_t begin = data.find('(');
size_t end = data.rfind(')');
if (begin == std::string::npos || end == std::string::npos || end <= begin)
return "";

boost::property_tree::ptree root;
try
{
std::istringstream is(data.substr(begin+1, end-begin-1));
boost::property_tree::read_json(is, root);
}
catch (...)
{
return "";
}

const std::string expectedArtist = letrasSlug(artist);
const std::string expectedTitle = letrasNormTitle(title);
std::string fallback;

if (auto docs = root.get_child_optional("response.docs"))
{
for (const auto &kv : *docs)
{
const boost::property_tree::ptree &doc = kv.second;
std::string art = doc.get<std::string>("art", "");
std::string txt = doc.get<std::string>("txt", "");
std::string url = doc.get<std::string>("url", "");
std::string dns = doc.get<std::string>("dns", "");
if (art.empty() || txt.empty() || dns.empty() || url.empty())
continue;
if (letrasSlug(art) != expectedArtist)
continue;
if (fallback.empty())
fallback = "https://www.letras.com/" + dns + "/" + url + "/";
if (letrasNormTitle(txt) == expectedTitle)
return "https://www.letras.com/" + dns + "/" + url + "/";
}
}

return fallback;
}

void LyricsFetcher::postProcess(std::string &data) const
{
data = unescapeHtmlUtf8(data);
Expand Down
11 changes: 11 additions & 0 deletions src/lyrics_fetcher.h
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ struct LyricsFetcher

protected:
virtual const char *urlTemplate() const = 0;
virtual std::string buildURL(const std::string &artist, const std::string &title) const;
virtual const char *regex() const = 0;

virtual bool notLyrics(const std::string &) const { return false; }
Expand Down Expand Up @@ -111,6 +112,16 @@ struct ZeneszovegFetcher : public GoogleLyricsFetcher
virtual const char *regex() const override { return "<div class=\"lyrics-plain-text trans_original\">(.*?)</div>"; }
};

struct LetrasFetcher : public LyricsFetcher
{
virtual const char *name() const override { return "letras.com"; }

protected:
virtual const char *urlTemplate() const override { return ""; }
virtual std::string buildURL(const std::string &artist, const std::string &title) const override;
virtual const char *regex() const override { return "<div class=\"[^\"]*lyric-original[^\"]*\">(.*?)</div>"; }
};

struct InternetLyricsFetcher : public GoogleLyricsFetcher
{
virtual const char *name() const override { return "the Internet"; }
Expand Down