Packages

An Elixir DuckDB library

Current section

Files

Jump to
exduckdb c_src duckdb src common string_util.cpp
Raw

c_src/duckdb/src/common/string_util.cpp

#include "duckdb/common/string_util.hpp"
#include "duckdb/common/pair.hpp"
#include "duckdb/common/to_string.hpp"
#include "duckdb/common/string_util.hpp"
#include "duckdb/common/exception.hpp"
#include <algorithm>
#include <cctype>
#include <iomanip>
#include <memory>
#include <sstream>
#include <stdarg.h>
#include <string.h>
namespace duckdb {
bool StringUtil::Contains(const string &haystack, const string &needle) {
return (haystack.find(needle) != string::npos);
}
void StringUtil::LTrim(string &str) {
auto it = str.begin();
while (CharacterIsSpace(*it)) {
it++;
}
str.erase(str.begin(), it);
}
// Remove trailing ' ', '\f', '\n', '\r', '\t', '\v'
void StringUtil::RTrim(string &str) {
str.erase(find_if(str.rbegin(), str.rend(), [](int ch) { return ch > 0 && !CharacterIsSpace(ch); }).base(),
str.end());
}
void StringUtil::Trim(string &str) {
StringUtil::LTrim(str);
StringUtil::RTrim(str);
}
bool StringUtil::StartsWith(string str, string prefix) {
if (prefix.size() > str.size()) {
return false;
}
return equal(prefix.begin(), prefix.end(), str.begin());
}
bool StringUtil::EndsWith(const string &str, const string &suffix) {
if (suffix.size() > str.size()) {
return false;
}
return equal(suffix.rbegin(), suffix.rend(), str.rbegin());
}
string StringUtil::Repeat(const string &str, idx_t n) {
std::ostringstream os;
for (idx_t i = 0; i < n; i++) {
os << str;
}
return (os.str());
}
vector<string> StringUtil::Split(const string &str, char delimiter) {
std::stringstream ss(str);
vector<string> lines;
string temp;
while (getline(ss, temp, delimiter)) {
lines.push_back(temp);
}
return (lines);
}
namespace string_util_internal {
inline void SkipSpaces(const string &str, idx_t &index) {
while (index < str.size() && std::isspace(str[index])) {
index++;
}
}
inline void ConsumeLetter(const string &str, idx_t &index, char expected) {
if (index >= str.size() || str[index] != expected) {
throw ParserException("Invalid quoted list: %s", str);
}
index++;
}
template <typename F>
inline void TakeWhile(const string &str, idx_t &index, const F &cond, string &taker) {
while (index < str.size() && cond(str[index])) {
taker.push_back(str[index]);
index++;
}
}
inline string TakePossiblyQuotedItem(const string &str, idx_t &index, char delimiter, char quote) {
string entry;
if (str[index] == quote) {
index++;
TakeWhile(
str, index, [quote](char c) { return c != quote; }, entry);
ConsumeLetter(str, index, quote);
} else {
TakeWhile(
str, index, [delimiter, quote](char c) { return c != delimiter && c != quote && !std::isspace(c); }, entry);
}
return entry;
}
} // namespace string_util_internal
vector<string> StringUtil::SplitWithQuote(const string &str, char delimiter, char quote) {
vector<string> entries;
idx_t i = 0;
string_util_internal::SkipSpaces(str, i);
while (i < str.size()) {
if (!entries.empty()) {
string_util_internal::ConsumeLetter(str, i, delimiter);
}
entries.emplace_back(string_util_internal::TakePossiblyQuotedItem(str, i, delimiter, quote));
string_util_internal::SkipSpaces(str, i);
}
return entries;
}
string StringUtil::Join(const vector<string> &input, const string &separator) {
return StringUtil::Join(input, input.size(), separator, [](const string &s) { return s; });
}
string StringUtil::BytesToHumanReadableString(idx_t bytes) {
string db_size;
auto kilobytes = bytes / 1000;
auto megabytes = kilobytes / 1000;
kilobytes -= megabytes * 1000;
auto gigabytes = megabytes / 1000;
megabytes -= gigabytes * 1000;
auto terabytes = gigabytes / 1000;
gigabytes -= terabytes * 1000;
if (terabytes > 0) {
return to_string(terabytes) + "." + to_string(gigabytes / 100) + "TB";
} else if (gigabytes > 0) {
return to_string(gigabytes) + "." + to_string(megabytes / 100) + "GB";
} else if (megabytes > 0) {
return to_string(megabytes) + "." + to_string(kilobytes / 100) + "MB";
} else if (kilobytes > 0) {
return to_string(kilobytes) + "KB";
} else {
return to_string(bytes) + " bytes";
}
}
string StringUtil::Upper(const string &str) {
string copy(str);
transform(copy.begin(), copy.end(), copy.begin(), [](unsigned char c) { return std::toupper(c); });
return (copy);
}
string StringUtil::Lower(const string &str) {
string copy(str);
transform(copy.begin(), copy.end(), copy.begin(), [](unsigned char c) { return std::tolower(c); });
return (copy);
}
vector<string> StringUtil::Split(const string &input, const string &split) {
vector<string> splits;
idx_t last = 0;
idx_t input_len = input.size();
idx_t split_len = split.size();
while (last <= input_len) {
idx_t next = input.find(split, last);
if (next == string::npos) {
next = input_len;
}
// Push the substring [last, next) on to splits
string substr = input.substr(last, next - last);
if (substr.empty() == false) {
splits.push_back(substr);
}
last = next + split_len;
}
return splits;
}
string StringUtil::Replace(string source, const string &from, const string &to) {
idx_t start_pos = 0;
while ((start_pos = source.find(from, start_pos)) != string::npos) {
source.replace(start_pos, from.length(), to);
start_pos += to.length(); // In case 'to' contains 'from', like
// replacing 'x' with 'yx'
}
return source;
}
vector<string> StringUtil::TopNStrings(vector<pair<string, idx_t>> scores, idx_t n, idx_t threshold) {
if (scores.empty()) {
return vector<string>();
}
sort(scores.begin(), scores.end(),
[](const pair<string, idx_t> &a, const pair<string, idx_t> &b) -> bool { return a.second < b.second; });
vector<string> result;
result.push_back(scores[0].first);
for (idx_t i = 1; i < MinValue<idx_t>(scores.size(), n); i++) {
if (scores[i].second > threshold) {
break;
}
result.push_back(scores[i].first);
}
return result;
}
struct LevenshteinArray {
LevenshteinArray(idx_t len1, idx_t len2) : len1(len1) {
dist = unique_ptr<idx_t[]>(new idx_t[len1 * len2]);
}
idx_t &Score(idx_t i, idx_t j) {
return dist[GetIndex(i, j)];
}
private:
idx_t len1;
unique_ptr<idx_t[]> dist;
idx_t GetIndex(idx_t i, idx_t j) {
return j * len1 + i;
}
};
// adapted from https://en.wikibooks.org/wiki/Algorithm_Implementation/Strings/Levenshtein_distance#C++
idx_t StringUtil::LevenshteinDistance(const string &s1, const string &s2) {
idx_t len1 = s1.size();
idx_t len2 = s2.size();
if (len1 == 0) {
return len2;
}
if (len2 == 0) {
return len1;
}
LevenshteinArray array(len1 + 1, len2 + 1);
array.Score(0, 0) = 0;
for (idx_t i = 0; i <= len1; i++) {
array.Score(i, 0) = i;
}
for (idx_t j = 0; j <= len2; j++) {
array.Score(0, j) = j;
}
for (idx_t i = 1; i <= len1; i++) {
for (idx_t j = 1; j <= len2; j++) {
// d[i][j] = std::min({ d[i - 1][j] + 1,
// d[i][j - 1] + 1,
// d[i - 1][j - 1] + (s1[i - 1] == s2[j - 1] ? 0 : 1) });
int equal = s1[i - 1] == s2[j - 1] ? 0 : 1;
idx_t adjacent_score1 = array.Score(i - 1, j) + 1;
idx_t adjacent_score2 = array.Score(i, j - 1) + 1;
idx_t adjacent_score3 = array.Score(i - 1, j - 1) + equal;
idx_t t = MinValue<idx_t>(adjacent_score1, adjacent_score2);
array.Score(i, j) = MinValue<idx_t>(t, adjacent_score3);
}
}
return array.Score(len1, len2);
}
vector<string> StringUtil::TopNLevenshtein(const vector<string> &strings, const string &target, idx_t n,
idx_t threshold) {
vector<pair<string, idx_t>> scores;
scores.reserve(strings.size());
for (auto &str : strings) {
scores.emplace_back(str, LevenshteinDistance(str, target));
}
return TopNStrings(scores, n, threshold);
}
string StringUtil::CandidatesMessage(const vector<string> &candidates, const string &candidate) {
string result_str;
if (!candidates.empty()) {
result_str = "\n" + candidate + ": ";
for (idx_t i = 0; i < candidates.size(); i++) {
if (i > 0) {
result_str += ", ";
}
result_str += "\"" + candidates[i] + "\"";
}
}
return result_str;
}
} // namespace duckdb