| #include <set> |
| 6 | #include <stdexcept> |
| 7 |
| 8 | #include "SyntaxHighlighter.h" |
| 9 |
| 10 | namespace prisma_android { |
| 11 | namespace { |
| 12 |
| 13 | class GrammarReader { |
| 14 | public: |
| 15 | explicit GrammarReader(std::string_view bytes) : bytes_(bytes) {} |
| 16 |
| 17 | uint8_t readByte() { |
| 18 | require(1); |
| 19 | return static_cast<uint8_t>(bytes_[position_++]); |
| 20 | } |
| 21 |
| 22 | uint16_t readShort() { |
| 23 | const uint16_t low = readByte(); |
| 24 | return static_cast<uint16_t>(low | (readByte() << 8)); |
| 25 | } |
| 26 |
| 27 | std::string_view readString() { |
| 28 | size_t length = readByte(); |
| 29 | if (length >= 254) { |
| 30 | const size_t a = readByte(); |
| 31 | const size_t b = readByte(); |
| 32 | const size_t c = readByte(); |
| 33 | length = a | (b << 8) | (c << 16); |
| 34 | } |
| 35 | require(length); |
| 36 | const auto value = bytes_.substr(position_, length); |
| 37 | position_ += length; |
| 38 | return value; |
| 39 | } |
| 40 |
| 41 | bool finished() const { return position_ == bytes_.size(); } |
| 42 |
| 43 | private: |
| 44 | void require(size_t length) const { |
| 45 | if (length > bytes_.size() - position_) { |
| 46 | throw std::invalid_argument("Truncated grammars.dat"); |
| 47 | } |
| 48 | } |
| 49 |
| 50 | std::string_view bytes_; |
| 51 | size_t position_ = 0; |
| 52 | }; |
| 53 |
| 54 | // The upstream binary reader assumes that its input and all references are |
| 55 | // valid. Check structure before exposing a byte[] constructor through JNI. |
| 56 | void validateGrammars(std::string_view bytes, std::set<std::string>& names) { |
| 57 | GrammarReader reader(bytes); |
| 58 | const size_t patternCount = reader.readShort(); |
| 59 | if (patternCount == 0) { |
| 60 | throw std::invalid_argument("Empty grammar table"); |
| 61 | } |
| 62 | std::vector<size_t> insideGrammars; |
| 63 | for (size_t i = 0; i < patternCount; ++i) { |
| 64 | const auto pattern = reader.readString(); |
| 65 | const size_t firstSlash = pattern.find('/'); |
| 66 | const size_t lastSlash = pattern.rfind('/'); |
| 67 | if (firstSlash == std::string_view::npos || lastSlash <= firstSlash) { |
| 68 | throw std::invalid_argument("Invalid grammar pattern"); |
| 69 | } |
| 70 | const auto options = pattern.substr(lastSlash + 1); |
| 71 | const size_t firstComma = options.find(','); |
| 72 | const size_t lastComma = options.rfind(','); |
| 73 | if (firstComma == std::string_view::npos || lastComma <= firstComma) { |
| 74 | throw std::invalid_argument("Invalid grammar pattern options"); |
| 75 | } |
| 76 | names.emplace(options.substr(firstComma + 1, lastComma - firstComma - 1)); |
| 77 | if (lastComma + 1 < options.size()) { |
| 78 | size_t index = 0; |
| 79 | for (char c : options.substr(lastComma + 1)) { |
| 80 | if (c < '0' || c > '9' || index > 65535) { |
| 81 | throw std::invalid_argument("Invalid nested grammar index"); |
| 82 | } |
| 83 | index = index * 10 + static_cast<size_t>(c - '0'); |
| 84 | } |
| 85 | insideGrammars.push_back(index); |
| 86 | } |
| 87 | } |
| 88 | const size_t grammarCount = reader.readShort(); |
| 89 | for (size_t index : insideGrammars) { |
| 90 | if (index >= grammarCount) { |
| 91 | throw std::invalid_argument("Nested grammar index out of bounds"); |
| 92 | } |
| 93 | } |
| 94 | for (size_t i = 0; i < grammarCount; ++i) { |
| 95 | const size_t tokenCount = reader.readByte(); |
| 96 | for (size_t j = 0; j < tokenCount; ++j) { |
| 97 | names.emplace(reader.readString()); |
| 98 | const size_t references = reader.readByte(); |
| 99 | for (size_t k = 0; k < references; ++k) { |
| 100 | if (reader.readShort() >= patternCount) { |
| 101 | throw std::invalid_argument("Pattern index out of bounds"); |
| 102 | } |
| 103 | } |
| 104 | } |
| 105 | } |
| 106 | const size_t languageCount = reader.readShort(); |
| 107 | for (size_t i = 0; i < languageCount; ++i) { |
| 108 | reader.readString(); |
| 109 | reader.readString(); |
| 110 | if (reader.readShort() >= grammarCount) { |
| 111 | throw std::invalid_argument("Language grammar index out of bounds"); |
| 112 | } |
| 113 | } |
| 114 | if (!reader.finished()) { |
| 115 | throw std::invalid_argument("Unexpected trailing grammar data"); |
| 116 | } |
| 117 | } |
| 118 |
| 119 | struct Utf8Text { |
| 120 | std::string bytes; |
| 121 | std::vector<int32_t> offsets; |
| 122 |
| 123 | int32_t endOffset(size_t byteOffset) const { |
| 124 | // A byte-oriented regex can split a UTF-8 sequence. Keep spans on full |
| 125 | // UTF-16 code points, including both halves of a surrogate pair. |
| 126 | while (byteOffset < bytes.size() && |
| 127 | (static_cast<uint8_t>(bytes[byteOffset]) & 0xC0) == 0x80) { |
| 128 | ++byteOffset; |
| 129 | } |
| 130 | return offsets.at(byteOffset); |
| 131 | } |
| 132 | }; |
| 133 |
| 134 | Utf8Text encodeUtf8(std::u16string_view text) { |
| 135 | if (text.size() > static_cast<size_t>(std::numeric_limits<int32_t>::max())) { |
| 136 | throw std::length_error("Text exceeds JNI range capacity"); |
| 137 | } |
| 138 | Utf8Text result; |
| 139 | result.bytes.reserve(text.size()); |
| 140 | result.offsets.reserve(text.size() + 1); |
| 141 | for (size_t i = 0; i < text.size(); ++i) { |
| 142 | const int32_t start = static_cast<int32_t>(i); |
| 143 | uint32_t cp = text[i]; |
| 144 | if (cp >= 0xD800 && cp <= 0xDBFF && i + 1 < text.size() && text[i + 1] >= 0xDC00 && |
| 145 | text[i + 1] <= 0xDFFF) { |
| 146 | cp = 0x10000 + ((cp - 0xD800) << 10) + (text[++i] - 0xDC00); |
| 147 | } else if (cp >= 0xD800 && cp <= 0xDFFF) { |
| 148 | cp = 0xFFFD; |
| 149 | } |
| 150 | const auto append = [&](uint8_t byte) { |
| 151 | result.bytes.push_back(static_cast<char>(byte)); |
| 152 | result.offsets.push_back(start); |
| 153 | }; |
| 154 | if (cp < 0x80) { |
| 155 | append(static_cast<uint8_t>(cp)); |
| 156 | } else if (cp < 0x800) { |
| 157 | append(static_cast<uint8_t>(0xC0 | (cp >> 6))); |
| 158 | append(static_cast<uint8_t>(0x80 | (cp & 0x3F))); |
| 159 | } else if (cp < 0x10000) { |
| 160 | append(static_cast<uint8_t>(0xE0 | (cp >> 12))); |
| 161 | append(static_cast<uint8_t>(0x80 | ((cp >> 6) & 0x3F))); |
| 162 | append(static_cast<uint8_t>(0x80 | (cp & 0x3F))); |
| 163 | } else { |
| 164 | append(static_cast<uint8_t>(0xF0 | (cp >> 18))); |
| 165 | append(static_cast<uint8_t>(0x80 | ((cp >> 12) & 0x3F))); |
| 166 | append(static_cast<uint8_t>(0x80 | ((cp >> 6) & 0x3F))); |
| 167 | append(static_cast<uint8_t>(0x80 | (cp & 0x3F))); |
| 168 | } |
| 169 | } |
| 170 | result.offsets.push_back(static_cast<int32_t>(text.size())); |
| 171 | return result; |
| 172 | } |
| 173 |
| 174 | void flatten(const TokenList& list, const Utf8Text& source, size_t start, size_t limit, |
| 175 | int32_t parent, const std::unordered_map<std::string, int32_t>& names, |
| 176 | std::vector<int32_t>& ranges) { |
| 177 | size_t offset = start; |
| 178 | for (const auto& node : list) { |
| 179 | if (offset > limit || node.length() > limit - offset) { |
| 180 | throw std::runtime_error("Invalid tokenizer range"); |
| 181 | } |
| 182 | if (node.isSyntax()) { |
| 183 | const auto& syntax = static_cast<const Syntax&>(node); |
| 184 | // Rounding both boundaries up gives every full code point to the |
| 185 | // first byte fragment, without overlapping sibling token ranges. |
| 186 | const int32_t begin = source.endOffset(offset); |
| 187 | const int32_t end = source.endOffset(offset + node.length()); |
| 188 | int32_t childParent = parent; |
| 189 | if (begin < end) { |
| 190 | if (ranges.size() > static_cast<size_t>(std::numeric_limits<int32_t>::max()) - 5) { |
| 191 | throw std::length_error("Too many syntax tokens"); |
| 192 | } |
| 193 | childParent = static_cast<int32_t>(ranges.size() / 5); |
| 194 | ranges.insert(ranges.end(), {begin, end, names.at(syntax.m_type), |
| 195 | names.at(syntax.m_alias), parent}); |
| 196 | } |
| 197 | flatten(syntax.children(), source, offset, offset + node.length(), childParent, names, |
| 198 | ranges); |
| 199 | } |
| 200 | offset += node.length(); |
| 201 | } |
| 202 | if (offset != limit) throw std::runtime_error("Tokenizer changed text length"); |
| 203 | } |
| 204 |
| 205 | } // namespace |
| 206 |
| 207 | Engine::Engine(std::string_view grammars) { |
| 208 | std::set<std::string> names; |
| 209 | names.emplace(""); |
| 210 | validateGrammars(grammars, names); |
| 211 | tokenNames_.assign(names.begin(), names.end()); |
| 212 | for (size_t i = 0; i < tokenNames_.size(); ++i) { |
| 213 | tokenNameIds_.emplace(tokenNames_[i], static_cast<int32_t>(i)); |
| 214 | } |
| 215 | highlighter_ = std::make_unique<SyntaxHighlighter>(std::string(grammars)); |
| 216 | } |
| 217 |
| 218 | Engine::~Engine() = default; |
| 219 |
| 220 | std::vector<int32_t> Engine::highlight(std::u16string_view text, std::u16string_view language) { |
| 221 | std::vector<int32_t> result; |
| 222 | if (text.empty() || language.empty()) return result; |
| 223 | const auto source = encodeUtf8(text); |
| 224 | auto languageUtf8 = encodeUtf8(language).bytes; |
| 225 | for (char& c : languageUtf8) { |
| 226 | if (c >= 'A' && c <= 'Z') c = static_cast<char>(c + ('a' - 'A')); |
| 227 | } |
| 228 | // TokenList holds string_views into source.bytes. Flatten before source dies. |
| 229 | const auto tokens = highlighter_->tokenize(source.bytes, languageUtf8); |
| 230 | flatten(tokens, source, 0, source.bytes.size(), -1, tokenNameIds_, result); |
| 231 | return result; |
| 232 | } |
| 233 |
| 234 | std::vector<std::string> Engine::languages() const { |
| 235 | std::vector<std::string> result; |
| 236 | // File extensions (rs, h, ...) have no display name: they only resolve a diff's |
| 237 | // file names and stay out of the language picker. |
| 238 | for (const auto& item : highlighter_->languages()) { |
| 239 | if (!item.second.empty()) result.push_back(item.first); |
| 240 | } |
| 241 | return result; |
| 242 | } |
| 243 |
| 244 | } // namespace prisma_android |
| 245 |