| const Prism = require('prismjs') |
| 4 | const components = require('prismjs/components.js') |
| 5 |
| 6 | global.Prism = Prism |
| 7 |
| 8 | const SCRIPTS = {} |
| 9 | const include = function (src) { |
| 10 | // Some black magic of eval. Load the script from src to global scope. Source: https://stackoverflow.com/a/23699187/17140794 |
| 11 | (1, eval)(src.toString()) |
| 12 | } |
| 13 |
| 14 | async function loadLanguages(lngs) { |
| 15 | if (lngs) { |
| 16 | lngs = Array.isArray(lngs) ? lngs : [lngs]; |
| 17 |
| 18 | for (const lng of lngs) { |
| 19 | await loadLanguage(lng) |
| 20 | } |
| 21 | } |
| 22 | } |
| 23 | let langNumber = 0 |
| 24 | async function loadLanguage(lng) { |
| 25 | if (!components.languages[lng].title) { |
| 26 | return |
| 27 | } |
| 28 |
| 29 | await loadLanguages(components.languages[lng].optional) |
| 30 | await loadLanguages(components.languages[lng].require) |
| 31 | await loadLanguages(components.languages[lng].modify) |
| 32 |
| 33 | if (!SCRIPTS[lng]) { |
| 34 | SCRIPTS[lng] = true |
| 35 |
| 36 | langNumber += 1 |
| 37 | console.log(`${langNumber} | Loading ${lng}`); |
| 38 | // TODO: version should probably not be hardcoded |
| 39 |
| 40 | require(`prismjs/components/prism-${lng}.js`) |
| 41 | } |
| 42 | } |
| 43 |
| 44 | function loadLocalLanguage(path, code, title, alias) { |
| 45 | include(fs.readFileSync(path)) |
| 46 |
| 47 | components.languages[code] = { |
| 48 | title: title, |
| 49 | alias: alias |
| 50 | } |
| 51 | } |
| 52 |
| 53 | // Spell out the constructs whose meaning differs between regex engines, so one table can |
| 54 | // serve all of them. java.util.regex reads a [ inside a character class as the start of a |
| 55 | // nested class, requires a { outside one to open a quantifier, reads \0 as the start of an |
| 56 | // octal escape, and reads \v as the whole vertical whitespace class rather than as U+000B. |
| 57 | // All four rewrites are no-ops for ECMAScript and for Boost, which is what libprisma |
| 58 | // compiles the patterns with. |
| 59 | const QUANTIFIER = /^\{\d+(?:,\d*)?\}/ |
| 60 |
| 61 | function normalizeEscapes(pattern) { |
| 62 | let result = '' |
| 63 | let inClass = false |
| 64 |
| 65 | for (let i = 0; i < pattern.length; i++) { |
| 66 | const c = pattern[i] |
| 67 |
| 68 | if (c === '\\') { |
| 69 | if (pattern[i + 1] === '0' && !/[0-7]/.test(pattern[i + 2] || '')) { |
| 70 | result += '\\x00' |
| 71 | } else if (pattern[i + 1] === 'v') { |
| 72 | result += '\\x0B' |
| 73 | } else { |
| 74 | result += pattern.substr(i, 2) |
| 75 | } |
| 76 | i++ |
| 77 | continue |
| 78 | } |
| 79 |
| 80 | if (inClass) { |
| 81 | if (c === '[') { |
| 82 | result += '\\[' |
| 83 | continue |
| 84 | } |
| 85 | // && is set intersection in java.util.regex |
| 86 | if (c === '&' && pattern[i + 1] === '&') { |
| 87 | result += '\\&\\&' |
| 88 | i++ |
| 89 | continue |
| 90 | } |
| 91 | if (c === ']') { |
| 92 | inClass = false |
| 93 | } |
| 94 | } else if (c === '[') { |
| 95 | inClass = true |
| 96 | } else if (c === '{' && !QUANTIFIER.test(pattern.substr(i))) { |
| 97 | result += '\\{' |
| 98 | continue |
| 99 | } |
| 100 |
| 101 | result += c |
| 102 | } |
| 103 |
| 104 | return result |
| 105 | } |
| 106 |
| 107 | // A character written literally in a grammar - BQN's •, APL's ⍵ - has to reach the readers as |
| 108 | // an escape rather than as itself: they disagree about what the bytes of a pattern mean, and |
| 109 | // \uXXXX is the one spelling all of them read as a code point. An astral character is two code |
| 110 | // units here, so it becomes a surrogate pair, which UnicodeEscapes.h puts back together. |
| 111 | function escapeNonAscii(pattern) { |
| 112 | if (!/[^\x00-\x7F]/.test(pattern)) { |
| 113 | return pattern |
| 114 | } |
| 115 |
| 116 | let result = '' |
| 117 |
| 118 | for (let i = 0; i < pattern.length; i++) { |
| 119 | const code = pattern.charCodeAt(i) |
| 120 | result += code > 0x7F |
| 121 | ? '\\u' + code.toString(16).padStart(4, '0') |
| 122 | : pattern[i] |
| 123 | } |
| 124 |
| 125 | return result |
| 126 | } |
| 127 |
| 128 | function unique(a, fn) { |
| 129 | if (a.length === 0 || a.length === 1) { |
| 130 | return a; |
| 131 | } |
| 132 | if (!fn) { |
| 133 | return a; |
| 134 | } |
| 135 |
| 136 | for (let i = 0; i < a.length; i++) { |
| 137 | for (let j = i + 1; j < a.length; j++) { |
| 138 | if (fn(a[i], a[j])) { |
| 139 | a.splice(i, 1); |
| 140 | } |
| 141 | } |
| 142 | } |
| 143 | return a; |
| 144 | } |
| 145 |
| 146 | function uniqlo(a, fn) { |
| 147 | var size = a.length; |
| 148 |
| 149 | do { |
| 150 | size = a.length |
| 151 | a = unique(a, fn) |
| 152 | } |
| 153 | while (size > a.length) |
| 154 | return a |
| 155 | } |
| 156 |
| 157 | // An extension more than one language uses is here only when one of them is by far the most |
| 158 | // common: .h is C++, whose grammar reads C too, .pl Perl and .fs F#. Left out are .m, as much |
| 159 | // Objective-C as MATLAB, and .s/.asm, which name no one dialect. |
| 160 | const EXTENSIONS = { |
| 161 | rs: 'rust', |
| 162 | h: 'cpp', |
| 163 | hpp: 'cpp', |
| 164 | hh: 'cpp', |
| 165 | hxx: 'cpp', |
| 166 | cc: 'cpp', |
| 167 | cxx: 'cpp', |
| 168 | inl: 'cpp', |
| 169 | ipp: 'cpp', |
| 170 | mm: 'objectivec', |
| 171 | mts: 'typescript', |
| 172 | cts: 'typescript', |
| 173 | mjs: 'javascript', |
| 174 | cjs: 'javascript', |
| 175 | pyi: 'python', |
| 176 | htm: 'html', |
| 177 | xaml: 'xml', |
| 178 | csproj: 'xml', |
| 179 | vcxproj: 'xml', |
| 180 | props: 'xml', |
| 181 | targets: 'xml', |
| 182 | resx: 'xml', |
| 183 | resw: 'xml', |
| 184 | xsd: 'xml', |
| 185 | xsl: 'xml', |
| 186 | xslt: 'xml', |
| 187 | config: 'xml', |
| 188 | manifest: 'xml', |
| 189 | appxmanifest: 'xml', |
| 190 | nuspec: 'xml', |
| 191 | plist: 'xml', |
| 192 | zsh: 'bash', |
| 193 | ksh: 'bash', |
| 194 | ps1: 'powershell', |
| 195 | psm1: 'powershell', |
| 196 | bat: 'batch', |
| 197 | cmd: 'batch', |
| 198 | mk: 'makefile', |
| 199 | pl: 'perl', |
| 200 | pm: 'perl', |
| 201 | rake: 'ruby', |
| 202 | gemspec: 'ruby', |
| 203 | jl: 'julia', |
| 204 | fs: 'fsharp', |
| 205 | fsx: 'fsharp', |
| 206 | ex: 'elixir', |
| 207 | exs: 'elixir', |
| 208 | erl: 'erlang', |
| 209 | hrl: 'erlang', |
| 210 | clj: 'clojure', |
| 211 | cljs: 'clojure', |
| 212 | cljc: 'clojure', |
| 213 | edn: 'clojure', |
| 214 | ml: 'ocaml', |
| 215 | mli: 'ocaml', |
| 216 | sv: 'verilog', |
| 217 | svh: 'verilog', |
| 218 | vhd: 'vhdl', |
| 219 | styl: 'stylus', |
| 220 | gql: 'graphql', |
| 221 | proto: 'protobuf', |
| 222 | tf: 'hcl', |
| 223 | tfvars: 'hcl', |
| 224 | htaccess: 'apacheconf', |
| 225 | } |
| 226 |
| 227 | async function generate() { |
| 228 |
| 229 | var tempPatterns = [] |
| 230 | var tempLanguages = {} |
| 231 | var tempTokens = [] |
| 232 | var tempGrammars = [] |
| 233 | var weak = new WeakMap |
| 234 |
| 235 | function flatten(grammar) { |
| 236 | var keys = {} |
| 237 |
| 238 | var cache = weak.get(grammar) |
| 239 | if (cache !== undefined) { |
| 240 | return cache |
| 241 | } |
| 242 |
| 243 | weak.set(grammar, keys) |
| 244 |
| 245 | var copy = grammar; |
| 246 | var rest = copy.rest; |
| 247 | if (rest) { |
| 248 | copy = {} |
| 249 |
| 250 | Object.keys(grammar).forEach(name => { |
| 251 | copy[name] = grammar[name] |
| 252 | }) |
| 253 |
| 254 | for (var token in rest) { |
| 255 | copy[token] = rest[token]; |
| 256 | } |
| 257 |
| 258 | delete copy.rest; |
| 259 | } |
| 260 |
| 261 | function sanitize(pattern) { |
| 262 | // Unsupported: |
| 263 | // UTF-16 ranges |
| 264 | // [^] => [\s\S] <- matches any character, including new line |
| 265 | // [] => ?? <- matches _empty_ string |
| 266 |
| 267 | // All the whitelisted languages have 0xFFFF as maximum range |
| 268 | // This is not the case for all the grammars supported by Prisma. |
| 269 |
| 270 | pattern = pattern.replaceAll("\\uFFFF", "\\xFF"); |
| 271 | pattern = pattern.replaceAll("[^]", "[\\s\\S]"); |
| 272 |
| 273 | // TODO: This just bruteforces the regex to work, but of course |
| 274 | // result may vary from the original one. |
| 275 | //static const boost::regex hex(R"(\\u([0-9a-fA-F]{4}))"); |
| 276 | //pattern = boost::regex_replace(pattern, hex, R"(\\xFF)"); |
| 277 |
| 278 | // TODO: Again, none of the whitelisted languages use [], but others do. |
| 279 | // Howhever, it is unclear to me how [] is supposed to work. |
| 280 | pattern = pattern.replaceAll("|[])", ")"); |
| 281 | pattern = pattern.replaceAll(":[]", ":"); |
| 282 |
| 283 | return escapeNonAscii(normalizeEscapes(pattern)) |
| 284 | } |
| 285 |
| 286 | for (var token in copy) { |
| 287 | if (!copy.hasOwnProperty(token) || !copy[token]) { |
| 288 | continue; |
| 289 | } |
| 290 |
| 291 | var patterns = copy[token]; |
| 292 | patterns = Array.isArray(patterns) ? patterns : [patterns]; |
| 293 |
| 294 | var indexes = [] |
| 295 |
| 296 | for (var j = 0; j < patterns.length; ++j) { |
| 297 | var patternObj = patterns[j]; |
| 298 | var inside = patternObj.inside; |
| 299 | var lookbehind = !!patternObj.lookbehind; |
| 300 | var greedy = !!patternObj.greedy; |
| 301 | var alias = patternObj.alias; |
| 302 |
| 303 | // Preserve every Prism CSS class in the existing alias string field. |
| 304 | alias = Array.isArray(alias) ? alias.join(' ') : alias; |
| 305 |
| 306 | var pattern = patternObj.pattern || patternObj; |
| 307 | var patternStr = sanitize(pattern.toString()) |
| 308 |
| 309 | if (lookbehind) { |
| 310 | patternStr += "l" |
| 311 | } |
| 312 | if (greedy) { |
| 313 | patternStr += "y" |
| 314 | } |
| 315 |
| 316 | var np |
| 317 |
| 318 | if (alias || inside) { |
| 319 | np = { |
| 320 | pattern: patternStr |
| 321 | } |
| 322 |
| 323 | if (alias) { |
| 324 | np.alias = alias |
| 325 | } |
| 326 | if (inside) { |
| 327 | np.inside = flatten(inside) |
| 328 | } |
| 329 |
| 330 | } else if (pattern instanceof RegExp) { |
| 331 | np = patternStr |
| 332 | } else { |
| 333 | debugger |
| 334 | } |
| 335 |
| 336 | tempPatterns.push(np) |
| 337 | indexes.push(np) |
| 338 | } |
| 339 |
| 340 | keys[token] = indexes |
| 341 | tempTokens.push(indexes) |
| 342 | } |
| 343 |
| 344 | tempGrammars.push(keys) |
| 345 | return keys |
| 346 | } |
| 347 |
| 348 | var unsupported = [ |
| 349 | "bsl", |
| 350 | "coq", |
| 351 | "gherkin", |
| 352 | "jexl", |
| 353 | "kumir", |
| 354 | "pure", |
| 355 | "purescript", |
| 356 | "turtle", |
| 357 | "sparql" // requires turtle |
| 358 | ] |
| 359 |
| 360 | await loadLanguages(Object.keys(components.languages)) |
| 361 | console.log(`\nLoaded all ${langNumber} languages`) |
| 362 | console.log("Processing...") |
| 363 |
| 364 | // Manually add local definitions |
| 365 | loadLocalLanguage('./components/prism-tl.js', 'typelanguage', 'TypeLanguage', 'tl') |
| 366 | loadLocalLanguage('./components/prism-tlb.js', 'tlb', 'TypeLanguage-Binary', 'tlb') |
| 367 | loadLocalLanguage('./components/prism-fift.js', 'fift', 'Fift', 'fift') |
| 368 | loadLocalLanguage('./components/prism-func.js', 'func', 'FunC', ['func', 'fc']) |
| 369 | loadLocalLanguage('./components/prism-tact.js', 'tact', 'Tact', 'tact') |
| 370 | loadLocalLanguage('./components/prism-tolk.js', 'tolk', 'Tolk', 'tolk') |
| 371 |
| 372 | // Changes to Prism's own grammars |
| 373 | include(fs.readFileSync('./components/prism-diff-embedded.js')) |
| 374 | include(fs.readFileSync('./components/prism-kotlin.js')) |
| 375 |
| 376 | Object.keys(Prism.languages).forEach(lng => { |
| 377 | if (unsupported.includes(lng) || !components.languages[lng]) { |
| 378 | return |
| 379 | } |
| 380 |
| 381 | tempLanguages[lng] = flatten(Prism.languages[lng]) |
| 382 | }) |
| 383 |
| 384 | var allTokens = uniqlo(tempTokens, isEqual) |
| 385 | var allGrammars = uniqlo(tempGrammars, isEqual) |
| 386 | var allPatterns = uniqlo(tempPatterns, isEqual) |
| 387 |
| 388 | Object.keys(tempLanguages).forEach(name => { |
| 389 | var find = allGrammars.find(x => isEqual(x, tempLanguages[name])) |
| 390 | if (find === undefined) { |
| 391 | debugger |
| 392 | } |
| 393 |
| 394 | tempLanguages[name] = find |
| 395 | }) |
| 396 |
| 397 | for (var i = 0; i < allPatterns.length; i++) { |
| 398 | if (allPatterns[i].inside) { |
| 399 | var find = allGrammars.find(x => isEqual(x, allPatterns[i].inside)) |
| 400 | if (find === undefined) { |
| 401 | debugger |
| 402 | } |
| 403 |
| 404 | allPatterns[i].inside = find |
| 405 | } |
| 406 | } |
| 407 |
| 408 | for (var i = 0; i < allTokens.length; i++) { |
| 409 | var token = allTokens[i] |
| 410 |
| 411 | for (var j = 0; j < token.length; j++) { |
| 412 | var find = allPatterns.find(x => isEqual(x, token[j])) |
| 413 | if (find === undefined) { |
| 414 | debugger |
| 415 | } |
| 416 |
| 417 | token[j] = find |
| 418 | } |
| 419 | } |
| 420 |
| 421 | for (var i = 0; i < allGrammars.length; i++) { |
| 422 | Object.keys(allGrammars[i]).forEach(name => { |
| 423 | var find = allTokens.find(x => isEqual(x, allGrammars[i][name])) |
| 424 | if (find === undefined) { |
| 425 | debugger |
| 426 | } |
| 427 |
| 428 | allGrammars[i][name] = find |
| 429 | }) |
| 430 | } |
| 431 |
| 432 | for (var i = 0; i < allPatterns.length; i++) { |
| 433 | if (allPatterns[i].inside) { |
| 434 | allPatterns[i].inside = allGrammars.indexOf(allPatterns[i].inside) |
| 435 | } |
| 436 | } |
| 437 |
| 438 | for (var i = 0; i < allTokens.length; i++) { |
| 439 | var token = allTokens[i] |
| 440 |
| 441 | for (var j = 0; j < token.length; j++) { |
| 442 | token[j] = allPatterns.indexOf(token[j]) |
| 443 | } |
| 444 | } |
| 445 |
| 446 | /*for (var i = 0; i < allGrammars.length; i++) { |
| 447 | Object.keys(allGrammars[i]).forEach(name => { |
| 448 | if (allGrammars[i][name].length == 1) { |
| 449 | allGrammars[i][name] = allGrammars[i][name][0] |
| 450 | } |
| 451 | }) |
| 452 | }*/ |
| 453 |
| 454 | for (var i = 0; i < allPatterns.length; i++) { |
| 455 | if (allPatterns[i].pattern) { |
| 456 | var patternStr = allPatterns[i].pattern + ","; |
| 457 | if (allPatterns[i].alias) { |
| 458 | patternStr += allPatterns[i].alias |
| 459 | } |
| 460 | patternStr += "," |
| 461 | if (allPatterns[i].inside) { |
| 462 | patternStr += allPatterns[i].inside |
| 463 | } |
| 464 |
| 465 | allPatterns[i] = patternStr |
| 466 | } else { |
| 467 | allPatterns[i] += ",," |
| 468 | } |
| 469 | } |
| 470 |
| 471 | var allLanguages = {} |
| 472 | var languageNames = {} |
| 473 |
| 474 | Object.keys(tempLanguages).forEach(name => { |
| 475 | var find = allGrammars.find(x => isEqual(x, tempLanguages[name])) |
| 476 | if (find === undefined) { |
| 477 | debugger |
| 478 | } |
| 479 |
| 480 | allLanguages[name] = allGrammars.indexOf(find) |
| 481 | languageNames[name] = components.languages[name].title |
| 482 |
| 483 | var alias = components.languages[name].alias |
| 484 | if (alias) { |
| 485 | alias = Array.isArray(alias) ? alias : [alias]; |
| 486 |
| 487 | for (const lng of alias) { |
| 488 | allLanguages[lng] = allGrammars.indexOf(find) |
| 489 |
| 490 | // aliasTitles only names the aliases whose title differs from the language's, |
| 491 | // so an alias missing from it takes the language title rather than no title: |
| 492 | // an empty one drops the alias out of SyntaxHighlighter::languages(). |
| 493 | const aliasTitles = components.languages[name].aliasTitles |
| 494 | languageNames[lng] = (aliasTitles && aliasTitles[lng]) || components.languages[name].title |
| 495 | } |
| 496 | } |
| 497 | }) |
| 498 |
| 499 | // File extensions that are no language name, so that a diff's file names find their |
| 500 | // language. Untitled, they stay out of the list a user picks a language from. |
| 501 | Object.entries(EXTENSIONS).forEach(([extension, name]) => { |
| 502 | if (allLanguages[name] === undefined) { |
| 503 | throw new Error(`.${extension} names ${name}, which is no language`) |
| 504 | } |
| 505 | if (allLanguages[extension] === undefined) { |
| 506 | allLanguages[extension] = allLanguages[name] |
| 507 | languageNames[extension] = '' |
| 508 | } else { |
| 509 | console.log(`EXTENSIONS.${extension} is not needed: it is a language name already`) |
| 510 | } |
| 511 | }) |
| 512 |
| 513 | var final = { |
| 514 | patterns: allPatterns, |
| 515 | grammars: allGrammars, |
| 516 | languages: allLanguages |
| 517 | } |
| 518 |
| 519 | const chunks = []; |
| 520 |
| 521 | const writeUint16 = i => chunks.push(new Uint16Array([i])) |
| 522 | const writeUint8 = i => chunks.push(new Uint8Array([i])) |
| 523 | const writeString = str => { |
| 524 | // one byte per code unit truncated anything above U+00FF; the length is the byte |
| 525 | // count, which is what the readers advance by |
| 526 | const bytes = new TextEncoder().encode(str) |
| 527 | if (bytes.length < 253) { |
| 528 | writeUint8(bytes.length) |
| 529 | } else { |
| 530 | writeUint8(254 & 0xFF) |
| 531 | writeUint8(bytes.length & 0xFF) |
| 532 | writeUint8((bytes.length >> 8) & 0xFF) |
| 533 | writeUint8((bytes.length >> 16) & 0xFF) |
| 534 | } |
| 535 | chunks.push(bytes) |
| 536 | } |
| 537 |
| 538 | // Patterns |
| 539 | writeUint16(allPatterns.length) |
| 540 |
| 541 | allPatterns.forEach(pattern => { |
| 542 | writeString(pattern) |
| 543 | }) |
| 544 |
| 545 | // Grammars |
| 546 | writeUint16(allGrammars.length) |
| 547 |
| 548 | for (var i = 0; i < allGrammars.length; i++) { |
| 549 | writeUint8(Object.keys(allGrammars[i]).length) |
| 550 |
| 551 | Object.keys(allGrammars[i]).forEach(name => { |
| 552 | writeString(name) |
| 553 | writeUint8(allGrammars[i][name].length) |
| 554 | allGrammars[i][name].forEach(id => { |
| 555 | writeUint16(id) |
| 556 | }) |
| 557 | }) |
| 558 | } |
| 559 |
| 560 | // Languages |
| 561 | writeUint16(Object.keys(allLanguages).length) |
| 562 |
| 563 | Object.keys(allLanguages).forEach(name => { |
| 564 | writeString(name) |
| 565 | if (languageNames[name]) { |
| 566 | writeString(languageNames[name]) |
| 567 | } else { |
| 568 | writeString("") |
| 569 | } |
| 570 | writeUint16(allLanguages[name]) |
| 571 | }) |
| 572 |
| 573 | const blob = new Blob(chunks, {type: 'application/octet-binary'}); |
| 574 | console.log(blob) |
| 575 | return blob; |
| 576 | } |
| 577 |
| 578 | async function saveBlob(blob, filename) { |
| 579 | console.log(`Saving ${blob} to ${filename}`) |
| 580 | const buffer = Buffer.from(await blob.arrayBuffer()) |
| 581 | fs.writeFileSync(filename, buffer) |
| 582 | } |
| 583 |
| 584 | const filepath = "libprisma/grammars.dat"; |
| 585 |
| 586 | generate().then(blob => { |
| 587 | saveBlob(blob, filepath).then(() => { |
| 588 | console.log("Done! Saved to " + filepath) |
| 589 | }) |
| 590 | }) |
| 591 |