671 lines
17 KiB
Plaintext
671 lines
17 KiB
Plaintext
|
/**
|
|||
|
|
* @typedef {import('micromark-util-types').Chunk} Chunk
|
|||
|
|
* @typedef {import('micromark-util-types').Code} Code
|
|||
|
|
* @typedef {import('micromark-util-types').Construct} Construct
|
|||
|
|
* @typedef {import('micromark-util-types').ConstructRecord} ConstructRecord
|
|||
|
|
* @typedef {import('micromark-util-types').Effects} Effects
|
|||
|
|
* @typedef {import('micromark-util-types').InitialConstruct} InitialConstruct
|
|||
|
|
* @typedef {import('micromark-util-types').ParseContext} ParseContext
|
|||
|
|
* @typedef {import('micromark-util-types').Point} Point
|
|||
|
|
* @typedef {import('micromark-util-types').State} State
|
|||
|
|
* @typedef {import('micromark-util-types').Token} Token
|
|||
|
|
* @typedef {import('micromark-util-types').TokenType} TokenType
|
|||
|
|
* @typedef {import('micromark-util-types').TokenizeContext} TokenizeContext
|
|||
|
|
*/
|
|||
|
|
|
|||
|
|
/**
|
|||
|
|
* @callback Restore
|
|||
|
|
* @returns {undefined}
|
|||
|
|
*
|
|||
|
|
* @typedef Info
|
|||
|
|
* @property {Restore} restore
|
|||
|
|
* @property {number} from
|
|||
|
|
*
|
|||
|
|
* @callback ReturnHandle
|
|||
|
|
* Handle a successful run.
|
|||
|
|
* @param {Construct} construct
|
|||
|
|
* @param {Info} info
|
|||
|
|
* @returns {undefined}
|
|||
|
|
*/
|
|||
|
|
|
|||
|
|
import createDebug from 'debug'
|
|||
|
|
import {markdownLineEnding} from 'micromark-util-character'
|
|||
|
|
import {push, splice} from 'micromark-util-chunked'
|
|||
|
|
import {resolveAll} from 'micromark-util-resolve-all'
|
|||
|
|
import {codes, values} from 'micromark-util-symbol'
|
|||
|
|
import {ok as assert} from 'devlop'
|
|||
|
|
|
|||
|
|
const debug = createDebug('micromark')
|
|||
|
|
|
|||
|
|
/**
|
|||
|
|
* Create a tokenizer.
|
|||
|
|
* Tokenizers deal with one type of data (e.g., containers, flow, text).
|
|||
|
|
* The parser is the object dealing with it all.
|
|||
|
|
* `initialize` works like other constructs, except that only its `tokenize`
|
|||
|
|
* function is used, in which case it doesn’t receive an `ok` or `nok`.
|
|||
|
|
* `from` can be given to set the point before the first character, although
|
|||
|
|
* when further lines are indented, they must be set with `defineSkip`.
|
|||
|
|
*
|
|||
|
|
* @param {ParseContext} parser
|
|||
|
|
* @param {InitialConstruct} initialize
|
|||
|
|
* @param {Omit<Point, '_bufferIndex' | '_index'> | undefined} [from]
|
|||
|
|
* @returns {TokenizeContext}
|
|||
|
|
*/
|
|||
|
|
export function createTokenizer(parser, initialize, from) {
|
|||
|
|
/** @type {Point} */
|
|||
|
|
let point = Object.assign(
|
|||
|
|
from ? Object.assign({}, from) : {line: 1, column: 1, offset: 0},
|
|||
|
|
{_index: 0, _bufferIndex: -1}
|
|||
|
|
)
|
|||
|
|
/** @type {Record<string, number>} */
|
|||
|
|
const columnStart = {}
|
|||
|
|
/** @type {Array<Construct>} */
|
|||
|
|
const resolveAllConstructs = []
|
|||
|
|
/** @type {Array<Chunk>} */
|
|||
|
|
let chunks = []
|
|||
|
|
/** @type {Array<Token>} */
|
|||
|
|
let stack = []
|
|||
|
|
/** @type {boolean | undefined} */
|
|||
|
|
let consumed = true
|
|||
|
|
|
|||
|
|
/**
|
|||
|
|
* Tools used for tokenizing.
|
|||
|
|
*
|
|||
|
|
* @type {Effects}
|
|||
|
|
*/
|
|||
|
|
const effects = {
|
|||
|
|
consume,
|
|||
|
|
enter,
|
|||
|
|
exit,
|
|||
|
|
attempt: constructFactory(onsuccessfulconstruct),
|
|||
|
|
check: constructFactory(onsuccessfulcheck),
|
|||
|
|
interrupt: constructFactory(onsuccessfulcheck, {interrupt: true})
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
/**
|
|||
|
|
* State and tools for resolving and serializing.
|
|||
|
|
*
|
|||
|
|
* @type {TokenizeContext}
|
|||
|
|
*/
|
|||
|
|
const context = {
|
|||
|
|
previous: codes.eof,
|
|||
|
|
code: codes.eof,
|
|||
|
|
containerState: {},
|
|||
|
|
events: [],
|
|||
|
|
parser,
|
|||
|
|
sliceStream,
|
|||
|
|
sliceSerialize,
|
|||
|
|
now,
|
|||
|
|
defineSkip,
|
|||
|
|
write
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
/**
|
|||
|
|
* The state function.
|
|||
|
|
*
|
|||
|
|
* @type {State | undefined}
|
|||
|
|
*/
|
|||
|
|
let state = initialize.tokenize.call(context, effects)
|
|||
|
|
|
|||
|
|
/**
|
|||
|
|
* Track which character we expect to be consumed, to catch bugs.
|
|||
|
|
*
|
|||
|
|
* @type {Code}
|
|||
|
|
*/
|
|||
|
|
let expectedCode
|
|||
|
|
|
|||
|
|
if (initialize.resolveAll) {
|
|||
|
|
resolveAllConstructs.push(initialize)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
return context
|
|||
|
|
|
|||
|
|
/** @type {TokenizeContext['write']} */
|
|||
|
|
function write(slice) {
|
|||
|
|
chunks = push(chunks, slice)
|
|||
|
|
|
|||
|
|
main()
|
|||
|
|
|
|||
|
|
// Exit if we’re not done, resolve might change stuff.
|
|||
|
|
if (chunks[chunks.length - 1] !== codes.eof) {
|
|||
|
|
return []
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
addResult(initialize, 0)
|
|||
|
|
|
|||
|
|
// Otherwise, resolve, and exit.
|
|||
|
|
context.events = resolveAll(resolveAllConstructs, context.events, context)
|
|||
|
|
|
|||
|
|
return context.events
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
//
|
|||
|
|
// Tools.
|
|||
|
|
//
|
|||
|
|
|
|||
|
|
/** @type {TokenizeContext['sliceSerialize']} */
|
|||
|
|
function sliceSerialize(token, expandTabs) {
|
|||
|
|
return serializeChunks(sliceStream(token), expandTabs)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
/** @type {TokenizeContext['sliceStream']} */
|
|||
|
|
function sliceStream(token) {
|
|||
|
|
return sliceChunks(chunks, token)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
/** @type {TokenizeContext['now']} */
|
|||
|
|
function now() {
|
|||
|
|
// This is a hot path, so we clone manually instead of `Object.assign({}, point)`
|
|||
|
|
const {line, column, offset, _index, _bufferIndex} = point
|
|||
|
|
return {line, column, offset, _index, _bufferIndex}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
/** @type {TokenizeContext['defineSkip']} */
|
|||
|
|
function defineSkip(value) {
|
|||
|
|
columnStart[value.line] = value.column
|
|||
|
|
accountForPotentialSkip()
|
|||
|
|
debug('position: define skip: `%j`', point)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
//
|
|||
|
|
// State management.
|
|||
|
|
//
|
|||
|
|
|
|||
|
|
/**
|
|||
|
|
* Main loop (note that `_index` and `_bufferIndex` in `point` are modified by
|
|||
|
|
* `consume`).
|
|||
|
|
* Here is where we walk through the chunks, which either include strings of
|
|||
|
|
* several characters, or numerical character codes.
|
|||
|
|
* The reason to do this in a loop instead of a call is so the stack can
|
|||
|
|
* drain.
|
|||
|
|
*
|
|||
|
|
* @returns {undefined}
|
|||
|
|
*/
|
|||
|
|
function main() {
|
|||
|
|
/** @type {number} */
|
|||
|
|
let chunkIndex
|
|||
|
|
|
|||
|
|
while (point._index < chunks.length) {
|
|||
|
|
const chunk = chunks[point._index]
|
|||
|
|
|
|||
|
|
// If we’re in a buffer chunk, loop through it.
|
|||
|
|
if (typeof chunk === 'string') {
|
|||
|
|
chunkIndex = point._index
|
|||
|
|
|
|||
|
|
if (point._bufferIndex < 0) {
|
|||
|
|
point._bufferIndex = 0
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
while (
|
|||
|
|
point._index === chunkIndex &&
|
|||
|
|
point._bufferIndex < chunk.length
|
|||
|
|
) {
|
|||
|
|
go(chunk.charCodeAt(point._bufferIndex))
|
|||
|
|
}
|
|||
|
|
} else {
|
|||
|
|
go(chunk)
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
/**
|
|||
|
|
* Deal with one code.
|
|||
|
|
*
|
|||
|
|
* @param {Code} code
|
|||
|
|
* @returns {undefined}
|
|||
|
|
*/
|
|||
|
|
function go(code) {
|
|||
|
|
assert(consumed === true, 'expected character to be consumed')
|
|||
|
|
consumed = undefined
|
|||
|
|
debug('main: passing `%s` to %s', code, state && state.name)
|
|||
|
|
expectedCode = code
|
|||
|
|
assert(typeof state === 'function', 'expected state')
|
|||
|
|
state = state(code)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
/** @type {Effects['consume']} */
|
|||
|
|
function consume(code) {
|
|||
|
|
assert(code === expectedCode, 'expected given code to equal expected code')
|
|||
|
|
|
|||
|
|
debug('consume: `%s`', code)
|
|||
|
|
|
|||
|
|
assert(
|
|||
|
|
consumed === undefined,
|
|||
|
|
'expected code to not have been consumed: this might be because `return x(code)` instead of `return x` was used'
|
|||
|
|
)
|
|||
|
|
assert(
|
|||
|
|
code === null
|
|||
|
|
? context.events.length === 0 ||
|
|||
|
|
context.events[context.events.length - 1][0] === 'exit'
|
|||
|
|
: context.events[context.events.length - 1][0] === 'enter',
|
|||
|
|
'expected last token to be open'
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
if (markdownLineEnding(code)) {
|
|||
|
|
point.line++
|
|||
|
|
point.column = 1
|
|||
|
|
point.offset += code === codes.carriageReturnLineFeed ? 2 : 1
|
|||
|
|
accountForPotentialSkip()
|
|||
|
|
debug('position: after eol: `%j`', point)
|
|||
|
|
} else if (code !== codes.virtualSpace) {
|
|||
|
|
point.column++
|
|||
|
|
point.offset++
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// Not in a string chunk.
|
|||
|
|
if (point._bufferIndex < 0) {
|
|||
|
|
point._index++
|
|||
|
|
} else {
|
|||
|
|
point._bufferIndex++
|
|||
|
|
|
|||
|
|
// At end of string chunk.
|
|||
|
|
// @ts-expect-error Points w/ non-negative `_bufferIndex` reference
|
|||
|
|
// strings.
|
|||
|
|
if (point._bufferIndex === chunks[point._index].length) {
|
|||
|
|
point._bufferIndex = -1
|
|||
|
|
point._index++
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// Expose the previous character.
|
|||
|
|
context.previous = code
|
|||
|
|
|
|||
|
|
// Mark as consumed.
|
|||
|
|
consumed = true
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
/** @type {Effects['enter']} */
|
|||
|
|
function enter(type, fields) {
|
|||
|
|
/** @type {Token} */
|
|||
|
|
// @ts-expect-error Patch instead of assign required fields to help GC.
|
|||
|
|
const token = fields || {}
|
|||
|
|
token.type = type
|
|||
|
|
token.start = now()
|
|||
|
|
|
|||
|
|
assert(typeof type === 'string', 'expected string type')
|
|||
|
|
assert(type.length > 0, 'expected non-empty string')
|
|||
|
|
debug('enter: `%s`', type)
|
|||
|
|
|
|||
|
|
context.events.push(['enter', token, context])
|
|||
|
|
|
|||
|
|
stack.push(token)
|
|||
|
|
|
|||
|
|
return token
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
/** @type {Effects['exit']} */
|
|||
|
|
function exit(type) {
|
|||
|
|
assert(typeof type === 'string', 'expected string type')
|
|||
|
|
assert(type.length > 0, 'expected non-empty string')
|
|||
|
|
|
|||
|
|
const token = stack.pop()
|
|||
|
|
assert(token, 'cannot close w/o open tokens')
|
|||
|
|
token.end = now()
|
|||
|
|
|
|||
|
|
assert(type === token.type, 'expected exit token to match current token')
|
|||
|
|
|
|||
|
|
assert(
|
|||
|
|
!(
|
|||
|
|
token.start._index === token.end._index &&
|
|||
|
|
token.start._bufferIndex === token.end._bufferIndex
|
|||
|
|
),
|
|||
|
|
'expected non-empty token (`' + type + '`)'
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
debug('exit: `%s`', token.type)
|
|||
|
|
context.events.push(['exit', token, context])
|
|||
|
|
|
|||
|
|
return token
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
/**
|
|||
|
|
* Use results.
|
|||
|
|
*
|
|||
|
|
* @type {ReturnHandle}
|
|||
|
|
*/
|
|||
|
|
function onsuccessfulconstruct(construct, info) {
|
|||
|
|
addResult(construct, info.from)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
/**
|
|||
|
|
* Discard results.
|
|||
|
|
*
|
|||
|
|
* @type {ReturnHandle}
|
|||
|
|
*/
|
|||
|
|
function onsuccessfulcheck(_, info) {
|
|||
|
|
info.restore()
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
/**
|
|||
|
|
* Factory to attempt/check/interrupt.
|
|||
|
|
*
|
|||
|
|
* @param {ReturnHandle} onreturn
|
|||
|
|
* @param {{interrupt?: boolean | undefined} | undefined} [fields]
|
|||
|
|
*/
|
|||
|
|
function constructFactory(onreturn, fields) {
|
|||
|
|
return hook
|
|||
|
|
|
|||
|
|
/**
|
|||
|
|
* Handle either an object mapping codes to constructs, a list of
|
|||
|
|
* constructs, or a single construct.
|
|||
|
|
*
|
|||
|
|
* @param {Array<Construct> | Construct | ConstructRecord} constructs
|
|||
|
|
* @param {State} returnState
|
|||
|
|
* @param {State | undefined} [bogusState]
|
|||
|
|
* @returns {State}
|
|||
|
|
*/
|
|||
|
|
function hook(constructs, returnState, bogusState) {
|
|||
|
|
/** @type {Array<Construct>} */
|
|||
|
|
let listOfConstructs
|
|||
|
|
/** @type {number} */
|
|||
|
|
let constructIndex
|
|||
|
|
/** @type {Construct} */
|
|||
|
|
let currentConstruct
|
|||
|
|
/** @type {Info} */
|
|||
|
|
let info
|
|||
|
|
|
|||
|
|
return Array.isArray(constructs)
|
|||
|
|
? /* c8 ignore next 1 */
|
|||
|
|
handleListOfConstructs(constructs)
|
|||
|
|
: 'tokenize' in constructs
|
|||
|
|
? // @ts-expect-error Looks like a construct.
|
|||
|
|
handleListOfConstructs([constructs])
|
|||
|
|
: handleMapOfConstructs(constructs)
|
|||
|
|
|
|||
|
|
/**
|
|||
|
|
* Handle a list of construct.
|
|||
|
|
*
|
|||
|
|
* @param {ConstructRecord} map
|
|||
|
|
* @returns {State}
|
|||
|
|
*/
|
|||
|
|
function handleMapOfConstructs(map) {
|
|||
|
|
return start
|
|||
|
|
|
|||
|
|
/** @type {State} */
|
|||
|
|
function start(code) {
|
|||
|
|
const def = code !== null && map[code]
|
|||
|
|
const all = code !== null && map.null
|
|||
|
|
const list = [
|
|||
|
|
// To do: add more extension tests.
|
|||
|
|
/* c8 ignore next 2 */
|
|||
|
|
...(Array.isArray(def) ? def : def ? [def] : []),
|
|||
|
|
...(Array.isArray(all) ? all : all ? [all] : [])
|
|||
|
|
]
|
|||
|
|
|
|||
|
|
return handleListOfConstructs(list)(code)
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
/**
|
|||
|
|
* Handle a list of construct.
|
|||
|
|
*
|
|||
|
|
* @param {Array<Construct>} list
|
|||
|
|
* @returns {State}
|
|||
|
|
*/
|
|||
|
|
function handleListOfConstructs(list) {
|
|||
|
|
listOfConstructs = list
|
|||
|
|
constructIndex = 0
|
|||
|
|
|
|||
|
|
if (list.length === 0) {
|
|||
|
|
assert(bogusState, 'expected `bogusState` to be given')
|
|||
|
|
return bogusState
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
return handleConstruct(list[constructIndex])
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
/**
|
|||
|
|
* Handle a single construct.
|
|||
|
|
*
|
|||
|
|
* @param {Construct} construct
|
|||
|
|
* @returns {State}
|
|||
|
|
*/
|
|||
|
|
function handleConstruct(construct) {
|
|||
|
|
return start
|
|||
|
|
|
|||
|
|
/** @type {State} */
|
|||
|
|
function start(code) {
|
|||
|
|
// To do: not needed to store if there is no bogus state, probably?
|
|||
|
|
// Currently doesn’t work because `inspect` in document does a check
|
|||
|
|
// w/o a bogus, which doesn’t make sense. But it does seem to help perf
|
|||
|
|
// by not storing.
|
|||
|
|
info = store()
|
|||
|
|
currentConstruct = construct
|
|||
|
|
|
|||
|
|
if (!construct.partial) {
|
|||
|
|
context.currentConstruct = construct
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// Always populated by defaults.
|
|||
|
|
assert(
|
|||
|
|
context.parser.constructs.disable.null,
|
|||
|
|
'expected `disable.null` to be populated'
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
if (
|
|||
|
|
construct.name &&
|
|||
|
|
context.parser.constructs.disable.null.includes(construct.name)
|
|||
|
|
) {
|
|||
|
|
return nok(code)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
return construct.tokenize.call(
|
|||
|
|
// If we do have fields, create an object w/ `context` as its
|
|||
|
|
// prototype.
|
|||
|
|
// This allows a “live binding”, which is needed for `interrupt`.
|
|||
|
|
fields ? Object.assign(Object.create(context), fields) : context,
|
|||
|
|
effects,
|
|||
|
|
ok,
|
|||
|
|
nok
|
|||
|
|
)(code)
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
/** @type {State} */
|
|||
|
|
function ok(code) {
|
|||
|
|
assert(code === expectedCode, 'expected code')
|
|||
|
|
consumed = true
|
|||
|
|
onreturn(currentConstruct, info)
|
|||
|
|
return returnState
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
/** @type {State} */
|
|||
|
|
function nok(code) {
|
|||
|
|
assert(code === expectedCode, 'expected code')
|
|||
|
|
consumed = true
|
|||
|
|
info.restore()
|
|||
|
|
|
|||
|
|
if (++constructIndex < listOfConstructs.length) {
|
|||
|
|
return handleConstruct(listOfConstructs[constructIndex])
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
return bogusState
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
/**
|
|||
|
|
* @param {Construct} construct
|
|||
|
|
* @param {number} from
|
|||
|
|
* @returns {undefined}
|
|||
|
|
*/
|
|||
|
|
function addResult(construct, from) {
|
|||
|
|
if (construct.resolveAll && !resolveAllConstructs.includes(construct)) {
|
|||
|
|
resolveAllConstructs.push(construct)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
if (construct.resolve) {
|
|||
|
|
splice(
|
|||
|
|
context.events,
|
|||
|
|
from,
|
|||
|
|
context.events.length - from,
|
|||
|
|
construct.resolve(context.events.slice(from), context)
|
|||
|
|
)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
if (construct.resolveTo) {
|
|||
|
|
context.events = construct.resolveTo(context.events, context)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
assert(
|
|||
|
|
construct.partial ||
|
|||
|
|
context.events.length === 0 ||
|
|||
|
|
context.events[context.events.length - 1][0] === 'exit',
|
|||
|
|
'expected last token to end'
|
|||
|
|
)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
/**
|
|||
|
|
* Store state.
|
|||
|
|
*
|
|||
|
|
* @returns {Info}
|
|||
|
|
*/
|
|||
|
|
function store() {
|
|||
|
|
const startPoint = now()
|
|||
|
|
const startPrevious = context.previous
|
|||
|
|
const startCurrentConstruct = context.currentConstruct
|
|||
|
|
const startEventsIndex = context.events.length
|
|||
|
|
const startStack = Array.from(stack)
|
|||
|
|
|
|||
|
|
return {restore, from: startEventsIndex}
|
|||
|
|
|
|||
|
|
/**
|
|||
|
|
* Restore state.
|
|||
|
|
*
|
|||
|
|
* @returns {undefined}
|
|||
|
|
*/
|
|||
|
|
function restore() {
|
|||
|
|
point = startPoint
|
|||
|
|
context.previous = startPrevious
|
|||
|
|
context.currentConstruct = startCurrentConstruct
|
|||
|
|
context.events.length = startEventsIndex
|
|||
|
|
stack = startStack
|
|||
|
|
accountForPotentialSkip()
|
|||
|
|
debug('position: restore: `%j`', point)
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
/**
|
|||
|
|
* Move the current point a bit forward in the line when it’s on a column
|
|||
|
|
* skip.
|
|||
|
|
*
|
|||
|
|
* @returns {undefined}
|
|||
|
|
*/
|
|||
|
|
function accountForPotentialSkip() {
|
|||
|
|
if (point.line in columnStart && point.column < 2) {
|
|||
|
|
point.column = columnStart[point.line]
|
|||
|
|
point.offset += columnStart[point.line] - 1
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
/**
|
|||
|
|
* Get the chunks from a slice of chunks in the range of a token.
|
|||
|
|
*
|
|||
|
|
* @param {Array<Chunk>} chunks
|
|||
|
|
* @param {Pick<Token, 'end' | 'start'>} token
|
|||
|
|
* @returns {Array<Chunk>}
|
|||
|
|
*/
|
|||
|
|
function sliceChunks(chunks, token) {
|
|||
|
|
const startIndex = token.start._index
|
|||
|
|
const startBufferIndex = token.start._bufferIndex
|
|||
|
|
const endIndex = token.end._index
|
|||
|
|
const endBufferIndex = token.end._bufferIndex
|
|||
|
|
/** @type {Array<Chunk>} */
|
|||
|
|
let view
|
|||
|
|
|
|||
|
|
if (startIndex === endIndex) {
|
|||
|
|
assert(endBufferIndex > -1, 'expected non-negative end buffer index')
|
|||
|
|
assert(startBufferIndex > -1, 'expected non-negative start buffer index')
|
|||
|
|
// @ts-expect-error `_bufferIndex` is used on string chunks.
|
|||
|
|
view = [chunks[startIndex].slice(startBufferIndex, endBufferIndex)]
|
|||
|
|
} else {
|
|||
|
|
view = chunks.slice(startIndex, endIndex)
|
|||
|
|
|
|||
|
|
if (startBufferIndex > -1) {
|
|||
|
|
const head = view[0]
|
|||
|
|
if (typeof head === 'string') {
|
|||
|
|
view[0] = head.slice(startBufferIndex)
|
|||
|
|
} else {
|
|||
|
|
assert(startBufferIndex === 0, 'expected `startBufferIndex` to be `0`')
|
|||
|
|
view.shift()
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
if (endBufferIndex > 0) {
|
|||
|
|
// @ts-expect-error `_bufferIndex` is used on string chunks.
|
|||
|
|
view.push(chunks[endIndex].slice(0, endBufferIndex))
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
return view
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
/**
|
|||
|
|
* Get the string value of a slice of chunks.
|
|||
|
|
*
|
|||
|
|
* @param {Array<Chunk>} chunks
|
|||
|
|
* @param {boolean | undefined} [expandTabs=false]
|
|||
|
|
* @returns {string}
|
|||
|
|
*/
|
|||
|
|
function serializeChunks(chunks, expandTabs) {
|
|||
|
|
let index = -1
|
|||
|
|
/** @type {Array<string>} */
|
|||
|
|
const result = []
|
|||
|
|
/** @type {boolean | undefined} */
|
|||
|
|
let atTab
|
|||
|
|
|
|||
|
|
while (++index < chunks.length) {
|
|||
|
|
const chunk = chunks[index]
|
|||
|
|
/** @type {string} */
|
|||
|
|
let value
|
|||
|
|
|
|||
|
|
if (typeof chunk === 'string') {
|
|||
|
|
value = chunk
|
|||
|
|
} else
|
|||
|
|
switch (chunk) {
|
|||
|
|
case codes.carriageReturn: {
|
|||
|
|
value = values.cr
|
|||
|
|
|
|||
|
|
break
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
case codes.lineFeed: {
|
|||
|
|
value = values.lf
|
|||
|
|
|
|||
|
|
break
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
case codes.carriageReturnLineFeed: {
|
|||
|
|
value = values.cr + values.lf
|
|||
|
|
|
|||
|
|
break
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
case codes.horizontalTab: {
|
|||
|
|
value = expandTabs ? values.space : values.ht
|
|||
|
|
|
|||
|
|
break
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
case codes.virtualSpace: {
|
|||
|
|
if (!expandTabs && atTab) continue
|
|||
|
|
value = values.space
|
|||
|
|
|
|||
|
|
break
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
default: {
|
|||
|
|
assert(typeof chunk === 'number', 'expected number')
|
|||
|
|
// Currently only replacement character.
|
|||
|
|
value = String.fromCharCode(chunk)
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
atTab = chunk === codes.horizontalTab
|
|||
|
|
result.push(value)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
return result.join('')
|
|||
|
|
}
|