[][src]Struct gf::pivot::tokenizer::TokenizerInstance

pub struct TokenizerInstance<'a> { /* fields omitted */ }

🟢 Instance of a tokenizer.


impl<'a> TokenizerInstance<'a>[src]

pub fn new(tokenizer: Tokenizer<'a>) -> Self[src]

Create an instance without specialization.

pub fn invoke(&mut self, _input: BitSlice<'_>) -> TokenizerResult[src]

Invoke the tokenizer on remaining input.

Tokens may be produced regardless of whether the result is Accept or Reject.

pub fn tokens(&self) -> &[AggregateValue<'a>][src]

Return tokens produced by the previous invocation.

Even if the previous invocation was not an Accept, some tokens might have been produced before the tokenizer stopped, and will be returned.

Trait Implementations

impl<'a> Clone for TokenizerInstance<'a>[src]

Auto Trait Implementations

impl<'a> RefUnwindSafe for TokenizerInstance<'a>

impl<'a> Send for TokenizerInstance<'a>

impl<'a> Sync for TokenizerInstance<'a>

impl<'a> Unpin for TokenizerInstance<'a>

impl<'a> UnwindSafe for TokenizerInstance<'a>

Blanket Implementations

impl<T> Any for T where
    T: 'static + ?Sized

impl<T> Borrow<T> for T where
    T: ?Sized

impl<T> BorrowMut<T> for T where
    T: ?Sized

impl<T> From<T> for T[src]

impl<T, U> Into<U> for T where
    U: From<T>, 

impl<T> ToOwned for T where
    T: Clone

type Owned = T

The resulting type after obtaining ownership.

impl<T, U> TryFrom<U> for T where
    U: Into<T>, 

type Error = Infallible

The type returned in the event of a conversion error.

impl<T, U> TryInto<U> for T where
    U: TryFrom<T>, 

type Error = <U as TryFrom<T>>::Error

The type returned in the event of a conversion error.