Skip to content

tokens

The building blocks of the TIM language,

Use pytermgui.markup.parsing.tokenize_markup and pytermgui.markup.parsing.tokenize_ansi to generate.

AliasToken dataclass

Bases: Token

A way to reference a set of tags from one central name.

Source code in pytermgui/markup/tokens.py 
260
261
262
263
264
265
266
@dataclass(frozen=True, repr=False)
class AliasToken(Token):
    """A way to reference a set of tags from one central name."""

    __slots__ = ("value",)

    value: str

ClearToken dataclass

Bases: Token

A tag-clearer.

These tokens are prefixed by /, and followed by the name of the tag they target.

To reset color information in the current text, use the /fg and /bg special tags. We cannot unset a specific color due to how the terminal works; all these do is "reset" the current stroke color to the default of the terminal.

Additionally, there are some other special identifiers:

  • /: Clears all tags, including styles, colors, macros, links and more.
  • /!: Clears all currently applied macros.
  • /~: Clears all currently applied links.
Source code in pytermgui/markup/tokens.py 
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
@dataclass(frozen=True, repr=False)
class ClearToken(Token):
    """A tag-clearer.

    These tokens are prefixed by `/`, and followed by the name of the tag they target.

    To reset color information in the current text, use the `/fg` and `/bg` special
    tags. We cannot unset a specific color due to how the terminal works; all these do
    is "reset" the current stroke color to the default of the terminal.

    Additionally, there are some other special identifiers:

    - `/`:  Clears all tags, including styles, colors, macros, links and more.
    - `/!`: Clears all currently applied macros.
    - `/~`: Clears all currently applied links.
    """

    __slots__ = ("value",)

    value: str

    @cached_property
    def prettified_markup(self) -> str:
        target = self.markup[1:]

        return f"[210 strikethrough]/[/fg]{target}[/]"

    def __eq__(self, other: object) -> bool:
        if not isinstance(other, Token):
            return False

        return super().__eq__(other) or all(
            obj.markup in ["/dim", "/bold"] for obj in [self, other]
        )

    def targets(  # pylint: disable=too-many-return-statements
        self, token: Token
    ) -> bool:
        """Returns True if this token targets the one given as an argument."""

        if token.is_clear() or token.is_cursor():
            return False

        if self.value in ("/", f"/{token.value}"):
            return True

        if token.is_hyperlink() and self.value == "/~":
            return True

        if token.is_macro() and self.value == "/!":
            return True

        if not Token.is_color(token):
            return False

        if self.value == "/fg" and not token.color.background:
            return True

        return self.value == "/bg" and token.color.background

targets(token)

Returns True if this token targets the one given as an argument.

Source code in pytermgui/markup/tokens.py 
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
def targets(  # pylint: disable=too-many-return-statements
    self, token: Token
) -> bool:
    """Returns True if this token targets the one given as an argument."""

    if token.is_clear() or token.is_cursor():
        return False

    if self.value in ("/", f"/{token.value}"):
        return True

    if token.is_hyperlink() and self.value == "/~":
        return True

    if token.is_macro() and self.value == "/!":
        return True

    if not Token.is_color(token):
        return False

    if self.value == "/fg" and not token.color.background:
        return True

    return self.value == "/bg" and token.color.background

ColorToken dataclass

Bases: Token

A color identifier.

It stores the markup that created it, as well as the pytermgui.colors.Color object that it represents.

Source code in pytermgui/markup/tokens.py 
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
@dataclass(frozen=True, repr=False)
class ColorToken(Token):
    """A color identifier.

    It stores the markup that created it, as well as the `pytermgui.colors.Color` object
    that it represents.
    """

    __slots__ = ("value",)

    value: str
    color: Color

    @cached_property
    def markup(self) -> str:
        return self.color.markup

    @cached_property
    def prettified_markup(self) -> str:
        clearer = "bg" if self.color.background else "fg"

        return f"[{self.markup}]{self.markup}[/{clearer}]"

CursorToken dataclass

Bases: Token

A cursor location.

These can be used to move the terminal's cursor.

Source code in pytermgui/markup/tokens.py 
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
@dataclass(frozen=True, repr=False)
class CursorToken(Token):
    """A cursor location.

    These can be used to move the terminal's cursor.
    """

    __slots__ = ("value", "y", "x")

    value: str
    y: int | None
    x: int | None

    def __iter__(self) -> Iterator[int | None]:
        return iter((self.y, self.x))

    def __repr__(self) -> str:
        return f"<{type(self).__name__} position: {(';'.join(map(str, self)))}>"

    def __fancy_repr__(self) -> Generator[FancyYield, None, None]:
        yield self.__repr__()

    @cached_property
    def markup(self) -> str:
        return f"({self.value})"

HLinkToken dataclass

Bases: Token

A terminal hyperlink.

See https://gist.github.com/egmontkob/eb114294efbcd5adb1944c9f3cb5feda.

Source code in pytermgui/markup/tokens.py 
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
@dataclass(frozen=True, repr=False)
class HLinkToken(Token):
    """A terminal hyperlink.

    See https://gist.github.com/egmontkob/eb114294efbcd5adb1944c9f3cb5feda.
    """

    __slots__ = ("value",)

    value: str

    @cached_property
    def markup(self) -> str:
        return f"~{self.value}"

    @cached_property
    def prettified_markup(self) -> str:
        return f"[{self.markup}]~[blue underline]{self.value}[/fg /underline /~]"

MacroToken dataclass

Bases: Token

A binding of a Python function to a markup name.

See the docs on information about syntax & semantics.

Source code in pytermgui/markup/tokens.py 
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
@dataclass(frozen=True, repr=False)
class MacroToken(Token):
    """A binding of a Python function to a markup name.

    See the docs on information about syntax & semantics.
    """

    __slots__ = ("value", "arguments")

    value: str
    arguments: tuple[str, ...]

    def __iter__(self) -> Iterator[Any]:
        return iter((self.value, self.arguments))

    @cached_property
    def prettified_markup(self) -> str:
        target = self.markup[1:]

        return f"[210 bold]![/]{target}"

    @cached_property
    def markup(self) -> str:
        return f"{self.value}" + (
            f"({':'.join(self.arguments)})" if len(self.arguments) > 0 else ""
        )

PlainToken dataclass

Bases: Token

A plain piece of text.

These are the parts of data in-between markup tag groups.

Source code in pytermgui/markup/tokens.py 
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
@dataclass(frozen=True, repr=False)
class PlainToken(Token):
    """A plain piece of text.

    These are the parts of data in-between markup tag groups.
    """

    __slots__ = ("value",)

    value: str

    def __repr__(self) -> str:
        return f"<{type(self).__name__} markup: {self.markup!r}>"

    def __fancy_repr__(self) -> Generator[FancyYield, None, None]:
        yield f"<{type(self).__name__} markup: {self.markup!r}>"

PseudoToken dataclass

Bases: Token

A token that can modify it's context, but doesn't hold information of its own.

Source code in pytermgui/markup/tokens.py 
136
137
138
139
140
141
142
143
144
@dataclass(frozen=True, repr=False)
class PseudoToken(Token):
    """A token that can modify it's context, but doesn't hold information of its own."""

    value: str

    @cached_property
    def prettified_markup(self) -> str:
        return f"[245 italic]{self.markup}[/]"

StyleToken dataclass

Bases: Token

A terminal-style identifier.

Most terminals support a set of 9 styles:

  • bold
  • dim
  • italic
  • underline
  • blink
  • blink2
  • inverse
  • invisible
  • strikethrough

This token will store the style it represents by its name in the value field. Note that other, less widely supported styles may be available; for an up-to-date list, run ptg -i pytermgui.markup.style_maps.STYLES. ```

Source code in pytermgui/markup/tokens.py 
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
@dataclass(frozen=True, repr=False)
class StyleToken(Token):
    """A terminal-style identifier.

    Most terminals support a set of 9 styles:

    - bold
    - dim
    - italic
    - underline
    - blink
    - blink2
    - inverse
    - invisible
    - strikethrough

    This token will store the style it represents by its name in the `value` field. Note
    that other, less widely supported styles *may* be available; for an up-to-date list,
    run `ptg -i pytermgui.markup.style_maps.STYLES`.
    ```

    """

    __slots__ = ("value",)

    value: str

Token

A piece of markup information.

All tokens must have at least a value field, and have markup and prettified_markup properties derived from it in some manner.

They are meant to be immutable (frozen), and generated by some tokenization. They are also static representations of the data in its pre-parsed form.

Source code in pytermgui/markup/tokens.py 
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
class Token:
    """A piece of markup information.

    All tokens must have at least a `value` field, and have `markup` and `prettified_markup`
    properties derived from it in some manner.

    They are meant to be immutable (frozen), and generated by some tokenization. They are also
    static representations of the data in its pre-parsed form.
    """

    value: str

    @cached_property
    def markup(self) -> str:
        """Returns markup representing this token."""

        return self.value

    @cached_property
    def prettified_markup(self) -> str:
        """Returns syntax-highlighted markup representing this token."""

        return f"[{self.markup}]{self.markup}[/{self.markup}]"

    def __eq__(self, other: object) -> bool:
        return isinstance(other, type(self)) and other.value == self.value

    def __repr__(self) -> str:
        return f"<{type(self).__name__} markup: '{self.markup}'>"

    def __fancy_repr__(self) -> Generator[FancyYield, None, None]:
        yield f"<{type(self).__name__} markup: "
        yield {
            "text": self.prettified_markup,
            "highlight": False,
        }
        yield ">"

    def is_plain(self) -> TypeGuard["PlainToken"]:
        """Returns True if this token is an instance of PlainToken."""

        return isinstance(self, PlainToken)

    def is_pseudo(self) -> TypeGuard["PseudoToken"]:
        """Returns True if this token is an instance of PseudoToken."""

        return isinstance(self, PseudoToken)

    def is_color(self) -> TypeGuard["ColorToken"]:
        """Returns True if this token is an instance of ColorToken."""

        return isinstance(self, ColorToken)

    def is_style(self) -> TypeGuard["StyleToken"]:
        """Returns True if this token is an instance of StyleToken."""

        return isinstance(self, StyleToken)

    def is_alias(self) -> TypeGuard["AliasToken"]:
        """Returns True if this token is an instance of AliasToken."""

        return isinstance(self, AliasToken)

    def is_macro(self) -> TypeGuard["MacroToken"]:
        """Returns True if this token is an instance of MacroToken."""

        return isinstance(self, MacroToken)

    def is_clear(self) -> TypeGuard["ClearToken"]:
        """Returns True if this token is an instance of ClearToken."""

        return isinstance(self, ClearToken)

    def is_hyperlink(self) -> TypeGuard["HLinkToken"]:
        """Returns True if this token is an instance of HLinkToken."""

        return isinstance(self, HLinkToken)

    def is_cursor(self) -> TypeGuard["CursorToken"]:
        """Returns True if this token is an instance of CursorToken."""

        return isinstance(self, CursorToken)

markup: str cached property

Returns markup representing this token.

prettified_markup: str cached property

Returns syntax-highlighted markup representing this token.

is_alias()

Returns True if this token is an instance of AliasToken.

Source code in pytermgui/markup/tokens.py 
92
93
94
95
def is_alias(self) -> TypeGuard["AliasToken"]:
    """Returns True if this token is an instance of AliasToken."""

    return isinstance(self, AliasToken)

is_clear()

Returns True if this token is an instance of ClearToken.

Source code in pytermgui/markup/tokens.py 
102
103
104
105
def is_clear(self) -> TypeGuard["ClearToken"]:
    """Returns True if this token is an instance of ClearToken."""

    return isinstance(self, ClearToken)

is_color()

Returns True if this token is an instance of ColorToken.

Source code in pytermgui/markup/tokens.py 
82
83
84
85
def is_color(self) -> TypeGuard["ColorToken"]:
    """Returns True if this token is an instance of ColorToken."""

    return isinstance(self, ColorToken)

is_cursor()

Returns True if this token is an instance of CursorToken.

Source code in pytermgui/markup/tokens.py 
112
113
114
115
def is_cursor(self) -> TypeGuard["CursorToken"]:
    """Returns True if this token is an instance of CursorToken."""

    return isinstance(self, CursorToken)

Returns True if this token is an instance of HLinkToken.

Source code in pytermgui/markup/tokens.py 
107
108
109
110
def is_hyperlink(self) -> TypeGuard["HLinkToken"]:
    """Returns True if this token is an instance of HLinkToken."""

    return isinstance(self, HLinkToken)

is_macro()

Returns True if this token is an instance of MacroToken.

Source code in pytermgui/markup/tokens.py 
 97
 98
 99
100
def is_macro(self) -> TypeGuard["MacroToken"]:
    """Returns True if this token is an instance of MacroToken."""

    return isinstance(self, MacroToken)

is_plain()

Returns True if this token is an instance of PlainToken.

Source code in pytermgui/markup/tokens.py 
72
73
74
75
def is_plain(self) -> TypeGuard["PlainToken"]:
    """Returns True if this token is an instance of PlainToken."""

    return isinstance(self, PlainToken)

is_pseudo()

Returns True if this token is an instance of PseudoToken.

Source code in pytermgui/markup/tokens.py 
77
78
79
80
def is_pseudo(self) -> TypeGuard["PseudoToken"]:
    """Returns True if this token is an instance of PseudoToken."""

    return isinstance(self, PseudoToken)

is_style()

Returns True if this token is an instance of StyleToken.

Source code in pytermgui/markup/tokens.py 
87
88
89
90
def is_style(self) -> TypeGuard["StyleToken"]:
    """Returns True if this token is an instance of StyleToken."""

    return isinstance(self, StyleToken)