Coverage for C:\Repos\leo-editor\leo\core\leoColorizer.py: 29%
1729 statements
« prev ^ index » next coverage.py v6.4, created at 2022-05-24 10:21 -0500
« prev ^ index » next coverage.py v6.4, created at 2022-05-24 10:21 -0500
1# -*- coding: utf-8 -*-
2#@+leo-ver=5-thin
3#@+node:ekr.20140827092102.18574: * @file leoColorizer.py
4#@@first
5"""All colorizing code for Leo."""
7# Indicated code are copyright (c) Jupyter Development Team.
8# Distributed under the terms of the Modified BSD License.
10#@+<< imports >>
11#@+node:ekr.20140827092102.18575: ** << imports >> (leoColorizer.py)
12import re
13import string
14import time
15from typing import Any, Callable, Dict, List, Tuple
16#
17# Third-part tools.
18try:
19 import pygments # type:ignore
20except ImportError:
21 pygments = None # type:ignore
22#
23# Leo imports...
24from leo.core import leoGlobals as g
26from leo.core.leoColor import leo_color_database
27#
28# Qt imports. May fail from the bridge.
29try: # #1973
30 from leo.core.leoQt import Qsci, QtGui, QtWidgets
31 from leo.core.leoQt import UnderlineStyle, Weight # #2330
32except Exception:
33 Qsci = QtGui = QtWidgets = None
34 UnderlineStyle = Weight = None
35#@-<< imports >>
36#@+others
37#@+node:ekr.20190323044524.1: ** function: make_colorizer
38def make_colorizer(c, widget):
39 """Return an instance of JEditColorizer or PygmentsColorizer."""
40 use_pygments = pygments and c.config.getBool('use-pygments', default=False)
41 if use_pygments:
42 return PygmentsColorizer(c, widget)
43 return JEditColorizer(c, widget)
44#@+node:ekr.20170127141855.1: ** class BaseColorizer
45class BaseColorizer:
46 """The base class for all Leo colorizers."""
47 #@+others
48 #@+node:ekr.20220317050513.1: *3* BaseColorizer: birth
49 #@+node:ekr.20190324044744.1: *4* BaseColorizer.__init__
50 def __init__(self, c, widget=None):
51 """ctor for BaseColorizer class."""
52 # Copy args...
53 self.c = c
54 self.widget = widget
55 if widget: # #503: widget may be None during unit tests.
56 widget.leo_colorizer = self
57 # Configuration dicts...
58 self.configDict: Dict[str, Any] = {} # Keys are tags, values are colors (names or values).
59 self.configUnderlineDict: Dict[str, bool] = {} # Keys are tags, values are bools.
60 # Common state ivars...
61 self.enabled = False # Per-node enable/disable flag set by updateSyntaxColorer.
62 self.highlighter = g.NullObject() # May be overridden in subclass...
63 self.language = 'python' # set by scanLanguageDirectives.
64 self.prev = None # Used by setTag.
65 self.showInvisibles = False
66 # Statistics....
67 self.count = 0
68 self.full_recolor_count = 0 # For unit tests.
69 self.recolorCount = 0
70 # For traces...
71 self.matcher_name = ''
72 self.rulesetName = ''
73 self.delegate_name = ''
74 #@+node:ekr.20190324045134.1: *4* BaseColorizer.init
75 def init(self):
76 """May be over-ridden in subclasses."""
77 pass
78 #@+node:ekr.20110605121601.18578: *4* BaseColorizer.configureTags & helpers
79 def configureTags(self):
80 """Configure all tags."""
81 self.configure_fonts()
82 self.configure_colors()
83 self.configure_variable_tags()
84 if 'coloring' in g.app.debug:
85 g.printObj(self.configDict, tag='configDict')
86 g.printObj(self.configUnderlineDict, tag='configUnderlineDict')
87 #@+node:ekr.20190324172632.1: *5* BaseColorizer.configure_colors
88 def configure_colors(self):
89 """Configure all colors in the default colors dict."""
90 c = self.c
91 # getColor puts the color name in standard form:
92 # color = color.replace(' ', '').lower().strip()
93 getColor = c.config.getColor
94 for key in sorted(self.default_colors_dict.keys()):
95 option_name, default_color = self.default_colors_dict[key]
96 color = (
97 getColor(f"{self.language}_{option_name}") or
98 getColor(option_name) or
99 default_color
100 )
101 self.configDict[key] = color
102 #@+node:ekr.20190324172242.1: *5* BaseColorizer.configure_fonts & helper
103 def configure_fonts(self):
104 """Configure all fonts in the default fonts dict."""
105 c = self.c
106 isQt = g.app.gui.guiName().startswith('qt')
107 #
108 # Get the default body font.
109 defaultBodyfont = self.fonts.get('default_body_font')
110 if not defaultBodyfont:
111 defaultBodyfont = c.config.getFontFromParams(
112 "body_text_font_family", "body_text_font_size",
113 "body_text_font_slant", "body_text_font_weight",
114 c.config.defaultBodyFontSize)
115 self.fonts['default_body_font'] = defaultBodyfont
116 #
117 # Set all fonts.
118 for key in sorted(self.default_font_dict.keys()):
119 option_name = self.default_font_dict[key]
120 # Find language specific setting before general setting.
121 table = (
122 f"{self.language}_{option_name}",
123 option_name,
124 )
125 for name in table:
126 font = self.fonts.get(name)
127 if font:
128 break
129 font = self.find_font(key, name)
130 if font:
131 self.fonts[key] = font
132 if isQt and key == 'url':
133 font.setUnderline(True)
134 # #1919: This really isn't correct.
135 self.configure_hard_tab_width(font)
136 break
137 else:
138 # Neither setting exists.
139 self.fonts[key] = None # Essential
140 #@+node:ekr.20190326034006.1: *6* BaseColorizer.find_font
141 # Keys are key::settings_names, values are cumulative font size.
142 zoom_dict: Dict[str, int] = {}
144 def find_font(self, key, setting_name):
145 """
146 Return the font for the given setting name.
147 """
148 trace = 'zoom' in g.app.debug
149 c, get = self.c, self.c.config.get
150 default_size = c.config.defaultBodyFontSize
151 for name in (setting_name, setting_name.rstrip('_font')):
152 size_error = False
153 family = get(name + '_family', 'family')
154 size = get(name + '_size', 'size')
155 slant = get(name + '_slant', 'slant')
156 weight = get(name + '_weight', 'weight')
157 if family or slant or weight or size:
158 family = family or g.app.config.defaultFontFamily
159 key = f"{key}::{setting_name}"
160 if key in self.zoom_dict:
161 old_size = self.zoom_dict.get(key)
162 else:
163 # It's a good idea to set size explicitly.
164 old_size = size or default_size
165 if isinstance(old_size, str):
166 # All settings should be in units of points.
167 try:
168 if old_size.endswith(('pt', 'px'),):
169 old_size = int(old_size[:-2])
170 else:
171 old_size = int(old_size)
172 except ValueError:
173 size_error = True
174 elif not isinstance(old_size, int):
175 size_error = True
176 if size_error:
177 g.trace('bad old_size:', old_size.__class__, old_size)
178 size = old_size
179 else:
180 # #490: Use c.zoom_size if it exists.
181 zoom_delta = getattr(c, 'zoom_delta', 0)
182 if zoom_delta:
183 size = old_size + zoom_delta
184 self.zoom_dict[key] = size
185 slant = slant or 'roman'
186 weight = weight or 'normal'
187 size = str(size)
188 font = g.app.gui.getFontFromParams(family, size, slant, weight)
189 # A good trace: the key shows what is happening.
190 if font:
191 if trace:
192 g.trace(
193 f"key: {key:>35} family: {family or 'None'} "
194 f"size: {size or 'None'} {slant} {weight}")
195 return font
196 return None
197 #@+node:ekr.20111024091133.16702: *5* BaseColorizer.configure_hard_tab_width
198 def configure_hard_tab_width(self, font):
199 """
200 Set the width of a hard tab.
202 Qt does not appear to have the required methods. Indeed,
203 https://stackoverflow.com/questions/13027091/how-to-override-tab-width-in-qt
204 assumes that QTextEdit's have only a single font(!).
206 This method probabably only works probably if the body text contains
207 a single @language directive, and it may not work properly even then.
208 """
209 c, widget = self.c, self.widget
210 if isinstance(widget, QtWidgets.QTextEdit):
211 # #1919: https://forum.qt.io/topic/99371/how-to-set-tab-stop-width-and-space-width
212 fm = QtGui.QFontMetrics(font)
213 try: # fm.horizontalAdvance
214 width = fm.horizontalAdvance(' ') * abs(c.tab_width)
215 widget.setTabStopDistance(width)
216 except Exception:
217 width = fm.width(' ') * abs(c.tab_width)
218 widget.setTabStopWidth(width) # Obsolete.
219 else:
220 # To do: configure the QScintilla widget.
221 pass
222 #@+node:ekr.20110605121601.18579: *5* BaseColorizer.configure_variable_tags
223 def configure_variable_tags(self):
224 c = self.c
225 use_pygments = pygments and c.config.getBool('use-pygments', default=False)
226 name = 'name.other' if use_pygments else 'name'
227 self.configUnderlineDict[name] = self.underline_undefined
228 for name, option_name, default_color in (
229 # ("blank", "show_invisibles_space_background_color", "Gray90"),
230 # ("tab", "show_invisibles_tab_background_color", "Gray80"),
231 ("elide", None, "yellow"),
232 ):
233 if self.showInvisibles:
234 color = c.config.getColor(option_name) if option_name else default_color
235 else:
236 option_name, default_color = self.default_colors_dict.get(name, (None, None))
237 color = c.config.getColor(option_name) if option_name else ''
238 self.configDict[name] = color # 2022/05/20: Discovered by pyflakes.
239 #@+node:ekr.20110605121601.18574: *4* BaseColorizer.defineDefaultColorsDict
240 #@@nobeautify
242 def defineDefaultColorsDict (self):
244 # These defaults are sure to exist.
245 self.default_colors_dict = {
246 #
247 # Used in Leo rules...
248 # tag name :( option name, default color),
249 'blank' :('show_invisibles_space_color', '#E5E5E5'), # gray90
250 'docpart' :('doc_part_color', 'red'),
251 'leokeyword' :('leo_keyword_color', 'blue'),
252 'link' :('section_name_color', 'red'),
253 'name' :('undefined_section_name_color','red'),
254 'namebrackets' :('section_name_brackets_color', 'blue'),
255 'tab' :('show_invisibles_tab_color', '#CCCCCC'), # gray80
256 'url' :('url_color', 'purple'),
257 #
258 # Pygments tags. Non-default values are taken from 'default' style.
259 #
260 # Top-level...
261 # tag name :( option name, default color),
262 'error' :('error', '#FF0000'), # border
263 'other' :('other', 'white'),
264 'punctuation' :('punctuation', 'white'),
265 'whitespace' :('whitespace', '#bbbbbb'),
266 'xt' :('xt', '#bbbbbb'),
267 #
268 # Comment...
269 # tag name :( option name, default color),
270 'comment' :('comment', '#408080'), # italic
271 'comment.hashbang' :('comment.hashbang', '#408080'),
272 'comment.multiline' :('comment.multiline', '#408080'),
273 'comment.special' :('comment.special', '#408080'),
274 'comment.preproc' :('comment.preproc', '#BC7A00'), # noitalic
275 'comment.single' :('comment.single', '#BC7A00'), # italic
276 #
277 # Generic...
278 # tag name :( option name, default color),
279 'generic' :('generic', '#A00000'),
280 'generic.deleted' :('generic.deleted', '#A00000'),
281 'generic.emph' :('generic.emph', '#000080'), # italic
282 'generic.error' :('generic.error', '#FF0000'),
283 'generic.heading' :('generic.heading', '#000080'), # bold
284 'generic.inserted' :('generic.inserted', '#00A000'),
285 'generic.output' :('generic.output', '#888'),
286 'generic.prompt' :('generic.prompt', '#000080'), # bold
287 'generic.strong' :('generic.strong', '#000080'), # bold
288 'generic.subheading':('generic.subheading', '#800080'), # bold
289 'generic.traceback' :('generic.traceback', '#04D'),
290 #
291 # Keyword...
292 # tag name :( option name, default color),
293 'keyword' :('keyword', '#008000'), # bold
294 'keyword.constant' :('keyword.constant', '#008000'),
295 'keyword.declaration' :('keyword.declaration', '#008000'),
296 'keyword.namespace' :('keyword.namespace', '#008000'),
297 'keyword.pseudo' :('keyword.pseudo', '#008000'), # nobold
298 'keyword.reserved' :('keyword.reserved', '#008000'),
299 'keyword.type' :('keyword.type', '#B00040'),
300 #
301 # Literal...
302 # tag name :( option name, default color),
303 'literal' :('literal', 'white'),
304 'literal.date' :('literal.date', 'white'),
305 #
306 # Name...
307 # tag name :( option name, default color
308 # 'name' defined below.
309 'name.attribute' :('name.attribute', '#7D9029'), # bold
310 'name.builtin' :('name.builtin', '#008000'),
311 'name.builtin.pseudo' :('name.builtin.pseudo','#008000'),
312 'name.class' :('name.class', '#0000FF'), # bold
313 'name.constant' :('name.constant', '#880000'),
314 'name.decorator' :('name.decorator', '#AA22FF'),
315 'name.entity' :('name.entity', '#999999'), # bold
316 'name.exception' :('name.exception', '#D2413A'), # bold
317 'name.function' :('name.function', '#0000FF'),
318 'name.function.magic' :('name.function.magic','#0000FF'),
319 'name.label' :('name.label', '#A0A000'),
320 'name.namespace' :('name.namespace', '#0000FF'), # bold
321 'name.other' :('name.other', 'red'),
322 # A hack: getLegacyFormat returns name.pygments instead of name.
323 'name.pygments' :('name.pygments', 'white'),
324 'name.tag' :('name.tag', '#008000'), # bold
325 'name.variable' :('name.variable', '#19177C'),
326 'name.variable.class' :('name.variable.class', '#19177C'),
327 'name.variable.global' :('name.variable.global', '#19177C'),
328 'name.variable.instance':('name.variable.instance', '#19177C'),
329 'name.variable.magic' :('name.variable.magic', '#19177C'),
330 #
331 # Number...
332 # tag name :( option name, default color
333 'number' :('number', '#666666'),
334 'number.bin' :('number.bin', '#666666'),
335 'number.float' :('number.float', '#666666'),
336 'number.hex' :('number.hex', '#666666'),
337 'number.integer' :('number.integer', '#666666'),
338 'number.integer.long' :('number.integer.long','#666666'),
339 'number.oct' :('number.oct', '#666666'),
340 #
341 # Operator...
342 # tag name :( option name, default color
343 # 'operator' defined below.
344 'operator.word' :('operator.Word', '#AA22FF'), # bold
345 #
346 # String...
347 # tag name :( option name, default color
348 'string' :('string', '#BA2121'),
349 'string.affix' :('string.affix', '#BA2121'),
350 'string.backtick' :('string.backtick', '#BA2121'),
351 'string.char' :('string.char', '#BA2121'),
352 'string.delimiter' :('string.delimiter', '#BA2121'),
353 'string.doc' :('string.doc', '#BA2121'), # italic
354 'string.double' :('string.double', '#BA2121'),
355 'string.escape' :('string.escape', '#BB6622'), # bold
356 'string.heredoc' :('string.heredoc', '#BA2121'),
357 'string.interpol' :('string.interpol', '#BB6688'), # bold
358 'string.other' :('string.other', '#008000'),
359 'string.regex' :('string.regex', '#BB6688'),
360 'string.single' :('string.single', '#BA2121'),
361 'string.symbol' :('string.symbol', '#19177C'),
362 #
363 # jEdit tags.
364 # tag name :( option name, default color),
365 'comment1' :('comment1_color', 'red'),
366 'comment2' :('comment2_color', 'red'),
367 'comment3' :('comment3_color', 'red'),
368 'comment4' :('comment4_color', 'red'),
369 'function' :('function_color', 'black'),
370 'keyword1' :('keyword1_color', 'blue'),
371 'keyword2' :('keyword2_color', 'blue'),
372 'keyword3' :('keyword3_color', 'blue'),
373 'keyword4' :('keyword4_color', 'blue'),
374 'keyword5' :('keyword5_color', 'blue'),
375 'label' :('label_color', 'black'),
376 'literal1' :('literal1_color', '#00aa00'),
377 'literal2' :('literal2_color', '#00aa00'),
378 'literal3' :('literal3_color', '#00aa00'),
379 'literal4' :('literal4_color', '#00aa00'),
380 'markup' :('markup_color', 'red'),
381 'null' :('null_color', None), #'black'),
382 'operator' :('operator_color', 'black'),
383 'trailing_whitespace': ('trailing_whitespace_color', '#808080'),
384 }
385 #@+node:ekr.20110605121601.18575: *4* BaseColorizer.defineDefaultFontDict
386 #@@nobeautify
388 def defineDefaultFontDict (self):
390 self.default_font_dict = {
391 #
392 # Used in Leo rules...
393 # tag name : option name
394 'blank' :'show_invisibles_space_font', # 2011/10/24.
395 'docpart' :'doc_part_font',
396 'leokeyword' :'leo_keyword_font',
397 'link' :'section_name_font',
398 'name' :'undefined_section_name_font',
399 'namebrackets' :'section_name_brackets_font',
400 'tab' :'show_invisibles_tab_font', # 2011/10/24.
401 'url' :'url_font',
402 #
403 # Pygments tags (lower case)...
404 # tag name : option name
405 "comment" :'comment1_font',
406 "comment.preproc" :'comment2_font',
407 "comment.single" :'comment1_font',
408 "error" :'null_font',
409 "generic.deleted" :'literal4_font',
410 "generic.emph" :'literal4_font',
411 "generic.error" :'literal4_font',
412 "generic.heading" :'literal4_font',
413 "generic.inserted" :'literal4_font',
414 "generic.output" :'literal4_font',
415 "generic.prompt" :'literal4_font',
416 "generic.strong" :'literal4_font',
417 "generic.subheading":'literal4_font',
418 "generic.traceback" :'literal4_font',
419 "keyword" :'keyword1_font',
420 "keyword.pseudo" :'keyword2_font',
421 "keyword.type" :'keyword3_font',
422 "name.attribute" :'null_font',
423 "name.builtin" :'null_font',
424 "name.class" :'null_font',
425 "name.constant" :'null_font',
426 "name.decorator" :'null_font',
427 "name.entity" :'null_font',
428 "name.exception" :'null_font',
429 "name.function" :'null_font',
430 "name.label" :'null_font',
431 "name.namespace" :'null_font',
432 "name.tag" :'null_font',
433 "name.variable" :'null_font',
434 "number" :'null_font',
435 "operator.word" :'keyword4_font',
436 "string" :'literal1_font',
437 "string.doc" :'literal1_font',
438 "string.escape" :'literal1_font',
439 "string.interpol" :'literal1_font',
440 "string.other" :'literal1_font',
441 "string.regex" :'literal1_font',
442 "string.single" :'literal1_font',
443 "string.symbol" :'literal1_font',
444 'xt' :'text_font',
445 "whitespace" :'text_font',
446 #
447 # jEdit tags.
448 # tag name : option name
449 'comment1' :'comment1_font',
450 'comment2' :'comment2_font',
451 'comment3' :'comment3_font',
452 'comment4' :'comment4_font',
453 #'default' :'default_font',
454 'function' :'function_font',
455 'keyword1' :'keyword1_font',
456 'keyword2' :'keyword2_font',
457 'keyword3' :'keyword3_font',
458 'keyword4' :'keyword4_font',
459 'keyword5' :'keyword5_font',
460 'label' :'label_font',
461 'literal1' :'literal1_font',
462 'literal2' :'literal2_font',
463 'literal3' :'literal3_font',
464 'literal4' :'literal4_font',
465 'markup' :'markup_font',
466 # 'nocolor' This tag is used, but never generates code.
467 'null' :'null_font',
468 'operator' :'operator_font',
469 'trailing_whitespace' :'trailing_whitespace_font',
470 }
471 #@+node:ekr.20110605121601.18573: *4* BaseColorizer.defineLeoKeywordsDict
472 def defineLeoKeywordsDict(self):
473 self.leoKeywordsDict = {}
474 for key in g.globalDirectiveList:
475 self.leoKeywordsDict[key] = 'leokeyword'
476 #@+node:ekr.20171114041307.1: *3* BaseColorizer.reloadSettings
477 #@@nobeautify
478 def reloadSettings(self):
479 c, getBool = self.c, self.c.config.getBool
480 #
481 # Init all settings ivars.
482 self.color_tags_list = []
483 self.showInvisibles = getBool("show-invisibles-by-default")
484 self.underline_undefined = getBool("underline-undefined-section-names")
485 self.use_hyperlinks = getBool("use-hyperlinks")
486 self.use_pygments = None # Set in report_changes.
487 self.use_pygments_styles = getBool('use-pygments-styles', default=True)
488 #
489 # Report changes to pygments settings.
490 self.report_changes()
491 #
492 # Init the default fonts.
493 self.bold_font = c.config.getFontFromParams(
494 "body_text_font_family", "body_text_font_size",
495 "body_text_font_slant", "body_text_font_weight",
496 c.config.defaultBodyFontSize)
497 self.italic_font = c.config.getFontFromParams(
498 "body_text_font_family", "body_text_font_size",
499 "body_text_font_slant", "body_text_font_weight",
500 c.config.defaultBodyFontSize)
501 self.bolditalic_font = c.config.getFontFromParams(
502 "body_text_font_family", "body_text_font_size",
503 "body_text_font_slant", "body_text_font_weight",
504 c.config.defaultBodyFontSize)
505 # Init everything else.
506 self.init_style_ivars()
507 self.defineLeoKeywordsDict()
508 self.defineDefaultColorsDict()
509 self.defineDefaultFontDict()
510 self.configureTags()
511 self.init()
512 #@+node:ekr.20190327053604.1: *4* BaseColorizer.report_changes
513 prev_use_pygments = None
514 prev_use_styles = None
515 prev_style = None
517 def report_changes(self):
518 """Report changes to pygments settings"""
519 c = self.c
520 use_pygments = c.config.getBool('use-pygments', default=False)
521 if not use_pygments: # 1696.
522 return
523 trace = 'coloring' in g.app.debug and not g.unitTesting
524 if trace:
525 g.es_print('\nreport changes...')
527 def show(setting, val):
528 if trace:
529 g.es_print(f"{setting:35}: {val}")
531 # Set self.use_pygments only once: it can't be changed later.
532 # There is no easy way to re-instantiate classes created by make_colorizer.
533 if self.prev_use_pygments is None:
534 self.use_pygments = self.prev_use_pygments = use_pygments
535 show('@bool use-pygments', use_pygments)
536 elif use_pygments == self.prev_use_pygments:
537 show('@bool use-pygments', use_pygments)
538 else:
539 g.es_print(
540 f"{'Can not change @bool use-pygments':35}: "
541 f"{self.prev_use_pygments}",
542 color='red')
543 # This setting is used only in the LeoHighlighter class
544 style_name = c.config.getString('pygments-style-name') or 'default'
545 # Report everything if we are tracing.
546 show('@bool use-pytments-styles', self.use_pygments_styles)
547 show('@string pygments-style-name', style_name)
548 # Report changes to @bool use-pygments-style
549 if self.prev_use_styles is None:
550 self.prev_use_styles = self.use_pygments_styles
551 elif self.use_pygments_styles != self.prev_use_styles:
552 g.es_print(f"using pygments styles: {self.use_pygments_styles}")
553 # Report @string pygments-style-name only if we are using styles.
554 if not self.use_pygments_styles:
555 return
556 # Report changes to @string pygments-style-name
557 if self.prev_style is None:
558 self.prev_style = style_name
559 elif style_name != self.prev_style:
560 g.es_print(f"New pygments style: {style_name}")
561 self.prev_style = style_name
562 #@+node:ekr.20190324050727.1: *4* BaseColorizer.init_style_ivars
563 def init_style_ivars(self):
564 """Init Style data common to JEdit and Pygments colorizers."""
565 # init() properly sets these for each language.
566 self.actualColorDict = {} # Used only by setTag.
567 self.hyperCount = 0
568 # Attributes dict ivars: defaults are as shown...
569 self.default = 'null'
570 self.digit_re = ''
571 self.escape = ''
572 self.highlight_digits = True
573 self.ignore_case = True
574 self.no_word_sep = ''
575 # Debugging...
576 self.allow_mark_prev = True
577 self.n_setTag = 0
578 self.tagCount = 0
579 self.trace_leo_matches = False
580 self.trace_match_flag = False
581 # Profiling...
582 self.recolorCount = 0 # Total calls to recolor
583 self.stateCount = 0 # Total calls to setCurrentState
584 self.totalStates = 0
585 self.maxStateNumber = 0
586 self.totalKeywordsCalls = 0
587 self.totalLeoKeywordsCalls = 0
588 # Mode data...
589 self.defaultRulesList = []
590 self.importedRulesets = {}
591 self.initLanguage = None
592 self.prev = None # The previous token.
593 self.fonts = {} # Keys are config names. Values are actual fonts.
594 self.keywords = {} # Keys are keywords, values are 0..5.
595 self.modes = {} # Keys are languages, values are modes.
596 self.mode = None # The mode object for the present language.
597 self.modeBunch = None # A bunch fully describing a mode.
598 self.modeStack = []
599 self.rulesDict = {}
600 # self.defineAndExtendForthWords()
601 self.word_chars = {} # Inited by init_keywords().
602 self.tags = [
603 # 8 Leo-specific tags.
604 "blank", # show_invisibles_space_color
605 "docpart",
606 "leokeyword",
607 "link", # section reference.
608 "name",
609 "namebrackets",
610 "tab", # show_invisibles_space_color
611 "url",
612 # jEdit tags.
613 'comment1', 'comment2', 'comment3', 'comment4',
614 # default, # exists, but never generated.
615 'function',
616 'keyword1', 'keyword2', 'keyword3', 'keyword4',
617 'label', 'literal1', 'literal2', 'literal3', 'literal4',
618 'markup', 'operator',
619 'trailing_whitespace',
620 ]
621 #@+node:ekr.20110605121601.18641: *3* BaseColorizer.setTag
622 def setTag(self, tag, s, i, j):
623 """Set the tag in the highlighter."""
624 trace = 'coloring' in g.app.debug and not g.unitTesting
625 self.n_setTag += 1
626 if i == j:
627 return
628 if not tag.strip():
629 return
630 tag = tag.lower().strip()
631 # A hack to allow continuation dots on any tag.
632 dots = tag.startswith('dots')
633 if dots:
634 tag = tag[len('dots') :]
635 colorName = self.configDict.get(tag) # This color name should already be valid.
636 if not colorName:
637 return
638 # New in Leo 5.8.1: allow symbolic color names here.
639 # This now works because all keys in leo_color_database are normalized.
640 colorName = colorName.replace(
641 ' ', '').replace('-', '').replace('_', '').lower().strip()
642 colorName = leo_color_database.get(colorName, colorName)
643 # Get the actual color.
644 color = self.actualColorDict.get(colorName)
645 if not color:
646 color = QtGui.QColor(colorName)
647 if color.isValid():
648 self.actualColorDict[colorName] = color
649 else:
650 g.trace('unknown color name', colorName, g.callers())
651 return
652 underline = self.configUnderlineDict.get(tag)
653 format = QtGui.QTextCharFormat()
654 font = self.fonts.get(tag)
655 if font:
656 format.setFont(font)
657 self.configure_hard_tab_width(font) # #1919.
658 if tag in ('blank', 'tab'):
659 if tag == 'tab' or colorName == 'black':
660 format.setFontUnderline(True)
661 if colorName != 'black':
662 format.setBackground(color)
663 elif underline:
664 format.setForeground(color)
665 format.setUnderlineStyle(UnderlineStyle.SingleUnderline)
666 format.setFontUnderline(True)
667 elif dots or tag == 'trailing_whitespace':
668 format.setForeground(color)
669 format.setUnderlineStyle(UnderlineStyle.DotLine)
670 else:
671 format.setForeground(color)
672 format.setUnderlineStyle(UnderlineStyle.NoUnderline)
673 self.tagCount += 1
674 if trace:
675 # A superb trace.
676 if len(repr(s[i:j])) <= 20:
677 s2 = repr(s[i:j])
678 else:
679 s2 = repr(s[i : i + 17 - 2] + '...')
680 kind_s = f"{self.language}.{tag}"
681 kind_s2 = f"{self.delegate_name}:" if self.delegate_name else ''
682 print(
683 f"setTag: {kind_s:32} {i:3} {j:3} {s2:>22} "
684 f"{self.rulesetName}:{kind_s2}{self.matcher_name}"
685 )
686 self.highlighter.setFormat(i, j - i, format)
687 #@+node:ekr.20170127142001.1: *3* BaseColorizer.updateSyntaxColorer & helpers
688 # Note: these are used by unit tests.
690 def updateSyntaxColorer(self, p):
691 """
692 Scan for color directives in p and its ancestors.
693 Return True unless an coloring is unambiguously disabled.
694 Called from Leo's node-selection logic and from the colorizer.
695 """
696 if p: # This guard is required.
697 try:
698 self.enabled = self.useSyntaxColoring(p)
699 self.language = self.scanLanguageDirectives(p)
700 except Exception:
701 g.es_print('unexpected exception in updateSyntaxColorer')
702 g.es_exception()
703 #@+node:ekr.20170127142001.2: *4* BaseColorizer.scanLanguageDirectives
704 def scanLanguageDirectives(self, p):
705 """Return language based on the directives in p's ancestors."""
706 c = self.c
707 language = g.getLanguageFromAncestorAtFileNode(p)
708 return language or c.target_language
709 #@+node:ekr.20170127142001.7: *4* BaseColorizer.useSyntaxColoring & helper
710 def useSyntaxColoring(self, p):
711 """True if p's parents enable coloring in p."""
712 # Special cases for the selected node.
713 d = self.findColorDirectives(p)
714 if 'killcolor' in d:
715 return False
716 if 'nocolor-node' in d:
717 return False
718 # Now look at the parents.
719 for p in p.parents():
720 d = self.findColorDirectives(p)
721 # @killcolor anywhere disables coloring.
722 if 'killcolor' in d:
723 return False
724 # unambiguous @color enables coloring.
725 if 'color' in d and 'nocolor' not in d:
726 return True
727 # Unambiguous @nocolor disables coloring.
728 if 'nocolor' in d and 'color' not in d:
729 return False
730 return True
731 #@+node:ekr.20170127142001.8: *5* BaseColorizer.findColorDirectives
732 # Order is important: put longest matches first.
733 color_directives_pat = re.compile(
734 r'(^@color|^@killcolor|^@nocolor-node|^@nocolor)'
735 , re.MULTILINE)
737 def findColorDirectives(self, p):
738 """Return a dict with each color directive in p.b, without the leading '@'."""
739 d = {}
740 for m in self.color_directives_pat.finditer(p.b):
741 word = m.group(0)[1:]
742 d[word] = word
743 return d
744 #@-others
745#@+node:ekr.20110605121601.18569: ** class JEditColorizer(BaseColorizer)
746# This is c.frame.body.colorizer
749class JEditColorizer(BaseColorizer):
750 """
751 The JEditColorizer class adapts jEdit pattern matchers for QSyntaxHighlighter.
752 For full documentation, see:
753 https://github.com/leo-editor/leo-editor/blob/master/leo/doc/colorizer.md
754 """
755 #@+others
756 #@+node:ekr.20220317050804.1: *3* jedit: Birth
757 #@+node:ekr.20110605121601.18572: *4* jedit.__init__ & helpers
758 def __init__(self, c, widget):
759 """Ctor for JEditColorizer class."""
760 super().__init__(c, widget)
761 #
762 # Create the highlighter. The default is NullObject.
763 if isinstance(widget, QtWidgets.QTextEdit):
764 self.highlighter = LeoHighlighter(c,
765 colorizer=self,
766 document=widget.document(),
767 )
768 #
769 # State data used only by this class...
770 self.after_doc_language = None
771 self.initialStateNumber = -1
772 self.old_v = None
773 self.nextState = 1 # Dont use 0.
774 self.n2languageDict = {-1: c.target_language}
775 self.restartDict = {} # Keys are state numbers, values are restart functions.
776 self.stateDict = {} # Keys are state numbers, values state names.
777 self.stateNameDict = {} # Keys are state names, values are state numbers.
778 # #2276: Set by init_section_delims.
779 self.section_delim1 = '<<'
780 self.section_delim2 = '>>'
781 #
782 # Init common data...
783 self.reloadSettings()
784 #@+node:ekr.20110605121601.18580: *5* jedit.init
785 def init(self):
786 """Init the colorizer, but *not* state."""
787 #
788 # These *must* be recomputed.
789 self.initialStateNumber = self.setInitialStateNumber()
790 #
791 # Fix #389. Do *not* change these.
792 # self.nextState = 1 # Dont use 0.
793 # self.stateDict = {}
794 # self.stateNameDict = {}
795 # self.restartDict = {}
796 self.init_mode(self.language)
797 self.clearState()
798 # Used by matchers.
799 self.prev = None
800 # Must be done to support per-language @font/@color settings.
801 self.init_section_delims() # #2276
802 #@+node:ekr.20170201082248.1: *5* jedit.init_all_state
803 def init_all_state(self, v):
804 """Completely init all state data."""
805 assert self.language, g.callers(8)
806 self.old_v = v
807 self.n2languageDict = {-1: self.language}
808 self.nextState = 1 # Dont use 0.
809 self.restartDict = {}
810 self.stateDict = {}
811 self.stateNameDict = {}
812 #@+node:ekr.20211029073553.1: *5* jedit.init_section_delims
813 def init_section_delims(self):
815 p = self.c.p
817 def find_delims(v):
818 for s in g.splitLines(v.b):
819 m = g.g_section_delims_pat.match(s)
820 if m:
821 return m
822 return None
824 v = g.findAncestorVnodeByPredicate(p, v_predicate=find_delims)
825 if v:
826 m = find_delims(v)
827 self.section_delim1 = m.group(1)
828 self.section_delim2 = m.group(2)
829 else:
830 self.section_delim1 = '<<'
831 self.section_delim2 = '>>'
832 #@+node:ekr.20110605121601.18576: *4* jedit.addImportedRules
833 def addImportedRules(self, mode, rulesDict, rulesetName):
834 """Append any imported rules at the end of the rulesets specified in mode.importDict"""
835 if self.importedRulesets.get(rulesetName):
836 return
837 self.importedRulesets[rulesetName] = True
838 names = mode.importDict.get(
839 rulesetName, []) if hasattr(mode, 'importDict') else []
840 for name in names:
841 savedBunch = self.modeBunch
842 ok = self.init_mode(name)
843 if ok:
844 rulesDict2 = self.rulesDict
845 for key in rulesDict2.keys():
846 aList = self.rulesDict.get(key, [])
847 aList2 = rulesDict2.get(key)
848 if aList2:
849 # Don't add the standard rules again.
850 rules = [z for z in aList2 if z not in aList]
851 if rules:
852 aList.extend(rules)
853 self.rulesDict[key] = aList
854 self.initModeFromBunch(savedBunch)
855 #@+node:ekr.20110605121601.18577: *4* jedit.addLeoRules
856 def addLeoRules(self, theDict):
857 """Put Leo-specific rules to theList."""
858 # pylint: disable=no-member
859 table = [
860 # Rules added at front are added in **reverse** order.
861 # Debatable: Leo keywords override langauge keywords.
862 ('@', self.match_leo_keywords, True), # Called after all other Leo matchers.
863 ('@', self.match_at_color, True),
864 ('@', self.match_at_killcolor, True),
865 ('@', self.match_at_language, True), # 2011/01/17
866 ('@', self.match_at_nocolor, True),
867 ('@', self.match_at_nocolor_node, True),
868 ('@', self.match_at_wrap, True), # 2015/06/22
869 ('@', self.match_doc_part, True),
870 ('f', self.match_url_f, True),
871 ('g', self.match_url_g, True),
872 ('h', self.match_url_h, True),
873 ('m', self.match_url_m, True),
874 ('n', self.match_url_n, True),
875 ('p', self.match_url_p, True),
876 ('t', self.match_url_t, True),
877 ('u', self.match_unl, True),
878 ('w', self.match_url_w, True),
879 # ('<', self.match_image, True),
880 ('<', self.match_section_ref, True), # Called **first**.
881 # Rules added at back are added in normal order.
882 (' ', self.match_blanks, False),
883 ('\t', self.match_tabs, False),
884 ]
885 if self.c.config.getBool("color-trailing-whitespace"):
886 table += [
887 (' ', self.match_trailing_ws, True),
888 ('\t', self.match_trailing_ws, True),
889 ]
890 for ch, rule, atFront, in table:
891 # Replace the bound method by an unbound method.
892 rule = rule.__func__
893 theList = theDict.get(ch, [])
894 if rule not in theList:
895 if atFront:
896 theList.insert(0, rule)
897 else:
898 theList.append(rule)
899 theDict[ch] = theList
900 #@+node:ekr.20170514054524.1: *4* jedit.getFontFromParams
901 def getFontFromParams(self, family, size, slant, weight, defaultSize=12):
902 return None
904 # def setFontFromConfig(self):
905 # pass
906 #@+node:ekr.20110605121601.18581: *4* jedit.init_mode & helpers
907 def init_mode(self, name):
908 """Name may be a language name or a delegate name."""
909 if not name:
910 return False
911 if name == 'latex':
912 name = 'tex' # #1088: use tex mode for both tex and latex.
913 language, rulesetName = self.nameToRulesetName(name)
914 # if 'coloring' in g.app.debug and not g.unitTesting:
915 # print(f"language: {language!r}, rulesetName: {rulesetName!r}")
916 bunch = self.modes.get(rulesetName)
917 if bunch:
918 if bunch.language == 'unknown-language':
919 return False
920 self.initModeFromBunch(bunch)
921 self.language = language # 2011/05/30
922 return True
923 # Don't try to import a non-existent language.
924 path = g.os_path_join(g.app.loadDir, '..', 'modes')
925 fn = g.os_path_join(path, f"{language}.py")
926 if g.os_path_exists(fn):
927 mode = g.import_module(name=f"leo.modes.{language}")
928 else:
929 mode = None
930 return self.init_mode_from_module(name, mode)
931 #@+node:btheado.20131124162237.16303: *5* jedit.init_mode_from_module
932 def init_mode_from_module(self, name, mode):
933 """
934 Name may be a language name or a delegate name.
935 Mode is a python module or class containing all
936 coloring rule attributes for the mode.
937 """
938 language, rulesetName = self.nameToRulesetName(name)
939 if mode:
940 # A hack to give modes/forth.py access to c.
941 if hasattr(mode, 'pre_init_mode'):
942 mode.pre_init_mode(self.c)
943 else:
944 # Create a dummy bunch to limit recursion.
945 self.modes[rulesetName] = self.modeBunch = g.Bunch(
946 attributesDict={},
947 defaultColor=None,
948 keywordsDict={},
949 language='unknown-language',
950 mode=mode,
951 properties={},
952 rulesDict={},
953 rulesetName=rulesetName,
954 word_chars=self.word_chars, # 2011/05/21
955 )
956 self.rulesetName = rulesetName
957 self.language = 'unknown-language'
958 return False
959 self.language = language
960 self.rulesetName = rulesetName
961 self.properties = getattr(mode, 'properties', None) or {}
962 #
963 # #1334: Careful: getattr(mode, ivar, {}) might be None!
964 #
965 d: Dict[Any, Any] = getattr(mode, 'keywordsDictDict', {}) or {}
966 self.keywordsDict = d.get(rulesetName, {})
967 self.setKeywords()
968 d = getattr(mode, 'attributesDictDict', {}) or {}
969 self.attributesDict: Dict[str, Any] = d.get(rulesetName, {})
970 self.setModeAttributes()
971 d = getattr(mode, 'rulesDictDict', {}) or {}
972 self.rulesDict: Dict[str, Any] = d.get(rulesetName, {})
973 self.addLeoRules(self.rulesDict)
974 self.defaultColor = 'null'
975 self.mode = mode
976 self.modes[rulesetName] = self.modeBunch = g.Bunch(
977 attributesDict=self.attributesDict,
978 defaultColor=self.defaultColor,
979 keywordsDict=self.keywordsDict,
980 language=self.language,
981 mode=self.mode,
982 properties=self.properties,
983 rulesDict=self.rulesDict,
984 rulesetName=self.rulesetName,
985 word_chars=self.word_chars, # 2011/05/21
986 )
987 # Do this after 'officially' initing the mode, to limit recursion.
988 self.addImportedRules(mode, self.rulesDict, rulesetName)
989 self.updateDelimsTables()
990 initialDelegate = self.properties.get('initialModeDelegate')
991 if initialDelegate:
992 # Replace the original mode by the delegate mode.
993 self.init_mode(initialDelegate)
994 language2, rulesetName2 = self.nameToRulesetName(initialDelegate)
995 self.modes[rulesetName] = self.modes.get(rulesetName2)
996 self.language = language2 # 2017/01/31
997 else:
998 self.language = language # 2017/01/31
999 return True
1000 #@+node:ekr.20110605121601.18582: *5* jedit.nameToRulesetName
1001 def nameToRulesetName(self, name):
1002 """
1003 Compute language and rulesetName from name, which is either a language
1004 name or a delegate name.
1005 """
1006 if not name:
1007 return ''
1008 # #1334. Lower-case the name, regardless of the spelling in @language.
1009 name = name.lower()
1010 i = name.find('::')
1011 if i == -1:
1012 language = name
1013 # New in Leo 5.0: allow delegated language names.
1014 language = g.app.delegate_language_dict.get(language, language)
1015 rulesetName = f"{language}_main"
1016 else:
1017 language = name[:i]
1018 delegate = name[i + 2 :]
1019 rulesetName = self.munge(f"{language}_{delegate}")
1020 return language, rulesetName
1021 #@+node:ekr.20110605121601.18583: *5* jedit.setKeywords
1022 def setKeywords(self):
1023 """
1024 Initialize the keywords for the present language.
1026 Set self.word_chars ivar to string.letters + string.digits
1027 plus any other character appearing in any keyword.
1028 """
1029 # Add any new user keywords to leoKeywordsDict.
1030 d = self.keywordsDict
1031 keys = list(d.keys())
1032 for s in g.globalDirectiveList:
1033 key = '@' + s
1034 if key not in keys:
1035 d[key] = 'leokeyword'
1036 # Create a temporary chars list. It will be converted to a dict later.
1037 chars = [z for z in string.ascii_letters + string.digits]
1038 for key in list(d.keys()):
1039 for ch in key:
1040 if ch not in chars:
1041 chars.append(g.checkUnicode(ch))
1042 # jEdit2Py now does this check, so this isn't really needed.
1043 # But it is needed for forth.py.
1044 for ch in (' ', '\t'):
1045 if ch in chars:
1046 # g.es_print('removing %s from word_chars' % (repr(ch)))
1047 chars.remove(ch)
1048 # Convert chars to a dict for faster access.
1049 self.word_chars: Dict[str, str] = {}
1050 for z in chars:
1051 self.word_chars[z] = z
1052 #@+node:ekr.20110605121601.18584: *5* jedit.setModeAttributes
1053 def setModeAttributes(self):
1054 """
1055 Set the ivars from self.attributesDict,
1056 converting 'true'/'false' to True and False.
1057 """
1058 d = self.attributesDict
1059 aList = (
1060 ('default', 'null'),
1061 ('digit_re', ''),
1062 ('escape', ''), # New in Leo 4.4.2.
1063 ('highlight_digits', True),
1064 ('ignore_case', True),
1065 ('no_word_sep', ''),
1066 )
1067 for key, default in aList:
1068 val = d.get(key, default)
1069 if val in ('true', 'True'):
1070 val = True
1071 if val in ('false', 'False'):
1072 val = False
1073 setattr(self, key, val)
1074 #@+node:ekr.20110605121601.18585: *5* jedit.initModeFromBunch
1075 def initModeFromBunch(self, bunch):
1076 self.modeBunch = bunch
1077 self.attributesDict = bunch.attributesDict
1078 self.setModeAttributes()
1079 self.defaultColor = bunch.defaultColor
1080 self.keywordsDict = bunch.keywordsDict
1081 self.language = bunch.language
1082 self.mode = bunch.mode
1083 self.properties = bunch.properties
1084 self.rulesDict = bunch.rulesDict
1085 self.rulesetName = bunch.rulesetName
1086 self.word_chars = bunch.word_chars # 2011/05/21
1087 #@+node:ekr.20110605121601.18586: *5* jedit.updateDelimsTables
1088 def updateDelimsTables(self):
1089 """Update g.app.language_delims_dict if no entry for the language exists."""
1090 d = self.properties
1091 lineComment = d.get('lineComment')
1092 startComment = d.get('commentStart')
1093 endComment = d.get('commentEnd')
1094 if lineComment and startComment and endComment:
1095 delims = f"{lineComment} {startComment} {endComment}"
1096 elif startComment and endComment:
1097 delims = f"{startComment} {endComment}"
1098 elif lineComment:
1099 delims = f"{lineComment}"
1100 else:
1101 delims = None
1102 if delims:
1103 d = g.app.language_delims_dict
1104 if not d.get(self.language):
1105 d[self.language] = delims
1106 #@+node:ekr.20110605121601.18587: *4* jedit.munge
1107 def munge(self, s):
1108 """Munge a mode name so that it is a valid python id."""
1109 valid = string.ascii_letters + string.digits + '_'
1110 return ''.join([ch.lower() if ch in valid else '_' for ch in s])
1111 #@+node:ekr.20170205055743.1: *4* jedit.set_wikiview_patterns
1112 def set_wikiview_patterns(self, leadins, patterns):
1113 """
1114 Init the colorizer so it will *skip* all patterns.
1115 The wikiview plugin calls this method.
1116 """
1117 d = self.rulesDict
1118 for leadins_list, pattern in zip(leadins, patterns):
1119 for ch in leadins_list:
1121 def wiki_rule(self, s, i, pattern=pattern):
1122 """Bind pattern and leadin for jedit.match_wiki_pattern."""
1123 return self.match_wiki_pattern(s, i, pattern)
1125 aList = d.get(ch, [])
1126 if wiki_rule not in aList:
1127 aList.insert(0, wiki_rule)
1128 d[ch] = aList
1129 self.rulesDict = d
1130 #@+node:ekr.20110605121601.18638: *3* jedit.mainLoop
1131 last_v = None
1132 tot_time = 0.0
1134 def mainLoop(self, n, s):
1135 """Colorize a *single* line s, starting in state n."""
1136 f = self.restartDict.get(n)
1137 if 'coloring' in g.app.debug:
1138 p = self.c and self.c.p
1139 if p and p.v != self.last_v:
1140 self.last_v = p.v
1141 g.trace(f"\nNEW NODE: {p.h}\n")
1142 t1 = time.process_time()
1143 i = f(s) if f else 0
1144 while i < len(s):
1145 progress = i
1146 functions = self.rulesDict.get(s[i], [])
1147 for f in functions:
1148 n = f(self, s, i)
1149 if n is None:
1150 g.trace('Can not happen: n is None', repr(f))
1151 break
1152 elif n > 0: # Success. The match has already been colored.
1153 self.matcher_name = f.__name__ # For traces.
1154 i += n
1155 break
1156 elif n < 0: # Total failure.
1157 i += -n
1158 break
1159 else: # Partial failure: Do not break or change i!
1160 pass
1161 else:
1162 i += 1
1163 assert i > progress
1164 # Don't even *think* about changing state here.
1165 self.tot_time += time.process_time() - t1
1166 #@+node:ekr.20110605121601.18640: *3* jedit.recolor & helpers
1167 def recolor(self, s):
1168 """
1169 jEdit.recolor: Recolor a *single* line, s.
1170 QSyntaxHighligher calls this method repeatedly and automatically.
1171 """
1172 p = self.c.p
1173 self.recolorCount += 1
1174 block_n = self.currentBlockNumber()
1175 n = self.prevState()
1176 if p.v == self.old_v:
1177 new_language = self.n2languageDict.get(n)
1178 if new_language != self.language:
1179 self.language = new_language
1180 self.init()
1181 else:
1182 self.updateSyntaxColorer(p) # Force a full recolor
1183 assert self.language
1184 self.init_all_state(p.v)
1185 self.init()
1186 if block_n == 0:
1187 n = self.initBlock0()
1188 n = self.setState(n) # Required.
1189 # Always color the line, even if colorizing is disabled.
1190 if s:
1191 self.mainLoop(n, s)
1192 #@+node:ekr.20170126100139.1: *4* jedit.initBlock0
1193 def initBlock0(self):
1194 """
1195 Init *local* ivars when handling block 0.
1196 This prevents endless recalculation of the proper default state.
1197 """
1198 if self.enabled:
1199 n = self.setInitialStateNumber()
1200 else:
1201 n = self.setRestart(self.restartNoColor)
1202 return n
1203 #@+node:ekr.20170126101049.1: *4* jedit.setInitialStateNumber
1204 def setInitialStateNumber(self):
1205 """
1206 Init the initialStateNumber ivar for clearState()
1207 This saves a lot of work.
1209 Called from init() and initBlock0.
1210 """
1211 state = self.languageTag(self.language)
1212 n = self.stateNameToStateNumber(None, state)
1213 self.initialStateNumber = n
1214 self.blankStateNumber = self.stateNameToStateNumber(None, state + ';blank')
1215 return n
1216 #@+node:ekr.20170126103925.1: *4* jedit.languageTag
1217 def languageTag(self, name):
1218 """
1219 Return the standardized form of the language name.
1220 Doing this consistently prevents subtle bugs.
1221 """
1222 if name:
1223 table = (
1224 ('markdown', 'md'),
1225 ('python', 'py'),
1226 ('javascript', 'js'),
1227 )
1228 for pattern, s in table:
1229 name = name.replace(pattern, s)
1230 return name
1231 return 'no-language'
1232 #@+node:ekr.20110605121601.18589: *3* jedit:Pattern matchers
1233 #@+node:ekr.20110605121601.18590: *4* About the pattern matchers
1234 #@@language rest
1235 #@+at
1236 # The following jEdit matcher methods return the length of the matched text if the
1237 # match succeeds, and zero otherwise. In most cases, these methods colorize all
1238 # the matched text.
1239 #
1240 # The following arguments affect matching:
1241 #
1242 # - at_line_start True: sequence must start the line.
1243 # - at_whitespace_end True: sequence must be first non-whitespace text of the line.
1244 # - at_word_start True: sequence must start a word.
1245 # - hash_char The first character that must match in a regular expression.
1246 # - no_escape: True: ignore an 'end' string if it is preceded by
1247 # the ruleset's escape character.
1248 # - no_line_break True: the match will not succeed across line breaks.
1249 # - no_word_break: True: the match will not cross word breaks.
1250 #
1251 # The following arguments affect coloring when a match succeeds:
1252 #
1253 # - delegate A ruleset name. The matched text will be colored recursively
1254 # by the indicated ruleset.
1255 # - exclude_match If True, the actual text that matched will not be colored.
1256 # - kind The color tag to be applied to colored text.
1257 #@+node:ekr.20110605121601.18637: *4* jedit.colorRangeWithTag
1258 def colorRangeWithTag(self, s, i, j, tag, delegate='', exclude_match=False):
1259 """
1260 Actually colorize the selected range.
1262 This is called whenever a pattern matcher succeed.
1263 """
1264 # setTag does most tracing.
1265 trace = 'coloring' in g.app.debug and not g.unitTesting
1266 if not self.inColorState():
1267 # Do *not* check x.flag here. It won't work.
1268 if trace:
1269 g.trace('not in color state')
1270 return
1271 self.delegate_name = delegate
1272 if delegate:
1273 if trace:
1274 if len(repr(s[i:j])) <= 20:
1275 s2 = repr(s[i:j])
1276 else:
1277 s2 = repr(s[i : i + 17 - 2] + '...')
1278 kind_s = f"{delegate}:{tag}"
1279 print(
1280 f"\ncolorRangeWithTag: {kind_s:25} {i:3} {j:3} "
1281 f"{s2:>20} {self.matcher_name}\n")
1282 self.modeStack.append(self.modeBunch)
1283 self.init_mode(delegate)
1284 while 0 <= i < j and i < len(s):
1285 progress = i
1286 assert j >= 0, j
1287 for f in self.rulesDict.get(s[i], []):
1288 n = f(self, s, i)
1289 if n is None:
1290 g.trace('Can not happen: delegate matcher returns None')
1291 elif n > 0:
1292 self.matcher_name = f.__name__
1293 i += n
1294 break
1295 else:
1296 # Use the default chars for everything else.
1297 # Use the *delegate's* default characters if possible.
1298 default_tag = self.attributesDict.get('default')
1299 self.setTag(default_tag or tag, s, i, i + 1)
1300 i += 1
1301 assert i > progress
1302 bunch = self.modeStack.pop()
1303 self.initModeFromBunch(bunch)
1304 elif not exclude_match:
1305 self.setTag(tag, s, i, j)
1306 if tag != 'url':
1307 # Allow UNL's and URL's *everywhere*.
1308 j = min(j, len(s))
1309 while i < j:
1310 ch = s[i].lower()
1311 if ch == 'u':
1312 n = self.match_unl(s, i)
1313 i += max(1, n)
1314 elif ch in 'fh': # file|ftp|http|https
1315 n = self.match_any_url(s, i)
1316 i += max(1, n)
1317 else:
1318 i += 1
1319 #@+node:ekr.20110605121601.18591: *4* jedit.dump
1320 def dump(self, s):
1321 if s.find('\n') == -1:
1322 return s
1323 return '\n' + s + '\n'
1324 #@+node:ekr.20110605121601.18592: *4* jedit.Leo rule functions
1325 #@+node:ekr.20110605121601.18593: *5* jedit.match_at_color
1326 def match_at_color(self, s, i):
1327 if self.trace_leo_matches:
1328 g.trace()
1329 # Only matches at start of line.
1330 if i == 0 and g.match_word(s, 0, '@color'):
1331 n = self.setRestart(self.restartColor)
1332 self.setState(n) # Enable coloring of *this* line.
1333 # Now required. Sets state.
1334 self.colorRangeWithTag(s, 0, len('@color'), 'leokeyword')
1335 return len('@color')
1336 return 0
1337 #@+node:ekr.20170125140113.1: *6* restartColor
1338 def restartColor(self, s):
1339 """Change all lines up to the next color directive."""
1340 if g.match_word(s, 0, '@killcolor'):
1341 self.colorRangeWithTag(s, 0, len('@color'), 'leokeyword')
1342 self.setRestart(self.restartKillColor)
1343 return -len(s) # Continue to suppress coloring.
1344 if g.match_word(s, 0, '@nocolor-node'):
1345 self.setRestart(self.restartNoColorNode)
1346 return -len(s) # Continue to suppress coloring.
1347 if g.match_word(s, 0, '@nocolor'):
1348 self.setRestart(self.restartNoColor)
1349 return -len(s) # Continue to suppress coloring.
1350 n = self.setRestart(self.restartColor)
1351 self.setState(n) # Enables coloring of *this* line.
1352 return 0 # Allow colorizing!
1353 #@+node:ekr.20110605121601.18597: *5* jedit.match_at_killcolor & restarter
1354 def match_at_killcolor(self, s, i):
1356 # Only matches at start of line.
1357 if i == 0 and g.match_word(s, i, '@killcolor'):
1358 self.setRestart(self.restartKillColor)
1359 return len(s) # Match everything.
1360 return 0
1361 #@+node:ekr.20110605121601.18598: *6* jedit.restartKillColor
1362 def restartKillColor(self, s):
1363 self.setRestart(self.restartKillColor)
1364 return len(s) + 1
1365 #@+node:ekr.20110605121601.18594: *5* jedit.match_at_language
1366 def match_at_language(self, s, i):
1367 """Match Leo's @language directive."""
1368 # Only matches at start of line.
1369 if i != 0:
1370 return 0
1371 if g.match_word(s, i, '@language'):
1372 old_name = self.language
1373 j = g.skip_ws(s, i + len('@language'))
1374 k = g.skip_c_id(s, j)
1375 name = s[j:k]
1376 ok = self.init_mode(name)
1377 if ok:
1378 self.colorRangeWithTag(s, i, k, 'leokeyword')
1379 if name != old_name:
1380 # Solves the recoloring problem!
1381 n = self.setInitialStateNumber()
1382 self.setState(n)
1383 return k - i
1384 return 0
1385 #@+node:ekr.20110605121601.18595: *5* jedit.match_at_nocolor & restarter
1386 def match_at_nocolor(self, s, i):
1388 if self.trace_leo_matches:
1389 g.trace(i, repr(s))
1390 # Only matches at start of line.
1391 if i == 0 and not g.match(s, i, '@nocolor-') and g.match_word(s, i, '@nocolor'):
1392 self.setRestart(self.restartNoColor)
1393 return len(s) # Match everything.
1394 return 0
1395 #@+node:ekr.20110605121601.18596: *6* jedit.restartNoColor
1396 def restartNoColor(self, s):
1397 if self.trace_leo_matches:
1398 g.trace(repr(s))
1399 if g.match_word(s, 0, '@color'):
1400 n = self.setRestart(self.restartColor)
1401 self.setState(n) # Enables coloring of *this* line.
1402 self.colorRangeWithTag(s, 0, len('@color'), 'leokeyword')
1403 return len('@color')
1404 self.setRestart(self.restartNoColor)
1405 return len(s) # Match everything.
1406 #@+node:ekr.20110605121601.18599: *5* jedit.match_at_nocolor_node & restarter
1407 def match_at_nocolor_node(self, s, i):
1409 # Only matches at start of line.
1410 if i == 0 and g.match_word(s, i, '@nocolor-node'):
1411 self.setRestart(self.restartNoColorNode)
1412 return len(s) # Match everything.
1413 return 0
1414 #@+node:ekr.20110605121601.18600: *6* jedit.restartNoColorNode
1415 def restartNoColorNode(self, s):
1416 self.setRestart(self.restartNoColorNode)
1417 return len(s) + 1
1418 #@+node:ekr.20150622072456.1: *5* jedit.match_at_wrap
1419 def match_at_wrap(self, s, i):
1420 """Match Leo's @wrap directive."""
1421 c = self.c
1422 # Only matches at start of line.
1423 seq = '@wrap'
1424 if i == 0 and g.match_word(s, i, seq):
1425 j = i + len(seq)
1426 k = g.skip_ws(s, j)
1427 self.colorRangeWithTag(s, i, k, 'leokeyword')
1428 c.frame.forceWrap(c.p)
1429 return k - i
1430 return 0
1431 #@+node:ekr.20110605121601.18601: *5* jedit.match_blanks
1432 def match_blanks(self, s, i):
1433 # Use Qt code to show invisibles.
1434 return 0
1435 #@+node:ekr.20110605121601.18602: *5* jedit.match_doc_part & restarter
1436 def match_doc_part(self, s, i):
1437 """
1438 Colorize Leo's @ and @ doc constructs.
1439 Matches only at the start of the line.
1440 """
1441 if i != 0:
1442 return 0
1443 if g.match_word(s, i, '@doc'):
1444 j = i + 4
1445 elif g.match(s, i, '@') and (i + 1 >= len(s) or s[i + 1] in (' ', '\t', '\n')):
1446 j = i + 1
1447 else:
1448 return 0
1449 c = self.c
1450 self.colorRangeWithTag(s, 0, j, 'leokeyword')
1451 # New in Leo 5.5: optionally colorize doc parts using reStructuredText
1452 if c.config.getBool('color-doc-parts-as-rest'):
1453 # Switch langauges.
1454 self.after_doc_language = self.language
1455 self.language = 'rest'
1456 self.clearState()
1457 self.init()
1458 # Restart.
1459 self.setRestart(self.restartDocPart)
1460 # Do *not* color the text here!
1461 return j
1462 self.clearState()
1463 self.setRestart(self.restartDocPart)
1464 self.colorRangeWithTag(s, j, len(s), 'docpart')
1465 return len(s)
1466 #@+node:ekr.20110605121601.18603: *6* jedit.restartDocPart
1467 def restartDocPart(self, s):
1468 """
1469 Restarter for @ and @ contructs.
1470 Continue until an @c, @code or @language at the start of the line.
1471 """
1472 for tag in ('@c', '@code', '@language'):
1473 if g.match_word(s, 0, tag):
1474 if tag == '@language':
1475 return self.match_at_language(s, 0)
1476 j = len(tag)
1477 self.colorRangeWithTag(s, 0, j, 'leokeyword') # 'docpart')
1478 # Switch languages.
1479 self.language = self.after_doc_language
1480 self.clearState()
1481 self.init()
1482 self.after_doc_language = None
1483 return j
1484 # Color the next line.
1485 self.setRestart(self.restartDocPart)
1486 if self.c.config.getBool('color-doc-parts-as-rest'):
1487 # Do *not* colorize the text here.
1488 return 0
1489 self.colorRangeWithTag(s, 0, len(s), 'docpart')
1490 return len(s)
1491 #@+node:ekr.20170204072452.1: *5* jedit.match_image
1492 image_url = re.compile(r'^\s*<\s*img\s+.*src=\"(.*)\".*>\s*$')
1494 def match_image(self, s, i):
1495 """Matcher for <img...>"""
1496 m = self.image_url.match(s, i)
1497 if m:
1498 self.image_src = src = m.group(1)
1499 j = len(src)
1500 doc = self.highlighter.document()
1501 block_n = self.currentBlockNumber()
1502 text_block = doc.findBlockByNumber(block_n)
1503 g.trace(f"block_n: {block_n:2} {s!r}")
1504 # How to get the cursor of the colorized line.
1505 # body = self.c.frame.body
1506 # s = body.wrapper.getAllText()
1507 # wrapper.delete(0, j)
1508 # cursor.insertHtml(src)
1509 g.trace(f"block text: {repr(text_block.text())}")
1510 return j
1511 return 0
1512 #@+node:ekr.20110605121601.18604: *5* jedit.match_leo_keywords
1513 def match_leo_keywords(self, s, i):
1514 """Succeed if s[i:] is a Leo keyword."""
1515 self.totalLeoKeywordsCalls += 1
1516 if s[i] != '@':
1517 return 0
1518 # fail if something besides whitespace precedes the word on the line.
1519 i2 = i - 1
1520 while i2 >= 0:
1521 ch = s[i2]
1522 if ch == '\n':
1523 break
1524 elif ch in (' ', '\t'):
1525 i2 -= 1
1526 else:
1527 return 0
1528 # Get the word as quickly as possible.
1529 j = i + 1
1530 while j < len(s) and s[j] in self.word_chars:
1531 j += 1
1532 word = s[i + 1 : j] # entries in leoKeywordsDict do not start with '@'.
1533 if j < len(s) and s[j] not in (' ', '\t', '\n'):
1534 return 0 # Fail, but allow a rescan, as in objective_c.
1535 if self.leoKeywordsDict.get(word):
1536 kind = 'leokeyword'
1537 self.colorRangeWithTag(s, i, j, kind)
1538 self.prev = (i, j, kind)
1539 result = j - i + 1 # Bug fix: skip the last character.
1540 self.trace_match(kind, s, i, j)
1541 return result
1542 # 2010/10/20: also check the keywords dict here.
1543 # This allows for objective_c keywords starting with '@'
1544 # This will not slow down Leo, because it is called
1545 # for things that look like Leo directives.
1546 word = '@' + word
1547 kind = self.keywordsDict.get(word)
1548 if kind:
1549 self.colorRangeWithTag(s, i, j, kind)
1550 self.prev = (i, j, kind)
1551 self.trace_match(kind, s, i, j)
1552 return j - i
1553 # Bug fix: allow rescan. Affects @language patch.
1554 return 0
1555 #@+node:ekr.20110605121601.18605: *5* jedit.match_section_ref
1556 def match_section_ref(self, s, i):
1557 p = self.c.p
1558 if self.trace_leo_matches:
1559 g.trace(self.section_delim1, self.section_delim2, s)
1560 #
1561 # Special case for @language patch: section references are not honored.
1562 if self.language == 'patch':
1563 return 0
1564 n1, n2 = len(self.section_delim1), len(self.section_delim2)
1565 if not g.match(s, i, self.section_delim1):
1566 return 0
1567 k = g.find_on_line(s, i + n1, self.section_delim2)
1568 if k == -1:
1569 return 0
1570 j = k + n2
1571 # Special case for @section-delims.
1572 if s.startswith('@section-delims'):
1573 self.colorRangeWithTag(s, i, i + n1, 'namebrackets')
1574 self.colorRangeWithTag(s, k, j, 'namebrackets')
1575 return j - i
1576 # An actual section reference.
1577 self.colorRangeWithTag(s, i, i + n1, 'namebrackets')
1578 ref = g.findReference(s[i:j], p)
1579 if ref:
1580 if self.use_hyperlinks:
1581 #@+<< set the hyperlink >>
1582 #@+node:ekr.20110605121601.18606: *6* << set the hyperlink >> (jedit)
1583 # Set the bindings to VNode callbacks.
1584 tagName = "hyper" + str(self.hyperCount)
1585 self.hyperCount += 1
1586 ref.tagName = tagName
1587 #@-<< set the hyperlink >>
1588 else:
1589 self.colorRangeWithTag(s, i + n1, k, 'link')
1590 else:
1591 self.colorRangeWithTag(s, i + n1, k, 'name')
1592 self.colorRangeWithTag(s, k, j, 'namebrackets')
1593 return j - i
1594 #@+node:ekr.20110605121601.18607: *5* jedit.match_tabs
1595 def match_tabs(self, s, i):
1596 # Use Qt code to show invisibles.
1597 return 0
1598 # Old code...
1599 # if not self.showInvisibles:
1600 # return 0
1601 # if self.trace_leo_matches: g.trace()
1602 # j = i; n = len(s)
1603 # while j < n and s[j] == '\t':
1604 # j += 1
1605 # if j > i:
1606 # self.colorRangeWithTag(s, i, j, 'tab')
1607 # return j - i
1608 # return 0
1609 #@+node:tbrown.20170707150713.1: *5* jedit.match_tabs
1610 def match_trailing_ws(self, s, i):
1611 """match trailing whitespace"""
1612 j = i
1613 n = len(s)
1614 while j < n and s[j] in ' \t':
1615 j += 1
1616 if j > i and j == n:
1617 self.colorRangeWithTag(s, i, j, 'trailing_whitespace')
1618 return j - i
1619 return 0
1620 #@+node:ekr.20170225103140.1: *5* jedit.match_unl
1621 def match_unl(self, s, i):
1622 if g.match(s.lower(), i, 'unl://'):
1623 j = len(s) # By default, color the whole line.
1624 # #2410: Limit the coloring if possible.
1625 if i > 0:
1626 ch = s[i - 1]
1627 if ch in ('"', "'", '`'):
1628 k = s.find(ch, i)
1629 if k > -1:
1630 j = k
1631 self.colorRangeWithTag(s, i, j, 'url')
1632 return j
1633 return 0
1634 #@+node:ekr.20110605121601.18608: *5* jedit.match_url_any/f/h
1635 # Fix bug 893230: URL coloring does not work for many Internet protocols.
1636 # Added support for: gopher, mailto, news, nntp, prospero, telnet, wais
1638 url_regex_f = re.compile(r"""(file|ftp)://[^\s'"`>]+[\w=/]""")
1639 url_regex_g = re.compile(r"""gopher://[^\s'"`>]+[\w=/]""")
1640 url_regex_h = re.compile(r"""(http|https)://[^\s'"]+[\w=/]""")
1641 url_regex_h = re.compile(r"""(http|https)://[^\s'"`>]+[\w=/]""")
1642 url_regex_m = re.compile(r"""mailto://[^\s'"`>]+[\w=/]""")
1643 url_regex_n = re.compile(r"""(news|nntp)://[^\s'"`>]+[\w=/]""")
1644 url_regex_p = re.compile(r"""prospero://[^\s'"`>]+[\w=/]""")
1645 url_regex_t = re.compile(r"""telnet://[^\s'"`>]+[\w=/]""")
1646 url_regex_w = re.compile(r"""wais://[^\s'"`>]+[\w=/]""")
1647 url_regex = g.url_regex
1649 def match_any_url(self, s, i):
1650 return self.match_compiled_regexp(s, i, kind='url', regexp=self.url_regex)
1652 def match_url_f(self, s, i):
1653 return self.match_compiled_regexp(s, i, kind='url', regexp=self.url_regex_f)
1655 def match_url_g(self, s, i):
1656 return self.match_compiled_regexp(s, i, kind='url', regexp=self.url_regex_g)
1658 def match_url_h(self, s, i):
1659 return self.match_compiled_regexp(s, i, kind='url', regexp=self.url_regex_h)
1661 def match_url_m(self, s, i):
1662 return self.match_compiled_regexp(s, i, kind='url', regexp=self.url_regex_m)
1664 def match_url_n(self, s, i):
1665 return self.match_compiled_regexp(s, i, kind='url', regexp=self.url_regex_n)
1667 def match_url_p(self, s, i):
1668 return self.match_compiled_regexp(s, i, kind='url', regexp=self.url_regex_p)
1670 def match_url_t(self, s, i):
1671 return self.match_compiled_regexp(s, i, kind='url', regexp=self.url_regex_t)
1673 def match_url_w(self, s, i):
1674 return self.match_compiled_regexp(s, i, kind='url', regexp=self.url_regex_w)
1675 #@+node:ekr.20110605121601.18609: *4* jedit.match_compiled_regexp
1676 def match_compiled_regexp(self, s, i, kind, regexp, delegate=''):
1677 """Succeed if the compiled regular expression regexp matches at s[i:]."""
1678 n = self.match_compiled_regexp_helper(s, i, regexp)
1679 if n > 0:
1680 j = i + n
1681 self.colorRangeWithTag(s, i, j, kind, delegate=delegate)
1682 self.prev = (i, j, kind)
1683 self.trace_match(kind, s, i, j)
1684 return n
1685 return 0
1686 #@+node:ekr.20110605121601.18610: *5* jedit.match_compiled_regexp_helper
1687 def match_compiled_regexp_helper(self, s, i, regex):
1688 """
1689 Return the length of the matching text if
1690 seq (a regular expression) matches the present position.
1691 """
1692 # Match succeeds or fails more quickly than search.
1693 self.match_obj = mo = regex.match(s, i) # re_obj.search(s,i)
1694 if mo is None:
1695 return 0
1696 start, end = mo.start(), mo.end()
1697 if start != i:
1698 return 0
1699 return end - start
1700 #@+node:ekr.20110605121601.18611: *4* jedit.match_eol_span
1701 def match_eol_span(self, s, i,
1702 kind=None, seq='',
1703 at_line_start=False, at_whitespace_end=False, at_word_start=False,
1704 delegate='', exclude_match=False
1705 ):
1706 """Succeed if seq matches s[i:]"""
1707 if at_line_start and i != 0 and s[i - 1] != '\n':
1708 return 0
1709 if at_whitespace_end and i != g.skip_ws(s, 0):
1710 return 0
1711 if at_word_start and i > 0 and s[i - 1] in self.word_chars:
1712 return 0
1713 if at_word_start and i + len(
1714 seq) + 1 < len(s) and s[i + len(seq)] in self.word_chars:
1715 return 0
1716 if g.match(s, i, seq):
1717 j = len(s)
1718 self.colorRangeWithTag(
1719 s, i, j, kind, delegate=delegate, exclude_match=exclude_match)
1720 self.prev = (i, j, kind)
1721 self.trace_match(kind, s, i, j)
1722 return j # (was j-1) With a delegate, this could clear state.
1723 return 0
1724 #@+node:ekr.20110605121601.18612: *4* jedit.match_eol_span_regexp
1725 def match_eol_span_regexp(self, s, i,
1726 kind='', regexp='',
1727 at_line_start=False, at_whitespace_end=False, at_word_start=False,
1728 delegate='', exclude_match=False
1729 ):
1730 """Succeed if the regular expression regex matches s[i:]."""
1731 if at_line_start and i != 0 and s[i - 1] != '\n':
1732 return 0
1733 if at_whitespace_end and i != g.skip_ws(s, 0):
1734 return 0
1735 if at_word_start and i > 0 and s[i - 1] in self.word_chars:
1736 return 0 # 7/5/2008
1737 n = self.match_regexp_helper(s, i, regexp)
1738 if n > 0:
1739 j = len(s)
1740 self.colorRangeWithTag(
1741 s, i, j, kind, delegate=delegate, exclude_match=exclude_match)
1742 self.prev = (i, j, kind)
1743 self.trace_match(kind, s, i, j)
1744 return j - i
1745 return 0
1746 #@+node:ekr.20110605121601.18613: *4* jedit.match_everything
1747 # def match_everything (self,s,i,kind=None,delegate='',exclude_match=False):
1748 # """Match the entire rest of the string."""
1749 # j = len(s)
1750 # self.colorRangeWithTag(s,i,j,kind,delegate=delegate)
1751 # return j
1752 #@+node:ekr.20110605121601.18614: *4* jedit.match_keywords
1753 # This is a time-critical method.
1755 def match_keywords(self, s, i):
1756 """
1757 Succeed if s[i:] is a keyword.
1758 Returning -len(word) for failure greatly reduces the number of times this
1759 method is called.
1760 """
1761 self.totalKeywordsCalls += 1
1762 # We must be at the start of a word.
1763 if i > 0 and s[i - 1] in self.word_chars:
1764 return 0
1765 # Get the word as quickly as possible.
1766 j = i
1767 n = len(s)
1768 chars = self.word_chars
1769 # Special cases...
1770 if self.language in ('haskell', 'clojure'):
1771 chars["'"] = "'"
1772 if self.language == 'c':
1773 chars['_'] = '_'
1774 while j < n and s[j] in chars:
1775 j += 1
1776 word = s[i:j]
1777 # Fix part of #585: A kludge for css.
1778 if self.language == 'css' and word.endswith(':'):
1779 j -= 1
1780 word = word[:-1]
1781 if not word:
1782 g.trace(
1783 'can not happen',
1784 repr(s[i : max(j, i + 1)]),
1785 repr(s[i : i + 10]),
1786 g.callers(),
1787 )
1788 return 0
1789 if self.ignore_case:
1790 word = word.lower()
1791 kind = self.keywordsDict.get(word)
1792 if kind:
1793 self.colorRangeWithTag(s, i, j, kind)
1794 self.prev = (i, j, kind)
1795 result = j - i
1796 self.trace_match(kind, s, i, j)
1797 return result
1798 return -len(word) # An important new optimization.
1799 #@+node:ekr.20110605121601.18615: *4* jedit.match_line
1800 def match_line(self, s, i, kind=None, delegate='', exclude_match=False):
1801 """Match the rest of the line."""
1802 j = g.skip_to_end_of_line(s, i)
1803 self.colorRangeWithTag(s, i, j, kind, delegate=delegate)
1804 return j - i
1805 #@+node:ekr.20190606201152.1: *4* jedit.match_lua_literal
1806 def match_lua_literal(self, s, i, kind):
1807 """Succeed if s[i:] is a lua literal. See #1175"""
1808 k = self.match_span(s, i, kind=kind, begin="[[", end="]]")
1809 if k not in (None, 0):
1810 return k
1811 if not g.match(s, i, '[='):
1812 return 0
1813 # Calculate begin and end, then just call match_span
1814 j = i + 2
1815 while g.match(s, j, '='):
1816 j += 1
1817 if not g.match(s, j, '['):
1818 return 0
1819 return self.match_span(s, i, kind=kind, begin=s[i:j], end=s[i + 1 : j] + ']')
1820 #@+node:ekr.20110605121601.18616: *4* jedit.match_mark_following & getNextToken
1821 def match_mark_following(self, s, i,
1822 kind='', pattern='',
1823 at_line_start=False, at_whitespace_end=False, at_word_start=False,
1824 exclude_match=False
1825 ):
1826 """Succeed if s[i:] matches pattern."""
1827 if not self.allow_mark_prev:
1828 return 0
1829 if at_line_start and i != 0 and s[i - 1] != '\n':
1830 return 0
1831 if at_whitespace_end and i != g.skip_ws(s, 0):
1832 return 0
1833 if at_word_start and i > 0 and s[i - 1] in self.word_chars:
1834 return 0 # 7/5/2008
1835 if (
1836 at_word_start
1837 and i + len(pattern) + 1 < len(s)
1838 and s[i + len(pattern)] in self.word_chars
1839 ):
1840 return 0
1841 if g.match(s, i, pattern):
1842 j = i + len(pattern)
1843 # self.colorRangeWithTag(s,i,j,kind,exclude_match=exclude_match)
1844 k = self.getNextToken(s, j)
1845 # 2011/05/31: Do not match *anything* unless there is a token following.
1846 if k > j:
1847 self.colorRangeWithTag(s, i, j, kind, exclude_match=exclude_match)
1848 self.colorRangeWithTag(s, j, k, kind, exclude_match=False)
1849 j = k
1850 self.prev = (i, j, kind)
1851 self.trace_match(kind, s, i, j)
1852 return j - i
1853 return 0
1854 #@+node:ekr.20110605121601.18617: *5* jedit.getNextToken
1855 def getNextToken(self, s, i):
1856 """
1857 Return the index of the end of the next token for match_mark_following.
1859 The jEdit docs are not clear about what a 'token' is, but experiments with jEdit
1860 show that token means a word, as defined by word_chars.
1861 """
1862 # 2011/05/31: Might we extend the concept of token?
1863 # If s[i] is not a word char, should we return just it?
1864 i0 = i
1865 while i < len(s) and s[i].isspace():
1866 i += 1
1867 i1 = i
1868 while i < len(s) and s[i] in self.word_chars:
1869 i += 1
1870 if i == i1:
1871 return i0
1872 return min(len(s), i)
1873 #@+node:ekr.20110605121601.18618: *4* jedit.match_mark_previous
1874 def match_mark_previous(self, s, i,
1875 kind='', pattern='',
1876 at_line_start=False, at_whitespace_end=False, at_word_start=False,
1877 exclude_match=False
1878 ):
1879 """
1880 Return the length of a matched SEQ or 0 if no match.
1882 'at_line_start': True: sequence must start the line.
1883 'at_whitespace_end':True: sequence must be first non-whitespace text of the line.
1884 'at_word_start': True: sequence must start a word.
1885 """
1886 # This match was causing most of the syntax-color problems.
1887 return 0 # 2009/6/23
1888 #@+node:ekr.20110605121601.18619: *4* jedit.match_regexp_helper
1889 def match_regexp_helper(self, s, i, pattern):
1890 """
1891 Return the length of the matching text if
1892 seq (a regular expression) matches the present position.
1893 """
1894 try:
1895 flags = re.MULTILINE
1896 if self.ignore_case:
1897 flags |= re.IGNORECASE
1898 re_obj = re.compile(pattern, flags)
1899 except Exception:
1900 # Do not call g.es here!
1901 g.trace(f"Invalid regular expression: {pattern}")
1902 return 0
1903 # Match succeeds or fails more quickly than search.
1904 self.match_obj = mo = re_obj.match(s, i) # re_obj.search(s,i)
1905 if mo is None:
1906 return 0
1907 start, end = mo.start(), mo.end()
1908 if start != i: # Bug fix 2007-12-18: no match at i
1909 return 0
1910 return end - start
1911 #@+node:ekr.20110605121601.18620: *4* jedit.match_seq
1912 def match_seq(self, s, i,
1913 kind='', seq='',
1914 at_line_start=False,
1915 at_whitespace_end=False,
1916 at_word_start=False,
1917 delegate=''
1918 ):
1919 """Succeed if s[:] mathces seq."""
1920 if at_line_start and i != 0 and s[i - 1] != '\n':
1921 j = i
1922 elif at_whitespace_end and i != g.skip_ws(s, 0):
1923 j = i
1924 elif at_word_start and i > 0 and s[i - 1] in self.word_chars: # 7/5/2008
1925 j = i
1926 if at_word_start and i + len(
1927 seq) + 1 < len(s) and s[i + len(seq)] in self.word_chars:
1928 j = i # 7/5/2008
1929 elif g.match(s, i, seq):
1930 j = i + len(seq)
1931 self.colorRangeWithTag(s, i, j, kind, delegate=delegate)
1932 self.prev = (i, j, kind)
1933 self.trace_match(kind, s, i, j)
1934 else:
1935 j = i
1936 return j - i
1937 #@+node:ekr.20110605121601.18621: *4* jedit.match_seq_regexp
1938 def match_seq_regexp(self, s, i,
1939 kind='', regexp='',
1940 at_line_start=False, at_whitespace_end=False, at_word_start=False,
1941 delegate=''
1942 ):
1943 """Succeed if the regular expression regexp matches at s[i:]."""
1944 if at_line_start and i != 0 and s[i - 1] != '\n':
1945 return 0
1946 if at_whitespace_end and i != g.skip_ws(s, 0):
1947 return 0
1948 if at_word_start and i > 0 and s[i - 1] in self.word_chars:
1949 return 0
1950 n = self.match_regexp_helper(s, i, regexp)
1951 j = i + n
1952 assert j - i == n
1953 self.colorRangeWithTag(s, i, j, kind, delegate=delegate)
1954 self.prev = (i, j, kind)
1955 self.trace_match(kind, s, i, j)
1956 return j - i
1957 #@+node:ekr.20110605121601.18622: *4* jedit.match_span & helper & restarter
1958 def match_span(self, s, i,
1959 kind='', begin='', end='',
1960 at_line_start=False, at_whitespace_end=False, at_word_start=False,
1961 delegate='', exclude_match=False,
1962 no_escape=False, no_line_break=False, no_word_break=False
1963 ):
1964 """Succeed if s[i:] starts with 'begin' and contains a following 'end'."""
1965 dots = False # A flag that we are using dots as a continuation.
1966 if i >= len(s):
1967 return 0
1968 if at_line_start and i != 0 and s[i - 1] != '\n':
1969 j = i
1970 elif at_whitespace_end and i != g.skip_ws(s, 0):
1971 j = i
1972 elif at_word_start and i > 0 and s[i - 1] in self.word_chars:
1973 j = i
1974 elif at_word_start and i + len(
1975 begin) + 1 < len(s) and s[i + len(begin)] in self.word_chars:
1976 j = i
1977 elif not g.match(s, i, begin):
1978 j = i
1979 else:
1980 # We have matched the start of the span.
1981 j = self.match_span_helper(s, i + len(begin), end,
1982 no_escape, no_line_break, no_word_break=no_word_break)
1983 if j == -1:
1984 j = i # A real failure.
1985 else:
1986 # A hack to handle continued strings. Should work for most languages.
1987 # Prepend "dots" to the kind, as a flag to setTag.
1988 dots = j > len(
1989 s) and begin in "'\"" and end in "'\"" and kind.startswith('literal')
1990 dots = dots and self.language not in ('lisp', 'elisp', 'rust')
1991 if dots:
1992 kind = 'dots' + kind
1993 # A match
1994 i2 = i + len(begin)
1995 j2 = j + len(end)
1996 if delegate:
1997 self.colorRangeWithTag(
1998 s, i, i2, kind, delegate=None, exclude_match=exclude_match)
1999 self.colorRangeWithTag(
2000 s, i2, j, kind, delegate=delegate, exclude_match=exclude_match)
2001 self.colorRangeWithTag(
2002 s, j, j2, kind, delegate=None, exclude_match=exclude_match)
2003 else:
2004 self.colorRangeWithTag(
2005 s, i, j2, kind, delegate=None, exclude_match=exclude_match)
2006 j = j2
2007 self.prev = (i, j, kind)
2008 self.trace_match(kind, s, i, j)
2009 # New in Leo 5.5: don't recolor everything after continued strings.
2010 if j > len(s) and not dots:
2011 j = len(s) + 1
2013 def span(s):
2014 # Note: bindings are frozen by this def.
2015 return self.restart_match_span(s, # Positional args, in alpha order
2016 delegate, end, exclude_match, kind,
2017 no_escape, no_line_break, no_word_break)
2019 self.setRestart(span, # These must be keyword args.
2020 delegate=delegate, end=end,
2021 exclude_match=exclude_match,
2022 kind=kind,
2023 no_escape=no_escape,
2024 no_line_break=no_line_break,
2025 no_word_break=no_word_break)
2026 return j - i # Correct, whatever j is.
2027 #@+node:ekr.20110605121601.18623: *5* jedit.match_span_helper
2028 def match_span_helper(self, s, i, pattern, no_escape, no_line_break, no_word_break):
2029 """
2030 Return n >= 0 if s[i] ends with a non-escaped 'end' string.
2031 """
2032 esc = self.escape
2033 # pylint: disable=inconsistent-return-statements
2034 while 1:
2035 j = s.find(pattern, i)
2036 if j == -1:
2037 # Match to end of text if not found and no_line_break is False
2038 if no_line_break:
2039 return -1
2040 return len(s) + 1
2041 if no_word_break and j > 0 and s[j - 1] in self.word_chars:
2042 return -1 # New in Leo 4.5.
2043 if no_line_break and '\n' in s[i:j]:
2044 return -1
2045 if esc and not no_escape:
2046 # Only an odd number of escapes is a 'real' escape.
2047 escapes = 0
2048 k = 1
2049 while j - k >= 0 and s[j - k] == esc:
2050 escapes += 1
2051 k += 1
2052 if (escapes % 2) == 1:
2053 assert s[j - 1] == esc
2054 i += 1 # 2013/08/26: just advance past the *one* escaped character.
2055 else:
2056 return j
2057 else:
2058 return j
2059 # For pylint.
2060 return -1
2061 #@+node:ekr.20110605121601.18624: *5* jedit.restart_match_span
2062 def restart_match_span(self, s,
2063 delegate, end, exclude_match, kind,
2064 no_escape, no_line_break, no_word_break
2065 ):
2066 """Remain in this state until 'end' is seen."""
2067 self.matcher_name = 'restart:' + self.matcher_name.replace('restart:', '')
2068 i = 0
2069 j = self.match_span_helper(s, i, end, no_escape, no_line_break, no_word_break)
2070 if j == -1:
2071 j2 = len(s) + 1
2072 elif j > len(s):
2073 j2 = j
2074 else:
2075 j2 = j + len(end)
2076 if delegate:
2077 self.colorRangeWithTag(s, i, j, kind,
2078 delegate=delegate, exclude_match=exclude_match)
2079 self.colorRangeWithTag(s, j, j2, kind,
2080 delegate=None, exclude_match=exclude_match)
2081 else: # avoid having to merge ranges in addTagsToList.
2082 self.colorRangeWithTag(s, i, j2, kind,
2083 delegate=None, exclude_match=exclude_match)
2084 j = j2
2085 self.trace_match(kind, s, i, j)
2086 if j > len(s):
2088 def span(s):
2089 return self.restart_match_span(s, # Positional args, in alpha order
2090 delegate, end, exclude_match, kind,
2091 no_escape, no_line_break, no_word_break)
2093 self.setRestart(span, # These must be keywords args.
2094 delegate=delegate, end=end, kind=kind,
2095 no_escape=no_escape,
2096 no_line_break=no_line_break,
2097 no_word_break=no_word_break)
2098 else:
2099 self.clearState()
2100 return j # Return the new i, *not* the length of the match.
2101 #@+node:ekr.20110605121601.18625: *4* jedit.match_span_regexp
2102 def match_span_regexp(self, s, i,
2103 kind='', begin='', end='',
2104 at_line_start=False, at_whitespace_end=False, at_word_start=False,
2105 delegate='', exclude_match=False,
2106 no_escape=False, no_line_break=False, no_word_break=False,
2107 ):
2108 """
2109 Succeed if s[i:] starts with 'begin' (a regular expression) and
2110 contains a following 'end'.
2111 """
2112 if at_line_start and i != 0 and s[i - 1] != '\n':
2113 return 0
2114 if at_whitespace_end and i != g.skip_ws(s, 0):
2115 return 0
2116 if at_word_start and i > 0 and s[i - 1] in self.word_chars:
2117 return 0 # 7/5/2008
2118 if (
2119 at_word_start
2120 and i + len(begin) + 1 < len(s)
2121 and s[i + len(begin)] in self.word_chars
2122 ):
2123 return 0 # 7/5/2008
2124 n = self.match_regexp_helper(s, i, begin)
2125 # We may have to allow $n here, in which case we must use a regex object?
2126 if n > 0:
2127 j = i + n
2128 j2 = s.find(end, j)
2129 if j2 == -1:
2130 return 0
2131 if self.escape and not no_escape:
2132 # Only an odd number of escapes is a 'real' escape.
2133 escapes = 0
2134 k = 1
2135 while j - k >= 0 and s[j - k] == self.escape:
2136 escapes += 1
2137 k += 1
2138 if (escapes % 2) == 1:
2139 # An escaped end **aborts the entire match**:
2140 # there is no way to 'restart' the regex.
2141 return 0
2142 i2 = j2 - len(end)
2143 if delegate:
2144 self.colorRangeWithTag(
2145 s, i, j, kind, delegate=None, exclude_match=exclude_match)
2146 self.colorRangeWithTag(
2147 s, j, i2, kind, delegate=delegate, exclude_match=False)
2148 self.colorRangeWithTag(
2149 s, i2, j2, kind, delegate=None, exclude_match=exclude_match)
2150 else: # avoid having to merge ranges in addTagsToList.
2151 self.colorRangeWithTag(
2152 s, i, j2, kind, delegate=None, exclude_match=exclude_match)
2153 self.prev = (i, j, kind)
2154 self.trace_match(kind, s, i, j2)
2155 return j2 - i
2156 return 0
2157 #@+node:ekr.20190623132338.1: *4* jedit.match_tex_backslash
2158 ascii_letters = re.compile(r'[a-zA-Z]+')
2160 def match_tex_backslash(self, s, i, kind):
2161 """
2162 Match the tex s[i:].
2164 (Conventional) acro names are a backslashe followed by either:
2165 1. One or more ascii letters, or
2166 2. Exactly one character, of any kind.
2167 """
2168 assert s[i] == '\\'
2169 m = self.ascii_letters.match(s, i + 1)
2170 if m:
2171 n = len(m.group(0))
2172 j = i + n + 1
2173 else:
2174 # Colorize the backslash plus exactly one more character.
2175 j = i + 2
2176 self.colorRangeWithTag(s, i, j, kind, delegate='')
2177 self.prev = (i, j, kind)
2178 self.trace_match(kind, s, i, j)
2179 return j - i
2180 #@+node:ekr.20170205074106.1: *4* jedit.match_wiki_pattern
2181 def match_wiki_pattern(self, s, i, pattern):
2182 """Show or hide a regex pattern managed by the wikiview plugin."""
2183 m = pattern.match(s, i)
2184 if m:
2185 n = len(m.group(0))
2186 self.colorRangeWithTag(s, i, i + n, 'url')
2187 return n
2188 return 0
2189 #@+node:ekr.20110605121601.18626: *4* jedit.match_word_and_regexp
2190 def match_word_and_regexp(self, s, i,
2191 kind1='', word='',
2192 kind2='', pattern='',
2193 at_line_start=False, at_whitespace_end=False, at_word_start=False,
2194 exclude_match=False
2195 ):
2196 """Succeed if s[i:] matches pattern."""
2197 if not self.allow_mark_prev:
2198 return 0
2199 if at_line_start and i != 0 and s[i - 1] != '\n':
2200 return 0
2201 if at_whitespace_end and i != g.skip_ws(s, 0):
2202 return 0
2203 if at_word_start and i > 0 and s[i - 1] in self.word_chars:
2204 return 0
2205 if (
2206 at_word_start
2207 and i + len(word) + 1 < len(s)
2208 and s[i + len(word)] in self.word_chars
2209 ):
2210 j = i
2211 if not g.match(s, i, word):
2212 return 0
2213 j = i + len(word)
2214 n = self.match_regexp_helper(s, j, pattern)
2215 if n == 0:
2216 return 0
2217 self.colorRangeWithTag(s, i, j, kind1, exclude_match=exclude_match)
2218 k = j + n
2219 self.colorRangeWithTag(s, j, k, kind2, exclude_match=False)
2220 self.prev = (j, k, kind2)
2221 self.trace_match(kind1, s, i, j)
2222 self.trace_match(kind2, s, j, k)
2223 return k - i
2224 #@+node:ekr.20110605121601.18627: *4* jedit.skip_line
2225 def skip_line(self, s, i):
2226 if self.escape:
2227 escape = self.escape + '\n'
2228 n = len(escape)
2229 while i < len(s):
2230 j = g.skip_line(s, i)
2231 if not g.match(s, j - n, escape):
2232 return j
2233 i = j
2234 return i
2235 # Include the newline so we don't get a flash at the end of the line.
2236 return g.skip_line(s, i)
2237 #@+node:ekr.20110605121601.18628: *4* jedit.trace_match
2238 def trace_match(self, kind, s, i, j):
2240 if j != i and self.trace_match_flag:
2241 g.trace(kind, i, j, g.callers(2), self.dump(s[i:j]))
2242 #@+node:ekr.20110605121601.18629: *3* jedit:State methods
2243 #@+node:ekr.20110605121601.18630: *4* jedit.clearState
2244 def clearState(self):
2245 """
2246 Create a *language-specific* default state.
2247 This properly forces a full recoloring when @language changes.
2248 """
2249 n = self.initialStateNumber
2250 self.setState(n)
2251 return n
2252 #@+node:ekr.20110605121601.18631: *4* jedit.computeState
2253 def computeState(self, f, keys):
2254 """
2255 Compute the state name associated with f and all the keys.
2256 Return a unique int n representing that state.
2257 """
2258 # Abbreviate arg names.
2259 d = {
2260 'delegate': '=>',
2261 'end': 'end',
2262 'at_line_start': 'start',
2263 'at_whitespace_end': 'ws-end',
2264 'exclude_match': '!match',
2265 'no_escape': '!esc',
2266 'no_line_break': '!lbrk',
2267 'no_word_break': '!wbrk',
2268 }
2269 result = [self.languageTag(self.language)]
2270 if not self.rulesetName.endswith('_main'):
2271 result.append(self.rulesetName)
2272 if f:
2273 result.append(f.__name__)
2274 for key in sorted(keys):
2275 keyVal = keys.get(key)
2276 val = d.get(key)
2277 if val is None:
2278 val = keys.get(key)
2279 result.append(f"{key}={val}")
2280 elif keyVal is True:
2281 result.append(f"{val}")
2282 elif keyVal is False:
2283 pass
2284 elif keyVal not in (None, ''):
2285 result.append(f"{key}={keyVal}")
2286 state = ';'.join(result).lower()
2287 table = (
2288 ('kind=', ''),
2289 ('literal', 'lit'),
2290 ('restart', '@'),
2291 )
2292 for pattern, s in table:
2293 state = state.replace(pattern, s)
2294 n = self.stateNameToStateNumber(f, state)
2295 return n
2296 #@+node:ekr.20110605121601.18632: *4* jedit.getters & setters
2297 def currentBlockNumber(self):
2298 block = self.highlighter.currentBlock()
2299 return block.blockNumber() if block and block.isValid() else -1
2301 def currentState(self):
2302 return self.highlighter.currentBlockState()
2304 def prevState(self):
2305 return self.highlighter.previousBlockState()
2307 def setState(self, n):
2308 self.highlighter.setCurrentBlockState(n)
2309 return n
2310 #@+node:ekr.20170125141148.1: *4* jedit.inColorState
2311 def inColorState(self):
2312 """True if the *current* state is enabled."""
2313 n = self.currentState()
2314 state = self.stateDict.get(n, 'no-state')
2315 enabled = (
2316 not state.endswith('@nocolor') and
2317 not state.endswith('@nocolor-node') and
2318 not state.endswith('@killcolor'))
2319 return enabled
2320 #@+node:ekr.20110605121601.18633: *4* jedit.setRestart
2321 def setRestart(self, f, **keys):
2322 n = self.computeState(f, keys)
2323 self.setState(n)
2324 return n
2325 #@+node:ekr.20110605121601.18635: *4* jedit.show...
2326 def showState(self, n):
2327 state = self.stateDict.get(n, 'no-state')
2328 return f"{n:2}:{state}"
2330 def showCurrentState(self):
2331 n = self.currentState()
2332 return self.showState(n)
2334 def showPrevState(self):
2335 n = self.prevState()
2336 return self.showState(n)
2337 #@+node:ekr.20110605121601.18636: *4* jedit.stateNameToStateNumber
2338 def stateNameToStateNumber(self, f, stateName):
2339 """
2340 stateDict: Keys are state numbers, values state names.
2341 stateNameDict: Keys are state names, values are state numbers.
2342 restartDict: Keys are state numbers, values are restart functions
2343 """
2344 n = self.stateNameDict.get(stateName)
2345 if n is None:
2346 n = self.nextState
2347 self.stateNameDict[stateName] = n
2348 self.stateDict[n] = stateName
2349 self.restartDict[n] = f
2350 self.nextState += 1
2351 self.n2languageDict[n] = self.language
2352 return n
2353 #@-others
2354#@+node:ekr.20110605121601.18565: ** class LeoHighlighter (QSyntaxHighlighter)
2355# Careful: we may be running from the bridge.
2357if QtGui:
2360 class LeoHighlighter(QtGui.QSyntaxHighlighter): # type:ignore
2361 """
2362 A subclass of QSyntaxHighlighter that overrides
2363 the highlightBlock and rehighlight methods.
2365 All actual syntax coloring is done in the highlighter class.
2367 Used by both the JeditColorizer and PYgmentsColorizer classes.
2368 """
2369 # This is c.frame.body.colorizer.highlighter
2370 #@+others
2371 #@+node:ekr.20110605121601.18566: *3* leo_h.ctor (sets style)
2372 def __init__(self, c, colorizer, document):
2373 """ctor for LeoHighlighter class."""
2374 self.c = c
2375 self.colorizer = colorizer
2376 self.n_calls = 0
2377 # Alas, a QsciDocument is not a QTextDocument.
2378 assert isinstance(document, QtGui.QTextDocument), document
2379 self.leo_document = document
2380 super().__init__(document)
2381 self.reloadSettings()
2382 #@+node:ekr.20110605121601.18567: *3* leo_h.highlightBlock
2383 def highlightBlock(self, s):
2384 """ Called by QSyntaxHighlighter """
2385 self.n_calls += 1
2386 s = g.toUnicode(s)
2387 self.colorizer.recolor(s) # Highlight just one line.
2388 #@+node:ekr.20190327052228.1: *3* leo_h.reloadSettings
2389 def reloadSettings(self):
2390 """Reload all reloadable settings."""
2391 c, document = self.c, self.leo_document
2392 if not pygments:
2393 return
2394 if not c.config.getBool('use-pygments', default=False):
2395 return
2396 # Init pygments ivars.
2397 self._brushes = {}
2398 self._document = document
2399 self._formats = {}
2400 self.colorizer.style_name = 'default'
2401 # Style gallery: https://help.farbox.com/pygments.html
2402 # Dark styles: fruity, monokai, native, vim
2403 # https://github.com/gthank/solarized-dark-pygments
2404 style_name = c.config.getString('pygments-style-name') or 'default'
2405 if not c.config.getBool('use-pygments-styles', default=True):
2406 return
2407 # Init pygments style.
2408 try:
2409 self.setStyle(style_name)
2410 # print('using %r pygments style in %r' % (style_name, c.shortFileName()))
2411 except Exception:
2412 print(f'pygments {style_name!r} style not found. Using "default" style')
2413 self.setStyle('default')
2414 style_name = 'default'
2415 self.colorizer.style_name = style_name
2416 assert self._style
2417 #@+node:ekr.20190320154014.1: *3* leo_h: From PygmentsHighlighter
2418 #
2419 # All code in this tree is based on PygmentsHighlighter.
2420 #
2421 # Copyright (c) Jupyter Development Team.
2422 # Distributed under the terms of the Modified BSD License.
2423 #@+others
2424 #@+node:ekr.20190320153605.1: *4* leo_h._get_format & helpers
2425 def _get_format(self, token):
2426 """ Returns a QTextCharFormat for token or None.
2427 """
2428 if token in self._formats:
2429 return self._formats[token]
2430 if self._style is None:
2431 result = self._get_format_from_document(token, self._document)
2432 else:
2433 result = self._get_format_from_style(token, self._style)
2434 result = self._get_format_from_style(token, self._style)
2435 self._formats[token] = result
2436 return result
2437 #@+node:ekr.20190320162831.1: *5* pyg_h._get_format_from_document
2438 def _get_format_from_document(self, token, document):
2439 """ Returns a QTextCharFormat for token by
2440 """
2441 # Modified by EKR.
2442 # These lines cause unbounded recursion.
2443 # code, html = next(self._formatter._format_lines([(token, u'dummy')]))
2444 # self._document.setHtml(html)
2445 return QtGui.QTextCursor(self._document).charFormat()
2446 #@+node:ekr.20190320153716.1: *5* leo_h._get_format_from_style
2447 key_error_d: Dict[str, bool] = {}
2449 def _get_format_from_style(self, token, style):
2450 """ Returns a QTextCharFormat for token by reading a Pygments style.
2451 """
2452 result = QtGui.QTextCharFormat()
2453 #
2454 # EKR: handle missing tokens.
2455 try:
2456 data = style.style_for_token(token).items()
2457 except KeyError as err:
2458 key = repr(err)
2459 if key not in self.key_error_d:
2460 self.key_error_d[key] = True
2461 g.trace(err)
2462 return result
2463 for key, value in data:
2464 if value:
2465 if key == 'color':
2466 result.setForeground(self._get_brush(value))
2467 elif key == 'bgcolor':
2468 result.setBackground(self._get_brush(value))
2469 elif key == 'bold':
2470 result.setFontWeight(Weight.Bold)
2471 elif key == 'italic':
2472 result.setFontItalic(True)
2473 elif key == 'underline':
2474 result.setUnderlineStyle(UnderlineStyle.SingleUnderline)
2475 elif key == 'sans':
2476 result.setFontStyleHint(Weight.SansSerif)
2477 elif key == 'roman':
2478 result.setFontStyleHint(Weight.Times)
2479 elif key == 'mono':
2480 result.setFontStyleHint(Weight.TypeWriter)
2481 return result
2482 #@+node:ekr.20190320153958.1: *4* leo_h.setStyle
2483 def setStyle(self, style):
2484 """ Sets the style to the specified Pygments style.
2485 """
2486 from pygments.styles import get_style_by_name # type:ignore
2488 if isinstance(style, str):
2489 style = get_style_by_name(style)
2490 self._style = style
2491 self._clear_caches()
2492 #@+node:ekr.20190320154604.1: *4* leo_h.clear_caches
2493 def _clear_caches(self):
2494 """ Clear caches for brushes and formats.
2495 """
2496 self._brushes = {}
2497 self._formats = {}
2498 #@+node:ekr.20190320154752.1: *4* leo_h._get_brush/color
2499 def _get_brush(self, color):
2500 """ Returns a brush for the color.
2501 """
2502 result = self._brushes.get(color)
2503 if result is None:
2504 qcolor = self._get_color(color)
2505 result = QtGui.QBrush(qcolor)
2506 self._brushes[color] = result
2507 return result
2509 def _get_color(self, color):
2510 """ Returns a QColor built from a Pygments color string.
2511 """
2512 qcolor = QtGui.QColor()
2513 qcolor.setRgb(int(color[:2], base=16),
2514 int(color[2:4], base=16),
2515 int(color[4:6], base=16))
2516 return qcolor
2517 #@-others
2518 #@-others
2519#@+node:ekr.20140906095826.18717: ** class NullScintillaLexer (QsciLexerCustom)
2520if Qsci:
2523 class NullScintillaLexer(Qsci.QsciLexerCustom): # type:ignore
2524 """A do-nothing colorizer for Scintilla."""
2526 def __init__(self, c, parent=None):
2527 super().__init__(parent) # Init the pase class
2528 self.leo_c = c
2529 self.configure_lexer()
2531 def description(self, style):
2532 return 'NullScintillaLexer'
2534 def setStyling(self, length, style):
2535 g.trace('(NullScintillaLexer)', length, style)
2537 def styleText(self, start, end):
2538 """Style the text from start to end."""
2540 def configure_lexer(self):
2541 """Configure the QScintilla lexer."""
2542 # c = self.leo_c
2543 lexer = self
2544 # To do: use c.config setting.
2545 # pylint: disable=no-member
2546 font = QtGui.QFont("DejaVu Sans Mono", 14)
2547 lexer.setFont(font)
2548#@+node:ekr.20190319151826.1: ** class PygmentsColorizer(BaseColorizer)
2549class PygmentsColorizer(BaseColorizer):
2550 """
2551 This class adapts pygments tokens to QSyntaxHighlighter.
2552 """
2553 # This is c.frame.body.colorizer
2554 #@+others
2555 #@+node:ekr.20220317053040.1: *3* pyg_c: Birth
2556 #@+node:ekr.20190319151826.3: *4* pyg_c.__init__
2557 def __init__(self, c, widget):
2558 """Ctor for PygmentsColorizer class."""
2559 super().__init__(c, widget)
2560 # Create the highlighter. The default is NullObject.
2561 if isinstance(widget, QtWidgets.QTextEdit):
2562 self.highlighter = LeoHighlighter(c,
2563 colorizer=self,
2564 document=widget.document(),
2565 )
2566 # State unique to this class...
2567 self.color_enabled = self.enabled
2568 self.old_v = None
2569 # Monkey-patch g.isValidLanguage.
2570 g.isValidLanguage = self.pygments_isValidLanguage
2571 # Init common data...
2572 self.reloadSettings()
2573 #@+node:ekr.20190324063349.1: *4* pyg_c.format getters
2574 def getLegacyDefaultFormat(self):
2575 return None
2577 def getLegacyFormat(self, token, text):
2578 """Return a jEdit tag for the given pygments token."""
2579 # Tables and setTag assume lower-case.
2580 r = repr(token).lstrip('Token.').lstrip('Literal.').lower()
2581 if r == 'name':
2582 # Avoid a colision with existing Leo tag.
2583 r = 'name.pygments'
2584 return r
2586 def getPygmentsFormat(self, token, text):
2587 """Return a pygments format."""
2588 format = self.highlighter._formats.get(token)
2589 if not format:
2590 format = self.highlighter._get_format(token)
2591 return format
2592 #@+node:ekr.20190324064341.1: *4* pyg_c.format setters
2593 def setLegacyFormat(self, index, length, format, s):
2594 """Call the jEdit style setTag."""
2595 super().setTag(format, s, index, index + length)
2597 def setPygmentsFormat(self, index, length, format, s):
2598 """Call the base setTag to set the Qt format."""
2599 self.highlighter.setFormat(index, length, format)
2600 #@+node:ekr.20220316200022.1: *3* pyg_c.pygments_isValidLanguage
2601 def pygments_isValidLanguage(self, language: str) -> bool:
2602 """
2603 A hack: we will monkey-patch g.isValidLanguage to be this method.
2605 Without this hack this class would have to define its own copy of the
2606 (complex!) g.getLanguageFromAncestorAtFileNode function.
2607 """
2608 lexer_name = 'python3' if language == 'python' else language
2609 try:
2610 import pygments.lexers as lexers # type: ignore
2611 lexers.get_lexer_by_name(lexer_name)
2612 return True
2613 except Exception:
2614 return False
2615 #@+node:ekr.20190324051704.1: *3* pyg_c.reloadSettings
2616 def reloadSettings(self):
2617 """Reload the base settings, plus pygments settings."""
2618 # Do basic inits.
2619 super().reloadSettings()
2620 # Bind methods.
2621 if self.use_pygments_styles:
2622 self.getDefaultFormat = QtGui.QTextCharFormat
2623 self.getFormat = self.getPygmentsFormat
2624 self.setFormat = self.setPygmentsFormat
2625 else:
2626 self.getDefaultFormat = self.getLegacyDefaultFormat
2627 self.getFormat = self.getLegacyFormat
2628 self.setFormat = self.setLegacyFormat
2629 #@+node:ekr.20190319151826.78: *3* pyg_c.mainLoop & helpers
2630 format_dict: Dict[str, str] = {} # Keys are repr(Token), values are formats.
2631 lexers_dict: Dict[str, Callable] = {} # Keys are language names, values are instantiated, patched lexers.
2632 state_s_dict: Dict[str, int] = {} # Keys are strings, values are ints.
2633 state_n_dict: Dict[int, str] = {} # # Keys are ints, values are strings.
2634 state_index = 1 # Index of state number to be allocated.
2635 # For traces.
2636 last_v = None
2637 tot_time = 0.0
2639 def mainLoop(self, s):
2640 """Colorize a *single* line s"""
2641 if 'coloring' in g.app.debug:
2642 p = self.c and self.c.p
2643 if p and p.v != self.last_v:
2644 self.last_v = p.v
2645 g.trace(f"\nNEW NODE: {p.h}\n")
2646 t1 = time.process_time()
2647 highlighter = self.highlighter
2648 #
2649 # First, set the *expected* lexer. It may change later.
2650 lexer = self.set_lexer()
2651 #
2652 # Restore the state.
2653 # Based on Jupyter code: (c) Jupyter Development Team.
2654 stack_ivar = '_saved_state_stack'
2655 prev_data = highlighter.currentBlock().previous().userData()
2656 if prev_data is not None:
2657 # New code by EKR. Restore the language if necessary.
2658 if self.language != prev_data.leo_language:
2659 # Change the language and the lexer!
2660 self.language = prev_data.leo_language
2661 lexer = self.set_lexer()
2662 setattr(lexer, stack_ivar, prev_data.syntax_stack)
2663 elif hasattr(lexer, stack_ivar):
2664 delattr(lexer, stack_ivar)
2665 #
2666 # The main loop. Warning: this can change self.language.
2667 index = 0
2668 for token, text in lexer.get_tokens(s):
2669 length = len(text)
2670 if self.color_enabled:
2671 format = self.getFormat(token, text)
2672 else:
2673 format = self.getDefaultFormat()
2674 self.setFormat(index, length, format, s)
2675 index += length
2676 #
2677 # Save the state.
2678 # Based on Jupyter code: (c) Jupyter Development Team.
2679 stack = getattr(lexer, stack_ivar, None)
2680 if stack:
2681 data = PygmentsBlockUserData(syntax_stack=stack, leo_language=self.language)
2682 highlighter.currentBlock().setUserData(data)
2683 # Clean up for the next go-round.
2684 delattr(lexer, stack_ivar)
2685 #
2686 # New code by EKR:
2687 # - Fixes a bug so multiline tokens work.
2688 # - State supports Leo's color directives.
2689 state_s = f"{self.language}; {self.color_enabled}: {stack!r}"
2690 state_n = self.state_s_dict.get(state_s)
2691 if state_n is None:
2692 state_n = self.state_index
2693 self.state_index += 1
2694 self.state_s_dict[state_s] = state_n
2695 self.state_n_dict[state_n] = state_s
2696 highlighter.setCurrentBlockState(state_n)
2697 self.tot_time += time.process_time() - t1
2698 #@+node:ekr.20190323045655.1: *4* pyg_c.at_color_callback
2699 def at_color_callback(self, lexer, match):
2700 from pygments.token import Name, Text # type: ignore
2701 kind = match.group(0)
2702 self.color_enabled = kind == '@color'
2703 if self.color_enabled:
2704 yield match.start(), Name.Decorator, kind
2705 else:
2706 yield match.start(), Text, kind
2707 #@+node:ekr.20190323045735.1: *4* pyg_c.at_language_callback
2708 def at_language_callback(self, lexer, match):
2709 """Colorize the name only if the language has a lexer."""
2710 from pygments.token import Name
2711 language = match.group(2)
2712 # #2484: The language is known if there is a lexer for it.
2713 if self.pygments_isValidLanguage(language):
2714 self.language = language
2715 yield match.start(), Name.Decorator, match.group(0)
2716 else:
2717 # Color only the @language, indicating an unknown language.
2718 yield match.start(), Name.Decorator, match.group(1)
2719 #@+node:ekr.20190322082533.1: *4* pyg_c.get_lexer
2720 unknown_languages: List[str] = []
2722 def get_lexer(self, language):
2723 """Return the lexer for self.language, creating it if necessary."""
2724 import pygments.lexers as lexers # type: ignore
2725 trace = 'coloring' in g.app.debug
2726 try:
2727 # #1520: always define lexer_language.
2728 lexer_name = 'python3' if language == 'python' else language
2729 lexer = lexers.get_lexer_by_name(lexer_name)
2730 except Exception:
2731 # One of the lexer's will not exist.
2732 # pylint: disable=no-member
2733 if trace and language not in self.unknown_languages:
2734 self.unknown_languages.append(language)
2735 g.trace(f"\nno lexer for {language!r}. Using python 3 lexer\n")
2736 lexer = lexers.Python3Lexer()
2737 return lexer
2738 #@+node:ekr.20190322094034.1: *4* pyg_c.patch_lexer
2739 def patch_lexer(self, language, lexer):
2741 from pygments.token import Comment # type:ignore
2742 from pygments.lexer import inherit # type:ignore
2745 class PatchedLexer(lexer.__class__): # type:ignore
2747 leo_sec_ref_pat = r'(?-m:\<\<(.*?)\>\>)'
2748 tokens = {
2749 'root': [
2750 (r'^@(color|nocolor|killcolor)\b', self.at_color_callback),
2751 (r'^(@language)\s+(\w+)', self.at_language_callback),
2752 # Single-line, non-greedy match.
2753 (leo_sec_ref_pat, self.section_ref_callback),
2754 # Multi-line, non-greedy match.
2755 (r'(^\s*@doc|@)(\s+|\n)(.|\n)*?^@c', Comment.Leo.DocPart),
2756 inherit,
2757 ],
2758 }
2760 try:
2761 return PatchedLexer()
2762 except Exception:
2763 g.trace(f"can not patch {language!r}")
2764 g.es_exception()
2765 return lexer
2766 #@+node:ekr.20190322133358.1: *4* pyg_c.section_ref_callback
2767 def section_ref_callback(self, lexer, match):
2768 """pygments callback for section references."""
2769 c = self.c
2770 from pygments.token import Comment, Name
2771 name, ref, start = match.group(1), match.group(0), match.start()
2772 found = g.findReference(ref, c.p)
2773 found_tok = Name.Entity if found else Name.Other
2774 yield match.start(), Comment, '<<'
2775 yield start + 2, found_tok, name
2776 yield start + 2 + len(name), Comment, '>>'
2777 #@+node:ekr.20190323064820.1: *4* pyg_c.set_lexer
2778 def set_lexer(self):
2779 """Return the lexer for self.language."""
2780 if self.language == 'patch':
2781 self.language = 'diff'
2782 key = f"{self.language}:{id(self)}"
2783 lexer = self.lexers_dict.get(key)
2784 if not lexer:
2785 lexer = self.get_lexer(self.language)
2786 lexer = self.patch_lexer(self.language, lexer)
2787 self.lexers_dict[key] = lexer
2788 return lexer
2789 #@+node:ekr.20190319151826.79: *3* pyg_c.recolor
2790 def recolor(self, s):
2791 """
2792 PygmentsColorizer.recolor: Recolor a *single* line, s.
2793 QSyntaxHighligher calls this method repeatedly and automatically.
2794 """
2795 p = self.c.p
2796 self.recolorCount += 1
2797 if p.v != self.old_v:
2798 # Force a full recolor
2799 # sets self.language and self.enabled.
2800 self.updateSyntaxColorer(p)
2801 self.color_enabled = self.enabled
2802 self.old_v = p.v # Fix a major performance bug.
2803 self.init()
2804 assert self.language
2805 if s is not None:
2806 # For pygments, we *must* call for all lines.
2807 self.mainLoop(s)
2808 #@-others
2809#@+node:ekr.20140906081909.18689: ** class QScintillaColorizer(BaseColorizer)
2810# This is c.frame.body.colorizer
2813class QScintillaColorizer(BaseColorizer):
2814 """A colorizer for a QsciScintilla widget."""
2815 #@+others
2816 #@+node:ekr.20140906081909.18709: *3* qsc.__init__ & reloadSettings
2817 def __init__(self, c, widget):
2818 """Ctor for QScintillaColorizer. widget is a """
2819 super().__init__(c)
2820 self.count = 0 # For unit testing.
2821 self.colorCacheFlag = False
2822 self.error = False # Set if there is an error in jeditColorizer.recolor
2823 self.flag = True # Per-node enable/disable flag.
2824 self.full_recolor_count = 0 # For unit testing.
2825 self.language = 'python' # set by scanLanguageDirectives.
2826 self.highlighter = None
2827 self.lexer = None # Set in changeLexer.
2828 widget.leo_colorizer = self
2829 # Define/configure various lexers.
2830 self.reloadSettings()
2831 if Qsci:
2832 self.lexersDict = self.makeLexersDict()
2833 self.nullLexer = NullScintillaLexer(c)
2834 else:
2835 self.lexersDict = {} # type:ignore
2836 self.nullLexer = g.NullObject() # type:ignore
2838 def reloadSettings(self):
2839 c = self.c
2840 self.enabled = c.config.getBool('use-syntax-coloring')
2841 #@+node:ekr.20170128141158.1: *3* qsc.scanColorDirectives (over-ride)
2842 def scanColorDirectives(self, p):
2843 """
2844 Return language based on the directives in p's ancestors.
2845 Same as BaseColorizer.scanColorDirectives, except it also scans p.b.
2846 """
2847 c = self.c
2848 root = p.copy()
2849 for p in root.self_and_parents(copy=False):
2850 language = g.findFirstValidAtLanguageDirective(p.b)
2851 if language:
2852 return language
2853 # Get the language from the nearest ancestor @<file> node.
2854 language = g.getLanguageFromAncestorAtFileNode(root) or c.target_language
2855 return language
2856 #@+node:ekr.20140906081909.18718: *3* qsc.changeLexer
2857 def changeLexer(self, language):
2858 """Set the lexer for the given language."""
2859 c = self.c
2860 wrapper = c.frame.body.wrapper
2861 w = wrapper.widget # A Qsci.QsciSintilla object.
2862 self.lexer = self.lexersDict.get(language, self.nullLexer) # type:ignore
2863 w.setLexer(self.lexer)
2864 #@+node:ekr.20140906081909.18707: *3* qsc.colorize
2865 def colorize(self, p):
2866 """The main Scintilla colorizer entry point."""
2867 # It would be much better to use QSyntaxHighlighter.
2868 # Alas, a QSciDocument is not a QTextDocument.
2869 self.updateSyntaxColorer(p)
2870 self.changeLexer(self.language)
2871 # if self.NEW:
2872 # # Works, but QScintillaWrapper.tag_configuration is presently a do-nothing.
2873 # for s in g.splitLines(p.b):
2874 # self.jeditColorizer.recolor(s)
2875 #@+node:ekr.20140906095826.18721: *3* qsc.configure_lexer
2876 def configure_lexer(self, lexer):
2877 """Configure the QScintilla lexer using @data qt-scintilla-styles."""
2878 c = self.c
2879 qcolor, qfont = QtGui.QColor, QtGui.QFont
2880 font = qfont("DejaVu Sans Mono", 14)
2881 lexer.setFont(font)
2882 lexer.setEolFill(False, -1)
2883 if hasattr(lexer, 'setStringsOverNewlineAllowed'):
2884 lexer.setStringsOverNewlineAllowed(False)
2885 table: List[Tuple[str, str]] = []
2886 aList = c.config.getData('qt-scintilla-styles')
2887 if aList:
2888 aList = [s.split(',') for s in aList]
2889 for z in aList:
2890 if len(z) == 2:
2891 color, style = z
2892 table.append((color.strip(), style.strip()),)
2893 else: g.trace(f"entry: {z}")
2894 if not table:
2895 black = '#000000'
2896 firebrick3 = '#CD2626'
2897 leo_green = '#00aa00'
2898 # See http://pyqt.sourceforge.net/Docs/QScintilla2/classQsciLexerPython.html
2899 # for list of selector names.
2900 table = [
2901 # EKR's personal settings are reasonable defaults.
2902 (black, 'ClassName'),
2903 (firebrick3, 'Comment'),
2904 (leo_green, 'Decorator'),
2905 (leo_green, 'DoubleQuotedString'),
2906 (black, 'FunctionMethodName'),
2907 ('blue', 'Keyword'),
2908 (black, 'Number'),
2909 (leo_green, 'SingleQuotedString'),
2910 (leo_green, 'TripleSingleQuotedString'),
2911 (leo_green, 'TripleDoubleQuotedString'),
2912 (leo_green, 'UnclosedString'),
2913 # End of line where string is not closed
2914 # style.python.13=fore:#000000,$(font.monospace),back:#E0C0E0,eolfilled
2915 ]
2916 for color, style in table:
2917 if hasattr(lexer, style):
2918 style_number = getattr(lexer, style)
2919 try:
2920 lexer.setColor(qcolor(color), style_number)
2921 except Exception:
2922 g.trace('bad color', color)
2923 else:
2924 pass
2925 # Not an error. Not all lexers have all styles.
2926 # g.trace('bad style: %s.%s' % (lexer.__class__.__name__, style))
2927 #@+node:ekr.20170128031840.1: *3* qsc.init
2928 def init(self):
2929 """QScintillaColorizer.init"""
2930 self.updateSyntaxColorer(self.c.p)
2931 self.changeLexer(self.language)
2932 #@+node:ekr.20170128133525.1: *3* qsc.makeLexersDict
2933 def makeLexersDict(self):
2934 """Make a dictionary of Scintilla lexers, and configure each one."""
2935 c = self.c
2936 # g.printList(sorted(dir(Qsci)))
2937 parent = c.frame.body.wrapper.widget
2938 table = (
2939 # 'Asm', 'Erlang', 'Forth', 'Haskell',
2940 # 'LaTeX', 'Lisp', 'Markdown', 'Nsis', 'R',
2941 'Bash', 'Batch', 'CPP', 'CSS', 'CMake', 'CSharp', 'CoffeeScript',
2942 'D', 'Diff', 'Fortran', 'Fortran77', 'HTML',
2943 'Java', 'JavaScript', 'Lua', 'Makefile', 'Matlab',
2944 'Pascal', 'Perl', 'Python', 'PostScript', 'Properties',
2945 'Ruby', 'SQL', 'TCL', 'TeX', 'XML', 'YAML',
2946 )
2947 d = {}
2948 for language_name in table:
2949 class_name = 'QsciLexer' + language_name
2950 lexer_class = getattr(Qsci, class_name, None)
2951 if lexer_class:
2952 # pylint: disable=not-callable
2953 lexer = lexer_class(parent=parent)
2954 self.configure_lexer(lexer)
2955 d[language_name.lower()] = lexer
2956 elif 0:
2957 g.trace('no lexer for', class_name)
2958 return d
2959 #@-others
2960#@+node:ekr.20190320062618.1: ** Jupyter classes
2961# Copyright (c) Jupyter Development Team.
2962# Distributed under the terms of the Modified BSD License.
2964if pygments:
2965 #@+others
2966 #@+node:ekr.20190320062624.2: *3* RegexLexer.get_tokens_unprocessed
2967 # Copyright (c) Jupyter Development Team.
2968 # Distributed under the terms of the Modified BSD License.
2970 from pygments.lexer import RegexLexer, _TokenType, Text, Error
2972 def get_tokens_unprocessed(self, text, stack=('root',)):
2973 """
2974 Split ``text`` into (tokentype, text) pairs.
2976 Monkeypatched to store the final stack on the object itself.
2978 The `text` parameter this gets passed is only the current line, so to
2979 highlight things like multiline strings correctly, we need to retrieve
2980 the state from the previous line (this is done in PygmentsHighlighter,
2981 below), and use it to continue processing the current line.
2982 """
2983 pos = 0
2984 tokendefs = self._tokens
2985 if hasattr(self, '_saved_state_stack'):
2986 statestack = list(self._saved_state_stack)
2987 else:
2988 statestack = list(stack)
2989 # Fix #1113...
2990 try:
2991 statetokens = tokendefs[statestack[-1]]
2992 except Exception:
2993 # g.es_exception()
2994 return
2995 while 1:
2996 for rexmatch, action, new_state in statetokens:
2997 m = rexmatch(text, pos)
2998 if m:
2999 if action is not None:
3000 # pylint: disable=unidiomatic-typecheck
3001 # EKR: Why not use isinstance?
3002 if type(action) is _TokenType:
3003 yield pos, action, m.group()
3004 else:
3005 for item in action(self, m):
3006 yield item
3007 pos = m.end()
3008 if new_state is not None:
3009 # state transition
3010 if isinstance(new_state, tuple):
3011 for state in new_state:
3012 if state == '#pop':
3013 statestack.pop()
3014 elif state == '#push':
3015 statestack.append(statestack[-1])
3016 else:
3017 statestack.append(state)
3018 elif isinstance(new_state, int):
3019 # pop
3020 del statestack[new_state:]
3021 elif new_state == '#push':
3022 statestack.append(statestack[-1])
3023 else:
3024 assert False, f"wrong state def: {new_state!r}"
3025 statetokens = tokendefs[statestack[-1]]
3026 break
3027 else:
3028 try:
3029 if text[pos] == '\n':
3030 # at EOL, reset state to "root"
3031 pos += 1
3032 statestack = ['root']
3033 statetokens = tokendefs['root']
3034 yield pos, Text, '\n'
3035 continue
3036 yield pos, Error, text[pos]
3037 pos += 1
3038 except IndexError:
3039 break
3040 self._saved_state_stack = list(statestack)
3042 # Monkeypatch!
3044 if pygments:
3045 RegexLexer.get_tokens_unprocessed = get_tokens_unprocessed
3046 #@+node:ekr.20190320062624.3: *3* class PygmentsBlockUserData(QTextBlockUserData)
3047 # Copyright (c) Jupyter Development Team.
3048 # Distributed under the terms of the Modified BSD License.
3050 if QtGui:
3053 class PygmentsBlockUserData(QtGui.QTextBlockUserData): # type:ignore
3054 """ Storage for the user data associated with each line."""
3056 syntax_stack = ('root',)
3058 def __init__(self, **kwds):
3059 for key, value in kwds.items():
3060 setattr(self, key, value)
3061 super().__init__()
3063 def __repr__(self):
3064 attrs = ['syntax_stack']
3065 kwds = ', '.join([
3066 f"{attr}={getattr(self, attr)!r}"
3067 for attr in attrs
3068 ])
3069 return f"PygmentsBlockUserData({kwds})"
3070 #@-others
3071#@-others
3072#@@language python
3073#@@tabwidth -4
3074#@@pagewidth 70
3075#@-leo