Coverage for C:\leo.repo\leo-editor\leo\core\leoAtFile.py: 100%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# -*- coding: utf-8 -*-
2#@+leo-ver=5-thin
3#@+node:ekr.20150323150718.1: * @file leoAtFile.py
4#@@first
5"""Classes to read and write @file nodes."""
6#@+<< imports >>
7#@+node:ekr.20041005105605.2: ** << imports >> (leoAtFile.py)
8import io
9import os
10import re
11import sys
12import tabnanny
13import time
14import tokenize
15from typing import List
16from leo.core import leoGlobals as g
17from leo.core import leoNodes
18#@-<< imports >>
19#@+others
20#@+node:ekr.20150509194251.1: ** cmd (decorator)
21def cmd(name): # pragma: no cover
22 """Command decorator for the AtFileCommands class."""
23 return g.new_cmd_decorator(name, ['c', 'atFileCommands',])
24#@+node:ekr.20160514120655.1: ** class AtFile
25class AtFile:
26 """A class implementing the atFile subcommander."""
27 #@+<< define class constants >>
28 #@+node:ekr.20131224053735.16380: *3* << define class constants >>
29 #@@nobeautify
31 # directives...
32 noDirective = 1 # not an at-directive.
33 allDirective = 2 # at-all (4.2)
34 docDirective = 3 # @doc.
35 atDirective = 4 # @<space> or @<newline>
36 codeDirective = 5 # @code
37 cDirective = 6 # @c<space> or @c<newline>
38 othersDirective = 7 # at-others
39 miscDirective = 8 # All other directives
40 startVerbatim = 9 # @verbatim Not a real directive. Used to issue warnings.
41 #@-<< define class constants >>
42 #@+others
43 #@+node:ekr.20041005105605.7: *3* at.Birth & init
44 #@+node:ekr.20041005105605.8: *4* at.ctor & helpers
45 # Note: g.getScript also call the at.__init__ and at.finishCreate().
47 def __init__(self, c):
48 """ctor for atFile class."""
49 # **Warning**: all these ivars must **also** be inited in initCommonIvars.
50 self.c = c
51 self.encoding = 'utf-8' # 2014/08/13
52 self.fileCommands = c.fileCommands
53 self.errors = 0 # Make sure at.error() works even when not inited.
54 # #2276: allow different section delims.
55 self.section_delim1 = '<<'
56 self.section_delim2 = '>>'
57 # **Only** at.writeAll manages these flags.
58 self.unchangedFiles = 0
59 # promptForDangerousWrite sets cancelFlag and yesToAll only if canCancelFlag is True.
60 self.canCancelFlag = False
61 self.cancelFlag = False
62 self.yesToAll = False
63 # User options: set in reloadSettings.
64 self.checkPythonCodeOnWrite = False
65 self.runPyFlakesOnWrite = False
66 self.underindentEscapeString = '\\-'
67 self.reloadSettings()
68 #@+node:ekr.20171113152939.1: *5* at.reloadSettings
69 def reloadSettings(self):
70 """AtFile.reloadSettings"""
71 c = self.c
72 self.checkPythonCodeOnWrite = c.config.getBool(
73 'check-python-code-on-write', default=True)
74 self.runPyFlakesOnWrite = c.config.getBool(
75 'run-pyflakes-on-write', default=False)
76 self.underindentEscapeString = c.config.getString(
77 'underindent-escape-string') or '\\-'
78 #@+node:ekr.20041005105605.10: *4* at.initCommonIvars
79 def initCommonIvars(self):
80 """
81 Init ivars common to both reading and writing.
83 The defaults set here may be changed later.
84 """
85 at = self
86 c = at.c
87 at.at_auto_encoding = c.config.default_at_auto_file_encoding
88 at.encoding = c.config.default_derived_file_encoding
89 at.endSentinelComment = ""
90 at.errors = 0
91 at.inCode = True
92 at.indent = 0 # The unit of indentation is spaces, not tabs.
93 at.language = None
94 at.output_newline = g.getOutputNewline(c=c)
95 at.page_width = None
96 at.root = None # The root (a position) of tree being read or written.
97 at.startSentinelComment = ""
98 at.startSentinelComment = ""
99 at.tab_width = c.tab_width or -4
100 at.writing_to_shadow_directory = False
101 #@+node:ekr.20041005105605.13: *4* at.initReadIvars
102 def initReadIvars(self, root, fileName):
103 at = self
104 at.initCommonIvars()
105 at.bom_encoding = None # The encoding implied by any BOM (set by g.stripBOM)
106 at.cloneSibCount = 0 # n > 1: Make sure n cloned sibs exists at next @+node sentinel
107 at.correctedLines = 0 # For perfect import.
108 at.docOut = [] # The doc part being accumulated.
109 at.done = False # True when @-leo seen.
110 at.fromString = False
111 at.importRootSeen = False
112 at.indentStack = []
113 at.lastLines = [] # The lines after @-leo
114 at.leadingWs = ""
115 at.lineNumber = 0 # New in Leo 4.4.8.
116 at.out = None
117 at.outStack = []
118 at.read_i = 0
119 at.read_lines = []
120 at.readVersion = '' # "5" for new-style thin files.
121 at.readVersion5 = False # Synonym for at.readVersion >= '5'
122 at.root = root
123 at.rootSeen = False
124 at.targetFileName = fileName # For at.writeError only.
125 at.v = None
126 at.vStack = [] # Stack of at.v values.
127 at.thinChildIndexStack = [] # number of siblings at this level.
128 at.thinNodeStack = [] # Entries are vnodes.
129 at.updateWarningGiven = False
130 #@+node:ekr.20041005105605.15: *4* at.initWriteIvars
131 def initWriteIvars(self, root):
132 """
133 Compute default values of all write-related ivars.
134 Return the finalized name of the output file.
135 """
136 at, c = self, self.c
137 if not c and c.config:
138 return None # pragma: no cover
139 make_dirs = c.config.create_nonexistent_directories
140 assert root
141 self.initCommonIvars()
142 assert at.checkPythonCodeOnWrite is not None
143 assert at.underindentEscapeString is not None
144 #
145 # Copy args
146 at.root = root
147 at.sentinels = True
148 #
149 # Override initCommonIvars.
150 if g.unitTesting:
151 at.output_newline = '\n'
152 #
153 # Set other ivars.
154 at.force_newlines_in_at_nosent_bodies = c.config.getBool(
155 'force-newlines-in-at-nosent-bodies')
156 # For at.putBody only.
157 at.outputList = []
158 # For stream output.
159 at.scanAllDirectives(root)
160 # Sets the following ivars:
161 # at.encoding
162 # at.explicitLineEnding
163 # at.language
164 # at.output_newline
165 # at.page_width
166 # at.tab_width
167 #
168 # Overrides of at.scanAllDirectives...
169 if at.language == 'python':
170 # Encoding directive overrides everything else.
171 encoding = g.getPythonEncodingFromString(root.b)
172 if encoding:
173 at.encoding = encoding
174 #
175 # Clean root.v.
176 if not at.errors and at.root:
177 at.root.v._p_changed = True
178 #
179 # #1907: Compute the file name and create directories as needed.
180 targetFileName = g.os_path_realpath(g.fullPath(c, root))
181 at.targetFileName = targetFileName # For at.writeError only.
182 #
183 # targetFileName can be empty for unit tests & @command nodes.
184 if not targetFileName: # pragma: no cover
185 targetFileName = root.h if g.unitTesting else None
186 at.targetFileName = targetFileName # For at.writeError only.
187 return targetFileName
188 #
189 # #2276: scan for section delims
190 at.scanRootForSectionDelims(root)
191 #
192 # Do nothing more if the file already exists.
193 if os.path.exists(targetFileName):
194 return targetFileName
195 #
196 # Create directories if enabled.
197 root_dir = g.os_path_dirname(targetFileName)
198 if make_dirs and root_dir: # pragma: no cover
199 ok = g.makeAllNonExistentDirectories(root_dir)
200 if not ok:
201 g.error(f"Error creating directories: {root_dir}")
202 return None
203 #
204 # Return the target file name, regardless of future problems.
205 return targetFileName
206 #@+node:ekr.20041005105605.17: *3* at.Reading
207 #@+node:ekr.20041005105605.18: *4* at.Reading (top level)
208 #@+node:ekr.20070919133659: *5* at.checkExternalFile
209 @cmd('check-external-file')
210 def checkExternalFile(self, event=None): # pragma: no cover
211 """Make sure an external file written by Leo may be read properly."""
212 c, p = self.c, self.c.p
213 if not p.isAtFileNode() and not p.isAtThinFileNode():
214 g.red('Please select an @thin or @file node')
215 return
216 fn = g.fullPath(c, p) # #1910.
217 if not g.os_path_exists(fn):
218 g.red(f"file not found: {fn}")
219 return
220 s, e = g.readFileIntoString(fn)
221 if s is None:
222 g.red(f"empty file: {fn}")
223 return
224 #
225 # Create a dummy, unconnected, VNode as the root.
226 root_v = leoNodes.VNode(context=c)
227 root = leoNodes.Position(root_v)
228 FastAtRead(c, gnx2vnode={}).read_into_root(s, fn, root)
229 #@+node:ekr.20041005105605.19: *5* at.openFileForReading & helper
230 def openFileForReading(self, fromString=False):
231 """
232 Open the file given by at.root.
233 This will be the private file for @shadow nodes.
234 """
235 at, c = self, self.c
236 is_at_shadow = self.root.isAtShadowFileNode()
237 if fromString: # pragma: no cover
238 if is_at_shadow: # pragma: no cover
239 return at.error(
240 'can not call at.read from string for @shadow files')
241 at.initReadLine(fromString)
242 return None, None
243 #
244 # Not from a string. Carefully read the file.
245 # Returns full path, including file name.
246 fn = g.fullPath(c, at.root)
247 # Remember the full path to this node.
248 at.setPathUa(at.root, fn)
249 if is_at_shadow: # pragma: no cover
250 fn = at.openAtShadowFileForReading(fn)
251 if not fn:
252 return None, None
253 assert fn
254 try:
255 # Sets at.encoding, regularizes whitespace and calls at.initReadLines.
256 s = at.readFileToUnicode(fn)
257 # #1466.
258 if s is None: # pragma: no cover
259 # The error has been given.
260 at._file_bytes = g.toEncodedString('')
261 return None, None
262 at.warnOnReadOnlyFile(fn)
263 except Exception: # pragma: no cover
264 at.error(f"unexpected exception opening: '@file {fn}'")
265 at._file_bytes = g.toEncodedString('')
266 fn, s = None, None
267 return fn, s
268 #@+node:ekr.20150204165040.4: *6* at.openAtShadowFileForReading
269 def openAtShadowFileForReading(self, fn): # pragma: no cover
270 """Open an @shadow for reading and return shadow_fn."""
271 at = self
272 x = at.c.shadowController
273 # readOneAtShadowNode should already have checked these.
274 shadow_fn = x.shadowPathName(fn)
275 shadow_exists = (g.os_path_exists(shadow_fn) and g.os_path_isfile(shadow_fn))
276 if not shadow_exists:
277 g.trace('can not happen: no private file',
278 shadow_fn, g.callers())
279 at.error(f"can not happen: private file does not exist: {shadow_fn}")
280 return None
281 # This method is the gateway to the shadow algorithm.
282 x.updatePublicAndPrivateFiles(at.root, fn, shadow_fn)
283 return shadow_fn
284 #@+node:ekr.20041005105605.21: *5* at.read & helpers
285 def read(self, root, fromString=None):
286 """Read an @thin or @file tree."""
287 at, c = self, self.c
288 fileName = g.fullPath(c, root) # #1341. #1889.
289 if not fileName: # pragma: no cover
290 at.error("Missing file name. Restoring @file tree from .leo file.")
291 return False
292 # Fix bug 760531: always mark the root as read, even if there was an error.
293 # Fix bug 889175: Remember the full fileName.
294 at.rememberReadPath(g.fullPath(c, root), root)
295 at.initReadIvars(root, fileName)
296 at.fromString = fromString
297 if at.errors:
298 return False # pragma: no cover
299 fileName, file_s = at.openFileForReading(fromString=fromString)
300 # #1798:
301 if file_s is None:
302 return False # pragma: no cover
303 #
304 # Set the time stamp.
305 if fileName:
306 c.setFileTimeStamp(fileName)
307 elif not fileName and not fromString and not file_s: # pragma: no cover
308 return False
309 root.clearVisitedInTree()
310 at.scanAllDirectives(root)
311 # Sets the following ivars:
312 # at.encoding: **changed later** by readOpenFile/at.scanHeader.
313 # at.explicitLineEnding
314 # at.language
315 # at.output_newline
316 # at.page_width
317 # at.tab_width
318 gnx2vnode = c.fileCommands.gnxDict
319 contents = fromString or file_s
320 FastAtRead(c, gnx2vnode).read_into_root(contents, fileName, root)
321 root.clearDirty()
322 return True
323 #@+node:ekr.20071105164407: *6* at.deleteUnvisitedNodes
324 def deleteUnvisitedNodes(self, root): # pragma: no cover
325 """
326 Delete unvisited nodes in root's subtree, not including root.
328 Before Leo 5.6: Move unvisited node to be children of the 'Resurrected
329 Nodes'.
330 """
331 at, c = self, self.c
332 # Find the unvisited nodes.
333 aList = [z for z in root.subtree() if not z.isVisited()]
334 if aList:
335 at.c.deletePositionsInList(aList)
336 c.redraw()
338 #@+node:ekr.20041005105605.26: *5* at.readAll & helpers
339 def readAll(self, root):
340 """Scan positions, looking for @<file> nodes to read."""
341 at, c = self, self.c
342 old_changed = c.changed
343 t1 = time.time()
344 c.init_error_dialogs()
345 files = at.findFilesToRead(root, all=True)
346 for p in files:
347 at.readFileAtPosition(p)
348 for p in files:
349 p.v.clearDirty()
350 if not g.unitTesting and files: # pragma: no cover
351 t2 = time.time()
352 g.es(f"read {len(files)} files in {t2 - t1:2.2f} seconds")
353 c.changed = old_changed
354 c.raise_error_dialogs()
355 #@+node:ekr.20190108054317.1: *6* at.findFilesToRead
356 def findFilesToRead(self, root, all): # pragma: no cover
358 c = self.c
359 p = root.copy()
360 scanned_nodes = set()
361 files = []
362 after = None if all else p.nodeAfterTree()
363 while p and p != after:
364 data = (p.gnx, g.fullPath(c, p))
365 # skip clones referring to exactly the same paths.
366 if data in scanned_nodes:
367 p.moveToNodeAfterTree()
368 continue
369 scanned_nodes.add(data)
370 if not p.h.startswith('@'):
371 p.moveToThreadNext()
372 elif p.isAtIgnoreNode():
373 if p.isAnyAtFileNode():
374 c.ignored_at_file_nodes.append(p.h)
375 p.moveToNodeAfterTree()
376 elif (
377 p.isAtThinFileNode() or
378 p.isAtAutoNode() or
379 p.isAtEditNode() or
380 p.isAtShadowFileNode() or
381 p.isAtFileNode() or
382 p.isAtCleanNode() # 1134.
383 ):
384 files.append(p.copy())
385 p.moveToNodeAfterTree()
386 elif p.isAtAsisFileNode() or p.isAtNoSentFileNode():
387 # Note (see #1081): @asis and @nosent can *not* be updated automatically.
388 # Doing so using refresh-from-disk will delete all child nodes.
389 p.moveToNodeAfterTree()
390 else:
391 p.moveToThreadNext()
392 return files
393 #@+node:ekr.20190108054803.1: *6* at.readFileAtPosition
394 def readFileAtPosition(self, p): # pragma: no cover
395 """Read the @<file> node at p."""
396 at, c, fileName = self, self.c, p.anyAtFileNodeName()
397 if p.isAtThinFileNode() or p.isAtFileNode():
398 at.read(p)
399 elif p.isAtAutoNode():
400 at.readOneAtAutoNode(p)
401 elif p.isAtEditNode():
402 at.readOneAtEditNode(fileName, p)
403 elif p.isAtShadowFileNode():
404 at.readOneAtShadowNode(fileName, p)
405 elif p.isAtAsisFileNode() or p.isAtNoSentFileNode():
406 at.rememberReadPath(g.fullPath(c, p), p)
407 elif p.isAtCleanNode():
408 at.readOneAtCleanNode(p)
409 #@+node:ekr.20220121052056.1: *5* at.readAllSelected
410 def readAllSelected(self, root): # pragma: no cover
411 """Read all @<file> nodes in root's tree."""
412 at, c = self, self.c
413 old_changed = c.changed
414 t1 = time.time()
415 c.init_error_dialogs()
416 files = at.findFilesToRead(root, all=False)
417 for p in files:
418 at.readFileAtPosition(p)
419 for p in files:
420 p.v.clearDirty()
421 if not g.unitTesting: # pragma: no cover
422 if files:
423 t2 = time.time()
424 g.es(f"read {len(files)} files in {t2 - t1:2.2f} seconds")
425 else:
426 g.es("no @<file> nodes in the selected tree")
427 c.changed = old_changed
428 c.raise_error_dialogs()
429 #@+node:ekr.20080801071227.7: *5* at.readAtShadowNodes
430 def readAtShadowNodes(self, p): # pragma: no cover
431 """Read all @shadow nodes in the p's tree."""
432 at = self
433 after = p.nodeAfterTree()
434 p = p.copy() # Don't change p in the caller.
435 while p and p != after: # Don't use iterator.
436 if p.isAtShadowFileNode():
437 fileName = p.atShadowFileNodeName()
438 at.readOneAtShadowNode(fileName, p)
439 p.moveToNodeAfterTree()
440 else:
441 p.moveToThreadNext()
442 #@+node:ekr.20070909100252: *5* at.readOneAtAutoNode
443 def readOneAtAutoNode(self, p): # pragma: no cover
444 """Read an @auto file into p. Return the *new* position."""
445 at, c, ic = self, self.c, self.c.importCommands
446 fileName = g.fullPath(c, p) # #1521, #1341, #1914.
447 if not g.os_path_exists(fileName):
448 g.error(f"not found: {p.h!r}", nodeLink=p.get_UNL())
449 return p
450 # Remember that we have seen the @auto node.
451 # #889175: Remember the full fileName.
452 at.rememberReadPath(fileName, p)
453 old_p = p.copy()
454 try:
455 at.scanAllDirectives(p)
456 p.v.b = '' # Required for @auto API checks.
457 p.v._deleteAllChildren()
458 p = ic.createOutline(parent=p.copy())
459 # Do *not* call c.selectPosition(p) here.
460 # That would improperly expand nodes.
461 except Exception:
462 p = old_p
463 ic.errors += 1
464 g.es_print('Unexpected exception importing', fileName)
465 g.es_exception()
466 if ic.errors:
467 g.error(f"errors inhibited read @auto {fileName}")
468 elif c.persistenceController:
469 c.persistenceController.update_after_read_foreign_file(p)
470 # Finish.
471 if ic.errors or not g.os_path_exists(fileName):
472 p.clearDirty()
473 else:
474 g.doHook('after-auto', c=c, p=p)
475 return p # For #451: return p.
476 #@+node:ekr.20090225080846.3: *5* at.readOneAtEditNode
477 def readOneAtEditNode(self, fn, p): # pragma: no cover
478 at = self
479 c = at.c
480 ic = c.importCommands
481 # #1521
482 fn = g.fullPath(c, p)
483 junk, ext = g.os_path_splitext(fn)
484 # Fix bug 889175: Remember the full fileName.
485 at.rememberReadPath(fn, p)
486 # if not g.unitTesting: g.es("reading: @edit %s" % (g.shortFileName(fn)))
487 s, e = g.readFileIntoString(fn, kind='@edit')
488 if s is None:
489 return
490 encoding = 'utf-8' if e is None else e
491 # Delete all children.
492 while p.hasChildren():
493 p.firstChild().doDelete()
494 head = ''
495 ext = ext.lower()
496 if ext in ('.html', '.htm'):
497 head = '@language html\n'
498 elif ext in ('.txt', '.text'):
499 head = '@nocolor\n'
500 else:
501 language = ic.languageForExtension(ext)
502 if language and language != 'unknown_language':
503 head = f"@language {language}\n"
504 else:
505 head = '@nocolor\n'
506 p.b = head + g.toUnicode(s, encoding=encoding, reportErrors=True)
507 g.doHook('after-edit', p=p)
508 #@+node:ekr.20190201104956.1: *5* at.readOneAtAsisNode
509 def readOneAtAsisNode(self, fn, p): # pragma: no cover
510 """Read one @asis node. Used only by refresh-from-disk"""
511 at, c = self, self.c
512 # #1521 & #1341.
513 fn = g.fullPath(c, p)
514 junk, ext = g.os_path_splitext(fn)
515 # Remember the full fileName.
516 at.rememberReadPath(fn, p)
517 # if not g.unitTesting: g.es("reading: @asis %s" % (g.shortFileName(fn)))
518 s, e = g.readFileIntoString(fn, kind='@edit')
519 if s is None:
520 return
521 encoding = 'utf-8' if e is None else e
522 # Delete all children.
523 while p.hasChildren():
524 p.firstChild().doDelete()
525 old_body = p.b
526 p.b = g.toUnicode(s, encoding=encoding, reportErrors=True)
527 if not c.isChanged() and p.b != old_body:
528 c.setChanged()
529 #@+node:ekr.20150204165040.5: *5* at.readOneAtCleanNode & helpers
530 def readOneAtCleanNode(self, root): # pragma: no cover
531 """Update the @clean/@nosent node at root."""
532 at, c, x = self, self.c, self.c.shadowController
533 fileName = g.fullPath(c, root)
534 if not g.os_path_exists(fileName):
535 g.es_print(f"not found: {fileName}", color='red', nodeLink=root.get_UNL())
536 return False
537 at.rememberReadPath(fileName, root)
538 at.initReadIvars(root, fileName)
539 # Must be called before at.scanAllDirectives.
540 at.scanAllDirectives(root)
541 # Sets at.startSentinelComment/endSentinelComment.
542 new_public_lines = at.read_at_clean_lines(fileName)
543 old_private_lines = self.write_at_clean_sentinels(root)
544 marker = x.markerFromFileLines(old_private_lines, fileName)
545 old_public_lines, junk = x.separate_sentinels(old_private_lines, marker)
546 if old_public_lines:
547 new_private_lines = x.propagate_changed_lines(
548 new_public_lines, old_private_lines, marker, p=root)
549 else:
550 new_private_lines = []
551 root.b = ''.join(new_public_lines)
552 return True
553 if new_private_lines == old_private_lines:
554 return True
555 if not g.unitTesting:
556 g.es("updating:", root.h)
557 root.clearVisitedInTree()
558 gnx2vnode = at.fileCommands.gnxDict
559 contents = ''.join(new_private_lines)
560 FastAtRead(c, gnx2vnode).read_into_root(contents, fileName, root)
561 return True # Errors not detected.
562 #@+node:ekr.20150204165040.7: *6* at.dump_lines
563 def dump(self, lines, tag): # pragma: no cover
564 """Dump all lines."""
565 print(f"***** {tag} lines...\n")
566 for s in lines:
567 print(s.rstrip())
568 #@+node:ekr.20150204165040.8: *6* at.read_at_clean_lines
569 def read_at_clean_lines(self, fn): # pragma: no cover
570 """Return all lines of the @clean/@nosent file at fn."""
571 at = self
572 # Use the standard helper. Better error reporting.
573 # Important: uses 'rb' to open the file.
574 s = at.openFileHelper(fn)
575 # #1798.
576 if s is None:
577 s = ''
578 else:
579 s = g.toUnicode(s, encoding=at.encoding)
580 s = s.replace('\r\n', '\n') # Suppress meaningless "node changed" messages.
581 return g.splitLines(s)
582 #@+node:ekr.20150204165040.9: *6* at.write_at_clean_sentinels
583 def write_at_clean_sentinels(self, root): # pragma: no cover
584 """
585 Return all lines of the @clean tree as if it were
586 written as an @file node.
587 """
588 at = self
589 result = at.atFileToString(root, sentinels=True)
590 s = g.toUnicode(result, encoding=at.encoding)
591 return g.splitLines(s)
592 #@+node:ekr.20080711093251.7: *5* at.readOneAtShadowNode & helper
593 def readOneAtShadowNode(self, fn, p): # pragma: no cover
595 at, c = self, self.c
596 x = c.shadowController
597 if not fn == p.atShadowFileNodeName():
598 at.error(
599 f"can not happen: fn: {fn} != atShadowNodeName: "
600 f"{p.atShadowFileNodeName()}")
601 return
602 fn = g.fullPath(c, p) # #1521 & #1341.
603 # #889175: Remember the full fileName.
604 at.rememberReadPath(fn, p)
605 shadow_fn = x.shadowPathName(fn)
606 shadow_exists = g.os_path_exists(shadow_fn) and g.os_path_isfile(shadow_fn)
607 # Delete all children.
608 while p.hasChildren():
609 p.firstChild().doDelete()
610 if shadow_exists:
611 at.read(p)
612 else:
613 ok = at.importAtShadowNode(p)
614 if ok:
615 # Create the private file automatically.
616 at.writeOneAtShadowNode(p)
617 #@+node:ekr.20080712080505.1: *6* at.importAtShadowNode
618 def importAtShadowNode(self, p): # pragma: no cover
619 c, ic = self.c, self.c.importCommands
620 fn = g.fullPath(c, p) # #1521, #1341, #1914.
621 if not g.os_path_exists(fn):
622 g.error(f"not found: {p.h!r}", nodeLink=p.get_UNL())
623 return p
624 # Delete all the child nodes.
625 while p.hasChildren():
626 p.firstChild().doDelete()
627 # Import the outline, exactly as @auto does.
628 ic.createOutline(parent=p.copy())
629 if ic.errors:
630 g.error('errors inhibited read @shadow', fn)
631 if ic.errors or not g.os_path_exists(fn):
632 p.clearDirty()
633 return ic.errors == 0
634 #@+node:ekr.20180622110112.1: *4* at.fast_read_into_root
635 def fast_read_into_root(self, c, contents, gnx2vnode, path, root): # pragma: no cover
636 """A convenience wrapper for FastAtRead.read_into_root()"""
637 return FastAtRead(c, gnx2vnode).read_into_root(contents, path, root)
638 #@+node:ekr.20041005105605.116: *4* at.Reading utils...
639 #@+node:ekr.20041005105605.119: *5* at.createImportedNode
640 def createImportedNode(self, root, headline): # pragma: no cover
641 at = self
642 if at.importRootSeen:
643 p = root.insertAsLastChild()
644 p.initHeadString(headline)
645 else:
646 # Put the text into the already-existing root node.
647 p = root
648 at.importRootSeen = True
649 p.v.setVisited() # Suppress warning about unvisited node.
650 return p
651 #@+node:ekr.20130911110233.11286: *5* at.initReadLine
652 def initReadLine(self, s):
653 """Init the ivars so that at.readLine will read all of s."""
654 at = self
655 at.read_i = 0
656 at.read_lines = g.splitLines(s)
657 at._file_bytes = g.toEncodedString(s)
658 #@+node:ekr.20041005105605.120: *5* at.parseLeoSentinel
659 def parseLeoSentinel(self, s):
660 """
661 Parse the sentinel line s.
662 If the sentinel is valid, set at.encoding, at.readVersion, at.readVersion5.
663 """
664 at, c = self, self.c
665 # Set defaults.
666 encoding = c.config.default_derived_file_encoding
667 readVersion, readVersion5 = None, None
668 new_df, start, end, isThin = False, '', '', False
669 # Example: \*@+leo-ver=5-thin-encoding=utf-8,.*/
670 pattern = re.compile(
671 r'(.+)@\+leo(-ver=([0123456789]+))?(-thin)?(-encoding=(.*)(\.))?(.*)')
672 # The old code weirdly allowed '.' in version numbers.
673 # group 1: opening delim
674 # group 2: -ver=
675 # group 3: version number
676 # group(4): -thin
677 # group(5): -encoding=utf-8,.
678 # group(6): utf-8,
679 # group(7): .
680 # group(8): closing delim.
681 m = pattern.match(s)
682 valid = bool(m)
683 if valid:
684 start = m.group(1) # start delim
685 valid = bool(start)
686 if valid:
687 new_df = bool(m.group(2)) # -ver=
688 if new_df:
689 # Set the version number.
690 if m.group(3):
691 readVersion = m.group(3)
692 readVersion5 = readVersion >= '5'
693 else:
694 valid = False # pragma: no cover
695 if valid:
696 # set isThin
697 isThin = bool(m.group(4))
698 if valid and m.group(5):
699 # set encoding.
700 encoding = m.group(6)
701 if encoding and encoding.endswith(','):
702 # Leo 4.2 or after.
703 encoding = encoding[:-1]
704 if not g.isValidEncoding(encoding): # pragma: no cover
705 g.es_print("bad encoding in derived file:", encoding)
706 valid = False
707 if valid:
708 end = m.group(8) # closing delim
709 if valid:
710 at.encoding = encoding
711 at.readVersion = readVersion
712 at.readVersion5 = readVersion5
713 return valid, new_df, start, end, isThin
714 #@+node:ekr.20130911110233.11284: *5* at.readFileToUnicode & helpers
715 def readFileToUnicode(self, fileName): # pragma: no cover
716 """
717 Carefully sets at.encoding, then uses at.encoding to convert the file
718 to a unicode string.
720 Sets at.encoding as follows:
721 1. Use the BOM, if present. This unambiguously determines the encoding.
722 2. Use the -encoding= field in the @+leo header, if present and valid.
723 3. Otherwise, uses existing value of at.encoding, which comes from:
724 A. An @encoding directive, found by at.scanAllDirectives.
725 B. The value of c.config.default_derived_file_encoding.
727 Returns the string, or None on failure.
728 """
729 at = self
730 s = at.openFileHelper(fileName) # Catches all exceptions.
731 # #1798.
732 if s is None:
733 return None
734 e, s = g.stripBOM(s)
735 if e:
736 # The BOM determines the encoding unambiguously.
737 s = g.toUnicode(s, encoding=e)
738 else:
739 # Get the encoding from the header, or the default encoding.
740 s_temp = g.toUnicode(s, 'ascii', reportErrors=False)
741 e = at.getEncodingFromHeader(fileName, s_temp)
742 s = g.toUnicode(s, encoding=e)
743 s = s.replace('\r\n', '\n')
744 at.encoding = e
745 at.initReadLine(s)
746 return s
747 #@+node:ekr.20130911110233.11285: *6* at.openFileHelper
748 def openFileHelper(self, fileName):
749 """Open a file, reporting all exceptions."""
750 at = self
751 # #1798: return None as a flag on any error.
752 s = None
753 try:
754 with open(fileName, 'rb') as f:
755 s = f.read()
756 except IOError: # pragma: no cover
757 at.error(f"can not open {fileName}")
758 except Exception: # pragma: no cover
759 at.error(f"Exception reading {fileName}")
760 g.es_exception()
761 return s
762 #@+node:ekr.20130911110233.11287: *6* at.getEncodingFromHeader
763 def getEncodingFromHeader(self, fileName, s):
764 """
765 Return the encoding given in the @+leo sentinel, if the sentinel is
766 present, or the previous value of at.encoding otherwise.
767 """
768 at = self
769 if at.errors: # pragma: no cover
770 g.trace('can not happen: at.errors > 0', g.callers())
771 e = at.encoding
772 if g.unitTesting:
773 assert False, g.callers()
774 else:
775 at.initReadLine(s)
776 old_encoding = at.encoding
777 assert old_encoding
778 at.encoding = None
779 # Execute scanHeader merely to set at.encoding.
780 at.scanHeader(fileName, giveErrors=False)
781 e = at.encoding or old_encoding
782 assert e
783 return e
784 #@+node:ekr.20041005105605.128: *5* at.readLine
785 def readLine(self):
786 """
787 Read one line from file using the present encoding.
788 Returns at.read_lines[at.read_i++]
789 """
790 # This is an old interface, now used only by at.scanHeader.
791 # For now, it's not worth replacing.
792 at = self
793 if at.read_i < len(at.read_lines):
794 s = at.read_lines[at.read_i]
795 at.read_i += 1
796 return s
797 # Not an error.
798 return '' # pragma: no cover
799 #@+node:ekr.20041005105605.129: *5* at.scanHeader
800 def scanHeader(self, fileName, giveErrors=True):
801 """
802 Scan the @+leo sentinel, using the old readLine interface.
804 Sets self.encoding, and self.start/endSentinelComment.
806 Returns (firstLines,new_df,isThinDerivedFile) where:
807 firstLines contains all @first lines,
808 new_df is True if we are reading a new-format derived file.
809 isThinDerivedFile is True if the file is an @thin file.
810 """
811 at = self
812 new_df, isThinDerivedFile = False, False
813 firstLines: List[str] = [] # The lines before @+leo.
814 s = self.scanFirstLines(firstLines)
815 valid = len(s) > 0
816 if valid:
817 valid, new_df, start, end, isThinDerivedFile = at.parseLeoSentinel(s)
818 if valid:
819 at.startSentinelComment = start
820 at.endSentinelComment = end
821 elif giveErrors: # pragma: no cover
822 at.error(f"No @+leo sentinel in: {fileName}")
823 g.trace(g.callers())
824 return firstLines, new_df, isThinDerivedFile
825 #@+node:ekr.20041005105605.130: *6* at.scanFirstLines
826 def scanFirstLines(self, firstLines): # pragma: no cover
827 """
828 Append all lines before the @+leo line to firstLines.
830 Empty lines are ignored because empty @first directives are
831 ignored.
833 We can not call sentinelKind here because that depends on the comment
834 delimiters we set here.
835 """
836 at = self
837 s = at.readLine()
838 while s and s.find("@+leo") == -1:
839 firstLines.append(s)
840 s = at.readLine()
841 return s
842 #@+node:ekr.20050103163224: *5* at.scanHeaderForThin (import code)
843 def scanHeaderForThin(self, fileName): # pragma: no cover
844 """
845 Return true if the derived file is a thin file.
847 This is a kludgy method used only by the import code."""
848 at = self
849 # Set at.encoding, regularize whitespace and call at.initReadLines.
850 at.readFileToUnicode(fileName)
851 # scanHeader uses at.readline instead of its args.
852 # scanHeader also sets at.encoding.
853 junk, junk, isThin = at.scanHeader(None)
854 return isThin
855 #@+node:ekr.20041005105605.132: *3* at.Writing
856 #@+node:ekr.20041005105605.133: *4* Writing (top level)
857 #@+node:ekr.20190111153551.1: *5* at.commands
858 #@+node:ekr.20070806105859: *6* at.writeAtAutoNodes
859 @cmd('write-at-auto-nodes')
860 def writeAtAutoNodes(self, event=None): # pragma: no cover
861 """Write all @auto nodes in the selected outline."""
862 at, c, p = self, self.c, self.c.p
863 c.init_error_dialogs()
864 after, found = p.nodeAfterTree(), False
865 while p and p != after:
866 if p.isAtAutoNode() and not p.isAtIgnoreNode():
867 ok = at.writeOneAtAutoNode(p)
868 if ok:
869 found = True
870 p.moveToNodeAfterTree()
871 else:
872 p.moveToThreadNext()
873 else:
874 p.moveToThreadNext()
875 if g.unitTesting:
876 return
877 if found:
878 g.es("finished")
879 else:
880 g.es("no @auto nodes in the selected tree")
881 c.raise_error_dialogs(kind='write')
883 #@+node:ekr.20220120072251.1: *6* at.writeDirtyAtAutoNodes
884 @cmd('write-dirty-at-auto-nodes') # pragma: no cover
885 def writeDirtyAtAutoNodes(self, event=None):
886 """Write all dirty @auto nodes in the selected outline."""
887 at, c, p = self, self.c, self.c.p
888 c.init_error_dialogs()
889 after, found = p.nodeAfterTree(), False
890 while p and p != after:
891 if p.isAtAutoNode() and not p.isAtIgnoreNode() and p.isDirty():
892 ok = at.writeOneAtAutoNode(p)
893 if ok:
894 found = True
895 p.moveToNodeAfterTree()
896 else:
897 p.moveToThreadNext()
898 else:
899 p.moveToThreadNext()
900 if g.unitTesting:
901 return
902 if found:
903 g.es("finished")
904 else:
905 g.es("no dirty @auto nodes in the selected tree")
906 c.raise_error_dialogs(kind='write')
907 #@+node:ekr.20080711093251.3: *6* at.writeAtShadowNodes
908 @cmd('write-at-shadow-nodes')
909 def writeAtShadowNodes(self, event=None): # pragma: no cover
910 """Write all @shadow nodes in the selected outline."""
911 at, c, p = self, self.c, self.c.p
912 c.init_error_dialogs()
913 after, found = p.nodeAfterTree(), False
914 while p and p != after:
915 if p.atShadowFileNodeName() and not p.isAtIgnoreNode():
916 ok = at.writeOneAtShadowNode(p)
917 if ok:
918 found = True
919 g.blue(f"wrote {p.atShadowFileNodeName()}")
920 p.moveToNodeAfterTree()
921 else:
922 p.moveToThreadNext()
923 else:
924 p.moveToThreadNext()
925 if g.unitTesting:
926 return found
927 if found:
928 g.es("finished")
929 else:
930 g.es("no @shadow nodes in the selected tree")
931 c.raise_error_dialogs(kind='write')
932 return found
934 #@+node:ekr.20220120072917.1: *6* at.writeDirtyAtShadowNodes
935 @cmd('write-dirty-at-shadow-nodes')
936 def writeDirtyAtShadowNodes(self, event=None): # pragma: no cover
937 """Write all @shadow nodes in the selected outline."""
938 at, c, p = self, self.c, self.c.p
939 c.init_error_dialogs()
940 after, found = p.nodeAfterTree(), False
941 while p and p != after:
942 if p.atShadowFileNodeName() and not p.isAtIgnoreNode() and p.isDirty():
943 ok = at.writeOneAtShadowNode(p)
944 if ok:
945 found = True
946 g.blue(f"wrote {p.atShadowFileNodeName()}")
947 p.moveToNodeAfterTree()
948 else:
949 p.moveToThreadNext()
950 else:
951 p.moveToThreadNext()
952 if g.unitTesting:
953 return found
954 if found:
955 g.es("finished")
956 else:
957 g.es("no dirty @shadow nodes in the selected tree")
958 c.raise_error_dialogs(kind='write')
959 return found
961 #@+node:ekr.20041005105605.157: *5* at.putFile
962 def putFile(self, root, fromString='', sentinels=True):
963 """Write the contents of the file to the output stream."""
964 at = self
965 s = fromString if fromString else root.v.b
966 root.clearAllVisitedInTree()
967 at.putAtFirstLines(s)
968 at.putOpenLeoSentinel("@+leo-ver=5")
969 at.putInitialComment()
970 at.putOpenNodeSentinel(root)
971 at.putBody(root, fromString=fromString)
972 # The -leo sentinel is required to handle @last.
973 at.putSentinel("@-leo")
974 root.setVisited()
975 at.putAtLastLines(s)
976 #@+node:ekr.20041005105605.147: *5* at.writeAll & helpers
977 def writeAll(self, all=False, dirty=False):
978 """Write @file nodes in all or part of the outline"""
979 at = self
980 # This is the *only* place where these are set.
981 # promptForDangerousWrite sets cancelFlag only if canCancelFlag is True.
982 at.unchangedFiles = 0
983 at.canCancelFlag = True
984 at.cancelFlag = False
985 at.yesToAll = False
986 files, root = at.findFilesToWrite(all)
987 for p in files:
988 try:
989 at.writeAllHelper(p, root)
990 except Exception: # pragma: no cover
991 at.internalWriteError(p)
992 # Make *sure* these flags are cleared for other commands.
993 at.canCancelFlag = False
994 at.cancelFlag = False
995 at.yesToAll = False
996 # Say the command is finished.
997 at.reportEndOfWrite(files, all, dirty)
998 # #2338: Never call at.saveOutlineIfPossible().
999 #@+node:ekr.20190108052043.1: *6* at.findFilesToWrite
1000 def findFilesToWrite(self, force): # pragma: no cover
1001 """
1002 Return a list of files to write.
1003 We must do this in a prepass, so as to avoid errors later.
1004 """
1005 trace = 'save' in g.app.debug and not g.unitTesting
1006 if trace:
1007 g.trace(f"writing *{'selected' if force else 'all'}* files")
1008 c = self.c
1009 if force:
1010 # The Write @<file> Nodes command.
1011 # Write all nodes in the selected tree.
1012 root = c.p
1013 p = c.p
1014 after = p.nodeAfterTree()
1015 else:
1016 # Write dirty nodes in the entire outline.
1017 root = c.rootPosition()
1018 p = c.rootPosition()
1019 after = None
1020 seen = set()
1021 files = []
1022 while p and p != after:
1023 if p.isAtIgnoreNode() and not p.isAtAsisFileNode():
1024 # Honor @ignore in *body* text, but *not* in @asis nodes.
1025 if p.isAnyAtFileNode():
1026 c.ignored_at_file_nodes.append(p.h)
1027 p.moveToNodeAfterTree()
1028 elif p.isAnyAtFileNode():
1029 data = p.v, g.fullPath(c, p)
1030 if data in seen:
1031 if trace and force:
1032 g.trace('Already seen', p.h)
1033 else:
1034 seen.add(data)
1035 files.append(p.copy())
1036 # Don't scan nested trees???
1037 p.moveToNodeAfterTree()
1038 else:
1039 p.moveToThreadNext()
1040 # When scanning *all* nodes, we only actually write dirty nodes.
1041 if not force:
1042 files = [z for z in files if z.isDirty()]
1043 if trace:
1044 g.printObj([z.h for z in files], tag='Files to be saved')
1045 return files, root
1046 #@+node:ekr.20190108053115.1: *6* at.internalWriteError
1047 def internalWriteError(self, p): # pragma: no cover
1048 """
1049 Fix bug 1260415: https://bugs.launchpad.net/leo-editor/+bug/1260415
1050 Give a more urgent, more specific, more helpful message.
1051 """
1052 g.es_exception()
1053 g.es(f"Internal error writing: {p.h}", color='red')
1054 g.es('Please report this error to:', color='blue')
1055 g.es('https://groups.google.com/forum/#!forum/leo-editor', color='blue')
1056 g.es('Warning: changes to this file will be lost', color='red')
1057 g.es('unless you can save the file successfully.', color='red')
1058 #@+node:ekr.20190108112519.1: *6* at.reportEndOfWrite
1059 def reportEndOfWrite(self, files, all, dirty): # pragma: no cover
1061 at = self
1062 if g.unitTesting:
1063 return
1064 if files:
1065 n = at.unchangedFiles
1066 g.es(f"finished: {n} unchanged file{g.plural(n)}")
1067 elif all:
1068 g.warning("no @<file> nodes in the selected tree")
1069 elif dirty:
1070 g.es("no dirty @<file> nodes in the selected tree")
1071 #@+node:ekr.20041005105605.149: *6* at.writeAllHelper & helper
1072 def writeAllHelper(self, p, root):
1073 """
1074 Write one file for at.writeAll.
1076 Do *not* write @auto files unless p == root.
1078 This prevents the write-all command from needlessly updating
1079 the @persistence data, thereby annoyingly changing the .leo file.
1080 """
1081 at = self
1082 at.root = root
1083 if p.isAtIgnoreNode(): # pragma: no cover
1084 # Should have been handled in findFilesToWrite.
1085 g.trace(f"Can not happen: {p.h} is an @ignore node")
1086 return
1087 try:
1088 at.writePathChanged(p)
1089 except IOError: # pragma: no cover
1090 return
1091 table = (
1092 (p.isAtAsisFileNode, at.asisWrite),
1093 (p.isAtAutoNode, at.writeOneAtAutoNode),
1094 (p.isAtCleanNode, at.writeOneAtCleanNode),
1095 (p.isAtEditNode, at.writeOneAtEditNode),
1096 (p.isAtFileNode, at.writeOneAtFileNode),
1097 (p.isAtNoSentFileNode, at.writeOneAtNosentNode),
1098 (p.isAtShadowFileNode, at.writeOneAtShadowNode),
1099 (p.isAtThinFileNode, at.writeOneAtFileNode),
1100 )
1101 for pred, func in table:
1102 if pred():
1103 func(p) # type:ignore
1104 break
1105 else: # pragma: no cover
1106 g.trace(f"Can not happen: {p.h}")
1107 return
1108 #
1109 # Clear the dirty bits in all descendant nodes.
1110 # The persistence data may still have to be written.
1111 for p2 in p.self_and_subtree(copy=False):
1112 p2.v.clearDirty()
1113 #@+node:ekr.20190108105509.1: *7* at.writePathChanged
1114 def writePathChanged(self, p): # pragma: no cover
1115 """
1116 raise IOError if p's path has changed *and* user forbids the write.
1117 """
1118 at, c = self, self.c
1119 #
1120 # Suppress this message during save-as and save-to commands.
1121 if c.ignoreChangedPaths:
1122 return # pragma: no cover
1123 oldPath = g.os_path_normcase(at.getPathUa(p))
1124 newPath = g.os_path_normcase(g.fullPath(c, p))
1125 try: # #1367: samefile can throw an exception.
1126 changed = oldPath and not os.path.samefile(oldPath, newPath)
1127 except Exception:
1128 changed = True
1129 if not changed:
1130 return
1131 ok = at.promptForDangerousWrite(
1132 fileName=None,
1133 message=(
1134 f"{g.tr('path changed for %s' % (p.h))}\n"
1135 f"{g.tr('write this file anyway?')}"
1136 ),
1137 )
1138 if not ok:
1139 raise IOError
1140 at.setPathUa(p, newPath) # Remember that we have changed paths.
1141 #@+node:ekr.20190109172025.1: *5* at.writeAtAutoContents
1142 def writeAtAutoContents(self, fileName, root): # pragma: no cover
1143 """Common helper for atAutoToString and writeOneAtAutoNode."""
1144 at, c = self, self.c
1145 # Dispatch the proper writer.
1146 junk, ext = g.os_path_splitext(fileName)
1147 writer = at.dispatch(ext, root)
1148 if writer:
1149 at.outputList = []
1150 writer(root)
1151 return '' if at.errors else ''.join(at.outputList)
1152 if root.isAtAutoRstNode():
1153 # An escape hatch: fall back to the theRst writer
1154 # if there is no rst writer plugin.
1155 at.outputFile = outputFile = io.StringIO()
1156 ok = c.rstCommands.writeAtAutoFile(root, fileName, outputFile)
1157 return outputFile.close() if ok else None
1158 # leo 5.6: allow undefined section references in all @auto files.
1159 ivar = 'allow_undefined_refs'
1160 try:
1161 setattr(at, ivar, True)
1162 at.outputList = []
1163 at.putFile(root, sentinels=False)
1164 return '' if at.errors else ''.join(at.outputList)
1165 except Exception:
1166 return None
1167 finally:
1168 if hasattr(at, ivar):
1169 delattr(at, ivar)
1170 #@+node:ekr.20190111153522.1: *5* at.writeX...
1171 #@+node:ekr.20041005105605.154: *6* at.asisWrite & helper
1172 def asisWrite(self, root): # pragma: no cover
1173 at, c = self, self.c
1174 try:
1175 c.endEditing()
1176 c.init_error_dialogs()
1177 fileName = at.initWriteIvars(root)
1178 # #1450.
1179 if not fileName or not at.precheck(fileName, root):
1180 at.addToOrphanList(root)
1181 return
1182 at.outputList = []
1183 for p in root.self_and_subtree(copy=False):
1184 at.writeAsisNode(p)
1185 if not at.errors:
1186 contents = ''.join(at.outputList)
1187 at.replaceFile(contents, at.encoding, fileName, root)
1188 except Exception:
1189 at.writeException(fileName, root)
1191 silentWrite = asisWrite # Compatibility with old scripts.
1192 #@+node:ekr.20170331141933.1: *7* at.writeAsisNode
1193 def writeAsisNode(self, p): # pragma: no cover
1194 """Write the p's node to an @asis file."""
1195 at = self
1197 def put(s):
1198 """Append s to self.output_list."""
1199 # #1480: Avoid calling at.os().
1200 s = g.toUnicode(s, at.encoding, reportErrors=True)
1201 at.outputList.append(s)
1203 # Write the headline only if it starts with '@@'.
1205 s = p.h
1206 if g.match(s, 0, "@@"):
1207 s = s[2:]
1208 if s:
1209 put('\n') # Experimental.
1210 put(s)
1211 put('\n')
1212 # Write the body.
1213 s = p.b
1214 if s:
1215 put(s)
1216 #@+node:ekr.20041005105605.151: *6* at.writeMissing & helper
1217 def writeMissing(self, p): # pragma: no cover
1218 at, c = self, self.c
1219 writtenFiles = False
1220 c.init_error_dialogs()
1221 # #1450.
1222 at.initWriteIvars(root=p.copy())
1223 p = p.copy()
1224 after = p.nodeAfterTree()
1225 while p and p != after: # Don't use iterator.
1226 if (
1227 p.isAtAsisFileNode() or (p.isAnyAtFileNode() and not p.isAtIgnoreNode())
1228 ):
1229 fileName = p.anyAtFileNodeName()
1230 if fileName:
1231 fileName = g.fullPath(c, p) # #1914.
1232 if at.precheck(fileName, p):
1233 at.writeMissingNode(p)
1234 writtenFiles = True
1235 else:
1236 at.addToOrphanList(p)
1237 p.moveToNodeAfterTree()
1238 elif p.isAtIgnoreNode():
1239 p.moveToNodeAfterTree()
1240 else:
1241 p.moveToThreadNext()
1242 if not g.unitTesting:
1243 if writtenFiles > 0:
1244 g.es("finished")
1245 else:
1246 g.es("no @file node in the selected tree")
1247 c.raise_error_dialogs(kind='write')
1248 #@+node:ekr.20041005105605.152: *7* at.writeMissingNode
1249 def writeMissingNode(self, p): # pragma: no cover
1251 at = self
1252 table = (
1253 (p.isAtAsisFileNode, at.asisWrite),
1254 (p.isAtAutoNode, at.writeOneAtAutoNode),
1255 (p.isAtCleanNode, at.writeOneAtCleanNode),
1256 (p.isAtEditNode, at.writeOneAtEditNode),
1257 (p.isAtFileNode, at.writeOneAtFileNode),
1258 (p.isAtNoSentFileNode, at.writeOneAtNosentNode),
1259 (p.isAtShadowFileNode, at.writeOneAtShadowNode),
1260 (p.isAtThinFileNode, at.writeOneAtFileNode),
1261 )
1262 for pred, func in table:
1263 if pred():
1264 func(p) # type:ignore
1265 return
1266 g.trace(f"Can not happen unknown @<file> kind: {p.h}")
1267 #@+node:ekr.20070806141607: *6* at.writeOneAtAutoNode & helpers
1268 def writeOneAtAutoNode(self, p): # pragma: no cover
1269 """
1270 Write p, an @auto node.
1271 File indices *must* have already been assigned.
1272 Return True if the node was written successfully.
1273 """
1274 at, c = self, self.c
1275 root = p.copy()
1276 try:
1277 c.endEditing()
1278 if not p.atAutoNodeName():
1279 return False
1280 fileName = at.initWriteIvars(root)
1281 at.sentinels = False
1282 # #1450.
1283 if not fileName or not at.precheck(fileName, root):
1284 at.addToOrphanList(root)
1285 return False
1286 if c.persistenceController:
1287 c.persistenceController.update_before_write_foreign_file(root)
1288 contents = at.writeAtAutoContents(fileName, root)
1289 if contents is None:
1290 g.es("not written:", fileName)
1291 at.addToOrphanList(root)
1292 return False
1293 at.replaceFile(contents, at.encoding, fileName, root,
1294 ignoreBlankLines=root.isAtAutoRstNode())
1295 return True
1296 except Exception:
1297 at.writeException(fileName, root)
1298 return False
1299 #@+node:ekr.20140728040812.17993: *7* at.dispatch & helpers
1300 def dispatch(self, ext, p): # pragma: no cover
1301 """Return the correct writer function for p, an @auto node."""
1302 at = self
1303 # Match @auto type before matching extension.
1304 return at.writer_for_at_auto(p) or at.writer_for_ext(ext)
1305 #@+node:ekr.20140728040812.17995: *8* at.writer_for_at_auto
1306 def writer_for_at_auto(self, root): # pragma: no cover
1307 """A factory returning a writer function for the given kind of @auto directive."""
1308 at = self
1309 d = g.app.atAutoWritersDict
1310 for key in d:
1311 aClass = d.get(key)
1312 if aClass and g.match_word(root.h, 0, key):
1314 def writer_for_at_auto_cb(root):
1315 # pylint: disable=cell-var-from-loop
1316 try:
1317 writer = aClass(at.c)
1318 s = writer.write(root)
1319 return s
1320 except Exception:
1321 g.es_exception()
1322 return None
1324 return writer_for_at_auto_cb
1325 return None
1326 #@+node:ekr.20140728040812.17997: *8* at.writer_for_ext
1327 def writer_for_ext(self, ext): # pragma: no cover
1328 """A factory returning a writer function for the given file extension."""
1329 at = self
1330 d = g.app.writersDispatchDict
1331 aClass = d.get(ext)
1332 if aClass:
1334 def writer_for_ext_cb(root):
1335 try:
1336 return aClass(at.c).write(root)
1337 except Exception:
1338 g.es_exception()
1339 return None
1341 return writer_for_ext_cb
1343 return None
1344 #@+node:ekr.20210501064359.1: *6* at.writeOneAtCleanNode
1345 def writeOneAtCleanNode(self, root): # pragma: no cover
1346 """Write one @clean file..
1347 root is the position of an @clean node.
1348 """
1349 at, c = self, self.c
1350 try:
1351 c.endEditing()
1352 fileName = at.initWriteIvars(root)
1353 at.sentinels = False
1354 if not fileName or not at.precheck(fileName, root):
1355 return
1356 at.outputList = []
1357 at.putFile(root, sentinels=False)
1358 at.warnAboutOrphandAndIgnoredNodes()
1359 if at.errors:
1360 g.es("not written:", g.shortFileName(fileName))
1361 at.addToOrphanList(root)
1362 else:
1363 contents = ''.join(at.outputList)
1364 at.replaceFile(contents, at.encoding, fileName, root)
1365 except Exception:
1366 at.writeException(fileName, root)
1367 #@+node:ekr.20090225080846.5: *6* at.writeOneAtEditNode
1368 def writeOneAtEditNode(self, p): # pragma: no cover
1369 """Write one @edit node."""
1370 at, c = self, self.c
1371 root = p.copy()
1372 try:
1373 c.endEditing()
1374 c.init_error_dialogs()
1375 if not p.atEditNodeName():
1376 return False
1377 if p.hasChildren():
1378 g.error('@edit nodes must not have children')
1379 g.es('To save your work, convert @edit to @auto, @file or @clean')
1380 return False
1381 fileName = at.initWriteIvars(root)
1382 at.sentinels = False
1383 # #1450.
1384 if not fileName or not at.precheck(fileName, root):
1385 at.addToOrphanList(root)
1386 return False
1387 contents = ''.join([s for s in g.splitLines(p.b)
1388 if at.directiveKind4(s, 0) == at.noDirective])
1389 at.replaceFile(contents, at.encoding, fileName, root)
1390 c.raise_error_dialogs(kind='write')
1391 return True
1392 except Exception:
1393 at.writeException(fileName, root)
1394 return False
1395 #@+node:ekr.20210501075610.1: *6* at.writeOneAtFileNode
1396 def writeOneAtFileNode(self, root): # pragma: no cover
1397 """Write @file or @thin file."""
1398 at, c = self, self.c
1399 try:
1400 c.endEditing()
1401 fileName = at.initWriteIvars(root)
1402 at.sentinels = True
1403 if not fileName or not at.precheck(fileName, root):
1404 # Raise dialog warning of data loss.
1405 at.addToOrphanList(root)
1406 return
1407 at.outputList = []
1408 at.putFile(root, sentinels=True)
1409 at.warnAboutOrphandAndIgnoredNodes()
1410 if at.errors:
1411 g.es("not written:", g.shortFileName(fileName))
1412 at.addToOrphanList(root)
1413 else:
1414 contents = ''.join(at.outputList)
1415 at.replaceFile(contents, at.encoding, fileName, root)
1416 except Exception:
1417 at.writeException(fileName, root)
1418 #@+node:ekr.20210501065352.1: *6* at.writeOneAtNosentNode
1419 def writeOneAtNosentNode(self, root): # pragma: no cover
1420 """Write one @nosent node.
1421 root is the position of an @<file> node.
1422 sentinels will be False for @clean and @nosent nodes.
1423 """
1424 at, c = self, self.c
1425 try:
1426 c.endEditing()
1427 fileName = at.initWriteIvars(root)
1428 at.sentinels = False
1429 if not fileName or not at.precheck(fileName, root):
1430 return
1431 at.outputList = []
1432 at.putFile(root, sentinels=False)
1433 at.warnAboutOrphandAndIgnoredNodes()
1434 if at.errors:
1435 g.es("not written:", g.shortFileName(fileName))
1436 at.addToOrphanList(root)
1437 else:
1438 contents = ''.join(at.outputList)
1439 at.replaceFile(contents, at.encoding, fileName, root)
1440 except Exception:
1441 at.writeException(fileName, root)
1442 #@+node:ekr.20080711093251.5: *6* at.writeOneAtShadowNode & helper
1443 def writeOneAtShadowNode(self, p, testing=False): # pragma: no cover
1444 """
1445 Write p, an @shadow node.
1446 File indices *must* have already been assigned.
1448 testing: set by unit tests to suppress the call to at.precheck.
1449 Testing is not the same as g.unitTesting.
1450 """
1451 at, c = self, self.c
1452 root = p.copy()
1453 x = c.shadowController
1454 try:
1455 c.endEditing() # Capture the current headline.
1456 fn = p.atShadowFileNodeName()
1457 assert fn, p.h
1458 self.adjustTargetLanguage(fn)
1459 # A hack to support unknown extensions. May set c.target_language.
1460 full_path = g.fullPath(c, p)
1461 at.initWriteIvars(root)
1462 # Force python sentinels to suppress an error message.
1463 # The actual sentinels will be set below.
1464 at.endSentinelComment = None
1465 at.startSentinelComment = "#"
1466 # Make sure we can compute the shadow directory.
1467 private_fn = x.shadowPathName(full_path)
1468 if not private_fn:
1469 return False
1470 if not testing and not at.precheck(full_path, root):
1471 return False
1472 #
1473 # Bug fix: Leo 4.5.1:
1474 # use x.markerFromFileName to force the delim to match
1475 # what is used in x.propegate changes.
1476 marker = x.markerFromFileName(full_path)
1477 at.startSentinelComment, at.endSentinelComment = marker.getDelims()
1478 if g.unitTesting:
1479 ivars_dict = g.getIvarsDict(at)
1480 #
1481 # Write the public and private files to strings.
1483 def put(sentinels):
1484 at.outputList = []
1485 at.sentinels = sentinels
1486 at.putFile(root, sentinels=sentinels)
1487 return '' if at.errors else ''.join(at.outputList)
1489 at.public_s = put(False)
1490 at.private_s = put(True)
1491 at.warnAboutOrphandAndIgnoredNodes()
1492 if g.unitTesting:
1493 exceptions = ('public_s', 'private_s', 'sentinels', 'outputList')
1494 assert g.checkUnchangedIvars(
1495 at, ivars_dict, exceptions), 'writeOneAtShadowNode'
1496 if not at.errors:
1497 # Write the public and private files.
1498 x.makeShadowDirectory(full_path)
1499 # makeShadowDirectory takes a *public* file name.
1500 x.replaceFileWithString(at.encoding, private_fn, at.private_s)
1501 x.replaceFileWithString(at.encoding, full_path, at.public_s)
1502 at.checkPythonCode(contents=at.private_s, fileName=full_path, root=root)
1503 if at.errors:
1504 g.error("not written:", full_path)
1505 at.addToOrphanList(root)
1506 else:
1507 root.clearDirty()
1508 return not at.errors
1509 except Exception:
1510 at.writeException(full_path, root)
1511 return False
1512 #@+node:ekr.20080819075811.13: *7* at.adjustTargetLanguage
1513 def adjustTargetLanguage(self, fn): # pragma: no cover
1514 """Use the language implied by fn's extension if
1515 there is a conflict between it and c.target_language."""
1516 at = self
1517 c = at.c
1518 junk, ext = g.os_path_splitext(fn)
1519 if ext:
1520 if ext.startswith('.'):
1521 ext = ext[1:]
1522 language = g.app.extension_dict.get(ext)
1523 if language:
1524 c.target_language = language
1525 else:
1526 # An unknown language.
1527 # Use the default language, **not** 'unknown_language'
1528 pass
1529 #@+node:ekr.20190111153506.1: *5* at.XToString
1530 #@+node:ekr.20190109160056.1: *6* at.atAsisToString
1531 def atAsisToString(self, root): # pragma: no cover
1532 """Write the @asis node to a string."""
1533 # pylint: disable=used-before-assignment
1534 at, c = self, self.c
1535 try:
1536 c.endEditing()
1537 fileName = at.initWriteIvars(root)
1538 at.outputList = []
1539 for p in root.self_and_subtree(copy=False):
1540 at.writeAsisNode(p)
1541 return '' if at.errors else ''.join(at.outputList)
1542 except Exception:
1543 at.writeException(fileName, root)
1544 return ''
1545 #@+node:ekr.20190109160056.2: *6* at.atAutoToString
1546 def atAutoToString(self, root): # pragma: no cover
1547 """Write the root @auto node to a string, and return it."""
1548 at, c = self, self.c
1549 try:
1550 c.endEditing()
1551 fileName = at.initWriteIvars(root)
1552 at.sentinels = False
1553 # #1450.
1554 if not fileName:
1555 at.addToOrphanList(root)
1556 return ''
1557 return at.writeAtAutoContents(fileName, root) or ''
1558 except Exception:
1559 at.writeException(fileName, root)
1560 return ''
1561 #@+node:ekr.20190109160056.3: *6* at.atEditToString
1562 def atEditToString(self, root): # pragma: no cover
1563 """Write one @edit node."""
1564 at, c = self, self.c
1565 try:
1566 c.endEditing()
1567 if root.hasChildren():
1568 g.error('@edit nodes must not have children')
1569 g.es('To save your work, convert @edit to @auto, @file or @clean')
1570 return False
1571 fileName = at.initWriteIvars(root)
1572 at.sentinels = False
1573 # #1450.
1574 if not fileName:
1575 at.addToOrphanList(root)
1576 return ''
1577 contents = ''.join([
1578 s for s in g.splitLines(root.b)
1579 if at.directiveKind4(s, 0) == at.noDirective])
1580 return contents
1581 except Exception:
1582 at.writeException(fileName, root)
1583 return ''
1584 #@+node:ekr.20190109142026.1: *6* at.atFileToString
1585 def atFileToString(self, root, sentinels=True): # pragma: no cover
1586 """Write an external file to a string, and return its contents."""
1587 at, c = self, self.c
1588 try:
1589 c.endEditing()
1590 at.initWriteIvars(root)
1591 at.sentinels = sentinels
1592 at.outputList = []
1593 at.putFile(root, sentinels=sentinels)
1594 assert root == at.root, 'write'
1595 contents = '' if at.errors else ''.join(at.outputList)
1596 return contents
1597 except Exception:
1598 at.exception("exception preprocessing script")
1599 root.v._p_changed = True
1600 return ''
1601 #@+node:ekr.20050506084734: *6* at.stringToString
1602 def stringToString(self, root, s, forcePythonSentinels=True, sentinels=True): # pragma: no cover
1603 """
1604 Write an external file from a string.
1606 This is at.write specialized for scripting.
1607 """
1608 at, c = self, self.c
1609 try:
1610 c.endEditing()
1611 at.initWriteIvars(root)
1612 if forcePythonSentinels:
1613 at.endSentinelComment = None
1614 at.startSentinelComment = "#"
1615 at.language = "python"
1616 at.sentinels = sentinels
1617 at.outputList = []
1618 at.putFile(root, fromString=s, sentinels=sentinels)
1619 contents = '' if at.errors else ''.join(at.outputList)
1620 # Major bug: failure to clear this wipes out headlines!
1621 # Sometimes this causes slight problems...
1622 if root:
1623 root.v._p_changed = True
1624 return contents
1625 except Exception:
1626 at.exception("exception preprocessing script")
1627 return ''
1628 #@+node:ekr.20041005105605.160: *4* Writing helpers
1629 #@+node:ekr.20041005105605.161: *5* at.putBody & helper
1630 def putBody(self, p, fromString=''):
1631 """
1632 Generate the body enclosed in sentinel lines.
1633 Return True if the body contains an @others line.
1634 """
1635 at = self
1636 #
1637 # New in 4.3 b2: get s from fromString if possible.
1638 s = fromString if fromString else p.b
1639 p.v.setVisited()
1640 # Make sure v is never expanded again.
1641 # Suppress orphans check.
1642 #
1643 # #1048 & #1037: regularize most trailing whitespace.
1644 if s and (at.sentinels or at.force_newlines_in_at_nosent_bodies):
1645 if not s.endswith('\n'):
1646 s = s + '\n'
1649 class Status:
1650 at_comment_seen = False
1651 at_delims_seen = False
1652 at_warning_given = False
1653 has_at_others = False
1654 in_code = True
1657 i = 0
1658 status = Status()
1659 while i < len(s):
1660 next_i = g.skip_line(s, i)
1661 assert next_i > i, 'putBody'
1662 kind = at.directiveKind4(s, i)
1663 at.putLine(i, kind, p, s, status)
1664 i = next_i
1665 if not status.in_code:
1666 at.putEndDocLine()
1667 return status.has_at_others
1668 #@+node:ekr.20041005105605.163: *6* at.putLine
1669 def putLine(self, i, kind, p, s, status):
1670 """Put the line at s[i:] of the given kind, updating the status."""
1671 at = self
1672 if kind == at.noDirective:
1673 if status.in_code:
1674 # Important: the so-called "name" must include brackets.
1675 name, n1, n2 = at.findSectionName(s, i, p)
1676 if name:
1677 at.putRefLine(s, i, n1, n2, name, p)
1678 else:
1679 at.putCodeLine(s, i)
1680 else:
1681 at.putDocLine(s, i)
1682 elif kind in (at.docDirective, at.atDirective):
1683 if not status.in_code:
1684 # Bug fix 12/31/04: handle adjacent doc parts.
1685 at.putEndDocLine()
1686 at.putStartDocLine(s, i, kind)
1687 status.in_code = False
1688 elif kind in (at.cDirective, at.codeDirective):
1689 # Only @c and @code end a doc part.
1690 if not status.in_code:
1691 at.putEndDocLine()
1692 at.putDirective(s, i, p)
1693 status.in_code = True
1694 elif kind == at.allDirective:
1695 if status.in_code:
1696 if p == self.root:
1697 at.putAtAllLine(s, i, p)
1698 else:
1699 at.error(f"@all not valid in: {p.h}") # pragma: no cover
1700 else:
1701 at.putDocLine(s, i)
1702 elif kind == at.othersDirective:
1703 if status.in_code:
1704 if status.has_at_others:
1705 at.error(f"multiple @others in: {p.h}") # pragma: no cover
1706 else:
1707 at.putAtOthersLine(s, i, p)
1708 status.has_at_others = True
1709 else:
1710 at.putDocLine(s, i)
1711 elif kind == at.startVerbatim: # pragma: no cover
1712 # Fix bug 778204: @verbatim not a valid Leo directive.
1713 if g.unitTesting:
1714 # A hack: unit tests for @shadow use @verbatim as a kind of directive.
1715 pass
1716 else:
1717 at.error(f"@verbatim is not a Leo directive: {p.h}")
1718 elif kind == at.miscDirective:
1719 # Fix bug 583878: Leo should warn about @comment/@delims clashes.
1720 if g.match_word(s, i, '@comment'):
1721 status.at_comment_seen = True
1722 elif g.match_word(s, i, '@delims'):
1723 status.at_delims_seen = True
1724 if (
1725 status.at_comment_seen and
1726 status.at_delims_seen and not
1727 status.at_warning_given
1728 ): # pragma: no cover
1729 status.at_warning_given = True
1730 at.error(f"@comment and @delims in node {p.h}")
1731 at.putDirective(s, i, p)
1732 else:
1733 at.error(f"putBody: can not happen: unknown directive kind: {kind}") # pragma: no cover
1734 #@+node:ekr.20041005105605.164: *5* writing code lines...
1735 #@+node:ekr.20041005105605.165: *6* at: @all
1736 #@+node:ekr.20041005105605.166: *7* at.putAtAllLine
1737 def putAtAllLine(self, s, i, p):
1738 """Put the expansion of @all."""
1739 at = self
1740 j, delta = g.skip_leading_ws_with_indent(s, i, at.tab_width)
1741 k = g.skip_to_end_of_line(s, i)
1742 at.putLeadInSentinel(s, i, j)
1743 at.indent += delta
1744 at.putSentinel("@+" + s[j + 1 : k].strip())
1745 # s[j:k] starts with '@all'
1746 for child in p.children():
1747 at.putAtAllChild(child)
1748 at.putSentinel("@-all")
1749 at.indent -= delta
1750 #@+node:ekr.20041005105605.167: *7* at.putAtAllBody
1751 def putAtAllBody(self, p):
1752 """ Generate the body enclosed in sentinel lines."""
1753 at = self
1754 s = p.b
1755 p.v.setVisited()
1756 # Make sure v is never expanded again.
1757 # Suppress orphans check.
1758 if at.sentinels and s and s[-1] != '\n':
1759 s = s + '\n'
1760 i = 0
1761 # Leo 6.6. This code never changes at.in_code status!
1762 while i < len(s):
1763 next_i = g.skip_line(s, i)
1764 assert next_i > i
1765 at.putCodeLine(s, i)
1766 i = next_i
1767 #@+node:ekr.20041005105605.169: *7* at.putAtAllChild
1768 def putAtAllChild(self, p):
1769 """
1770 This code puts only the first of two or more cloned siblings, preceding
1771 the clone with an @clone n sentinel.
1773 This is a debatable choice: the cloned tree appears only once in the
1774 external file. This should be benign; the text created by @all is
1775 likely to be used only for recreating the outline in Leo. The
1776 representation in the derived file doesn't matter much.
1777 """
1778 at = self
1779 at.putOpenNodeSentinel(p, inAtAll=True)
1780 # Suppress warnings about @file nodes.
1781 at.putAtAllBody(p)
1782 for child in p.children():
1783 at.putAtAllChild(child) # pragma: no cover (recursive call)
1784 #@+node:ekr.20041005105605.170: *6* at: @others
1785 #@+node:ekr.20041005105605.173: *7* at.putAtOthersLine & helper
1786 def putAtOthersLine(self, s, i, p):
1787 """Put the expansion of @others."""
1788 at = self
1789 j, delta = g.skip_leading_ws_with_indent(s, i, at.tab_width)
1790 k = g.skip_to_end_of_line(s, i)
1791 at.putLeadInSentinel(s, i, j)
1792 at.indent += delta
1793 # s[j:k] starts with '@others'
1794 # Never write lws in new sentinels.
1795 at.putSentinel("@+" + s[j + 1 : k].strip())
1796 for child in p.children():
1797 p = child.copy()
1798 after = p.nodeAfterTree()
1799 while p and p != after:
1800 if at.validInAtOthers(p):
1801 at.putOpenNodeSentinel(p)
1802 at_others_flag = at.putBody(p)
1803 if at_others_flag:
1804 p.moveToNodeAfterTree()
1805 else:
1806 p.moveToThreadNext()
1807 else:
1808 p.moveToNodeAfterTree()
1809 # This is the same in both old and new sentinels.
1810 at.putSentinel("@-others")
1811 at.indent -= delta
1812 #@+node:ekr.20041005105605.171: *8* at.validInAtOthers
1813 def validInAtOthers(self, p):
1814 """
1815 Return True if p should be included in the expansion of the @others
1816 directive in the body text of p's parent.
1817 """
1818 at = self
1819 i = g.skip_ws(p.h, 0)
1820 isSection, junk = at.isSectionName(p.h, i)
1821 if isSection:
1822 return False # A section definition node.
1823 if at.sentinels:
1824 # @ignore must not stop expansion here!
1825 return True
1826 if p.isAtIgnoreNode(): # pragma: no cover
1827 g.error('did not write @ignore node', p.v.h)
1828 return False
1829 return True
1830 #@+node:ekr.20041005105605.199: *6* at.findSectionName
1831 def findSectionName(self, s, i, p):
1832 """
1833 Return n1, n2 representing a section name.
1835 Return the reference, *including* brackes.
1836 """
1837 at = self
1839 def is_space(i1, i2):
1840 """A replacement for s[i1 : i2] that doesn't create any substring."""
1841 return i == j or all(s[z] in ' \t\n' for z in range(i1, i2))
1843 end = s.find('\n', i)
1844 j = len(s) if end == -1 else end
1845 # Careful: don't look beyond the end of the line!
1846 if end == -1:
1847 n1 = s.find(at.section_delim1, i)
1848 n2 = s.find(at.section_delim2, i)
1849 else:
1850 n1 = s.find(at.section_delim1, i, end)
1851 n2 = s.find(at.section_delim2, i, end)
1852 n3 = n2 + len(at.section_delim2)
1853 if -1 < n1 < n2: # A *possible* section reference.
1854 if is_space(i, n1) and is_space(n3, j): # A *real* section reference.
1855 return s[n1:n3], n1, n3
1856 # An apparent section reference.
1857 if 'sections' in g.app.debug and not g.unitTesting: # pragma: no cover
1858 i1, i2 = g.getLine(s, i)
1859 g.es_print('Ignoring apparent section reference:', color='red')
1860 g.es_print('Node: ', p.h)
1861 g.es_print('Line: ', s[i1:i2].rstrip())
1862 return None, 0, 0
1863 #@+node:ekr.20041005105605.174: *6* at.putCodeLine
1864 def putCodeLine(self, s, i):
1865 """Put a normal code line."""
1866 at = self
1867 # Put @verbatim sentinel if required.
1868 k = g.skip_ws(s, i)
1869 if g.match(s, k, self.startSentinelComment + '@'):
1870 self.putSentinel('@verbatim')
1871 j = g.skip_line(s, i)
1872 line = s[i:j]
1873 # Don't put any whitespace in otherwise blank lines.
1874 if len(line) > 1: # Preserve *anything* the user puts on the line!!!
1875 at.putIndent(at.indent, line)
1876 if line[-1:] == '\n':
1877 at.os(line[:-1])
1878 at.onl()
1879 else:
1880 at.os(line)
1881 elif line and line[-1] == '\n':
1882 at.onl()
1883 elif line:
1884 at.os(line) # Bug fix: 2013/09/16
1885 else:
1886 g.trace('Can not happen: completely empty line') # pragma: no cover
1887 #@+node:ekr.20041005105605.176: *6* at.putRefLine
1888 def putRefLine(self, s, i, n1, n2, name, p):
1889 """
1890 Put a line containing one or more references.
1892 Important: the so-called name *must* include brackets.
1893 """
1894 at = self
1895 ref = g.findReference(name, p)
1896 if ref:
1897 junk, delta = g.skip_leading_ws_with_indent(s, i, at.tab_width)
1898 at.putLeadInSentinel(s, i, n1)
1899 at.indent += delta
1900 at.putSentinel("@+" + name)
1901 at.putOpenNodeSentinel(ref)
1902 at.putBody(ref)
1903 at.putSentinel("@-" + name)
1904 at.indent -= delta
1905 return
1906 if hasattr(at, 'allow_undefined_refs'): # pragma: no cover
1907 p.v.setVisited() # #2311
1908 # Allow apparent section reference: just write the line.
1909 at.putCodeLine(s, i)
1910 else: # pragma: no cover
1911 # Do give this error even if unit testing.
1912 at.writeError(
1913 f"undefined section: {g.truncate(name, 60)}\n"
1914 f" referenced from: {g.truncate(p.h, 60)}")
1915 #@+node:ekr.20041005105605.180: *5* writing doc lines...
1916 #@+node:ekr.20041005105605.181: *6* at.putBlankDocLine
1917 def putBlankDocLine(self):
1918 at = self
1919 if not at.endSentinelComment:
1920 at.putIndent(at.indent)
1921 at.os(at.startSentinelComment)
1922 # #1496: Retire the @doc convention.
1923 # Remove the blank.
1924 # at.oblank()
1925 at.onl()
1926 #@+node:ekr.20041005105605.183: *6* at.putDocLine
1927 def putDocLine(self, s, i):
1928 """Handle one line of a doc part."""
1929 at = self
1930 j = g.skip_line(s, i)
1931 s = s[i:j]
1932 #
1933 # #1496: Retire the @doc convention:
1934 # Strip all trailing ws here.
1935 if not s.strip():
1936 # A blank line.
1937 at.putBlankDocLine()
1938 return
1939 # Write the line as it is.
1940 at.putIndent(at.indent)
1941 if not at.endSentinelComment:
1942 at.os(at.startSentinelComment)
1943 # #1496: Retire the @doc convention.
1944 # Leave this blank. The line is not blank.
1945 at.oblank()
1946 at.os(s)
1947 if not s.endswith('\n'):
1948 at.onl() # pragma: no cover
1949 #@+node:ekr.20041005105605.185: *6* at.putEndDocLine
1950 def putEndDocLine(self):
1951 """Write the conclusion of a doc part."""
1952 at = self
1953 # Put the closing delimiter if we are using block comments.
1954 if at.endSentinelComment:
1955 at.putIndent(at.indent)
1956 at.os(at.endSentinelComment)
1957 at.onl() # Note: no trailing whitespace.
1958 #@+node:ekr.20041005105605.182: *6* at.putStartDocLine
1959 def putStartDocLine(self, s, i, kind):
1960 """Write the start of a doc part."""
1961 at = self
1962 sentinel = "@+doc" if kind == at.docDirective else "@+at"
1963 directive = "@doc" if kind == at.docDirective else "@"
1964 # Put whatever follows the directive in the sentinel.
1965 # Skip past the directive.
1966 i += len(directive)
1967 j = g.skip_to_end_of_line(s, i)
1968 follow = s[i:j]
1969 # Put the opening @+doc or @-doc sentinel, including whatever follows the directive.
1970 at.putSentinel(sentinel + follow)
1971 # Put the opening comment if we are using block comments.
1972 if at.endSentinelComment:
1973 at.putIndent(at.indent)
1974 at.os(at.startSentinelComment)
1975 at.onl()
1976 #@+node:ekr.20041005105605.187: *4* Writing sentinels...
1977 #@+node:ekr.20041005105605.188: *5* at.nodeSentinelText & helper
1978 def nodeSentinelText(self, p):
1979 """Return the text of a @+node or @-node sentinel for p."""
1980 at = self
1981 h = at.removeCommentDelims(p)
1982 if getattr(at, 'at_shadow_test_hack', False): # pragma: no cover
1983 # A hack for @shadow unit testing.
1984 # see AtShadowTestCase.makePrivateLines.
1985 return h
1986 gnx = p.v.fileIndex
1987 level = 1 + p.level() - self.root.level()
1988 if level > 2:
1989 return f"{gnx}: *{level}* {h}"
1990 return f"{gnx}: {'*' * level} {h}"
1991 #@+node:ekr.20041005105605.189: *6* at.removeCommentDelims
1992 def removeCommentDelims(self, p):
1993 """
1994 If the present @language/@comment settings do not specify a single-line comment
1995 we remove all block comment delims from h. This prevents headline text from
1996 interfering with the parsing of node sentinels.
1997 """
1998 at = self
1999 start = at.startSentinelComment
2000 end = at.endSentinelComment
2001 h = p.h
2002 if end:
2003 h = h.replace(start, "")
2004 h = h.replace(end, "")
2005 return h
2006 #@+node:ekr.20041005105605.190: *5* at.putLeadInSentinel
2007 def putLeadInSentinel(self, s, i, j):
2008 """
2009 Set at.leadingWs as needed for @+others and @+<< sentinels.
2011 i points at the start of a line.
2012 j points at @others or a section reference.
2013 """
2014 at = self
2015 at.leadingWs = "" # Set the default.
2016 if i == j:
2017 return # The @others or ref starts a line.
2018 k = g.skip_ws(s, i)
2019 if j == k:
2020 # Remember the leading whitespace, including its spelling.
2021 at.leadingWs = s[i:j]
2022 else:
2023 self.putIndent(at.indent) # 1/29/04: fix bug reported by Dan Winkler.
2024 at.os(s[i:j])
2025 at.onl_sent()
2026 #@+node:ekr.20041005105605.192: *5* at.putOpenLeoSentinel 4.x
2027 def putOpenLeoSentinel(self, s):
2028 """Write @+leo sentinel."""
2029 at = self
2030 if at.sentinels or hasattr(at, 'force_sentinels'):
2031 s = s + "-thin"
2032 encoding = at.encoding.lower()
2033 if encoding != "utf-8": # pragma: no cover
2034 # New in 4.2: encoding fields end in ",."
2035 s = s + f"-encoding={encoding},."
2036 at.putSentinel(s)
2037 #@+node:ekr.20041005105605.193: *5* at.putOpenNodeSentinel
2038 def putOpenNodeSentinel(self, p, inAtAll=False):
2039 """Write @+node sentinel for p."""
2040 # Note: lineNumbers.py overrides this method.
2041 at = self
2042 if not inAtAll and p.isAtFileNode() and p != at.root: # pragma: no cover
2043 at.writeError("@file not valid in: " + p.h)
2044 return
2045 s = at.nodeSentinelText(p)
2046 at.putSentinel("@+node:" + s)
2047 # Leo 4.7: we never write tnodeLists.
2048 #@+node:ekr.20041005105605.194: *5* at.putSentinel (applies cweb hack) 4.x
2049 def putSentinel(self, s):
2050 """
2051 Write a sentinel whose text is s, applying the CWEB hack if needed.
2053 This method outputs all sentinels.
2054 """
2055 at = self
2056 if at.sentinels or hasattr(at, 'force_sentinels'):
2057 at.putIndent(at.indent)
2058 at.os(at.startSentinelComment)
2059 # #2194. The following would follow the black convention,
2060 # but doing so is a dubious idea.
2061 # at.os(' ')
2062 # Apply the cweb hack to s:
2063 # If the opening comment delim ends in '@',
2064 # double all '@' signs except the first.
2065 start = at.startSentinelComment
2066 if start and start[-1] == '@':
2067 s = s.replace('@', '@@')[1:]
2068 at.os(s)
2069 if at.endSentinelComment:
2070 at.os(at.endSentinelComment)
2071 at.onl()
2072 #@+node:ekr.20041005105605.196: *4* Writing utils...
2073 #@+node:ekr.20181024134823.1: *5* at.addToOrphanList
2074 def addToOrphanList(self, root): # pragma: no cover
2075 """Mark the root as erroneous for c.raise_error_dialogs()."""
2076 c = self.c
2077 # Fix #1050:
2078 root.setOrphan()
2079 c.orphan_at_file_nodes.append(root.h)
2080 #@+node:ekr.20220120210617.1: *5* at.checkPyflakes
2081 def checkPyflakes(self, contents, fileName, root): # pragma: no cover
2082 at = self
2083 ok = True
2084 if g.unitTesting or not at.runPyFlakesOnWrite:
2085 return ok
2086 if not contents or not fileName or not fileName.endswith('.py'):
2087 return ok
2088 ok = self.runPyflakes(root)
2089 if not ok:
2090 g.app.syntax_error_files.append(g.shortFileName(fileName))
2091 return ok
2092 #@+node:ekr.20090514111518.5661: *5* at.checkPythonCode & helpers
2093 def checkPythonCode(self, contents, fileName, root): # pragma: no cover
2094 """Perform python-related checks on root."""
2095 at = self
2096 if g.unitTesting or not contents or not fileName or not fileName.endswith('.py'):
2097 return
2098 ok = True
2099 if at.checkPythonCodeOnWrite:
2100 ok = at.checkPythonSyntax(root, contents)
2101 if ok and at.runPyFlakesOnWrite:
2102 ok = self.runPyflakes(root)
2103 if not ok:
2104 g.app.syntax_error_files.append(g.shortFileName(fileName))
2105 #@+node:ekr.20090514111518.5663: *6* at.checkPythonSyntax
2106 def checkPythonSyntax(self, p, body):
2107 at = self
2108 try:
2109 body = body.replace('\r', '')
2110 fn = f"<node: {p.h}>"
2111 compile(body + '\n', fn, 'exec')
2112 return True
2113 except SyntaxError: # pragma: no cover
2114 if not g.unitTesting:
2115 at.syntaxError(p, body)
2116 except Exception: # pragma: no cover
2117 g.trace("unexpected exception")
2118 g.es_exception()
2119 return False
2120 #@+node:ekr.20090514111518.5666: *7* at.syntaxError (leoAtFile)
2121 def syntaxError(self, p, body): # pragma: no cover
2122 """Report a syntax error."""
2123 g.error(f"Syntax error in: {p.h}")
2124 typ, val, tb = sys.exc_info()
2125 message = hasattr(val, 'message') and val.message
2126 if message:
2127 g.es_print(message)
2128 if val is None:
2129 return
2130 lines = g.splitLines(body)
2131 n = val.lineno
2132 offset = val.offset or 0
2133 if n is None:
2134 return
2135 i = val.lineno - 1
2136 for j in range(max(0, i - 2), min(i + 2, len(lines) - 1)):
2137 line = lines[j].rstrip()
2138 if j == i:
2139 unl = p.get_UNL()
2140 g.es_print(f"{j+1:5}:* {line}", nodeLink=f"{unl}::-{j+1:d}") # Global line.
2141 g.es_print(' ' * (7 + offset) + '^')
2142 else:
2143 g.es_print(f"{j+1:5}: {line}")
2144 #@+node:ekr.20161021084954.1: *6* at.runPyflakes
2145 def runPyflakes(self, root): # pragma: no cover
2146 """Run pyflakes on the selected node."""
2147 try:
2148 from leo.commands import checkerCommands
2149 if checkerCommands.pyflakes:
2150 x = checkerCommands.PyflakesCommand(self.c)
2151 ok = x.run(root)
2152 return ok
2153 return True # Suppress error if pyflakes can not be imported.
2154 except Exception:
2155 g.es_exception()
2156 return True # Pretend all is well
2157 #@+node:ekr.20041005105605.198: *5* at.directiveKind4 (write logic)
2158 # These patterns exclude constructs such as @encoding.setter or @encoding(whatever)
2159 # However, they must allow @language python, @nocolor-node, etc.
2161 at_directive_kind_pattern = re.compile(r'\s*@([\w-]+)\s*')
2163 def directiveKind4(self, s, i):
2164 """
2165 Return the kind of at-directive or noDirective.
2167 Potential simplifications:
2168 - Using strings instead of constants.
2169 - Using additional regex's to recognize directives.
2170 """
2171 at = self
2172 n = len(s)
2173 if i >= n or s[i] != '@':
2174 j = g.skip_ws(s, i)
2175 if g.match_word(s, j, "@others"):
2176 return at.othersDirective
2177 if g.match_word(s, j, "@all"):
2178 return at.allDirective
2179 return at.noDirective
2180 table = (
2181 ("@all", at.allDirective),
2182 ("@c", at.cDirective),
2183 ("@code", at.codeDirective),
2184 ("@doc", at.docDirective),
2185 ("@others", at.othersDirective),
2186 ("@verbatim", at.startVerbatim))
2187 # ("@end_raw", at.endRawDirective), # #2276.
2188 # ("@raw", at.rawDirective), # #2276
2189 # Rewritten 6/8/2005.
2190 if i + 1 >= n or s[i + 1] in (' ', '\t', '\n'):
2191 # Bare '@' not recognized in cweb mode.
2192 return at.noDirective if at.language == "cweb" else at.atDirective
2193 if not s[i + 1].isalpha():
2194 return at.noDirective # Bug fix: do NOT return miscDirective here!
2195 if at.language == "cweb" and g.match_word(s, i, '@c'):
2196 return at.noDirective
2197 # When the language is elixir, @doc followed by a space and string delimiter
2198 # needs to be treated as plain text; the following does not enforce the
2199 # 'string delimiter' part of that. An @doc followed by something other than
2200 # a space will fall through to usual Leo @doc processing.
2201 if at.language == "elixir" and g.match_word(s, i, '@doc '): # pragma: no cover
2202 return at.noDirective
2203 for name, directive in table:
2204 if g.match_word(s, i, name):
2205 return directive
2206 # Support for add_directives plugin.
2207 # Use regex to properly distinguish between Leo directives
2208 # and python decorators.
2209 s2 = s[i:]
2210 m = self.at_directive_kind_pattern.match(s2)
2211 if m:
2212 word = m.group(1)
2213 if word not in g.globalDirectiveList:
2214 return at.noDirective
2215 s3 = s2[m.end(1) :]
2216 if s3 and s3[0] in ".(":
2217 return at.noDirective
2218 return at.miscDirective
2219 # An unusual case.
2220 return at.noDirective # pragma: no cover
2221 #@+node:ekr.20041005105605.200: *5* at.isSectionName
2222 # returns (flag, end). end is the index of the character after the section name.
2224 def isSectionName(self, s, i): # pragma: no cover
2226 at = self
2227 # Allow leading periods.
2228 while i < len(s) and s[i] == '.':
2229 i += 1
2230 if not g.match(s, i, at.section_delim1):
2231 return False, -1
2232 i = g.find_on_line(s, i, at.section_delim2)
2233 if i > -1:
2234 return True, i + len(at.section_delim2)
2235 return False, -1
2236 #@+node:ekr.20190111112442.1: *5* at.isWritable
2237 def isWritable(self, path): # pragma: no cover
2238 """Return True if the path is writable."""
2239 try:
2240 # os.access() may not exist on all platforms.
2241 ok = os.access(path, os.W_OK)
2242 except AttributeError:
2243 return True
2244 if not ok:
2245 g.es('read only:', repr(path), color='red')
2246 return ok
2247 #@+node:ekr.20041005105605.201: *5* at.os and allies
2248 #@+node:ekr.20041005105605.202: *6* at.oblank, oblanks & otabs
2249 def oblank(self):
2250 self.os(' ')
2252 def oblanks(self, n): # pragma: no cover
2253 self.os(' ' * abs(n))
2255 def otabs(self, n): # pragma: no cover
2256 self.os('\t' * abs(n))
2257 #@+node:ekr.20041005105605.203: *6* at.onl & onl_sent
2258 def onl(self):
2259 """Write a newline to the output stream."""
2260 self.os('\n') # **not** self.output_newline
2262 def onl_sent(self):
2263 """Write a newline to the output stream, provided we are outputting sentinels."""
2264 if self.sentinels:
2265 self.onl()
2266 #@+node:ekr.20041005105605.204: *6* at.os
2267 def os(self, s):
2268 """
2269 Append a string to at.outputList.
2271 All output produced by leoAtFile module goes here.
2272 """
2273 at = self
2274 if s.startswith(self.underindentEscapeString): # pragma: no cover
2275 try:
2276 junk, s = at.parseUnderindentTag(s)
2277 except Exception:
2278 at.exception("exception writing:" + s)
2279 return
2280 s = g.toUnicode(s, at.encoding)
2281 at.outputList.append(s)
2282 #@+node:ekr.20041005105605.205: *5* at.outputStringWithLineEndings
2283 def outputStringWithLineEndings(self, s): # pragma: no cover
2284 """
2285 Write the string s as-is except that we replace '\n' with the proper line ending.
2287 Calling self.onl() runs afoul of queued newlines.
2288 """
2289 at = self
2290 s = g.toUnicode(s, at.encoding)
2291 s = s.replace('\n', at.output_newline)
2292 self.os(s)
2293 #@+node:ekr.20190111045822.1: *5* at.precheck (calls shouldPrompt...)
2294 def precheck(self, fileName, root): # pragma: no cover
2295 """
2296 Check whether a dirty, potentially dangerous, file should be written.
2298 Return True if so. Return False *and* issue a warning otherwise.
2299 """
2300 at = self
2301 #
2302 # #1450: First, check that the directory exists.
2303 theDir = g.os_path_dirname(fileName)
2304 if theDir and not g.os_path_exists(theDir):
2305 at.error(f"Directory not found:\n{theDir}")
2306 return False
2307 #
2308 # Now check the file.
2309 if not at.shouldPromptForDangerousWrite(fileName, root):
2310 # Fix bug 889175: Remember the full fileName.
2311 at.rememberReadPath(fileName, root)
2312 return True
2313 #
2314 # Prompt if the write would overwrite the existing file.
2315 ok = self.promptForDangerousWrite(fileName)
2316 if ok:
2317 # Fix bug 889175: Remember the full fileName.
2318 at.rememberReadPath(fileName, root)
2319 return True
2320 #
2321 # Fix #1031: do not add @ignore here!
2322 g.es("not written:", fileName)
2323 return False
2324 #@+node:ekr.20050506090446.1: *5* at.putAtFirstLines
2325 def putAtFirstLines(self, s):
2326 """
2327 Write any @firstlines from string s.
2328 These lines are converted to @verbatim lines,
2329 so the read logic simply ignores lines preceding the @+leo sentinel.
2330 """
2331 at = self
2332 tag = "@first"
2333 i = 0
2334 while g.match(s, i, tag):
2335 i += len(tag)
2336 i = g.skip_ws(s, i)
2337 j = i
2338 i = g.skip_to_end_of_line(s, i)
2339 # Write @first line, whether empty or not
2340 line = s[j:i]
2341 at.os(line)
2342 at.onl()
2343 i = g.skip_nl(s, i)
2344 #@+node:ekr.20050506090955: *5* at.putAtLastLines
2345 def putAtLastLines(self, s):
2346 """
2347 Write any @last lines from string s.
2348 These lines are converted to @verbatim lines,
2349 so the read logic simply ignores lines following the @-leo sentinel.
2350 """
2351 at = self
2352 tag = "@last"
2353 # Use g.splitLines to preserve trailing newlines.
2354 lines = g.splitLines(s)
2355 n = len(lines)
2356 j = k = n - 1
2357 # Scan backwards for @last directives.
2358 while j >= 0:
2359 line = lines[j]
2360 if g.match(line, 0, tag):
2361 j -= 1
2362 elif not line.strip():
2363 j -= 1
2364 else:
2365 break # pragma: no cover (coverage bug)
2366 # Write the @last lines.
2367 for line in lines[j + 1 : k + 1]:
2368 if g.match(line, 0, tag):
2369 i = len(tag)
2370 i = g.skip_ws(line, i)
2371 at.os(line[i:])
2372 #@+node:ekr.20041005105605.206: *5* at.putDirective & helper
2373 def putDirective(self, s, i, p):
2374 r"""
2375 Output a sentinel a directive or reference s.
2377 It is important for PHP and other situations that \@first and \@last
2378 directives get translated to verbatim lines that do *not* include what
2379 follows the @first & @last directives.
2380 """
2381 at = self
2382 k = i
2383 j = g.skip_to_end_of_line(s, i)
2384 directive = s[i:j]
2385 if g.match_word(s, k, "@delims"):
2386 at.putDelims(directive, s, k)
2387 elif g.match_word(s, k, "@language"):
2388 self.putSentinel("@" + directive)
2389 elif g.match_word(s, k, "@comment"):
2390 self.putSentinel("@" + directive)
2391 elif g.match_word(s, k, "@last"):
2392 # #1307.
2393 if p.isAtCleanNode(): # pragma: no cover
2394 at.error(f"ignoring @last directive in {p.h!r}")
2395 g.es_print('@last is not valid in @clean nodes')
2396 # #1297.
2397 elif g.app.inScript or g.unitTesting or p.isAnyAtFileNode():
2398 self.putSentinel("@@last")
2399 # Convert to an verbatim line _without_ anything else.
2400 else:
2401 at.error(f"ignoring @last directive in {p.h!r}") # pragma: no cover
2402 elif g.match_word(s, k, "@first"):
2403 # #1307.
2404 if p.isAtCleanNode(): # pragma: no cover
2405 at.error(f"ignoring @first directive in {p.h!r}")
2406 g.es_print('@first is not valid in @clean nodes')
2407 # #1297.
2408 elif g.app.inScript or g.unitTesting or p.isAnyAtFileNode():
2409 self.putSentinel("@@first")
2410 # Convert to an verbatim line _without_ anything else.
2411 else:
2412 at.error(f"ignoring @first directive in {p.h!r}") # pragma: no cover
2413 else:
2414 self.putSentinel("@" + directive)
2415 i = g.skip_line(s, k)
2416 return i
2417 #@+node:ekr.20041005105605.207: *6* at.putDelims
2418 def putDelims(self, directive, s, k):
2419 """Put an @delims directive."""
2420 at = self
2421 # Put a space to protect the last delim.
2422 at.putSentinel(directive + " ") # 10/23/02: put @delims, not @@delims
2423 # Skip the keyword and whitespace.
2424 j = i = g.skip_ws(s, k + len("@delims"))
2425 # Get the first delim.
2426 while i < len(s) and not g.is_ws(s[i]) and not g.is_nl(s, i):
2427 i += 1
2428 if j < i:
2429 at.startSentinelComment = s[j:i]
2430 # Get the optional second delim.
2431 j = i = g.skip_ws(s, i)
2432 while i < len(s) and not g.is_ws(s[i]) and not g.is_nl(s, i):
2433 i += 1
2434 at.endSentinelComment = s[j:i] if j < i else ""
2435 else:
2436 at.writeError("Bad @delims directive") # pragma: no cover
2437 #@+node:ekr.20041005105605.210: *5* at.putIndent
2438 def putIndent(self, n, s=''): # pragma: no cover
2439 """Put tabs and spaces corresponding to n spaces,
2440 assuming that we are at the start of a line.
2442 Remove extra blanks if the line starts with the underindentEscapeString"""
2443 tag = self.underindentEscapeString
2444 if s.startswith(tag):
2445 n2, s2 = self.parseUnderindentTag(s)
2446 if n2 >= n:
2447 return
2448 if n > 0:
2449 n -= n2
2450 else:
2451 n += n2
2452 if n > 0:
2453 w = self.tab_width
2454 if w > 1:
2455 q, r = divmod(n, w)
2456 self.otabs(q)
2457 self.oblanks(r)
2458 else:
2459 self.oblanks(n)
2460 #@+node:ekr.20041005105605.211: *5* at.putInitialComment
2461 def putInitialComment(self): # pragma: no cover
2462 c = self.c
2463 s2 = c.config.output_initial_comment
2464 if s2:
2465 lines = s2.split("\\n")
2466 for line in lines:
2467 line = line.replace("@date", time.asctime())
2468 if line:
2469 self.putSentinel("@comment " + line)
2470 #@+node:ekr.20190111172114.1: *5* at.replaceFile & helpers
2471 def replaceFile(self, contents, encoding, fileName, root, ignoreBlankLines=False):
2472 """
2473 Write or create the given file from the contents.
2474 Return True if the original file was changed.
2475 """
2476 at, c = self, self.c
2477 if root:
2478 root.clearDirty()
2479 #
2480 # Create the timestamp (only for messages).
2481 if c.config.getBool('log-show-save-time', default=False): # pragma: no cover
2482 format = c.config.getString('log-timestamp-format') or "%H:%M:%S"
2483 timestamp = time.strftime(format) + ' '
2484 else:
2485 timestamp = ''
2486 #
2487 # Adjust the contents.
2488 assert isinstance(contents, str), g.callers()
2489 if at.output_newline != '\n': # pragma: no cover
2490 contents = contents.replace('\r', '').replace('\n', at.output_newline)
2491 #
2492 # If file does not exist, create it from the contents.
2493 fileName = g.os_path_realpath(fileName)
2494 sfn = g.shortFileName(fileName)
2495 if not g.os_path_exists(fileName):
2496 ok = g.writeFile(contents, encoding, fileName)
2497 if ok:
2498 c.setFileTimeStamp(fileName)
2499 if not g.unitTesting:
2500 g.es(f"{timestamp}created: {fileName}") # pragma: no cover
2501 if root:
2502 # Fix bug 889175: Remember the full fileName.
2503 at.rememberReadPath(fileName, root)
2504 at.checkPythonCode(contents, fileName, root)
2505 else:
2506 at.addToOrphanList(root) # pragma: no cover
2507 # No original file to change. Return value tested by a unit test.
2508 return False # No change to original file.
2509 #
2510 # Compare the old and new contents.
2511 old_contents = g.readFileIntoUnicodeString(fileName,
2512 encoding=at.encoding, silent=True)
2513 if not old_contents:
2514 old_contents = ''
2515 unchanged = (
2516 contents == old_contents
2517 or (not at.explicitLineEnding and at.compareIgnoringLineEndings(old_contents, contents))
2518 or ignoreBlankLines and at.compareIgnoringBlankLines(old_contents, contents))
2519 if unchanged:
2520 at.unchangedFiles += 1
2521 if not g.unitTesting and c.config.getBool(
2522 'report-unchanged-files', default=True):
2523 g.es(f"{timestamp}unchanged: {sfn}") # pragma: no cover
2524 # Leo 5.6: Check unchanged files.
2525 at.checkPyflakes(contents, fileName, root)
2526 return False # No change to original file.
2527 #
2528 # Warn if we are only adjusting the line endings.
2529 if at.explicitLineEnding: # pragma: no cover
2530 ok = (
2531 at.compareIgnoringLineEndings(old_contents, contents) or
2532 ignoreBlankLines and at.compareIgnoringLineEndings(
2533 old_contents, contents))
2534 if not ok:
2535 g.warning("correcting line endings in:", fileName)
2536 #
2537 # Write a changed file.
2538 ok = g.writeFile(contents, encoding, fileName)
2539 if ok:
2540 c.setFileTimeStamp(fileName)
2541 if not g.unitTesting:
2542 g.es(f"{timestamp}wrote: {sfn}") # pragma: no cover
2543 else: # pragma: no cover
2544 g.error('error writing', sfn)
2545 g.es('not written:', sfn)
2546 at.addToOrphanList(root)
2547 at.checkPythonCode(contents, fileName, root)
2548 # Check *after* writing the file.
2549 return ok
2550 #@+node:ekr.20190114061452.27: *6* at.compareIgnoringBlankLines
2551 def compareIgnoringBlankLines(self, s1, s2): # pragma: no cover
2552 """Compare two strings, ignoring blank lines."""
2553 assert isinstance(s1, str), g.callers()
2554 assert isinstance(s2, str), g.callers()
2555 if s1 == s2:
2556 return True
2557 s1 = g.removeBlankLines(s1)
2558 s2 = g.removeBlankLines(s2)
2559 return s1 == s2
2560 #@+node:ekr.20190114061452.28: *6* at.compareIgnoringLineEndings
2561 def compareIgnoringLineEndings(self, s1, s2): # pragma: no cover
2562 """Compare two strings, ignoring line endings."""
2563 assert isinstance(s1, str), (repr(s1), g.callers())
2564 assert isinstance(s2, str), (repr(s2), g.callers())
2565 if s1 == s2:
2566 return True
2567 # Wrong: equivalent to ignoreBlankLines!
2568 # s1 = s1.replace('\n','').replace('\r','')
2569 # s2 = s2.replace('\n','').replace('\r','')
2570 s1 = s1.replace('\r', '')
2571 s2 = s2.replace('\r', '')
2572 return s1 == s2
2573 #@+node:ekr.20211029052041.1: *5* at.scanRootForSectionDelims
2574 def scanRootForSectionDelims(self, root):
2575 """
2576 Scan root.b for an "@section-delims" directive.
2577 Set section_delim1 and section_delim2 ivars.
2578 """
2579 at = self
2580 # Set defaults.
2581 at.section_delim1 = '<<'
2582 at.section_delim2 = '>>'
2583 # Scan root.b.
2584 lines = []
2585 for s in g.splitLines(root.b):
2586 m = g.g_section_delims_pat.match(s)
2587 if m:
2588 lines.append(s)
2589 at.section_delim1 = m.group(1)
2590 at.section_delim2 = m.group(2)
2591 # Disallow multiple directives.
2592 if len(lines) > 1: # pragma: no cover
2593 at.error(f"Multiple @section-delims directives in {root.h}")
2594 g.es_print('using default delims')
2595 at.section_delim1 = '<<'
2596 at.section_delim2 = '>>'
2597 #@+node:ekr.20090514111518.5665: *5* at.tabNannyNode
2598 def tabNannyNode(self, p, body):
2599 try:
2600 readline = g.ReadLinesClass(body).next
2601 tabnanny.process_tokens(tokenize.generate_tokens(readline))
2602 except IndentationError: # pragma: no cover
2603 if g.unitTesting:
2604 raise
2605 junk2, msg, junk = sys.exc_info()
2606 g.error("IndentationError in", p.h)
2607 g.es('', str(msg))
2608 except tokenize.TokenError: # pragma: no cover
2609 if g.unitTesting:
2610 raise
2611 junk3, msg, junk = sys.exc_info()
2612 g.error("TokenError in", p.h)
2613 g.es('', str(msg))
2614 except tabnanny.NannyNag: # pragma: no cover
2615 if g.unitTesting:
2616 raise
2617 junk4, nag, junk = sys.exc_info()
2618 badline = nag.get_lineno()
2619 line = nag.get_line()
2620 message = nag.get_msg()
2621 g.error("indentation error in", p.h, "line", badline)
2622 g.es(message)
2623 line2 = repr(str(line))[1:-1]
2624 g.es("offending line:\n", line2)
2625 except Exception: # pragma: no cover
2626 g.trace("unexpected exception")
2627 g.es_exception()
2628 raise
2629 #@+node:ekr.20041005105605.216: *5* at.warnAboutOrpanAndIgnoredNodes
2630 # Called from putFile.
2632 def warnAboutOrphandAndIgnoredNodes(self): # pragma: no cover
2633 # Always warn, even when language=="cweb"
2634 at, root = self, self.root
2635 if at.errors:
2636 return # No need to repeat this.
2637 for p in root.self_and_subtree(copy=False):
2638 if not p.v.isVisited():
2639 at.writeError("Orphan node: " + p.h)
2640 if p.hasParent():
2641 g.blue("parent node:", p.parent().h)
2642 p = root.copy()
2643 after = p.nodeAfterTree()
2644 while p and p != after:
2645 if p.isAtAllNode():
2646 p.moveToNodeAfterTree()
2647 else:
2648 # #1050: test orphan bit.
2649 if p.isOrphan():
2650 at.writeError("Orphan node: " + p.h)
2651 if p.hasParent():
2652 g.blue("parent node:", p.parent().h)
2653 p.moveToThreadNext()
2654 #@+node:ekr.20041005105605.217: *5* at.writeError
2655 def writeError(self, message): # pragma: no cover
2656 """Issue an error while writing an @<file> node."""
2657 at = self
2658 if at.errors == 0:
2659 fn = at.targetFileName or 'unnamed file'
2660 g.es_error(f"errors writing: {fn}")
2661 at.error(message)
2662 at.addToOrphanList(at.root)
2663 #@+node:ekr.20041005105605.218: *5* at.writeException
2664 def writeException(self, fileName, root): # pragma: no cover
2665 at = self
2666 g.error("exception writing:", fileName)
2667 g.es_exception()
2668 if getattr(at, 'outputFile', None):
2669 at.outputFile.flush()
2670 at.outputFile.close()
2671 at.outputFile = None
2672 at.remove(fileName)
2673 at.addToOrphanList(root)
2674 #@+node:ekr.20041005105605.219: *3* at.Utilites
2675 #@+node:ekr.20041005105605.220: *4* at.error & printError
2676 def error(self, *args): # pragma: no cover
2677 at = self
2678 at.printError(*args)
2679 at.errors += 1
2681 def printError(self, *args): # pragma: no cover
2682 """Print an error message that may contain non-ascii characters."""
2683 at = self
2684 if at.errors:
2685 g.error(*args)
2686 else:
2687 g.warning(*args)
2688 #@+node:ekr.20041005105605.221: *4* at.exception
2689 def exception(self, message): # pragma: no cover
2690 self.error(message)
2691 g.es_exception()
2692 #@+node:ekr.20050104131929: *4* at.file operations...
2693 # Error checking versions of corresponding functions in Python's os module.
2694 #@+node:ekr.20050104131820: *5* at.chmod
2695 def chmod(self, fileName, mode): # pragma: no cover
2696 # Do _not_ call self.error here.
2697 if mode is None:
2698 return
2699 try:
2700 os.chmod(fileName, mode)
2701 except Exception:
2702 g.es("exception in os.chmod", fileName)
2703 g.es_exception()
2705 #@+node:ekr.20050104132018: *5* at.remove
2706 def remove(self, fileName): # pragma: no cover
2707 if not fileName:
2708 g.trace('No file name', g.callers())
2709 return False
2710 try:
2711 os.remove(fileName)
2712 return True
2713 except Exception:
2714 if not g.unitTesting:
2715 self.error(f"exception removing: {fileName}")
2716 g.es_exception()
2717 return False
2718 #@+node:ekr.20050104132026: *5* at.stat
2719 def stat(self, fileName): # pragma: no cover
2720 """Return the access mode of named file, removing any setuid, setgid, and sticky bits."""
2721 # Do _not_ call self.error here.
2722 try:
2723 mode = (os.stat(fileName))[0] & (7 * 8 * 8 + 7 * 8 + 7) # 0777
2724 except Exception:
2725 mode = None
2726 return mode
2728 #@+node:ekr.20090530055015.6023: *4* at.get/setPathUa
2729 def getPathUa(self, p):
2730 if hasattr(p.v, 'tempAttributes'):
2731 d = p.v.tempAttributes.get('read-path', {})
2732 return d.get('path')
2733 return ''
2735 def setPathUa(self, p, path):
2736 if not hasattr(p.v, 'tempAttributes'):
2737 p.v.tempAttributes = {}
2738 d = p.v.tempAttributes.get('read-path', {})
2739 d['path'] = path
2740 p.v.tempAttributes['read-path'] = d
2741 #@+node:ekr.20081216090156.4: *4* at.parseUnderindentTag
2742 # Important: this is part of the *write* logic.
2743 # It is called from at.os and at.putIndent.
2745 def parseUnderindentTag(self, s): # pragma: no cover
2746 tag = self.underindentEscapeString
2747 s2 = s[len(tag) :]
2748 # To be valid, the escape must be followed by at least one digit.
2749 i = 0
2750 while i < len(s2) and s2[i].isdigit():
2751 i += 1
2752 if i > 0:
2753 n = int(s2[:i])
2754 # Bug fix: 2012/06/05: remove any period following the count.
2755 # This is a new convention.
2756 if i < len(s2) and s2[i] == '.':
2757 i += 1
2758 return n, s2[i:]
2759 return 0, s
2760 #@+node:ekr.20090712050729.6017: *4* at.promptForDangerousWrite
2761 def promptForDangerousWrite(self, fileName, message=None): # pragma: no cover
2762 """Raise a dialog asking the user whether to overwrite an existing file."""
2763 at, c, root = self, self.c, self.root
2764 if at.cancelFlag:
2765 assert at.canCancelFlag
2766 return False
2767 if at.yesToAll:
2768 assert at.canCancelFlag
2769 return True
2770 if root and root.h.startswith('@auto-rst'):
2771 # Fix bug 50: body text lost switching @file to @auto-rst
2772 # Refuse to convert any @<file> node to @auto-rst.
2773 d = root.v.at_read if hasattr(root.v, 'at_read') else {}
2774 aList = sorted(d.get(fileName, []))
2775 for h in aList:
2776 if not h.startswith('@auto-rst'):
2777 g.es('can not convert @file to @auto-rst!', color='red')
2778 g.es('reverting to:', h)
2779 root.h = h
2780 c.redraw()
2781 return False
2782 if message is None:
2783 message = (
2784 f"{g.splitLongFileName(fileName)}\n"
2785 f"{g.tr('already exists.')}\n"
2786 f"{g.tr('Overwrite this file?')}")
2787 result = g.app.gui.runAskYesNoCancelDialog(c,
2788 title='Overwrite existing file?',
2789 yesToAllMessage="Yes To &All",
2790 message=message,
2791 cancelMessage="&Cancel (No To All)",
2792 )
2793 if at.canCancelFlag:
2794 # We are in the writeAll logic so these flags can be set.
2795 if result == 'cancel':
2796 at.cancelFlag = True
2797 elif result == 'yes-to-all':
2798 at.yesToAll = True
2799 return result in ('yes', 'yes-to-all')
2800 #@+node:ekr.20120112084820.10001: *4* at.rememberReadPath
2801 def rememberReadPath(self, fn, p):
2802 """
2803 Remember the files that have been read *and*
2804 the full headline (@<file> type) that caused the read.
2805 """
2806 v = p.v
2807 # Fix bug #50: body text lost switching @file to @auto-rst
2808 if not hasattr(v, 'at_read'):
2809 v.at_read = {} # pragma: no cover
2810 d = v.at_read
2811 aSet = d.get(fn, set())
2812 aSet.add(p.h)
2813 d[fn] = aSet
2814 #@+node:ekr.20080923070954.4: *4* at.scanAllDirectives
2815 def scanAllDirectives(self, p):
2816 """
2817 Scan p and p's ancestors looking for directives,
2818 setting corresponding AtFile ivars.
2819 """
2820 at, c = self, self.c
2821 d = c.scanAllDirectives(p)
2822 #
2823 # Language & delims: Tricky.
2824 lang_dict = d.get('lang-dict') or {}
2825 delims, language = None, None
2826 if lang_dict:
2827 # There was an @delims or @language directive.
2828 language = lang_dict.get('language')
2829 delims = lang_dict.get('delims')
2830 if not language:
2831 # No language directive. Look for @<file> nodes.
2832 # Do *not* used.get('language')!
2833 language = g.getLanguageFromAncestorAtFileNode(p) or 'python'
2834 at.language = language
2835 if not delims:
2836 delims = g.set_delims_from_language(language)
2837 #
2838 # Previously, setting delims was sometimes skipped, depending on kwargs.
2839 #@+<< Set comment strings from delims >>
2840 #@+node:ekr.20080923070954.13: *5* << Set comment strings from delims >> (at.scanAllDirectives)
2841 delim1, delim2, delim3 = delims
2842 # Use single-line comments if we have a choice.
2843 # delim1,delim2,delim3 now correspond to line,start,end
2844 if delim1:
2845 at.startSentinelComment = delim1
2846 at.endSentinelComment = "" # Must not be None.
2847 elif delim2 and delim3:
2848 at.startSentinelComment = delim2
2849 at.endSentinelComment = delim3
2850 else: # pragma: no cover
2851 #
2852 # Emergency!
2853 #
2854 # Issue an error only if at.language has been set.
2855 # This suppresses a message from the markdown importer.
2856 if not g.unitTesting and at.language:
2857 g.trace(repr(at.language), g.callers())
2858 g.es_print("unknown language: using Python comment delimiters")
2859 g.es_print("c.target_language:", c.target_language)
2860 at.startSentinelComment = "#" # This should never happen!
2861 at.endSentinelComment = ""
2862 #@-<< Set comment strings from delims >>
2863 #
2864 # Easy cases
2865 at.encoding = d.get('encoding') or c.config.default_derived_file_encoding
2866 lineending = d.get('lineending')
2867 at.explicitLineEnding = bool(lineending)
2868 at.output_newline = lineending or g.getOutputNewline(c=c)
2869 at.page_width = d.get('pagewidth') or c.page_width
2870 at.tab_width = d.get('tabwidth') or c.tab_width
2871 return {
2872 "encoding": at.encoding,
2873 "language": at.language,
2874 "lineending": at.output_newline,
2875 "pagewidth": at.page_width,
2876 "path": d.get('path'),
2877 "tabwidth": at.tab_width,
2878 }
2879 #@+node:ekr.20120110174009.9965: *4* at.shouldPromptForDangerousWrite
2880 def shouldPromptForDangerousWrite(self, fn, p): # pragma: no cover
2881 """
2882 Return True if Leo should warn the user that p is an @<file> node that
2883 was not read during startup. Writing that file might cause data loss.
2885 See #50: https://github.com/leo-editor/leo-editor/issues/50
2886 """
2887 trace = 'save' in g.app.debug
2888 sfn = g.shortFileName(fn)
2889 c = self.c
2890 efc = g.app.externalFilesController
2891 if p.isAtNoSentFileNode():
2892 # #1450.
2893 # No danger of overwriting a file.
2894 # It was never read.
2895 return False
2896 if not g.os_path_exists(fn):
2897 # No danger of overwriting fn.
2898 if trace:
2899 g.trace('Return False: does not exist:', sfn)
2900 return False
2901 # #1347: Prompt if the external file is newer.
2902 if efc:
2903 # Like c.checkFileTimeStamp.
2904 if c.sqlite_connection and c.mFileName == fn:
2905 # sqlite database file is never actually overwriten by Leo,
2906 # so do *not* check its timestamp.
2907 pass
2908 elif efc.has_changed(fn):
2909 if trace:
2910 g.trace('Return True: changed:', sfn)
2911 return True
2912 if hasattr(p.v, 'at_read'):
2913 # Fix bug #50: body text lost switching @file to @auto-rst
2914 d = p.v.at_read
2915 for k in d:
2916 # Fix bug # #1469: make sure k still exists.
2917 if (
2918 os.path.exists(k) and os.path.samefile(k, fn)
2919 and p.h in d.get(k, set())
2920 ):
2921 d[fn] = d[k]
2922 if trace:
2923 g.trace('Return False: in p.v.at_read:', sfn)
2924 return False
2925 aSet = d.get(fn, set())
2926 if trace:
2927 g.trace(f"Return {p.h not in aSet()}: p.h not in aSet(): {sfn}")
2928 return p.h not in aSet
2929 if trace:
2930 g.trace('Return True: never read:', sfn)
2931 return True # The file was never read.
2932 #@+node:ekr.20041005105605.20: *4* at.warnOnReadOnlyFile
2933 def warnOnReadOnlyFile(self, fn):
2934 # os.access() may not exist on all platforms.
2935 try:
2936 read_only = not os.access(fn, os.W_OK)
2937 except AttributeError: # pragma: no cover
2938 read_only = False
2939 if read_only:
2940 g.error("read only:", fn) # pragma: no cover
2941 #@-others
2942atFile = AtFile # compatibility
2943#@+node:ekr.20180602102448.1: ** class FastAtRead
2944class FastAtRead:
2945 """
2946 Read an exteral file, created from an @file tree.
2947 This is Vitalije's code, edited by EKR.
2948 """
2950 #@+others
2951 #@+node:ekr.20211030193146.1: *3* fast_at.__init__
2952 def __init__(self, c, gnx2vnode):
2954 self.c = c
2955 assert gnx2vnode is not None
2956 self.gnx2vnode = gnx2vnode # The global fc.gnxDict. Keys are gnx's, values are vnodes.
2957 self.path = None
2958 self.root = None
2959 # compiled patterns...
2960 self.after_pat = None
2961 self.all_pat = None
2962 self.code_pat = None
2963 self.comment_pat = None
2964 self.delims_pat = None
2965 self.doc_pat = None
2966 self.first_pat = None
2967 self.last_pat = None
2968 self.node_start_pat = None
2969 self.others_pat = None
2970 self.ref_pat = None
2971 self.section_delims_pat = None
2972 #@+node:ekr.20180602103135.3: *3* fast_at.get_patterns
2973 #@@nobeautify
2975 def get_patterns(self, comment_delims):
2976 """Create regex patterns for the given comment delims."""
2977 # This must be a function, because of @comments & @delims.
2978 comment_delim_start, comment_delim_end = comment_delims
2979 delim1 = re.escape(comment_delim_start)
2980 delim2 = re.escape(comment_delim_end or '')
2981 ref = g.angleBrackets(r'(.*)')
2982 table = (
2983 # These patterns must be mutually exclusive.
2984 ('after', fr'^\s*{delim1}@afterref{delim2}$'), # @afterref
2985 ('all', fr'^(\s*){delim1}@(\+|-)all\b(.*){delim2}$'), # @all
2986 ('code', fr'^\s*{delim1}@@c(ode)?{delim2}$'), # @c and @code
2987 ('comment', fr'^\s*{delim1}@@comment(.*){delim2}'), # @comment
2988 ('delims', fr'^\s*{delim1}@delims(.*){delim2}'), # @delims
2989 ('doc', fr'^\s*{delim1}@\+(at|doc)?(\s.*?)?{delim2}\n'), # @doc or @
2990 ('first', fr'^\s*{delim1}@@first{delim2}$'), # @first
2991 ('last', fr'^\s*{delim1}@@last{delim2}$'), # @last
2992 # @node
2993 ('node_start', fr'^(\s*){delim1}@\+node:([^:]+): \*(\d+)?(\*?) (.*){delim2}$'),
2994 ('others', fr'^(\s*){delim1}@(\+|-)others\b(.*){delim2}$'), # @others
2995 ('ref', fr'^(\s*){delim1}@(\+|-){ref}\s*{delim2}$'), # section ref
2996 # @section-delims
2997 ('section_delims', fr'^\s*{delim1}@@section-delims[ \t]+([^ \w\n\t]+)[ \t]+([^ \w\n\t]+)[ \t]*{delim2}$'),
2998 )
2999 # Set the ivars.
3000 for (name, pattern) in table:
3001 ivar = f"{name}_pat"
3002 assert hasattr(self, ivar), ivar
3003 setattr(self, ivar, re.compile(pattern))
3004 #@+node:ekr.20180602103135.2: *3* fast_at.scan_header
3005 header_pattern = re.compile(
3006 r'''
3007 ^(.+)@\+leo
3008 (-ver=(\d+))?
3009 (-thin)?
3010 (-encoding=(.*)(\.))?
3011 (.*)$''',
3012 re.VERBOSE,
3013 )
3015 def scan_header(self, lines):
3016 """
3017 Scan for the header line, which follows any @first lines.
3018 Return (delims, first_lines, i+1) or None
3019 """
3020 first_lines: List[str] = []
3021 i = 0 # To keep some versions of pylint happy.
3022 for i, line in enumerate(lines):
3023 m = self.header_pattern.match(line)
3024 if m:
3025 delims = m.group(1), m.group(8) or ''
3026 return delims, first_lines, i + 1
3027 first_lines.append(line)
3028 return None # pragma: no cover (defensive)
3029 #@+node:ekr.20180602103135.8: *3* fast_at.scan_lines
3030 def scan_lines(self, comment_delims, first_lines, lines, path, start):
3031 """Scan all lines of the file, creating vnodes."""
3032 #@+<< init scan_lines >>
3033 #@+node:ekr.20180602103135.9: *4* << init scan_lines >>
3034 #
3035 # Simple vars...
3036 afterref = False # True: the next line follows @afterref.
3037 clone_v = None # The root of the clone tree.
3038 comment_delim1, comment_delim2 = comment_delims # The start/end *comment* delims.
3039 doc_skip = (comment_delim1 + '\n', comment_delim2 + '\n') # To handle doc parts.
3040 first_i = 0 # Index into first array.
3041 in_doc = False # True: in @doc parts.
3042 is_cweb = comment_delim1 == '@q@' and comment_delim2 == '@>' # True: cweb hack in effect.
3043 indent = 0 # The current indentation.
3044 level_stack = [] # Entries are (vnode, in_clone_tree)
3045 n_last_lines = 0 # The number of @@last directives seen.
3046 root_gnx_adjusted = False # True: suppress final checks.
3047 # #1065 so reads will not create spurious child nodes.
3048 root_seen = False # False: The next +@node sentinel denotes the root, regardless of gnx.
3049 section_delim1 = '<<'
3050 section_delim2 = '>>'
3051 section_reference_seen = False
3052 sentinel = comment_delim1 + '@' # Faster than a regex!
3053 # The stack is updated when at+others, at+<section>, or at+all is seen.
3054 stack = [] # Entries are (gnx, indent, body)
3055 # The spelling of at-verbatim sentinel
3056 verbatim_line = comment_delim1 + '@verbatim' + comment_delim2 + '\n'
3057 verbatim = False # True: the next line must be added without change.
3058 #
3059 # Init the parent vnode.
3060 #
3061 root_gnx = gnx = self.root.gnx
3062 context = self.c
3063 parent_v = self.root.v
3064 root_v = parent_v # Does not change.
3065 level_stack.append((root_v, False),)
3066 #
3067 # Init the gnx dict last.
3068 #
3069 gnx2vnode = self.gnx2vnode # Keys are gnx's, values are vnodes.
3070 gnx2body = {} # Keys are gnxs, values are list of body lines.
3071 gnx2vnode[gnx] = parent_v # Add gnx to the keys
3072 # Add gnx to the keys.
3073 # Body is the list of lines presently being accumulated.
3074 gnx2body[gnx] = body = first_lines
3075 #
3076 # Set the patterns
3077 self.get_patterns(comment_delims)
3078 #@-<< init scan_lines >>
3079 i = 0 # To keep pylint happy.
3080 for i, line in enumerate(lines[start:]):
3081 # Strip the line only once.
3082 strip_line = line.strip()
3083 if afterref:
3084 #@+<< handle afterref line>>
3085 #@+node:ekr.20211102052251.1: *4* << handle afterref line >>
3086 if body: # a List of lines.
3087 body[-1] = body[-1].rstrip() + line
3088 else:
3089 body = [line] # pragma: no cover
3090 afterref = False
3091 #@-<< handle afterref line>>
3092 continue
3093 if verbatim:
3094 #@+<< handle verbatim line >>
3095 #@+node:ekr.20211102052518.1: *4* << handle verbatim line >>
3096 # Previous line was verbatim *sentinel*. Append this line as it is.
3097 body.append(line)
3098 verbatim = False
3099 #@-<< handle verbatim line >>
3100 continue
3101 if line == verbatim_line: # <delim>@verbatim.
3102 verbatim = True
3103 continue
3104 #@+<< finalize line >>
3105 #@+node:ekr.20180602103135.10: *4* << finalize line >>
3106 # Undo the cweb hack.
3107 if is_cweb and line.startswith(sentinel):
3108 line = line[: len(sentinel)] + line[len(sentinel) :].replace('@@', '@')
3109 # Adjust indentation.
3110 if indent and line[:indent].isspace() and len(line) > indent:
3111 line = line[indent:]
3112 #@-<< finalize line >>
3113 if not in_doc and not strip_line.startswith(sentinel): # Faster than a regex!
3114 body.append(line)
3115 continue
3116 # These three sections might clear in_doc.
3117 #@+<< handle @others >>
3118 #@+node:ekr.20180602103135.14: *4* << handle @others >>
3119 m = self.others_pat.match(line)
3120 if m:
3121 in_doc = False
3122 if m.group(2) == '+': # opening sentinel
3123 body.append(f"{m.group(1)}@others{m.group(3) or ''}\n")
3124 stack.append((gnx, indent, body))
3125 indent += m.end(1) # adjust current identation
3126 else: # closing sentinel.
3127 # m.group(2) is '-' because the pattern matched.
3128 gnx, indent, body = stack.pop()
3129 continue
3130 #@-<< handle @others >>
3131 #@+<< handle section refs >>
3132 #@+node:ekr.20180602103135.18: *4* << handle section refs >>
3133 # Note: scan_header sets *comment* delims, not *section* delims.
3134 # This section coordinates with the section that handles @section-delims.
3135 m = self.ref_pat.match(line)
3136 if m:
3137 in_doc = False
3138 if m.group(2) == '+':
3139 # Any later @section-delims directive is a serious error.
3140 # This kind of error should have been caught by Leo's atFile write logic.
3141 section_reference_seen = True
3142 # open sentinel.
3143 body.append(m.group(1) + section_delim1 + m.group(3) + section_delim2 + '\n')
3144 stack.append((gnx, indent, body))
3145 indent += m.end(1)
3146 elif stack:
3147 # m.group(2) is '-' because the pattern matched.
3148 gnx, indent, body = stack.pop() # #1232: Only if the stack exists.
3149 continue # 2021/10/29: *always* continue.
3150 #@-<< handle section refs >>
3151 #@+<< handle node_start >>
3152 #@+node:ekr.20180602103135.19: *4* << handle node_start >>
3153 m = self.node_start_pat.match(line)
3154 if m:
3155 in_doc = False
3156 gnx, head = m.group(2), m.group(5)
3157 # m.group(3) is the level number, m.group(4) is the number of stars.
3158 level = int(m.group(3)) if m.group(3) else 1 + len(m.group(4))
3159 v = gnx2vnode.get(gnx)
3160 #
3161 # Case 1: The root @file node. Don't change the headline.
3162 if not root_seen and not v and not g.unitTesting:
3163 # Don't warn about a gnx mismatch in the root.
3164 root_gnx_adjusted = True # pragma: no cover
3165 if not root_seen:
3166 # Fix #1064: The node represents the root, regardless of the gnx!
3167 root_seen = True
3168 clone_v = None
3169 gnx2body[gnx] = body = []
3170 # This case can happen, but not in unit tests.
3171 if not v: # pragma: no cover
3172 # Fix #1064.
3173 v = root_v
3174 # This message is annoying when using git-diff.
3175 # if gnx != root_gnx:
3176 # g.es_print("using gnx from external file: %s" % (v.h), color='blue')
3177 gnx2vnode[gnx] = v
3178 v.fileIndex = gnx
3179 v.children = []
3180 continue
3181 #
3182 # Case 2: We are scanning the descendants of a clone.
3183 parent_v, clone_v = level_stack[level - 2]
3184 if v and clone_v:
3185 # The last version of the body and headline wins..
3186 gnx2body[gnx] = body = []
3187 v._headString = head
3188 # Update the level_stack.
3189 level_stack = level_stack[: level - 1]
3190 level_stack.append((v, clone_v),)
3191 # Always clear the children!
3192 v.children = []
3193 parent_v.children.append(v)
3194 continue
3195 #
3196 # Case 3: we are not already scanning the descendants of a clone.
3197 if v:
3198 # The *start* of a clone tree. Reset the children.
3199 clone_v = v
3200 v.children = []
3201 else:
3202 # Make a new vnode.
3203 v = leoNodes.VNode(context=context, gnx=gnx)
3204 #
3205 # The last version of the body and headline wins.
3206 gnx2vnode[gnx] = v
3207 gnx2body[gnx] = body = []
3208 v._headString = head
3209 #
3210 # Update the stack.
3211 level_stack = level_stack[: level - 1]
3212 level_stack.append((v, clone_v),)
3213 #
3214 # Update the links.
3215 assert v != root_v
3216 parent_v.children.append(v)
3217 v.parents.append(parent_v)
3218 continue
3219 #@-<< handle node_start >>
3220 if in_doc:
3221 #@+<< handle @c or @code >>
3222 #@+node:ekr.20211031033532.1: *4* << handle @c or @code >>
3223 # When delim_end exists the doc block:
3224 # - begins with the opening delim, alone on its own line
3225 # - ends with the closing delim, alone on its own line.
3226 # Both of these lines should be skipped.
3227 #
3228 # #1496: Retire the @doc convention.
3229 # An empty line is no longer a sentinel.
3230 if comment_delim2 and line in doc_skip:
3231 # doc_skip is (comment_delim1 + '\n', delim_end + '\n')
3232 continue
3233 #
3234 # Check for @c or @code.
3235 m = self.code_pat.match(line)
3236 if m:
3237 in_doc = False
3238 body.append('@code\n' if m.group(1) else '@c\n')
3239 continue
3240 #@-<< handle @c or @code >>
3241 else:
3242 #@+<< handle @ or @doc >>
3243 #@+node:ekr.20211031033754.1: *4* << handle @ or @doc >>
3244 m = self.doc_pat.match(line)
3245 if m:
3246 # @+at or @+doc?
3247 doc = '@doc' if m.group(1) == 'doc' else '@'
3248 doc2 = m.group(2) or '' # Trailing text.
3249 if doc2:
3250 body.append(f"{doc}{doc2}\n")
3251 else:
3252 body.append(doc + '\n')
3253 # Enter @doc mode.
3254 in_doc = True
3255 continue
3256 #@-<< handle @ or @doc >>
3257 if line.startswith(comment_delim1 + '@-leo'): # Faster than a regex!
3258 # The @-leo sentinel adds *nothing* to the text.
3259 i += 1
3260 break
3261 # Order doesn't matter.
3262 #@+<< handle @all >>
3263 #@+node:ekr.20180602103135.13: *4* << handle @all >>
3264 m = self.all_pat.match(line)
3265 if m:
3266 # @all tells Leo's *write* code not to check for undefined sections.
3267 # Here, in the read code, we merely need to add it to the body.
3268 # Pushing and popping the stack may not be necessary, but it can't hurt.
3269 if m.group(2) == '+': # opening sentinel
3270 body.append(f"{m.group(1)}@all{m.group(3) or ''}\n")
3271 stack.append((gnx, indent, body))
3272 else: # closing sentinel.
3273 # m.group(2) is '-' because the pattern matched.
3274 gnx, indent, body = stack.pop()
3275 gnx2body[gnx] = body
3276 continue
3277 #@-<< handle @all >>
3278 #@+<< handle afterref >>
3279 #@+node:ekr.20180603063102.1: *4* << handle afterref >>
3280 m = self.after_pat.match(line)
3281 if m:
3282 afterref = True
3283 continue
3284 #@-<< handle afterref >>
3285 #@+<< handle @first and @last >>
3286 #@+node:ekr.20180606053919.1: *4* << handle @first and @last >>
3287 m = self.first_pat.match(line)
3288 if m:
3289 # pylint: disable=no-else-continue
3290 if 0 <= first_i < len(first_lines):
3291 body.append('@first ' + first_lines[first_i])
3292 first_i += 1
3293 continue
3294 else: # pragma: no cover
3295 g.trace(f"\ntoo many @first lines: {path}")
3296 print('@first is valid only at the start of @<file> nodes\n')
3297 g.printObj(first_lines, tag='first_lines')
3298 g.printObj(lines[start : i + 2], tag='lines[start:i+2]')
3299 continue
3300 m = self.last_pat.match(line)
3301 if m:
3302 # Just increment the count of the expected last lines.
3303 # We'll fill in the @last line directives after we see the @-leo directive.
3304 n_last_lines += 1
3305 continue
3306 #@-<< handle @first and @last >>
3307 #@+<< handle @comment >>
3308 #@+node:ekr.20180621050901.1: *4* << handle @comment >>
3309 # http://leoeditor.com/directives.html#part-4-dangerous-directives
3310 m = self.comment_pat.match(line)
3311 if m:
3312 # <1, 2 or 3 comment delims>
3313 delims = m.group(1).strip()
3314 # Whatever happens, retain the @delims line.
3315 body.append(f"@comment {delims}\n")
3316 delim1, delim2, delim3 = g.set_delims_from_string(delims)
3317 # delim1 is always the single-line delimiter.
3318 if delim1:
3319 comment_delim1, comment_delim2 = delim1, ''
3320 else:
3321 comment_delim1, comment_delim2 = delim2, delim3
3322 #
3323 # Within these delimiters:
3324 # - double underscores represent a newline.
3325 # - underscores represent a significant space,
3326 comment_delim1 = comment_delim1.replace('__', '\n').replace('_', ' ')
3327 comment_delim2 = comment_delim2.replace('__', '\n').replace('_', ' ')
3328 # Recalculate all delim-related values
3329 doc_skip = (comment_delim1 + '\n', comment_delim2 + '\n')
3330 is_cweb = comment_delim1 == '@q@' and comment_delim2 == '@>'
3331 sentinel = comment_delim1 + '@'
3332 #
3333 # Recalculate the patterns.
3334 comment_delims = comment_delim1, comment_delim2
3335 self.get_patterns(comment_delims)
3336 continue
3337 #@-<< handle @comment >>
3338 #@+<< handle @delims >>
3339 #@+node:ekr.20180608104836.1: *4* << handle @delims >>
3340 m = self.delims_pat.match(line)
3341 if m:
3342 # Get 1 or 2 comment delims
3343 # Whatever happens, retain the original @delims line.
3344 delims = m.group(1).strip()
3345 body.append(f"@delims {delims}\n")
3346 #
3347 # Parse the delims.
3348 self.delims_pat = re.compile(r'^([^ ]+)\s*([^ ]+)?')
3349 m2 = self.delims_pat.match(delims)
3350 if not m2: # pragma: no cover
3351 g.trace(f"Ignoring invalid @delims: {line!r}")
3352 continue
3353 comment_delim1 = m2.group(1)
3354 comment_delim2 = m2.group(2) or ''
3355 #
3356 # Within these delimiters:
3357 # - double underscores represent a newline.
3358 # - underscores represent a significant space,
3359 comment_delim1 = comment_delim1.replace('__', '\n').replace('_', ' ')
3360 comment_delim2 = comment_delim2.replace('__', '\n').replace('_', ' ')
3361 # Recalculate all delim-related values
3362 doc_skip = (comment_delim1 + '\n', comment_delim2 + '\n')
3363 is_cweb = comment_delim1 == '@q@' and comment_delim2 == '@>'
3364 sentinel = comment_delim1 + '@'
3365 #
3366 # Recalculate the patterns
3367 comment_delims = comment_delim1, comment_delim2
3368 self.get_patterns(comment_delims)
3369 continue
3370 #@-<< handle @delims >>
3371 #@+<< handle @section-delims >>
3372 #@+node:ekr.20211030033211.1: *4* << handle @section-delims >>
3373 m = self.section_delims_pat.match(line)
3374 if m:
3375 if section_reference_seen: # pragma: no cover
3376 # This is a serious error.
3377 # This kind of error should have been caught by Leo's atFile write logic.
3378 g.es_print('section-delims seen after a section reference', color='red')
3379 else:
3380 # Carefully update the section reference pattern!
3381 section_delim1 = d1 = re.escape(m.group(1))
3382 section_delim2 = d2 = re.escape(m.group(2) or '')
3383 self.ref_pat = re.compile(fr'^(\s*){comment_delim1}@(\+|-){d1}(.*){d2}\s*{comment_delim2}$')
3384 body.append(f"@section-delims {m.group(1)} {m.group(2)}\n")
3385 continue
3386 #@-<< handle @section-delims >>
3387 # These sections must be last, in this order.
3388 #@+<< handle remaining @@ lines >>
3389 #@+node:ekr.20180603135602.1: *4* << handle remaining @@ lines >>
3390 # @first, @last, @delims and @comment generate @@ sentinels,
3391 # So this must follow all of those.
3392 if line.startswith(comment_delim1 + '@@'):
3393 ii = len(comment_delim1) + 1 # on second '@'
3394 jj = line.rfind(comment_delim2) if comment_delim2 else -1
3395 body.append(line[ii:jj] + '\n')
3396 continue
3397 #@-<< handle remaining @@ lines >>
3398 if in_doc:
3399 #@+<< handle remaining @doc lines >>
3400 #@+node:ekr.20180606054325.1: *4* << handle remaining @doc lines >>
3401 if comment_delim2:
3402 # doc lines are unchanged.
3403 body.append(line)
3404 continue
3405 # Doc lines start with start_delim + one blank.
3406 # #1496: Retire the @doc convention.
3407 # #2194: Strip lws.
3408 tail = line.lstrip()[len(comment_delim1) + 1 :]
3409 if tail.strip():
3410 body.append(tail)
3411 else:
3412 body.append('\n')
3413 continue
3414 #@-<< handle remaining @doc lines >>
3415 #@+<< handle remaining @ lines >>
3416 #@+node:ekr.20180602103135.17: *4* << handle remaining @ lines >>
3417 # Handle an apparent sentinel line.
3418 # This *can* happen after the git-diff or refresh-from-disk commands.
3419 #
3420 if 1: # pragma: no cover (defensive)
3421 # This assert verifies the short-circuit test.
3422 assert strip_line.startswith(sentinel), (repr(sentinel), repr(line))
3423 # A useful trace.
3424 g.trace(
3425 f"{g.shortFileName(self.path)}: "
3426 f"warning: inserting unexpected line: {line.rstrip()!r}"
3427 )
3428 # #2213: *Do* insert the line, with a warning.
3429 body.append(line)
3430 #@-<< handle remaining @ lines >>
3431 else:
3432 # No @-leo sentinel!
3433 return # pragma: no cover
3434 #@+<< final checks >>
3435 #@+node:ekr.20211104054823.1: *4* << final checks >>
3436 if g.unitTesting:
3437 # Unit tests must use the proper value for root.gnx.
3438 assert not root_gnx_adjusted
3439 assert not stack, stack
3440 assert root_gnx == gnx, (root_gnx, gnx)
3441 elif root_gnx_adjusted: # pragma: no cover
3442 pass # Don't check!
3443 elif stack: # pragma: no cover
3444 g.error('scan_lines: Stack should be empty')
3445 g.printObj(stack, tag='stack')
3446 elif root_gnx != gnx: # pragma: no cover
3447 g.error('scan_lines: gnx error')
3448 g.es_print(f"root_gnx: {root_gnx} != gnx: {gnx}")
3449 #@-<< final checks >>
3450 #@+<< insert @last lines >>
3451 #@+node:ekr.20211103101453.1: *4* << insert @last lines >>
3452 tail_lines = lines[start + i :]
3453 if tail_lines:
3454 # Convert the trailing lines to @last directives.
3455 last_lines = [f"@last {z.rstrip()}\n" for z in tail_lines]
3456 # Add the lines to the dictionary of lines.
3457 gnx2body[gnx] = gnx2body[gnx] + last_lines
3458 # Warn if there is an unexpected number of last lines.
3459 if n_last_lines != len(last_lines): # pragma: no cover
3460 n1 = n_last_lines
3461 n2 = len(last_lines)
3462 g.trace(f"Expected {n1} trailing line{g.plural(n1)}, got {n2}")
3463 #@-<< insert @last lines >>
3464 #@+<< post pass: set all body text>>
3465 #@+node:ekr.20211104054426.1: *4* << post pass: set all body text>>
3466 # Set the body text.
3467 assert root_v.gnx in gnx2vnode, root_v
3468 assert root_v.gnx in gnx2body, root_v
3469 for key in gnx2body:
3470 body = gnx2body.get(key)
3471 v = gnx2vnode.get(key)
3472 assert v, (key, v)
3473 v._bodyString = g.toUnicode(''.join(body))
3474 #@-<< post pass: set all body text>>
3475 #@+node:ekr.20180603170614.1: *3* fast_at.read_into_root
3476 def read_into_root(self, contents, path, root):
3477 """
3478 Parse the file's contents, creating a tree of vnodes
3479 anchored in root.v.
3480 """
3481 self.path = path
3482 self.root = root
3483 sfn = g.shortFileName(path)
3484 contents = contents.replace('\r', '')
3485 lines = g.splitLines(contents)
3486 data = self.scan_header(lines)
3487 if not data: # pragma: no cover
3488 g.trace(f"Invalid external file: {sfn}")
3489 return False
3490 # Clear all children.
3491 # Previously, this had been done in readOpenFile.
3492 root.v._deleteAllChildren()
3493 comment_delims, first_lines, start_i = data
3494 self.scan_lines(comment_delims, first_lines, lines, path, start_i)
3495 return True
3496 #@-others
3497#@-others
3498#@@language python
3499#@@tabwidth -4
3500#@@pagewidth 60
3502#@-leo